From d6bb1f6fa4a431fd6fbf64928c968cb90aa4c4fc Mon Sep 17 00:00:00 2001 From: mgostIH <13892786+mgostIH@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:50:01 +0200 Subject: [PATCH 1/6] [AUTOMATED] Derive free Varnode definition order from location index Share compact comparator snapshots and precompute address and flag ordering. Preserve creation order, SeqNum quirks, and unification failure states. IR instructions fall 7.06% with identical output; 66 focused bank and Funcdata checks pass. --- .../kuna-decomp/src/substrate/varnode.rs | 411 +++++++++++++----- docs/spec/00-overview.md | 43 +- 2 files changed, 322 insertions(+), 132 deletions(-) diff --git a/decompiler/crates/kuna-decomp/src/substrate/varnode.rs b/decompiler/crates/kuna-decomp/src/substrate/varnode.rs index c774a5ae64..d54ae217e9 100644 --- a/decompiler/crates/kuna-decomp/src/substrate/varnode.rs +++ b/decompiler/crates/kuna-decomp/src/substrate/varnode.rs @@ -11,23 +11,22 @@ //! `Varnode`s in a `Funcdata`-owned slotmap arena keyed by [`VarnodeId`]; here //! [`VarnodeBank`] owns that arena directly (the bank *is* the varnode store). //! `def` becomes `Option`, `descend` a [`SmallVec`] of [`OpId`] with the -//! exact C++ `push_back`/erase-by-value order, and the cached `set::iterator`s -//! become **stored keys** ([`LocKey`]/[`DefKey`]) so the xref operations -//! (`xref`/`makeFree`/`setInput`/`setDef`/`destroy`) do the same precise -//! `erase(iter)` + reinsert the C++ does. +//! exact C++ `push_back`/erase-by-value order. Each node caches one [`LocKey`] +//! snapshot; the definition index derives its [`DefKey`] from that snapshot. //! //! ## ADR 0002 (ordered containers) realization //! -//! `loc_tree`/`def_tree` are `BTreeMap` / -//! `BTreeMap`. [`LocKey::cmp`]/[`DefKey::cmp`] transcribe -//! `VarnodeCompareLocDef`/`VarnodeCompareDefLoc` (`varnode.cc:34-79`) -//! field-by-field, **including** the `(f1-1) < (f2-1)` wrap that orders -//! input < written < free and the `create_index` tie-break for frees. The -//! key carries everything the comparator reads — address, size, flag class, -//! the def's `SeqNum` (written) or `create_index` (free) — so the trees never -//! dereference an op to compare, and `beginLoc`/`endLoc`/`beginDef`/`endDef` -//! become `BTreeMap::range` queries over *constructed bound keys* that mirror -//! the C++ `searchvn` template assignments exactly. +//! `loc_tree` holds all Varnodes in location order; `def_tree` holds inputs and +//! written Varnodes in definition order. Free Varnodes have the same relative +//! order in both indexes, so definition queries use a filtered location view +//! for their free tail instead of maintaining duplicate entries. +//! +//! [`LocKey::cmp`]/[`DefKey::cmp`] preserve `VarnodeCompareLocDef`/`DefLoc` +//! (`varnode.cc:34-79`), including input < written < free, the creation-index +//! tie-break for frees, and SeqNum's uniq-only equality. Keys contain plain +//! integer ordering fields, with the free creation index sharing the written +//! sequence's uniq slot. Constructed range bounds mirror the C++ `searchvn` +//! assignments and lower_bound/upper_bound operations. use std::collections::BTreeMap; use std::ops::Bound; @@ -168,38 +167,28 @@ type DescendVec = SmallVec<[OpId; 4]>; // Comparator keys (ADR 0002): transcribe VarnodeCompareLocDef / DefLoc exactly // --------------------------------------------------------------------------- -/// The ordering triple of an [`Address`] flattened into plain integers. -/// -/// `Address::cmp` orders by `(sentinel rank, space index, offset)`; storing -/// those three fields directly keeps the comparator identical while making the -/// tree key `Copy` — no `Rc` traffic on clone or drop, and no pointer chase -/// into the `AddrSpace` on a comparison. The trees are the decompiler's -/// hottest container (millions of inserts and removals per function). +/// Address ordering without reference counts or address-space dereferences. +/// Sentinel rank and signed space index share a u64 prefix; flipping the index's +/// sign bit preserves signed order within each rank. The offset follows it. #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct AddrKey { - /// Offset within the space (last in the comparison order). offset: u64, - /// `AddrSpace::get_index()`, or `0` for the two sentinels. - index: i32, - /// `0` = null / invalid, `1` = a real space, `2` = the `m_maximal` sentinel. - rank: u8, + space: u64, } impl AddrKey { #[inline] fn of(a: &Address) -> AddrKey { let (rank, index, offset) = a.sort_key(); - AddrKey { offset, index, rank } + let space = ((rank as u64) << 32) | ((index as u32 ^ 0x8000_0000) as u64); + AddrKey { offset, space } } } impl Ord for AddrKey { #[inline] fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.rank - .cmp(&other.rank) - .then_with(|| self.index.cmp(&other.index)) - .then_with(|| self.offset.cmp(&other.offset)) + self.space.cmp(&other.space).then_with(|| self.offset.cmp(&other.offset)) } } impl PartialOrd for AddrKey { @@ -220,7 +209,7 @@ struct SeqKey { impl Default for SeqKey { /// Matches `SeqNum::default()` (a null pc and `uniq == 0`). fn default() -> SeqKey { - SeqKey { pc: AddrKey { offset: 0, index: 0, rank: 0 }, uniq: 0 } + SeqKey { pc: AddrKey { offset: 0, space: 0x8000_0000 }, uniq: 0 } } } @@ -231,22 +220,21 @@ impl SeqKey { } } -/// Masked input/written flags, ordered as input < written < free using -/// wrapping subtraction; the free class is zero before subtracting one. +/// Masked input/written flags with wrapping subtraction precomputed, ordering +/// input < written < free. `raw` restores the original masked flags. #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct FlagClass(uint4); impl FlagClass { /// The raw `flags & (input|written)` value (one of 0, input, written). fn raw(self) -> uint4 { - self.0 + self.0.wadd(1) } } impl Ord for FlagClass { fn cmp(&self, other: &Self) -> std::cmp::Ordering { - // -1 forces free varnodes to come last - self.0.wsub(1).cmp(&other.0.wsub(1)) + self.0.cmp(&other.0) } } impl PartialOrd for FlagClass { @@ -258,7 +246,7 @@ impl PartialOrd for FlagClass { /// Extract the `(input|written)` class from a flag word (C++ /// `flags & (Varnode::input|Varnode::written)`). fn flag_class_of(flags: uint4) -> FlagClass { - FlagClass(flags & (varnode_flags::input | varnode_flags::written)) + FlagClass((flags & (varnode_flags::input | varnode_flags::written)).wsub(1)) } /// The seqnum step of the varnode comparators. @@ -283,7 +271,7 @@ fn seqnum_step(a: &SeqKey, b: &SeqKey) -> std::cmp::Ordering { /// /// Captures every field the comparator reads so the `BTreeMap` orders without /// dereferencing an op. For written varnodes `seqnum` holds the def's -/// `SeqNum`; for free varnodes `create_index` provides the final tie-break. +/// `SeqNum`; for free varnodes its uniq slot holds the creation-index tie-break. #[derive(Debug, Clone, Copy)] pub struct LocKey { /// Storage location (C++ `getAddr()`) @@ -292,10 +280,8 @@ pub struct LocKey { size: int4, /// `(input|written)` flag class (C++ `getFlags()&(input|written)`) flagclass: FlagClass, - /// The def `SeqNum`, consulted only when `flagclass == written` + /// The def sequence, or the free creation index in its uniq slot. seqnum: SeqKey, - /// Creation index, consulted only when `flagclass == 0` (free) - create_index: uint4, } impl PartialEq for LocKey { @@ -328,7 +314,7 @@ impl Ord for LocKey { return ord; } } else if self.flagclass.raw() == 0 { - return self.create_index.cmp(&other.create_index); + return self.seqnum.uniq.cmp(&other.seqnum.uniq); } Ordering::Equal } @@ -349,10 +335,20 @@ pub struct DefKey { size: int4, /// `(input|written)` flag class flagclass: FlagClass, - /// The def `SeqNum`, consulted only when `flagclass == written` + /// The def sequence, or the free creation index in its uniq slot. seqnum: SeqKey, - /// Creation index, consulted only when `flagclass == 0` (free) - create_index: uint4, +} + +impl From for DefKey { + fn from(k: LocKey) -> Self { + Self { addr: k.addr, size: k.size, flagclass: k.flagclass, seqnum: k.seqnum } + } +} + +impl From for LocKey { + fn from(k: DefKey) -> Self { + Self { addr: k.addr, size: k.size, flagclass: k.flagclass, seqnum: k.seqnum } + } } impl PartialEq for DefKey { @@ -386,7 +382,7 @@ impl Ord for DefKey { return ord; } if self.flagclass.raw() == 0 { - return self.create_index.cmp(&other.create_index); + return self.seqnum.uniq.cmp(&other.seqnum.uniq); } Ordering::Equal } @@ -520,10 +516,8 @@ pub struct Varnode { /// *dataType; ... } temp` — the `dataType` field). Set/read only by /// `ActionInferTypes`; `None` outside a propagation pass. temp_type: Option>, - /// Cached key into the loc tree (replaces the C++ `lociter`) + /// Shared ordering snapshot for the location and definition indexes. lociter: Option, - /// Cached key into the def tree (replaces the C++ `defiter`) - defiter: Option, /// Every op reading this varnode as input (C++ `list descend`) descend: DescendVec, /// Addresses covered by def->use of this Varnode (STUB(W7)) @@ -561,7 +555,6 @@ impl Varnode { type_: dt, temp_type: None, lociter: None, - defiter: None, descend: SmallVec::new(), cover: None, consumed: u64::MAX, // ~((uintb)0) @@ -1568,7 +1561,7 @@ pub struct VarnodeBank { arena: SlotMap, /// Varnodes sorted by location then def (C++ `loc_tree`) loc_tree: BTreeMap, - /// Varnodes sorted by def then location (C++ `def_tree`) + /// Inputs and written Varnodes sorted by def then location (C++ `def_tree`). def_tree: BTreeMap, /// The unique space, cached at construction (C++ `uniq_space`) uniq_space: Rc, @@ -1650,24 +1643,15 @@ impl VarnodeBank { addr: AddrKey::of(&vn.loc), size: vn.size, flagclass: flag_class_of(vn.flags), - seqnum: vn.def_seqnum.as_ref().map(SeqKey::of).unwrap_or_default(), - create_index: vn.create_index, - } - } - - /// Build the definition-tree key for the varnode at `id`. - fn def_key_of(&self, id: VarnodeId) -> DefKey { - let vn = &self.arena[id]; - DefKey { - addr: AddrKey::of(&vn.loc), - size: vn.size, - flagclass: flag_class_of(vn.flags), - seqnum: vn.def_seqnum.as_ref().map(SeqKey::of).unwrap_or_default(), - create_index: vn.create_index, + seqnum: if vn.is_free() { + SeqKey { uniq: vn.create_index, ..SeqKey::default() } + } else { + vn.def_seqnum.as_ref().map(SeqKey::of).unwrap_or_default() + }, } } - /// Create a \e free Varnode and insert it into both trees (C++ `create`). + /// Create a \e free Varnode in the location tree (C++ `create`). pub fn create(&mut self, s: int4, m: Address, ct: Rc) -> VarnodeId { let mut vn = Varnode::new(s, m, ct); vn.create_index = self.create_index; @@ -1675,11 +1659,8 @@ impl VarnodeBank { let id = self.arena.insert(vn); // Frees can always be inserted without duplication. let lk = self.loc_key_of(id); - let dk = self.def_key_of(id); self.arena[id].lociter = Some(lk); - self.arena[id].defiter = Some(dk); self.loc_tree.insert(lk, id); - self.def_tree.insert(dk, id); id } @@ -1702,15 +1683,16 @@ impl VarnodeBank { return Err(KunaError::lowlevel("Deleting integrated varnode")); } let lk = self.arena[vn].lociter.take().expect("destroy: missing lociter"); - let dk = self.arena[vn].defiter.take().expect("destroy: missing defiter"); self.loc_tree.remove(&lk); - self.def_tree.remove(&dk); + if lk.flagclass.raw() != 0 { + self.def_tree.remove(&DefKey::from(lk)); + } self.arena.remove(vn); Ok(()) } - /// Insert a Varnode into both sorted trees, unifying with an equivalent - /// existing free varnode if present (C++ `xref`, `varnode.cc:1310`). + /// Insert an input or written Varnode into both trees, unifying with an + /// equivalent existing Varnode if present (C++ `xref`, `varnode.cc:1310`). /// /// If the loc tree already contains an equivalent varnode, the existing one /// is kept: `replace_reads` patches every op reading `vn` onto it, `vn` is @@ -1738,9 +1720,8 @@ impl VarnodeBank { } self.arena[vn].set_flags(varnode_flags::insert); self.arena[vn].lociter = Some(lk); - let dk = self.def_key_of(vn); + let dk = DefKey::from(lk); self.def_tree.insert(dk, vn); // new in def_tree - self.arena[vn].defiter = Some(dk); Ok(vn) } @@ -1757,11 +1738,12 @@ impl VarnodeBank { if v.is_constant() { return Err(KunaError::lowlevel("Making input out of constant varnode")); } - // Erase the free version of the varnode from both trees. + // Erase the previous ordering snapshot. let lk = self.arena[vn].lociter.take().expect("set_input: missing lociter"); - let dk = self.arena[vn].defiter.take().expect("set_input: missing defiter"); self.loc_tree.remove(&lk); - self.def_tree.remove(&dk); + if lk.flagclass.raw() != 0 { + self.def_tree.remove(&DefKey::from(lk)); + } self.arena[vn].set_input_flag(); // Set the input flag self.xref(vn, replace_reads) } @@ -1785,9 +1767,10 @@ impl VarnodeBank { return Err(KunaError::lowlevel("Assignment to constant")); } let lk = self.arena[vn].lociter.take().expect("set_def: missing lociter"); - let dk = self.arena[vn].defiter.take().expect("set_def: missing defiter"); self.loc_tree.remove(&lk); - self.def_tree.remove(&dk); + if lk.flagclass.raw() != 0 { + self.def_tree.remove(&DefKey::from(lk)); + } self.arena[vn].set_def(Some(def.id)); // Change the varnode to be defined self.arena[vn].def_seqnum = Some(def.seqnum); self.xref(vn, replace_reads) @@ -1796,9 +1779,10 @@ impl VarnodeBank { /// Convert a Varnode to be \e free (C++ `makeFree`). pub fn make_free(&mut self, vn: VarnodeId) { let lk = self.arena[vn].lociter.take().expect("make_free: missing lociter"); - let dk = self.arena[vn].defiter.take().expect("make_free: missing defiter"); self.loc_tree.remove(&lk); - self.def_tree.remove(&dk); + if lk.flagclass.raw() != 0 { + self.def_tree.remove(&DefKey::from(lk)); + } self.arena[vn].set_def(None); // Clear things that make vn non-free self.arena[vn].def_seqnum = None; @@ -1808,11 +1792,8 @@ impl VarnodeBank { // Re-insert as free varnode. let lk = self.loc_key_of(vn); - let dk = self.def_key_of(vn); self.arena[vn].lociter = Some(lk); - self.arena[vn].defiter = Some(dk); self.loc_tree.insert(lk, vn); - self.def_tree.insert(dk, vn); } /// Create a Varnode already in the def tree as if it were the output of the @@ -1890,7 +1871,6 @@ impl VarnodeBank { size: 0, flagclass: flag_class_of(varnode_flags::input), seqnum: SeqKey::of(&(SeqNum::default())), - create_index: 0, }); let finish = if end.get_offset() < start.get_offset() { LocProbe::End @@ -1900,7 +1880,6 @@ impl VarnodeBank { size: 0, flagclass: flag_class_of(varnode_flags::input), seqnum: SeqKey::of(&(SeqNum::default())), - create_index: 0, }) }; self.iter_loc_probe(begin, finish).take_while(move |&id| { @@ -1910,7 +1889,9 @@ impl VarnodeBank { /// Iterate all varnode ids in definition order (C++ `beginDef()`..`endDef()`). pub fn iter_def(&self) -> impl Iterator + '_ { - self.def_tree.values().copied() + self.def_tree.values().copied().chain( + self.loc_tree.iter().filter_map(|(k, &id)| (k.flagclass.raw() == 0).then_some(id)) + ) } /// Iterate varnode ids whose location lies in `[start, end)` of the loc @@ -1996,7 +1977,6 @@ impl VarnodeBank { size: s, flagclass: flag_class_of(varnode_flags::input), seqnum: SeqKey::of(&(SeqNum::default())), - create_index: 0, }); } if fl == varnode_flags::written { @@ -2006,7 +1986,6 @@ impl VarnodeBank { size: s, flagclass: flag_class_of(varnode_flags::written), seqnum: SeqKey::of(&(SeqNum::new_extreme(mach_extreme::m_minimal))), - create_index: 0, }); } // fl == 0 (free): searchvn{size=s, loc=addr, flags=written, def=&searchop(maximal seq)} ; upper_bound @@ -2015,7 +1994,6 @@ impl VarnodeBank { size: s, flagclass: flag_class_of(varnode_flags::written), seqnum: SeqKey::of(&(SeqNum::new_extreme(mach_extreme::m_maximal))), - create_index: 0, }) } @@ -2028,7 +2006,6 @@ impl VarnodeBank { size: s, flagclass: flag_class_of(varnode_flags::written), seqnum: SeqKey::of(&(SeqNum::new_extreme(mach_extreme::m_maximal))), - create_index: 0, }); } if fl == varnode_flags::input { @@ -2038,7 +2015,6 @@ impl VarnodeBank { size: s, flagclass: flag_class_of(varnode_flags::input), seqnum: SeqKey::of(&(SeqNum::default())), - create_index: 0, }); } // fl == 0 (free): searchvn{loc=addr, size=s+1, flags=input} ; lower_bound @@ -2047,7 +2023,6 @@ impl VarnodeBank { size: s + 1, flagclass: flag_class_of(varnode_flags::input), seqnum: SeqKey::of(&(SeqNum::default())), - create_index: 0, }) } @@ -2062,7 +2037,6 @@ impl VarnodeBank { size: s, flagclass: flag_class_of(varnode_flags::written), seqnum: SeqKey::of(&(SeqNum::new(pc.clone(), u))), - create_index: 0, }) } @@ -2075,7 +2049,6 @@ impl VarnodeBank { size: s, flagclass: flag_class_of(varnode_flags::written), seqnum: SeqKey::of(&(SeqNum::new(pc.clone(), uniq))), - create_index: 0, }) } @@ -2097,7 +2070,6 @@ impl VarnodeBank { size: 0, flagclass: flag_class_of(varnode_flags::input), seqnum: SeqKey::of(&(SeqNum::default())), - create_index: 0, }); let mut out = Vec::new(); for id in self.iter_loc_probe(lo, LocProbe::End) { @@ -2122,7 +2094,6 @@ impl VarnodeBank { size: s, flagclass: flag_class_of(varnode_flags::input), seqnum: SeqKey::of(&(SeqNum::default())), - create_index: 0, }); // endLoc: searchvn{size=s+1, loc=addr} ; lower_bound let end = LocProbe::Lower(LocKey { @@ -2130,7 +2101,6 @@ impl VarnodeBank { size: s + 1, flagclass: flag_class_of(varnode_flags::input), seqnum: SeqKey::of(&(SeqNum::default())), - create_index: 0, }); self.iter_loc_probe(begin, end) } @@ -2171,8 +2141,32 @@ impl VarnodeBank { &self, start: DefProbe, end: DefProbe, - ) -> impl Iterator + '_ { - self.def_tree.range((start.into_start(), end.into_end())).map(|(_, &id)| id) + ) -> impl DoubleEndedIterator + '_ { + let bounds = (start.into_start(), end.into_end()); + self.def_tree.range(bounds).map(|(_, &id)| id).chain(self.iter_free_def(bounds)) + } + + /// Free nodes have identical relative order in the location and definition indexes. + fn iter_free_def( + &self, + (start, end): (Bound, Bound), + ) -> impl DoubleEndedIterator + '_ { + let start = match start { + Bound::Included(k) if k.flagclass.raw() == 0 => Bound::Included(LocKey::from(k)), + Bound::Excluded(k) if k.flagclass.raw() == 0 => Bound::Excluded(LocKey::from(k)), + _ => Bound::Unbounded, + }; + let end = match end { + Bound::Included(k) if k.flagclass.raw() == 0 => Some(Bound::Included(LocKey::from(k))), + Bound::Excluded(k) if k.flagclass.raw() == 0 => Some(Bound::Excluded(LocKey::from(k))), + Bound::Unbounded => Some(Bound::Unbounded), + _ => None, + }; + end.into_iter().flat_map(move |end| { + self.loc_tree + .range((start, end)) + .filter_map(|(k, &id)| (k.flagclass.raw() == 0).then_some(id)) + }) } /// `beginDef(uint4 fl)` (`varnode.cc:1850`). @@ -2188,7 +2182,6 @@ impl VarnodeBank { size: 0, flagclass: flag_class_of(varnode_flags::written), seqnum: SeqKey::of(&(SeqNum::new_extreme(mach_extreme::m_minimal))), - create_index: 0, }); } // fl == 0 (free): searchvn{loc=maximal, flags=written, def=&searchop(maximal seq)} ; upper_bound @@ -2197,7 +2190,6 @@ impl VarnodeBank { size: 0, flagclass: flag_class_of(varnode_flags::written), seqnum: SeqKey::of(&(SeqNum::new_extreme(mach_extreme::m_maximal))), - create_index: 0, }) } @@ -2210,7 +2202,6 @@ impl VarnodeBank { size: 0, flagclass: flag_class_of(varnode_flags::written), seqnum: SeqKey::of(&(SeqNum::new_extreme(mach_extreme::m_minimal))), - create_index: 0, }); } if fl == varnode_flags::written { @@ -2220,7 +2211,6 @@ impl VarnodeBank { size: 0, flagclass: flag_class_of(varnode_flags::written), seqnum: SeqKey::of(&(SeqNum::new_extreme(mach_extreme::m_maximal))), - create_index: 0, }); } // fl == 0 (free): def_tree.end() @@ -2249,7 +2239,6 @@ impl VarnodeBank { size: 0, flagclass: flag_class_of(varnode_flags::input), seqnum: SeqKey::of(&(SeqNum::default())), - create_index: 0, })); } // fl == 0 (free): searchvn{loc=addr, flags=0(free)} ; upper_bound @@ -2258,7 +2247,6 @@ impl VarnodeBank { size: 0, flagclass: flag_class_of(0), seqnum: SeqKey::of(&(SeqNum::default())), - create_index: 0, })) } @@ -2274,7 +2262,6 @@ impl VarnodeBank { size: 1000000, flagclass: flag_class_of(varnode_flags::input), seqnum: SeqKey::of(&(SeqNum::default())), - create_index: 0, })); } // fl == 0 (free): searchvn{loc=addr, size=1000000, flags=0(free)} ; lower_bound @@ -2283,7 +2270,6 @@ impl VarnodeBank { size: 1000000, flagclass: flag_class_of(0), seqnum: SeqKey::of(&(SeqNum::default())), - create_index: 0, })) } @@ -2525,8 +2511,9 @@ mod tests { addr: AddrKey::of(&addr), size, flagclass: flag_class_of(fl), - seqnum: SeqKey::of(&seq), - create_index: ci, + seqnum: if fl & (varnode_flags::input | varnode_flags::written) == 0 { + SeqKey { uniq: ci, ..SeqKey::of(&seq) } + } else { SeqKey::of(&seq) }, } } fn make_def_key(addr: Address, size: int4, fl: uint4, seq: SeqNum, ci: uint4) -> DefKey { @@ -2534,9 +2521,42 @@ mod tests { addr: AddrKey::of(&addr), size, flagclass: flag_class_of(fl), - seqnum: SeqKey::of(&seq), - create_index: ci, + seqnum: if fl & (varnode_flags::input | varnode_flags::written) == 0 { + SeqKey { uniq: ci, ..SeqKey::of(&seq) } + } else { SeqKey::of(&seq) }, + } + } + + + #[test] + fn packed_address_order_matches_sentinels_and_signed_space_indexes() { + let mut addrs = vec![ + Address::default(), + Address::new_extreme(mach_extreme::m_minimal), + Address::new_extreme(mach_extreme::m_maximal), + ]; + for index in [i32::MIN, -1, 0, 1, i32::MAX] { + let sp = Rc::new(AddrSpace::new( + spacetype::IPTR_PROCESSOR, + "test", + false, + 8, + 1, + index, + 0, + 0, + 0, + )); + for offset in [0, 1, u64::MAX] { + addrs.push(Address::new(Rc::clone(&sp), offset)); + } + } + for a in &addrs { + for b in &addrs { + assert_eq!(AddrKey::of(a).cmp(&AddrKey::of(b)), a.cmp(b)); + } } + assert_eq!(SeqKey::default(), SeqKey::of(&SeqNum::default())); } /// GOLDEN (varnodesort): the `tests/golden/vectors/` corpus has no varnode @@ -2797,6 +2817,169 @@ mod tests { assert_eq!(all, vec![input, written]); } + + #[test] + fn free_definition_ranges_match_full_tree() { + eprintln!( + "index layout: LocKey={} DefKey={} Varnode={}", + std::mem::size_of::(), + std::mem::size_of::(), + std::mem::size_of::() + ); + let m = build_manager(); + let mut bank = VarnodeBank::new(&m, 0).unwrap(); + let mut idc = 0; + for i in 0..48 { + let addr = Address::new(space(&m, 2), (i % 6) * 16); + let id = bank.create(1 << (i % 4), addr, dt(1 << (i % 4))); + if i % 3 == 0 { + bank.set_input(id, &mut no_replace()).unwrap(); + } else if i % 3 == 1 { + let def = def_info(&mut idc, Address::new(space(&m, 2), i * 4), i as uint4 + 1); + bank.set_def(id, def, &mut no_replace()).unwrap(); + } + } + for pass in 0..3 { + let full: BTreeMap = bank + .arena + .iter() + .map(|(id, v)| { + let key = make_def_key( + v.loc.clone(), + v.size, + v.flags, + v.def_seqnum.clone().unwrap_or_default(), + v.create_index, + ); + (key, id) + }) + .collect(); + assert_eq!( + bank.iter_def().collect::>(), + full.values().copied().collect::>() + ); + let mut probes = vec![DefProbe::Begin]; + for class in [varnode_flags::input, varnode_flags::written, 0] { + for off in [0, 16, 48, 96] { + let key = make_def_key( + Address::new(space(&m, 2), off), + 4, + class, + SeqNum::new(Address::new(space(&m, 2), 40), 11), + 17, + ); + probes.push(DefProbe::Lower(key)); + probes.push(DefProbe::Upper(key)); + } + } + probes.push(DefProbe::End); + for start in &probes { + for end in &probes { + let bounds = (start.clone().into_start(), end.clone().into_end()); + match bounds { + ( + Bound::Included(a) | Bound::Excluded(a), + Bound::Included(b) | Bound::Excluded(b), + ) if a > b => continue, + (Bound::Excluded(a), Bound::Excluded(b)) if a == b => continue, + _ => {} + } + let expected: Vec<_> = full.range(bounds).map(|(_, &id)| id).collect(); + let actual: Vec<_> = bank.iter_def_probe(start.clone(), end.clone()).collect(); + assert_eq!(actual, expected, "pass={pass} bounds={bounds:?}"); + let reverse: Vec<_> = bank + .iter_def_probe(start.clone(), end.clone()) + .rev() + .collect(); + assert_eq!(reverse, expected.into_iter().rev().collect::>()); + } + } + let written: Vec<_> = bank.iter_def_flag(varnode_flags::written).take(3).collect(); + for id in written { + bank.make_free(id); + } + let free = bank.iter_def_flag(0).next().unwrap(); + bank.destroy(free).unwrap(); + bank.create(4, Address::new(space(&m, 2), 16), dt(4)); + } + } + + + #[test] + fn definition_view_preserves_uniq_only_seqnum_equality() { + let m = build_manager(); + let mut bank = VarnodeBank::new(&m, 0).unwrap(); + let mut idc = 0; + let high = bank.create(4, Address::new(space(&m, 2), 0x400), dt(4)); + let low = bank.create(4, Address::new(space(&m, 2), 0x100), dt(4)); + let early = def_info(&mut idc, Address::new(space(&m, 2), 0x10), 7); + let late = def_info(&mut idc, Address::new(space(&m, 2), 0x20), 7); + bank.set_def(high, early, &mut no_replace()).unwrap(); + bank.set_def(low, late, &mut no_replace()).unwrap(); + assert_eq!( + bank.iter_def_flag(varnode_flags::written) + .collect::>(), + vec![low, high] + ); + let duplicate = bank.create(4, Address::new(space(&m, 2), 0x400), dt(4)); + let other_pc = def_info(&mut idc, Address::new(space(&m, 2), 0x30), 7); + let retained = bank + .set_def(duplicate, other_pc, &mut no_replace()) + .unwrap(); + assert_eq!(retained, high); + assert!(bank.get(duplicate).is_none()); + bank.make_free(low); + assert_eq!(bank.iter_def().collect::>(), vec![high, low]); + } + + #[test] + fn definition_views_use_index_snapshots() { + let m = build_manager(); + let mut bank = VarnodeBank::new(&m, 0).unwrap(); + let mut idc = 0; + let older = bank.create(4, Address::new(space(&m, 2), 16), dt(4)); + let def = def_info(&mut idc, Address::new(space(&m, 2), 8), 7); + bank.set_def(older, def, &mut no_replace()).unwrap(); + let newer = bank.create(4, Address::new(space(&m, 2), 16), dt(4)); + bank.get_mut(newer) + .unwrap() + .set_flags_pub(varnode_flags::written); + assert_eq!(bank.iter_def_flag(0).collect::>(), vec![newer]); + bank.get_mut(newer) + .unwrap() + .clear_flags_pub(varnode_flags::written); + bank.make_free(older); + assert_eq!( + bank.iter_def_flag(0).collect::>(), + vec![older, newer] + ); + bank.destroy(older).unwrap(); + assert_eq!(bank.iter_def_flag(0).collect::>(), vec![newer]); + bank.clear(); + assert!(bank.iter_def().next().is_none()); + } + + #[test] + fn failed_unification_stays_out_of_definition_views() { + let m = build_manager(); + let mut bank = VarnodeBank::new(&m, 0).unwrap(); + let addr = Address::new(space(&m, 2), 16); + let kept = bank.create(4, addr.clone(), dt(4)); + bank.set_input(kept, &mut no_replace()).unwrap(); + let orphan = bank.create(4, addr, dt(4)); + assert!(bank + .set_input(orphan, &mut |_, _, _| Err(KunaError::lowlevel( + "replacement failed" + ))) + .is_err()); + assert!(bank.get(orphan).is_some()); + assert_eq!(bank.num_varnodes(), 1); + assert_eq!(bank.iter_loc().collect::>(), vec![kept]); + assert_eq!(bank.iter_def().collect::>(), vec![kept]); + bank.clear(); + assert_eq!(bank.num_varnodes(), 0); + } + /// find_input / find_covering_input / find_covered_input on inputs. /// /// The def-tree finders match the C++'s `lower_bound`-then-step-back idiom, diff --git a/docs/spec/00-overview.md b/docs/spec/00-overview.md index 29e03ff4d7..b0f4fdd10e 100644 --- a/docs/spec/00-overview.md +++ b/docs/spec/00-overview.md @@ -2771,24 +2771,31 @@ return-address effects. This is unconditional registration behavior, not a separate feature gate; `funcdata_varnode.rs (apply_input_effect_marking)` uses the canonical `fspec.rs (effect_type)` values rather than maintaining copies. -The varnode bank's two sorted trees are the container the decompiler touches most -— a large function creates and destroys well over a million Varnodes, each one -inserted into and removed from both — so their keys -(`decompiler/crates/kuna-decomp/src/substrate/varnode.rs (LocKey, DefKey)`) do not -store an `Address` or a `SeqNum` directly. They store the ordering triple those -compare by, flattened into plain integers: the sentinel rank, the space index and -the offset (`decompiler/crates/kuna-base/src/address.rs (Address::sort_key)`). -Lexicographic comparison of the triple reproduces `Address::cmp` exactly — two -Addresses sharing a space pointer share a rank and an index and fall through to -the offset, which is what the pointer-equality fast path does — so the tree order -is the C++ comparator's order, while the key itself becomes `Copy`: no reference -counting on clone or drop, no pointer chase into an `AddrSpace` to compare, and a -smaller node. Insertion also takes a single descent -(`decompiler/crates/kuna-decomp/src/substrate/varnode.rs (VarnodeBank::xref)`): -the "is an equivalent varnode already present" lookup and the insertion that -follows it are the same search, because the `insert` flag set afterwards is -outside the `(input|written)` mask the key is built from and so cannot move the -entry. +The varnode bank maintains a location tree for all Varnodes and a definition +tree for inputs and written Varnodes. Free Varnodes have identical relative +order in both indexes: address, size, then creation index. Definition iteration +therefore appends a filtered location-tree view for the free class instead of +inserting and erasing a duplicate definition entry. Free definition bounds are +translated to location bounds with the same inclusion or exclusion; ranges +ending in the input or written class never traverse that view +(`decompiler/crates/kuna-decomp/src/substrate/varnode.rs (iter_free_def)`). +Creation inserts a free node only in the location tree. Becoming input or written +adds its definition entry; becoming free removes that entry and rekeys its +location entry. Destroying a free node removes only its location entry. Both +logical orders retain the same ids and comparator semantics. + +The keys (`decompiler/crates/kuna-decomp/src/substrate/varnode.rs (LocKey, DefKey)`) +store integer ordering fields rather than `Address` or `SeqNum` objects. Sentinel +rank and signed space index share a packed prefix, followed by the offset; +flipping the index's sign bit preserves signed order within each rank. The +input/written/free wrap ordering is computed once when a key is constructed. +Written keys retain SeqNum's uniq-only equality check before comparing PC and +time; free keys use the same uniq slot for their creation index. Each Varnode +caches one location snapshot, from which definition keys are copied, so removals +use the indexed state even if other Varnode flags have changed. Insertion takes +a single location-tree descent (`decompiler/crates/kuna-decomp/src/substrate/varnode.rs +(VarnodeBank::xref)`): the equivalent-node lookup and insertion share one search, +and the subsequent `insert` flag is outside the key's input/written mask. Read-only block queries walk the stored Varnode descendants without copying them. Common-subexpression lookup returns the first eligible equal op in that From ece019472b4042f7a9895715bbf7434e9cd16fee Mon Sep 17 00:00:00 2001 From: mgostIH <13892786+mgostIH@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:29:02 +0200 Subject: [PATCH 2/6] [AUTOMATED] Reuse compact Varnode keys during class changes Pack signed size and class ordering into one location-key prefix. Reuse immutable address and size snapshots when a node changes class, and construct birth keys directly. Immediate teardown clears the live definition without inserting a temporary free entry before removing the indexed snapshot. Preserve combined class masks, signed bounds and SeqNum's uniq-only equality; add comparator, range, snapshot and teardown coverage and update the spec. Focused validation: 127 unit/integration tests, test layout, spec anchors and diff checks passed. Final candidate reduces instructions:u by 8.16% on ir and 9.07% on core against edba0094, with identical outputs; smoke also matches. Three fresh core CPU pairs remain pending at the user's requested pause. --- .../src/substrate/funcdata_varnode.rs | 6 +- .../kuna-decomp/src/substrate/varnode.rs | 380 +++++++++++------- docs/spec/00-overview.md | 22 +- 3 files changed, 263 insertions(+), 145 deletions(-) diff --git a/decompiler/crates/kuna-decomp/src/substrate/funcdata_varnode.rs b/decompiler/crates/kuna-decomp/src/substrate/funcdata_varnode.rs index 6ceda988e1..9ce5526d0e 100644 --- a/decompiler/crates/kuna-decomp/src/substrate/funcdata_varnode.rs +++ b/decompiler/crates/kuna-decomp/src/substrate/funcdata_varnode.rs @@ -876,11 +876,7 @@ impl Funcdata { if let Some(o) = self.obank_mut().get_mut(defop) { o.set_output(None); } - // vn->def = 0: the C++ nulls the raw `def` field (no tree reinsert, - // since destroy() follows immediately). `make_free` clears the - // written class and re-keys the varnode as free so `destroy`'s - // "integrated varnode" guard (def != 0) passes. - self.vbank_mut().make_free(vn); + self.vbank_mut().make_free_for_destroy(vn); } // ~Varnode(): if (high != 0) { high->remove(this); if (high->isUnattached()) // delete high; } (varnode.cc:629). See `purge_high_on_free`. diff --git a/decompiler/crates/kuna-decomp/src/substrate/varnode.rs b/decompiler/crates/kuna-decomp/src/substrate/varnode.rs index d54ae217e9..1de4d89224 100644 --- a/decompiler/crates/kuna-decomp/src/substrate/varnode.rs +++ b/decompiler/crates/kuna-decomp/src/substrate/varnode.rs @@ -226,7 +226,7 @@ impl SeqKey { struct FlagClass(uint4); impl FlagClass { - /// The raw `flags & (input|written)` value (one of 0, input, written). + /// The raw `flags & (input|written)` value, including the combined mask. fn raw(self) -> uint4 { self.0.wadd(1) } @@ -276,14 +276,34 @@ fn seqnum_step(a: &SeqKey, b: &SeqKey) -> std::cmp::Ordering { pub struct LocKey { /// Storage location (C++ `getAddr()`) addr: AddrKey, - /// Size in bytes (C++ `getSize()`) - size: int4, - /// `(input|written)` flag class (C++ `getFlags()&(input|written)`) - flagclass: FlagClass, + /// Signed size, sign-biased, followed by the preordered flag class. + size_class: u64, /// The def sequence, or the free creation index in its uniq slot. seqnum: SeqKey, } +impl LocKey { + #[inline] + fn new(addr: AddrKey, size: int4, flagclass: FlagClass, seqnum: SeqKey) -> Self { + let size_class = ((size as u32 ^ 0x8000_0000) as u64) << 32 | flagclass.0 as u64; + Self { addr, size_class, seqnum } + } + + #[inline] + fn flag_class(&self) -> FlagClass { + FlagClass(self.size_class as u32) + } + + /// Class changes preserve the immutable storage address and size. + fn with_class(self, flagclass: FlagClass, seqnum: SeqKey) -> Self { + Self { + addr: self.addr, + size_class: (self.size_class & 0xffff_ffff_0000_0000) | flagclass.0 as u64, + seqnum, + } + } +} + impl PartialEq for LocKey { fn eq(&self, other: &Self) -> bool { self.cmp(other) == std::cmp::Ordering::Equal @@ -299,21 +319,13 @@ impl Ord for LocKey { if ord != Ordering::Equal { return ord; } - let ord = self.size.cmp(&other.size); - if ord != Ordering::Equal { - return ord; - } - let ord = self.flagclass.cmp(&other.flagclass); + let ord = self.size_class.cmp(&other.size_class); if ord != Ordering::Equal { return ord; } - // f1 == f2 from here on - if self.flagclass.raw() == varnode_flags::written { - let ord = seqnum_step(&self.seqnum, &other.seqnum); - if ord != Ordering::Equal { - return ord; - } - } else if self.flagclass.raw() == 0 { + if self.flag_class().raw() == varnode_flags::written { + return seqnum_step(&self.seqnum, &other.seqnum); + } else if self.flag_class().raw() == 0 { return self.seqnum.uniq.cmp(&other.seqnum.uniq); } Ordering::Equal @@ -341,13 +353,18 @@ pub struct DefKey { impl From for DefKey { fn from(k: LocKey) -> Self { - Self { addr: k.addr, size: k.size, flagclass: k.flagclass, seqnum: k.seqnum } + Self { + addr: k.addr, + size: ((k.size_class >> 32) as u32 ^ 0x8000_0000) as int4, + flagclass: k.flag_class(), + seqnum: k.seqnum, + } } } impl From for LocKey { fn from(k: DefKey) -> Self { - Self { addr: k.addr, size: k.size, flagclass: k.flagclass, seqnum: k.seqnum } + Self::new(k.addr, k.size, k.flagclass, k.seqnum) } } @@ -1282,6 +1299,14 @@ impl Varnode { } } + fn set_free(&mut self) { + self.set_def(None); + self.def_seqnum = None; + self.clear_flags( + varnode_flags::insert | varnode_flags::input | varnode_flags::indirect_creation, + ); + } + /// Put a new operator in the descendant list and set the cover dirty flag /// (C++ `Varnode::addDescend`, `varnode.cc:330`). /// @@ -1636,29 +1661,19 @@ impl VarnodeBank { self.arena.get_mut(id) } - /// Build the location-tree key for the varnode at `id`. - fn loc_key_of(&self, id: VarnodeId) -> LocKey { - let vn = &self.arena[id]; - LocKey { - addr: AddrKey::of(&vn.loc), - size: vn.size, - flagclass: flag_class_of(vn.flags), - seqnum: if vn.is_free() { - SeqKey { uniq: vn.create_index, ..SeqKey::default() } - } else { - vn.def_seqnum.as_ref().map(SeqKey::of).unwrap_or_default() - }, - } - } - /// Create a \e free Varnode in the location tree (C++ `create`). pub fn create(&mut self, s: int4, m: Address, ct: Rc) -> VarnodeId { + let lk = LocKey::new( + AddrKey::of(&m), + s, + flag_class_of(0), + SeqKey { uniq: self.create_index, ..SeqKey::default() }, + ); let mut vn = Varnode::new(s, m, ct); vn.create_index = self.create_index; self.create_index = self.create_index.wadd(1); let id = self.arena.insert(vn); // Frees can always be inserted without duplication. - let lk = self.loc_key_of(id); self.arena[id].lociter = Some(lk); self.loc_tree.insert(lk, id); id @@ -1684,7 +1699,7 @@ impl VarnodeBank { } let lk = self.arena[vn].lociter.take().expect("destroy: missing lociter"); self.loc_tree.remove(&lk); - if lk.flagclass.raw() != 0 { + if lk.flag_class().raw() != 0 { self.def_tree.remove(&DefKey::from(lk)); } self.arena.remove(vn); @@ -1698,9 +1713,13 @@ impl VarnodeBank { /// is kept: `replace_reads` patches every op reading `vn` onto it, `vn` is /// removed from the arena, and the existing id is returned. Otherwise the /// varnode is freshly inserted, marked `insert`, and added to the def tree. - fn xref(&mut self, vn: VarnodeId, replace_reads: &mut ReplaceReads) -> KunaResult { + fn xref( + &mut self, + vn: VarnodeId, + lk: LocKey, + replace_reads: &mut ReplaceReads, + ) -> KunaResult { use std::collections::btree_map::Entry; - let lk = self.loc_key_of(vn); // One descent, not two: the lookup and the insertion are the same // search. The `insert` flag set below is outside the (input|written) // mask, so it does not change the loc/def keys and the entry placed @@ -1741,11 +1760,13 @@ impl VarnodeBank { // Erase the previous ordering snapshot. let lk = self.arena[vn].lociter.take().expect("set_input: missing lociter"); self.loc_tree.remove(&lk); - if lk.flagclass.raw() != 0 { + if lk.flag_class().raw() != 0 { self.def_tree.remove(&DefKey::from(lk)); } self.arena[vn].set_input_flag(); // Set the input flag - self.xref(vn, replace_reads) + let seqnum = self.arena[vn].def_seqnum.as_ref().map(SeqKey::of).unwrap_or_default(); + let lk = lk.with_class(flag_class_of(varnode_flags::input), seqnum); + self.xref(vn, lk, replace_reads) } /// Change a Varnode to be defined by the given op (C++ `setDef`). @@ -1768,34 +1789,41 @@ impl VarnodeBank { } let lk = self.arena[vn].lociter.take().expect("set_def: missing lociter"); self.loc_tree.remove(&lk); - if lk.flagclass.raw() != 0 { + if lk.flag_class().raw() != 0 { self.def_tree.remove(&DefKey::from(lk)); } self.arena[vn].set_def(Some(def.id)); // Change the varnode to be defined + let seqnum = SeqKey::of(&def.seqnum); self.arena[vn].def_seqnum = Some(def.seqnum); - self.xref(vn, replace_reads) + let lk = lk.with_class(flag_class_of(varnode_flags::written), seqnum); + self.xref(vn, lk, replace_reads) } /// Convert a Varnode to be \e free (C++ `makeFree`). pub fn make_free(&mut self, vn: VarnodeId) { let lk = self.arena[vn].lociter.take().expect("make_free: missing lociter"); self.loc_tree.remove(&lk); - if lk.flagclass.raw() != 0 { + if lk.flag_class().raw() != 0 { self.def_tree.remove(&DefKey::from(lk)); } - self.arena[vn].set_def(None); // Clear things that make vn non-free - self.arena[vn].def_seqnum = None; - self.arena[vn].clear_flags( - varnode_flags::insert | varnode_flags::input | varnode_flags::indirect_creation, - ); + self.arena[vn].set_free(); // Re-insert as free varnode. - let lk = self.loc_key_of(vn); + let seqnum = SeqKey { uniq: self.arena[vn].create_index, ..SeqKey::default() }; + let lk = lk.with_class(flag_class_of(0), seqnum); self.arena[vn].lociter = Some(lk); self.loc_tree.insert(lk, vn); } + /// Clear a definition for immediate destruction, retaining the indexed key. + /// Teardown must not query either index before calling `destroy`. + pub(crate) fn make_free_for_destroy(&mut self, vn: VarnodeId) { + let v = &mut self.arena[vn]; + v.lociter.as_ref().expect("make_free: missing lociter"); + v.set_free(); + } + /// Create a Varnode already in the def tree as if it were the output of the /// given op (C++ `createDef`). pub fn create_def( @@ -1806,13 +1834,19 @@ impl VarnodeBank { def: DefOpInfo, replace_reads: &mut ReplaceReads, ) -> KunaResult { + let lk = LocKey::new( + AddrKey::of(&m), + s, + flag_class_of(varnode_flags::written), + SeqKey::of(&def.seqnum), + ); let mut vn = Varnode::new(s, m, ct); vn.create_index = self.create_index; self.create_index = self.create_index.wadd(1); vn.set_def(Some(def.id)); vn.def_seqnum = Some(def.seqnum); let id = self.arena.insert(vn); - self.xref(id, replace_reads) + self.xref(id, lk, replace_reads) } /// Create a unique Varnode as output of the given op (C++ `createDefUnique`). @@ -1866,21 +1900,21 @@ impl VarnodeBank { end: &Address, ) -> impl Iterator + '_ { let space_index = start.get_space().map(|s| s.get_index()); - let begin = LocProbe::Lower(LocKey { - addr: AddrKey::of(start), - size: 0, - flagclass: flag_class_of(varnode_flags::input), - seqnum: SeqKey::of(&(SeqNum::default())), - }); + let begin = LocProbe::Lower(LocKey::new( + AddrKey::of(start), + 0, + flag_class_of(varnode_flags::input), + SeqKey::of(&(SeqNum::default())), + )); let finish = if end.get_offset() < start.get_offset() { LocProbe::End } else { - LocProbe::Lower(LocKey { - addr: AddrKey::of(end), - size: 0, - flagclass: flag_class_of(varnode_flags::input), - seqnum: SeqKey::of(&(SeqNum::default())), - }) + LocProbe::Lower(LocKey::new( + AddrKey::of(end), + 0, + flag_class_of(varnode_flags::input), + SeqKey::of(&(SeqNum::default())), + )) }; self.iter_loc_probe(begin, finish).take_while(move |&id| { self.arena[id].get_addr().get_space().map(|s| s.get_index()) == space_index @@ -1890,7 +1924,7 @@ impl VarnodeBank { /// Iterate all varnode ids in definition order (C++ `beginDef()`..`endDef()`). pub fn iter_def(&self) -> impl Iterator + '_ { self.def_tree.values().copied().chain( - self.loc_tree.iter().filter_map(|(k, &id)| (k.flagclass.raw() == 0).then_some(id)) + self.loc_tree.iter().filter_map(|(k, &id)| (k.flag_class().raw() == 0).then_some(id)) ) } @@ -1972,58 +2006,58 @@ impl VarnodeBank { fn begin_loc_flag(&self, s: int4, addr: &Address, fl: uint4) -> LocProbe { if fl == varnode_flags::input { // searchvn{size=s, loc=addr, flags=input} ; lower_bound - return LocProbe::Lower(LocKey { - addr: AddrKey::of(addr), - size: s, - flagclass: flag_class_of(varnode_flags::input), - seqnum: SeqKey::of(&(SeqNum::default())), - }); + return LocProbe::Lower(LocKey::new( + AddrKey::of(addr), + s, + flag_class_of(varnode_flags::input), + SeqKey::of(&(SeqNum::default())), + )); } if fl == varnode_flags::written { // searchvn{size=s, loc=addr, flags=written, def=&searchop(minimal seq)} ; lower_bound - return LocProbe::Lower(LocKey { - addr: AddrKey::of(addr), - size: s, - flagclass: flag_class_of(varnode_flags::written), - seqnum: SeqKey::of(&(SeqNum::new_extreme(mach_extreme::m_minimal))), - }); + return LocProbe::Lower(LocKey::new( + AddrKey::of(addr), + s, + flag_class_of(varnode_flags::written), + SeqKey::of(&(SeqNum::new_extreme(mach_extreme::m_minimal))), + )); } // fl == 0 (free): searchvn{size=s, loc=addr, flags=written, def=&searchop(maximal seq)} ; upper_bound - LocProbe::Upper(LocKey { - addr: AddrKey::of(addr), - size: s, - flagclass: flag_class_of(varnode_flags::written), - seqnum: SeqKey::of(&(SeqNum::new_extreme(mach_extreme::m_maximal))), - }) + LocProbe::Upper(LocKey::new( + AddrKey::of(addr), + s, + flag_class_of(varnode_flags::written), + SeqKey::of(&(SeqNum::new_extreme(mach_extreme::m_maximal))), + )) } /// `endLoc(int4 s,const Address &addr,uint4 fl)` (`varnode.cc:1712`). fn end_loc_flag(&self, s: int4, addr: &Address, fl: uint4) -> LocProbe { if fl == varnode_flags::written { // searchvn{loc=addr, size=s, flags=written, def=&searchop(maximal seq)} ; upper_bound - return LocProbe::Upper(LocKey { - addr: AddrKey::of(addr), - size: s, - flagclass: flag_class_of(varnode_flags::written), - seqnum: SeqKey::of(&(SeqNum::new_extreme(mach_extreme::m_maximal))), - }); + return LocProbe::Upper(LocKey::new( + AddrKey::of(addr), + s, + flag_class_of(varnode_flags::written), + SeqKey::of(&(SeqNum::new_extreme(mach_extreme::m_maximal))), + )); } if fl == varnode_flags::input { // searchvn{loc=addr, size=s, flags=input} ; upper_bound - return LocProbe::Upper(LocKey { - addr: AddrKey::of(addr), - size: s, - flagclass: flag_class_of(varnode_flags::input), - seqnum: SeqKey::of(&(SeqNum::default())), - }); + return LocProbe::Upper(LocKey::new( + AddrKey::of(addr), + s, + flag_class_of(varnode_flags::input), + SeqKey::of(&(SeqNum::default())), + )); } // fl == 0 (free): searchvn{loc=addr, size=s+1, flags=input} ; lower_bound - LocProbe::Lower(LocKey { - addr: AddrKey::of(addr), - size: s + 1, - flagclass: flag_class_of(varnode_flags::input), - seqnum: SeqKey::of(&(SeqNum::default())), - }) + LocProbe::Lower(LocKey::new( + AddrKey::of(addr), + s + 1, + flag_class_of(varnode_flags::input), + SeqKey::of(&(SeqNum::default())), + )) } /// `beginLoc(int4 s,const Address &addr,const Address &pc,uintm uniq)` @@ -2032,24 +2066,24 @@ impl VarnodeBank { // find earliest let u = uniq.unwrap_or(0); // searchvn{size=s, loc=addr, flags=written, def=&searchop(pc,u)} ; lower_bound - LocProbe::Lower(LocKey { - addr: AddrKey::of(addr), - size: s, - flagclass: flag_class_of(varnode_flags::written), - seqnum: SeqKey::of(&(SeqNum::new(pc.clone(), u))), - }) + LocProbe::Lower(LocKey::new( + AddrKey::of(addr), + s, + flag_class_of(varnode_flags::written), + SeqKey::of(&(SeqNum::new(pc.clone(), u))), + )) } /// `endLoc(int4 s,const Address &addr,const Address &pc,uintm uniq)` /// (`varnode.cc:1781`). fn end_loc_pc(&self, s: int4, addr: &Address, pc: &Address, uniq: uintm) -> LocProbe { // (the C++ does NOT remap ~0 here) ; upper_bound - LocProbe::Upper(LocKey { - addr: AddrKey::of(addr), - size: s, - flagclass: flag_class_of(varnode_flags::written), - seqnum: SeqKey::of(&(SeqNum::new(pc.clone(), uniq))), - }) + LocProbe::Upper(LocKey::new( + AddrKey::of(addr), + s, + flag_class_of(varnode_flags::written), + SeqKey::of(&(SeqNum::new(pc.clone(), uniq))), + )) } // --- Public location range queries (the beginLoc/endLoc overloads) ----- @@ -2065,12 +2099,12 @@ impl VarnodeBank { /// next-space-or-Max endpoint. pub fn loc_space_ids(&self, space: &Rc) -> Vec { let begin = Address::new(Rc::clone(space), 0); - let lo = LocProbe::Lower(LocKey { - addr: AddrKey::of(&begin), - size: 0, - flagclass: flag_class_of(varnode_flags::input), - seqnum: SeqKey::of(&(SeqNum::default())), - }); + let lo = LocProbe::Lower(LocKey::new( + AddrKey::of(&begin), + 0, + flag_class_of(varnode_flags::input), + SeqKey::of(&(SeqNum::default())), + )); let mut out = Vec::new(); for id in self.iter_loc_probe(lo, LocProbe::End) { if !Rc::ptr_eq(self.arena[id].get_space(), space) { @@ -2089,19 +2123,19 @@ impl VarnodeBank { addr: &Address, ) -> impl Iterator + '_ { // beginLoc: searchvn{size=s, loc=addr} ; lower_bound (flag = input) - let begin = LocProbe::Lower(LocKey { - addr: AddrKey::of(addr), - size: s, - flagclass: flag_class_of(varnode_flags::input), - seqnum: SeqKey::of(&(SeqNum::default())), - }); + let begin = LocProbe::Lower(LocKey::new( + AddrKey::of(addr), + s, + flag_class_of(varnode_flags::input), + SeqKey::of(&(SeqNum::default())), + )); // endLoc: searchvn{size=s+1, loc=addr} ; lower_bound - let end = LocProbe::Lower(LocKey { - addr: AddrKey::of(addr), - size: s + 1, - flagclass: flag_class_of(varnode_flags::input), - seqnum: SeqKey::of(&(SeqNum::default())), - }); + let end = LocProbe::Lower(LocKey::new( + AddrKey::of(addr), + s + 1, + flag_class_of(varnode_flags::input), + SeqKey::of(&(SeqNum::default())), + )); self.iter_loc_probe(begin, end) } @@ -2165,7 +2199,7 @@ impl VarnodeBank { end.into_iter().flat_map(move |end| { self.loc_tree .range((start, end)) - .filter_map(|(k, &id)| (k.flagclass.raw() == 0).then_some(id)) + .filter_map(|(k, &id)| (k.flag_class().raw() == 0).then_some(id)) }) } @@ -2507,14 +2541,14 @@ mod tests { } fn make_loc_key(addr: Address, size: int4, fl: uint4, seq: SeqNum, ci: uint4) -> LocKey { - LocKey { - addr: AddrKey::of(&addr), + LocKey::new( + AddrKey::of(&addr), size, - flagclass: flag_class_of(fl), - seqnum: if fl & (varnode_flags::input | varnode_flags::written) == 0 { + flag_class_of(fl), + if fl & (varnode_flags::input | varnode_flags::written) == 0 { SeqKey { uniq: ci, ..SeqKey::of(&seq) } } else { SeqKey::of(&seq) }, - } + ) } fn make_def_key(addr: Address, size: int4, fl: uint4, seq: SeqNum, ci: uint4) -> DefKey { DefKey { @@ -2559,6 +2593,49 @@ mod tests { assert_eq!(SeqKey::default(), SeqKey::of(&SeqNum::default())); } + #[test] + fn packed_size_class_preserves_signed_order_and_roundtrips() { + let addr = AddrKey::of(&Address::default()); + let mut keys = Vec::new(); + for size in [int4::MIN, -1, 0, 1, 1000000, int4::MAX] { + for flags in [ + 0, + varnode_flags::input, + varnode_flags::written, + varnode_flags::input | varnode_flags::written, + ] { + let key = LocKey::new(addr, size, flag_class_of(flags), SeqKey::default()); + let def = DefKey::from(key); + assert_eq!(def.size, size); + assert_eq!(def.flagclass.raw(), flags); + assert_eq!(LocKey::from(def), key); + for next in [ + 0, + varnode_flags::input, + varnode_flags::written, + varnode_flags::input | varnode_flags::written, + ] { + let seq = SeqKey { + uniq: uint4::MAX, + ..SeqKey::default() + }; + let changed = DefKey::from(key.with_class(flag_class_of(next), seq)); + assert_eq!(changed.addr, addr); + assert_eq!(changed.size, size); + assert_eq!(changed.flagclass.raw(), next); + assert_eq!(changed.seqnum, seq); + } + keys.push((size, flags, key)); + } + } + for &(sa, fa, a) in &keys { + for &(sb, fb, b) in &keys { + let expected = sa.cmp(&sb).then_with(|| fa.wsub(1).cmp(&fb.wsub(1))); + assert_eq!(a.cmp(&b), expected); + } + } + } + /// GOLDEN (varnodesort): the `tests/golden/vectors/` corpus has no varnode /// section (only addrsort/opbehavior/float/lift — verified against the /// goldengen emitter), so per the W3 item brief we pin the comparators with @@ -2576,7 +2653,10 @@ mod tests { Address::new(space(&m, 3), 0x10), // register:0x10 ]; let sizes: [int4; 3] = [1, 4, 8]; - let classes: [uint4; 3] = [0, varnode_flags::input, varnode_flags::written]; + let classes: [uint4; 4] = [ + 0, varnode_flags::input, varnode_flags::written, + varnode_flags::input | varnode_flags::written, + ]; let seqs: Vec = vec![ SeqNum::new(Address::new(space(&m, 2), 0x100), 0), SeqNum::new(Address::new(space(&m, 2), 0x100), 5), @@ -2821,10 +2901,11 @@ mod tests { #[test] fn free_definition_ranges_match_full_tree() { eprintln!( - "index layout: LocKey={} DefKey={} Varnode={}", + "index layout: LocKey={} DefKey={} Varnode={} snapshot={}", std::mem::size_of::(), std::mem::size_of::(), - std::mem::size_of::() + std::mem::size_of::(), + std::mem::size_of::>() ); let m = build_manager(); let mut bank = VarnodeBank::new(&m, 0).unwrap(); @@ -3089,6 +3170,29 @@ mod tests { assert_eq!(frees, vec![w]); } + #[test] + fn immediate_destroy_removes_the_written_snapshot() { + let m = build_manager(); + let mut bank = VarnodeBank::new(&m, 0).unwrap(); + let addr = Address::new(space(&m, 2), 0x700); + let written = bank.create(4, addr.clone(), dt(4)); + let free = bank.create(4, addr, dt(4)); + let mut idc = 0; + let def = def_info(&mut idc, Address::new(space(&m, 2), 0x30), 1); + bank.set_def(written, def, &mut no_replace()).unwrap(); + assert!(bank.destroy(written).is_err()); + let key = bank.arena[written].lociter.unwrap(); + bank.make_free_for_destroy(written); + assert!(bank.arena[written].is_free()); + assert!(bank.arena[written].get_def().is_none()); + assert_eq!(bank.arena[written].lociter, Some(key)); + assert_eq!(bank.loc_tree.get(&key), Some(&written)); + bank.destroy(written).unwrap(); + assert_eq!(bank.iter_loc().collect::>(), vec![free]); + assert_eq!(bank.iter_def().collect::>(), vec![free]); + assert_eq!(bank.num_varnodes(), 1); + } + /// destroy removes a free varnode; rejects an integrated one. #[test] fn destroy_semantics() { diff --git a/docs/spec/00-overview.md b/docs/spec/00-overview.md index b0f4fdd10e..68904f1297 100644 --- a/docs/spec/00-overview.md +++ b/docs/spec/00-overview.md @@ -2782,13 +2782,31 @@ ending in the input or written class never traverse that view Creation inserts a free node only in the location tree. Becoming input or written adds its definition entry; becoming free removes that entry and rekeys its location entry. Destroying a free node removes only its location entry. Both -logical orders retain the same ids and comparator semantics. +logical orders retain the same ids and comparator semantics. Immediate teardown +(`decompiler/crates/kuna-decomp/src/substrate/funcdata_varnode.rs (destroy_varnode)`) +clears the definition and class flags without reinserting a temporary free entry. +High-variable and descendant cleanup read nodes by id, then destruction erases +the retained indexed snapshot; no location or definition query runs in between +(`decompiler/crates/kuna-decomp/src/substrate/varnode.rs (make_free_for_destroy)`). The keys (`decompiler/crates/kuna-decomp/src/substrate/varnode.rs (LocKey, DefKey)`) store integer ordering fields rather than `Address` or `SeqNum` objects. Sentinel rank and signed space index share a packed prefix, followed by the offset; flipping the index's sign bit preserves signed order within each rank. The -input/written/free wrap ordering is computed once when a key is constructed. +input/written/free wrap ordering is computed once when a key is constructed, +including the combined input-and-written mask. Location keys also pack signed +size, sign-biased, and the preordered class into a single unsigned prefix. +One comparison therefore retains the original size-then-class ordering, +including negative sizes and extreme range bounds. Converting a location key +to a definition key restores the signed size and class without consulting the +live Varnode. +Storage address and size are immutable within a Varnode. Changing its input, +written, or free class therefore reuses those fields from the old snapshot and +replaces only the class and sequence payload, avoiding another address-space +lookup (`decompiler/crates/kuna-decomp/src/substrate/varnode.rs (LocKey::with_class)`). +Creation constructs the key directly from the allocation's address, size, +creation index, and optional definition; its free or written class is already +known. A newly allocated node does not need a separate general key reconstruction. Written keys retain SeqNum's uniq-only equality check before comparing PC and time; free keys use the same uniq slot for their creation index. Each Varnode caches one location snapshot, from which definition keys are copied, so removals From b1259a493019e2e2e4c084fead162cab4f7226eb Mon Sep 17 00:00:00 2001 From: mgostIH <13892786+mgostIH@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:11:29 +0200 Subject: [PATCH 3/6] [AUTOMATED] Cache full IR keys for inline arena checks Keep SlotMap storage and ID allocation while validating Varnode and op accesses against a compact array of full FFI keys. Invalidate cached keys before removal or clear, and retain SlotMap validation on debug reads. Expose the checked bank accessors and two hot record getters to inlining. The IR panel uses 5.56% fewer user instructions and smoke uses 6.98% fewer; all outputs are identical. All 52 focused arena/bank tests and the spec, test-layout, formatting, and diff checks pass. Core measurements continue. --- .../src/substrate/kuna_checkedarena.rs | 101 ++++++++++++++++++ .../src/substrate/kuna_checkedarena/tests.rs | 60 +++++++++++ .../crates/kuna-decomp/src/substrate/mod.rs | 1 + .../crates/kuna-decomp/src/substrate/op.rs | 31 +++++- .../kuna-decomp/src/substrate/varnode.rs | 35 +++++- docs/spec/00-overview.md | 19 ++++ 6 files changed, 243 insertions(+), 4 deletions(-) create mode 100644 decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena.rs create mode 100644 decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena/tests.rs diff --git a/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena.rs b/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena.rs new file mode 100644 index 0000000000..f36cb44efe --- /dev/null +++ b/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena.rs @@ -0,0 +1,101 @@ +//! SlotMap storage with compact, inlineable live-id validation in every build. +//! +//! Only this wrapper changes slot membership. Cached full FFI encodings identify +//! live keys; removal and clear invalidate them before changing storage. The +//! vacancy marker's index is never allocated, so it cannot match an in-bounds key. +//! SlotMap additionally checks the original generation on reads in debug builds. + +use slotmap::{Key, SlotMap}; +use std::ops::{Index, IndexMut}; + +const VACANT: u64 = u64::MAX; + +pub(crate) struct Arena { + values: SlotMap, + keys: Vec, +} + +/// The locked SlotMap version encodes the slot index in the low 32 bits. +#[inline(always)] +fn index(encoded: u64) -> usize { + encoded as u32 as usize +} + +impl Arena { + pub(crate) fn with_key() -> Self { + Self { + values: SlotMap::with_key(), + keys: Vec::new(), + } + } + + pub(crate) fn insert(&mut self, value: V) -> K { + let id = self.values.insert(value); + let encoded = id.data().as_ffi(); + let slot = index(encoded); + assert!(slot < u32::MAX as usize, "SlotMap key encoding changed"); + if self.keys.len() <= slot { + self.keys.resize(slot + 1, VACANT); + } + assert_eq!(self.keys[slot], VACANT, "SlotMap key encoding changed"); + self.keys[slot] = encoded; + id + } + + pub(crate) fn remove(&mut self, id: K) -> Option { + let encoded = id.data().as_ffi(); + let key = self.keys.get_mut(index(encoded))?; + if *key != encoded { + return None; + } + *key = VACANT; + self.values.remove(id) + } + + pub(crate) fn clear(&mut self) { + self.keys.fill(VACANT); + self.values.clear(); + } + + #[inline(always)] + pub(crate) fn get(&self, id: K) -> Option<&V> { + let encoded = id.data().as_ffi(); + if self.keys.get(index(encoded)) != Some(&encoded) { + return None; + } + Some(unsafe { self.values.get_unchecked(id) }) + } + + #[inline(always)] + pub(crate) fn get_mut(&mut self, id: K) -> Option<&mut V> { + let encoded = id.data().as_ffi(); + if self.keys.get(index(encoded)) != Some(&encoded) { + return None; + } + Some(unsafe { self.values.get_unchecked_mut(id) }) + } + + #[cfg(test)] + pub(crate) fn iter(&self) -> impl Iterator { + self.values.iter() + } +} + +impl Index for Arena { + type Output = V; + + #[inline(always)] + fn index(&self, id: K) -> &V { + self.get(id).expect("invalid SlotMap key used") + } +} + +impl IndexMut for Arena { + #[inline(always)] + fn index_mut(&mut self, id: K) -> &mut V { + self.get_mut(id).expect("invalid SlotMap key used") + } +} + +#[cfg(test)] +mod tests; diff --git a/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena/tests.rs b/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena/tests.rs new file mode 100644 index 0000000000..2899aabe71 --- /dev/null +++ b/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena/tests.rs @@ -0,0 +1,60 @@ +use super::*; + +slotmap::new_key_type! { struct TestId; } + +#[test] +fn cached_ids_match_slotmap_through_churn() { + let mut arena = Arena::::with_key(); + let mut reference = SlotMap::::with_key(); + let mut ids = vec![ + TestId::null(), + slotmap::KeyData::from_ffi(u64::MAX).into(), + slotmap::KeyData::from_ffi(0).into(), + ]; + let mut random = 1u64; + for step in 0..1024u64 { + random = random.wrapping_mul(6364136223846793005).wrapping_add(1); + let id = ids[(random >> 32) as usize % ids.len()]; + if step % 127 == 126 { + arena.clear(); + reference.clear(); + } else if step % 3 == 0 { + assert_eq!(arena.remove(id), reference.remove(id)); + } else { + let allocated = arena.insert(step); + assert_eq!(allocated, reference.insert(step)); + ids.push(allocated); + } + if let Some(value) = arena.get_mut(id) { + *value ^= step; + } + if let Some(value) = reference.get_mut(id) { + *value ^= step; + } + for &key in &ids { + assert_eq!(arena.get(key), reference.get(key), "step {step}, {key:?}"); + assert_eq!(arena.get_mut(key), reference.get_mut(key)); + } + assert_eq!( + arena.iter().collect::>(), + reference.iter().collect::>() + ); + } +} + +#[test] +fn indexing_keeps_the_original_validity_checks() { + let mut arena = Arena::::with_key(); + let stale = arena.insert(4); + assert_eq!(arena[stale], 4); + arena[stale] = 9; + assert_eq!(arena.remove(stale), Some(9)); + let live = arena.insert(12); + assert_ne!(stale, live); + assert!(std::panic::catch_unwind(|| arena[stale]).is_err()); + assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + arena[stale] = 1; + })) + .is_err()); + assert_eq!(arena[live], 12); +} diff --git a/decompiler/crates/kuna-decomp/src/substrate/mod.rs b/decompiler/crates/kuna-decomp/src/substrate/mod.rs index 00528cb591..802a184c8d 100644 --- a/decompiler/crates/kuna-decomp/src/substrate/mod.rs +++ b/decompiler/crates/kuna-decomp/src/substrate/mod.rs @@ -17,3 +17,4 @@ pub mod context; pub mod expression; pub mod transform; pub mod addtreestate; +pub(crate) mod kuna_checkedarena; diff --git a/decompiler/crates/kuna-decomp/src/substrate/op.rs b/decompiler/crates/kuna-decomp/src/substrate/op.rs index 5647925b9d..db861dc076 100644 --- a/decompiler/crates/kuna-decomp/src/substrate/op.rs +++ b/decompiler/crates/kuna-decomp/src/substrate/op.rs @@ -428,7 +428,7 @@ impl Iterator for IntrusiveIter<'_> { } /// The op arena alias (ADR 0001): a slotmap keyed by [`OpId`]. -type OpArena = slotmap::SlotMap; +type OpArena = crate::kuna_checkedarena::Arena; // --------------------------------------------------------------------------- // PcodeOp @@ -873,6 +873,7 @@ impl PcodeOp { /// Panics if the opcode is unset (the C++ would dereference a null `opcode` /// — an internal-invariant violation, ADR 0004): `code()` is only ever /// called on ops that have been through `PcodeOpBank::change_opcode`. + #[inline] pub fn code(&self) -> OpCode { self.opcode .as_ref() @@ -1637,10 +1638,12 @@ impl PcodeOpBank { } /// Borrow a PcodeOp by id. + #[inline(always)] pub fn get(&self, id: OpId) -> Option<&PcodeOp> { self.arena.get(id) } /// Mutably borrow a PcodeOp by id. + #[inline(always)] pub fn get_mut(&mut self, id: OpId) -> Option<&mut PcodeOp> { self.arena.get_mut(id) } @@ -2093,6 +2096,32 @@ mod tests { TypeOp::new(opc, 0, format!("{opc:?}")) } + #[test] + fn accessors_keep_id_validity() { + let m = build_manager(); + let mut bank = PcodeOpBank::new(); + let retired = bank.create_at(1, ram(&m, 0x100)); + bank.get_mut(retired).unwrap().set_order(7); + assert_eq!(bank.get(retired).unwrap().get_seq_num().get_order(), 7); + bank.destroy(retired); + assert!(bank.get(retired).is_some()); + assert!(bank.get_mut(retired).is_some()); + + bank.clear(); + let live = bank.create_at(1, ram(&m, 0x200)); + assert_ne!(retired, live); + let null = ::null(); + for id in [retired, null] { + assert!(bank.get(id).is_none()); + assert!(bank.get_mut(id).is_none()); + } + assert!(bank.get(live).is_some()); + assert!(bank.get_mut(live).is_some()); + bank.clear(); + assert!(bank.get(live).is_none()); + assert!(bank.get_mut(live).is_none()); + } + // --- allocation-order monotonicity ------------------------------------ #[test] diff --git a/decompiler/crates/kuna-decomp/src/substrate/varnode.rs b/decompiler/crates/kuna-decomp/src/substrate/varnode.rs index 1de4d89224..7cc4df571f 100644 --- a/decompiler/crates/kuna-decomp/src/substrate/varnode.rs +++ b/decompiler/crates/kuna-decomp/src/substrate/varnode.rs @@ -36,7 +36,7 @@ use kuna_base::address::{mach_extreme, Address, SeqNum}; use kuna_base::error::{KunaError, KunaResult}; use kuna_base::space::{spacetype, AddrSpace, AddrSpaceManager}; use kuna_base::types::{int2, int4, uint2, uint4, uintb, uintm, Wrap}; -use slotmap::SlotMap; +use crate::kuna_checkedarena::Arena; use smallvec::SmallVec; use crate::cover::{Cover, CoverContext}; @@ -1025,6 +1025,7 @@ impl Varnode { == varnode_flags::insert } /// C++ `hasNoDescend`. + #[inline] pub fn has_no_descend(&self) -> bool { self.descend.is_empty() } @@ -1583,7 +1584,7 @@ pub struct VarnodeBank { /// Number of varnodes created (C++ `create_index`) create_index: uint4, /// The varnode arena (ADR 0001): the bank owns the store - arena: SlotMap, + arena: Arena, /// Varnodes sorted by location then def (C++ `loc_tree`) loc_tree: BTreeMap, /// Inputs and written Varnodes sorted by def then location (C++ `def_tree`). @@ -1619,7 +1620,7 @@ impl VarnodeBank { uniqbase: uniq_start, uniqid: uniq_start, create_index: 0, - arena: SlotMap::with_key(), + arena: Arena::with_key(), loc_tree: BTreeMap::new(), def_tree: BTreeMap::new(), uniq_space, @@ -1653,10 +1654,12 @@ impl VarnodeBank { } /// Borrow a Varnode by id. + #[inline(always)] pub fn get(&self, id: VarnodeId) -> Option<&Varnode> { self.arena.get(id) } /// Mutably borrow a Varnode by id. + #[inline(always)] pub fn get_mut(&mut self, id: VarnodeId) -> Option<&mut Varnode> { self.arena.get_mut(id) } @@ -2468,6 +2471,32 @@ mod tests { |_: &mut VarnodeBank, _: VarnodeId, _: VarnodeId| -> KunaResult<()> { Ok(()) } } + #[test] + fn accessors_keep_id_validity() { + let m = build_manager(); + let mut bank = VarnodeBank::new(&m, 0).unwrap(); + let addr = Address::new(space(&m, 2), 0x800); + let stale = bank.create(4, addr.clone(), dt(4)); + bank.get_mut(stale).unwrap().set_consume(7); + assert_eq!(bank.get(stale).unwrap().get_consume(), 7); + bank.destroy(stale).unwrap(); + assert!(bank.get(stale).is_none()); + assert!(bank.get_mut(stale).is_none()); + + let live = bank.create(4, addr, dt(4)); + assert_ne!(stale, live); + let null = ::null(); + for id in [stale, null] { + assert!(bank.get(id).is_none()); + assert!(bank.get_mut(id).is_none()); + } + assert!(bank.get(live).is_some()); + assert!(bank.get_mut(live).is_some()); + bank.clear(); + assert!(bank.get(live).is_none()); + assert!(bank.get_mut(live).is_none()); + } + // -- Reference comparators: a literal re-transcription of varnode.cc:34-79 // used to cross-check LocKey/DefKey::cmp exhaustively. `seq_a != seq_b` // and `seq_a < seq_b` use SeqNum's own (uniq-only Eq, (pc,uniq) Ord), diff --git a/docs/spec/00-overview.md b/docs/spec/00-overview.md index 68904f1297..8c5fb32514 100644 --- a/docs/spec/00-overview.md +++ b/docs/spec/00-overview.md @@ -2766,6 +2766,25 @@ the CFG, and `sblocks`, the structuring tree — physically distinct, seeded as `BlockCopy` mirror of the CFG when structuring begins (`decompiler/crates/kuna-decomp/src/substrate/funcdata.rs (seed_sblocks_copy)`). +The Varnode and op banks' `get` and `get_mut` accessors are inlineable, exposing +repeated accesses to the optimizer without changing their lookup contract. +Bounds checks and full-id comparisons remain enabled in release and debug builds; +stale and null ids still return `None`. Retired PcodeOps remain accessible until +the bank is cleared. The checked arena +(`decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena.rs`) retains +SlotMap storage and allocation order, and caches the full FFI encoding of each +live id in a compact array indexed by slot. A lookup checks that array's bounds and +compares the full cached id before reading the original slot. Removal and clear +invalidate cached ids before modifying storage; only the arena wrapper can change +slot membership. The all-ones encoding marks vacancies. Its index is `u32::MAX`, +which the cache rejects during insertion, so a checked id cannot match a vacancy. +This adds eight bytes per slot, while avoiding out-of-line +SlotMap validation calls and allowing repeated checks to fold together. Debug +builds also perform SlotMap's original checks at the final slot access. +The record getters `PcodeOp::code` and +`Varnode::has_no_descend` are inlineable as well; `code` still panics if the +opcode has not been assigned. + Registering an input varnode also applies the prototype's saved-register and return-address effects. This is unconditional registration behavior, not a separate feature gate; `funcdata_varnode.rs (apply_input_effect_marking)` uses From 3e42e5ccdcc90e1c7bd9b20647671fde5406fa63 Mon Sep 17 00:00:00 2001 From: mahaloz Date: Thu, 29 Oct 2026 16:29:49 +0000 Subject: [PATCH 4/6] [AUTOMATED] Store each arena key beside its value, not in a side array The separate key array made lookups about 2% slower in user CPU on an Ice Lake-SP Xeon. Every lookup loaded one more cache line, and it had to build the 64-bit key from its two halves first. SlotMap's version word already sat next to the flag words that most lookups read. Each slot now stores its full key word directly after the value, so the inline check reads a line the caller already touches. Allocation, free-list order and versions replicate slotmap::SlotMap, so ids and output are unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013BBaWTixHX2E5cBwneY5aW --- .../src/substrate/kuna_checkedarena.rs | 129 +++++++++++++----- .../src/substrate/kuna_checkedarena/tests.rs | 1 + docs/spec/00-overview.md | 21 +-- 3 files changed, 107 insertions(+), 44 deletions(-) diff --git a/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena.rs b/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena.rs index f36cb44efe..d5b0fb17e0 100644 --- a/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena.rs +++ b/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena.rs @@ -1,83 +1,144 @@ -//! SlotMap storage with compact, inlineable live-id validation in every build. +//! Slot storage with compact, inlineable live-id validation in every build. //! -//! Only this wrapper changes slot membership. Cached full FFI encodings identify -//! live keys; removal and clear invalidate them before changing storage. The -//! vacancy marker's index is never allocated, so it cannot match an in-bounds key. -//! SlotMap additionally checks the original generation on reads in debug builds. +//! Each slot stores its value followed by a full 64-bit key word. An occupied +//! slot's word is the FFI encoding of its live id, so a lookup compares one word +//! that sits beside the value's trailing fields instead of reading a separate +//! array. A vacant slot's word holds an even version and the next free index; +//! that index never names the slot itself, so no id can match a vacancy. The +//! sentinel slot's word is zero, which no id encodes. Allocation, free-list +//! order and versions replicate `slotmap::SlotMap`, so ids are unchanged. -use slotmap::{Key, SlotMap}; +use slotmap::{Key, KeyData}; +use std::marker::PhantomData; +use std::mem::MaybeUninit; use std::ops::{Index, IndexMut}; -const VACANT: u64 = u64::MAX; +#[repr(C)] +struct Slot { + value: MaybeUninit, + key: u64, +} + +impl Slot { + #[inline(always)] + fn occupied(&self) -> bool { + (self.key >> 32) & 1 == 1 + } +} pub(crate) struct Arena { - values: SlotMap, - keys: Vec, + slots: Vec>, + free_head: u32, + _k: PhantomData K>, } -/// The locked SlotMap version encodes the slot index in the low 32 bits. #[inline(always)] -fn index(encoded: u64) -> usize { - encoded as u32 as usize +fn encode(version: u32, low: u32) -> u64 { + (u64::from(version) << 32) | u64::from(low) } impl Arena { pub(crate) fn with_key() -> Self { Self { - values: SlotMap::with_key(), - keys: Vec::new(), + slots: vec![Slot { + value: MaybeUninit::uninit(), + key: 0, + }], + free_head: 1, + _k: PhantomData, } } pub(crate) fn insert(&mut self, value: V) -> K { - let id = self.values.insert(value); - let encoded = id.data().as_ffi(); - let slot = index(encoded); - assert!(slot < u32::MAX as usize, "SlotMap key encoding changed"); - if self.keys.len() <= slot { - self.keys.resize(slot + 1, VACANT); + let idx = self.free_head; + if let Some(slot) = self.slots.get_mut(idx as usize) { + let next = slot.key as u32; + let key = encode((slot.key >> 32) as u32 | 1, idx); + slot.value.write(value); + slot.key = key; + self.free_head = next; + return KeyData::from_ffi(key).into(); } - assert_eq!(self.keys[slot], VACANT, "SlotMap key encoding changed"); - self.keys[slot] = encoded; - id + assert!(self.slots.len() < u32::MAX as usize, "SlotMap is full"); + let idx = self.slots.len() as u32; + let key = encode(1, idx); + self.slots.push(Slot { + value: MaybeUninit::new(value), + key, + }); + self.free_head = idx + 1; + KeyData::from_ffi(key).into() + } + + fn take(&mut self, idx: usize) -> V { + let free_head = self.free_head; + let slot = &mut self.slots[idx]; + debug_assert!(slot.occupied()); + let value = unsafe { slot.value.assume_init_read() }; + slot.key = encode(((slot.key >> 32) as u32).wrapping_add(1), free_head); + self.free_head = idx as u32; + value } pub(crate) fn remove(&mut self, id: K) -> Option { let encoded = id.data().as_ffi(); - let key = self.keys.get_mut(index(encoded))?; - if *key != encoded { + let idx = encoded as u32 as usize; + if self.slots.get(idx)?.key != encoded { return None; } - *key = VACANT; - self.values.remove(id) + Some(self.take(idx)) } pub(crate) fn clear(&mut self) { - self.keys.fill(VACANT); - self.values.clear(); + for idx in 1..self.slots.len() { + if self.slots[idx].occupied() { + drop(self.take(idx)); + } + } } #[inline(always)] pub(crate) fn get(&self, id: K) -> Option<&V> { let encoded = id.data().as_ffi(); - if self.keys.get(index(encoded)) != Some(&encoded) { + let slot = self.slots.get(encoded as u32 as usize)?; + if slot.key != encoded { return None; } - Some(unsafe { self.values.get_unchecked(id) }) + Some(unsafe { slot.value.assume_init_ref() }) } #[inline(always)] pub(crate) fn get_mut(&mut self, id: K) -> Option<&mut V> { let encoded = id.data().as_ffi(); - if self.keys.get(index(encoded)) != Some(&encoded) { + let slot = self.slots.get_mut(encoded as u32 as usize)?; + if slot.key != encoded { return None; } - Some(unsafe { self.values.get_unchecked_mut(id) }) + Some(unsafe { slot.value.assume_init_mut() }) } #[cfg(test)] pub(crate) fn iter(&self) -> impl Iterator { - self.values.iter() + self.slots + .iter() + .filter(|slot| slot.occupied()) + .map(|slot| { + (KeyData::from_ffi(slot.key).into(), unsafe { + slot.value.assume_init_ref() + }) + }) + } +} + +impl Drop for Arena { + fn drop(&mut self) { + if std::mem::needs_drop::() { + for slot in &mut self.slots { + if slot.occupied() { + unsafe { slot.value.assume_init_drop() }; + } + } + } } } diff --git a/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena/tests.rs b/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena/tests.rs index 2899aabe71..290a6e669c 100644 --- a/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena/tests.rs +++ b/decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena/tests.rs @@ -1,4 +1,5 @@ use super::*; +use slotmap::SlotMap; slotmap::new_key_type! { struct TestId; } diff --git a/docs/spec/00-overview.md b/docs/spec/00-overview.md index 92081ddb65..f7bfba6b87 100644 --- a/docs/spec/00-overview.md +++ b/docs/spec/00-overview.md @@ -2936,16 +2936,17 @@ repeated accesses to the optimizer without changing their lookup contract. Bounds checks and full-id comparisons remain enabled in release and debug builds; stale and null ids still return `None`. Retired PcodeOps remain accessible until the bank is cleared. The checked arena -(`decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena.rs`) retains -SlotMap storage and allocation order, and caches the full FFI encoding of each -live id in a compact array indexed by slot. A lookup checks that array's bounds and -compares the full cached id before reading the original slot. Removal and clear -invalidate cached ids before modifying storage; only the arena wrapper can change -slot membership. The all-ones encoding marks vacancies. Its index is `u32::MAX`, -which the cache rejects during insertion, so a checked id cannot match a vacancy. -This adds eight bytes per slot, while avoiding out-of-line -SlotMap validation calls and allowing repeated checks to fold together. Debug -builds also perform SlotMap's original checks at the final slot access. +(`decompiler/crates/kuna-decomp/src/substrate/kuna_checkedarena.rs`) stores each +value in a slot followed by a 64-bit key word. An occupied slot's word is the full +FFI encoding of its live id, so a lookup checks the slot index against the slot +count and compares that one word. The word sits directly after the value's last +fields, which for Varnode and PcodeOp currently include their flag words, so the check +normally reads a cache line the caller touches anyway; there is no separate key +array. A vacant slot's word holds an even version and the index of the next free slot, +which never names the slot itself, and the sentinel slot's word is zero, which no +id encodes, so a checked id cannot match a vacancy. Allocation, free-list order, +version increments and clearing order replicate `slotmap::SlotMap`, so every id is +unchanged, and these two slot types keep SlotMap's slot size. The record getters `PcodeOp::code` and `Varnode::has_no_descend` are inlineable as well; `code` still panics if the opcode has not been assigned. From 2ef8f7aefd1b2db5b90998f67310354d80c4d298 Mon Sep 17 00:00:00 2001 From: Paulo Matias Date: Thu, 29 Oct 2026 14:08:46 -0300 Subject: [PATCH 5/6] [AUTOMATED] Preserve SPARC32 __divdi3 quotient results (#1252) SPARC32 callers of libgcc's `__divdi3` decompile with four 32-bit arguments and an unsigned, word-swapped quotient: ```sh printf '%s\n' 'void g(void);' \ 'long long q(long long a,long long b){return a/b;}' \ 'long long keep(long long a,long long b){long long r=a/b;g();return r;}' \ 'long long guard(long long a,long long b){if(b==0)return -1;return a/b;}' \ > /tmp/di-returns.c clang -target sparc-linux-gnu -integrated-as -O2 -c /tmp/di-returns.c -o /tmp/di-returns.o kuna decompile-all /tmp/di-returns.o --option libgccsigs on ``` Default output is `unsigned long long keep(...) { unsigned long long v1; // o1:o0 v1 = __divdi3(a0,a1,a2,a3); ...`. With `--option libgccsigs on`, `q`, `keep` and `guard` return `long long` from `__divdi3(CONCAT44(a0,a1),CONCAT44(a2,a3))`; `keep` keeps the intervening call and `guard` returns `-1` without dividing. - `libgccsigs` (default off) seeds `long long __divdi3(long long, long long)` on SPARC32 big-endian ELF images that name `__divdi3`; DWARF and `--assert` declarations keep precedence. It stays opt-in because callers that test or clamp the quotient before returning it still lose words. - On high-word-first 32-bit pair models (SPARC32, PowerPC32, MIPS32 BE, ARM/Thumb BE), a caller's return of a declared eight-byte integer call result is kept whole and in ABI order when nothing but the RETURN (or a later call's unresolved input) reads it: return recovery waits for frame SSA, and contained-output guards on four-byte-word pairs are split into per-word trials that only the whole-pair check can activate. A `void` caller that stores or tests such a result stays `void`. - `RuleLoadVarnode` reads `(base + c) | k` as an offset when the base's known-zero bits prove the OR is an addition. - Known limit: a SPARC32 `-O0` caller without its own debug info that stores and then returns a declared 64-bit result returns only the high word (main printed `void`). Tests: `libgcc_divdi3` (generated C recompiled and run at `-O0`/`-O2`, option on and off, BE void-caller regressions), `call_result_pair_returns`, `call_wide`, a `tests/stages/` testcase, and the restored ARM stage assertions. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013BBaWTixHX2E5cBwneY5aW --- .../src/analyzers/protos/kuna_libgcc.rs | 46 +++ .../kuna-analysis/src/analyzers/protos/mod.rs | 11 +- decompiler/crates/kuna-analysis/src/passes.rs | 4 + .../crates/kuna-cli/tests/integration.rs | 3 + .../crates/kuna-cli/tests/libgcc_divdi3.rs | 374 ++++++++++++++++++ decompiler/crates/kuna-console/src/engine.rs | 1 + .../crates/kuna-console/src/kuna_console.rs | 1 + decompiler/crates/kuna-decomp/phases.toml | 19 + .../kuna-decomp/src/infra/architecture.rs | 7 + .../src/p0_knowledge/kuna_option_dispatch.rs | 1 + .../src/p0_knowledge/kuna_phases/tests.rs | 15 +- .../kuna-decomp/src/p0_knowledge/modes.rs | 6 +- .../kuna-decomp/src/p3_dataflow/heritage.rs | 48 ++- .../src/p3_dataflow/ruleaction_4.rs | 26 +- .../src/p3_dataflow/ruleaction_4/tests.rs | 41 ++ .../src/p4_calls/coreaction_protos.rs | 16 +- .../kuna-decomp/src/p4_calls/kuna_bejoin.rs | 2 +- .../src/p4_calls/kuna_retcallhalf.rs | 347 +++++++++++++++- .../kuna-decomp/tests/catalog_bytecompat.rs | 8 +- .../tests/fixtures/phase_catalog.json | 1 + docs/baseline-stages.json | 2 +- docs/modes.md | 9 +- docs/options.md | 11 + docs/spec/01-program-prep.md | 18 + docs/spec/04-calls-and-prototypes.md | 60 +++ tests/stages/kuna-catalog.xml | 2 +- 26 files changed, 1051 insertions(+), 28 deletions(-) create mode 100644 decompiler/crates/kuna-analysis/src/analyzers/protos/kuna_libgcc.rs create mode 100644 decompiler/crates/kuna-cli/tests/libgcc_divdi3.rs diff --git a/decompiler/crates/kuna-analysis/src/analyzers/protos/kuna_libgcc.rs b/decompiler/crates/kuna-analysis/src/analyzers/protos/kuna_libgcc.rs new file mode 100644 index 0000000000..49e6152ac5 --- /dev/null +++ b/decompiler/crates/kuna-analysis/src/analyzers/protos/kuna_libgcc.rs @@ -0,0 +1,46 @@ +//! (kuna `libgccsigs`) Compiler-runtime contracts whose integer widths come from +//! machine modes, not the host C data model. Off by default: a caller that +//! returns arithmetic on the locked eight-byte quotient still loses a word. +//! +//! `__divdi3` is `di_int (di_int, di_int)`, with `di_int = int64_t`: +//! https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/divdi3.c +//! https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/int_types.h + +use object::{Architecture, BinaryFormat, Object}; + +use super::{seed_named_prototypes, seed_resolved_prototypes, Sig, Ty, L}; +use crate::pass::{AnalysisCtx, AnalysisOutput, AnalysisPass, Phase}; + +const SIGNATURES: &[(&str, Sig)] = &[ + ("__divdi3", Sig { ret: Ty::Int64, params: &[Ty::Int64, Ty::Int64], vararg: -1 }), +]; + +pub struct LibgccSigsPass; + +impl AnalysisPass for LibgccSigsPass { + fn phase(&self) -> Phase { + Phase::P1 + } + + fn id(&self) -> &'static str { + "libgccsigs" + } + + /// Seed the evidenced SPARC32 ELF runtime ABI without retyping other targets. + fn run(&self, ctx: &AnalysisCtx) -> AnalysisOutput { + let mut out = AnalysisOutput::default(); + if ctx.file.format() != BinaryFormat::Elf + || !matches!(ctx.file.architecture(), Architecture::Sparc | Architecture::Sparc32Plus) + || ctx.arch.get_description() != "sparc:BE:32:default:default" + { + return out; + } + let (imports, reloc_imports) = super::loaded_import_addrs(ctx.file, ctx.bytes, ctx.image); + let names = super::loaded_function_names(ctx.file, ctx.bytes, &reloc_imports, false); + let types = ctx.arch.types(); + let (_, word_size) = ctx.arch.data_org(); + seed_named_prototypes(&mut out, &names, SIGNATURES, types, word_size, L); + seed_resolved_prototypes(&mut out, &imports, SIGNATURES, types, word_size, L); + out + } +} diff --git a/decompiler/crates/kuna-analysis/src/analyzers/protos/mod.rs b/decompiler/crates/kuna-analysis/src/analyzers/protos/mod.rs index cd163a30e0..ed2ba66eb9 100644 --- a/decompiler/crates/kuna-analysis/src/analyzers/protos/mod.rs +++ b/decompiler/crates/kuna-analysis/src/analyzers/protos/mod.rs @@ -38,6 +38,7 @@ use kuna_decomp::fspec::PrototypePieces; use crate::pass::{AnalysisCtx, AnalysisOutput, AnalysisPass, Phase}; +pub mod kuna_libgcc; mod kuna_libcnames; pub mod kuna_libcsigs; pub mod kuna_libctypes; @@ -49,10 +50,9 @@ pub struct LibProtoPass; /// A primitive type slot in a built-in libc signature. /// -/// Every variant is **width-stable**: it is either `void`, exactly 4 bytes, or -/// exactly pointer-width on every ILP32/LP64 target. A C type that is neither -/// (`off_t`, `time_t`, `long long`, `char`/`short` parameters) has no spelling -/// here on purpose — see [`kuna_libcsigs`]. +/// Integer slots are four bytes or pointer-width for libc declarations. Compiler +/// runtime machine-mode contracts can additionally state an exact eight-byte +/// integer; that does not establish the width of libc `long long` parameters. #[derive(Clone, Copy)] enum Ty { /// `void` (return only). @@ -65,6 +65,8 @@ enum Ty { Size, /// `ssize_t` / `long` / `ptrdiff_t` (pointer-width signed). Long, + /// Signed DI-mode integer (exactly eight bytes in the compiler runtime ABI). + Int64, /// `char *`. CharPtr, /// `char **`. @@ -182,6 +184,7 @@ fn build_ty( Ty::UInt => types.get_base(4, type_metatype::TYPE_UINT), Ty::Size => types.get_base(ptr, type_metatype::TYPE_UINT), Ty::Long => types.get_base(ptr, type_metatype::TYPE_INT), + Ty::Int64 => types.get_base(8, type_metatype::TYPE_INT), Ty::CharPtr => { let c = types.get_type_char(types.get_size_of_char())?; types.get_type_pointer(ptr, c, word_size) diff --git a/decompiler/crates/kuna-analysis/src/passes.rs b/decompiler/crates/kuna-analysis/src/passes.rs index 8774d73880..7bbbc3579c 100644 --- a/decompiler/crates/kuna-analysis/src/passes.rs +++ b/decompiler/crates/kuna-analysis/src/passes.rs @@ -88,6 +88,10 @@ pub fn passes_for(compiler: Compiler, format: object::BinaryFormat) -> Vec Vec { + let mut image = write::Object::new(BinaryFormat::Elf, Architecture::Sparc, Endianness::Big); + let text = image.section_id(write::StandardSection::Text); + // save; mov 62,o0; mov i0,o3; sra i0,31,o2; call; sethi 0x200000,o1; + // st o1,[i1]; ret; restore. Independently authored, with the call's delay slot. + let mut words: [u32; 9] = [ + 0x9de3bfa0, 0x9010203e, 0x96100018, 0x953e201f, 0x40000000, 0x13200000, 0xd2264000, + 0x81c7e008, 0x81e80000, + ]; + if !low_word { + words[6] = 0xd0264000; + } + let bytes: Vec = words.into_iter().flat_map(u32::to_be_bytes).collect(); + image.append_section_data(text, &bytes, 4); + image.add_symbol(write::Symbol { + name: b"divide_low".to_vec(), + value: 0, + size: bytes.len() as u64, + kind: SymbolKind::Text, + scope: SymbolScope::Linkage, + weak: false, + section: write::SymbolSection::Section(text), + flags: SymbolFlags::None, + }); + let helper = image.add_symbol(write::Symbol { + name: b"__divdi3".to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Unknown, + scope: SymbolScope::Dynamic, + weak: false, + section: write::SymbolSection::Undefined, + flags: SymbolFlags::None, + }); + image + .add_relocation( + text, + write::Relocation { + offset: 16, + symbol: helper, + addend: 0, + flags: RelocationFlags::Elf { + r_type: object::elf::R_SPARC_WDISP30, + }, + }, + ) + .unwrap(); + image.write().unwrap() +} + +#[test] +fn signed_di_quotient_low_word_survives_an_opaque_call() { + let root = tempfile::tempdir().unwrap(); + let image = root.path().join("divide.o"); + std::fs::write(&image, caller(true)).unwrap(); + let compilers: Vec<_> = ["gcc", "clang"] + .into_iter() + .filter(|cc| Command::new(cc).arg("--version").output().is_ok()) + .collect(); + assert!(!compilers.is_empty(), "a C compiler is required"); + let decompile = |option: Option<&str>| { + let mut args = vec![ + "decompile", + image.to_str().unwrap(), + "divide_low", + "--json", + "--assert", + "prototype divide_low void divide_low(int divisor,unsigned int *result)", + ]; + if let Some(value) = option { + args.extend(["--option", "libgccsigs", value]); + } + let (output, error, status) = common::run_kuna(&args); + assert_eq!(status, 0, "{error}"); + let document: serde_json::Value = serde_json::from_str(&output).unwrap(); + assert!(document["functions"][0]["error"].is_null(), "{output}"); + document["functions"][0]["code"].as_str().unwrap().to_string() + }; + let off = decompile(Some("off")); + assert_eq!(decompile(None), off, "libgccsigs is off by default"); + assert!( + !off.contains("CONCAT44") && off.contains("__divdi3(0x3e,0x80000000,"), + "off keeps the inferred four-word call: {off}" + ); + let body = decompile(Some("on")); + assert!(body.contains("__divdi3(0x3e80000000,"), "{body}"); + let source = root.path().join("quotient.c"); + std::fs::write( + &source, + format!( + r#" +#include +#include +long long __divdi3(long long a,long long b) {{ return a/b; }} +{body} +int main(void) {{ + const int divisors[] = {{1,-1,2,-2,3,-3,7,-7,999,-999,INT_MAX,INT_MIN}}; + for(unsigned i=0;i +#include +unsigned int __divdi3(unsigned int a,unsigned int b,unsigned int c,unsigned int d) {{ return a^b^c^d; }} +{body} +int main(void) {{ + const int inputs[] = {{0,1,-1,37,-37,INT_MAX,INT_MIN}}; + for(unsigned i=0;i 0, "a C compiler is required"); +} + +#[test] +fn declared_wide_results_survive_caller_return_recovery() { + if Command::new("clang").arg("--version").output().is_err() { + return; + } + let root = tempfile::tempdir().unwrap(); + let fixture = root.path().join("returns.c"); + std::fs::write( + &fixture, + r#" +void g(void); +long long q(long long a,long long b){return a/b;} +long long keep(long long a,long long b){long long r=a/b;g();return r;} +long long guard(long long a,long long b){if(b==0)return -1;return a/b;} +long long choose(long long a,long long b,int flag){if(flag)return a/b;return 0x123456789abcdefLL;} +long long mixed(long long a,long long b,int flag){long long r=a/b;if(flag)r=(unsigned int)r;return r;} +int low(long long a,long long b){return (int)(a/b);} +int high(long long a,long long b){return (int)((a/b)>>32);} +"#, + ) + .unwrap(); + for level in ["-O0", "-O2"] { + let image = root.path().join("returns.o"); + common::process::required_output( + Command::new("clang") + .args(["-target", "sparc-linux-gnu", "-integrated-as", level, "-c"]) + .arg(&fixture) + .arg("-o") + .arg(&image), + ); + for explicit in [false, true] { + let mut bodies = String::new(); + for function in ["q", "keep", "guard", "low", "high", "choose", "mixed"] { + let mut args = vec!["decompile", image.to_str().unwrap(), function, "--json"]; + if !explicit { + args.extend(["--option", "libgccsigs", "on"]); + } else { + args.extend([ + "--option", + "libproto", + "off", + "--assert", + "prototype __divdi3 long long __divdi3(long long a,long long b)", + ]); + } + let (output, error, status) = common::run_kuna(&args); + assert_eq!(status, 0, "{error}"); + let document: serde_json::Value = serde_json::from_str(&output).unwrap(); + assert!(document["functions"][0]["error"].is_null(), "{output}"); + bodies.push_str(document["functions"][0]["code"].as_str().unwrap()); + bodies.push('\n'); + } + let source = root.path().join("driver.c"); + std::fs::write(&source, format!(r#" +#include +#include +#include +#define CONCAT44(hi,lo) (((uint64_t)(uint32_t)(hi)<<32)|(uint32_t)(lo)) +static unsigned calls, effects; +long long __divdi3(long long a,long long b) {{ ++calls; return a/b; }} +void g(void) {{ ++effects; }} +{bodies} +#define ARGS(a,b) (uint32_t)((uint64_t)(a)>>32),(uint32_t)(a),(uint32_t)((uint64_t)(b)>>32),(uint32_t)(b) +_Static_assert(sizeof(low(0,0,0,1)) == 4, "the low-word caller returns one word"); +_Static_assert(sizeof(high(0,0,0,1)) == 4, "the high-word caller returns one word"); +_Static_assert(sizeof(choose(0,0,0,1,0)) == 8, "a nontrivial constant exit stays wide"); +_Static_assert(sizeof(mixed(0,0,0,1,0)) == 8, "one changed word does not discard the other"); +int main(void) {{ + const int64_t edges[] = {{0,1,-1,2,-3,INT64_MIN,INT64_MAX,INT64_C(0x123456789abcdef),-INT64_C(0x123456789abcdef)}}; + uint64_t random = UINT64_C(0x79b153c5a7d931e5); + for(unsigned i=0;i<2000;++i) {{ + random ^= random<<13; random ^= random>>7; random ^= random<<17; + int64_t a = i<81 ? edges[i/9] : (int64_t)random; + random ^= random<<13; random ^= random>>7; random ^= random<<17; + int64_t b = i<81 ? edges[i%9] : (int64_t)random; + if(!b || (a==INT64_MIN && b==-1)) continue; + uint64_t expected = (uint64_t)(a/b); + calls = effects = 0; + if(q(ARGS(a,b)) != expected || calls!=1 || effects) return 1; + if(keep(ARGS(a,b)) != expected || calls!=2 || effects!=1) return 2; + if(guard(ARGS(a,b)) != expected || calls!=3 || effects!=1) return 3; + if(low(ARGS(a,b)) != (uint32_t)expected || calls!=4) return 4; + if(high(ARGS(a,b)) != (uint32_t)(expected>>32) || calls!=5) return 5; + if(guard(ARGS(a,0)) != UINT64_MAX || calls!=5 || effects!=1) return 6; + if(choose(ARGS(a,b),1) != expected || calls!=6 || effects!=1) return 7; + if(choose(ARGS(a,b),0) != UINT64_C(0x123456789abcdef) || calls!=6) return 8; + if(mixed(ARGS(a,b),0) != expected || calls!=7) return 9; + if(mixed(ARGS(a,b),1) != (uint32_t)expected || calls!=8) return 10; + }} + return 0; +}} +"#)).unwrap(); + for cc in ["gcc", "clang"] { + if Command::new(cc).arg("--version").output().is_err() { + continue; + } + for opt in ["-O0", "-O2"] { + let executable = root.path().join(format!("run-{cc}")); + let built = Command::new(cc) + .args([ + "-std=c11", + opt, + "-Werror=uninitialized", + "-fsanitize=undefined", + "-fno-sanitize-recover=all", + ]) + .arg(&source) + .arg("-o") + .arg(&executable) + .output() + .unwrap(); + assert!( + built.status.success(), + "{level} explicit={explicit} {cc} {opt}: {}\n{bodies}", + String::from_utf8_lossy(&built.stderr) + ); + let ran = Command::new(&executable).output().unwrap(); + assert!( + ran.status.success(), + "{level} explicit={explicit} {cc} {opt}: {:?}\n{}\n{bodies}", + ran.status.code(), + String::from_utf8_lossy(&ran.stderr) + ); + } + } + } + } +} + +#[test] +fn a_consumed_wide_result_is_not_a_return() { + if Command::new("clang").arg("--version").output().is_err() { + return; + } + let root = tempfile::tempdir().unwrap(); + let fixture = root.path().join("consumed.c"); + std::fs::write( + &fixture, + r#" +long long f(long long a); +long long g64; +long long plain(long long a){return f(a);} +void store(long long a,long long *p){*p=f(a);} +void global(long long a){g64=f(a);} +void twice(long long a,long long *p){p[0]=f(a);p[1]=f(a+1);} +"#, + ) + .unwrap(); + for level in ["-O0", "-O2"] { + let image = root.path().join("consumed.o"); + common::process::required_output( + Command::new("clang") + .args(["-target", "armeb-linux-gnueabi", "-fno-pic", level, "-c"]) + .arg(&fixture) + .arg("-o") + .arg(&image), + ); + for (function, signature) in [ + ("plain", "long long plain("), + ("store", "void store("), + ("global", "void global("), + ("twice", "void twice("), + ] { + let (output, error, status) = common::run_kuna(&[ + "decompile", image.to_str().unwrap(), function, "--json", + "--assert", "prototype f long long f(long long a)", + ]); + assert_eq!(status, 0, "{error}"); + let document: serde_json::Value = serde_json::from_str(&output).unwrap(); + let body = document["functions"][0]["code"].as_str().unwrap(); + assert!(body.starts_with(signature), "{level} {function}: {body}"); + } + } +} diff --git a/decompiler/crates/kuna-console/src/engine.rs b/decompiler/crates/kuna-console/src/engine.rs index 64866765c1..4b2da9c7f8 100644 --- a/decompiler/crates/kuna-console/src/engine.rs +++ b/decompiler/crates/kuna-console/src/engine.rs @@ -3009,6 +3009,7 @@ fn analysis_pass_enabled(arch: &Architecture, pass_id: &str) -> bool { // facts are computed at LOAD and COMMITTED only when this gate is on, so // `off` renders exactly what the base table alone renders. "libcsigs" => arch.analysis_libcsigs, + "libgccsigs" => arch.analysis_libgccsigs, // (kuna) The LOAD-TIME format-string resolver — a Listing consumer whose // per-call-site overrides are computed at the deferred commit point and // applied only when the gate asks for the static half (`static` on x86, diff --git a/decompiler/crates/kuna-console/src/kuna_console.rs b/decompiler/crates/kuna-console/src/kuna_console.rs index 3b18802500..2bc65a06f2 100644 --- a/decompiler/crates/kuna-console/src/kuna_console.rs +++ b/decompiler/crates/kuna-console/src/kuna_console.rs @@ -270,6 +270,7 @@ pub fn kuna_live_value(conf: &Architecture, option: &str) -> Option on_off(conf.analysis_noreturn_known), "libproto" => on_off(conf.analysis_libproto), "libcsigs" => on_off(conf.analysis_libcsigs), + "libgccsigs" => on_off(conf.analysis_libgccsigs), // (kuna `libctypes`) Valued `off|opaque|glibc`, so the live token is the // value, not an on/off word. "libctypes" => match (conf.analysis_libctypes, conf.analysis_libctypes_glibc) { diff --git a/decompiler/crates/kuna-decomp/phases.toml b/decompiler/crates/kuna-decomp/phases.toml index 9dcf9ee349..ad94554eb7 100644 --- a/decompiler/crates/kuna-decomp/phases.toml +++ b/decompiler/crates/kuna-decomp/phases.toml @@ -5971,6 +5971,25 @@ change_kind = "correctness-fix" tier = "analysis" symptoms = "a Windows API call renders with an empty argument list|LoadLibraryExW() or CreateFileW() has no arguments but the pushes are visible above it|a PE function declares dozens of stack locals that only ever hold outgoing call arguments|CALL fall-through addresses are assigned to stack variables|a Win32 call's wide string argument renders as a bare 0x... constant|calleearity and varargstackargs both on and a Windows API call is still argumentless" +[[settable]] +option = "libgccsigs" +values = "on|off" +default = "off" +destructive = false +phase = "P1" +subphase = "external-refinement" +strength = "HARD" +rewind = "P1" +issue = "pr-1252-libgcc-divdi3" +summary = "Seed the libgcc DI-mode contract `long long __divdi3(long long, long long)` onto a SPARC32 big-endian ELF (`sparc:BE:32:default:default`) that names `__divdi3`, defined or imported. The helper's widths come from GCC machine modes, not the C data model: it takes two eight-byte operands in %o0:%o1 and %o2:%o3 and returns the eight-byte quotient in %o0:%o1, high word first. Without the contract the caller sees an unknown callee that writes only %o0, so a caller that stores only the low word (`*p = (int)(a / b);`, the `st %o1` after the call) stores a local nothing assigns. With it the call prints `__divdi3(CONCAT44(a0,a1),CONCAT44(a2,a3))` and the stored word is `(int)__divdi3(..)`. A DWARF or explicit `--assert prototype` declaration keeps precedence, a stripped image names no helper and is unchanged, and every other target and helper (`__moddi3`, `__udivdi3`, `__umoddi3`) keeps its inferred contract." +use_when = "Off by default: the locked eight-byte result still costs a caller that RETURNS arithmetic on the quotient. Return recovery keeps a caller's two return words only when each is a bare slice of the call result or a literal and nothing else reads the quotient (`return a / b;`, a quotient kept across a call, a guarded `return -1`), so `return a / b + 1;` prints `int inc(..)` with only the high word, `return -(a / b);` prints `int neg(..)`, and `return f ? a / b : c;` prints `void sel(..)`, dropping both values; with the option off all three return every word (in the first-register-low join `bejoin` leaves without evidence). Of 21 authored caller shapes at clang -O2, 14 lose words that way: +1, +c, negation, abs, `>> 3`, an unsigned `>> 32`, ^c, |c, *c, a sum of two quotients, an accumulation loop, a quotient an argument replaces on one path, a clamp (`if (x > 100) x = 100;`), and the `sel` phi. Turn on for a SPARC32 binary whose callers store or pass the quotient (`*p = (int)(a / b);`, `g(a / b)`) or return it unchanged, where the off output reads an unassigned local; check any `int` or `void` caller that should return a `long long` against the disassembly." +example = "option libgccsigs on" +source_decompiler = "kuna" +inspiration = "libgcc/compiler-rt `__divdi3` (di_int __divdi3(di_int a, di_int b), di_int = int64_t); Ghidra ApplyDataArchiveAnalyzer" +change_kind = "correctness-fix" +tier = "analysis" +symptoms = "a SPARC32 caller stores the low word of a 64-bit quotient from a local nothing assigns|__divdi3 is called with four 32-bit arguments and only %o0 is read back|*p = v1; after a __divdi3 call where v1 is never written" + [[settable]] option = "declaredlibcproto" values = "on|off" diff --git a/decompiler/crates/kuna-decomp/src/infra/architecture.rs b/decompiler/crates/kuna-decomp/src/infra/architecture.rs index c57792f76a..4694051edd 100644 --- a/decompiler/crates/kuna-decomp/src/infra/architecture.rs +++ b/decompiler/crates/kuna-decomp/src/infra/architecture.rs @@ -1694,6 +1694,9 @@ pub struct Architecture { pub analysis_libproto: bool, /// (kuna) Gate the measured libc signature extension (`libcsigs`); default on. pub analysis_libcsigs: bool, + /// (kuna) Gate the libgcc DI-mode `__divdi3` contract on SPARC32 BE ELF + /// (`libgccsigs`); default off. + pub analysis_libgccsigs: bool, /// (kuna) Gate the named libc/POSIX aggregate types in the prototype tables /// (`libctypes`); default on (`opaque`). Catalog-visible twin of the load-time /// [`crate::kuna_libctypes`] **env var** (the named shells are interned at @@ -3161,6 +3164,7 @@ impl Architecture { analysis_peimportcall: false, analysis_libproto: false, analysis_libcsigs: false, + analysis_libgccsigs: false, analysis_libctypes: false, analysis_libctypes_glibc: false, analysis_win32sigs: false, @@ -3571,6 +3575,9 @@ impl Architecture { self.analysis_libproto = true; // (kuna) DIV-65 measured libc signature extension — default-ON. self.analysis_libcsigs = true; + // (kuna) libgcc `__divdi3` contract -- default-OFF: a caller returning + // arithmetic on the quotient still loses a word. + self.analysis_libgccsigs = false; // (kuna) Named libc/POSIX aggregate pointers in the prototype tables -- // default-ON (`opaque`). Real-ELF analysis tier only, so the XML corpora // cannot observe it; the evidence is the corpus sweep in diff --git a/decompiler/crates/kuna-decomp/src/p0_knowledge/kuna_option_dispatch.rs b/decompiler/crates/kuna-decomp/src/p0_knowledge/kuna_option_dispatch.rs index 0136fc6214..bbb6662717 100644 --- a/decompiler/crates/kuna-decomp/src/p0_knowledge/kuna_option_dispatch.rs +++ b/decompiler/crates/kuna-decomp/src/p0_knowledge/kuna_option_dispatch.rs @@ -890,6 +890,7 @@ kuna_options! { self, p1; "peimportcall" => on_off!(analysis_peimportcall, "PE/Mach-O import-slot call binding"), "libproto" => on_off!(analysis_libproto, "Library-prototype analysis pass"), "libcsigs" => on_off!(analysis_libcsigs, "Measured libc signature extension"), + "libgccsigs" => on_off!(analysis_libgccsigs, "libgcc __divdi3 DI-mode contract"), "libctypes" => { use crate::kuna_libctypes::LibcTypesLayout; let layout = match p1.trim().to_ascii_lowercase().as_str() { diff --git a/decompiler/crates/kuna-decomp/src/p0_knowledge/kuna_phases/tests.rs b/decompiler/crates/kuna-decomp/src/p0_knowledge/kuna_phases/tests.rs index fb2431fd41..7bb3c2097a 100644 --- a/decompiler/crates/kuna-decomp/src/p0_knowledge/kuna_phases/tests.rs +++ b/decompiler/crates/kuna-decomp/src/p0_knowledge/kuna_phases/tests.rs @@ -38,13 +38,13 @@ fn surface_count_is_120() { } #[test] -fn settable_count_is_339() { - assert_eq!(kuna_num_settables(), 339); - assert_eq!(SETTABLE_TABLE.len(), 339); +fn settable_count_is_340() { + assert_eq!(kuna_num_settables(), 340); + assert_eq!(SETTABLE_TABLE.len(), 340); } #[test] -fn tier_counts_are_120_core_130_transform_89_analysis() { +fn tier_counts_are_120_core_130_transform_90_analysis() { let mut core = 0; let mut transform = 0; let mut analysis = 0; @@ -56,7 +56,7 @@ fn tier_counts_are_120_core_130_transform_89_analysis() { other => panic!("invalid tier {other:?} on {}", s.option), } } - assert_eq!((core, transform, analysis), (120, 130, 89)); + assert_eq!((core, transform, analysis), (120, 130, 90)); } #[test] @@ -318,6 +318,9 @@ fn option_values_live_value_present_for_154() { // with no codegen live reader (read console-side via kuna_live_value), // same as `libcsigs` above. Default-ON (DIV-141). "win32sigs", + // (kuna) The libgcc `__divdi3` contract -- an analysis-pass gate read + // console-side via kuna_live_value, like `libcsigs`. Default-off. + "libgccsigs", "strings", // (kuna) The 2-byte (UTF-16LE) width of the string-literal pass — an // analysis-pass gate read at the commit boundary (console-side via @@ -1109,7 +1112,7 @@ fn emit_catalog_json_static_form_brackets_and_commas() { // +1 for `goreg`; its P1 row sits beside `goabi`. // +1 for `gobool`; its P4 row sits beside `sourcelang`. // +1 for `swiftdemangle`; its P1 row sits beside `sourcelang`. - assert_eq!(json.matches("},\n").count(), 338); + assert_eq!(json.matches("},\n").count(), 339); } #[test] diff --git a/decompiler/crates/kuna-decomp/src/p0_knowledge/modes.rs b/decompiler/crates/kuna-decomp/src/p0_knowledge/modes.rs index 7b5c4523cc..5f07ba3e94 100644 --- a/decompiler/crates/kuna-decomp/src/p0_knowledge/modes.rs +++ b/decompiler/crates/kuna-decomp/src/p0_knowledge/modes.rs @@ -595,7 +595,11 @@ mod tests { // the web front-end, the benchmark) materialize every target they walk // anyway, so preset membership would buy nothing there and has not been // measured on them. Flip it into the preset with that measurement. - "lazyfuncs"]; + "lazyfuncs", + // `libgccsigs` costs a SPARC32 caller that returns arithmetic on + // the `__divdi3` quotient its low word or its whole return (see its + // catalog row), so the preset must not turn it on. + "libgccsigs"]; /// Default-off options that predate this test and are **not** in the preset, /// i.e. are currently unreachable on the default path. Each is a genuine open diff --git a/decompiler/crates/kuna-decomp/src/p3_dataflow/heritage.rs b/decompiler/crates/kuna-decomp/src/p3_dataflow/heritage.rs index 077f0cf2aa..bce3a3b1c7 100644 --- a/decompiler/crates/kuna-decomp/src/p3_dataflow/heritage.rs +++ b/decompiler/crates/kuna-decomp/src/p3_dataflow/heritage.rs @@ -2758,29 +2758,63 @@ impl Heritage { } } - /// Guard RETURN ops when the heritaged range *contains* the output storage, - /// truncating to the recovered output (C++ `Heritage::guardReturnsOverlapping`, - /// `heritage.cc:1610`). Registers the truncated output trial and inserts a - /// SUBPIECE before each RETURN, appending the truncated value as a new input. + /// Guard the biggest model output contained by a wider heritaged range. On a + /// model whose four-byte pair returns the high word first, also guard every + /// other contained output: a locked register-pair call result covers both + /// words, and [`crate::kuna_retcallhalf::whole_pair`] may need the second one. fn guard_returns_overlapping( &mut self, fd: &mut crate::funcdata::Funcdata, addr: &Address, size: int4, + ) { + let split = fd.get_func_proto().has_model() + && crate::kuna_bejoin::wide_pair(fd.get_func_proto().model(), fd.get_arch().manage()) + .is_some_and(|pair| pair.hi.1 == 4 && pair.lo.1 == 4); + self.guard_returns_in_range(fd, addr, size, addr, size, true, split); + } + + fn guard_returns_in_range( + &mut self, + fd: &mut crate::funcdata::Funcdata, + addr: &Address, + size: int4, + range: &Address, + range_size: int4, + primary: bool, + split: bool, ) { use kuna_num::opcodes::OpCode; use kuna_num::pcoderaw::VarnodeData; let mut vdata = VarnodeData::default(); - if !fd.get_func_proto().get_biggest_contained_output(addr, size, &mut vdata) { + if !fd.get_func_proto().get_biggest_contained_output(range, range_size, &mut vdata) { return; } let vspace = match vdata.space.clone() { Some(s) => s, None => return, }; + let Some(before) = vdata.offset.checked_sub(range.get_offset()) + .and_then(|n| int4::try_from(n).ok()) + else { + return; + }; + let piece_size = vdata.size as int4; + if before > range_size || piece_size <= 0 || piece_size > range_size - before { + return; + } + if split && before > 0 { + self.guard_returns_in_range(fd, addr, size, range, before, false, true); + } let trunc_addr = Address::new(Rc::clone(&vspace), vdata.offset); if let Some(active) = fd.get_active_output_mut() { active.register_trial(&trunc_addr, vdata.size as int4); + if !primary { + // The wide range alone is no evidence that the caller returns + // this extra word. Activate it only after a whole-pair proof. + let index = active.get_num_trials() - 1; + active.get_trial_mut(index).mark_inactive(); + } } // offset = least-significant bytes to truncate (endian-adjusted). let mut offset = (vdata.offset.wrapping_sub(addr.get_offset())) as i64 as int4; @@ -2815,6 +2849,10 @@ impl Heritage { .set_active_heritage(); let _ = fd.op_insert_input(op, ret_val, n); } + let after = range_size - before - piece_size; + if split && after > 0 { + self.guard_returns_in_range(fd, addr, size, &(&trunc_addr + piece_size as i64), after, false, true); + } } /// (kuna `stackstoreguard`) May this pass guard the frame of `addr`'s diff --git a/decompiler/crates/kuna-decomp/src/p3_dataflow/ruleaction_4.rs b/decompiler/crates/kuna-decomp/src/p3_dataflow/ruleaction_4.rs index b6708b3847..8e5b1fdc13 100644 --- a/decompiler/crates/kuna-decomp/src/p3_dataflow/ruleaction_4.rs +++ b/decompiler/crates/kuna-decomp/src/p3_dataflow/ruleaction_4.rs @@ -155,15 +155,39 @@ impl RuleLoadVarnode { data: &Funcdata, vn: VarnodeId, spc: &Rc, + depth: u32, ) -> Option<(Rc, uintb)> { if let Some(retspace) = RuleLoadVarnode::correct_spacebase(data, vn, spc) { return Some((retspace, 0)); } + if depth == 8 { + return None; + } let v = data.vbank().get(vn)?; if !v.is_written() { return None; } let op = v.get_def()?; + if code(data, op) == OpCode::CPUI_INT_OR { + let a = in_vn(data, op, 0); + let b = in_vn(data, op, 1); + let (base, constant) = if is_constant(data, b) { + (a, offset_of(data, b)) + } else if is_constant(data, a) { + (b, offset_of(data, a)) + } else { + return None; + }; + // Clang can address a word inside an aligned frame slot as + // (spacebase + offset) | word_offset. It is addition only when + // known-zero bits prove that no set bit can overlap. + if data.vbank().get(base)?.get_nz_mask() & constant != 0 { + return None; + } + let (space, offset) = Self::vn_spacebase(data, base, spc, depth + 1)?; + let offset = space.wrap_offset(offset.wrapping_add(constant)); + return Some((space, offset)); + } if code(data, op) != OpCode::CPUI_INT_ADD { return None; } @@ -205,7 +229,7 @@ impl RuleLoadVarnode { // Check for a literal constant offset into `loadspace`. return Some((loadspace, offset_of(data, offvn))); } - RuleLoadVarnode::vn_spacebase(data, offvn, &loadspace) + RuleLoadVarnode::vn_spacebase(data, offvn, &loadspace, 0) } } diff --git a/decompiler/crates/kuna-decomp/src/p3_dataflow/ruleaction_4/tests.rs b/decompiler/crates/kuna-decomp/src/p3_dataflow/ruleaction_4/tests.rs index f3cab57b15..72628b8ae4 100644 --- a/decompiler/crates/kuna-decomp/src/p3_dataflow/ruleaction_4/tests.rs +++ b/decompiler/crates/kuna-decomp/src/p3_dataflow/ruleaction_4/tests.rs @@ -1432,6 +1432,47 @@ fn concatcommute_or_lower() { assert_eq!(fd.vbank().get(cc).unwrap().get_offset(), 0x00ff); } +#[test] +fn loadvarnode_promotes_only_proven_disjoint_or_offsets() { + for (offset, mask, expected) in [ + (0x120, 0x120, Some(0x124)), + (0x124, 0x124, None), + (0x120, u64::MAX, None), + ] { + for commuted in [false, true] { + let mut fd = build_fd(); + let bl = mk_block(&mut fd); + // A constant pseudo-spacebase is the existing global-space form. + // The CLI regression exercises the same rewrite on real frame SSA. + let base = fd.new_constant(8, 0); + fd.vbank_mut().get_mut(base).unwrap().set_spacebase(); + let displacement = fd.new_constant(8, offset); + let (_, address) = mk_def( + &mut fd, bl, OpCode::CPUI_INT_ADD, 8, 0x20, &[base, displacement], + ); + fd.vbank_mut().get_mut(address).unwrap().set_nz_mask(mask); + let word = fd.new_constant(8, 4); + let inputs = if commuted { [word, address] } else { [address, word] }; + let (_, pointer) = mk_def(&mut fd, bl, OpCode::CPUI_INT_OR, 8, 0x30, &inputs); + let space = fd.new_constant(8, ram_space(&fd).get_index() as u64); + let (load, _) = mk_def(&mut fd, bl, OpCode::CPUI_LOAD, 4, 0x40, &[space, pointer]); + let changed = RuleLoadVarnode::new().apply_op(load, &mut fd); + if let Some(expected) = expected { + assert_eq!(changed, 1); + assert_eq!(fd.obank().get(load).unwrap().code(), OpCode::CPUI_COPY); + let input = fd.obank().get(load).unwrap().get_in(0).unwrap(); + let value = fd.vbank().get(input).unwrap(); + assert_eq!(value.get_offset(), expected); + assert_eq!(value.get_size(), 4); + assert_eq!(value.get_space().get_name(), "ram"); + } else { + assert_eq!(changed, 0); + assert_eq!(fd.obank().get(load).unwrap().code(), OpCode::CPUI_LOAD); + } + } + } +} + // --------------------------------------------------------------------------- // Stub-affected rules: assert the documented early-out // --------------------------------------------------------------------------- diff --git a/decompiler/crates/kuna-decomp/src/p4_calls/coreaction_protos.rs b/decompiler/crates/kuna-decomp/src/p4_calls/coreaction_protos.rs index d036cb8dcf..0cf1231489 100644 --- a/decompiler/crates/kuna-decomp/src/p4_calls/coreaction_protos.rs +++ b/decompiler/crates/kuna-decomp/src/p4_calls/coreaction_protos.rs @@ -1728,6 +1728,10 @@ impl Action for ActionReturnRecovery { let mut cond_failed = Vec::new(); let return_ops: Vec = data.obank().iter_code(OpCode::CPUI_RETURN).collect(); + if crate::kuna_retcallhalf::awaiting_stack(data, &active, &return_ops) { + data.restore_active_output(active); + return 0; + } for &op in &return_ops { let o = match data.obank().get(op) { Some(o) => o, @@ -1818,11 +1822,21 @@ impl Action for ActionReturnRecovery { crate::p4_calls::kuna_x87emptyreturn::drop_empty(data, &mut active, &return_ops); let own_input = crate::kuna_retcallhalf::accept(data, &mut active, &return_ops); crate::p4_calls::kuna_callwide::retain_pair(data, &mut active, &return_ops); + let whole_pair = crate::kuna_retcallhalf::whole_pair(data, &active, &return_ops); + if let Some(indices) = whole_pair { + for index in indices { + active.get_trial_mut(index).mark_active(); + } + } let manager_rc = data.get_arch().manage.clone(); let _ = data.get_func_proto().derive_output_map(&mut active, &manager_rc); crate::p4_calls::kuna_voidret::whole_or_none(data, &mut active); crate::kuna_retinputhalf::note_moved_back_returns(data, &active, &return_ops, own_input); - let order = crate::kuna_bejoin::join_order(&active, data, &return_ops); + let order = if whole_pair.is_some() && crate::kuna_bejoin::live(data) { + active.join_pair_order() + } else { + crate::kuna_bejoin::join_order(&active, data, &return_ops) + }; data.kuna_set_pairs_first_low(crate::kuna_bejoin::joins_first_low(&active, order)); let return_single = data.get_arch().return_single; for &op in &return_ops { diff --git a/decompiler/crates/kuna-decomp/src/p4_calls/kuna_bejoin.rs b/decompiler/crates/kuna-decomp/src/p4_calls/kuna_bejoin.rs index af3817896f..cf0af1b02f 100644 --- a/decompiler/crates/kuna-decomp/src/p4_calls/kuna_bejoin.rs +++ b/decompiler/crates/kuna-decomp/src/p4_calls/kuna_bejoin.rs @@ -2265,7 +2265,7 @@ fn ones(size: int4) -> u64 { /// The value of `vn` when it is a constant or integer arithmetic and logic on /// constants alone, within `depth` operations, truncated to its size. -fn literal_value(data: &Funcdata, vn: VarnodeId, depth: u32) -> Option { +pub(crate) fn literal_value(data: &Funcdata, vn: VarnodeId, depth: u32) -> Option { let v = data.vbank().get(vn)?; let size = v.get_size(); let mask = |x: u64, sz: int4| if sz >= 8 { x } else { x & ((1u64 << (8 * sz)) - 1) }; diff --git a/decompiler/crates/kuna-decomp/src/p4_calls/kuna_retcallhalf.rs b/decompiler/crates/kuna-decomp/src/p4_calls/kuna_retcallhalf.rs index df1b3c0f25..94d8517e50 100644 --- a/decompiler/crates/kuna-decomp/src/p4_calls/kuna_retcallhalf.rs +++ b/decompiler/crates/kuna-decomp/src/p4_calls/kuna_retcallhalf.rs @@ -61,8 +61,9 @@ //! is not changed: `bl g; orr r0,r0,#255` is byte for byte `int f(void) { //! return (int)g() | 255; }` as well as a 64-bit return. Nor is one whose //! second register feeds the first (`asr r1,r0,#2; add r0,r1,r0,lsr #31` is -//! an `int` division by a constant). And a big-endian pair is left alone, -//! because a pair is still joined in little-endian order there. +//! an `int` division by a constant). These heuristics leave high-first pairs +//! alone; [`whole_pair`] separately retains both halves of one declared +//! integer call result, using the model's word order. //! //! # A pair the function names only in part //! @@ -101,6 +102,348 @@ const MAX_DEPTH: u32 = 8; /// How many defining ops the zero test follows. const ZERO_DEPTH: u32 = 8; +/// Preserve pending return reads until scheduled frame SSA can resolve spills. +/// A LOAD is not itself proof: it may reload unused incoming arguments. +pub(crate) fn awaiting_stack( + data: &Funcdata, + active: &ParamActive, + returns: &[OpId], +) -> bool { + let Some(stack) = data.get_arch().manage().get_stack_space() else { return false }; + let Ok(passes) = data.num_heritage_passes(stack) else { return false }; + if !data.get_func_proto().has_model() || active.get_num_trials() < 2 { + return false; + } + if !(0..data.num_calls()).any(|i| wide_integer_call(data, data.get_call_specs(i))) { + return false; + } + if !crate::kuna_bejoin::wide_pair(data.get_func_proto().model(), data.get_arch().manage()) + .is_some_and(|pair| pair.hi.1 == 4 && pair.lo.1 == 4) + { + return false; + } + returns.iter().any(|&ret| { + data.obank().get(ret).is_some_and(|op| { + !op.is_dead() && op.get_halt_type() == 0 + && !crate::kuna_bejoin::never_reached(data, ret) + && (0..active.get_num_trials()).any(|i| { + op.get_in(active.get_trial(i).get_slot()).is_some_and(|vn| { + pending_stack_word(data, (vn, 0), stack.get_index(), passes <= 1, MAX_DEPTH) + }) + }) + }) + }) +} + +/// A frame read which the next simplification pool can materialize for heritage. +/// In particular, a narrow read underneath a phi or extension can survive the +/// first stack SSA pass even after the final eight-byte reload was resolved. +fn pending_stack_word( + data: &Funcdata, + word: (VarnodeId, u64), + stack: i32, + early: bool, + depth: u32, +) -> bool { + if depth == 0 { + return false; + } + let Some((vn, offset)) = word_source(data, word) else { return false }; + let Some(value) = data.vbank().get(vn) else { return false }; + let Some(id) = value.get_def() else { return false }; + let Some(op) = data.obank().get(id) else { return false }; + match op.code() { + OpCode::CPUI_LOAD => { + // Before spacebase propagation, preserve the original wide reload. + // Later, wait only for a proven frame access that RuleLoadVarnode + // can rewrite, not an arbitrary load or an extra fixed pass count. + early && value.get_size() == 8 + || crate::ruleaction_4::RuleLoadVarnode::check_spacebase(data, id) + .is_some_and(|(space, _)| space.get_index() == stack) + } + OpCode::CPUI_MULTIEQUAL => (0..op.num_input()).any(|slot| { + op.get_in(slot).is_some_and(|input| { + pending_stack_word(data, (input, offset), stack, early, depth - 1) + }) + }), + _ => false, + } +} + +/// A high-first pair with at least one complete declared integer call result. +/// Other paths may replace either word with a literal, including before the +/// simplification pool folds it, but may not introduce unrelated live values. +pub(crate) fn whole_pair( + data: &Funcdata, + active: &ParamActive, + returns: &[OpId], +) -> Option<[i32; 2]> { + let proto = data.get_func_proto(); + if proto.is_output_locked() || !proto.has_model() || active.get_num_trials() < 2 { + return None; + } + // No word walk can establish a complete declared result without such a call. + if !(0..data.num_calls()).any(|i| wide_integer_call(data, data.get_call_specs(i))) { + return None; + } + let pair = crate::kuna_bejoin::wide_pair(proto.model(), data.get_arch().manage())?; + if pair.hi.1 != 4 || pair.lo.1 != 4 { + return None; + } + let mut indices = [0; 2]; + for (slot, (address, size)) in [&pair.hi, &pair.lo].into_iter().enumerate() { + indices[slot] = (0..active.get_num_trials()).find(|&i| { + let trial = active.get_trial(i); + trial.get_address() == address && trial.get_size() == *size && !trial.is_unref() + })?; + } + if (0..active.get_num_trials()) + .any(|i| !indices.contains(&i) && active.get_trial(i).is_active()) + { + return None; + } + let mut wide = false; + for &ret in returns { + let operation = data.obank().get(ret)?; + if operation.is_dead() || operation.get_halt_type() != 0 + || crate::kuna_bejoin::never_reached(data, ret) + { + continue; + } + let hi = operation.get_in(active.get_trial(indices[0]).get_slot())?; + let lo = operation.get_in(active.get_trial(indices[1]).get_slot())?; + wide |= whole_words(data, (hi, 0), (lo, 0), MAX_DEPTH)?; + } + wide.then_some(indices) +} + +/// Follow only representation-preserving moves and exact four-byte slices. +fn word_source(data: &Funcdata, mut word: (VarnodeId, u64)) -> Option<(VarnodeId, u64)> { + // Moves cannot cycle in SSA without a phi, where this walk stops. + // Use the graph's own size as the bound, not an arbitrary copy-chain depth. + for _ in 0..data.vbank().num_varnodes() { + let value = data.vbank().get(word.0)?; + if word.1.checked_add(4)? > value.get_size() as u64 { + return None; + } + let Some(op) = value.get_def().and_then(|id| data.obank().get(id)) else { + return Some(word); + }; + match op.code() { + OpCode::CPUI_COPY => word.0 = op.get_in(0)?, + OpCode::CPUI_INT_ZEXT | OpCode::CPUI_INT_SEXT => { + let input = op.get_in(0)?; + if word.1.checked_add(4)? > data.vbank().get(input)?.get_size() as u64 { + return Some(word); + } + word.0 = input; + } + OpCode::CPUI_INDIRECT if !op.is_indirect_creation() => word.0 = op.get_in(0)?, + OpCode::CPUI_INT_OR if op.get_in(0) == op.get_in(1) => word.0 = op.get_in(0)?, + OpCode::CPUI_INT_ADD | OpCode::CPUI_INT_OR | OpCode::CPUI_INT_XOR => { + let a = op.get_in(0)?; + let b = op.get_in(1)?; + word.0 = if is_zero(data, a, ZERO_DEPTH) { + b + } else if is_zero(data, b, ZERO_DEPTH) { + a + } else { + return Some(word); + }; + } + OpCode::CPUI_SUBPIECE => { + let offset = data.vbank().get(op.get_in(1)?)?; + if !offset.is_constant() { + return None; + } + word = (op.get_in(0)?, word.1.checked_add(offset.get_offset())?); + } + OpCode::CPUI_PIECE => { + let low = op.get_in(1)?; + let low_size = data.vbank().get(low)?.get_size() as u64; + if word.1.checked_add(4)? <= low_size { + word.0 = low; + } else if word.1 >= low_size { + word = (op.get_in(0)?, word.1 - low_size); + } else { + return None; + } + } + _ => return Some(word), + } + } + None +} + +fn whole_words( + data: &Funcdata, + hi: (VarnodeId, u64), + lo: (VarnodeId, u64), + depth: u32, +) -> Option { + if depth == 0 { + return None; + } + let hi = word_source(data, hi)?; + let lo = word_source(data, lo)?; + let literal = |word: (VarnodeId, u64)| { + crate::kuna_bejoin::literal_value(data, word.0, ZERO_DEPTH).is_some() + // A word above a zero-extension's input is literal zero even when + // the complete extended value depends on the call's low word. + || data.vbank().get(word.0) + .and_then(|value| value.get_def()) + .and_then(|id| data.obank().get(id)) + .filter(|op| op.code() == OpCode::CPUI_INT_ZEXT) + .and_then(|op| op.get_in(0)) + .and_then(|id| data.vbank().get(id)) + .is_some_and(|input| word.1 >= input.get_size() as u64) + }; + let hi_literal = literal(hi); + let lo_literal = literal(lo); + if hi_literal && lo_literal { + return Some(false); + } + let phi = |vn: VarnodeId| { + data.vbank().get(vn)?.get_def().and_then(|id| data.obank().get(id)) + .filter(|op| op.code() == OpCode::CPUI_MULTIEQUAL) + }; + match (phi(hi.0), phi(lo.0)) { + (Some(hop), Some(lop)) + if hop.get_parent() == lop.get_parent() && hop.num_input() == lop.num_input() => + { + let mut wide = false; + for slot in 0..hop.num_input() { + wide |= whole_words(data, (hop.get_in(slot)?, hi.1), (lop.get_in(slot)?, lo.1), depth - 1)?; + } + return Some(wide); + } + (Some(hop), None) => { + let mut wide = false; + for slot in 0..hop.num_input() { + wide |= whole_words(data, (hop.get_in(slot)?, hi.1), lo, depth - 1)?; + } + return Some(wide); + } + (None, Some(lop)) => { + let mut wide = false; + for slot in 0..lop.num_input() { + wide |= whole_words(data, hi, (lop.get_in(slot)?, lo.1), depth - 1)?; + } + return Some(wide); + } + _ => {} + } + match (call_word(data, hi, 4), call_word(data, lo, 0)) { + (Some(h), Some(l)) if h == l && only_returned(data, h) => Some(true), + (Some(_), None) if lo_literal => Some(false), + (None, Some(_)) if hi_literal => Some(false), + _ => None, + } +} + +/// Every reader of a call result, through moves, slices and phis, is a RETURN. +/// A caller that stores, passes or tests the result consumed it, so the words +/// left in the return registers are no evidence that it returns them. +fn only_returned(data: &Funcdata, result: VarnodeId) -> bool { + let mut seen = std::collections::BTreeSet::new(); + let mut work = vec![result]; + while let Some(cur) = work.pop() { + if !seen.insert(cur) { + continue; + } + if seen.len() > 64 { + return false; + } + let Some(v) = data.vbank().get(cur) else { return false }; + if v.is_persist() { + return false; + } + for reader in v.descend_iter() { + let Some(op) = data.obank().get(reader) else { return false }; + let moves = match op.code() { + OpCode::CPUI_RETURN => continue, + OpCode::CPUI_CALL | OpCode::CPUI_CALLIND => { + if pending_argument(data, reader, op.get_slot(cur)) { + continue; + } + false + } + OpCode::CPUI_COPY + | OpCode::CPUI_SUBPIECE + | OpCode::CPUI_PIECE + | OpCode::CPUI_MULTIEQUAL + | OpCode::CPUI_INT_ZEXT + | OpCode::CPUI_INT_SEXT => true, + OpCode::CPUI_INDIRECT => op.get_in(0) == Some(cur), + OpCode::CPUI_INT_OR if op.get_in(0) == op.get_in(1) => true, + OpCode::CPUI_INT_ADD | OpCode::CPUI_INT_OR | OpCode::CPUI_INT_XOR => (0..2).any(|slot| { + op.get_in(slot) + .is_some_and(|other| other != cur && is_zero(data, other, ZERO_DEPTH)) + }), + _ => false, + }; + match op.get_out() { + Some(out) if moves => work.push(out), + _ => return false, + } + } + } + true +} + +/// An input of a later call whose parameter recovery has not made it an +/// argument, as upstream's `checkCallDoubleUse` treats it. +fn pending_argument(data: &Funcdata, call: OpId, slot: i32) -> bool { + let Some(index) = data.get_call_specs_index(call) else { return false }; + let spec = data.get_call_specs(index); + let placeholder = spec.get_stack_placeholder_slot(); + if slot <= 0 || slot == placeholder || !spec.is_input_active() { + return false; + } + let active = spec.active_input(); + let trial = slot - if placeholder < 0 || slot < placeholder { 1 } else { 2 }; + if trial < 0 || trial >= active.get_num_trials() { + return false; + } + let trial = active.get_trial(trial); + !(trial.is_checked() && trial.is_active()) +} + +/// A word at its ABI position in one locked integer call result. +fn call_word(data: &Funcdata, word: (VarnodeId, u64), offset: u64) -> Option { + let value = data.vbank().get(word.0)?; + if word.1 != offset || value.get_size() != 8 { + return None; + } + let op = data.obank().get(value.get_def()?)?; + let call = match op.code() { + OpCode::CPUI_CALL | OpCode::CPUI_CALLIND => value.get_def()?, + OpCode::CPUI_INDIRECT if op.is_indirect_creation() => { + let iop = data.vbank().get(op.get_in(1)?)?; + OpId::from(slotmap::KeyData::from_ffi(iop.get_offset())) + } + _ => return None, + }; + let spec = data.get_call_specs(data.get_call_specs_index(call)?); + if !wide_integer_call(data, spec) + || op.code() == OpCode::CPUI_INDIRECT && spec.proto().get_output().get_address() != *value.get_addr() + { + return None; + } + Some(word.0) +} + +fn wide_integer_call(data: &Funcdata, spec: &crate::fspec::FuncCallSpecs) -> bool { + spec.proto().is_output_locked() + && spec.proto().has_matching_model(data.get_func_proto().model()) + && spec.proto().get_output().get_type().is_some_and(|ty| { + ty.get_size() == 8 && matches!( + ty.get_metatype(), + crate::dtype::type_metatype::TYPE_INT | crate::dtype::type_metatype::TYPE_UINT + ) + }) +} + /// What the first register of a pair holds at one RETURN, when upstream's /// scoring refused it. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/decompiler/crates/kuna-decomp/tests/catalog_bytecompat.rs b/decompiler/crates/kuna-decomp/tests/catalog_bytecompat.rs index d33041d029..afba7e9689 100644 --- a/decompiler/crates/kuna-decomp/tests/catalog_bytecompat.rs +++ b/decompiler/crates/kuna-decomp/tests/catalog_bytecompat.rs @@ -76,14 +76,14 @@ fn fixture_has_no_current_field() { } #[test] -fn fixture_has_all_339_settables() { +fn fixture_has_all_340_settables() { // One `"option":` per settable row; the authoritative per-option list is // phases.toml settableTable (counts asserted in kuna_phases/tests.rs). - assert_eq!(FIXTURE.matches("\"option\": ").count(), 339); + assert_eq!(FIXTURE.matches("\"option\": ").count(), 340); // Every row carries the tier field appended after change_kind. - assert_eq!(FIXTURE.matches("\"tier\": ").count(), 339); + assert_eq!(FIXTURE.matches("\"tier\": ").count(), 340); // ... and the symptoms array appended after tier (C3). - assert_eq!(FIXTURE.matches("\"symptoms\": ").count(), 339); + assert_eq!(FIXTURE.matches("\"symptoms\": ").count(), 340); } #[test] diff --git a/decompiler/crates/kuna-decomp/tests/fixtures/phase_catalog.json b/decompiler/crates/kuna-decomp/tests/fixtures/phase_catalog.json index 69c1bfafe1..320ede63d9 100644 --- a/decompiler/crates/kuna-decomp/tests/fixtures/phase_catalog.json +++ b/decompiler/crates/kuna-decomp/tests/fixtures/phase_catalog.json @@ -216,6 +216,7 @@ {"option": "libcsigs", "values": ["on", "off"], "default": "on", "destructive_as_default": false, "phase": "P1", "subphase": "external-refinement", "strength": "HARD", "rewind": "P1", "issue": "kuna-analysis-libcsigs", "summary": "Seed the MEASURED extension of the built-in libc prototype table: the 539 further signatures the 28-entry libproto table does not carry (exit, gettext, dcgettext, error, strerror, strdup, strcasecmp, close, open, read, write, stat, getenv, syslog, the FORTIFY __printf_chk/__fprintf_chk/__memcpy_chk family, __errno_location, __stack_chk_fail, the *at family, the SELinux, account, xattr, socket and pthread surfaces, ...). Two mechanical selection rules built it, both over the frozen decbench C corpus and neither one a judgement about which names matter: the first 206 were ranked by PLT call-site frequency (>= 100 call sites, or >= 3 of the 1587 cases where IDA scores a perfect type_match and kuna does not); the 333 that widened it are every name that is an UNDEFINED FUNC symbol in at least 3 of the corpus's 665 dynamically linked binaries and is not already carried. A name in either rule is admitted only if its platform declaration reduces WHOLE to the width-stable Ty vocabulary -- a slot with no honest spelling (off_t, time_t, long long, char) rejects the declaration rather than approximating it, which is why lseek, time, strtoll, mmap, signal and qsort are deliberately absent. Every signature is reduced by script from gcc -aux-info over the platform headers (_GNU_SOURCE + _FORTIFY_SOURCE=2), never written from memory. Applied ONLY to names the image imports and does not itself define, so a program's own function that happens to share a libc spelling is never retyped.", "use_when": "On (default) types the arguments of every common libc call, so a caller whose parameter only flows into one gets a concrete type (char *path) instead of the inferred unsigned long, and the callee's return type is known. Flip OFF when a binary links a private library that reuses libc spellings with different signatures and is NOT statically linked (the pass already skips any name the image defines), or to ablate this table's contribution to a type-recovery difference; off declares exactly the names the 28-entry base table carries and nothing more. It is NOT an ablation of everything a declared prototype does to a call site: the stack placeholder that a call carries until its stack space is heritaged is stripped unconditionally, not by this option, so a base-table name renders with exactly its declared arity in both arms. A declared arity is also an assertion about the CALL SITE, not only about the types: a name this table carries renders with exactly the arguments the signature states, where an unsignatured name renders with whatever argument recovery found. That is the point where a wrong row would show, so the derivation, the rejected set and the whole-corpus diff of the widening live in docs/features/libcwiden/.", "example": "option libcsigs off", "source_decompiler": "kuna", "inspiration": "Ghidra ApplyDataArchiveAnalyzer (.gdt library archives); IDA Pro's til type libraries; measured on decbench C corpus (1587 ida-perfect/kuna-imperfect type_match cases)", "change_kind": "correctness-fix", "tier": "analysis", "symptoms": ["a caller's parameter is unsigned long where it is only ever passed to a libc function that takes a char */int", "a libc call renders its string argument as a bare 0x... constant instead of a literal", "a wrapper around a libc function loses its return value or return type", "__printf_chk / __fprintf_chk arguments are shifted by the fortify flag", "an exit or __stack_chk_fail call shows phantom arguments"]}, {"option": "libctypes", "values": ["off", "opaque", "glibc"], "default": "opaque", "destructive_as_default": false, "phase": "P1", "subphase": "external-refinement", "strength": "HARD", "rewind": "P1", "issue": "kuna-analysis-libctypes", "summary": "Spell the libc/POSIX aggregate pointers in the built-in prototype tables by NAME instead of void *: FILE *, DIR *, stat *, dirent *, passwd *, group *, tm *, option *, timespec *, timeval *, sigaction *, sigset_t *, mbstate_t *, termios *, sockaddr *, pthread_mutex_t *, obstack *, spwd *, utmpx *, utmp *, re_pattern_buffer *, lconv *, statfs *. The Ty vocabulary of those tables is width-stable, so every aggregate pointer in them is a void * today - honest about the width and silent about the pointee, which is the one thing a declaration actually knows. The retarget is enumerated slot by slot (a blanket void *->FILE * would type the va_list of vasprintf/vsnprintf/__vfprintf_chk/verr as a stream), and it brings the stdio names neither shipped table carries: __uflow, fgetc, rewind, freopen, popen, pclose, getdelim, flockfile, funlockfile. A second round of slots was mined from the benchmark corpus's own debug twins - which pointer-to-named-struct variables ground truth holds, and which libc slot each could be reached from - and adds the rest of the stream surface (fseeko, ftello, fread_unlocked, fputc_unlocked, feof_unlocked, fgets_unlocked, setbuf, vfprintf, __getdelim), the record-at-a-time and reentrant halves of the three account databases (getpwent/fgetpwent/putpwent/getpwnam_r and their group and spwd twins), more tm/timespec/termios slots, the regex entry points, localeconv and statfs/fstatfs. Two kinds of slot are left out because naming them splits the caller's frame object: the two-element timespec/timeval arrays of utimensat/futimens/utimes/futimesat, and further sigset_t slots, which a caller hands &sa.sa_mask of a struct sigaction local. Those names are new to both shipped tables, so each supplies an ARITY where there was none as well as a pointee. One table is matched against a name the image DEFINES rather than imports: the five _obstack_* entry points, which are the implementation-reserved half of obstack.h and can only be glibc's or gnulib's copy of the same file. That is what makes obstack reachable at all - gnulib links its copy in and the linker exports the symbols, so a stripped grep, tar or coreutils image carries _obstack_newchunk in .dynsym and nothing else in it says what the first argument addresses; obstack is the widest aggregate in the measured ground truth after FILE. Each named type is interned as a shell carrying its real glibc x86-64 width (FILE 216, stat 144, dirent 280, ...), never width 0 - a zero-width pointee keeps a PTRSUB alive through RulePtrsubUndo and the printer then emits the functional PTRSUB(p,0x28) form - and is left INCOMPLETE so the exported .h declares it opaque rather than as a struct with a width and no members. On an image that carries debug info the platform's own definition is used instead, fields and all: the pass runs after the DWARF importer and adopts an aggregate of the declared width already held under the name (or under glibc's _IO_FILE spelling of FILE) - complete or still being populated - and declines the signature rather than touch a name held by anything else, a different width included. The third value, `glibc`, fills in the nine aggregates whose layout the platform PUBLISHES and an application is meant to read - FILE, stat, timespec, timeval, tm, passwd, group, option, dirent - with their real x86-64 members, so an access renders under its own name (`f->_IO_read_ptr`, `st->st_size`, `opt[i].name`) instead of `f->field_0x8`, and the loaded value takes the FIELD's type rather than whatever the cast said - `st_size` is a signed long, so a wrapper that returns -1 on failure stops printing its sentinel as 0xffffffffffffffff. Every offset and width is measured with offsetof/sizeof against the installed headers, not restated from memory (docs/features/libctypes/glibc.md). Two claims the tables refuse to make: no field is spelled as a pointer to the struct being filled (_IO_FILE::_chain is void * - completing a struct re-keys it into a new Rc, so a self-pointer taken mid-completion would strand the program on two stream types), and no layout is installed unless it is TRUE of the image - x86-64 ELF whose .dynstr names glibc (the libc.so.6 soname or a GLIBC_2.x symbol version). musl, another libc and a statically linked image all keep the opaque shells, which for a static image costs nothing because these tables are matched against imported names only. The WIDTHS are a claim too, and about the caller's frame, not only about offsets inside the shell: a named pointer handed a frame object's address grows that object to the declared width, which is right on x86-64 against glibc (`char v1 [12]` becomes `termios v1` where the program touched twelve bytes of the struct `tcgetattr` fills) and on i386 folded the caller's neighbouring locals into an 8-byte timespec declared as 16 (`v1._12_4_ = a0 + 2`). So nothing at all is named - no slot, no stream symbol, no shell - unless the image is an x86-64 ELF not dynamically linked against another C library (no .dynstr, or one naming glibc); i386, ARM, AArch64, MIPS, PE and Mach-O get the shipped void * tables, byte for byte what `off` gives, and a name declared there by hand does too. A named vote is also held to what the caller does through the pointer. A libc aggregate is often the FIRST member of a larger object (grep's kwset begins with its obstack, an rsyslog instance record with its mutex), whose address is then the one the call reads; where the caller reads that argument past the aggregate's end or before its start, forms an address at or past the end, or steps it by a constant at least as large as the aggregate that is not exactly its size (coreutils wc walks &fstatus[i].st and reads `failed` 8 bytes before it), that call's vote falls back to void * (`kuna_libcfit`) and the declared type decides only the call's cast; type propagation likewise refuses to carry the named pointer onto a value read past the end, so `kwsincr` stops printing every kwset member as an element of an obstack array (`&v1[0x1b].field_0x38`). DIR, sigaction, sigset_t, mbstate_t, termios, sockaddr and pthread_mutex_t stay opaque under `glibc` too: glibc publishes no layout for DIR, and the rest are reserved words no caller reads by name. The same tables also type the stdio STREAMS themselves, which no prototype slot can reach: `stdin`/`stdout`/`stderr` are storage, not an argument, so a function that never calls a typed stdio function sees only an `undefined8` word, and on a shared object the slot has no name at all. Under any enabled value the pass emits a typed DATA symbol for each stream slot, read off the DYNAMIC RELOCATION that binds it - which is also the evidence that the name is the C library's and not a global the program spells `stdout` itself. A COPY relocation names a .bss word the run-time loader fills with libc's own `FILE *stdout`, so that slot is `FILE *` under the stream's own name; a GLOB_DAT on an UNDEFINED symbol names a GOT word holding the stream's ADDRESS, so it is `FILE **` under the minted name `stdout_ptr` - calling that one `stdout` would make the emitted `*stdout_ptr` read as an indirection the program does not perform. Because `stdout_ptr` is kuna's coinage and not the image's, the GLOB_DAT arm declines when either symbol table (.symtab as much as .dynsym) already spells it, rather than printing two addresses under one identifier. Linked ELF only, under the same width gate as the rest of the table, and a spelling that occurs twice in .dynsym is declined outright. The slots point at the same `FILE` the rest of the table uses, so `opaque` gives `stdout->field_0x28` and `glibc` `stdout->_IO_write_ptr`, and an image whose own debug info holds a different `FILE` contributes no stream symbol rather than a contradictory one.", "use_when": "On by default (opaque): a pointer the callee's own libc declaration names carries that name - get_prefix(FILE *f) instead of sub_2f30(long a0) on a stripped -O2 coreutils reader, fopen/fdopen/popen results as FILE *, stat/lstat/fstat filling a stat *, getopt_long reading an option * table, readdir returning a dirent *, localtime a tm *, an obstack handle as obstack * even where gnulib linked the obstack code into the program. Flip OFF to ablate the table: to check whether a pointee name came from this declaration rather than from inference, when a target statically links a private function under a libc spelling that the imports-only restriction did not catch (it would then be typed against libc's aggregate, not its own), or when a decompile-project export has to stay as close to compilable as it gets - the shells are deliberately INCOMPLETE, so the exported .c reads fields out of a type its own .h declares opaque (measured +58 cc errors on an ls export - 51 of them from reading fields out of the incomplete shells, the rest from a type name that collides with a function name of its own in the exported body; the .h itself is 0 errors either way). Off is byte-identical to the shipped void * tables, because with the gate off the pass does not run and not one named shell is interned. No datatest loads a file, so the 675 assertions cannot see this row either way; the stage corpus can, and tests/stages/kuna-libctypes.xml pins it in both arms. The flip's evidence is the corpus sweep in docs/features/libctypes/record.json. Flip to `glibc` for the field NAMES: on an x86-64 ELF against glibc it installs the published layouts of FILE/stat/timespec/timeval/tm/passwd/group/option/dirent, turning `*(unsigned char **)&f->field_0x8` into `f->_IO_read_ptr` and `*(unsigned long *)&st->field_0x30` into `st->st_size`, and giving the loaded value the field's type. Prefer it when reading stdio-heavy or stat-heavy code; stay on `opaque` when a decompile-project export must not declare a type the toolchain's own headers also declare. On a target that is not x86-64 against glibc every value is inert: no slot is named at all, because the widths are not that ABI's. The default also NAMES and TYPES the stream slots: `*(long *)(stdout + 0x28)` becomes `*(long *)&stdout->field_0x28` (`stdout->_IO_write_ptr` under `glibc`), and a shared object's `*(long *)(*dat_3fe0 + 0x28)` becomes `*(long *)&(*stdout_ptr)->field_0x28`. That reach is the function that makes no typed stdio call and the shared object whose stream never had a name - over twelve whole binaries 45 lines move, 38 of them a GOT slot gaining its name, and eight of the twelve are byte-identical because inference had already typed those streams from a typed stdio call in the same function. Its cost is local merging: a slot that takes `FILE *` stops merging with the unrelated values a scalar local had absorbed, which adds a declaration and renumbers the locals after it. Naming a data object stays `datasyms`' call - with that option off the stream slot goes back to `dat_`, since the type rides on the symbol - so ablate the stream half with `datasyms off` and the whole table with `libctypes off`.", "example": "option libctypes off", "source_decompiler": "ghidra", "inspiration": "Ghidra ApplyDataArchiveAnalyzer applying generic_clib_64.gdt, whose FILE */struct stat * slots carry the header's own pointee; signatures reduced from gcc -aux-info over the platform headers (_GNU_SOURCE + _FORTIFY_SOURCE=2), widths from sizeof/_Alignof", "change_kind": "analysis-enablement", "tier": "analysis", "symptoms": ["a function that only ever passes its argument to a libc stdio call still declares it long or unsigned long", "a fopen or fdopen result is a void * that nothing can name", "a stat or lstat call fills an untyped void * buffer", "getopt_long takes a bare (void *)0x... long-option table", "readdir returns a void * whose fields are casts and offsets", "a stream parameter is void * where the ground truth is FILE *", "a FILE * or stat * renders its members as field_0x8 / field_0x30 instead of _IO_read_ptr / st_size", "a value loaded out of a named libc struct keeps the cast's type instead of the field's", "stdin/stdout/stderr render as an untyped word and their fields as raw offsets ((long)stdout + 0x28)", "a shared object's stream slot has no name at all (*(long *)(*dat_3fe0 + 0x28))", "two functions in one binary disagree about what stdout is, one calling it FILE * and the other a long"]}, {"option": "win32sigs", "values": ["on", "off"], "default": "on", "destructive_as_default": false, "phase": "P1", "subphase": "external-refinement", "strength": "HARD", "rewind": "P1", "issue": "repipe-resource-loader-c-retains", "summary": "Seed built-in Win32 API prototypes onto a PE's imported API names, keyed by ENTRY ADDRESS. kuna carried libc signatures (libproto, libcsigs) and not one Windows API signature, so on a PE every LoadLibraryExW/CreateFileW/WriteFile call reached ActionDefaultParams with an empty prototype and its arguments had to be recovered from the call site alone; where that recovery loses -- the ordinary case for an image that writes its outgoing slots well before the call -- the call renders `LoadLibraryExW()` and the argument stores survive as mapped stack locals, along with the return-address push the call itself makes. The table is ~130 WINAPI exports ranked by an import histogram over the 54 PE images of the RE arena corpus (admitted at >= 5 images), plus the resource/loader family (FindResourceW, LoadResource, SizeofResource, EnumResourceTypesW/NamesW) below that bar. Each signature is reduced to the width-stable Ty vocabulary and a declaration with a slot that has no honest spelling (a by-value LARGE_INTEGER, a DWORD64) is rejected rather than approximated. Address is the key because a PE import is TWO FunctionSymbols -- the size-0 IAT slot the engine constant-folds through and the `FF 25` thunk veneer a direct `call` targets -- and the global by-name query used by libproto answers with the slot, which is not the address ActionDefaultParams asks about. Imported names only, so a program's own function sharing a Win32 spelling is never retyped; no CRT (__cdecl) spelling is admitted, because on x86 the PE default model is __stdcall and a locked arity is also the callee's stack pop.", "use_when": "On by default (DIV-141). On, a Windows PE renders `LoadLibraryExW(v1,0,8)` / `WriteFile(h,buf,n,&written,0)` instead of `LoadLibraryExW()` with the pushed values left behind as `v41 = 0x401baf;`-style stack stores. Flip OFF to ablate the table's contribution -- to check whether a Win32 call's arguments came from the parked signature rather than from recovery, or if a target statically links its own function under a Win32 spelling that the imports-only restriction did not catch. Every non-PE target is byte-identical either way.", "example": "option win32sigs off", "source_decompiler": "kuna", "inspiration": "Ghidra ApplyDataArchiveAnalyzer (.gdt library archives); IDA Pro's til type libraries; import histogram over the 54 PE images of the RE arena corpus", "change_kind": "correctness-fix", "tier": "analysis", "symptoms": ["a Windows API call renders with an empty argument list", "LoadLibraryExW() or CreateFileW() has no arguments but the pushes are visible above it", "a PE function declares dozens of stack locals that only ever hold outgoing call arguments", "CALL fall-through addresses are assigned to stack variables", "a Win32 call's wide string argument renders as a bare 0x... constant", "calleearity and varargstackargs both on and a Windows API call is still argumentless"]}, + {"option": "libgccsigs", "values": ["on", "off"], "default": "off", "destructive_as_default": false, "phase": "P1", "subphase": "external-refinement", "strength": "HARD", "rewind": "P1", "issue": "pr-1252-libgcc-divdi3", "summary": "Seed the libgcc DI-mode contract `long long __divdi3(long long, long long)` onto a SPARC32 big-endian ELF (`sparc:BE:32:default:default`) that names `__divdi3`, defined or imported. The helper's widths come from GCC machine modes, not the C data model: it takes two eight-byte operands in %o0:%o1 and %o2:%o3 and returns the eight-byte quotient in %o0:%o1, high word first. Without the contract the caller sees an unknown callee that writes only %o0, so a caller that stores only the low word (`*p = (int)(a / b);`, the `st %o1` after the call) stores a local nothing assigns. With it the call prints `__divdi3(CONCAT44(a0,a1),CONCAT44(a2,a3))` and the stored word is `(int)__divdi3(..)`. A DWARF or explicit `--assert prototype` declaration keeps precedence, a stripped image names no helper and is unchanged, and every other target and helper (`__moddi3`, `__udivdi3`, `__umoddi3`) keeps its inferred contract.", "use_when": "Off by default: the locked eight-byte result still costs a caller that RETURNS arithmetic on the quotient. Return recovery keeps a caller's two return words only when each is a bare slice of the call result or a literal and nothing else reads the quotient (`return a / b;`, a quotient kept across a call, a guarded `return -1`), so `return a / b + 1;` prints `int inc(..)` with only the high word, `return -(a / b);` prints `int neg(..)`, and `return f ? a / b : c;` prints `void sel(..)`, dropping both values; with the option off all three return every word (in the first-register-low join `bejoin` leaves without evidence). Of 21 authored caller shapes at clang -O2, 14 lose words that way: +1, +c, negation, abs, `>> 3`, an unsigned `>> 32`, ^c, |c, *c, a sum of two quotients, an accumulation loop, a quotient an argument replaces on one path, a clamp (`if (x > 100) x = 100;`), and the `sel` phi. Turn on for a SPARC32 binary whose callers store or pass the quotient (`*p = (int)(a / b);`, `g(a / b)`) or return it unchanged, where the off output reads an unassigned local; check any `int` or `void` caller that should return a `long long` against the disassembly.", "example": "option libgccsigs on", "source_decompiler": "kuna", "inspiration": "libgcc/compiler-rt `__divdi3` (di_int __divdi3(di_int a, di_int b), di_int = int64_t); Ghidra ApplyDataArchiveAnalyzer", "change_kind": "correctness-fix", "tier": "analysis", "symptoms": ["a SPARC32 caller stores the low word of a 64-bit quotient from a local nothing assigns", "__divdi3 is called with four 32-bit arguments and only %o0 is read back", "*p = v1; after a __divdi3 call where v1 is never written"]}, {"option": "declaredlibcproto", "values": ["on", "off"], "default": "on", "destructive_as_default": false, "phase": "P1", "subphase": "external-refinement", "strength": "HARD", "rewind": "P1", "issue": "kuna-analysis-declaredlibcproto", "summary": "Answer a DECLARED function name out of the built-in libc signature tables, so naming a callee also gives it a prototype. `libproto` and `libcsigs` match a name the IMAGE carries -- its FUNC symbols, its imports -- which is exactly the evidence a stripped, statically linked target does not have. There the name exists only because the operator supplied it (`--define-function 0x8048968=ptrace`, `--assert 'function 0x8048968 ptrace'`), and until now that named the entry and nothing more: the call sites still rendered `ptrace()` with no arguments, because the callee's arity has to come from somewhere and a glibc varargs wrapper (`long ptrace(enum __ptrace_request, ...)`, the rest fetched with `va_arg`) cannot yield it to any amount of body analysis. With this on, a declared name is looked up in both tables and the matching signature is parked on the DECLARED ENTRY ADDRESS -- the key `ActionDefaultParams` reads back per call site, and the only key that survives two symbols sharing a name. The imports-only restriction the `libcsigs` table carries is deliberately lifted here: it exists so a coincidental spelling cannot retype a function the image defines itself, and that is a judgement about evidence, which is different once a human or an agent has identified the entry outright.", "use_when": "On by default: naming a callee is the one thing an RE agent always does, and a name that buys no prototype buys almost nothing. The shape that needs it is a stripped or packed image where argument recovery has nothing to work from -- the tell is a call rendered `f()` with the pushed argument slots visible as raw stores on the lines above it (`*(int *)(esp - 4) = ...` four times, then `v += ptrace();`). Structurally inert unless a run declares a function name (`--define-function START=NAME`, `--assert function`, console `function bounds ... as NAME`), so it is byte-identical over both parity corpora, and inert again unless that name is one of the ~230 the built-in tables carry. Flip OFF when a target defines its own function that merely shares a libc spelling and the declaration is about that function, not the library one -- or to check whether the parked signature, rather than recovery, is what moved a call's arguments. An explicit `--assert prototype` on the same function still wins: it is applied after the declaration.", "example": "kuna decompile ./crackme sub_80483b0 --define-function 0x8048968=ptrace", "source_decompiler": "kuna", "inspiration": "Ghidra ApplyDataArchiveAnalyzer (.gdt archives applied on rename); IDA Pro's til type libraries, which retype a function the moment it is given a library name", "change_kind": "correctness-fix", "tier": "analysis", "symptoms": ["a call to a function I named with --define-function still renders with no arguments", "the argument stores are emitted as raw stack writes on the lines above an argumentless call", "naming a callee changes the spelling of the call and nothing else", "calleearity and varargstackargs both on and a stripped statically linked libc call is still argumentless", "a declared ptrace/read/write/open call loses every argument"]}, {"option": "strings", "values": ["on", "off"], "default": "on", "destructive_as_default": false, "phase": "P1", "subphase": "code-data-partition", "strength": "HARD", "rewind": "P1", "issue": "kuna-analysis-strings", "summary": "Run the string-literal analysis pass: detect NUL-terminated ASCII strings (minimum length 4) and plant a typelocked char[N] data symbol at each. The four-character threshold keeps command and mode literals such as quit, show, bind, fork, and dup2 visible while the separate UTF-16 arm retains Ghidra's length-5 default.", "use_when": "On (default) lays char[N] data at detected strings; off leaves those addresses undefined.", "example": "option strings off", "source_decompiler": "kuna", "inspiration": "Ghidra StringsAnalyzer; pwn.college CSCI 4250 module 5 game servers, where four-character commands and libc operation names rendered as addresses", "change_kind": "analysis-enablement", "tier": "analysis", "symptoms": ["string constants render as raw addresses or unnamed data instead of quoted char[N] literals", "no data symbols at ascii runs in rodata", "four-character commands such as quit, show, bind, or fork render as hexadecimal addresses"]}, {"option": "widestrings", "values": ["on", "off"], "default": "on", "destructive_as_default": false, "phase": "P1", "subphase": "code-data-partition", "strength": "HARD", "rewind": "P1", "issue": "re-wide-api-string-args", "summary": "Scan the loaded image at 2-byte (UTF-16LE) width as well as 1-byte, and plant a typelocked wchar2[N] data symbol at each wide literal -- the allCharWidths arm of Ghidra's StringsAnalyzer that kuna's 1-byte port left as a documented seam. Read at 1-byte width a UTF-16LE literal is a ONE-CHARACTER string (`L\"ntdll.dll\"` is 6e 00 74 00 ..., whose first NUL closes the run after `n`), so nothing is marked up there, the constant keeps whatever char * the type lattice gave it, and the printer reads to the same NUL: every wide Windows-API argument rendered as its own first character -- LoadLibraryW(\"n\"), FindWindowW(\"O\",0). The widened matcher is the 1-byte MinLengthCharSequenceMatcher over 2-byte little-endian code units and nothing else: the same printable-ASCII recognizer over each unit's low byte, the same require-NUL-end rule, the same minimum length of 5, over exactly the address set the 1-byte pass scans, on even addresses only. The two widths cannot both claim a run -- a wide unit demands a zero high byte, so five consecutive 1-byte-charset bytes never occur inside a wide run -- and the 1-byte facts are committed first regardless, so the established width owns any address both could reach. With the symbol planted the existing printer path does the rest: the char type's size 2 is what emits the L prefix and reads the bytes two at a time.", "use_when": "On by default. A wide (UTF-16) string argument renders as a one-character literal -- LoadLibraryW(\"n\") where the binary says L\"ntdll.dll\", FindWindowW(\"O\",0) where it says L\"OllyDbg - [CPU]\" -- which is most of what a Windows anti-debugging or anti-VM check contains. `kuna strings --encoding utf16` already reported those literals in full while the decompiled C did not, and the difference between the two surfaces is the tell. Flip OFF to restore markup that is exactly the 1-byte pass's -- e.g. on an image whose 16-bit data tables read as plausible wide text, or to ablate this width's contribution.", "example": "option widestrings off", "source_decompiler": "ghidra-upstream", "inspiration": "Ghidra StringsAnalyzer's allCharWidths setting (the UTF-16 arm of MinLengthCharSequenceMatcher); measured on crackmes.one 65acadf3eef082e477ff5ede (ObfuscationFiesta.exe, PE32+), where sub_140002530's LoadLibraryW / FindWindowW anti-debug checks rendered as one-character strings", "change_kind": "analysis-enablement", "tier": "analysis", "symptoms": ["a wide Windows API argument renders as a one-character string literal", "LoadLibraryW(\"n\") or FindWindowW(\"O\",0) in the emitted C", "kuna strings --encoding utf16 reports a literal the decompiled C truncates to its first character", "a UTF-16 string address carries no data symbol while the ascii ones do", "an anti-debugging check names no DLL or window class"]}, diff --git a/docs/baseline-stages.json b/docs/baseline-stages.json index 26c30cfdce..7c710f808d 100644 --- a/docs/baseline-stages.json +++ b/docs/baseline-stages.json @@ -973,7 +973,7 @@ "data:KUNA-CATALOG #12: scalar ARM inputs are a default-off opt-in", "data:KUNA-CATALOG #13: validated A32 inventory frames are the default", "data:KUNA-CATALOG #14: full ARM frames are explicitly enabled", - "data:KUNA-CATALOG #15: 339 registry rows plus six individual queries", + "data:KUNA-CATALOG #15: 340 registry rows plus six individual queries", "data:KUNA-CATALOG #16: standard C byte-array rendering is explicitly opt-in", "data:KUNA-CATALOG #17: standard C byte-array rendering exposes its live state", "data:KUNA-CATALOG #18: bounded local views are default-on and live", diff --git a/docs/modes.md b/docs/modes.md index 78e6e4ae7a..7761aa96b8 100644 --- a/docs/modes.md +++ b/docs/modes.md @@ -196,9 +196,16 @@ not, and the invariant test's `EXCLUDED_ON_PURPOSE` list is the authority: the preset would make a near-neutral judgment call the default output under 500 KiB. +- **`libgccsigs`** — the SPARC32 `long long __divdi3(long long, long long)` + contract fixes a caller that stores only the quotient's low word, but a caller + that returns arithmetic on the quotient (`a / b + 1`, `-(a / b)`, + `f ? a / b : c`) then keeps only the high word or returns nothing, where the + default returns every word. + These therefore stay manual per-run opt-ins (`--option v850indirectbranch on`, `--option dwarf_lines on`, `--option formatstring full`, `--option ifuncfpret on`, -`--option msvcstrappend on`, `--option bejoin on`, `--option regsigned on`) +`--option msvcstrappend on`, `--option bejoin on`, `--option regsigned on`, +`--option libgccsigs on`) even under `--mode aggressive`; a named `--option` still wins over the preset by last-write precedence. For `formatstring` only the `full` loop above is excluded: its `static` default reads the same format constants out of the image at load, so diff --git a/docs/options.md b/docs/options.md index b2b288ec60..b3ceefba42 100644 --- a/docs/options.md +++ b/docs/options.md @@ -509,6 +509,9 @@ Three tiers: | CALL fall-through addresses are assigned to stack variables | [`win32sigs`](#win32sigs) | | a Win32 call's wide string argument renders as a bare 0x... constant | [`win32sigs`](#win32sigs) | | calleearity and varargstackargs both on and a Windows API call is still argumentless | [`win32sigs`](#win32sigs) | +| a SPARC32 caller stores the low word of a 64-bit quotient from a local nothing assigns | [`libgccsigs`](#libgccsigs) | +| __divdi3 is called with four 32-bit arguments and only %o0 is read back | [`libgccsigs`](#libgccsigs) | +| *p = v1; after a __divdi3 call where v1 is never written | [`libgccsigs`](#libgccsigs) | | a call to a function I named with --define-function still renders with no arguments | [`declaredlibcproto`](#declaredlibcproto) | | the argument stores are emitted as raw stack writes on the lines above an argumentless call | [`declaredlibcproto`](#declaredlibcproto) | | naming a callee changes the spelling of the call and nothing else | [`declaredlibcproto`](#declaredlibcproto) | @@ -2411,6 +2414,14 @@ Program-prep enablement: what is discovered, decoded, and named before any funct - **Where / provenance:** P1/external-refinement · kuna · correctness-fix · repipe-resource-loader-c-retains - **Example:** `option win32sigs off` +### `libgccsigs` -- on | off, default `off` + +- **Symptoms:** a SPARC32 caller stores the low word of a 64-bit quotient from a local nothing assigns; __divdi3 is called with four 32-bit arguments and only %o0 is read back; *p = v1; after a __divdi3 call where v1 is never written. +- **What it does:** Seed the libgcc DI-mode contract `long long __divdi3(long long, long long)` onto a SPARC32 big-endian ELF (`sparc:BE:32:default:default`) that names `__divdi3`, defined or imported. The helper's widths come from GCC machine modes, not the C data model: it takes two eight-byte operands in %o0:%o1 and %o2:%o3 and returns the eight-byte quotient in %o0:%o1, high word first. Without the contract the caller sees an unknown callee that writes only %o0, so a caller that stores only the low word (`*p = (int)(a / b);`, the `st %o1` after the call) stores a local nothing assigns. With it the call prints `__divdi3(CONCAT44(a0,a1),CONCAT44(a2,a3))` and the stored word is `(int)__divdi3(..)`. A DWARF or explicit `--assert prototype` declaration keeps precedence, a stripped image names no helper and is unchanged, and every other target and helper (`__moddi3`, `__udivdi3`, `__umoddi3`) keeps its inferred contract. +- **When to flip:** Off by default: the locked eight-byte result still costs a caller that RETURNS arithmetic on the quotient. Return recovery keeps a caller's two return words only when each is a bare slice of the call result or a literal and nothing else reads the quotient (`return a / b;`, a quotient kept across a call, a guarded `return -1`), so `return a / b + 1;` prints `int inc(..)` with only the high word, `return -(a / b);` prints `int neg(..)`, and `return f ? a / b : c;` prints `void sel(..)`, dropping both values; with the option off all three return every word (in the first-register-low join `bejoin` leaves without evidence). Of 21 authored caller shapes at clang -O2, 14 lose words that way: +1, +c, negation, abs, `>> 3`, an unsigned `>> 32`, ^c, |c, *c, a sum of two quotients, an accumulation loop, a quotient an argument replaces on one path, a clamp (`if (x > 100) x = 100;`), and the `sel` phi. Turn on for a SPARC32 binary whose callers store or pass the quotient (`*p = (int)(a / b);`, `g(a / b)`) or return it unchanged, where the off output reads an unassigned local; check any `int` or `void` caller that should return a `long long` against the disassembly. +- **Where / provenance:** P1/external-refinement · kuna · correctness-fix · pr-1252-libgcc-divdi3 +- **Example:** `option libgccsigs on` + ### `declaredlibcproto` -- on | off, default `on` - **Symptoms:** a call to a function I named with --define-function still renders with no arguments; the argument stores are emitted as raw stack writes on the lines above an argumentless call; naming a callee changes the spelling of the call and nothing else; calleearity and varargstackargs both on and a stripped statically linked libc call is still argumentless; a declared ptrace/read/write/open call loses every argument. diff --git a/docs/spec/01-program-prep.md b/docs/spec/01-program-prep.md index c05cc77443..9889d88629 100644 --- a/docs/spec/01-program-prep.md +++ b/docs/spec/01-program-prep.md @@ -1849,6 +1849,24 @@ moves. (stale i386 `-O0` outgoing stack slots, the AArch64 `x8` result register), and in an i386 PIC object, whose `GOTOFF` addresses the loader leaves unresolved, a `char *` parameter renders such a garbage address as a string literal. +- **(kuna) libgcc DI-mode contracts** (`libgccsigs`, off by default, + `decompiler/crates/kuna-analysis/src/analyzers/protos/kuna_libgcc.rs (LibgccSigsPass)`): + seeds the reserved compiler-runtime name `__divdi3` on SPARC32 big-endian ELF + under `sparc:BE:32:default:default`. Its DI-mode contract is a signed + eight-byte quotient of two signed eight-byte operands, independent of the + target's four-byte C `long`. The normal compiler model assigns the argument + and return register pairs. Thus a caller storing only `%o1` stores the low + word of the computed quotient instead of reading an unassigned local. Other + names and targets retain their existing inference. Existing import-address + and name-ambiguity rules apply, and DWARF or explicit prototype assertions + retain precedence over the built-in signature. The pass's facts are computed + at load and committed only when the option is on. It ships off because the + locked eight-byte result still costs a caller that returns arithmetic on the + quotient: §4's whole-pair proof keeps both return words only when each is a + bare slice of the call result or a literal and nothing else reads the + quotient, so `return a / b + 1;`, + `return -(a / b);` and `return f ? a / b : c;` keep only the high word or + return nothing, where the off output returns every word. - **(kuna) Measured libc signatures** (`libcsigs`, `decompiler/crates/kuna-analysis/src/analyzers/protos/kuna_libcsigs.rs (LibcSigsPass)`): the second, larger half of the same table, closing most of the LOSS above. Which diff --git a/docs/spec/04-calls-and-prototypes.md b/docs/spec/04-calls-and-prototypes.md index 09e795622c..b70e843445 100644 --- a/docs/spec/04-calls-and-prototypes.md +++ b/docs/spec/04-calls-and-prototypes.md @@ -2077,6 +2077,66 @@ output sits at the constructed join address (falling back to the first piece if no join can be built); more pieces chain PIECEs over contiguous trials. The (kuna) `returnpair` gate intercepts this join — §4.4. +A wider heritaged register range can contain several separate output entries. +On a model whose four-byte register pair returns the high word first +(`kuna_bejoin`'s `wide_pair`: 32-bit SPARC, PowerPC, MIPS and ARM big-endian), +`Heritage::guard_returns_overlapping` keeps the extra entries as inactive +candidates instead of dropping their bytes or treating the range alone as +evidence that the caller returns every word. Every other model keeps +upstream's single biggest contained entry: there `whole_pair` never applies, +and an inactive leftover word would make `voidret`'s whole-or-none rule drop +an ARM `d0` or `r0:r1` return the caller still reads. With a declared wide callee, +return scoring lets frame slots materialized by spacebase simplification +receive stack SSA before finalizing. A wide load alone is not proof, since it +can reload unused incoming arguments. +This also preserves a narrow frame read beneath a phi or extension until +the standard spacebase-load rule can materialize it for the following +heritage pass; resolving only the final eight-byte reload is insufficient. +The spacebase-load consumer also recognizes a constant OR displacement, such +as `(spacebase - 32) | 4`, only when existing nonzero masks prove its set bits +disjoint from the base. Overlapping or unknown bits do not establish an +additive frame offset. +Before deriving the output map, `kuna_retcallhalf::whole_pair` can retain a +high-first pair when at least one path returns corresponding slices of one +locked, same-convention eight-byte integer call result. Corresponding phi +inputs are checked together; a phi in only one word is checked against the +other word on every incoming edge. Other paths may replace either word with +a literal, including constant-only expressions not yet folded by the +simplification pool and the upper zero word of a zero-extension. A lower word +inside an extension keeps its source provenance. Unrelated live values do not +establish a pair, and at least one path must return a whole wide value. +Every reader of that call result, through moves, slices and phis, must be a +RETURN or an input that a later call's parameter recovery has not made an +argument: a caller that stores, passes or tests the result consumed it, so a +void `*p = f(x);` or `G = f(x);` that leaves the words in the return +registers is not a return. The ABI supplies the join order. Explicit +caller outputs and competing output classes are unchanged. Provenance rejects +single-word truncations that move a different word into the return storage, +including the SPARC register-window controls in the regression. + +This does not prove an unknown caller's C return width from its callee's width. +On a high-first ABI, returning the whole result and returning only its high +word can have identical machine instructions: authored PowerPC32 callers +compiled with sibling-call optimization disabled exhibit exactly that case. +Recovery then retains the wide result; an explicit scalar caller prototype +is needed to state the otherwise indistinguishable narrow contract. + +The pair proof does not follow arithmetic. A caller that returns a value +computed from the call's words (`a / b + 1`, `-(a / b)`, an absolute value, a +shift) or a phi of the call's words and an incoming argument +(`f ? a / b : c`) leaves the second word inactive, so its return keeps only the +high word or none. That is why the SPARC32 `__divdi3` contract (`libgccsigs`, +§1) is off by default: without it the helper is an unknown callee, and such a +caller returns every word in the first-register-low join. With a declared wide +callee from DWARF or `--assert prototype`, the same limit applies. + +The authored `kuna-cli/tests/libgcc_divdi3.rs` round trips cover direct, +saved-across-call, guarded, nontrivial-constant and one-word-replaced quotient +returns at compiler `-O0` and `-O2`, +both with `libgccsigs` on and with an explicit wide prototype while +`libproto` is off. They also check single-word truncations, call effects, and +that the default (`libgccsigs` off) leaves a low-word store as before. + After native return trials are finalized, an otherwise unknown ten-byte scalar result in the full x87 ST0 register receives an extended floating type when the active convention explicitly classifies that exact output storage as floating. diff --git a/tests/stages/kuna-catalog.xml b/tests/stages/kuna-catalog.xml index 53b181d265..73258f1090 100644 --- a/tests/stages/kuna-catalog.xml +++ b/tests/stages/kuna-catalog.xml @@ -53,7 +53,7 @@ "option": "armfloatargs", "values": \["on", "off"\], "default": "off", "current": "off" "option": "armframes", "values": \["inventory", "on", "off"\], "default": "inventory", "current": "inventory" "option": "armframes", "values": \["inventory", "on", "off"\], "default": "inventory", "current": "on" -"option": +"option": "option": "globalarrayanchor", "values": \["on", "off"\], "default": "off", "current": "off" "option": "bytearrayc", "values": \["on", "off"\], "default": "off", "current": "off" "option": "bytearrayc", "values": \["on", "off"\], "default": "off", "current": "on" From 9acd318cd34cd58567c24862b6df9ed5a94bb7d7 Mon Sep 17 00:00:00 2001 From: alvistar Date: Thu, 29 Oct 2026 18:22:47 +0100 Subject: [PATCH 6/6] xrefs: follow ARM and Thumb -fPIC GOT loads and PC-relative displacements in images mapped low (#1264) ARM code compiled `-fPIC` but linked as a fixed-address executable at 0x10000 loses two kinds of reference: function pointers read through the GOT have no xref at all, and a string whose PC-relative displacement happens to be a mapped address gets none either. The raw displacement is filed as a pointer instead. Thumb code loses every one of these, because its literal loads were not recognised. ``` $ kuna xrefs decompiler/crates/kuna-analysis/tests/fixtures/gotxref_arm_le32 --from arm_timers ... 0x110dc data - @0x100f8 ldr r0,[0x10164] 0x110e4 data - @0x100d4 ldr r0,[0x1015c] 0x120f4 data - @0x100dc ldr r6,[0x10160] $ kuna xrefs .../gotxref_arm_le32 --to on_expire # 0 references to on_expire @ 0x10098 $ kuna xrefs .../gotxref_thumb_le32 --to on_alarm # 0 references to on_alarm @ 0x100b0 ``` 0x110dc, 0x110e4 and 0x120f4 are displacements, not pointers. The fix, in `listing/kuna_picpool.rs` with its plumbing in `xrefs.rs`: - In an image with a GOT or `.dynamic`, a pool word is a displacement whenever the PC is added to it, whatever it looks like, and Thumb literal loads (16-bit `ldr`, `ldr.w`) are read too. Static GOT-less firmware keeps the previous rule and cost. - A PC-composed value that lands in `.got` becomes a GOT base, tracked along every edge (switch cases, calls in callee-saved registers, predicated skips) with a meet at joins. A load through the base plus a pool word files a read of the slot and a reference to the slot's word. `poolref`'s pointer row for a consumed word is dropped. - An odd target in code names the Thumb function (bit 0 cleared) when the walk decoded code or knows a function there, or a `$t` mapping symbol covers it; under `$d`/`$a` it keeps its exact address. - Only data and read rows change; the decompiler reads call rows only. On ARM PIC images the new data references can reorder `decompile-all`'s callee-first pass (struct numbering only). Tests: `gotxref_{arm,thumb}_le32` and `picodd_thumb_le32` fixtures with console verify tests, three `tests/cli/` probes, `kuna_picpool` unit tests, spec prose for the shapes still not covered, and the closed `arm-pic-got-xrefs` need. Speed: GOT-less firmware reference walk within ~1% of main; ARM PIC libraries +7-8% walk time for their new rows; no measurable `decompile-all` change. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013BBaWTixHX2E5cBwneY5aW --- .../kuna-analysis/src/listing/kuna_picpool.rs | 1381 ++++++++++++++--- .../crates/kuna-analysis/src/listing/xrefs.rs | 139 +- .../kuna-analysis/tests/fixtures/README.md | 3 + .../tests/fixtures/gotxref_arm_le32 | Bin 0 -> 71396 bytes .../tests/fixtures/gotxref_arm_le32.c | 55 + .../tests/fixtures/gotxref_thumb_le32 | Bin 0 -> 73340 bytes .../tests/fixtures/gotxref_thumb_le32.c | 94 ++ .../tests/fixtures/picodd_thumb_le32 | Bin 0 -> 67004 bytes .../tests/fixtures/picodd_thumb_le32.c | 41 + .../tests/fixtures/picodd_thumb_le32.ld | 8 + .../crates/kuna-console/tests/integration.rs | 5 + .../kuna-console/tests/verify_gotxref.rs | 106 ++ .../tests/verify_gotxref_thumb.rs | 192 +++ docs/re-needs/arm-pic-got-xrefs.md | 135 ++ docs/spec/01-program-prep.md | 106 +- .../arm-pic-displacement-not-a-pointer.json | 43 + tests/cli/arm-pic-got-callback-xref.json | 53 + .../arm-pic-mapped-displacement-string.json | 53 + 18 files changed, 2206 insertions(+), 208 deletions(-) create mode 100755 decompiler/crates/kuna-analysis/tests/fixtures/gotxref_arm_le32 create mode 100644 decompiler/crates/kuna-analysis/tests/fixtures/gotxref_arm_le32.c create mode 100755 decompiler/crates/kuna-analysis/tests/fixtures/gotxref_thumb_le32 create mode 100644 decompiler/crates/kuna-analysis/tests/fixtures/gotxref_thumb_le32.c create mode 100755 decompiler/crates/kuna-analysis/tests/fixtures/picodd_thumb_le32 create mode 100644 decompiler/crates/kuna-analysis/tests/fixtures/picodd_thumb_le32.c create mode 100644 decompiler/crates/kuna-analysis/tests/fixtures/picodd_thumb_le32.ld create mode 100644 decompiler/crates/kuna-console/tests/verify_gotxref.rs create mode 100644 decompiler/crates/kuna-console/tests/verify_gotxref_thumb.rs create mode 100644 docs/re-needs/arm-pic-got-xrefs.md create mode 100644 tests/cli/arm-pic-displacement-not-a-pointer.json create mode 100644 tests/cli/arm-pic-got-callback-xref.json create mode 100644 tests/cli/arm-pic-mapped-displacement-string.json diff --git a/decompiler/crates/kuna-analysis/src/listing/kuna_picpool.rs b/decompiler/crates/kuna-analysis/src/listing/kuna_picpool.rs index 053f04d85e..13b2aa8757 100644 --- a/decompiler/crates/kuna-analysis/src/listing/kuna_picpool.rs +++ b/decompiler/crates/kuna-analysis/src/listing/kuna_picpool.rs @@ -35,9 +35,15 @@ //! as a *displacement* only when //! //! * it comes out of the same read-only, pointer-sized, pointer-aligned slot -//! [`PoolImage`] already vouches for — the image's copy of it is what runs; -//! * `poolref` **declined** it, so the two mechanisms never claim one word: a -//! value that is already a mapped address is a pointer, not a displacement; +//! [`PoolImage`] already vouches for — the image's copy of it is what runs — +//! by a `LOAD` from a constant address (A32) or a `COPY` of the slot itself +//! (Thumb's 16-bit `ldr` and `ldr.w`, whose SLEIGH exports the slot); +//! * whatever the word itself looks like: in an image mapped low (a non-PIE +//! ARM executable at 0x10000) a displacement often lands in a mapped section +//! by chance, and only the PC fold below decides that it is one. This, and +//! Thumb's `COPY`, hold only in an image that has a GOT or a dynamic +//! section ([`Image::extended`]); a static image without one (bare-metal +//! firmware) seeds only an A32 `LOAD` of a word that is not a mapped address; //! * a later instruction in the same straight-line run folds it, through //! `COPY`/`INT_ADD`/`INT_SUB` alone, together with a constant this //! instruction materialized from its **own address** (SLEIGH spells a PC read @@ -56,16 +62,112 @@ //! settled by the PC, and applying it would discard every reference in an image //! mapped low — which the filing Android PIE is, its whole layout living under //! 0x2828. +//! +//! # GOT bases +//! +//! ARM `-fPIC` code makes the GOT address once per function with the same pair +//! (`ldr r4,[pc,#a]; add r4,pc,r4`), keeps it in a callee-saved register, and +//! reads slots through it (`ldr r2,[pc,#b]; ldr r2,[r4,r2]`). A composed value +//! that lands in `.got` is therefore tracked as a base along every edge of the +//! body, and across calls when its register is preserved, until the register is +//! written. A load through base plus a pool word is filed as a +//! [`XrefKind::Read`] of the slot, and the slot's file word, when it is a mapped +//! address, as a [`XrefKind::Data`] reference: in a non-PIE executable the +//! linker writes a local function's address there with no relocation. On ARM a +//! Thumb function's word, and a Thumb function's PC-composed address, carry +//! bit 0; they are answered as they are, and the walk strips the bit once it +//! knows what is code ([`super::xrefs`]). Only base plus a pool +//! word (GOTOFF) is an address; a plain copy of the base is a base and files +//! nothing; a switch's cases receive the dispatch's bases. +//! +//! # Joins +//! +//! A tracked value holds at an instruction only if every predecessor in the body +//! hands it the same one; the walk decodes the entry first, so nothing is ever +//! carried into it from a back edge. The walk cannot apply that rule as it goes, +//! since it decodes a join before it has seen every edge into it, so +//! [`PicPool::finish`] applies it to the recorded body and answers only what +//! survives. A predicated instruction's skip is an edge of its own +//! (`Edge::Skip`). Nothing is recorded before the body's first pool load; +//! what the walk had queued by then holds nothing (`dry`), as does an +//! instruction another body's walk refused to enter. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::rc::Rc; use kuna_base::space::{spacetype, AddrSpace}; +use kuna_decomp::kuna_fasthash::FxBuildHasher; use kuna_num::opcodes::OpCode; use kuna_num::pcoderaw::VarnodeData; +use object::read::{Object, ObjectSection}; + use super::kuna_poolref::PoolImage; -use super::xrefs::{in_range, looks_like_address, FullOp}; +use super::xrefs::{in_range, looks_like_address, FullOp, XrefKind}; + +/// The file bytes of `.got`, which [`PoolImage`] leaves out because the section +/// is writable. A slot for a local function holds its address in a non-PIE +/// executable and the same word as an in-place `R_ARM_RELATIVE` addend in a PIE; +/// an import's slot holds 0 and names nothing. +pub(super) struct GotImage<'a> { + lo: u64, + hi: u64, + data: &'a [u8], + ptr: u32, + little_endian: bool, +} + +impl<'a> GotImage<'a> { + pub(super) fn new(file: &'a object::File<'a>) -> Option { + let sec = file.section_by_name(".got")?; + let lo = sec.address(); + let data = sec.data().ok()?; + if lo == 0 || data.is_empty() { + return None; + } + let hi = lo.saturating_add(sec.size().min(data.len() as u64)); + Some(GotImage { + lo, + hi, + data, + ptr: if file.is_64() { 8 } else { 4 }, + little_endian: file.is_little_endian(), + }) + } + + fn contains(&self, vma: u64) -> bool { + vma >= self.lo && vma < self.hi + } + + /// The pointer-sized, pointer-aligned slot word at `vma`. + fn word_at(&self, vma: u64) -> Option { + if !self.contains(vma) || vma % u64::from(self.ptr) != 0 { + return None; + } + let off = usize::try_from(vma - self.lo).ok()?; + let b = self.data.get(off..off.checked_add(self.ptr as usize)?)?; + Some(match (self.ptr, self.little_endian) { + (8, true) => u64::from_le_bytes(b.try_into().ok()?), + (8, false) => u64::from_be_bytes(b.try_into().ok()?), + (_, true) => u64::from(u32::from_le_bytes(b.try_into().ok()?)), + (_, false) => u64::from(u32::from_be_bytes(b.try_into().ok()?)), + }) + } +} + +/// What every [`PicPool::step`] over one image reads. +pub(super) struct Image<'a> { + pub pool: &'a PoolImage<'a>, + pub mapped: &'a [(u64, u64)], + /// The image has a GOT or a dynamic section, so its code may be + /// position-independent: a pool word that is a mapped address, and a Thumb + /// literal `COPY`, are seeded too. + pub extended: bool, + pub data_space: Option<&'a Rc>, + pub got: Option<&'a GotImage<'a>>, + /// The callee-saved register spans, which a value survives a call in. + pub preserved: &'a [(u64, u64)], +} /// How many instructions a pool value may travel before the `add` that composes /// it. GCC and Clang emit the pair adjacent (the displacement is measured from @@ -97,6 +199,8 @@ struct Pending { /// separate sign extension. word: u64, hops: u8, + /// The pool load the word came from; 0 for a GOT base. + origin: u64, } /// The pool values one function's walk is carrying, keyed by the address of the @@ -106,78 +210,417 @@ struct Pending { /// because [`super::xrefs::build_with_focus`] walks a function breadth-first: at /// a conditional branch the queue interleaves the two successors, so "the /// instruction decoded before this one" is not the instruction that precedes it. +/// +/// The same order means an instruction can be decoded before every edge into it +/// is known, so what the walk carries is only a candidate. Each instruction that +/// sees a tracked value is recorded as a [`Node`], every edge as it is taken, and +/// [`PicPool::finish`] keeps a value at an instruction only where every +/// predecessor delivers the same one (a meet), before it answers any address. #[derive(Default)] pub(super) struct PicPool { - at: HashMap>, + at: HashMap, FxBuildHasher>, + /// GOT bases, keyed by the instruction they reach like `at`, but carried + /// along every edge with no hop bound. + bases_at: HashMap, FxBuildHasher>, + nodes: Vec, + /// `(from, to, kind)`. + edges: Vec<(u64, u64, Edge)>, + /// Set by the body's first pool load; nothing is recorded before it. + live: bool, + /// What the walk had queued at the first pool load: each was queued by an + /// instruction that carried nothing, so nothing holds there. + dry: HashSet, +} + +/// One decoded instruction, as [`PicPool::step`] reads it. +#[derive(Clone, Copy)] +pub(super) struct Insn<'a> { + pub vma: u64, + pub len: u32, + /// `classify`'s fall-through: `None` when the instruction, once it runs, + /// does not fall through. + pub fall: Option, + pub ops: &'a [FullOp], + /// The static branch targets; none for a call. + pub branches: &'a [u64], +} + +/// How control reaches the next instruction, which decides what it carries. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Edge { + /// The instruction ran and fell through: every value it passes on. + Fall, + /// A static branch or a switch case: only GOT bases. + Branch, + /// A predicated instruction's condition failed and it did not run: every + /// value it was handed and its condition did not overwrite. + Skip, +} + +/// One instruction's effect on the values it was handed: `ins` are the values, +/// and every value it passes on and every address it forms names, as a bit mask +/// over `ins`, the values it was computed from. +struct Node { + vma: u64, + ins: Vec, + outs: Vec<(Fact, u32)>, + /// What the [`Edge::Skip`] edge carries, when the instruction is predicated. + skip: Vec<(Fact, u32)>, + refs: Vec<(u64, XrefKind, u32, u64)>, +} + +/// A tracked value. A GOT base travels every edge; a pool word only the +/// fall-through one. +#[derive(Debug, Clone, Copy)] +struct Fact { + p: Pending, + base: bool, +} + +impl Fact { + fn same(&self, o: &Fact) -> bool { + self.base == o.base && self.p.off == o.p.off && self.p.size == o.p.size && self.p.word == o.p.word + } +} + +/// What [`PicPool::finish`] answers for one body. +#[derive(Default)] +pub(super) struct Verdict { + /// `(from, to, kind)`. + pub refs: Vec<(u64, u64, XrefKind)>, + /// Pool loads whose word a PC fold consumed: a displacement, never a pointer. + pub displaced: HashSet, } impl PicPool { - /// Consume one decoded instruction, and answer the addresses it composed. + /// Consume one decoded instruction, and answer the pool word it loaded and + /// carries on, whose [`super::kuna_poolref`] pointer edge waits for + /// [`Verdict::displaced`]. /// /// Called for every instruction of the function in whatever order the walk - /// reaches them; the answer is empty for all but the `add` of a PIC pair. - pub(super) fn step( - &mut self, - vma: u64, - len: u32, - ops: &[FullOp], - pool: &PoolImage, - mapped: &[(u64, u64)], - data_space: Option<&Rc>, - ) -> Vec { + /// reaches them. Until the body's first pool load nothing can be in flight, + /// so nothing is recorded; at that load, `queued` names what the walk has + /// queued and not yet decoded. Every one of those was queued by an + /// instruction that carried nothing, whose edge the meet would count + /// against it, so marking them `dry` answers exactly what recording every + /// edge from the entry would, at no cost per instruction before the load. + /// + /// A predicated instruction (`if (!cond) goto inst_next` ahead of its body) + /// reaches the next instruction along two edges: [`Edge::Fall`] when it ran + /// and fell through, [`Edge::Skip`] when it did not run. A conditional + /// write to a base therefore kills it only on the path where it ran, and a + /// predicated return (`popeq {r4-r6,pc}`) leaves the base to the path that + /// falls past it. + pub(super) fn step(&mut self, img: &Image, insn: Insn, queued: impl FnOnce() -> Vec) -> Option { + let Insn { vma, len, fall, ops, branches: branch_targets } = insn; + let fresh = seed(vma, ops, img); + if !self.live { + fresh?; + self.live = true; + self.dry = queued().into_iter().collect(); + } + let next = vma.wrapping_add(u64::from(len)); + let skip_at = ops.iter().position(|op| op.opcode == OpCode::CPUI_CBRANCH && branches_to(op, next)); + if let Some(next) = fall { + self.edges.push((vma, next, Edge::Fall)); + } + if skip_at.is_some() { + self.edges.push((vma, next, Edge::Skip)); + } + self.edges.extend( + branch_targets + .iter() + .filter(|&&to| !(skip_at.is_some() && to == next) && Some(to) != fall) + .map(|&to| (vma, to, Edge::Branch)), + ); + // Hashing every instruction's address would be this walk's whole cost on // an architecture that never forms one of these pairs; nothing is in // flight until a pool load puts it there. let pending = if self.at.is_empty() { Vec::new() } else { self.at.remove(&vma).unwrap_or_default() }; - let (mut refs, mut carry) = if pending.is_empty() { - (Vec::new(), Vec::new()) + let bases = + if self.bases_at.is_empty() { Vec::new() } else { self.bases_at.remove(&vma).unwrap_or_default() }; + if pending.is_empty() && bases.is_empty() && fresh.is_none() { + return None; + } + let ins: Vec = if bases.is_empty() { + pending } else { - evaluate(vma, ops, &pending, mapped, data_space) + bases.iter().map(|&p| Fact { p, base: true }).chain(pending).collect() + }; + let ev = evaluate(vma, ops, &ins, img); + let calls = ops.iter().any(|op| matches!(op.opcode, OpCode::CPUI_CALL | OpCode::CPUI_CALLIND)); + let kept = |f: &Fact| { + !ev.written.iter().any(|&(off, size)| spans_overlap(f.p.off, f.p.size, off, size)) + && (!calls || survives_calls(&f.p, img.preserved)) }; - // A load out of the pool starts (or restarts) the value its register - // holds, so the pair `ldr; add` needs no separate entry point. - if let Some(fresh) = seed(ops, pool, mapped, data_space) { - carry.retain(|p| !overlaps(p, &fresh)); - carry.push(fresh); + let mut outs: Vec<(Fact, u32)> = Vec::new(); + for (i, f) in ins.iter().enumerate().filter(|(_, f)| f.base) { + if kept(f) { + outs.push((*f, 1 << i)); + } + } + for &(p, deps) in &ev.new_bases { + let f = Fact { p, base: true }; + if !calls || survives_calls(&p, img.preserved) { + outs.retain(|(o, _)| !(o.base && spans_overlap(o.p.off, o.p.size, p.off, p.size))); + outs.push((f, deps)); + } + } + if !leaves_the_run(ops) { + let loaded = fresh.filter(|p| !calls || survives_calls(p, img.preserved)); + let carry = ins + .iter() + .enumerate() + .filter(|(_, f)| !f.base && kept(f) && !loaded.is_some_and(|p| overlaps(&f.p, &p))) + .map(|(i, f)| (*f, 1u32 << i)) + .chain(loaded.map(|p| (Fact { p, base: false }, 0))); + let mut tracked = 0; + for (mut f, deps) in carry { + f.p.hops += 1; + if f.p.hops <= MAX_HOPS && tracked < MAX_TRACKED { + outs.push((f, deps)); + tracked += 1; + } + } } - if !carry.is_empty() && !leaves_the_run(ops) { - let next = vma.wrapping_add(u64::from(len)); - let slot = self.at.entry(next).or_default(); - for mut p in carry { - p.hops += 1; - if p.hops <= MAX_HOPS && slot.len() < MAX_TRACKED { - slot.push(p); + // The path where the instruction did not run: what it was handed, less + // what its condition wrote. Its pool words follow the instruction when + // that falls through, since a predicated pair (`ldrne; addne`) runs + // under one condition; only a base meets both paths. + let mut skip: Vec<(Fact, u32)> = Vec::new(); + if let Some(k) = skip_at { + let cond_writes: Vec<(u64, u32)> = ops[..k] + .iter() + .filter_map(|op| match op.out.as_ref().and_then(|o| key_of(o, img.data_space)) { + Some(Key::Reg(off, size)) => Some((off, size)), + _ => None, + }) + .collect(); + let untouched = + |f: &Fact| !cond_writes.iter().any(|&(off, size)| spans_overlap(f.p.off, f.p.size, off, size)); + for (i, f) in ins.iter().enumerate() { + if f.base && untouched(f) { + skip.push((*f, 1 << i)); + } else if !f.base && fall.is_none() && untouched(f) && f.p.hops < MAX_HOPS { + let mut f = *f; + f.p.hops += 1; + skip.push((f, 1 << i)); } } - if slot.is_empty() { - self.at.remove(&next); + if fall.is_some() { + skip.extend(outs.iter().filter(|(f, _)| !f.base).copied()); + } + } + + if let Some(next) = fall.filter(|_| outs.iter().any(|(f, _)| !f.base)) { + self.at.entry(next).or_default().extend(outs.iter().filter(|(f, _)| !f.base).map(|(f, _)| *f)); + } + if fall.is_none() { + if skip.iter().any(|(f, _)| !f.base) { + self.at.entry(next).or_default().extend(skip.iter().filter(|(f, _)| !f.base).map(|(f, _)| *f)); + } + } + let based: Vec = + outs.iter().filter(|(f, _)| f.base).map(|(f, _)| f.p).take(MAX_TRACKED).collect(); + let skipped: Vec = skip.iter().filter(|(f, _)| f.base).map(|(f, _)| f.p).collect(); + let reach = fall + .into_iter() + .chain(branch_targets.iter().copied().filter(|&to| !(skip_at.is_some() && to == next))) + .map(|to| (to, &based)) + .chain(skip_at.map(|_| (next, &skipped))); + if !(based.is_empty() && skipped.is_empty()) { + for (to, carry) in reach { + self.reach(to, carry); } } - refs.sort_unstable(); - refs.dedup(); - refs + let seeded = fresh.and_then(|p| { + outs.iter().any(|(f, deps)| *deps == 0 && !f.base && f.p.origin == p.origin).then_some(p.word) + }); + // A node that passes nothing on and forms nothing delivers nothing, + // which an absent one already does. + if !(outs.is_empty() && skip.is_empty() && ev.found.is_empty()) { + self.nodes.push(Node { vma, ins, outs, skip, refs: ev.found }); + } + seeded + } + + /// Record the case edges of a switch dispatch at `vma`, which the walk reads + /// after the dispatch's own [`PicPool::step`], and hand each case the GOT + /// bases the dispatch passes on: a case body is the dispatcher's own code. + /// Recorded whether or not the dispatch carried anything: a case target may + /// still be reached with a value along another edge, and a dispatch that + /// carried nothing must veto it there. Before the first pool load the walk + /// queues the targets, which [`PicPool::step`] then marks `dry`. + pub(super) fn cases(&mut self, vma: u64, targets: &[u64]) { + if !self.live { + return; + } + self.edges.extend(targets.iter().map(|&to| (vma, to, Edge::Branch))); + let Some(n) = self.nodes.iter().rev().find(|n| n.vma == vma) else { return }; + let based: Vec = n.outs.iter().filter(|(f, _)| f.base).map(|(f, _)| f.p).collect(); + for &to in targets { + self.reach(to, &based); + } + } + + /// Hand `carry` to the instruction at `to`. What the walk carries is a + /// candidate the meet in [`PicPool::finish`] settles, so a second edge into + /// an undecoded instruction only narrows it. + fn reach(&mut self, to: u64, carry: &[Pending]) { + if carry.is_empty() { + return; + } + match self.bases_at.get_mut(&to) { + Some(have) => have.retain(|h| carry.contains(h)), + None => { + self.bases_at.insert(to, carry.to_vec()); + } + } + } + + /// The addresses the body formed, once the whole body is known. + /// + /// Starting from what the walk carried, a value is dropped at an instruction + /// unless every recorded predecessor passes on the same one, and everything + /// computed from a dropped value goes with it, until nothing changes. An + /// address is answered only when all the values it was formed from survived. + /// The entry needs no rule of its own: the walk decodes it first, so nothing + /// is carried into it. An instruction in `entered` has a predecessor in + /// another body's walk, outside this record, so nothing holds there. + pub(super) fn finish(self, entered: &HashSet) -> Verdict { + let mut verdict = Verdict::default(); + if self.nodes.iter().all(|n| n.refs.is_empty()) { + return verdict; + } + let index: HashMap = + self.nodes.iter().enumerate().map(|(i, n)| (n.vma, i)).collect(); + let mut preds: HashMap, FxBuildHasher> = HashMap::default(); + for &(from, to, kind) in &self.edges { + if index.contains_key(&to) { + preds.entry(to).or_default().push((from, kind)); + } + } + let mut valid: Vec = + self.nodes.iter().map(|n| if entered.contains(&n.vma) || self.dry.contains(&n.vma) { 0 } else { all(n.ins.len()) }).collect(); + let delivers = |valid: &[u32], from: u64, kind: Edge, f: &Fact| { + index.get(&from).is_some_and(|&i| { + let n = &self.nodes[i]; + let carried = if kind == Edge::Skip { &n.skip } else { &n.outs }; + carried + .iter() + .any(|(o, deps)| deps & !valid[i] == 0 && (o.base || kind != Edge::Branch) && o.same(f)) + }) + }; + let mut changed = true; + while changed { + changed = false; + for (i, n) in self.nodes.iter().enumerate() { + let mut keep = valid[i]; + let ps = preds.get(&n.vma).map_or(&[][..], Vec::as_slice); + for (j, f) in n.ins.iter().enumerate() { + if keep & (1 << j) == 0 { + continue; + } + if ps.is_empty() || !ps.iter().all(|&(from, kind)| delivers(&valid, from, kind, f)) { + keep &= !(1 << j); + } + } + if keep != valid[i] { + valid[i] = keep; + changed = true; + } + } + } + for (i, n) in self.nodes.iter().enumerate() { + for &(to, kind, deps, origin) in &n.refs { + if deps & !valid[i] == 0 { + verdict.refs.push((n.vma, to, kind)); + if origin != 0 { + verdict.displaced.insert(origin); + } + } + } + } + verdict.refs.sort_unstable(); + verdict.refs.dedup(); + verdict } } -/// The composed addresses this instruction forms, and the tracked values that -/// survive it. -fn evaluate( - vma: u64, - ops: &[FullOp], - pending: &[Pending], - mapped: &[(u64, u64)], - data: Option<&Rc>, -) -> (Vec, Vec) { - let mut vals: Vec<(Key, Known)> = - pending.iter().map(|p| (Key::Reg(p.off, p.size), Known::seed(p.word))).collect(); +/// The mask with the low `n` bits set. +fn all(n: usize) -> u32 { + if n >= 32 { u32::MAX } else { (1u32 << n) - 1 } +} + +/// What one instruction did to the tracked values. +struct Eval { + /// `(to, kind, deps, origin)`. + found: Vec<(u64, XrefKind, u32, u64)>, + written: Vec<(u64, u32)>, + new_bases: Vec<(Pending, u32)>, +} + +/// The addresses this instruction forms from the values it was handed, the +/// registers it writes, and the GOT bases it establishes: a PC-composed value +/// in `.got`, or a plain copy of a base. +fn evaluate(vma: u64, ops: &[FullOp], ins: &[Fact], img: &Image) -> Eval { + let (mapped, data, got) = (img.mapped, img.data_space, img.got); + // Without a base among its inputs or its own PC among its constants, nothing + // this instruction computes is an address; only its writes matter. + let inert = !ins.iter().any(|f| f.base) + && !ops.iter().flat_map(|op| op.ins.iter()).any(|vn| is_constant(vn) && is_own_pc(vn.offset, vma)); + if inert + || !ops.iter().flat_map(|op| op.ins.iter()).any(|vn| match key_of(vn, data) { + Some(Key::Reg(off, size)) => ins.iter().any(|f| spans_overlap(f.p.off, f.p.size, off, size)), + _ => false, + }) + { + let written = ops + .iter() + .filter_map(|op| match op.out.as_ref().and_then(|o| key_of(o, data)) { + Some(Key::Reg(off, size)) if ins.iter().any(|f| spans_overlap(f.p.off, f.p.size, off, size)) => { + Some((off, size)) + } + _ => None, + }) + .collect(); + return Eval { found: Vec::new(), written, new_bases: Vec::new() }; + } + let mut vals: Vec<(Key, Known)> = ins + .iter() + .enumerate() + .map(|(i, f)| { + let k = if f.base { + Known { v: f.p.word, seed: false, pc: false, got: true, origin: 0, deps: 1 << i } + } else { + Known { deps: 1 << i, ..Known::seed(f.p.word, f.p.origin) } + }; + (Key::Reg(f.p.off, f.p.size), k) + }) + .collect(); let mut written: Vec<(u64, u32)> = Vec::new(); - let mut found: Vec = Vec::new(); + let mut found: Vec<(u64, XrefKind, u32, u64)> = Vec::new(); + let mut new_bases: Vec<(Pending, u32)> = Vec::new(); for op in ops { + if op.opcode == OpCode::CPUI_LOAD { + if let (Some(g), Some(addr)) = (got, op.ins.get(1).and_then(|vn| value_of(vn, vma, &vals, data))) { + let slot = addr.v & mask(op.ins[1].size); + if addr.got && addr.seed && !addr.pc && g.contains(slot) { + found.push((slot, XrefKind::Read, addr.deps, addr.origin)); + if let Some(word) = g.word_at(slot) { + if looks_like_address(word) && in_range(mapped, word) { + found.push((word, XrefKind::Data, addr.deps, addr.origin)); + } + } + } + } + } let folded = fold(op, vma, &vals, data); let Some(out) = &op.out else { continue }; let key = key_of(out, data); @@ -187,89 +630,112 @@ fn evaluate( invalidate(&mut vals, key); let (Some(key), Some(mut val)) = (key, folded) else { continue }; val.v &= mask(out.size); - if matches!(key, Key::Reg(..)) && val.seed && val.pc && in_range(mapped, val.v) { - found.push(val.v); + if let Key::Reg(off, size) = key { + let composed = val.seed && val.pc && !val.got; + let gotoff = val.seed && val.got && !val.pc; + if (composed || gotoff) && in_range(mapped, val.v) { + found.push((val.v, XrefKind::Data, val.deps, val.origin)); + } + let copied = val.got && !val.seed && !val.pc && ins.iter().any(|f| f.base && f.p.word == val.v); + let made = composed && in_range(mapped, val.v) && got.is_some_and(|g| g.contains(val.v)); + if made || copied { + new_bases.retain(|(b, _)| !spans_overlap(b.off, b.size, off, size)); + new_bases.push((Pending { off, size, word: val.v, hops: 0, origin: 0 }, val.deps)); + } } vals.push((key, val)); } - - let survivors = pending - .iter() - .filter(|p| { - !written.iter().any(|&(off, size)| { - spans_overlap(p.off, p.size, off, size) - }) - }) - .copied() - .collect(); - (found, survivors) + Eval { found, written, new_bases } } -/// The pool word this instruction loads into a register, when it loads one that -/// [`super::kuna_poolref`] declined to follow as a pointer. +/// The pool word this instruction loads into a register. /// /// The first such load, which on every architecture that forms these pairs is /// also the only one: a literal load's address is a decode-time constant, and a /// multi-register load (`ldm`) computes its addresses from a base register. -fn seed( - ops: &[FullOp], - pool: &PoolImage, - mapped: &[(u64, u64)], - data: Option<&Rc>, -) -> Option { +/// +/// A32 lifts `ldr rX,[pc,#..]` as a `LOAD` from a constant address. Thumb's +/// literal loads, both the 16-bit form and `ldr.w`, export the slot itself +/// (`*:4 reloc`, the PC already word-aligned), so they lift as a `COPY` out of +/// a data-space varnode. Outside an [`Image::extended`] image only the `LOAD` +/// is read, and only of a word that is not a mapped address. +fn seed(vma: u64, ops: &[FullOp], img: &Image) -> Option { + let (pool, data) = (img.pool, img.data_space); let data_index = data.map(|d| d.get_index()); for op in ops { - if op.opcode != OpCode::CPUI_LOAD { - continue; - } let Some(out) = &op.out else { continue }; + let addr = match op.opcode { + OpCode::CPUI_LOAD => { + // `in0` is the space id: a load through some other space is not + // a read of the image. + let through_image = op + .ins + .first() + .zip(data_index) + .is_some_and(|(vn, idx)| vn.offset == u64::try_from(idx).unwrap_or(u64::MAX)); + match op.ins.get(1).filter(|vn| through_image && is_constant(vn)) { + Some(vn) => vn.offset, + None => continue, + } + } + OpCode::CPUI_COPY if img.extended => match op.ins.first() { + Some(vn) if in_data_space(vn, data) && vn.size == out.size => vn.offset, + _ => continue, + }, + _ => continue, + }; if !matches!(key_of(out, data), Some(Key::Reg(..))) || out.size != pool.ptr_width() { continue; } - // `in0` is the space id: a load through some other space is not a read of - // the image. - let through_image = op - .ins - .first() - .zip(data_index) - .is_some_and(|(vn, idx)| vn.offset == u64::try_from(idx).unwrap_or(u64::MAX)); - if !through_image { + let Some(word) = pool.word_at(addr) else { continue }; + if !img.extended && looks_like_address(word) && in_range(img.mapped, word) { continue; } - let Some(addr) = op.ins.get(1).filter(|vn| is_constant(vn)) else { continue }; - let Some(word) = pool.word_at(addr.offset) else { continue }; - if looks_like_address(word) && in_range(mapped, word) { - continue; // a pointer, which is `poolref`'s to follow - } - return Some(Pending { off: out.offset, size: out.size, word, hops: 0 }); + return Some(Pending { off: out.offset, size: out.size, word, hops: 0, origin: vma }); } None } +fn in_data_space(vn: &VarnodeData, data: Option<&Rc>) -> bool { + matches!((vn.space.as_ref(), data), (Some(s), Some(d)) if Rc::ptr_eq(s, d)) +} + +/// Is `op` a branch to the absolute address `to`? A constant-space target is +/// p-code-relative, inside the instruction. +fn branches_to(op: &FullOp, to: u64) -> bool { + op.ins.first().is_some_and(|vn| !is_constant(vn) && vn.offset == to) +} + /// Does control leave the straight-line run here, so the fall-through is not the -/// next thing executed (or a call has clobbered the registers in between)? +/// next thing executed? /// /// A `CBRANCH` is deliberately not in the set: its fall-through IS a successor, -/// it writes nothing, and ARM spells a predicated instruction that way. +/// it writes nothing, and ARM spells a predicated instruction that way. A call is +/// not either: it ends only the values the calling convention does not preserve +/// (`survives_calls`), since GCC schedules `ldr r4,[pc,#..]` before a call and +/// its `add r4,pc,r4` after it. A `CALLOTHER` ends everything, except inside a +/// call, where it is the Thumb `bl`'s mode switch and the call already decides. fn leaves_the_run(ops: &[FullOp]) -> bool { - ops.iter().any(|op| { - matches!( - op.opcode, - OpCode::CPUI_BRANCH - | OpCode::CPUI_BRANCHIND - | OpCode::CPUI_CALL - | OpCode::CPUI_CALLIND - | OpCode::CPUI_CALLOTHER - | OpCode::CPUI_RETURN - ) + let calls = ops.iter().any(|op| matches!(op.opcode, OpCode::CPUI_CALL | OpCode::CPUI_CALLIND)); + ops.iter().any(|op| match op.opcode { + OpCode::CPUI_BRANCH | OpCode::CPUI_BRANCHIND | OpCode::CPUI_RETURN => true, + OpCode::CPUI_CALLOTHER => !calls, + _ => false, }) } +/// Is the value's whole register callee-saved? +fn survives_calls(p: &Pending, preserved: &[(u64, u64)]) -> bool { + preserved.iter().any(|&(lo, hi)| p.off >= lo && p.off + u64::from(p.size) <= hi) +} + // --- the one-instruction constant fold --------------------------------------- -/// A value the fold knows, and where it came from. Both provenance bits must be -/// set for the result to be a PIC address: `seed` alone is the pool word being -/// used as a number, `pc` alone is any PC-relative arithmetic. +/// A value the fold knows, and where it came from. A PIC address is a pool word +/// plus either the PC (`seed` and `pc`) or a GOT base (`seed` and `got`, the +/// GOTOFF form): `seed` alone is the pool word used as a number, `pc` alone is +/// any PC-relative arithmetic, and `got` alone is the base itself or base plus +/// an immediate, which names nothing. #[derive(Debug, Clone, Copy)] struct Known { v: u64, @@ -277,12 +743,27 @@ struct Known { seed: bool, /// This instruction's own PC contributed to this value. pc: bool, + /// A GOT base contributed to this value. + got: bool, + /// The pool load of the word that contributed, or 0. + origin: u64, + /// The tracked values that contributed, as a mask over the node's `ins`. + deps: u32, } impl Known { - fn seed(v: u64) -> Self { - Known { v, seed: true, pc: false } + fn seed(v: u64, origin: u64) -> Self { + Known { v, seed: true, pc: false, got: false, origin, deps: 0 } + } +} + +/// The value a varnode holds, as [`fold`] reads its inputs. +fn value_of(vn: &VarnodeData, vma: u64, vals: &[(Key, Known)], data: Option<&Rc>) -> Option { + if is_constant(vn) { + return Some(Known { v: vn.offset & mask(vn.size), seed: false, pc: is_own_pc(vn.offset, vma), got: false, origin: 0, deps: 0 }); } + let key = key_of(vn, data)?; + vals.iter().rev().find(|(k, _)| *k == key).map(|(_, v)| *v) } /// A varnode's identity, in the two classes the fold models. `register` and `ram` @@ -316,28 +797,31 @@ fn fold( vals: &[(Key, Known)], data: Option<&Rc>, ) -> Option { - let get = |i: usize| -> Option { - let vn = op.ins.get(i)?; - if is_constant(vn) { - return Some(Known { - v: vn.offset & mask(vn.size), - seed: false, - pc: is_own_pc(vn.offset, vma), - }); - } - let key = key_of(vn, data)?; - vals.iter().rev().find(|(k, _)| *k == key).map(|(_, v)| *v) - }; + let get = |i: usize| -> Option { value_of(op.ins.get(i)?, vma, vals, data) }; let a = get(0)?; match op.opcode { OpCode::CPUI_COPY => Some(a), OpCode::CPUI_INT_ADD => { let b = get(1)?; - Some(Known { v: a.v.wrapping_add(b.v), seed: a.seed || b.seed, pc: a.pc || b.pc }) + Some(Known { + v: a.v.wrapping_add(b.v), + seed: a.seed || b.seed, + pc: a.pc || b.pc, + got: a.got || b.got, + origin: if a.origin != 0 { a.origin } else { b.origin }, + deps: a.deps | b.deps, + }) } OpCode::CPUI_INT_SUB => { let b = get(1)?; - Some(Known { v: a.v.wrapping_sub(b.v), seed: a.seed || b.seed, pc: a.pc || b.pc }) + Some(Known { + v: a.v.wrapping_sub(b.v), + seed: a.seed || b.seed, + pc: a.pc || b.pc, + got: a.got || b.got, + origin: if a.origin != 0 { a.origin } else { b.origin }, + deps: a.deps | b.deps, + }) } _ => None, } @@ -390,6 +874,8 @@ fn mask(size: u32) -> u64 { #[cfg(test)] mod tests { use super::*; + use crate::listing::classify::classify; + use crate::listing::model::RawOp; /// The reduction of the filing image: a read-only pool at 0x6a0 holding the /// displacement -0x1c1, and `.rodata` at 0x400. @@ -445,6 +931,20 @@ mod tests { )] } + /// Thumb's `ldr rN,[pc,#imm]`: the slot is exported as a data-space + /// varnode, so the literal load is a `COPY`, not a `LOAD`. + fn thumb_ldr(s: &Spaces, reg: u64, at: u64) -> Vec { + vec![op(OpCode::CPUI_COPY, Some(vn(&s.reg, reg, 4)), vec![vn(&s.ram, at, 4)])] + } + + /// Thumb's `add rN,pc` at `vma`: the PC reads as `inst_start + 4`. + fn thumb_add_pc(s: &Spaces, reg: u64, vma: u64) -> Vec { + vec![ + op(OpCode::CPUI_INT_ADD, Some(vn(&s.uniq, 0x2f00, 4)), vec![vn(&s.konst, vma, 4), vn(&s.konst, 4, 4)]), + op(OpCode::CPUI_INT_ADD, Some(vn(&s.reg, reg, 4)), vec![vn(&s.reg, reg, 4), vn(&s.uniq, 0x2f00, 4)]), + ] + } + /// `add rN,pc,rN` at `vma`, exactly as the ARM SLEIGH lifts it: the PC is /// `inst_start + 8`, formed in a temporary first. fn add_pc(s: &Spaces, reg: u64, vma: u64) -> Vec { @@ -462,106 +962,167 @@ mod tests { ] } + /// One body's walk, driven in the order the test lists the instructions. + struct Walk<'a> { + pp: PicPool, + pool: &'a PoolImage<'a>, + mapped: &'a [(u64, u64)], + extended: bool, + got: Option<&'a GotImage<'a>>, + /// What the walk has queued, as `step` snapshots it at the first load. + queue: Vec, + preserved: &'a [(u64, u64)], + s: &'a Spaces, + } + + impl<'a> Walk<'a> { + fn new(s: &'a Spaces, pool: &'a PoolImage<'a>, mapped: &'a [(u64, u64)]) -> Self { + Walk { pp: PicPool::default(), pool, mapped, extended: true, got: None, queue: Vec::new(), preserved: &[], s } + } + + fn step(&mut self, vma: u64, ops: &[FullOp], branches: &[u64]) -> Option { + self.step_len(vma, 4, ops, branches) + } + + fn step_len(&mut self, vma: u64, len: u32, ops: &[FullOp], branches: &[u64]) -> Option { + let raw: Vec = + ops.iter().map(|op| RawOp { opcode: op.opcode, in0: op.ins.first().cloned() }).collect(); + let fall = classify(&raw, vma, len).fall_through; + let img = Image { + pool: self.pool, + mapped: self.mapped, + extended: self.extended, + data_space: Some(&self.s.ram), + got: self.got, + preserved: self.preserved, + }; + self.pp.step(&img, Insn { vma, len, fall, ops, branches }, || self.queue.clone()) + } + + fn finish(self) -> Verdict { + self.pp.finish(&HashSet::new()) + } + } + + fn refs(v: &Verdict) -> Vec<(u64, u64, XrefKind)> { + v.refs.clone() + } + /// The defect: the address of the literal is in no instruction and in no pool /// word — only in the two put together. #[test] fn a_pool_displacement_added_to_the_pc_is_the_address_it_forms() { - let s = spaces(); - let mut pp = PicPool::default(); - assert!(pp.step(0x660, 4, &ldr(&s, 0x20, 0x6a0), &image(), &MAPPED, Some(&s.ram)).is_empty()); - assert_eq!( - pp.step(0x664, 4, &add_pc(&s, 0x20, 0x664), &image(), &MAPPED, Some(&s.ram)), - vec![0x4ab], - "0x66c + (-0x1c1)" - ); + let (s, pool) = (spaces(), image()); + let mut w = Walk::new(&s, &pool, &MAPPED); + w.step(0x660, &ldr(&s, 0x20, 0x6a0), &[]); + w.step(0x664, &add_pc(&s, 0x20, 0x664), &[]); + assert_eq!(refs(&w.finish()), vec![(0x664, 0x4ab, XrefKind::Data)], "0x66c + (-0x1c1)"); } /// The pair need not be adjacent: instruction scheduling separates the load /// from the `add` when a function forms several references at once. #[test] fn the_add_may_sit_a_few_instructions_past_the_load() { - let s = spaces(); - let mut pp = PicPool::default(); + let (s, pool) = (spaces(), image()); let nop = vec![op( OpCode::CPUI_INT_ADD, Some(vn(&s.reg, 0x30, 4)), vec![vn(&s.reg, 0x30, 4), vn(&s.konst, 4, 4)], )]; - pp.step(0x660, 4, &ldr(&s, 0x20, 0x6a0), &image(), &MAPPED, Some(&s.ram)); - pp.step(0x664, 4, &nop, &image(), &MAPPED, Some(&s.ram)); - assert_eq!( - pp.step(0x668, 4, &add_pc(&s, 0x20, 0x668), &image(), &MAPPED, Some(&s.ram)), - vec![0x4af] - ); + let mut w = Walk::new(&s, &pool, &MAPPED); + w.step(0x660, &ldr(&s, 0x20, 0x6a0), &[]); + w.step(0x664, &nop, &[]); + w.step(0x668, &add_pc(&s, 0x20, 0x668), &[]); + assert_eq!(refs(&w.finish()), vec![(0x668, 0x4af, XrefKind::Data)]); } /// Overwriting the register between the load and the `add` ends the value: /// what the `add` composes is whatever was written last. #[test] fn a_write_to_the_register_ends_the_tracked_word() { - let s = spaces(); - let mut pp = PicPool::default(); + let (s, pool) = (spaces(), image()); let clobber = vec![op(OpCode::CPUI_COPY, Some(vn(&s.reg, 0x20, 4)), vec![vn(&s.konst, 0, 4)])]; - pp.step(0x660, 4, &ldr(&s, 0x20, 0x6a0), &image(), &MAPPED, Some(&s.ram)); - pp.step(0x664, 4, &clobber, &image(), &MAPPED, Some(&s.ram)); - assert!(pp - .step(0x668, 4, &add_pc(&s, 0x20, 0x668), &image(), &MAPPED, Some(&s.ram)) - .is_empty()); + let mut w = Walk::new(&s, &pool, &MAPPED); + w.step(0x660, &ldr(&s, 0x20, 0x6a0), &[]); + w.step(0x664, &clobber, &[]); + w.step(0x668, &add_pc(&s, 0x20, 0x668), &[]); + assert!(w.finish().refs.is_empty()); } - /// A call clobbers the caller-saved registers, so nothing survives it. + /// A call clobbers the caller-saved registers, so nothing in them survives it. #[test] fn a_call_ends_the_tracked_word() { - let s = spaces(); - let mut pp = PicPool::default(); + let (s, pool) = (spaces(), image()); let mut call = ldr(&s, 0x20, 0x6a0); call.push(op(OpCode::CPUI_CALL, None, vec![vn(&s.ram, 0x700, 4)])); - pp.step(0x660, 4, &call, &image(), &MAPPED, Some(&s.ram)); - assert!(pp - .step(0x664, 4, &add_pc(&s, 0x20, 0x664), &image(), &MAPPED, Some(&s.ram)) - .is_empty()); + let mut w = Walk::new(&s, &pool, &MAPPED); + w.step(0x660, &call, &[]); + w.step(0x664, &add_pc(&s, 0x20, 0x664), &[]); + assert!(w.finish().refs.is_empty()); } - /// The three refusals, each of which would be a fabricated reference. + /// A word in a callee-saved register survives the call between its load and + /// its `add`, as GCC schedules the GOT base's pair around a call. #[test] - fn arithmetic_without_the_pc_a_pointer_word_and_an_unmapped_sum_are_declined() { - let s = spaces(); + fn a_call_keeps_a_word_in_a_callee_saved_register() { + let (s, pool) = (spaces(), image()); + let mut call = ldr(&s, 0x30, 0x6a0); + call.push(op(OpCode::CPUI_CALL, None, vec![vn(&s.ram, 0x700, 4)])); + let mut w = Walk::new(&s, &pool, &MAPPED); + w.preserved = &[(0x30, 0x34)]; + w.step(0x660, &call, &[]); + w.step(0x664, &add_pc(&s, 0x30, 0x664), &[]); + assert_eq!(refs(&w.finish()), vec![(0x664, 0x4ab, XrefKind::Data)]); + } + + /// Thumb's `bl` also switches the instruction set through a `CALLOTHER`; + /// inside a call that is no more than the call, so a word in a + /// callee-saved register still survives it. + #[test] + fn a_thumb_call_keeps_a_word_in_a_callee_saved_register() { + let (s, pool) = (spaces(), image()); + let bl = vec![ + op(OpCode::CPUI_CALLOTHER, None, vec![vn(&s.konst, 62, 4), vn(&s.reg, 0x69, 1)]), + op(OpCode::CPUI_CALL, None, vec![vn(&s.ram, 0x700, 4)]), + ]; + let mut w = Walk::new(&s, &pool, &MAPPED); + w.preserved = &[(0x30, 0x34)]; + w.step_len(0x660, 2, &thumb_ldr(&s, 0x30, 0x6a0), &[]); + w.step_len(0x662, 4, &bl, &[]); + w.step_len(0x666, 2, &thumb_add_pc(&s, 0x30, 0x666), &[]); + assert_eq!(refs(&w.finish()), vec![(0x666, 0x4a9, XrefKind::Data)], "0x66a + (-0x1c1)"); + } + + /// The two refusals, each of which would be a fabricated reference. + #[test] + fn arithmetic_without_the_pc_and_an_unmapped_sum_are_declined() { + let (s, pool) = (spaces(), image()); // `add r0,r0,#8` — the pool word used as a number, not as a base. - let mut pp = PicPool::default(); - pp.step(0x660, 4, &ldr(&s, 0x20, 0x6a0), &image(), &MAPPED, Some(&s.ram)); let no_pc = vec![op( OpCode::CPUI_INT_ADD, Some(vn(&s.reg, 0x20, 4)), vec![vn(&s.reg, 0x20, 4), vn(&s.konst, 8, 4)], )]; - assert!(pp.step(0x664, 4, &no_pc, &image(), &MAPPED, Some(&s.ram)).is_empty()); - - // The word at 0x6a4 IS a mapped address, so it is `poolref`'s pointer to - // follow and never a displacement. - let mut pp = PicPool::default(); - pp.step(0x660, 4, &ldr(&s, 0x20, 0x6a4), &image(), &MAPPED, Some(&s.ram)); - assert!(pp - .step(0x664, 4, &add_pc(&s, 0x20, 0x664), &image(), &MAPPED, Some(&s.ram)) - .is_empty()); + let mut w = Walk::new(&s, &pool, &MAPPED); + w.step(0x660, &ldr(&s, 0x20, 0x6a0), &[]); + w.step(0x664, &no_pc, &[]); + assert!(w.finish().refs.is_empty()); // A sum landing in no section is not an address, whatever formed it. - let mut pp = PicPool::default(); - pp.step(0x660, 4, &ldr(&s, 0x20, 0x6a0), &image(), &MAPPED, Some(&s.ram)); - assert!(pp - .step(0x664, 4, &add_pc(&s, 0x20, 0x664), &image(), &[(0x9000, 0x9100)], Some(&s.ram)) - .is_empty()); + let mut w = Walk::new(&s, &pool, &[(0x9000, 0x9100)]); + w.step(0x660, &ldr(&s, 0x20, 0x6a0), &[]); + w.step(0x664, &add_pc(&s, 0x20, 0x664), &[]); + assert!(w.finish().refs.is_empty()); } - /// The GOT idiom `ldr r1,[pc,#imm]; ldr r1,[pc,r1]` composes its address into - /// a temporary and dereferences it. The address is never held in a register, - /// so nothing is filed — the double indirection is not this mechanism's. + /// `ldr r1,[pc,#imm]; ldr r1,[pc,r1]` composes its address into a temporary + /// and dereferences it. The address is never held in a register and is no + /// GOT base, so nothing is filed. #[test] fn a_composed_address_that_stays_in_a_temporary_files_nothing() { - let s = spaces(); - let mut pp = PicPool::default(); - pp.step(0x660, 4, &ldr(&s, 0x24, 0x6a0), &image(), &MAPPED, Some(&s.ram)); + let (s, pool) = (spaces(), image()); let indirect = vec![ op( OpCode::CPUI_INT_ADD, @@ -579,40 +1140,486 @@ mod tests { vec![vn(&s.konst, ram_id(&s), 8), vn(&s.uniq, 0x10900, 4)], ), ]; - assert!(pp.step(0x664, 4, &indirect, &image(), &MAPPED, Some(&s.ram)).is_empty()); + let mut w = Walk::new(&s, &pool, &MAPPED); + w.step(0x660, &ldr(&s, 0x24, 0x6a0), &[]); + w.step(0x664, &indirect, &[]); + assert!(w.finish().refs.is_empty()); } /// A narrow or unaligned pool read is not a slot at all, which is /// [`PoolImage`]'s rule and is inherited here. #[test] fn a_narrow_pool_read_is_not_a_displacement() { - let s = spaces(); - let mut pp = PicPool::default(); + let (s, pool) = (spaces(), image()); let narrow = vec![op( OpCode::CPUI_LOAD, Some(vn(&s.reg, 0x20, 2)), vec![vn(&s.konst, ram_id(&s), 8), vn(&s.konst, 0x6a0, 4)], )]; - pp.step(0x660, 4, &narrow, &image(), &MAPPED, Some(&s.ram)); - assert!(pp - .step(0x664, 4, &add_pc(&s, 0x20, 0x664), &image(), &MAPPED, Some(&s.ram)) - .is_empty()); + let mut w = Walk::new(&s, &pool, &MAPPED); + w.step(0x660, &narrow, &[]); + w.step(0x664, &add_pc(&s, 0x20, 0x664), &[]); + assert!(w.finish().refs.is_empty()); + } + + /// A displacement that happens to be a mapped address itself, as in a + /// non-PIE image mapped low, is still a displacement once the PC is added, + /// and `poolref` must not also file it as a pointer. + #[test] + fn a_displacement_that_is_also_a_mapped_address_still_composes() { + const P: [u8; 4] = [0x00, 0x05, 0x00, 0x00]; // 0x6a0 -> 0x500, mapped + const MAP: [(u64, u64); 1] = [(0x400, 0x1000)]; + let pool = PoolImage::from_ranges(vec![(0x6a0, 0x6a4, &P[..])], 4, true).unwrap(); + let s = spaces(); + let mut w = Walk::new(&s, &pool, &MAP); + assert_eq!(w.step(0x660, &ldr(&s, 0x20, 0x6a0), &[]), Some(0x500)); + w.step(0x664, &add_pc(&s, 0x20, 0x664), &[]); + let v = w.finish(); + assert_eq!(refs(&v), vec![(0x664, 0xb6c, XrefKind::Data)]); + assert!(v.displaced.contains(&0x660)); + } + + /// An image with no GOT and no dynamic section seeds neither a mapped word + /// nor a Thumb literal copy, and records nothing for them; an A32 load of + /// a word that is not an address still composes. + #[test] + fn a_static_image_without_a_got_seeds_only_an_a32_displacement() { + const P: [u8; 4] = [0x00, 0x15, 0x00, 0x00]; // 0x6a0 -> 0x1500, mapped + const MAP: [(u64, u64); 1] = [(0x400, 0x2000)]; + let mapped_pool = PoolImage::from_ranges(vec![(0x6a0, 0x6a4, &P[..])], 4, true).unwrap(); + let (s, pool) = (spaces(), image()); + let mut w = Walk::new(&s, &mapped_pool, &MAP); + w.extended = false; + assert_eq!(w.step(0x660, &ldr(&s, 0x20, 0x6a0), &[]), None); + w.step(0x664, &add_pc(&s, 0x20, 0x664), &[]); + assert!(!w.pp.live); + assert!(w.finish().refs.is_empty()); + + let mut w = Walk::new(&s, &pool, &MAPPED); + w.extended = false; + w.step_len(0x660, 2, &thumb_ldr(&s, 0x20, 0x6a0), &[]); + w.step_len(0x662, 2, &thumb_add_pc(&s, 0x20, 0x662), &[]); + assert!(!w.pp.live); + assert!(w.finish().refs.is_empty()); + + let mut w = Walk::new(&s, &pool, &MAPPED); + w.extended = false; + w.step(0x660, &ldr(&s, 0x20, 0x6a0), &[]); + w.step(0x664, &add_pc(&s, 0x20, 0x664), &[]); + assert_eq!(refs(&w.finish()), vec![(0x664, 0x4ab, XrefKind::Data)]); + } + + /// A mapped word that no PC fold consumes stays `poolref`'s pointer. + #[test] + fn a_mapped_word_used_as_a_number_is_not_displaced() { + let (s, pool) = (spaces(), image()); + let no_pc = vec![op( + OpCode::CPUI_INT_ADD, + Some(vn(&s.reg, 0x20, 4)), + vec![vn(&s.reg, 0x20, 4), vn(&s.konst, 8, 4)], + )]; + let mut w = Walk::new(&s, &pool, &MAPPED); + assert_eq!(w.step(0x660, &ldr(&s, 0x20, 0x6a4), &[]), Some(0x500)); + w.step(0x664, &no_pc, &[]); + let v = w.finish(); + assert!(v.refs.is_empty() && v.displaced.is_empty()); + } + + /// Thumb spells the literal load as a copy out of the image and reads the + /// PC as `inst_start + 4`; the pair composes the same way, and the word is + /// displaced, not `poolref`'s pointer. + #[test] + fn a_thumb_literal_copy_composes_with_the_thumb_pc() { + let (s, pool) = (spaces(), image()); + let mut w = Walk::new(&s, &pool, &MAPPED); + w.step_len(0x660, 2, &thumb_ldr(&s, 0x20, 0x6a0), &[]); + w.step_len(0x662, 2, &thumb_add_pc(&s, 0x20, 0x662), &[]); + let v = w.finish(); + assert_eq!(refs(&v), vec![(0x662, 0x4a5, XrefKind::Data)], "0x666 + (-0x1c1)"); + assert!(v.displaced.contains(&0x660)); + } + + /// A copy out of the image that is not a pool slot is no literal load. + #[test] + fn a_copy_from_outside_the_pool_is_not_a_literal_load() { + let (s, pool) = (spaces(), image()); + let mut w = Walk::new(&s, &pool, &MAPPED); + w.step_len(0x660, 2, &thumb_ldr(&s, 0x20, 0x600), &[]); + w.step_len(0x662, 2, &thumb_add_pc(&s, 0x20, 0x662), &[]); + assert!(w.finish().refs.is_empty()); } /// Nothing is tracked past the hop bound. #[test] fn a_word_is_not_believed_forever() { - let s = spaces(); - let mut pp = PicPool::default(); + let (s, pool) = (spaces(), image()); let nothing: Vec = Vec::new(); - pp.step(0x600, 4, &ldr(&s, 0x20, 0x6a0), &image(), &MAPPED, Some(&s.ram)); + let mut w = Walk::new(&s, &pool, &MAPPED); + w.step(0x600, &ldr(&s, 0x20, 0x6a0), &[]); let mut vma = 0x604; for _ in 0..MAX_HOPS { - pp.step(vma, 4, ¬hing, &image(), &MAPPED, Some(&s.ram)); + w.step(vma, ¬hing, &[]); vma += 4; } - assert!(pp - .step(vma, 4, &add_pc(&s, 0x20, vma), &image(), &MAPPED, Some(&s.ram)) - .is_empty()); + w.step(vma, &add_pc(&s, 0x20, vma), &[]); + assert!(w.finish().refs.is_empty()); + } + + // --- GOT bases ----------------------------------------------------------- + + /// Pool words for a GOT at 0x800: 0x6a0 holds the base's displacement from + /// an `add` at 0x604 (0x60c + 0x1f4 = 0x800), 0x6a4 the GOT offset 0x10 of + /// the slot at 0x810, which holds the function address 0x1450. + const GOT_POOL: [u8; 8] = [ + 0xf4, 0x01, 0x00, 0x00, // 0x6a0 -> 0x1f4 + 0x10, 0x00, 0x00, 0x00, // 0x6a4 -> 0x10 + ]; + const GOT_BYTES: [u8; 0x20] = { + let mut g = [0u8; 0x20]; + g[0x10] = 0x50; + g[0x11] = 0x14; + g + }; + const GOT_MAP: [(u64, u64); 1] = [(0x400, 0x2000)]; + const R4: u64 = 0x30; + const R5: u64 = 0x34; + const R2: u64 = 0x28; + + fn got_pool() -> PoolImage<'static> { + PoolImage::from_ranges(vec![(0x6a0, 0x6a8, &GOT_POOL[..])], 4, true).unwrap() + } + + fn got_image() -> GotImage<'static> { + GotImage { lo: 0x800, hi: 0x820, data: &GOT_BYTES[..], ptr: 4, little_endian: true } + } + + /// `ldr r2,[r4,r2]`. + fn slot_load(s: &Spaces) -> Vec { + vec![ + op(OpCode::CPUI_INT_ADD, Some(vn(&s.uniq, 0x2f00, 4)), vec![vn(&s.reg, R4, 4), vn(&s.reg, R2, 4)]), + op(OpCode::CPUI_LOAD, Some(vn(&s.reg, R2, 4)), vec![vn(&s.konst, ram_id(&s), 8), vn(&s.uniq, 0x2f00, 4)]), + ] + } + + fn jump(s: &Spaces, to: u64) -> Vec { + vec![op(OpCode::CPUI_BRANCH, None, vec![vn(&s.ram, to, 4)])] + } + + fn cbranch(s: &Spaces, to: u64) -> Vec { + vec![op(OpCode::CPUI_CBRANCH, None, vec![vn(&s.ram, to, 4), vn(&s.reg, 0x100, 1)])] + } + + const SLOT_REFS: [(u64, XrefKind); 2] = [(0x810, XrefKind::Read), (0x1450, XrefKind::Data)]; + + fn slot_refs(v: &Verdict, at: u64) -> Vec<(u64, XrefKind)> { + v.refs.iter().filter(|r| r.0 == at).map(|r| (r.1, r.2)).collect() + } + + /// The ARM GOT idiom of a `-fPIC` body: `ldr r4,[pc,#a]; add r4,pc,r4` + /// makes r4 the GOT, and `ldr r2,[pc,#b]; ldr r2,[r4,r2]` reads a slot. The + /// base survives a call (r4 is callee-saved) and a branch, and the slot's + /// word, a function address, is filed as the address the body takes. + #[test] + fn a_load_through_a_pc_composed_got_base_files_the_slot_and_its_word() { + let (s, pool, got) = (spaces(), got_pool(), got_image()); + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + w.preserved = &[(R4, R4 + 4)]; + let call = vec![op(OpCode::CPUI_CALL, None, vec![vn(&s.ram, 0x400, 4)])]; + w.step(0x600, &ldr(&s, R4, 0x6a0), &[]); + w.step(0x604, &add_pc(&s, R4, 0x604), &[]); + w.step(0x608, &call, &[]); + w.step(0x60c, &jump(&s, 0x700), &[0x700]); + w.step(0x700, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x704, &slot_load(&s), &[]); + let v = w.finish(); + assert_eq!(slot_refs(&v, 0x604), vec![(0x800, XrefKind::Data)]); + assert_eq!(slot_refs(&v, 0x704), SLOT_REFS.to_vec()); + } + + /// The Thumb GOT idiom: a slot holding a Thumb function's entry with bit 0 + /// set is answered as it is; whether the bit names code is the walk's to + /// decide, once it knows what it decoded. + #[test] + fn a_thumb_slot_word_is_answered_with_its_thumb_bit() { + const ODD: [u8; 0x20] = { + let mut g = [0u8; 0x20]; + g[0x10] = 0x51; + g[0x11] = 0x14; + g + }; + let (s, pool) = (spaces(), got_pool()); + let got = GotImage { lo: 0x800, hi: 0x820, data: &ODD[..], ptr: 4, little_endian: true }; + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + w.step_len(0x604, 2, &thumb_ldr(&s, R4, 0x6a0), &[]); + w.step_len(0x606, 2, &thumb_ldr(&s, R2, 0x6a4), &[]); + w.step_len(0x608, 2, &thumb_add_pc(&s, R4, 0x608), &[]); + w.step_len(0x60a, 2, &slot_load(&s), &[]); + assert_eq!(slot_refs(&w.finish(), 0x60a), vec![(0x810, XrefKind::Read), (0x1451, XrefKind::Data)]); + } + + /// `mov r5,r4` copies the base: the copy files nothing, and r5 is a base + /// the slot load can go through. + #[test] + fn a_copy_of_the_base_is_a_base_and_files_nothing() { + let (s, pool, got) = (spaces(), got_pool(), got_image()); + let mov = vec![op(OpCode::CPUI_COPY, Some(vn(&s.reg, R5, 4)), vec![vn(&s.reg, R4, 4)])]; + let through_r5 = vec![ + op(OpCode::CPUI_INT_ADD, Some(vn(&s.uniq, 0x2f00, 4)), vec![vn(&s.reg, R5, 4), vn(&s.reg, R2, 4)]), + op(OpCode::CPUI_LOAD, Some(vn(&s.reg, R2, 4)), vec![vn(&s.konst, ram_id(&s), 8), vn(&s.uniq, 0x2f00, 4)]), + ]; + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + w.step(0x600, &ldr(&s, R4, 0x6a0), &[]); + w.step(0x604, &add_pc(&s, R4, 0x604), &[]); + w.step(0x608, &mov, &[]); + w.step(0x60c, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x610, &through_r5, &[]); + let v = w.finish(); + assert!(slot_refs(&v, 0x608).is_empty(), "{:?}", v.refs); + assert_eq!(slot_refs(&v, 0x610), SLOT_REFS.to_vec()); + } + + /// Only base plus a pool word (GOTOFF) is an address: base plus an + /// immediate is arithmetic on the base and names nothing, even where it + /// lands in a mapped section. + #[test] + fn base_plus_an_immediate_files_nothing_and_base_plus_a_pool_word_does() { + let (s, pool, got) = (spaces(), got_pool(), got_image()); + let add_imm = vec![op(OpCode::CPUI_INT_ADD, Some(vn(&s.reg, R5, 4)), vec![vn(&s.reg, R4, 4), vn(&s.konst, 8, 4)])]; + let gotoff = vec![op(OpCode::CPUI_INT_ADD, Some(vn(&s.reg, 0x20, 4)), vec![vn(&s.reg, R4, 4), vn(&s.reg, R2, 4)])]; + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + w.step(0x600, &ldr(&s, R4, 0x6a0), &[]); + w.step(0x604, &add_pc(&s, R4, 0x604), &[]); + w.step(0x608, &add_imm, &[]); + w.step(0x60c, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x610, &gotoff, &[]); + let v = w.finish(); + assert!(slot_refs(&v, 0x608).is_empty(), "{:?}", v.refs); + assert_eq!(slot_refs(&v, 0x610), vec![(0x810, XrefKind::Data)]); + } + + /// The join rule. 0x620 is reached from 0x608, where r4 is the GOT, and from + /// 0x610, after 0x60c overwrote r4. The breadth-first walk decodes 0x620 + /// before 0x610, so only the meet over both edges, taken once the body is + /// known, can tell that r4 is not the GOT there. + #[test] + fn a_base_that_one_path_overwrites_is_not_believed_at_the_join() { + let (s, pool, got) = (spaces(), got_pool(), got_image()); + let clobber = vec![op(OpCode::CPUI_COPY, Some(vn(&s.reg, R4, 4)), vec![vn(&s.konst, 0x810, 4)])]; + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + w.step(0x600, &ldr(&s, R4, 0x6a0), &[]); + w.step(0x604, &add_pc(&s, R4, 0x604), &[]); + w.step(0x608, &cbranch(&s, 0x620), &[0x620]); + w.step(0x60c, &clobber, &[]); + w.step(0x620, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x610, &jump(&s, 0x620), &[0x620]); + w.step(0x624, &slot_load(&s), &[]); + let v = w.finish(); + assert!(slot_refs(&v, 0x624).is_empty(), "got {:?}", v.refs); + } + + /// The same join with both paths keeping r4: the base is equal on every + /// edge, so the meet keeps it. + #[test] + fn a_base_both_paths_keep_survives_the_join() { + let (s, pool, got) = (spaces(), got_pool(), got_image()); + let other = vec![op(OpCode::CPUI_COPY, Some(vn(&s.reg, 0x20, 4)), vec![vn(&s.konst, 0, 4)])]; + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + w.step(0x600, &ldr(&s, R4, 0x6a0), &[]); + w.step(0x604, &add_pc(&s, R4, 0x604), &[]); + w.step(0x608, &cbranch(&s, 0x620), &[0x620]); + w.step(0x60c, &other, &[]); + w.step(0x620, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x610, &jump(&s, 0x620), &[0x620]); + w.step(0x624, &slot_load(&s), &[]); + assert_eq!(slot_refs(&w.finish(), 0x624), SLOT_REFS.to_vec()); + } + + /// A predicated instruction (`ldrne`) spells its condition as a branch to its + /// own fall-through; that is not a join, and the word carries past it. + #[test] + fn a_predicated_instruction_between_the_load_and_the_add_keeps_the_word() { + let (s, pool) = (spaces(), image()); + let mut w = Walk::new(&s, &pool, &MAPPED); + w.step(0x660, &ldr(&s, 0x20, 0x6a0), &[]); + w.step(0x664, &cbranch(&s, 0x668), &[0x668]); + w.step(0x668, &add_pc(&s, 0x20, 0x668), &[]); + assert_eq!(refs(&w.finish()), vec![(0x668, 0x4af, XrefKind::Data)]); + } + + /// `popeq {r4-r6,pc}` as A32 lifts it: the condition, a branch to the next + /// instruction when it fails, then the pops and the return. + fn popeq(s: &Spaces, next: u64) -> Vec { + let pop = |reg: u64| { + op(OpCode::CPUI_LOAD, Some(vn(&s.reg, reg, 4)), vec![vn(&s.konst, ram_id(s), 8), vn(&s.reg, 0x54, 4)]) + }; + vec![ + op(OpCode::CPUI_BOOL_NEGATE, Some(vn(&s.uniq, 0x100, 1)), vec![vn(&s.reg, 0x101, 1)]), + op(OpCode::CPUI_CBRANCH, None, vec![vn(&s.ram, next, 4), vn(&s.uniq, 0x100, 1)]), + pop(R4), + pop(R5), + pop(0x38), + op(OpCode::CPUI_RETURN, None, vec![vn(&s.reg, 0x5c, 4)]), + ] + } + + /// A conditional return writes r4 only on the path where it runs, which + /// leaves the body; the path that falls past it keeps the base. + #[test] + fn a_base_survives_past_a_predicated_return_that_pops_it() { + let (s, pool, got) = (spaces(), got_pool(), got_image()); + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + w.step(0x600, &ldr(&s, R4, 0x6a0), &[]); + w.step(0x604, &add_pc(&s, R4, 0x604), &[]); + w.step(0x608, &popeq(&s, 0x60c), &[0x60c]); + w.step(0x60c, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x610, &slot_load(&s), &[]); + assert_eq!(slot_refs(&w.finish(), 0x610), SLOT_REFS.to_vec()); + } + + /// A conditional write to r4 that falls through (`ldrne r4,[sp]`) leaves + /// r4 the base on one path only, so it is not believed after it. + #[test] + fn a_predicated_write_that_falls_through_ends_the_base() { + let (s, pool, got) = (spaces(), got_pool(), got_image()); + let ldrne = vec![ + op(OpCode::CPUI_BOOL_NEGATE, Some(vn(&s.uniq, 0x100, 1)), vec![vn(&s.reg, 0x101, 1)]), + op(OpCode::CPUI_CBRANCH, None, vec![vn(&s.ram, 0x60c, 4), vn(&s.uniq, 0x100, 1)]), + op(OpCode::CPUI_LOAD, Some(vn(&s.reg, R4, 4)), vec![vn(&s.konst, ram_id(&s), 8), vn(&s.reg, 0x54, 4)]), + ]; + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + w.step(0x600, &ldr(&s, R4, 0x6a0), &[]); + w.step(0x604, &add_pc(&s, R4, 0x604), &[]); + w.step(0x608, &ldrne, &[0x60c]); + w.step(0x60c, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x610, &slot_load(&s), &[]); + assert!(slot_refs(&w.finish(), 0x610).is_empty()); + } + + /// A switch dispatched before any pool load: `cases` records nothing yet, + /// and a case target queued before the first load still does not believe + /// the base another path brings — the walk's dry snapshot vetoes it. + #[test] + fn a_switch_before_any_pool_load_still_vetoes_its_case_targets() { + let (s, pool, got) = (spaces(), got_pool(), got_image()); + let dispatch = vec![op(OpCode::CPUI_BRANCHIND, None, vec![vn(&s.reg, 0x20, 4)])]; + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + w.step(0x5f0, &dispatch, &[]); + w.pp.cases(0x5f0, &[0x600, 0x640]); + w.queue = vec![0x640]; + w.step(0x600, &ldr(&s, R4, 0x6a0), &[]); + w.step(0x604, &add_pc(&s, R4, 0x604), &[]); + w.step(0x608, &jump(&s, 0x640), &[0x640]); + w.step(0x640, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x644, &slot_load(&s), &[]); + let v = w.finish(); + assert!(slot_refs(&v, 0x644).is_empty(), "{:?}", v.refs); + } + + /// An instruction another body's walk refused to enter has a predecessor + /// there that this body never saw, so nothing holds at it. + #[test] + fn nothing_holds_where_another_walk_enters() { + let (s, pool, got) = (spaces(), got_pool(), got_image()); + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + w.step(0x600, &ldr(&s, R4, 0x6a0), &[]); + w.step(0x604, &add_pc(&s, R4, 0x604), &[]); + w.step(0x608, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x60c, &slot_load(&s), &[]); + let v = w.pp.finish(&HashSet::from([0x60c])); + assert!(slot_refs(&v, 0x60c).is_empty(), "{:?}", v.refs); + } + + /// A switch's case bodies are the dispatcher's own code: the base the + /// dispatch holds reaches each case, and a slot load there finds the slot. + #[test] + fn a_switch_case_inherits_the_dispatch_base() { + let (s, pool, got) = (spaces(), got_pool(), got_image()); + let dispatch = vec![op(OpCode::CPUI_BRANCHIND, None, vec![vn(&s.reg, 0x20, 4)])]; + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + w.step(0x600, &ldr(&s, R4, 0x6a0), &[]); + w.step(0x604, &add_pc(&s, R4, 0x604), &[]); + w.step(0x608, &dispatch, &[]); + w.pp.cases(0x608, &[0x700, 0x710]); + w.step(0x700, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x704, &slot_load(&s), &[]); + w.step(0x710, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x714, &slot_load(&s), &[]); + let v = w.finish(); + assert_eq!(slot_refs(&v, 0x704), SLOT_REFS.to_vec()); + assert_eq!(slot_refs(&v, 0x714), SLOT_REFS.to_vec()); + } + + /// Before the body's first pool load nothing is recorded at all: on an + /// architecture or a body that never forms one of these pairs the walk + /// costs one scan of each instruction's ops. + #[test] + fn nothing_is_recorded_before_the_first_pool_load() { + let (s, pool) = (spaces(), image()); + let mut w = Walk::new(&s, &pool, &MAPPED); + w.step(0x5f0, &cbranch(&s, 0x640), &[0x640]); + w.step(0x5f4, &jump(&s, 0x600), &[0x600]); + w.pp.cases(0x5f4, &[0x600]); + assert!(w.pp.edges.is_empty() && w.pp.nodes.is_empty() && !w.pp.live); + } + + /// An instruction queued before the first pool load was queued by one that + /// carried nothing, so it holds nothing, though the edge was never recorded. + #[test] + fn what_was_queued_before_the_first_pool_load_holds_nothing() { + let (s, pool, got) = (spaces(), got_pool(), got_image()); + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + let other = vec![op(OpCode::CPUI_COPY, Some(vn(&s.reg, 0x20, 4)), vec![vn(&s.konst, 0, 4)])]; + w.step(0x5f8, &cbranch(&s, 0x640), &[0x640]); + w.queue = vec![0x640]; + w.step(0x5fc, &ldr(&s, R4, 0x6a0), &[]); + w.step(0x600, &other, &[]); + w.step(0x604, &add_pc(&s, R4, 0x604), &[]); + w.step(0x608, &jump(&s, 0x640), &[0x640]); + w.step(0x640, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x644, &slot_load(&s), &[]); + let v = w.finish(); + assert!(slot_refs(&v, 0x644).is_empty(), "{:?}", v.refs); + assert_eq!(slot_refs(&v, 0x604), vec![(0x800, XrefKind::Data)]); + } + + /// A loop whose body does not write r4 keeps the base across its back edge. + #[test] + fn a_base_survives_a_loop_back_edge() { + let (s, pool, got) = (spaces(), got_pool(), got_image()); + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + w.step(0x600, &ldr(&s, R4, 0x6a0), &[]); + w.step(0x604, &add_pc(&s, R4, 0x604), &[]); + w.step(0x608, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x60c, &slot_load(&s), &[]); + w.step(0x610, &cbranch(&s, 0x608), &[0x608]); + assert_eq!(slot_refs(&w.finish(), 0x60c), SLOT_REFS.to_vec()); + } + + /// The walk decodes the entry before any edge into it, so a branch back to + /// the entry cannot make a base hold there. + #[test] + fn nothing_holds_at_the_entry() { + let (s, pool, got) = (spaces(), got_pool(), got_image()); + let mut w = Walk::new(&s, &pool, &GOT_MAP); + w.got = Some(&got); + w.step(0x5f8, &ldr(&s, R2, 0x6a4), &[]); + w.step(0x5fc, &slot_load(&s), &[]); + w.step(0x600, &ldr(&s, R4, 0x6a0), &[]); + w.step(0x604, &add_pc(&s, R4, 0x604), &[]); + w.step(0x608, &jump(&s, 0x5f8), &[0x5f8]); + assert!(slot_refs(&w.finish(), 0x5fc).is_empty()); } } diff --git a/decompiler/crates/kuna-analysis/src/listing/xrefs.rs b/decompiler/crates/kuna-analysis/src/listing/xrefs.rs index 038288d1e8..881afd5f4f 100644 --- a/decompiler/crates/kuna-analysis/src/listing/xrefs.rs +++ b/decompiler/crates/kuna-analysis/src/listing/xrefs.rs @@ -78,7 +78,7 @@ use object::SectionKind; use super::classify::classify; use super::context::ContextPainter; use super::kuna_picbase::{self, Ctx as PicCtx, PicBase}; -use super::kuna_picpool::PicPool; +use super::kuna_picpool::{self, GotImage, Insn, PicPool}; use super::kuna_poolref::PoolImage; use super::kuna_switchtable; use super::model::{FlowKind, RawOp}; @@ -718,6 +718,8 @@ fn descend( // read-only mapped section, and every relocatable object, whose sections are // not the runtime ones) leaves every reference below exactly as it was. let pool = if mapped.is_empty() { None } else { PoolImage::new(file) }; + // (kuna) `picpool`: the GOT's file bytes, read through a PC-composed base. + let got = if mapped.is_empty() { None } else { GotImage::new(file) }; let max_entries = match arch.max_jumptable_size { 0 => kuna_switchtable::DEFAULT_MAX_ENTRIES, n => n as usize, @@ -728,6 +730,17 @@ fn descend( preserved: if arch.analysis_switchreach { &preserved } else { &[] }, far: arch.analysis_switchreach, }; + let arm = file.architecture() == object::Architecture::Arm; + let pic_image = pool.as_ref().map(|pool| kuna_picpool::Image { + pool, + mapped: &mapped, + extended: [".got", ".got.plt", ".dynamic"].iter().any(|name| file.section_by_name(name).is_some()), + data_space: data_space.as_ref(), + got: got.as_ref(), + preserved: &preserved, + }); + // (kuna) `picpool`: odd composed addresses held for the Thumb-bit decision. + let mut thumb_refs: Vec<(u64, u64, XrefKind, String)> = Vec::new(); // (kuna) `switchtable`: the dispatches whose read stopped for want of a // decoded instruction ending at the key, as `(dispatch, mode, branch)`. let mut waiting: HashMap> = HashMap::new(); @@ -979,6 +992,9 @@ fn descend( // `add` that turns each into an address. Per function, because the values // it tracks are live only inside one straight-line run. let mut picpool = PicPool::default(); + // (kuna) `picpool`: `poolref` rows for pool words still in flight, filed + // once the body drains unless a PC fold consumed the word. + let mut held: Vec<(u64, u64, String)> = Vec::new(); // (kuna) `switchtable`: a dispatch can be decoded before the code that // falls into it -- breadth-first, or by a `.cold` fragment's jump back // into its parent's loop ahead of the parent's own walk -- and the slice @@ -1009,6 +1025,7 @@ fn descend( } let text = if read.cases.is_empty() { String::new() } else { assembly(translate, vma, &code_space) }; st.take_switch(vma, entry, &read.cases, read.stated, max_entries, &text); + picpool.cases(vma, &read.cases); insn_queue.extend(read.cases.iter().map(|&target| (target, mode))); } insn_queue.pop_front() @@ -1030,6 +1047,7 @@ fn descend( && frame_candidates.binary_search(&vma).is_ok() && !frame_cache.body_covers(entry, vma) { + st.refused.insert(vma); continue; } if sections_are_runtime && !in_range(&exec_set, vma) { @@ -1074,7 +1092,7 @@ fn descend( flow.jumps.extend(c.flows.iter().map(|&to| (vma, to))); } } - let drefs = if mapped.is_empty() { + let mut drefs = if mapped.is_empty() { Vec::new() } else { let fall_through = vma.wrapping_add(len as u64); @@ -1083,24 +1101,30 @@ fn descend( // (kuna) `picpool`: the address a PC-relative literal-pool pair forms, // which is in neither of its two instructions on its own. See // [`super::kuna_picpool`]. - let picrefs = match pool.as_ref() { - Some(p) => picpool.step( + let seeded = pic_image.as_ref().and_then(|img| { + let insn = Insn { vma, len, - cap.ops(), - p, - &mapped, - data_space.as_ref(), - ), - None => Vec::new(), - }; + fall: c.fall_through, + ops: cap.ops(), + branches: if c.flow.is_call { &[] } else { &c.flows }, + }; + picpool.step(img, insn, || insn_queue.iter().map(|&(at, _)| at).collect()) + }); + let pointer = seeded.and_then(|word| { + let i = drefs.iter().position(|&r| r == (word, XrefKind::Data))?; + Some(drefs.remove(i).0) + }); // Every row this instruction produces carries the same render, and an // instruction that produces none needs no render at all. - let mut text = if c.flows.is_empty() && drefs.is_empty() && picrefs.is_empty() { + let mut text = if c.flows.is_empty() && drefs.is_empty() && pointer.is_none() { String::new() } else { assembly(translate, vma, &code_space) }; + if let Some(word) = pointer { + held.push((vma, word, text.clone())); + } if cap.ops().iter().any(|op| op.opcode == OpCode::CPUI_CALLIND) { st.indirect_call_sites.insert(vma); @@ -1186,6 +1210,7 @@ fn descend( text = assembly(translate, vma, &code_space); } st.take_switch(vma, entry, &cases, stated, max_entries, &text); + picpool.cases(vma, &cases); insn_queue.extend(cases.iter().map(|&target| (target, mode))); } } @@ -1193,9 +1218,6 @@ fn descend( for &(to, kind) in &drefs { st.file(vma, to, kind, &text); } - for &to in &picrefs { - st.file(vma, to, XrefKind::Data, &text); - } // Both halves the deferred base-relative pass needs are pure // functions of this instruction's ops, so they are computed here // rather than by buffering (and cloning) the whole p-code. @@ -1215,6 +1237,24 @@ fn descend( } } + let verdict = picpool.finish(&st.refused); + for (vma, word, text) in held { + if !verdict.displaced.contains(&vma) { + st.file(vma, word, XrefKind::Data, &text); + } + } + // Rendered here for the same reason as `picbase`'s rows below: few + // instructions compose an address. An odd address whose even half is + // code waits for the whole walk to say whether it is a Thumb pointer. + for &(vma, to, kind) in &verdict.refs { + let text = assembly(translate, vma, &code_space); + if arm && to & 1 == 1 && in_range(&exec_set, to & !1) { + thumb_refs.push((vma, to, kind, text)); + } else { + st.file(vma, to, kind, &text); + } + } + // (kuna) `picbase`: the references this body forms THROUGH the base // register, filed only where the body cannot have changed it. if let Some((ctx, base)) = &picbase { @@ -1248,6 +1288,15 @@ fn descend( st.funcs.remove(&f); } } + // (kuna) `picpool`: a Thumb pointer is its entry with bit 0 set. See + // [`thumb_target`]. + if !thumb_refs.is_empty() { + let markers = mapping_markers(file); + for (vma, to, kind, text) in thumb_refs { + let to = thumb_target(to, &st, &markers); + st.file(vma, to, kind, &text); + } + } // The forwarding relation, over the entries the walk actually decoded. It // re-decodes at most `MAX_VENEER_INSNS` instructions per entry (a veneer is @@ -1503,6 +1552,10 @@ struct State { indirect_call_sites: BTreeSet, flow: Option, switches: Vec, + /// (kuna) `picpool`: instructions a recovered frame's walk refused to enter. + /// The frame's own edges into one are in no other body's record, so a body + /// that decodes it later cannot trust what it carries there. + refused: HashSet, } impl State { @@ -1518,6 +1571,7 @@ impl State { ..FlowGraph::default() }), switches: Vec::new(), + refused: HashSet::new(), } } @@ -1614,6 +1668,45 @@ impl State { } } +/// (kuna) `picpool`: the address an odd value `v` whose even half is in an +/// executable section names. It is a Thumb pointer, filed at `v & !1`, when the +/// walk decoded code or knows a function there, or when the nearest ARM mapping +/// symbol at or below it is `$t`; a `$d` or `$a` there makes it a datum at an +/// odd address (a string in a `.rodata` linked into `.text`), filed as it is. +/// An image with no mapping symbols says nothing, and the Thumb pointer, by far +/// the commoner of the two, is assumed. +fn thumb_target(v: u64, st: &State, markers: &[(u64, u8)]) -> u64 { + let entry = v & !1; + if st.decoded.contains(&entry) || st.funcs.contains(&entry) { + return entry; + } + let at = markers.partition_point(|&(a, _)| a <= entry); + match at.checked_sub(1).map(|i| markers[i].1) { + Some(b'd' | b'a') => v, + _ => entry, + } +} + +/// (kuna) `picpool`: the ARM mapping symbols (`$a`, `$t`, `$d`), by address. +fn mapping_markers(file: &object::File) -> Vec<(u64, u8)> { + let mut out: Vec<(u64, u8)> = file + .symbols() + .filter_map(|sym| Some((sym.address(), mapping_kind(sym.name().ok()?)?))) + .collect(); + out.sort_unstable(); + out +} + +/// The kind byte of an ARM mapping symbol name (`$a`, `$t`, `$d`, or one of +/// them followed by `.` and anything), read by byte so no other name can split +/// a character. +fn mapping_kind(name: &str) -> Option { + match name.as_bytes() { + [b'$', kind @ (b'a' | b't' | b'd')] | [b'$', kind @ (b'a' | b't' | b'd'), b'.', ..] => Some(*kind), + _ => None, + } +} + /// Lock one bucket's read ordering and collapse duplicates on `(from, to, kind)`, /// so a target reached twice from one site contributes exactly one row (the same /// contract [`super::Listing`]'s reference index holds). @@ -1971,6 +2064,19 @@ mod tests { use super::*; + /// Only `$a`, `$t`, `$d` and their `.`-suffixed forms are mapping symbols, + /// and a `$` name with a multi-byte character after it is read without + /// slicing through that character. + #[test] + fn mapping_symbol_names_are_read_by_byte() { + for (name, kind) in [("$a", b'a'), ("$t", b't'), ("$d", b'd'), ("$t.foo", b't'), ("$d.realdata", b'd')] { + assert_eq!(mapping_kind(name), Some(kind), "{name}"); + } + for name in ["$", "$x", "$é", "$\u{e9}t", "$ta", "$t_", "t", "", "é"] { + assert_eq!(mapping_kind(name), None, "{name:?}"); + } + } + /// A throwaway `(ram, constant)` space pair. `ram` stands in for the default /// data space; its index is what [`data_refs`] is told to match. fn spaces() -> (Rc, Rc) { @@ -2261,6 +2367,7 @@ mod tests { indirect_call_sites: BTreeSet::new(), flow: None, switches: Vec::new(), + refused: HashSet::new(), }; for e in edges { st.file(e.from, e.to, e.kind, ""); @@ -2318,6 +2425,7 @@ mod tests { indirect_call_sites: BTreeSet::from([0x1188]), flow: None, switches: Vec::new(), + refused: HashSet::new(), }; st.file(0x1188, 0x4008, XrefKind::Read, ""); let index = st.finish(BTreeMap::new(), false); @@ -2353,6 +2461,7 @@ mod tests { seeds: vec![0x3000, 0x1000, 0x3000, 0x2000], }), switches: Vec::new(), + refused: HashSet::new(), }; let index = st.finish(BTreeMap::new(), true); assert_eq!(index.function_containing(0x1001), Some(0x1000)); diff --git a/decompiler/crates/kuna-analysis/tests/fixtures/README.md b/decompiler/crates/kuna-analysis/tests/fixtures/README.md index 49308129f9..ccd079736a 100644 --- a/decompiler/crates/kuna-analysis/tests/fixtures/README.md +++ b/decompiler/crates/kuna-analysis/tests/fixtures/README.md @@ -143,6 +143,9 @@ real ELF parser. | `switchtable_pic_x86_64` (ELF, 9 KB, source `switchtable_pic_x86_64.s`, same `gcc -nostdlib -no-pie -Wl,-Ttext=0x100000 -e dispatch` recipe) / `pe_switchdelta_x86_64.exe` (PE32+, 3 KB, generator `pe_switchdelta_x86_64.py`, regenerate with `python3 pe_switchdelta_x86_64.py`) | the **delta-encoded** jump table (GH-456, same module and e2e as the pair above; CLI probe `tests/cli/xrefs-callees-not-complete.json`). Neither half of the shipped rule holds on these: the table base is materialized by an instruction of its own — so the dispatch names no address at all — and the entries are signed 32-bit displacements rather than pointers. The ELF is the gcc form (`lea jt(%rip),%rdx; movslq (%rdx,%rax,4),%rax; add %rdx,%rax; jmp *%rax`, base == table) with four cases each `LEA`ing a distinct literal; the PE is the MSVC form (`LEA RDX,[__ImageBase]; MOV ECX,[RDX + RBX*0x4 + 0x2000]; ADD RCX,RDX; JMP RCX`, base != table) whose four case bodies each hold a direct `CALL` nothing else in the image reaches, which is the reporter's missing-callee symptom. `.pdata` gives the PE's functions RUNTIME_FUNCTIONs so the inventory finds them without symbols. VMAs pinned in the e2e: `dispatch`@`0x100000` / `0x140001040`, the branch at `0x100015` / `0x14000105d`, the table at `0x101000` / `0x140002000`. No Windows toolchain on this host, hence the byte-by-byte PE generator | | `switchtable_gcc_x86_64` (ELF, 10 KB, source `switchtable_gcc_x86_64.s`, same recipe with `-e o0_dispatch`) | the jump-table dispatches gcc emits that the reference walk could not read: the `-O0` chain (base in the unscaled index slot, `cltq` of `eax`, range check ten instructions up), `-O2` with the case bodies' pushes scheduled between the `ja` and the jump, a flag-neutral store between the compare and its `ja` with a `sub` in the chain, a base hoisted into `%r13` across a call, a `.cold` fragment that reaches its parent's dispatch before the parent's prologue, a `jb` check that bounds the selector from below ahead of an adjacent table, and an `and`-masked index under a lower-bound-only check. Every case body calls or tail-jumps to a function nothing else reaches. `kuna-console/tests/verify_switchtable.rs` pins each dispatch's exact case list and each callee's caller; `kuna-cli/tests/xrefs_switch_case_calls.rs` builds the C repro with gcc and Clang | | `picpool_arm_le32` | 1,297-byte A32 ARM ELF assembled byte by byte by `picpool_arm_le32.py` (no ARM cross toolchain on this host), mapped entirely under 0x1000 | **PIC pool composition** in the on-demand xref walk (`listing/kuna_picpool.rs`; e2e `kuna-console/tests/verify_picpool.rs`, CLI probe `tests/cli/arm-pic-literal-pool.json`). The reduction of crackmes.one/68d40081224c0ec5dcedc2d2, whose `main` forms every string address as `ldr rX,[pool] ; add rX,pc,rX` — the pool word is a signed displacement, not a pointer, so following it as one declines and the literals were referenced by nothing. `uses_prompt`@`0x420` is the adjacent pair and `scheduled`@`0x430` the same pair with two instructions in between; `no_pc`@`0x448` is the control, `add r0,r0,#4` on a word whose sum IS a mapped literal. The low mapping is load-bearing: it holds the `checkOperands` 4096 floor off the composed address, as the witness does | +| `gotxref_arm_le32` | 71 KB static A32 ARM ELF built from `gotxref_arm_le32.c` (GCC 14.2 `arm-linux-musleabihf-gcc -O2 -marm -fPIC -no-pie`, command in the source; SHA-256 `7fd54d44cf3d1e1464abd08a9d593f59d2ac94225a50dc2f2eff311b0aaecfb2`), a fixed-address executable mapped at 0x10000 | **GOT bases and mapped-looking displacements** in `picpool` (`listing/kuna_picpool.rs`; e2e `kuna-console/tests/verify_gotxref.rs`). `arm_timers`@`0x100cc` makes the GOT base in r6 around a call, keeps it across a call and a branch join, and reads the slots of `on_expire` and `on_rearm` through it (`ldr r3,[r6,r3]`), which the linker filled with no relocation; `_start` reads `table`'s slot the same way. `table` sits between the code and the strings so every string displacement is itself a mapped address, which must be composed with the PC and never filed as a pointer | +| `gotxref_thumb_le32` | 73 KB static Thumb-2 ARM ELF built from `gotxref_thumb_le32.c` (GCC 14.2 `arm-linux-musleabihf-gcc -O2 -mthumb -march=armv7-a+fp -fPIC -no-pie`, command in the source; SHA-256 `1c40aee5537ecf6b073a3278ca0bf5ba27eba51afd4a7188b95b3fbd5bf56047`), a fixed-address executable mapped at 0x10000 | The Thumb counterpart of `gotxref_arm_le32` for `picpool` (e2e `kuna-console/tests/verify_gotxref_thumb.rs`). Thumb's literal loads (16-bit and `ldr.w`) lift as copies out of the image, not loads. `arm_timers`@`0x100f8` makes the GOT base in r6 and reads the slots of `on_expire` and `on_rearm` across a call and a join; `pick_timer`@`0x10164` dispatches through a `tbb` table whose cases each read a callback slot through the dispatch's base; `far_timer`@`0x10218` makes its base and a string with `ldr.w`; the slot words carry the Thumb bit | +| `picodd_thumb_le32` | 67 KB static Thumb-2 ARM ELF built from `picodd_thumb_le32.c` and `picodd_thumb_le32.ld` (GCC 10.3.1 `arm-none-eabi-gcc -Os -mcpu=cortex-m4 -mthumb -fPIC -nostdlib`, command in the source; SHA-256 `57b53ac8260b2286f2f5de5d4986f97a35c59c42bbd35a6ed647054af8048fb4`), `.rodata` linked into `.text` at 0x10000 | **The Thumb bit of a PC-composed address** in `picpool` (e2e `kuna-console/tests/verify_gotxref_thumb.rs`). The strings `"ab"`@`0x100a9` and `"ij"`@`0x100af` sit at odd addresses inside the executable section under a `$d` mapping symbol and must be filed at those addresses, not one byte low; `local_cb`'s PC-composed pointer and `f3`'s GOT slot word carry bit 0 and must name the even entries | | `retpushedhalf_x86_64` | 4.8 KB static ELF assembled from `retpushedhalf_x86_64.s` (`as && ld`), holding `xordec` -- the WeeperVM four-argument decryptor reduced to its prologue/epilogue, with an alignment `push %r8` popped into `RDX` -- and the control `regmove`, whose `RDX = R8` is an ordinary `mov` | the push-only placement rejection (`retpushedhalf`, default-on, DIV-156; `--option retpushedhalf off` restores the defect, where `xordec` grows a fifth argument and a 128-bit return). `tests/stages/kuna-retpushedhalf.xml` is the two-pass stage test; `tests/cli/alignment-push-pop-invents.json` the promoted RE-friction probe | | `raw_compat_int80_x86_64` | deterministic 17-byte headerless x86-64 image generated by `raw_compat_int80_x86_64.py`: `ENDBR64; MOV EAX,1; MOV EBX,0; INT 0x80; RET`, mapped at `0x1000` by the CLI probe | Linux's i386 compatibility entry from long mode (`linuxsyscall`): syscall 1 must render as standalone `sys_exit(0);`, never native x86-64 `write` or a generic `syscall()`. `tests/cli/linux-compatibility-int-0x80.json` is the promoted probe; `tests/stages/kuna-linuxsyscall-x64-compat.xml` adds a native `SYSCALL` control | | `raw_endptrbound_x86_64` | deterministic 271-byte headerless Win64 image generated by `raw_endptrbound_x86_64.py`, mapped at `0x140001000` by the CLI probes: `walk`@`0x140001000` walks an 8-byte stack buffer bytewise to one past its end, and that end address is also the object handed to `push_back`@`0x140001100`; `midpoint`@`0x140001080` counts its loop in `EBX` and compares the pointer against the same address only to skip a call inside the loop | the pointer-walk end bound (`endptrbound`): `walk`'s bound must render on its own buffer (`&v1[8]`), never as the neighbour, while `midpoint`'s comparison, which controls no loop exit, must not invent an 8-byte array. `tests/cli/endptrbound-walk-bound.json` and `tests/cli/endptrbound-midpoint-compare.json` are the probes; `tests/stages/kuna-endptrbound.xml` is the two-pass stage test | diff --git a/decompiler/crates/kuna-analysis/tests/fixtures/gotxref_arm_le32 b/decompiler/crates/kuna-analysis/tests/fixtures/gotxref_arm_le32 new file mode 100755 index 0000000000000000000000000000000000000000..6a29367d1bad21d15fc4b367af9a32589c707071 GIT binary patch literal 71396 zcmeIyKWrOS9KiA4voC3aD&a&)8v?4FDnlz#*Q7xprlcVW5-FvS4oH=*PJK-*#dc(0 zrBtd!$1G7Ks1Pi~1B1oPz{19W1VeWi@&{8&vmlTdkTAcW&zD|3F{AI3Uf%uQ@Aux_ zef%yT&Mz%E=Ok_^IVTo{IybH3y-|%k>6@0Uq%}7v1D$+SZ|Ds zGSk?GukK64hBG(W5Z|>~rJ;saO{SS5iMt-;DWgGkH{3nkt zN#ih`Y3zIX?Ut>1GZW=pW@qn&<^~>aA8gEg^Xq}$_vOU8x1WEy?nhnsi{^jMIQcW{ zU(Xq+wBn@Cw1LqcDI$z@9cSd7g{dUI80?fesofwH6?oQ>~`yW-QyqKV@=na ziJ8(6KmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R;a40>S1+sTO2!>z|^%Z7nF&%H6P3TtCj} zY^k!E4NK*qRzI$*1e@WrMon?jbWT3iadB?$jqJI_cdnn$zA~Mg%H?G?r`3g^uu>8? zsh54PF3(H9H{b@{bE#o(#ErVs=`r_$8&AFD&ZM5*`Z#)^ds|C)N4sAXM=OpdqA{%% z$(9x`8EUq#yF)bX$?xqEy^Qx4&7Rh{ITSxhsuO>4yvm8y+dD@ss{bkxTmI`rJl>5n zI;M2q_Ww4KxBBlAvE^@c-_M@M@?j#je4{6hJ?sw2X76`aCt@F`0%*S>{hoMB zV>?Un{vY+^KkJD<*Vz2x=abcHxLFHs`+9)=_2AM}?v_jzWU?sLif;#8{KciKmuHv! zs|yS7&oBGSvzM3VeX;WHVfX!&dfl&wg<9zQvR+;F%k@=T-uYG}EUefrouZ&p)RM#k zy^iLzJo?nc4~m7bphKl7^&srrrFusla`n4q-Au=@7L9B1RcPN?u2$`=%;_sh?pC#2 z4k~&v``(pYAq;D!l{;ZzpXPsVFKVTo+^WuXD8Rj(g8= zmY7*Yuk4>mQZEYfUEqX-GG#=2Q3z6?5g}+vImjZ>n1y=!{m#8-K8#-Uw)h@+_B_Ay z`<>@J=X1|__CoJK!dh#>BVyW&Mz-9VF6ry;l_+XTyG+cKNUqXU1o=XJP-#AIut6); zlQnX!bhC19c1 zn=_ts#y`#+UZ}a?jK2-WI`2X|5l!5)xo*pW>8YOT#6G##tbg>=+TgA;eogzCo+9{ZsHMt{`;9qsn~ks@I+YijeqIN0lD|d`>lCd`YL&j zhqi^P29J+HMmz*aIb@YY5T+9U7U;lUfsB=eM8C0ex0W=p?911>tm7WiPOvb zA4n`&Si=W?NpQBlzpkZF>(3wYOM`Q@{cVYG9dGy*!MW<-NuKtr`pXyUU-YYkoQa*i z(kW})B5T!k)=6uY7L#_i-WpwVhphSH&2v#{JuiuoKi}o=?KyVc?5APCJHR{eUw8+~ zWe=^4T@Lo~64|>uWbeLp>Y(g#hgMtDDEnMRzip{_?D?LW6YuSa$?kjomTA#`^=ATu>FlFP()%Ad48 zZ7!3_jTOVx=-qLCJez&asmpuWiSVkOsa!ZTBk9qFD!F=Od^qNfjAe59g}KKwQ=Tys zBSX(hQtp?}c%F`o(Rot4Q+sqB?s3#u(_U`Pap})XJ1y-iX-A~>ZQ8UU*4DS>skO0( zy5gPj4%6K!*QPS5!4YHIWLUcX!ERHwq`bSrRz{XYmqzPsy=^FIv`uz-q}i^Ftg`OS zobe%i!aYh%)D0zDTAZP9I%=-@%VZeNZzS6!aeG5yK4T(5 z{QkmK)@c2QPOSAGIkD!aomlgqII-qWIkDzvomlf4Q2>BO3EjNF}1^Eb-w#+o<9 z-!t@YT@;7Ew?2V;=kiMx|dW{p-?SExsQ$lJ8v(P0w zJa4=7dcSw2*Y7HIX0La&8+K;51Cfaw5CmVJgM7`N%jqF-Pk>l+?q&i z>)oE*-o0_4H)&)SN*1@EWO6W{Pv*T;&PyiEXm&U`mLJy3gIzD!lM1EAOAX3|k_q-e zBYRYFB&TGXGKHBk<7t^kP8D`X-N1_1J9i55vRleEg?gQAcgh>urJ2J1Ym%9C%1cQ% zo(^uN3(ii;J{iyN8I#3G@8t^pu5je(MjFp$gDd0WW8y>Ev9Zj!47#6uJf8Br+{oah zm(jiV-ySQZP`{EgX}UihlKveHibuG!wMeXW`upm8*R#)3#ok%rz05ls|W*Ze18iuQhsTuWJjBu10I# qmkVW?rd68WvqoB_G+kSGoLgbdd>BX{&Q~jAx9&`{{O|dPy1xPDhD6=~ literal 0 HcmV?d00001 diff --git a/decompiler/crates/kuna-analysis/tests/fixtures/gotxref_thumb_le32.c b/decompiler/crates/kuna-analysis/tests/fixtures/gotxref_thumb_le32.c new file mode 100644 index 0000000000..b1f2ab814f --- /dev/null +++ b/decompiler/crates/kuna-analysis/tests/fixtures/gotxref_thumb_le32.c @@ -0,0 +1,94 @@ +/* Fixture source for `gotxref_thumb_le32`: Thumb-2 code compiled -fPIC and + linked as a fixed-address (non-PIE) executable mapped at 0x10000, the + Thumb counterpart of `gotxref_arm_le32`. + + Thumb lifts its literal loads (`ldr rX,[pc,#..]`, 16-bit, and `ldr.w`) + as copies out of the image rather than as loads. `arm_timers` makes the + GOT base once in r6 (`ldr r6,[pc,#..]; add r6,pc`), keeps it across a call + and a branch join, and reads the slots of `on_expire` and `on_rearm` + through it (`ldr r3,[pc,#..]; ldr r3,[r6,r3]`). `pick_timer` dispatches + through a `tbb` table, and each case reads its callback's slot through + the base the dispatch holds. `far_timer` is long enough that its literal + loads are 32-bit `ldr.w`. + + `table` sits between the code and the strings so that every near string + displacement is itself a mapped address. + + Built with GCC 14.2 for armhf musl (homebrew `musl-cross` 0.9.11); the + SHA-256 below is that toolchain's output: + + arm-linux-musleabihf-gcc -O2 -mthumb -march=armv7-a+fp -fPIC -no-pie \ + -nostdlib -static -ffreestanding -fno-asynchronous-unwind-tables \ + -Wl,--build-id=none -Wl,--sort-section=name \ + -o gotxref_thumb_le32 gotxref_thumb_le32.c + + SHA-256 1c40aee5537ecf6b073a3278ca0bf5ba27eba51afd4a7188b95b3fbd5bf56047. */ + +const unsigned char table[0x11000] = { 1 }; + +void __attribute__((noinline)) log_msg(const char *s) +{ + __asm__ volatile("" : : "r"(s) : "memory"); +} + +void __attribute__((noinline)) set_timeout(int ms, void (*cb)(void *), void *arg) +{ + __asm__ volatile("" : : "r"(ms), "r"(cb), "r"(arg) : "memory"); +} + +void __attribute__((noinline)) set_alarm(void (*cb)(void *), int ms) +{ + __asm__ volatile("" : : "r"(cb), "r"(ms) : "memory"); +} + +void __attribute__((noinline)) set_watch(void *arg, void (*cb)(void *)) +{ + __asm__ volatile("" : : "r"(arg), "r"(cb) : "memory"); +} + +void on_expire(void *arg) { log_msg("expire fired"); } +void on_rearm(void *arg) { log_msg("rearm fired"); } +void on_alarm(void *arg) { log_msg("alarm fired"); } +void on_snooze(void *arg) { log_msg("snooze fired"); } +void on_stop(void *arg) { log_msg("stop fired"); } +void on_far(void *arg) { log_msg("far fired"); } +static void __attribute__((noinline)) on_idle(void *arg) { log_msg("idle fired"); } + +void arm_timers(void *arg, int extend) +{ + log_msg("arming timers"); + if (extend) + set_timeout(300, on_expire, arg); + log_msg("rearm next"); + set_timeout(51, on_rearm, arg); + set_timeout(7, on_idle, arg); +} + +int pick_timer(void *arg, int which) +{ + log_msg("picking"); + switch (which) { + case 0: set_timeout(10, on_alarm, arg); return 3; + case 1: set_alarm(on_snooze, 20); return 5; + case 2: set_watch(arg, on_stop); return 8; + case 3: log_msg("resetting"); set_alarm(on_expire, 7); return 13; + case 5: set_watch(on_rearm, on_rearm); return 21; + default: return 0; + } +} + +void far_timer(void *arg) +{ + log_msg("far away"); + __asm__ volatile(".rept 300\n\tnop.w\n\t.endr" ::: "memory"); + set_timeout(9, on_far, arg); +} + +void _start(void) +{ + arm_timers((void *)table, table[0]); + pick_timer((void *)table, table[1]); + far_timer((void *)table); + for (;;) + ; +} diff --git a/decompiler/crates/kuna-analysis/tests/fixtures/picodd_thumb_le32 b/decompiler/crates/kuna-analysis/tests/fixtures/picodd_thumb_le32 new file mode 100755 index 0000000000000000000000000000000000000000..751d2941f22d996a3e844f87032df9f30600d352 GIT binary patch literal 67004 zcmeIyy>DAp7{~GF*iO?(MbMIn&=%xEgj|6w+c6X}Ad=b%w2sP4BqCnq`sF${ICT`? zh?XKTuu%R4SQr?Y5X?qmjRX=f(1`_DSYXM9d7f)u_he&5`kv_AbAI=Ed~JX3bv~?A z*PU|`kA!5z(j7gPblpm6O2|w}(t0K%b8<4OOLVkOCx?3cN$0H2@%d=O0RaRMKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0 z009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{ z1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY** z5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0;reGNWiMO3>Se3H>}9K8 z_p;S{UbgxNUbgzjUbgz-G#~dhza_1IJE=SU#@7GEwBN~!W~YDix}MS1o=JPY?9QCl zJG7?jE1J(w>Q!BLz5Kdndlt>pUjK%!&w2T6&3C;1mS+3B@%6i!D_(z3bJokU*KPJ& zty%CgDA+?R_wYAl1?b`bKYn83q*6LcdQjSd@v&$7j# PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../..").canonicalize().unwrap() +} + +fn index() -> XrefIndex { + let bin = repo_root().join("decompiler/crates/kuna-analysis/tests/fixtures/gotxref_arm_le32"); + let specs = repo_root().join("specs"); + let spec_roots = vec![specs.to_str().unwrap().to_string()]; + let mut prog = bootstrap_from_object(bin.to_str().unwrap(), "", &spec_roots) + .expect("bootstrap fixture with built processor specs"); + prog.commit_pending_analysis().expect("analysis commit succeeds"); + + let bytes = std::fs::read(&bin).expect("fixture readable"); + let file = object::File::parse(&*bytes).expect("fixture parses"); + let seeds: Vec = + prog.function_entries_canonical().iter().map(|e| e.addr.get_offset()).collect(); + xrefs::build(&file, prog.arch(), prog.arch().translate(), &seeds) +} + +fn to(idx: &XrefIndex, target: u64) -> Vec<(u64, XrefKind)> { + idx.refs_to(target).iter().map(|r| (r.from, r.kind)).collect() +} + +/// The callbacks are reached only through GOT slots the linker filled with no +/// relocation, so before this nothing referenced them at all. +#[test] +fn a_callback_read_through_the_got_is_referenced_from_the_slot_load() { + let idx = index(); + assert_eq!(to(&idx, ON_EXPIRE), vec![(READS_EXPIRE, XrefKind::Data)]); + assert_eq!(to(&idx, ON_REARM), vec![(READS_REARM, XrefKind::Data)]); + assert_eq!(to(&idx, EXPIRE_SLOT), vec![(READS_EXPIRE, XrefKind::Read)]); + assert_eq!(to(&idx, REARM_SLOT), vec![(READS_REARM, XrefKind::Read)]); + assert!(to(&idx, GOT).contains(&(MAKES_BASE, XrefKind::Data)), "{:?}", to(&idx, GOT)); + assert_eq!(idx.function_containing(READS_REARM), Some(ARM_TIMERS)); +} + +/// A displacement that happens to be a mapped address is still composed with +/// the PC. +#[test] +fn a_string_formed_from_a_mapped_looking_displacement_is_referenced() { + let idx = index(); + assert_eq!(to(&idx, ARMING), vec![(FORMS_ARMING, XrefKind::Data)]); + assert_eq!(to(&idx, REARM_NEXT), vec![(FORMS_REARM_NEXT, XrefKind::Data)]); +} + +/// A data object read through the GOT is referenced the same way: `_start` +/// reads `table`'s slot to pass its address on. +#[test] +fn a_data_object_read_through_the_got_is_referenced() { + let idx = index(); + assert_eq!(to(&idx, TABLE), vec![(READS_TABLE, XrefKind::Data)]); +} + +/// The raw displacement words are not pointers, so nothing references them. +#[test] +fn a_composed_displacement_is_not_filed_as_a_pointer() { + let idx = index(); + for (load, word) in DISPLACEMENTS { + assert!(idx.refs_to(word).is_empty(), "{word:#x}: {:?}", idx.refs_to(word)); + assert!( + idx.refs_from_instruction(load).iter().all(|r| r.kind != XrefKind::Data), + "{load:#x}: {:?}", + idx.refs_from_instruction(load) + ); + } +} diff --git a/decompiler/crates/kuna-console/tests/verify_gotxref_thumb.rs b/decompiler/crates/kuna-console/tests/verify_gotxref_thumb.rs new file mode 100644 index 0000000000..0dacdf48be --- /dev/null +++ b/decompiler/crates/kuna-console/tests/verify_gotxref_thumb.rs @@ -0,0 +1,192 @@ +//! End-to-end gate for `picpool` on Thumb-2 `-fPIC` code linked as a +//! fixed-address executable mapped low, the Thumb counterpart of +//! `verify_gotxref`. +//! +//! Fixture: `gotxref_thumb_le32` (built from `gotxref_thumb_le32.c` with GCC). +//! Thumb lifts its literal loads, 16-bit and `ldr.w`, as copies out of the +//! image, and reads the PC as `inst_start + 4`. `arm_timers` makes the GOT +//! base in r6 and reads two callback slots through it across a call and a +//! join; `pick_timer` dispatches through a `tbb` table whose cases each read +//! a callback slot through the dispatch's base; `far_timer` makes its base +//! with `ldr.w`; the GOT words carry the Thumb bit. +//! +//! Fixture: `picodd_thumb_le32` (from `picodd_thumb_le32.c` and `.ld`), Thumb-2 +//! `-fPIC` with `.rodata` linked into `.text`: a string at an odd address in +//! the executable section is a datum, not a Thumb pointer. + +use std::path::PathBuf; + +use kuna_analysis::listing::xrefs::{self, XrefIndex, XrefKind}; +use kuna_console::engine::bootstrap_from_object; + +const ARM_TIMERS: u64 = 0x100f8; +const ON_EXPIRE: u64 = 0x10098; +const ON_REARM: u64 = 0x100a4; +const ON_FAR: u64 = 0x100d4; +/// `"arming timers"`, `"rearm next"` and `"far away"`. +const ARMING: u64 = 0x21788; +const REARM_NEXT: u64 = 0x21798; +const FAR_AWAY: u64 = 0x217b8; +/// `_GLOBAL_OFFSET_TABLE_`, the start of `.got`, and the slots read. +const GOT: u64 = 0x227c4; +const REARM_SLOT: u64 = 0x227d0; +const FAR_SLOT: u64 = 0x227dc; +const EXPIRE_SLOT: u64 = 0x227e8; +/// `table`, and the `ldr r4,[r3,r2]` in `_start` that reads its slot. +const TABLE: u64 = 0x1072c; +const READS_TABLE: u64 = 0x1070a; + +/// `add r6,pc`, which makes `arm_timers`' base. +const MAKES_BASE: u64 = 0x10106; +/// `ldr r3,[r6,r3]` on the conditional path, before the join. +const READS_EXPIRE: u64 = 0x10140; +/// `ldr r3,[r6,r3]` after the join at 0x1010e. +const READS_REARM: u64 = 0x1011c; +/// `ldr r3,[r4,r3]` in `far_timer`, whose base came from `ldr.w`. +const READS_FAR: u64 = 0x106e4; +const FORMS_ARMING: u64 = 0x10102; +const FORMS_REARM_NEXT: u64 = 0x10110; +const FORMS_FAR_AWAY: u64 = 0x10226; +/// The literal loads whose words are displacements that look like addresses, +/// the last two 32-bit `ldr.w`. +const DISPLACEMENTS: [(u64, u64); 6] = [ + (0x100fc, 0x11682), + (0x100fe, 0x126ba), + (0x1010e, 0x11684), + (0x10700, 0x120ba), + (0x1021c, 0x1158e), + (0x10220, 0x12594), +]; + +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../..").canonicalize().unwrap() +} + +fn index() -> XrefIndex { + index_of("gotxref_thumb_le32") +} + +fn index_of(fixture: &str) -> XrefIndex { + let bin = repo_root().join("decompiler/crates/kuna-analysis/tests/fixtures").join(fixture); + let specs = repo_root().join("specs"); + let spec_roots = vec![specs.to_str().unwrap().to_string()]; + let mut prog = bootstrap_from_object(bin.to_str().unwrap(), "", &spec_roots) + .expect("bootstrap fixture with built processor specs"); + prog.commit_pending_analysis().expect("analysis commit succeeds"); + + let bytes = std::fs::read(&bin).expect("fixture readable"); + let file = object::File::parse(&*bytes).expect("fixture parses"); + let seeds: Vec = + prog.function_entries_canonical().iter().map(|e| e.addr.get_offset()).collect(); + xrefs::build(&file, prog.arch(), prog.arch().translate(), &seeds) +} + +fn to(idx: &XrefIndex, target: u64) -> Vec<(u64, XrefKind)> { + idx.refs_to(target).iter().map(|r| (r.from, r.kind)).collect() +} + +/// The callbacks are reached only through GOT slots, which hold each entry +/// with the Thumb bit set. +#[test] +fn a_thumb_callback_read_through_the_got_is_referenced_from_the_slot_load() { + let idx = index(); + assert!(to(&idx, ON_EXPIRE).contains(&(READS_EXPIRE, XrefKind::Data)), "{:?}", to(&idx, ON_EXPIRE)); + assert!(to(&idx, ON_REARM).contains(&(READS_REARM, XrefKind::Data)), "{:?}", to(&idx, ON_REARM)); + assert_eq!(to(&idx, EXPIRE_SLOT).first(), Some(&(READS_EXPIRE, XrefKind::Read))); + assert_eq!(to(&idx, REARM_SLOT).first(), Some(&(READS_REARM, XrefKind::Read))); + assert!(to(&idx, GOT).contains(&(MAKES_BASE, XrefKind::Data)), "{:?}", to(&idx, GOT)); + assert_eq!(idx.function_containing(READS_REARM), Some(ARM_TIMERS)); +} + +/// `ldr.w` is a literal load too: the base it makes reaches a slot load 300 +/// instructions later. +#[test] +fn a_thumb2_wide_literal_load_makes_a_base_and_a_string() { + let idx = index(); + assert_eq!(to(&idx, ON_FAR), vec![(READS_FAR, XrefKind::Data)]); + assert_eq!(to(&idx, FAR_SLOT), vec![(READS_FAR, XrefKind::Read)]); + assert_eq!(to(&idx, FAR_AWAY), vec![(FORMS_FAR_AWAY, XrefKind::Data)]); +} + +/// A displacement that happens to be a mapped address is still composed with +/// Thumb's PC. +#[test] +fn a_thumb_string_formed_from_a_mapped_looking_displacement_is_referenced() { + let idx = index(); + assert_eq!(to(&idx, ARMING), vec![(FORMS_ARMING, XrefKind::Data)]); + assert_eq!(to(&idx, REARM_NEXT), vec![(FORMS_REARM_NEXT, XrefKind::Data)]); +} + +/// `_start` reads `table`'s slot through a base held in a scratch register. +#[test] +fn a_thumb_data_object_read_through_the_got_is_referenced() { + let idx = index(); + assert_eq!(to(&idx, TABLE), vec![(READS_TABLE, XrefKind::Data)]); +} + +/// `pick_timer`@0x10164: each case of the `tbb` switch reads its callback's +/// slot through r6, which only the dispatch hands it. +const PICK_TIMER: u64 = 0x10164; +const CASE_SLOT_LOADS: [(u64, u64, u64); 5] = [ + (0x10188, 0x227d0, ON_REARM), + (0x101a0, 0x227d8, 0x100b0), + (0x101b4, 0x227e0, 0x100bc), + (0x101c8, 0x227e4, 0x100c8), + (0x101e4, EXPIRE_SLOT, ON_EXPIRE), +]; +/// `"resetting"`, formed in case 3. +const RESETTING: u64 = 0x217ac; +const FORMS_RESETTING: u64 = 0x101da; + +#[test] +fn a_switch_case_reads_its_slot_through_the_dispatch_base() { + let idx = index(); + for (load, slot, callback) in CASE_SLOT_LOADS { + assert!(to(&idx, slot).contains(&(load, XrefKind::Read)), "{slot:#x}: {:?}", to(&idx, slot)); + assert!(to(&idx, callback).contains(&(load, XrefKind::Data)), "{callback:#x}: {:?}", to(&idx, callback)); + assert_eq!(idx.function_containing(load), Some(PICK_TIMER)); + } + assert_eq!(to(&idx, RESETTING), vec![(FORMS_RESETTING, XrefKind::Data)]); +} + +/// The raw displacement words are not pointers, so nothing references them. +#[test] +fn a_thumb_composed_displacement_is_not_filed_as_a_pointer() { + let idx = index(); + for (load, word) in DISPLACEMENTS { + assert!(idx.refs_to(word).is_empty(), "{word:#x}: {:?}", idx.refs_to(word)); + assert!( + idx.refs_from_instruction(load).iter().all(|r| r.kind != XrefKind::Data), + "{load:#x}: {:?}", + idx.refs_from_instruction(load) + ); + } +} + +/// `picodd_thumb_le32`: `"ab"` and `"ij"` sit at odd addresses inside `.text`, +/// under a `$d` mapping symbol, so each is filed at its own address and never +/// one byte low; the Thumb pointers to `local_cb` (formed PC-relatively) and +/// `f3` (read from its GOT slot) still name the even entries. +const AB: u64 = 0x100a9; +const IJ: u64 = 0x100af; +const F1_FORMS_AB: u64 = 0x1002e; +const LOCAL_CB_FORMS_AB: u64 = 0x10022; +const F4_FORMS_IJ: u64 = 0x10046; +const LOCAL_CB: u64 = 0x10020; +const SET_LOCAL_FORMS_CB: u64 = 0x1005a; +const F3: u64 = 0x10014; +const SET_GLOBAL_READS_F3: u64 = 0x10076; + +#[test] +fn an_odd_string_in_text_keeps_its_address_and_a_thumb_pointer_loses_its_bit() { + let idx = index_of("picodd_thumb_le32"); + assert_eq!( + to(&idx, AB), + vec![(LOCAL_CB_FORMS_AB, XrefKind::Data), (F1_FORMS_AB, XrefKind::Data)] + ); + assert_eq!(to(&idx, IJ), vec![(F4_FORMS_IJ, XrefKind::Data)]); + assert!(to(&idx, AB - 1).is_empty(), "{:?}", to(&idx, AB - 1)); + assert!(to(&idx, IJ - 1).is_empty(), "{:?}", to(&idx, IJ - 1)); + assert!(to(&idx, LOCAL_CB).contains(&(SET_LOCAL_FORMS_CB, XrefKind::Data)), "{:?}", to(&idx, LOCAL_CB)); + assert!(to(&idx, F3).contains(&(SET_GLOBAL_READS_F3, XrefKind::Data)), "{:?}", to(&idx, F3)); +} diff --git a/docs/re-needs/arm-pic-got-xrefs.md b/docs/re-needs/arm-pic-got-xrefs.md new file mode 100644 index 0000000000..2c23a9a20e --- /dev/null +++ b/docs/re-needs/arm-pic-got-xrefs.md @@ -0,0 +1,135 @@ +--- +need_id: arm-pic-got-xrefs +title: ARM -fPIC callback read through the GOT has no reference +track: tooling +status: closed +severity: major +probe_id: p-fd504c2f4ae3 +acceptance_id: a-336a275d03d0 +hypothesis_status: upheld +credibility: 0.9 +instances: 2 +challenges: [] +rounds: [] +first_seen_round: null +attempts: 1 +covered_by_option: null +touches: [decompiler/crates/kuna-analysis/src/listing/kuna_picpool.rs, decompiler/crates/kuna-analysis/src/listing/xrefs.rs] +scope: small +regression_of: null +pr: 1264 +closed_in_round: null +closing_pr: "1264" +reject_reason: null +--- + +## Symptom + +Find who registers the timer callback `on_expire`. + +> **ARM -fPIC callback read through the GOT has no reference** (major, synthetic `gotxref_arm_le32`) +> `kuna xrefs --to on_expire` reports 0 references, although `arm_timers` loads it from its GOT slot and passes it to `set_timeout`. The string literals of the same function also have no xref, and the raw pool words are filed as data references into `.text`. + +Seen first on a production ARM executable (GCC, `-fPIC`, linked non-PIE at 0x10000); `gotxref_arm_le32` reproduces it from a small C source. + +## Reproduction + +```json +{ + "schema": "re-probe/1", + "kind": "cli", + "timeout_s": 60, + "cmd": [ + "{{KUNA}}", + "xrefs", + "{{BIN}}", + "--to", + "on_expire", + "--json" + ], + "expect": { + "exit_code": { + "eq": 0 + }, + "json": [ + { + "path": "count", + "op": "eq", + "value": 0 + } + ] + } +} +``` + +## Acceptance + +```json +{ + "schema": "re-probe/1", + "kind": "cli", + "timeout_s": 60, + "cmd": [ + "{{KUNA}}", + "xrefs", + "{{BIN}}", + "--to", + "on_expire", + "--json" + ], + "expect": { + "exit_code": { + "eq": 0 + }, + "stdout_is_json": true, + "json": [ + { + "path": "count", + "op": "eq", + "value": 1 + }, + { + "path": "xrefs[0].from_address_hex", + "op": "eq", + "value": "0x10144" + }, + { + "path": "xrefs[0].kind", + "op": "eq", + "value": "data" + } + ] + }, + "target": { + "binary_rel": "decompiler/crates/kuna-analysis/tests/fixtures/gotxref_arm_le32", + "binary_sha256": "7fd54d44cf3d1e1464abd08a9d593f59d2ac94225a50dc2f2eff311b0aaecfb2", + "binary_size": 71396, + "binary_source": "in-repo" + } +} +``` + +## Hypothesis + +**Advisory — the builder is not bound by this.** + +- `picpool` declines a pool word that is itself a mapped address, so in an image mapped low a PC displacement that lands in a section by chance forms nothing, and `poolref` files it as a pointer. +- Nothing models the ARM GOT base (`ldr r6,[pc]; add r6,pc,r6`, kept in a callee-saved register across calls and joins), so `ldr r3,[r6,r3]` is never followed to the slot or to the address the linker wrote there. + +## Refutation + +_not yet refuted_ + +## Reference + +- Ghidra 12.1.4 finds both through `SymbolicPropogator` constant propagation and its ELF loader's `.got` pointer markup. + +## Instances + +- `gotxref_arm_le32` (in-repo, synthetic) +- `gotxref_thumb_le32` (in-repo, synthetic): the same in Thumb-2, where the literal loads lift as copies out of the image rather than loads, plus a `tbb` switch whose cases read slots through the dispatch's base + +## Decision log + +- filed by hand from one vendor binary and its synthetic reduction; prior art: `arm-pic-literal-pool` (#474) +- closed: acceptance `a-336a275d03d0` passes with #1264 (promoted to `tests/cli/arm-pic-got-callback-xref.json`, with `arm-pic-mapped-displacement-string` and `arm-pic-displacement-not-a-pointer`); the reproduction still passes on the unpatched tree. diff --git a/docs/spec/01-program-prep.md b/docs/spec/01-program-prep.md index 9889d88629..83ae595de1 100644 --- a/docs/spec/01-program-prep.md +++ b/docs/spec/01-program-prep.md @@ -6325,17 +6325,33 @@ carries instead is the pool word itself, from the load to whatever composes it. A pool word is admitted as a *displacement* only where it comes out of the same read-only, pointer-sized, pointer-aligned slot literal-pool following already -vouches for **and** literal-pool following declined it, so the two rules never -claim one word: a value that is already a mapped address is a pointer, not a -displacement. It is then carried forward along fall-through alone, for a bounded -run of instructions, and dropped at the first write of its register and at any -branch, call or return — the walk is breadth-first, so the values are keyed by +vouches for, whatever the word itself looks like: in an ARM executable linked at +a fixed low address (0x10000) a displacement often lands in a mapped section by +chance, and only the PC fold below decides that it is one. A32 reads the slot +with a `LOAD` from a constant address; Thumb's literal loads, the 16-bit `ldr` +and `ldr.w`, export the slot itself with the word-aligned PC already folded in, +so they lift as a `COPY` out of the image, and both forms are read. Both of +these, the mapped-looking word and the Thumb `COPY`, are admitted only in an +image that has a GOT (`.got` or `.got.plt`) or a `.dynamic` section, which is +what position-independent code links into; a static image with none of them, +which is what bare-metal firmware is, admits only an A32 `LOAD` of a word that +is not a mapped address. That keeps the walk on such firmware, whose Thumb code +is dense with literal loads of pointers, at the cost it had before Thumb was +read, and its output is unchanged: over the 98 decbench ARM firmware images, their +stripped copies, and firmware relinked at 0x0, the wider admission formed no +reference. It is then +carried forward along fall-through alone, for a bounded run of instructions, and +dropped at the first write of its register, at any branch or return, and at a +call unless its register is callee-saved (GCC schedules the load before a call +and its `add` after). A user-defined operation ends it too, except inside a +call, where it is the Thumb `bl`'s instruction-set switch and the call's own rule +decides — the walk is breadth-first, so the values are keyed by the address they reach rather than held as "the previous instruction's state", which is what makes the carry independent of the order the queue happens to visit a function in. The composition itself is a one-instruction constant fold over `COPY`/`INT_ADD`/`INT_SUB` and nothing else, and it reports a value only when **both** a pool word and a constant the instruction materialised from its own -address contributed to it. That second taint is the whole guard: `add r0,r0,#4` +address contributed to it (`inst_start + 8` on A32, `+ 4` on Thumb). That second taint is the whole guard: `add r0,r0,#4` on the same word can land on a real literal, and only the missing PC separates it from a reference. A value that stays in a temporary is not reported, for the same reason PIC base folding does not report one — the GOT idiom `ldr r1,[pool]; ldr @@ -6352,6 +6368,84 @@ vendored fixtures and a set of host binaries: no edge is added on any x86-64 or PE image (a RIP-relative load already encodes the address it forms), no attribution is lost anywhere, and the composed edges are all on ARM images. +When a pool word that is itself a mapped address is carried on, literal-pool +following's pointer edge for it is held until the function's walk ends and is +filed only if no PC fold consumed the word, so a displacement is never also +reported as an absolute pointer from its load. + +ARM `-fPIC` code also reaches data and functions through the GOT. The function +makes the GOT address once with the same pair (`ldr r4,[pool]; add r4,pc,r4`) and +keeps it in a callee-saved register; each use loads a GOT offset from the pool +and reads the slot (`ldr r2,[r4,r2]`). A composed value that lands in `.got` is +therefore tracked as a **GOT base** along every edge of the function, branches +included, and across a call when its register is callee-saved, until the register +is written. A load whose address is a GOT base plus a carried pool word files a +read of the slot and, when the slot's file word is a mapped address, a data +reference to that word from the load. `.got` is writable, so literal-pool +following does not hold it; its file bytes are what the loader starts from, and +in a non-PIE executable the linker writes a local function's or object's address +there with no relocation (an import's slot holds 0 and names nothing). Only base +plus a pool word (the GOTOFF form) is an address: base plus an immediate is +arithmetic on the base and names nothing, and a plain copy of the base files +nothing but is itself a base, so `mov r5,r4; ldr r3,[r5,r3]` still finds the +slot. A switch's case bodies are the dispatcher's own code, so each case target +receives the bases the dispatch holds. On ARM a Thumb function's slot word, and a +Thumb function's PC-composed address, carry bit 0. Whether an odd address whose +even half lies in an executable section is such a pointer or a datum at an odd +address (a string in a `.rodata` linked into `.text`) is decided once the whole +walk is done: it names the function, bit 0 cleared, when the walk decoded an +instruction or knows a function at the even address, or when the nearest ARM +mapping symbol at or below it is `$t`; a `$d` or `$a` there leaves it at its own +address. An image with no mapping symbols says nothing either way, and the +Thumb pointer, by far the commoner of the two, is assumed. + +Because the walk decodes each instruction once, breadth-first, it can decode a +join before it has seen every edge into it, so what it carries is only a +candidate. Each instruction that sees a tracked value is recorded with the +values it was handed, what it passes on and which addresses it forms, each +naming the incoming values it was computed from; every edge is recorded as it is +taken, switch cases included, whether or not the dispatch carried anything. When +the walk of the function ends, a value is kept at an instruction only if every +recorded predecessor passes on the same one, and everything computed from a +dropped value goes with it, repeated until nothing changes. Only addresses whose +inputs all survived are filed. The entry needs no rule of its own: the walk +decodes it first, so nothing is carried into it. This is stricter than Ghidra's +`SymbolicPropogator`, which follows each path with its own copy of the state and +keeps the union of what the paths find. + +A predicated instruction lifts as a branch to the next instruction when its +condition fails, ahead of its body, so it reaches the next instruction along two +edges: the one where it ran and fell through, and the one where it did not run, +which carries what it was handed less what its condition wrote. A base meets +both, so a conditional write to the base ends it after the instruction, while a +predicated return such as `popeq {r4-r6,pc}` leaves the base to the path that +falls past it. Pool words follow the instruction when it falls through, because +a predicated pair (`ldrne; addne`) runs under one condition. + +Nothing is recorded before a body's first pool load, since nothing can be in +flight; this keeps the walk's cost on bodies, and architectures, that never form +one of these pairs to one scan of each instruction's operations. Soundness does +not need the earlier edges: at the first load, everything the walk has queued and +not yet decoded was queued by an instruction that carried nothing, so nothing +holds there, which is exactly what the meet over every edge would answer. An +instruction that a recovered frame's walk refused to enter is treated the same +way, because the frame's edge into it is in no other body's record. + +Six shapes are not covered, or not exactly. A GOT base inherited from a caller +(one the function never forms itself) is not tracked: on ARM the register that holds it is +only callee-saved, not reserved, so assuming it would need every caller to +agree. A pool word does not cross a branch, so a slot load reached by a branch, as when +GCC merges the tails of switch cases into one shared `ldr r3,[r6,r3]`, files +nothing, even when every path brings the same offset. Code that a function walked +later reaches by jumping into a body already walked is not part of that body's +meet. A pool word the PC meets only inside an address (`ldr r3,[pc,r3]`, the A32 +read of a TLS initial-exec slot) composes nothing a register holds, so the slot is +not read, and a word that is itself a mapped address keeps its pointer edge from +the load. A position-independent static image with no GOT and no `.dynamic` +section (code that touches only its own strings and statics) is read as +firmware. And in an image stripped of its mapping symbols, a datum at an odd +address inside an executable section is filed one byte low. + (kuna) Every rule above widens what the walk reads out of an instruction it reached; the complementary defect is the instructions it never reaches at all. The descent's successors are `classify`'s static targets plus fall-through, and a diff --git a/tests/cli/arm-pic-displacement-not-a-pointer.json b/tests/cli/arm-pic-displacement-not-a-pointer.json new file mode 100644 index 0000000000..1b508a7a32 --- /dev/null +++ b/tests/cli/arm-pic-displacement-not-a-pointer.json @@ -0,0 +1,43 @@ +{ + "schema": "re-probe/1", + "probe_id": "a-25bbcc566b54", + "kind": "cli", + "cmd": [ + "{{KUNA}}", + "xrefs", + "{{BIN}}", + "--to", + "0x110e4", + "--json" + ], + "cwd": "{{WORK}}", + "env": { + "SLEIGHHOME": "{{SPECS}}" + }, + "stdin": null, + "timeout_s": 60, + "repeat": 1, + "target": { + "binary_rel": "decompiler/crates/kuna-analysis/tests/fixtures/gotxref_arm_le32", + "binary_sha256": "7fd54d44cf3d1e1464abd08a9d593f59d2ac94225a50dc2f2eff311b0aaecfb2", + "binary_size": 71396, + "binary_source": "in-repo", + "in_repo_path": "decompiler/crates/kuna-analysis/tests/fixtures/gotxref_arm_le32", + "selector": null, + "selector_kind": "none" + }, + "expect": { + "exit_code": { + "eq": 0 + }, + "stdout_is_json": true, + "json": [ + { + "path": "count", + "op": "eq", + "value": 0 + } + ] + }, + "notes": "Promoted acceptance of arm-pic-got-xrefs. 0x110e4 is the raw displacement word loaded at 0x100d4, not a pointer; before the fix it was filed as a data reference from the load." +} diff --git a/tests/cli/arm-pic-got-callback-xref.json b/tests/cli/arm-pic-got-callback-xref.json new file mode 100644 index 0000000000..3a8f376375 --- /dev/null +++ b/tests/cli/arm-pic-got-callback-xref.json @@ -0,0 +1,53 @@ +{ + "schema": "re-probe/1", + "probe_id": "a-336a275d03d0", + "kind": "cli", + "cmd": [ + "{{KUNA}}", + "xrefs", + "{{BIN}}", + "--to", + "on_expire", + "--json" + ], + "cwd": "{{WORK}}", + "env": { + "SLEIGHHOME": "{{SPECS}}" + }, + "stdin": null, + "timeout_s": 60, + "repeat": 1, + "target": { + "binary_rel": "decompiler/crates/kuna-analysis/tests/fixtures/gotxref_arm_le32", + "binary_sha256": "7fd54d44cf3d1e1464abd08a9d593f59d2ac94225a50dc2f2eff311b0aaecfb2", + "binary_size": 71396, + "binary_source": "in-repo", + "in_repo_path": "decompiler/crates/kuna-analysis/tests/fixtures/gotxref_arm_le32", + "selector": null, + "selector_kind": "none" + }, + "expect": { + "exit_code": { + "eq": 0 + }, + "stdout_is_json": true, + "json": [ + { + "path": "count", + "op": "eq", + "value": 1 + }, + { + "path": "xrefs[0].from_address_hex", + "op": "eq", + "value": "0x10144" + }, + { + "path": "xrefs[0].kind", + "op": "eq", + "value": "data" + } + ] + }, + "notes": "Promoted acceptance of arm-pic-got-xrefs. `arm_timers` reads `on_expire`'s GOT slot with `ldr r3,[r6,r3]` at 0x10144, through a base made by `ldr r6,[pc]; add r6,pc,r6` around a call. The linker filled the slot with no relocation, so before the fix nothing referenced the callback. The join and refusal cases are in verify_gotxref.rs and kuna_picpool.rs." +} diff --git a/tests/cli/arm-pic-mapped-displacement-string.json b/tests/cli/arm-pic-mapped-displacement-string.json new file mode 100644 index 0000000000..4dc4ef5ef5 --- /dev/null +++ b/tests/cli/arm-pic-mapped-displacement-string.json @@ -0,0 +1,53 @@ +{ + "schema": "re-probe/1", + "probe_id": "a-b42f7320990f", + "kind": "cli", + "cmd": [ + "{{KUNA}}", + "strings", + "{{BIN}}", + "--json", + "--filter", + "arming timers" + ], + "cwd": "{{WORK}}", + "env": { + "SLEIGHHOME": "{{SPECS}}" + }, + "stdin": null, + "timeout_s": 60, + "repeat": 1, + "target": { + "binary_rel": "decompiler/crates/kuna-analysis/tests/fixtures/gotxref_arm_le32", + "binary_sha256": "7fd54d44cf3d1e1464abd08a9d593f59d2ac94225a50dc2f2eff311b0aaecfb2", + "binary_size": 71396, + "binary_source": "in-repo", + "in_repo_path": "decompiler/crates/kuna-analysis/tests/fixtures/gotxref_arm_le32", + "selector": null, + "selector_kind": "none" + }, + "expect": { + "exit_code": { + "eq": 0 + }, + "stdout_is_json": true, + "json": [ + { + "path": "strings[0].address_hex", + "op": "eq", + "value": "0x211d0" + }, + { + "path": "strings[0].xrefs_count", + "op": "eq", + "value": 1 + }, + { + "path": "strings[0].functions", + "op": "len_gt", + "value": 0 + } + ] + }, + "notes": "Promoted acceptance of arm-pic-got-xrefs. The pool word of `ldr r0,[pc]; add r0,pc,r0` at 0x100d4/0x100e4 is 0x110e4, itself a mapped address because the image is linked at 0x10000; it must still be composed with the PC." +}