Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions docs/APP_PERFORMANCE.md
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,20 @@ functions rather than rescanning the project. Advisory draft reports share bound
trial bookkeeping; completed resume receipts flush immediately. Stopping a lane
flushes advisory reports and queues follow-ups for its already committed results.

Autopilot gives every machine draft its turn before running the saved-source return
repair lane. The second phase re-reads eligibility and acquires fresh claims, retains
its existing per-function search bounds, and reports separate repair progress with
the same Stop control. Direct single-function calls retain their saved-source repair
behavior. Refresh functions by ID inside the pass rather than resolving them through
project routes, which can trigger saved-file synchronization.

Try raw m2c output before generating optional variants or collecting callee evidence.
Autopilot trials use the existing lean search evaluation: full linking and exact-byte
verdicts remain mandatory; export/address/allocation diagnostics are generated by
interactive evaluation when requested. Pass workers share half-second claim cleanup
and identity reads. Selected expired or legacy speculative claims force immediate
cleanup, and the selected claim is still checked in the acquisition transaction.

### Releases queued behind a compile

A plain agent release takes the per-agent/function candidate lock before retaining the project
Expand Down
53 changes: 53 additions & 0 deletions src/decompnet/app/autopilot_claims.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
"""Coalesce pass-wide lease maintenance while checking each selected claim atomically."""
import threading
import time
from contextlib import contextmanager
from contextvars import ContextVar

CLEANUP_SECONDS=.5

_current=ContextVar('autopilot_claim_maintenance',default=None)


class Batch:
def __init__(self):
self.lock=threading.Lock()
self.identity_time=float('-inf')
self.identity_ids=None
self.expired_at=float('-inf')

def identities(self,net):
# Read the catalog before taking a separate project's write transaction.
with self.lock:
now=time.monotonic()
if self.identity_ids is None or now-self.identity_time>=CLEANUP_SECONDS:
self.identity_ids={'machine_ids':net._machine_agent_ids(),
'nonexclusive_ids':net._nonexclusive_agent_ids()}
self.identity_time=now
return self.identity_ids


def identities(net,store):
if batch:=_current.get():return batch.identities(net)
return {} if store is net.store else {'machine_ids':net._machine_agent_ids(),
'nonexclusive_ids':net._nonexclusive_agent_ids()}


def expire(net,db,fid,ids):
batch=_current.get()
if batch is None:
net._expire_claims(db,**ids);return
# Transactions on this pass's project store serialize this timestamp. Expired
# or legacy speculative claims on the selected function never wait for cadence.
claim=db.execute('SELECT agent_id,expires_at FROM claims WHERE function_id=?',(fid,)).fetchone()
if (time.monotonic()-batch.expired_at>=CLEANUP_SECONDS or claim and
(claim['expires_at']<time.time() or claim['agent_id'] in ids['nonexclusive_ids'])):
net._expire_claims(db,**ids)
batch.expired_at=time.monotonic()


@contextmanager
def scope(batch):
token=_current.set(batch)
try:yield
finally:_current.reset(token)
108 changes: 73 additions & 35 deletions src/decompnet/app/engine/autopilot.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import threading
import time
from contextlib import nullcontext
from contextvars import ContextVar
from itertools import chain

from ... import compiler, m2cfix, scoring, seeds, toolchains
Expand All @@ -21,6 +22,8 @@
from ..draft_context import candidate_context
from ..purpose_work import Executor

_draft_pipeline=ContextVar('autopilot_draft_pipeline',default=False)


class AutopilotMixin:
def start_autopilot(self, slug: str, workers: int | None = None, *, foreground=False) -> dict:
Expand Down Expand Up @@ -139,10 +142,17 @@ def check_stop():
outcome = "fail"
finally:
if acquired:background.release()
state["tried"] += 1
if outcome in ("match", "near"):
state[{"match": "matched", "near": "near"}[outcome]] += 1
counter='repair_done' if state['phase']=='repairs' else 'tried'
with queue_lock:
state[counter] = state.get(counter,0)+1
if outcome in ("match", "near"):
state[{"match": "matched", "near": "near"}[outcome]] += 1
if state['phase']=='repairs' and outcome=='match':
state['repair_matched']=state.get('repair_matched',0)+1

from ..autopilot_claims import Batch
from ..autopilot_claims import scope as claim_scope
claim_batch=Batch()
remaining = iter(ids)
queue_lock = threading.Lock()
def lane():
Expand All @@ -152,32 +162,49 @@ def lane():
from ..trial_bookkeeping import scope as bookkeeping
# Keep bounded preparation/output caches private to this worker across
# functions. Session.prepare rechecks exact include bytes on every use.
with session(), m2c_sessions.scope(state['_foreground']), streaming(self), bookkeeping():
with session(), m2c_sessions.scope(state['_foreground']), streaming(self), bookkeeping(), claim_scope(claim_batch):
while not state['stop'].is_set():
with queue_lock:
fid = next(remaining, None)
if fid is None:return
one(fid)
token=_draft_pipeline.set(True)
try:one(fid)
finally:_draft_pipeline.reset(token)
from ..restore_effects import checkpoint
checkpoint()
try:
count = min(len(ids), max(automatic_workers, width))
if count:
with Executor(max_workers=count, purpose='Autopilot repair search', project=slug, stop=state['stop'].set) as pool:
list(pool.map(lambda _: lane(), range(count)))
if not state['stop'].is_set() and self.is_rom(proj):
proj=self.project_store(proj['id']).one('SELECT * FROM projects WHERE id=?',proj['id'])
# Saved-source repair must not hold up the first machine draft
# of thousands of other functions. Re-read eligibility afterward.
repairs=[row['id'] for row in self.project_store(proj['id']).all(
"SELECT id FROM functions WHERE project_id=? AND status='open' AND best_source IS NOT NULL "
"AND id IN (SELECT value FROM json_each(?)) ORDER BY n_insns,vram",proj['id'],json.dumps(ids))]
state.update(phase='repairs',repair_total=len(repairs),repair_done=0)
remaining=iter(repairs)
try_one=self._autopilot_repair_one
if repairs:
with Executor(max_workers=min(count,len(repairs)),purpose='Autopilot saved-source repair',
project=slug,stop=state['stop'].set) as pool:
list(pool.map(lambda _: lane(),range(min(count,len(repairs)))))
finally:
state["state"] = "stopped" if state["stop"].is_set() else "done"
state["finished_at"] = time.time()
with self.project_store(proj['id']).tx() as db:
cfg = json.loads(db.execute("SELECT config FROM projects WHERE id=?", (proj["id"],)).fetchone()["config"])
cfg["autopilot"] = {k: state.get(k) for k in ("total", "tried", "matched", "near", "twins", "started_at",
"finished_at", "no_drafts")}
"finished_at", "no_drafts", "repair_total", "repair_done", "repair_matched")}
db.execute("UPDATE projects SET config=? WHERE id=?", (json.dumps(cfg), proj["id"]))
drafts = state["matched"] - state.get("twins", 0)
if drafts: # (twins announce their own)
with self.project_store(proj['id']).tx() as db:
method='machine drafts and saved-source repairs' if state.get('repair_matched') else 'machine drafts'
self._event(db, proj["id"], agent.id, "autopilot",
f"{drafts} of {state['tried']} functions matched from machine drafts")
f"{drafts} of {state['tried']} functions matched from {method}")

def on_improvement(self, proj, function_id: int) -> None:
from ..restore_effects import defer
Expand Down Expand Up @@ -267,20 +294,25 @@ def _autopilot_one(self, agent, proj, fid: int) -> str:
with self.progress.judging(proj["slug"], synchronize=False) if hasattr(self, "progress") else nullcontext():
return AutopilotMixin._autopilot_one_work(self, agent, proj, fid)

def _autopilot_one_work(self, agent, proj, fid: int) -> str:
def _autopilot_repair_one(self, agent, proj, fid):
with self.progress.judging(proj['slug'],synchronize=False) if hasattr(self,'progress') else nullcontext():
return self._autopilot_one_work(agent,proj,fid,repair_only=True)

def _autopilot_one_work(self, agent, proj, fid: int, *, repair_only=False) -> str:
now = time.time()
store = self.project_store(proj['id'])
identities = {} if store is self.store else {
'machine_ids': self._machine_agent_ids(), 'nonexclusive_ids': self._nonexclusive_agent_ids()}
from .. import autopilot_claims
identities = autopilot_claims.identities(self,store)
with store.tx() as db:
self._expire_claims(db, **identities)
autopilot_claims.expire(self,db,fid,identities)
fn = db.execute("SELECT * FROM functions WHERE id=?", (fid,)).fetchone()
if fn is None or fn["status"] != "open" or db.execute("SELECT 1 FROM claims WHERE function_id=?", (fid,)).fetchone():
return "skip"
db.execute("INSERT INTO claims VALUES(?,?,?,?)", (fid, agent.id, now, now + 300))
try:
from ..return_repair_autopilot import run as return_repair
if return_repair(self, agent, proj, fn):
if repair_only:return 'match' if return_repair(self,agent,proj,fn) else 'skip'
if not _draft_pipeline.get() and return_repair(self, agent, proj, fn):
return 'match'
ctx = self._context(proj["id"], fn["unit"])
plan, toolchain, context = self._draft_plan(fn, proj, ctx=ctx)
Expand Down Expand Up @@ -323,25 +355,28 @@ def _autopilot_one_work(self, agent, proj, fid: int) -> str:
return 'fail'
# (its compiler named in it: the same draft is worth judging again with another compiler)
note = f"/* machine draft (m2c) for {toolchain} {' '.join(own.get('flags', []))}, tried by autopilot */\n"
candidates = [('raw_m2c', f"{note}{seeds.M2C_MACROS}\n{draft}\n")]
from ..loop_storage_shapes import variants as loop_shapes
for family,source in loop_shapes(draft,fn['name'],ctx['header'],proj['arch'])[:2]:
candidates.append((family,f"{note}{seeds.M2C_MACROS}\n{source}\n"))
if (seeds.m2c_target(toolchain) or "").startswith("mips"):
# next to m2c's own draft, the same with its usual mistakes undone (m2cfix.py): few lines change
refined = m2cfix.refine_draft(draft, fn["name"], target_asm)
if refined:
candidates.append(('refined_m2c', f"{note}{seeds.M2C_MACROS}\n{refined}\n"))
from ..capture_followup_repairs import callee_evidence
from ..recent_capture_repairs import variants as captured_variants
known_callees=callee_evidence(self,proj,fn,draft)
for family,source in captured_variants(draft,fn['name'],'mips' if (seeds.m2c_target(toolchain) or '').startswith('mips') else '',target_asm,ctx['header'],known_callees):
if family in ('overlay_callee_alias','argument_home_slot','untouched_argument_forwarding'):
def draft_candidates():
yield 'raw_m2c', f"{note}{seeds.M2C_MACROS}\n{draft}\n"
candidates = []
from ..loop_storage_shapes import variants as loop_shapes
for family,source in loop_shapes(draft,fn['name'],ctx['header'],proj['arch'])[:2]:
candidates.append((family,f"{note}{seeds.M2C_MACROS}\n{source}\n"))
for canonical in (False,True):
if array := m2cfix.stack_array(draft,fn['name'],canonical):
family = 'stack_array_indexed' if canonical else 'stack_array'
candidates.append((family, f"{note}{seeds.M2C_MACROS}\n{array}\n"))
if (seeds.m2c_target(toolchain) or "").startswith("mips"):
# next to m2c's own draft, the same with its usual mistakes undone (m2cfix.py): few lines change
refined = m2cfix.refine_draft(draft, fn["name"], target_asm)
if refined:
candidates.append(('refined_m2c', f"{note}{seeds.M2C_MACROS}\n{refined}\n"))
from ..capture_followup_repairs import callee_evidence
from ..recent_capture_repairs import variants as captured_variants
known_callees=callee_evidence(self,proj,fn,draft)
for family,source in captured_variants(draft,fn['name'],'mips' if (seeds.m2c_target(toolchain) or '').startswith('mips') else '',target_asm,ctx['header'],known_callees):
if family in ('overlay_callee_alias','argument_home_slot','untouched_argument_forwarding'):
candidates.append((family,f"{note}{seeds.M2C_MACROS}\n{source}\n"))
for canonical in (False,True):
if array := m2cfix.stack_array(draft,fn['name'],canonical):
family = 'stack_array_indexed' if canonical else 'stack_array'
candidates.append((family, f"{note}{seeds.M2C_MACROS}\n{array}\n"))
yield from candidates
def local_candidates():
# Lazy fallback: successful normal drafts never pay for another m2c invocation.
# Imported projects use their file-context drafting route; forwarding their saved
Expand Down Expand Up @@ -377,7 +412,7 @@ def recovery_candidates():
from ..compiler_shape_repairs import machine_draft_copies
from ..narrow_store_repairs import machine_drafts as narrow_drafts
outcomes, seen = [], set()
for family, source in chain(narrow_drafts(machine_draft_copies(chain(candidates, local_candidates()), fn['name'], ctx['header']),fn['name'],ctx['header'],proj['arch']), recovery_candidates()):
for family, source in chain(narrow_drafts(machine_draft_copies(chain(draft_candidates(), local_candidates()), fn['name'], ctx['header']),fn['name'],ctx['header'],proj['arch']), recovery_candidates()):
if source in seen:
continue
seen.add(source)
Expand All @@ -389,7 +424,9 @@ def recovery_candidates():
report['compiled'] = True
report['family'] = family
report['compiled_source'] = source
cand, verdict, _ = self._evaluate(fn, proj, result.obj) if result.ok else (None, None, None)
from .judging import search_evaluation
with search_evaluation():
cand, verdict, _ = self._evaluate(fn, proj, result.obj) if result.ok else (None, None, None)
if cand is None:
if result.ok:
draft_reports.failure(report, 'link', family, 'Compiled C could not be evaluated against this target; review its symbols and data mapping.')
Expand All @@ -411,9 +448,10 @@ def recovery_candidates():
break
report['complete'] = True
draft_reports.save(self, fid, report, project_id=proj['id'])
fresh = self._function(fn['name'], proj['slug'])
if fresh['status'] == 'open' and fresh['best_source'] != fn['best_source'] and return_repair(self, agent, proj, fresh):
return 'match'
if not _draft_pipeline.get():
fresh = store.one('SELECT * FROM functions WHERE id=? AND project_id=?',fid,proj['id'])
if fresh is not None and fresh['status'] == 'open' and fresh['best_source'] != fn['best_source'] and return_repair(self, agent, proj, fresh):
return 'match'
return min(outcomes or ['fail'], key=("match", "near", "far", "skip", "fail").index)
finally:
with self.project_store(proj['id']).tx() as db:
Expand Down
6 changes: 6 additions & 0 deletions src/decompnet/app/static/js/steering.js
Original file line number Diff line number Diff line change
Expand Up @@ -614,6 +614,11 @@ function renderAutopilot(ov) {
if (!(Number(progress.total) > 0)) $('apSub').textContent += ' · Total not yet known';
$('apSub').textContent += ` · ${(tw?.matched || 0).toLocaleString()} verified matches${perm}`;
$("apBtn").textContent = "Stop";
} else if (running && ap.phase === 'repairs') {
$("apTitle").textContent = `Autopilot: repairing saved C · ${ap.matched.toLocaleString()} matched for free`;
$("apSub").textContent = `${(ap.repair_done || 0).toLocaleString()} of ${(ap.repair_total || 0).toLocaleString()} checked after ${ap.tried.toLocaleString()} machine drafts. Agents work alongside it.${perm}`;
updateWorkProgress({_workBar: $('apBar'), _workStatus: {textContent: ''}}, {phase: 'Repairing saved C', done: ap.repair_done || 0, total: ap.repair_total || 0});
$("apBtn").textContent = "Stop";
} else if (running) {
$("apTitle").textContent = `Autopilot: trying each function's machine draft · ${ap.matched.toLocaleString()} matched for free`;
$("apSub").textContent = `${ap.tried.toLocaleString()} of ${ap.total.toLocaleString()} tried${ap.near ? ` · ${ap.near} more are 90%+ there` : ""}${ap.twins ? ` · ${ap.twins.toLocaleString()} from reused C` : ""}. Agents work alongside it.${perm}`;
Expand All @@ -627,6 +632,7 @@ function renderAutopilot(ov) {
if (twinsDone) parts.push(`${twinsDone.toLocaleString()} from reused C`);
if (tw && tw.asm) parts.push(`${tw.asm.toLocaleString()} copies of assembly set aside (not for agents)`);
if (ap && ap.tried) parts.push(`${ap.tried.toLocaleString()} machine drafts tried${ap.near ? `, ${ap.near} more 90%+ there` : ""}`);
if (ap && ap.repair_done) parts.push(`${ap.repair_done.toLocaleString()} saved-source repairs checked`);
if (ap && ap.no_drafts && !ap.tried) $("apTitle").textContent = "Autopilot could not try machine drafts";
$("apSub").textContent = (parts.length ? parts.join(" · ") + ". " : "") + (ap && ap.no_drafts ? ap.no_drafts : "Run it again after agents add types and names: drafts get better.") + perm;
$("apBtn").textContent = "Run again";
Expand Down
Loading
Loading