Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions docs/APP_PERFORMANCE.md
Original file line number Diff line number Diff line change
Expand Up @@ -379,3 +379,20 @@ New feedback-generated children retain the existing ownership guard. Worker wait
checks cancellation and the original search deadline; shutdown terminates proposal
processes and releases their pipes. Specialized cooperative generation sessions retain
their in-process path so family/budget filters are unchanged.

### Autopilot draft pipeline

Explicit autopilot lanes keep a private pinned m2c interpreter warm across functions,
recycling it after 64 requests. Automatic runs keep the ordinary CLI path until
promoted by the user. Stop and timeout terminate the generator process; an unavailable
worker falls back to the pinned CLI. A local 12-request small MIPS draft check took
1.35 seconds through separate CLI launches and 0.17 seconds through the warmed
interpreter, including its first startup, with identical output. This measures draft
generation, not total autopilot throughput.

Each trial retains the saved-file revision guard but leaves project-wide file
synchronization to the watcher. Verified results are saved immediately. Optional
reuse/archive follow-ups run separately in bounded batches and index the changed
functions rather than rescanning the project. Advisory draft reports share bounded
trial bookkeeping; completed resume receipts flush immediately. Stopping a lane
flushes advisory reports and queues follow-ups for its already committed results.
13 changes: 9 additions & 4 deletions src/decompnet/app/draft_reports.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Persist bounded generation/compiler/link diagnostics independently of matching history."""
import copy
import hashlib
import json
import time
Expand All @@ -21,6 +22,9 @@ def key(plan, header, data, fn):

def read(net, fid, *, project_id=None):
store = net.project_store(project_id) if project_id is not None else net.store
from .trial_bookkeeping import lookup
cached=lookup(store,('draft-report',fid))
if cached is not None:return copy.deepcopy(cached)
row = store.one('SELECT value FROM settings WHERE key=?', PREFIX+str(fid))
if not row:
return {}
Expand All @@ -29,11 +33,12 @@ def read(net, fid, *, project_id=None):


def save(net, fid, report, *, project_id=None):
report = dict(report, updated_at=time.time())
report = dict(copy.deepcopy(report), updated_at=time.time())
store = net.project_store(project_id) if project_id is not None else net.store
with store.tx() as db:
db.execute('INSERT INTO settings(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',
(PREFIX+str(fid),json.dumps(report)))
from .trial_bookkeeping import submit
submit(store,[('INSERT INTO settings(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',
(PREFIX+str(fid),json.dumps(report)))],immediate=bool(report.get('complete')),
memo=(('draft-report',fid),report))


def failure(report, phase, family, reason):
Expand Down
13 changes: 10 additions & 3 deletions src/decompnet/app/engine/autopilot.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,9 @@ def one(fid: int) -> None:
manual = state['_foreground'].is_set()
try:
from ...compile_sessions import session
with maintenance.foreground(state['_foreground']), session(), \
def check_stop():
if state['stop'].is_set():raise NetworkError('Autopilot stopped.')
with maintenance.foreground(state['_foreground']), session(), compiler.cancellable(check_stop), \
compiler._SLOTS.capacity(width) if manual else nullcontext(), \
compiler.throughput_slots(manual):
outcome = try_one(agent, proj, fid)
Expand All @@ -144,15 +146,20 @@ def one(fid: int) -> None:
remaining = iter(ids)
queue_lock = threading.Lock()
def lane():
from ... import m2c_sessions
from ...compile_sessions import session
from ..restore_effects import streaming
from ..trial_bookkeeping import scope as bookkeeping
# Keep bounded preparation/output caches private to this worker across
# functions. Session.prepare rechecks exact include bytes on every use.
with session():
with session(), m2c_sessions.scope(state['_foreground']), streaming(self), bookkeeping():
while not state['stop'].is_set():
with queue_lock:
fid = next(remaining, None)
if fid is None:return
one(fid)
from ..restore_effects import checkpoint
checkpoint()
try:
count = min(len(ids), max(automatic_workers, width))
if count:
Expand Down Expand Up @@ -257,7 +264,7 @@ def _autopilot_one(self, agent, proj, fid: int) -> str:
"""match | near | far | fail | skip, for one function's draft. A draft autopilot already judged with the same
header is skipped: autopilot tries a function again when its draft changed (the background drafter asks it
to after the header or the compiler changed, app/drafts.py), not to judge the same C twice."""
with self.progress.judging(proj["slug"]) if hasattr(self, "progress") else nullcontext():
with self.progress.judging(proj["slug"], synchronize=False) if hasattr(self, "progress") else nullcontext():
return AutopilotMixin._autopilot_one_work(self, agent, proj, fid)

def _autopilot_one_work(self, agent, proj, fid: int) -> str:
Expand Down
39 changes: 37 additions & 2 deletions src/decompnet/app/restore_effects.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import json
import logging
import threading
import time
from contextlib import contextmanager
from contextvars import ContextVar

_current=ContextVar('saved_restore_effects',default=None)
Expand All @@ -14,6 +16,9 @@ def defer(net,proj,fid,*,matched):
if state is None or state['net'] is not net or state['owner']!=threading.get_ident():return False
project=state['projects'].setdefault(proj['id'],{'slug':proj['slug'],'matched':set(),'improved':set()})
project['matched' if matched else 'improved'].add(fid)
if state.get('streaming') and (sum(len(p['matched'])+len(p['improved']) for p in state['projects'].values())>=16
or time.monotonic()-state['last_flush']>=1):
_flush(state)
return True


Expand All @@ -28,7 +33,11 @@ def _run(net,pid,pending,logger):
rows=store.all("SELECT id,name FROM functions WHERE project_id=? AND status='matched' "
"AND id IN (SELECT value FROM json_each(?))",pid,json.dumps(sorted(pending['matched'])))
logger.progress({'phase':'Indexing restored matches','done':0,'total':len(rows)})
if rows:net.twins.index_project(proj)
if pending.get('incremental'):
for row in rows:
logger.check_cancel();maintenance.checkpoint()
net.twins.index_function(proj,row['id'])
elif rows:net.twins.index_project(proj)
if rows and (library:=getattr(net,'match_library',None)):library.archive_project(proj,logger,rows=rows)
for index,row in enumerate(rows):
logger.check_cancel();maintenance.checkpoint()
Expand All @@ -40,6 +49,32 @@ def _run(net,pid,pending,logger):
net.rangechecks.recheck(proj['slug'])


def _flush(state):
projects=state['projects'];state['projects']={};state['last_flush']=time.monotonic()
for pid,pending in projects.items():
pending['incremental']=True
try:_queue(state['net'],pid,pending)
except Exception:log.exception('Could not queue optional autopilot match indexing')


def checkpoint():
state=_current.get()
if state and state.get('streaming') and state['projects'] and time.monotonic()-state['last_flush']>=1:
_flush(state)


@contextmanager
def streaming(net):
if _current.get() is not None:
yield;return
state={'net':net,'owner':threading.get_ident(),'projects':{},'streaming':True,'last_flush':time.monotonic()-1}
token=_current.set(state)
try:yield
finally:
_current.reset(token)
_flush(state) # Already committed matches remain saved even if the pass is stopped.


def batched(function):
@functools.wraps(function)
def run(net,*args,**kwargs):
Expand All @@ -60,7 +95,7 @@ def run(net,*args,**kwargs):
def _queue(net,pid,pending):
def work(logger,pid=pid,pending=pending):return _run(net,pid,pending,logger)
if startup:=getattr(net,'startup',None):
startup.add('Index restored matches',work,project=pending['slug'],with_log=True)
startup.add('Index autopilot matches' if pending.get('incremental') else 'Index restored matches',work,project=pending['slug'],with_log=True)
startup.start()
else:
# Headless engines have no activity UI; reuse the same optional lane.
Expand Down
79 changes: 79 additions & 0 deletions src/decompnet/m2c_sessions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
"""One bounded, warmed draft generator per explicit autopilot worker."""
import json
import os
import queue
import subprocess
import threading
import time
from contextlib import contextmanager
from contextvars import ContextVar
from pathlib import Path

from . import compiler, limits
from .hostenv import NO_WINDOW, stop_process_tree

_current=ContextVar('m2c_session',default=None)


class Session:
def __init__(self, enabled):
self.enabled=enabled
self.owner=threading.get_ident()
self.process=None
self.key=None
self.requests=0
self.reader=None

def close(self):
if self.process is None:return
stop_process_tree(self.process)
self.reader.join(timeout=2)
for stream in (self.process.stdin,self.process.stdout):stream.close()
self.process=None

def run(self, argv, timeout):
executable=Path(argv[0])
python=executable.with_name('python.exe' if os.name=='nt' else 'python')
key=(str(executable),executable.stat().st_mtime_ns,executable.stat().st_size)
if self.process is not None and (self.key!=key or self.requests>=64):self.close()
if self.process is None:
self.lines=queue.Queue()
self.process=subprocess.Popen(limits.capped([str(python),str(Path(__file__).with_name('m2c_worker.py'))],memory=2<<30),
stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,creationflags=NO_WINDOW)
process=self.process
def read():
try:
for line in process.stdout:self.lines.put(line)
finally:self.lines.put(None)
self.reader=threading.Thread(target=read,name='autopilot-m2c-output',daemon=True)
self.reader.start();self.key=key;self.requests=0
try:
self.process.stdin.write(json.dumps(argv[1:]).encode()+b'\n');self.process.stdin.flush()
started=time.monotonic()
while True:
if check:=compiler._CANCEL_CHECK.get():check()
if time.monotonic()-started>=timeout:raise subprocess.TimeoutExpired(argv,timeout)
try:line=self.lines.get(timeout=.05);break
except queue.Empty:pass
if line is None:raise OSError('Draft generator stopped')
data=json.loads(line);self.requests+=1
return subprocess.CompletedProcess(argv,data['returncode'],data['stdout'],data['stderr'])
except BaseException:
self.close()
raise


def run(argv, timeout):
session=_current.get()
if session is None or session.owner!=threading.get_ident() or not session.enabled.is_set():return None
return session.run(argv,timeout)


@contextmanager
def scope(enabled):
if _current.get() is not None:
yield;return
session=Session(enabled);token=_current.set(session)
try:yield
finally:
_current.reset(token);session.close()
23 changes: 23 additions & 0 deletions src/decompnet/m2c_worker.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
"""Private warmed m2c interpreter; invoked by its pinned virtualenv, not the app."""
import contextlib
import gc
import io
import json
import sys

from m2c.main import parse_flags, run

sys.setrecursionlimit(10000)
for line in sys.stdin:
out, err = io.StringIO(), io.StringIO()
code = 0
try:
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
code = run(parse_flags(json.loads(line))) or 0
except SystemExit as error:
code = error.code if isinstance(error.code, int) else 1
except Exception as error:
code = 1
err.write(str(error))
print(json.dumps({'returncode':code,'stdout':out.getvalue(),'stderr':err.getvalue()}),flush=True)
gc.collect()
16 changes: 12 additions & 4 deletions src/decompnet/seeds.py
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,16 @@ def _failure(res: subprocess.CompletedProcess) -> str:
return note.strip().strip("*/").strip()[-MAX_REASON_CHARS:] or f"m2c stopped with code {res.returncode}"


def _execute_m2c(argv, timeout):
from . import m2c_sessions
try:
result=m2c_sessions.run(argv,timeout)
except (OSError,ValueError):
result=None # A missing/incompatible worker can use the ordinary pinned CLI.
return result if result is not None else subprocess.run(limits.capped(argv),capture_output=True,text=True,
timeout=timeout,encoding="utf-8",errors="replace",creationflags=NO_WINDOW)


def run_m2c(asm_text: str, name: str, target: str, context_header: str = "", data_asm: str | None = None,
timeout: float = 20, valid: bool = False, strict: bool = False, options: tuple[str, ...] = ()) -> str | None:
"""Run m2c for one function with an explicit target triple (e.g. ppc-mwcc-c++)."""
Expand Down Expand Up @@ -397,8 +407,7 @@ def run_m2c(asm_text: str, name: str, target: str, context_header: str = "", dat
ctx.write_text(_context_for_m2c(context_header), encoding="utf-8")
argv[1:1] = ["--context", str(ctx)]
try:
res = subprocess.run(limits.capped(argv), capture_output=True, text=True, timeout=timeout,
encoding="utf-8", errors="replace", creationflags=NO_WINDOW)
res = _execute_m2c(argv,timeout)
except subprocess.TimeoutExpired:
if strict:
raise DraftError(f"m2c took longer than {timeout:g} s") from None
Expand All @@ -407,8 +416,7 @@ def run_m2c(asm_text: str, name: str, target: str, context_header: str = "", dat
raise DraftError(_failure(res))
if res.returncode != 0 and "--context" in argv: # an unparsable context must not cost us the seed
i = argv.index("--context")
res = subprocess.run(limits.capped(argv[:i] + argv[i + 2:]), capture_output=True, text=True, timeout=timeout,
encoding="utf-8", errors="replace", creationflags=NO_WINDOW)
res = _execute_m2c(argv[:i] + argv[i + 2:],timeout)
out = res.stdout.strip()
# m2c can exit successfully after emitting a partial function and bare diagnostics.
# Its error placeholders discard instructions; they are explanations, never baseline C.
Expand Down
46 changes: 46 additions & 0 deletions tests/test_autopilot_pipeline.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
"""Critical-path reductions retain durable completions and synchronization guards."""
import pytest

from decompnet.app import draft_reports, restore_effects, trial_bookkeeping
from decompnet.app.engine import RomNetwork
from tests.test_loading import tiny_game


def test_receipts_batch_advisory_updates_but_completed_receipt_flushes(tmp_path):
net=RomNetwork(tmp_path/'db',tmp_path);tiny_game(net)
fn=net.store.one('SELECT id,project_id FROM functions LIMIT 1')
with trial_bookkeeping.scope():
draft_reports.save(net,fn['id'],{'failures':['diagnostic']},project_id=fn['project_id'])
assert not net.store.one('SELECT 1 FROM settings WHERE key=?',draft_reports.PREFIX+str(fn['id']))
receipt=draft_reports.read(net,fn['id'],project_id=fn['project_id'])
assert receipt['failures']==['diagnostic']
receipt['failures'].append('not saved')
assert draft_reports.read(net,fn['id'],project_id=fn['project_id'])['failures']==['diagnostic']
draft_reports.save(net,fn['id'],{'complete':True},project_id=fn['project_id'])
assert net.store.one('SELECT 1 FROM settings WHERE key=?',draft_reports.PREFIX+str(fn['id']))
assert draft_reports.read(net,fn['id'],project_id=fn['project_id'])['complete']


def test_streaming_followups_are_queued_and_retained_on_stop(monkeypatch):
net=object();queued=[]
monkeypatch.setattr(restore_effects,'_queue',lambda net,pid,pending:queued.append((pid,pending)))
with pytest.raises(RuntimeError):
with restore_effects.streaming(net):
assert restore_effects.defer(net,{'id':1,'slug':'game'},2,matched=True)
assert queued and queued[0][1]['incremental']
assert restore_effects.defer(net,{'id':1,'slug':'game'},3,matched=False)
raise RuntimeError('stopped')
assert any(3 in pending['improved'] for _,pending in queued)


def test_scoped_autopilot_trial_keeps_revision_guard_without_file_sync(tmp_path,monkeypatch):
net=RomNetwork(tmp_path/'db',tmp_path);tiny_game(net)
proj=net.project_row('game');fn=net.store.one('SELECT id FROM functions LIMIT 1')
agent,_=net.local_agent('autopilot')
monkeypatch.setattr(net.progress,'sync',lambda *a,**kw:pytest.fail('Trial must not synchronize the entire project'))
from decompnet.app.engine.autopilot import AutopilotMixin
def work(*args):
net.progress.check_result()
return 'skip'
monkeypatch.setattr(AutopilotMixin,'_autopilot_one_work',work)
assert net._autopilot_one(agent,proj,fn['id'])=='skip'
Loading
Loading