diff --git a/docs/APP_PERFORMANCE.md b/docs/APP_PERFORMANCE.md index 799384477..068bc0b1c 100644 --- a/docs/APP_PERFORMANCE.md +++ b/docs/APP_PERFORMANCE.md @@ -379,3 +379,20 @@ New feedback-generated children retain the existing ownership guard. Worker wait checks cancellation and the original search deadline; shutdown terminates proposal processes and releases their pipes. Specialized cooperative generation sessions retain their in-process path so family/budget filters are unchanged. + +### Autopilot draft pipeline + +Explicit autopilot lanes keep a private pinned m2c interpreter warm across functions, +recycling it after 64 requests. Automatic runs keep the ordinary CLI path until +promoted by the user. Stop and timeout terminate the generator process; an unavailable +worker falls back to the pinned CLI. A local 12-request small MIPS draft check took +1.35 seconds through separate CLI launches and 0.17 seconds through the warmed +interpreter, including its first startup, with identical output. This measures draft +generation, not total autopilot throughput. + +Each trial retains the saved-file revision guard but leaves project-wide file +synchronization to the watcher. Verified results are saved immediately. Optional +reuse/archive follow-ups run separately in bounded batches and index the changed +functions rather than rescanning the project. Advisory draft reports share bounded +trial bookkeeping; completed resume receipts flush immediately. Stopping a lane +flushes advisory reports and queues follow-ups for its already committed results. diff --git a/src/decompnet/app/draft_reports.py b/src/decompnet/app/draft_reports.py index 904de4e84..ec7e7804e 100644 --- a/src/decompnet/app/draft_reports.py +++ b/src/decompnet/app/draft_reports.py @@ -1,4 +1,5 @@ """Persist bounded generation/compiler/link diagnostics independently of matching history.""" +import copy import hashlib import json import time @@ -21,6 +22,9 @@ def key(plan, header, data, fn): def read(net, fid, *, project_id=None): store = net.project_store(project_id) if project_id is not None else net.store + from .trial_bookkeeping import lookup + cached=lookup(store,('draft-report',fid)) + if cached is not None:return copy.deepcopy(cached) row = store.one('SELECT value FROM settings WHERE key=?', PREFIX+str(fid)) if not row: return {} @@ -29,11 +33,12 @@ def read(net, fid, *, project_id=None): def save(net, fid, report, *, project_id=None): - report = dict(report, updated_at=time.time()) + report = dict(copy.deepcopy(report), updated_at=time.time()) store = net.project_store(project_id) if project_id is not None else net.store - with store.tx() as db: - db.execute('INSERT INTO settings(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value', - (PREFIX+str(fid),json.dumps(report))) + from .trial_bookkeeping import submit + submit(store,[('INSERT INTO settings(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value', + (PREFIX+str(fid),json.dumps(report)))],immediate=bool(report.get('complete')), + memo=(('draft-report',fid),report)) def failure(report, phase, family, reason): diff --git a/src/decompnet/app/engine/autopilot.py b/src/decompnet/app/engine/autopilot.py index 1937084ab..173ee9142 100644 --- a/src/decompnet/app/engine/autopilot.py +++ b/src/decompnet/app/engine/autopilot.py @@ -129,7 +129,9 @@ def one(fid: int) -> None: manual = state['_foreground'].is_set() try: from ...compile_sessions import session - with maintenance.foreground(state['_foreground']), session(), \ + def check_stop(): + if state['stop'].is_set():raise NetworkError('Autopilot stopped.') + with maintenance.foreground(state['_foreground']), session(), compiler.cancellable(check_stop), \ compiler._SLOTS.capacity(width) if manual else nullcontext(), \ compiler.throughput_slots(manual): outcome = try_one(agent, proj, fid) @@ -144,15 +146,20 @@ def one(fid: int) -> None: remaining = iter(ids) queue_lock = threading.Lock() def lane(): + from ... import m2c_sessions from ...compile_sessions import session + from ..restore_effects import streaming + from ..trial_bookkeeping import scope as bookkeeping # Keep bounded preparation/output caches private to this worker across # functions. Session.prepare rechecks exact include bytes on every use. - with session(): + with session(), m2c_sessions.scope(state['_foreground']), streaming(self), bookkeeping(): while not state['stop'].is_set(): with queue_lock: fid = next(remaining, None) if fid is None:return one(fid) + from ..restore_effects import checkpoint + checkpoint() try: count = min(len(ids), max(automatic_workers, width)) if count: @@ -257,7 +264,7 @@ def _autopilot_one(self, agent, proj, fid: int) -> str: """match | near | far | fail | skip, for one function's draft. A draft autopilot already judged with the same header is skipped: autopilot tries a function again when its draft changed (the background drafter asks it to after the header or the compiler changed, app/drafts.py), not to judge the same C twice.""" - with self.progress.judging(proj["slug"]) if hasattr(self, "progress") else nullcontext(): + with self.progress.judging(proj["slug"], synchronize=False) if hasattr(self, "progress") else nullcontext(): return AutopilotMixin._autopilot_one_work(self, agent, proj, fid) def _autopilot_one_work(self, agent, proj, fid: int) -> str: diff --git a/src/decompnet/app/restore_effects.py b/src/decompnet/app/restore_effects.py index fd8abc937..d9c65030b 100644 --- a/src/decompnet/app/restore_effects.py +++ b/src/decompnet/app/restore_effects.py @@ -3,6 +3,8 @@ import json import logging import threading +import time +from contextlib import contextmanager from contextvars import ContextVar _current=ContextVar('saved_restore_effects',default=None) @@ -14,6 +16,9 @@ def defer(net,proj,fid,*,matched): if state is None or state['net'] is not net or state['owner']!=threading.get_ident():return False project=state['projects'].setdefault(proj['id'],{'slug':proj['slug'],'matched':set(),'improved':set()}) project['matched' if matched else 'improved'].add(fid) + if state.get('streaming') and (sum(len(p['matched'])+len(p['improved']) for p in state['projects'].values())>=16 + or time.monotonic()-state['last_flush']>=1): + _flush(state) return True @@ -28,7 +33,11 @@ def _run(net,pid,pending,logger): rows=store.all("SELECT id,name FROM functions WHERE project_id=? AND status='matched' " "AND id IN (SELECT value FROM json_each(?))",pid,json.dumps(sorted(pending['matched']))) logger.progress({'phase':'Indexing restored matches','done':0,'total':len(rows)}) - if rows:net.twins.index_project(proj) + if pending.get('incremental'): + for row in rows: + logger.check_cancel();maintenance.checkpoint() + net.twins.index_function(proj,row['id']) + elif rows:net.twins.index_project(proj) if rows and (library:=getattr(net,'match_library',None)):library.archive_project(proj,logger,rows=rows) for index,row in enumerate(rows): logger.check_cancel();maintenance.checkpoint() @@ -40,6 +49,32 @@ def _run(net,pid,pending,logger): net.rangechecks.recheck(proj['slug']) +def _flush(state): + projects=state['projects'];state['projects']={};state['last_flush']=time.monotonic() + for pid,pending in projects.items(): + pending['incremental']=True + try:_queue(state['net'],pid,pending) + except Exception:log.exception('Could not queue optional autopilot match indexing') + + +def checkpoint(): + state=_current.get() + if state and state.get('streaming') and state['projects'] and time.monotonic()-state['last_flush']>=1: + _flush(state) + + +@contextmanager +def streaming(net): + if _current.get() is not None: + yield;return + state={'net':net,'owner':threading.get_ident(),'projects':{},'streaming':True,'last_flush':time.monotonic()-1} + token=_current.set(state) + try:yield + finally: + _current.reset(token) + _flush(state) # Already committed matches remain saved even if the pass is stopped. + + def batched(function): @functools.wraps(function) def run(net,*args,**kwargs): @@ -60,7 +95,7 @@ def run(net,*args,**kwargs): def _queue(net,pid,pending): def work(logger,pid=pid,pending=pending):return _run(net,pid,pending,logger) if startup:=getattr(net,'startup',None): - startup.add('Index restored matches',work,project=pending['slug'],with_log=True) + startup.add('Index autopilot matches' if pending.get('incremental') else 'Index restored matches',work,project=pending['slug'],with_log=True) startup.start() else: # Headless engines have no activity UI; reuse the same optional lane. diff --git a/src/decompnet/m2c_sessions.py b/src/decompnet/m2c_sessions.py new file mode 100644 index 000000000..d0b91df9f --- /dev/null +++ b/src/decompnet/m2c_sessions.py @@ -0,0 +1,79 @@ +"""One bounded, warmed draft generator per explicit autopilot worker.""" +import json +import os +import queue +import subprocess +import threading +import time +from contextlib import contextmanager +from contextvars import ContextVar +from pathlib import Path + +from . import compiler, limits +from .hostenv import NO_WINDOW, stop_process_tree + +_current=ContextVar('m2c_session',default=None) + + +class Session: + def __init__(self, enabled): + self.enabled=enabled + self.owner=threading.get_ident() + self.process=None + self.key=None + self.requests=0 + self.reader=None + + def close(self): + if self.process is None:return + stop_process_tree(self.process) + self.reader.join(timeout=2) + for stream in (self.process.stdin,self.process.stdout):stream.close() + self.process=None + + def run(self, argv, timeout): + executable=Path(argv[0]) + python=executable.with_name('python.exe' if os.name=='nt' else 'python') + key=(str(executable),executable.stat().st_mtime_ns,executable.stat().st_size) + if self.process is not None and (self.key!=key or self.requests>=64):self.close() + if self.process is None: + self.lines=queue.Queue() + self.process=subprocess.Popen(limits.capped([str(python),str(Path(__file__).with_name('m2c_worker.py'))],memory=2<<30), + stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,creationflags=NO_WINDOW) + process=self.process + def read(): + try: + for line in process.stdout:self.lines.put(line) + finally:self.lines.put(None) + self.reader=threading.Thread(target=read,name='autopilot-m2c-output',daemon=True) + self.reader.start();self.key=key;self.requests=0 + try: + self.process.stdin.write(json.dumps(argv[1:]).encode()+b'\n');self.process.stdin.flush() + started=time.monotonic() + while True: + if check:=compiler._CANCEL_CHECK.get():check() + if time.monotonic()-started>=timeout:raise subprocess.TimeoutExpired(argv,timeout) + try:line=self.lines.get(timeout=.05);break + except queue.Empty:pass + if line is None:raise OSError('Draft generator stopped') + data=json.loads(line);self.requests+=1 + return subprocess.CompletedProcess(argv,data['returncode'],data['stdout'],data['stderr']) + except BaseException: + self.close() + raise + + +def run(argv, timeout): + session=_current.get() + if session is None or session.owner!=threading.get_ident() or not session.enabled.is_set():return None + return session.run(argv,timeout) + + +@contextmanager +def scope(enabled): + if _current.get() is not None: + yield;return + session=Session(enabled);token=_current.set(session) + try:yield + finally: + _current.reset(token);session.close() diff --git a/src/decompnet/m2c_worker.py b/src/decompnet/m2c_worker.py new file mode 100644 index 000000000..fa2bb0bae --- /dev/null +++ b/src/decompnet/m2c_worker.py @@ -0,0 +1,23 @@ +"""Private warmed m2c interpreter; invoked by its pinned virtualenv, not the app.""" +import contextlib +import gc +import io +import json +import sys + +from m2c.main import parse_flags, run + +sys.setrecursionlimit(10000) +for line in sys.stdin: + out, err = io.StringIO(), io.StringIO() + code = 0 + try: + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = run(parse_flags(json.loads(line))) or 0 + except SystemExit as error: + code = error.code if isinstance(error.code, int) else 1 + except Exception as error: + code = 1 + err.write(str(error)) + print(json.dumps({'returncode':code,'stdout':out.getvalue(),'stderr':err.getvalue()}),flush=True) + gc.collect() diff --git a/src/decompnet/seeds.py b/src/decompnet/seeds.py index b7d45656e..044484da4 100644 --- a/src/decompnet/seeds.py +++ b/src/decompnet/seeds.py @@ -366,6 +366,16 @@ def _failure(res: subprocess.CompletedProcess) -> str: return note.strip().strip("*/").strip()[-MAX_REASON_CHARS:] or f"m2c stopped with code {res.returncode}" +def _execute_m2c(argv, timeout): + from . import m2c_sessions + try: + result=m2c_sessions.run(argv,timeout) + except (OSError,ValueError): + result=None # A missing/incompatible worker can use the ordinary pinned CLI. + return result if result is not None else subprocess.run(limits.capped(argv),capture_output=True,text=True, + timeout=timeout,encoding="utf-8",errors="replace",creationflags=NO_WINDOW) + + def run_m2c(asm_text: str, name: str, target: str, context_header: str = "", data_asm: str | None = None, timeout: float = 20, valid: bool = False, strict: bool = False, options: tuple[str, ...] = ()) -> str | None: """Run m2c for one function with an explicit target triple (e.g. ppc-mwcc-c++).""" @@ -397,8 +407,7 @@ def run_m2c(asm_text: str, name: str, target: str, context_header: str = "", dat ctx.write_text(_context_for_m2c(context_header), encoding="utf-8") argv[1:1] = ["--context", str(ctx)] try: - res = subprocess.run(limits.capped(argv), capture_output=True, text=True, timeout=timeout, - encoding="utf-8", errors="replace", creationflags=NO_WINDOW) + res = _execute_m2c(argv,timeout) except subprocess.TimeoutExpired: if strict: raise DraftError(f"m2c took longer than {timeout:g} s") from None @@ -407,8 +416,7 @@ def run_m2c(asm_text: str, name: str, target: str, context_header: str = "", dat raise DraftError(_failure(res)) if res.returncode != 0 and "--context" in argv: # an unparsable context must not cost us the seed i = argv.index("--context") - res = subprocess.run(limits.capped(argv[:i] + argv[i + 2:]), capture_output=True, text=True, timeout=timeout, - encoding="utf-8", errors="replace", creationflags=NO_WINDOW) + res = _execute_m2c(argv[:i] + argv[i + 2:],timeout) out = res.stdout.strip() # m2c can exit successfully after emitting a partial function and bare diagnostics. # Its error placeholders discard instructions; they are explanations, never baseline C. diff --git a/tests/test_autopilot_pipeline.py b/tests/test_autopilot_pipeline.py new file mode 100644 index 000000000..8e4dc0fb2 --- /dev/null +++ b/tests/test_autopilot_pipeline.py @@ -0,0 +1,46 @@ +"""Critical-path reductions retain durable completions and synchronization guards.""" +import pytest + +from decompnet.app import draft_reports, restore_effects, trial_bookkeeping +from decompnet.app.engine import RomNetwork +from tests.test_loading import tiny_game + + +def test_receipts_batch_advisory_updates_but_completed_receipt_flushes(tmp_path): + net=RomNetwork(tmp_path/'db',tmp_path);tiny_game(net) + fn=net.store.one('SELECT id,project_id FROM functions LIMIT 1') + with trial_bookkeeping.scope(): + draft_reports.save(net,fn['id'],{'failures':['diagnostic']},project_id=fn['project_id']) + assert not net.store.one('SELECT 1 FROM settings WHERE key=?',draft_reports.PREFIX+str(fn['id'])) + receipt=draft_reports.read(net,fn['id'],project_id=fn['project_id']) + assert receipt['failures']==['diagnostic'] + receipt['failures'].append('not saved') + assert draft_reports.read(net,fn['id'],project_id=fn['project_id'])['failures']==['diagnostic'] + draft_reports.save(net,fn['id'],{'complete':True},project_id=fn['project_id']) + assert net.store.one('SELECT 1 FROM settings WHERE key=?',draft_reports.PREFIX+str(fn['id'])) + assert draft_reports.read(net,fn['id'],project_id=fn['project_id'])['complete'] + + +def test_streaming_followups_are_queued_and_retained_on_stop(monkeypatch): + net=object();queued=[] + monkeypatch.setattr(restore_effects,'_queue',lambda net,pid,pending:queued.append((pid,pending))) + with pytest.raises(RuntimeError): + with restore_effects.streaming(net): + assert restore_effects.defer(net,{'id':1,'slug':'game'},2,matched=True) + assert queued and queued[0][1]['incremental'] + assert restore_effects.defer(net,{'id':1,'slug':'game'},3,matched=False) + raise RuntimeError('stopped') + assert any(3 in pending['improved'] for _,pending in queued) + + +def test_scoped_autopilot_trial_keeps_revision_guard_without_file_sync(tmp_path,monkeypatch): + net=RomNetwork(tmp_path/'db',tmp_path);tiny_game(net) + proj=net.project_row('game');fn=net.store.one('SELECT id FROM functions LIMIT 1') + agent,_=net.local_agent('autopilot') + monkeypatch.setattr(net.progress,'sync',lambda *a,**kw:pytest.fail('Trial must not synchronize the entire project')) + from decompnet.app.engine.autopilot import AutopilotMixin + def work(*args): + net.progress.check_result() + return 'skip' + monkeypatch.setattr(AutopilotMixin,'_autopilot_one_work',work) + assert net._autopilot_one(agent,proj,fn['id'])=='skip' diff --git a/tests/test_m2c_sessions.py b/tests/test_m2c_sessions.py new file mode 100644 index 000000000..7abba6449 --- /dev/null +++ b/tests/test_m2c_sessions.py @@ -0,0 +1,56 @@ +"""Warmed draft generators must agree with the CLI and stop with their owner.""" +import subprocess +import sys +import threading +import time + +import pytest + +from decompnet import compiler, m2c_sessions, seeds +from decompnet.network import NetworkError +from decompnet.rom.split import draft_asm +from tests.test_overlay_drafts import ADDRESS, ASSEMBLY + + +@pytest.mark.skipif(not seeds.available(),reason='needs pinned m2c') +def test_warmed_generator_matches_cli_across_inputs_and_reuses_process(): + target=seeds.m2c_target('gcc-2.7.2-kmc') + asm=draft_asm(ASSEMBLY,'example',ADDRESS) + inputs=[(asm,'example',''),(asm.replace('example','another_entry'),'another_entry','')] + cold=[seeds.run_m2c(asm,name,target,header,valid=True,strict=True) for asm,name,header in inputs] + enabled=threading.Event();enabled.set() + with m2c_sessions.scope(enabled): + warm=[seeds.run_m2c(asm,name,target,header,valid=True,strict=True) for asm,name,header in inputs] + session=m2c_sessions._current.get() + assert session.requests==2 + with pytest.raises(seeds.DraftError): + seeds.run_m2c('glabel broken\nbeqz $a0, .Lmissing\nnop','broken',target,valid=True,strict=True) + assert seeds.run_m2c(inputs[0][0],inputs[0][1],target,valid=True,strict=True)==cold[0] + old=session.process + session.requests=64 + assert seeds.run_m2c(inputs[0][0],inputs[0][1],target,valid=True,strict=True)==cold[0] + assert old.poll() is not None and session.process.pid!=old.pid + proc=session.process + assert proc.poll() is not None + assert warm==cold + + +def test_stop_terminates_stuck_generator(tmp_path,monkeypatch): + tool=tmp_path/'m2c';tool.write_text('test') + original=subprocess.Popen + processes=[] + def launch(*args,**kwargs): + proc=original([sys.executable,'-c','import time; time.sleep(30)'],**kwargs) + processes.append(proc);return proc + monkeypatch.setattr(m2c_sessions.subprocess,'Popen',launch) + enabled=threading.Event();enabled.set() + stop=threading.Event() + def check(): + if stop.is_set():raise NetworkError('stopped') + timer=threading.Timer(.1,stop.set);timer.start() + started=time.monotonic() + try: + with m2c_sessions.scope(enabled),compiler.cancellable(check),pytest.raises(NetworkError,match='stopped'): + m2c_sessions.run([str(tool)],20) + finally:timer.cancel() + assert time.monotonic()-started<5 and processes[0].poll() is not None