From ab01e6a8a05b0f589f62830f8742beaed1275ca8 Mon Sep 17 00:00:00 2001 From: RetroGameDeveloper <40120498+RetroGameDeveloper@users.noreply.github.com> Date: Sun, 8 Nov 2026 16:39:54 +0000 Subject: [PATCH] fix: keep backend locks and job admission off the server event loop --- docs/SERVER_LOCK_AUDIT.md | 28 +++ src/decompnet/app/cli.py | 2 +- src/decompnet/app/routes/__init__.py | 5 + src/decompnet/app/routes/browser_bridge.py | 12 +- src/decompnet/app/routes/forks.py | 22 +- src/decompnet/app/routes/life.py | 4 +- src/decompnet/app/routes/projects.py | 2 +- src/decompnet/app/routes/repair_feedback.py | 30 +-- src/decompnet/app/routes/repository_sync.py | 89 ++++---- src/decompnet/app/routes/rooms.py | 8 +- src/decompnet/app/routes/setup.py | 26 +-- src/decompnet/app/routes/state.py | 34 +-- src/decompnet/app/server.py | 6 +- src/decompnet/app/view_snapshots.py | 10 +- src/decompnet/app/web_github.py | 25 +- src/decompnet/app/web_login.py | 14 +- src/decompnet/hang_diagnostics.py | 55 +++++ src/decompnet/ui_diagnostics.py | 17 +- tests/test_server_lock_contention.py | 238 ++++++++++++++++++++ tests/test_web_server.py | 5 +- 20 files changed, 501 insertions(+), 131 deletions(-) create mode 100644 docs/SERVER_LOCK_AUDIT.md create mode 100644 src/decompnet/hang_diagnostics.py create mode 100644 tests/test_server_lock_contention.py diff --git a/docs/SERVER_LOCK_AUDIT.md b/docs/SERVER_LOCK_AUDIT.md new file mode 100644 index 000000000..fd8894427 --- /dev/null +++ b/docs/SERVER_LOCK_AUDIT.md @@ -0,0 +1,28 @@ +# Server responsiveness under contention + +The November 2026 live-process sample showed the ASGI main thread waiting for a Python lock while background workers continued. Two independent health requests timed out. Native sampling could not identify the precise Python lock owner; attaching a Python stack reader required unavailable elevated access. This audit fixes the independently reproducible blocking paths rather than assuming one lock explains every incident. + +## Audited boundaries + +| Area | Blocking risk | Treatment | +| --- | --- | --- | +| Repository setup, fork, push and sync | Job reservations acquired a thread RLock on the event loop; sync validation could enter progress import and database locks | Whole reservations run on one control worker, retaining atomic duplicate-job checks. Validation reads a progress snapshot; the job performs synchronization. | +| Import, restore, SDK/tool/agent installs and fork setup | `Jobs.start`, admission, logging and executor setup could wait synchronously | Job admission runs on reserved control workers. Actual work keeps its own job thread and progress/cancellation. | +| Repair replay/recovery | Duplicate checks and job creation held the job lock on the event loop | Each atomic reservation stays on a single worker. | +| Job cancellation and settings job discovery | Used the same generic worker capacity as long operations, or acquired a lock directly | Independent control capacity, separate from builds and UI snapshots. Large job JSON is encoded off the event loop. | +| Progress activity | Even an in-memory status call acquired the activity mutex | Activity status is read on a worker. Project GET routes retain existing snapshot-only contexts. | +| Browser bridge and login/OAuth | Capability/session locks and settings file locks could block HTTP dispatch | Locking operations are offloaded. Cookie authorization reads the current bounded session map without waiting for a writer, and still checks expiry. OAuth state consumption remains atomic. | +| Agent refresh, prompt previews, tool removal, background task admission | Synchronous helper calls could reach mutexes, subprocess or disk work | Offload at the request boundary. | +| Database/process writer gates | Waiting can be legitimate during transactions and must not be bypassed | Preserve writer ordering, transaction scopes, rollback and verification. Keep these operations off the event loop; a regression holds a real writer while serving health. | +| Read caches and map snapshots | Cache builders/consumers can wait for another worker; recursive snapshot waits could wait on themselves forever | Existing transaction cache bypass remains. Snapshot consumers have a bounded wait; recursive builders fail immediately. Successful builders remain valid after a consumer timeout. | +| Diagnostics | The stall logger itself could wait for a logging lock on the event loop; a completely blocked loop could not report itself | Log from workers. An independent heartbeat watchdog captures bounded thread code locations during a persistent stall. | +| Room host and MCP dispatch | Database, claim, compiler and member/auth operations | Checked the existing worker boundaries; synchronous room business logic and MCP tool/auth/discovery work already run on workers. Shared UI room admission now uses the control workers too. | +| Shutdown | Waiting for open requests can precede cleanup | Existing immediate signal-driven bounded shutdown remains. Do not force-release locks or commit a half-finished transaction. | + +## Diagnostics and regression gates + +A development or packaged runtime writes `.decomp/logs/event-loop-stacks.log` if its event loop misses heartbeats for five seconds. One bounded report is replaced at most every 30 seconds. It contains thread names and Python file/function/line locations, not locals, arguments, source lines, SQL bindings or request URLs. The watchdog stops with the server lifespan. + +Tests use one shared ASGI event loop, deliberately hold backend admission, job reservation, progress and database writer paths, and require concurrent `/health` responses. The database test checks that the delayed settings write commits and SQLite integrity remains valid. Separate tests cover reserved capacity, authorization expiry, recursive/bounded cache waits, watchdog capture and an AST gate against direct blocking lock/job/database admissions in asynchronous routes. + +Hardware exhaustion, the interpreter or a native extension can still prevent scheduling. This does not promise that every possible future hang is impossible. These boundaries prevent the reproduced backend lock waits from freezing the server and make another stall diagnosable without an elevated debugger. diff --git a/src/decompnet/app/cli.py b/src/decompnet/app/cli.py index b7a4d2c88..5ae4de0b4 100644 --- a/src/decompnet/app/cli.py +++ b/src/decompnet/app/cli.py @@ -231,7 +231,7 @@ def _start(args,home,internal,state_file,log,restarted,headless): mcp, manager, security, quit_app = build(net, internal, port, secret, **({"headless":True} if headless else {})) app = mcp.streamable_http_app(transport_security=security, max_request_body_size=REQUEST_BODY_LIMIT, host="127.0.0.1") from ..ui_diagnostics import Diagnostics - app = Diagnostics(app) + app = Diagnostics(app, stack_path=internal / "logs" / "event-loop-stacks.log") # with a tray icon, helpers kept working without the window have a way back to it window.IN_TRAY = False if headless else tray.start(window.open_app, quit_app) print(f"{edition.NAME} {__import__('decompnet').__version__}\nyour projects: {hostenv.shown_path(home)}" diff --git a/src/decompnet/app/routes/__init__.py b/src/decompnet/app/routes/__init__.py index f4a1dc582..957d4f3ed 100644 --- a/src/decompnet/app/routes/__init__.py +++ b/src/decompnet/app/routes/__init__.py @@ -49,6 +49,7 @@ class App: fail: Callable[..., Any] # fail(error, code=400): the error as JSON ui_call: Callable[..., Any] | None = None # reserved capacity for snapshot-only UI reads fast_call: Callable[..., Any] | None = None # lightweight counters, isolated from diff/map work + control_call: Callable[..., Any] | None = None # job admission/cancellation cannot queue behind builds updater: updates.Updater | None = None busy: Callable[[], bool] | None = None # whether helpers or a job are working quit_app: Callable[[], None] | None = None # quit: the window, every helper, the server @@ -58,6 +59,10 @@ class App: autopilot: Callable[[str], None] | None = None # state: start autopilot on a project with_project: Callable[..., Any] | None = None # projects: a project route from fn(slug, request, data) + async def control(self, fn, *args): + """Short locking operations use reserved workers, never the ASGI thread.""" + return await (self.control_call or self.call)(fn, *args) + async def read(self, fn, *args): """Read current state in a worker without waiting for saved-progress verification.""" def snapshot(): diff --git a/src/decompnet/app/routes/browser_bridge.py b/src/decompnet/app/routes/browser_bridge.py index 6c339c333..84d50fc09 100644 --- a/src/decompnet/app/routes/browser_bridge.py +++ b/src/decompnet/app/routes/browser_bridge.py @@ -34,7 +34,7 @@ async def authorize(request): body = await data(request) slug = str(body.get('slug') or '') if body.get('action') == 'revoke': - return JSONResponse(bridge.revoke(slug)) + return JSONResponse(await app.control(lambda: bridge.revoke(slug))) return JSONResponse(await app.call(bridge.create_code, slug, str(body.get('origin') or ''))) except (ValueError, NetworkError, OSError) as error: return app.fail(error) @@ -42,7 +42,7 @@ async def authorize(request): @app.mcp.custom_route('/api/browser-bridge/{action}', methods=['POST', 'OPTIONS']) async def browser(request): origin = request.headers.get('origin', '') - if request.headers.get('host') not in app.allowed_hosts or not bridge.allowed(origin): + if request.headers.get('host') not in app.allowed_hosts or not await app.control(lambda: bridge.allowed(origin)): return JSONResponse({'error':'This browser origin has not been paired in the desktop app.'}, status_code=403) headers = {'Access-Control-Allow-Origin':origin, 'Vary':'Origin', 'Cache-Control':'no-store', 'Access-Control-Allow-Methods':'POST, OPTIONS', 'Access-Control-Allow-Headers':'Authorization, Content-Type', @@ -53,11 +53,11 @@ async def browser(request): body = await data(request) action = request.path_params['action'] if action == 'pair': - return JSONResponse(bridge.pair(origin, body.get('code', '')), headers=headers) + return JSONResponse(await app.control(lambda: bridge.pair(origin, body.get('code', ''))), headers=headers) auth = request.headers.get('authorization', '') if not auth.startswith('Bearer '): raise PermissionError('Pair this browser first.') - key, session = bridge.auth(origin, auth[7:]) + key, session = await app.control(lambda: bridge.auth(origin, auth[7:])) if action == 'functions': result = await app.call(bridge.listing, session, body.get('search', '')) elif action == 'task': @@ -73,9 +73,9 @@ async def browser(request): variant_id=body.get('variantId') if variant_id is not None and (not isinstance(variant_id, str) or len(variant_id)>64): raise ValueError('Choose a host compiler variation.') - task = bridge.consume(key, session, str(body.get('taskId') or '')) + task = await app.control(lambda: bridge.consume(key, session, str(body.get('taskId') or ''))) try: - jid = app.jobs.start('browser-check', lambda log: bridge.submit(key, session, task, source, variant_id)) + jid = await app.control(lambda: app.jobs.start('browser-check', lambda log: bridge.submit(key, session, task, source, variant_id))) except Exception: session['busy'] = False raise diff --git a/src/decompnet/app/routes/forks.py b/src/decompnet/app/routes/forks.py index 62b22fd41..0b9638ca9 100644 --- a/src/decompnet/app/routes/forks.py +++ b/src/decompnet/app/routes/forks.py @@ -46,7 +46,7 @@ def work(log): settings.data["last_project"] = slug settings.save() return {"slug": slug} - return JSONResponse({"job": jobs.start("fork", work)}) + return JSONResponse({"job": await app.control(lambda: jobs.start("fork", work))}) @mcp.custom_route("/api/fork/local/scan", methods=["POST"]) async def api_local_scan(request: Request) -> Response: @@ -61,7 +61,7 @@ def work(log): result = net.forks.scan_local(hostenv.local_path(folder)) log(f"Found {len(result['projects'])} repositories.") return result - return JSONResponse({"job": jobs.start("local-scan", work)}) + return JSONResponse({"job": await app.control(lambda: jobs.start("local-scan", work))}) @mcp.custom_route("/api/fork/local/import", methods=["POST"]) async def api_local_import(request: Request) -> Response: @@ -75,7 +75,7 @@ async def api_local_import(request: Request) -> Response: return fail(ValueError("Choose the folder and at least one repository from its scan.")) def work(log): return net.forks.import_local(hostenv.local_path(folder), paths, copies=data.get("copies") is not False, log=log) - return JSONResponse({"job": jobs.start("local-import", work)}) + return JSONResponse({"job": await app.control(lambda: jobs.start("local-import", work))}) def fork_of(slug: str): if not net.fork_slug(slug): @@ -118,7 +118,7 @@ def work(log): settings.save() autopilot(active) return res - return JSONResponse({"job": jobs.start("fork-setup", work)}) + return JSONResponse({"job": await app.control(lambda: jobs.start("fork-setup", work))}) @mcp.custom_route("/api/fork/find-rom", methods=["POST"]) async def api_fork_find_rom(request: Request) -> Response: @@ -135,7 +135,7 @@ def work(log): _, project, _ = net.forks.get(slug) return find(project, net.forks.needs(slug), str(settings.data.get("rom_library") or ""), log) - return JSONResponse({"job": jobs.start("find-rom", work)}) + return JSONResponse({"job": await app.control(lambda: jobs.start("find-rom", work))}) @mcp.custom_route("/api/fork/setup", methods=["POST"]) async def api_fork_setup(request: Request) -> Response: @@ -163,7 +163,7 @@ def work(log): settings.save() autopilot(active) return res - return JSONResponse({"job": jobs.start("fork-setup", work)}) + return JSONResponse({"job": await app.control(lambda: jobs.start("fork-setup", work))}) @mcp.custom_route("/api/fork/check", methods=["POST"]) async def api_fork_check(request: Request) -> Response: @@ -176,7 +176,7 @@ async def api_fork_check(request: Request) -> Response: await app.read(fork_of, slug) except NetworkError as e: return fail(e) - return JSONResponse({"job": jobs.start("fork-check", lambda log: net.forks.check(slug, log=log, build=True))}) + return JSONResponse({"job": await app.control(lambda: jobs.start("fork-check", lambda log: net.forks.check(slug, log=log, build=True)))}) @mcp.custom_route("/api/fork/fix-setup", methods=["GET", "POST"]) async def api_fork_fix_setup(request: Request) -> Response: @@ -194,8 +194,8 @@ async def api_fork_fix_setup(request: Request) -> Response: folder = Path(proj["root"]) except (NetworkError, ValueError) as e: return fail(e) - return JSONResponse({"job": jobs.start("fix-setup", lambda log: setup_fix.launch( - app.home, folder, str(data.get("error") or ""), kind, log))}) + return JSONResponse({"job": await app.control(lambda: jobs.start("fix-setup", lambda log: setup_fix.launch( + app.home, folder, str(data.get("error") or ""), kind, log)))}) @mcp.custom_route("/api/fork/rom-check", methods=["POST"]) async def api_fork_rom_check(request: Request) -> Response: @@ -207,7 +207,7 @@ async def api_fork_rom_check(request: Request) -> Response: await app.read(fork_of, slug) except NetworkError as e: return fail(e) - return JSONResponse({"job": jobs.start("rom-check", lambda log: net.forks.check_rom(slug, log=log))}) + return JSONResponse({"job": await app.control(lambda: jobs.start("rom-check", lambda log: net.forks.check_rom(slug, log=log)))}) @mcp.custom_route("/api/fork/start-files", methods=["POST"]) async def api_fork_start_files(request: Request) -> Response: @@ -219,6 +219,6 @@ async def api_fork_start_files(request: Request) -> Response: await app.read(fork_of, slug) except NetworkError as e: return fail(e) - return JSONResponse({"job": jobs.start("fork-start-files", lambda log: net.forks.start_files(slug, log=log))}) + return JSONResponse({"job": await app.control(lambda: jobs.start("fork-start-files", lambda log: net.forks.start_files(slug, log=log)))}) mcp.custom_route("/api/fork/needs", methods=["GET"])(with_project(lambda s, r, d: net.forks.needs(fork_of(s)["slug"]))) diff --git a/src/decompnet/app/routes/life.py b/src/decompnet/app/routes/life.py index 1ad0cf404..4043ec9f6 100644 --- a/src/decompnet/app/routes/life.py +++ b/src/decompnet/app/routes/life.py @@ -148,8 +148,8 @@ async def api_log(request: Request) -> Response: if (bad := guard(request, write=True)): return bad data = await body(request) - page_log.warning("page: %s (%s)\n%s", str(data.get("message", ""))[:PAGE_ERROR_CHARS], - str(data.get("where", ""))[:PAGE_ERROR_CHARS], str(data.get("stack", ""))[:PAGE_STACK_CHARS]) + await app.control(lambda: page_log.warning("page: %s (%s)\n%s", str(data.get("message", ""))[:PAGE_ERROR_CHARS], + str(data.get("where", ""))[:PAGE_ERROR_CHARS], str(data.get("stack", ""))[:PAGE_STACK_CHARS])) return JSONResponse({"ok": True}) @mcp.custom_route("/api/quit", methods=["POST"]) diff --git a/src/decompnet/app/routes/projects.py b/src/decompnet/app/routes/projects.py index 48e4afcbb..9734f9e3a 100644 --- a/src/decompnet/app/routes/projects.py +++ b/src/decompnet/app/routes/projects.py @@ -88,7 +88,7 @@ def respond(): path = request.url.path if request.method == 'GET' and elapsed >= 0.5 and started - slow_requests.get(path, float('-inf')) >= 60: slow_requests[path] = started - jobs.logger.warning('Slow UI request %s %.3fs (includes worker queue and response encoding)', path, elapsed) + await app.control(lambda: jobs.logger.warning('Slow UI request %s %.3fs (includes worker queue and response encoding)', path, elapsed)) return handler def g_boundary_evidence(slug, request, data): diff --git a/src/decompnet/app/routes/repair_feedback.py b/src/decompnet/app/routes/repair_feedback.py index 831eb8b53..a4ac1452b 100644 --- a/src/decompnet/app/routes/repair_feedback.py +++ b/src/decompnet/app/routes/repair_feedback.py @@ -83,12 +83,14 @@ def select_examples(): ids = await app.read(select_examples) if not ids:raise NetworkError('No examples match these filters.') - with app.jobs._lock: - if any(j['kind']=='repair-replay' and j['state']=='running' for j in app.jobs.jobs.values()): - raise NetworkError('A repair replay is already running. Stop it or wait before starting another.') - job=app.jobs.start('repair-replay',lambda log:repair_replay_jobs.run(feedback,slug,ids,trials,seconds,log,minimize=minimize,diagnostics=diagnostics,combinations=combinations), - cancel_message='Repair replay stopped. No game progress or review states changed.') - return JSONResponse({'job':job}) + def start_replay(): + with app.jobs._lock: + if any(j['kind']=='repair-replay' and j['state']=='running' for j in app.jobs.jobs.values()): + raise NetworkError('A repair replay is already running. Stop it or wait before starting another.') + job=app.jobs.start('repair-replay',lambda log:repair_replay_jobs.run(feedback,slug,ids,trials,seconds,log,minimize=minimize,diagnostics=diagnostics,combinations=combinations), + cancel_message='Repair replay stopped. No game progress or review states changed.') + return JSONResponse({'job':job}) + return await app.control(start_replay) except (NetworkError,ValueError,KeyError,TypeError) as error: return app.fail(error) @@ -140,12 +142,14 @@ async def recovery(request): retry_failed = data.get('retry_failed', False) if not isinstance(retry_failed, bool): raise NetworkError('Retry failed measurements must be true or false.') - with app.jobs._lock: - if any(j['kind'] == 'candidate-recovery' and j['state'] == 'running' for j in app.jobs.jobs.values()): - raise NetworkError('A recovery batch is running. Stop it or wait before starting another.') - job = app.jobs.start('candidate-recovery', lambda log: candidate_rechecks.run(app.net, slug, limit, log, retry_failed), - cancel_message='Recovery stopped. Finished measurements are saved; selected C is unchanged.') - return JSONResponse({'job':job}) + def start_recovery(): + with app.jobs._lock: + if any(j['kind'] == 'candidate-recovery' and j['state'] == 'running' for j in app.jobs.jobs.values()): + raise NetworkError('A recovery batch is running. Stop it or wait before starting another.') + job = app.jobs.start('candidate-recovery', lambda log: candidate_rechecks.run(app.net, slug, limit, log, retry_failed), + cancel_message='Recovery stopped. Finished measurements are saved; selected C is unchanged.') + return JSONResponse({'job':job}) + return await app.control(start_recovery) except (NetworkError, ValueError, KeyError, TypeError) as error: return app.fail(error) @@ -198,7 +202,7 @@ async def investigate(request): kind = str(data.get('agent') or '') if not any(a['id'] == kind and a['installed'] for a in await app.call(repair_investigations.choices)): raise NetworkError('Choose an installed coding CLI for this investigation.') - job = app.jobs.start('repair investigation', lambda log: repair_investigations.launch(app.home, example, kind, log)) + job = await app.control(lambda: app.jobs.start('repair investigation', lambda log: repair_investigations.launch(app.home, example, kind, log))) return JSONResponse({'job': job}) except (NetworkError, ValueError, KeyError, TypeError) as error: return app.fail(error) diff --git a/src/decompnet/app/routes/repository_sync.py b/src/decompnet/app/routes/repository_sync.py index 4071adcf9..55035c7e3 100644 --- a/src/decompnet/app/routes/repository_sync.py +++ b/src/decompnet/app/routes/repository_sync.py @@ -30,20 +30,25 @@ async def setup(request): from .. import repository_setup if request.method == 'GET': slug = str(request.query_params.get('slug', '')) - with app.jobs._lock: - if job := active_setup(slug):return JSONResponse({'job':job}) + def pending_setup(): + with app.jobs._lock: + if job := active_setup(slug):return JSONResponse({'job':job}) + if (reserved := await app.control(pending_setup)) is not None: + return reserved return JSONResponse(await (app.fast_call or app.ui_call or app.call)(repository_setup.status, app.net, slug)) data = await app.body(request) slug = str(data.get('slug', '')) - with app.jobs._lock: - if job := active_setup(slug):return JSONResponse({'job':job}) - job = app.jobs.start('repository-setup', lambda log: repository_setup.run( - app.net, slug, str(data.get('action', '')), str(data.get('origin', '')).strip(), log, - github_name=str(data.get('github_name', '')).strip(), visibility=str(data.get('visibility', 'private'))), - cancel_message='Repository setup stopped. A created GitHub repository, Git metadata or completed commit may remain; check the log before retrying.') - app.jobs.jobs[job]['scope'] = {'slug':slug} - setups[slug] = job - return JSONResponse({'job': job}) + def start_setup(): + with app.jobs._lock: + if job := active_setup(slug):return JSONResponse({'job':job}) + job = app.jobs.start('repository-setup', lambda log: repository_setup.run( + app.net, slug, str(data.get('action', '')), str(data.get('origin', '')).strip(), log, + github_name=str(data.get('github_name', '')).strip(), visibility=str(data.get('visibility', 'private'))), + cancel_message='Repository setup stopped. A created GitHub repository, Git metadata or completed commit may remain; check the log before retrying.') + app.jobs.jobs[job]['scope'] = {'slug':slug} + setups[slug] = job + return JSONResponse({'job': job}) + return await app.control(start_setup) except (NetworkError, ValueError, KeyError, OSError) as error: return app.fail(error) @@ -69,18 +74,21 @@ async def fork(request): from .. import repository_fork data = await app.body(request) if request.method == 'POST' else request.query_params slug, folder = str(data.get('slug', '')), str(data.get('folder', '')).strip() - with app.jobs._lock: - active = active_fork(slug) - if active: - if active['scope']['folder'] != folder:raise NetworkError('Fork setup is already running for another checkout of this project.') - return JSONResponse({'job':active['id']}) - if request.method == 'POST': - job = app.jobs.start('repository-fork', lambda log:repository_fork.run(app.net,slug,folder, - str(data.get('owner','')).strip(),str(data.get('origin_revision','')),log), project=slug, - cancel_message='Fork setup stopped. A GitHub fork or completed remote change may remain. Reopen setup to check; no commits were pushed.') - app.jobs.jobs[job]['scope'] = {'slug':slug,'folder':folder} - forks[slug] = job - return JSONResponse({'job':job}) + def reserve_fork(): + with app.jobs._lock: + active = active_fork(slug) + if active: + if active['scope']['folder'] != folder:raise NetworkError('Fork setup is already running for another checkout of this project.') + return JSONResponse({'job':active['id']}) + if request.method == 'POST': + job = app.jobs.start('repository-fork', lambda log:repository_fork.run(app.net,slug,folder, + str(data.get('owner','')).strip(),str(data.get('origin_revision','')),log), project=slug, + cancel_message='Fork setup stopped. A GitHub fork or completed remote change may remain. Reopen setup to check; no commits were pushed.') + app.jobs.jobs[job]['scope'] = {'slug':slug,'folder':folder} + forks[slug] = job + return JSONResponse({'job':job}) + if (reserved := await app.control(reserve_fork)) is not None: + return reserved return JSONResponse(await app.call(repository_fork.info,app.net,slug,folder)) except (NetworkError,ValueError,KeyError,OSError) as error:return app.fail(error) @@ -126,11 +134,14 @@ async def push(request): try: from .. import repository_push if request.method == 'GET': - with app.jobs._lock: - setup = active_fork(str(request.query_params.get('slug', ''))) - if setup and setup['scope']['folder'] == str(request.query_params.get('folder', '')).strip(): - return JSONResponse({'fork_job':setup['id']}) - active = active_push(str(request.query_params.get('slug', ''))) + def pending_fork(): + with app.jobs._lock: + setup = active_fork(str(request.query_params.get('slug', ''))) + if setup and setup['scope']['folder'] == str(request.query_params.get('folder', '')).strip(): + return JSONResponse({'fork_job':setup['id']}) + if (reserved := await app.control(pending_fork)) is not None: + return reserved + active = await app.control(lambda: active_push(str(request.query_params.get('slug', '')))) if active: return JSONResponse({'job':active['id'], 'progress':active.get('progress', {}), **active['scope']}) return JSONResponse(await app.call(repository_push.status, app.net, @@ -140,14 +151,16 @@ async def push(request): folder = str(data.get('folder', '')).strip() # Reserve by project under the job lock; repeated clicks/windows attach # to the same queued push without blocking a sync's API reservation. - with app.jobs._lock: - active = active_push(slug) - if active:return JSONResponse({'job':active['id']}) - job = app.jobs.start('repository-push', lambda log: repository_push.run(app.net, slug, folder, log), - cancel_message='Push stopped. Waiting requests are removed from the queue; check origin if pushing had already started.') - app.jobs.jobs[job]['scope'] = {'slug':slug, 'folder':folder} - pushes[slug] = job - return JSONResponse({'job':job}) + def start_push(): + with app.jobs._lock: + active = active_push(slug) + if active:return JSONResponse({'job':active['id']}) + job = app.jobs.start('repository-push', lambda log: repository_push.run(app.net, slug, folder, log), + cancel_message='Push stopped. Waiting requests are removed from the queue; check origin if pushing had already started.') + app.jobs.jobs[job]['scope'] = {'slug':slug, 'folder':folder} + pushes[slug] = job + return JSONResponse({'job':job}) + return await app.control(start_push) except (NetworkError, ValueError, KeyError, OSError) as error: return app.fail(error) @@ -162,7 +175,7 @@ async def sync(request): slug=str(data.get('slug',''));folder=str(data.get('folder','')).strip() commit_push = data.get('commit_push', False) if not isinstance(commit_push, bool):raise NetworkError('Choose whether to commit and push with a boolean value.') - app.net.project_row(slug) + await app.read(app.net.project_row, slug) if not folder:raise NetworkError('Choose the root folder of your Git checkout.') # Reserve before starting a job; a second click must not launch another build. if not lock.acquire(blocking=False):raise NetworkError('A repository sync is already running.') @@ -172,7 +185,7 @@ def work(log): return repository_sync.run(app.net,slug,folder,log) finally:lock.release() try: - job=app.jobs.start('repository-sync',work,cancel_message='Repository sync stopped. Any published files or local commit remain in the checkout; check Git status before retrying.') + job=await app.control(lambda: app.jobs.start('repository-sync',work,cancel_message='Repository sync stopped. Any published files or local commit remain in the checkout; check Git status before retrying.')) except BaseException: lock.release();raise return JSONResponse({'job':job}) diff --git a/src/decompnet/app/routes/rooms.py b/src/decompnet/app/routes/rooms.py index 43e927686..24c06767a 100644 --- a/src/decompnet/app/routes/rooms.py +++ b/src/decompnet/app/routes/rooms.py @@ -212,7 +212,7 @@ def work(log): log.progress({'finished':len(result['results']),'total':result['total'], 'phase':'Saving verified room matches', 'results':result['results']}) return result - return JSONResponse({'job':jobs.start('room match verification',work)}) + return JSONResponse({'job':await app.control(lambda: jobs.start('room match verification',work))}) except (NetworkError,ValueError) as error:return fail(error) @mcp.custom_route("/api/collab/start", methods=["POST"]) @@ -232,7 +232,7 @@ def work(log): room.configure(data.get("visibility"), data.get("approve")) return {"link": open_room(slug, mode, str(data.get("address", "")), int(data.get("port") or 6481), log=log, keep=bool(data.get("keep")))} - return JSONResponse({"job": jobs.start("share", work)}) + return JSONResponse({"job": await app.control(lambda: jobs.start("share", work))}) @mcp.custom_route("/api/collab/manage", methods=["POST"]) async def api_collab_manage(request: Request) -> Response: @@ -331,7 +331,7 @@ def work(log): settings.data["last_project"] = slug settings.save() return {"slug": slug} - return JSONResponse({"job": jobs.start("join", work)}) + return JSONResponse({"job": await app.control(lambda: jobs.start("join", work))}) @mcp.custom_route("/api/collab/reconnect", methods=["POST"]) async def api_collab_reconnect(request: Request) -> Response: @@ -339,7 +339,7 @@ async def api_collab_reconnect(request: Request) -> Response: return bad data = await body(request) slug, link = str(data.get("slug", "")), str(data.get("link", "")) - return JSONResponse({"job": jobs.start("reconnect", lambda log: {"slug": collab.reconnect(net, slug, link, log=log)})}) + return JSONResponse({"job": await app.control(lambda: jobs.start("reconnect", lambda log: {"slug": collab.reconnect(net, slug, link, log=log)}))}) @mcp.custom_route("/api/collab/leave", methods=["POST"]) async def api_collab_leave(request: Request) -> Response: diff --git a/src/decompnet/app/routes/setup.py b/src/decompnet/app/routes/setup.py index 0ae4d8cf5..ec77f3947 100644 --- a/src/decompnet/app/routes/setup.py +++ b/src/decompnet/app/routes/setup.py @@ -40,10 +40,10 @@ async def api_sdk_library(request: Request) -> Response: if data.get('slug'): await call(lambda: sdk_library.choose(net, str(data['slug']), data.get('selection'))) from .. import purpose_work, sdk_names - purpose_work.start('SDK function naming', str(data['slug']), sdk_names.name_functions, net, str(data['slug'])) + await app.control(lambda: purpose_work.start('SDK function naming', str(data['slug']), sdk_names.name_functions, net, str(data['slug']))) return JSONResponse(await call(lambda: sdk_library.state(net, str(data['slug'])))) folders = data.get('folders', []) - return JSONResponse({'job': jobs.start('sdk-library', lambda log: sdk_library.scan(net, folders, log))}) + return JSONResponse({'job': await app.control(lambda: jobs.start('sdk-library', lambda log: sdk_library.scan(net, folders, log)))}) except (NetworkError, ValueError, OSError) as e: return fail(e) @@ -51,7 +51,7 @@ async def api_sdk_library(request: Request) -> Response: async def api_linux_vm(request: Request) -> Response: if (bad := guard(request, write=True)): return bad - return JSONResponse({'job': jobs.start('linux-vm', lambda log: linuxvm.setup(log=log))}) + return JSONResponse({'job': await app.control(lambda: jobs.start('linux-vm', lambda log: linuxvm.setup(log=log)))}) @mcp.custom_route("/api/toolchain", methods=["POST"]) async def api_toolchain(request: Request) -> Response: @@ -62,8 +62,8 @@ async def api_toolchain(request: Request) -> Response: folder = str(data.get("folder") or "").strip() or None if folder: folder = hostenv.local_path(folder) - return JSONResponse({"job": jobs.start("toolchain", lambda log: (toolchains.install(tid, accept_license=accept, force=bool(data.get("force")), folder=folder, log=log) - and {"id": tid}))}) + return JSONResponse({"job": await app.control(lambda: jobs.start("toolchain", lambda log: (toolchains.install(tid, accept_license=accept, force=bool(data.get("force")), folder=folder, log=log) + and {"id": tid})))}) @mcp.custom_route("/api/toolchain/remove", methods=["POST"]) async def api_toolchain_remove(request: Request) -> Response: @@ -72,7 +72,7 @@ async def api_toolchain_remove(request: Request) -> Response: return bad data = await body(request) try: - toolchains.remove_local(str(data.get("id", "")), str(data.get("folder", ""))) + await app.control(lambda: toolchains.remove_local(str(data.get("id", "")), str(data.get("folder", "")))) except (toolchains.ToolchainError, KeyError) as e: return fail(e) return JSONResponse({"ok": True}) @@ -84,7 +84,7 @@ async def api_toolchain_all(request: Request) -> Response: return bad data = await body(request) slug, accept = str(data.get("slug", "")), bool(data.get("accept_license")) - return JSONResponse({"job": jobs.start("toolchain", lambda log: net.install_compilers(slug, accept, log=log))}) + return JSONResponse({"job": await app.control(lambda: jobs.start("toolchain", lambda log: net.install_compilers(slug, accept, log=log)))}) @mcp.custom_route("/api/agents/install", methods=["POST"]) async def api_agent_install(request: Request) -> Response: @@ -112,7 +112,7 @@ def work(log): kinds.known_versions.pop(kind.id, None) manager.refresh_detected(wait=True) # the launcher shows it, with its models, when the job is done return {"kind": kind.id} - return JSONResponse({"job": jobs.start("agent-install", work)}) + return JSONResponse({"job": await app.control(lambda: jobs.start("agent-install", work))}) @mcp.custom_route("/api/agents/detect", methods=["POST"]) async def api_agents_detect(request: Request) -> Response: @@ -121,8 +121,8 @@ async def api_agents_detect(request: Request) -> Response: return bad from decompnet.agents import AGENTS for agent in AGENTS: - agent.forget_models() - manager.refresh_detected() + await app.control(agent.forget_models) + await app.control(manager.refresh_detected) return JSONResponse({"detecting": True}) @mcp.custom_route("/api/endpoints/models", methods=["GET"]) @@ -170,7 +170,7 @@ def create(log): finally: agents.start_progress.reset(token) try: - return JSONResponse({'job':jobs.start('Create agents',create)}) + return JSONResponse({'job':await app.control(lambda: jobs.start('Create agents',create))}) except ValueError as error: return fail(error) # Legacy callers keep the immediate response, but preparation must never block the event loop. @@ -323,8 +323,8 @@ async def api_agents_retry_paused(request: Request) -> Response: data=await body(request) try: if data.get('background'): - return JSONResponse({'job':jobs.start('Retry paused functions', - lambda log: manager.retry_paused(str(data.get('id','')),data.get('limit',5),log=log))}) + return JSONResponse({'job':await app.control(lambda: jobs.start('Retry paused functions', + lambda log: manager.retry_paused(str(data.get('id','')),data.get('limit',5),log=log)))}) result=await call(manager.retry_paused,str(data.get('id','')),data.get('limit',5)) except (NetworkError,ValueError) as error:return fail(error) return JSONResponse(result) diff --git a/src/decompnet/app/routes/state.py b/src/decompnet/app/routes/state.py index d053b9809..a4a3cccb2 100644 --- a/src/decompnet/app/routes/state.py +++ b/src/decompnet/app/routes/state.py @@ -52,10 +52,10 @@ async def api_match_library(request): if data.get('action') in ('name','apply-name'): fid=int(data['id']);name=await call(net.match_library.set_name,fid,data.get('name','')) if data['action']=='apply-name': - return JSONResponse({'job':jobs.start('Apply library function name', - lambda log:net.match_library.apply_name(fid,name,log))}) + return JSONResponse({'job':await app.control(lambda: jobs.start('Apply library function name', + lambda log:net.match_library.apply_name(fid,name,log)))}) return JSONResponse({'name':name}) - return JSONResponse({'job':jobs.start('Index verified match library', net.match_library.backfill)}) + return JSONResponse({'job':await app.control(lambda: jobs.start('Index verified match library', net.match_library.backfill))}) if request.query_params.get('id'): return JSONResponse(await call(net.match_library.detail,int(request.query_params['id']))) return JSONResponse(await call(net.match_library.summary, str(request.query_params.get('search',''))[:200])) @@ -69,9 +69,9 @@ async def api_settings_jobs(request): # Discover work before loading statistics: a busy/compacting database must # not hide a job which the user can still inspect or stop. return JSONResponse({ - 'storage':jobs.active(('Compress candidate history','Clean old unsuccessful source')), - 'headers':jobs.active(('C header cleanup','Database compaction')), - 'cleanup':jobs.active(('Database transcript cleanup',)), + 'storage':await app.control(lambda: jobs.active(('Compress candidate history','Clean old unsuccessful source'))), + 'headers':await app.control(lambda: jobs.active(('C header cleanup','Database compaction'))), + 'cleanup':await app.control(lambda: jobs.active(('Database transcript cleanup',))), }) @mcp.custom_route('/api/database-cleanup', methods=['GET','POST']) @@ -90,7 +90,7 @@ async def api_database_cleanup(request): return JSONResponse({'options':await call(database_cleanup.save,net,value)}) if data.get('action')!='start':raise ValueError('Choose preview, save or start for database cleanup.') await call(database_cleanup.save,net,value) - return JSONResponse({'job':database_cleanup.start(net,jobs,value)}) + return JSONResponse({'job':await app.control(lambda: database_cleanup.start(net,jobs,value))}) except (ValueError,NetworkError) as error:return fail(error) @mcp.custom_route('/api/database-storage', methods=['GET','POST']) @@ -106,14 +106,14 @@ async def api_database_storage(request): if data.get('action')=='save':return JSONResponse({'retention':await call(database_storage.save,net,data)}) if data.get('action') not in ('pack','prune'):raise ValueError('Choose pack, prune or save.') if data['action']=='prune':database_storage.options(data) - return JSONResponse({'job':database_storage.start(net,jobs,data['action'],data)}) + return JSONResponse({'job':await app.control(lambda: database_storage.start(net,jobs,data['action'],data))}) except (NetworkError,ValueError) as error:return fail(error) @mcp.custom_route('/api/performance', methods=['GET']) async def api_performance(request): if bad := guard(request):return bad from ...ui_diagnostics import snapshot - return JSONResponse({'slow_operations':snapshot()}) + return JSONResponse({'slow_operations':await app.control(snapshot)}) @mcp.custom_route('/api/header-storage', methods=['GET','POST']) async def api_header_storage(request): @@ -128,7 +128,7 @@ async def api_header_storage(request): if data.get('action')=='preview': result,_=await call(header_cleanup.summary,net) return JSONResponse(result) - return JSONResponse({'job':header_cleanup.start(net,jobs,data.get('action'))}) + return JSONResponse({'job':await app.control(lambda: header_cleanup.start(net,jobs,data.get('action')))}) except (ValueError,NetworkError) as error:return fail(error) def state() -> dict: @@ -214,7 +214,7 @@ async def agent_prompts(request: Request) -> Response: if helper: draft = replace(helper, options={**helper.options,'prompts':overrides}, prompt_config=None) return JSONResponse({'initial':await call(manager.prompt, draft), - 'instructions':manager.app_instructions(draft)}) + 'instructions':await app.control(lambda: manager.app_instructions(draft))}) templates = prompt_settings.defaults() | overrides return JSONResponse({'initial':'\n\n'.join(prompt_settings.LABELS[k]+'\n'+templates[k] for k in prompt_settings.FIELDS if k != 'instructions'), @@ -318,7 +318,7 @@ async def api_restore(request: Request) -> Response: game = hostenv.local_path(path) if path else None if not source or (game is not None and not await app.call(lambda: game.is_file())): return fail(ValueError("Choose the saved project and a valid game file, or leave the game field blank to find a matching local copy.")) - return JSONResponse({"job": jobs.start("restore", lambda log: saves.restore(net, source, game, log=log))}) + return JSONResponse({"job": await app.control(lambda: jobs.start("restore", lambda log: saves.restore(net, source, game, log=log)))}) @mcp.custom_route("/api/pick", methods=["POST"]) async def api_pick(request: Request) -> Response: @@ -370,7 +370,7 @@ async def api_import(request: Request) -> Response: if (bad := guard(request, write=True)): return bad try: - jobs.check_start() + await app.control(lambda: jobs.check_start()) except ValueError as e: return fail(e, 503) q = request.query_params @@ -437,7 +437,7 @@ def work(log): autopilot(slug) return {"slug": slug} try: - return JSONResponse({"job": jobs.start("import", work)}) + return JSONResponse({"job": await app.control(lambda: jobs.start("import", work))}) except ValueError as e: # A source update can queue while a disc is uploading, before it becomes a job. if uploaded: @@ -470,7 +470,7 @@ async def api_project_loading(request: Request) -> Response: if (bad := guard(request)): return bad # In-memory activity only: never start a sync or queue behind expensive UI reads. - return JSONResponse({'verification':net.progress.status(request.query_params.get('slug',''))}) + return JSONResponse({'verification':await app.control(lambda: net.progress.status(request.query_params.get('slug','')))}) @mcp.custom_route("/api/map", methods=["GET"]) async def api_map(request: Request) -> Response: @@ -558,13 +558,13 @@ async def api_job(request: Request) -> Response: if (bad := guard(request)): return bad job = jobs.jobs.get(request.query_params.get("id", "")) - return JSONResponse(job) if job else fail(ValueError("no such job"), 404) + return await app.control(lambda: JSONResponse(job)) if job else fail(ValueError("no such job"), 404) @mcp.custom_route("/api/job/cancel", methods=["POST"]) async def api_cancel_job(request: Request) -> Response: if (bad := guard(request, write=True)): return bad data = await body(request) - return JSONResponse({"cancelled": jobs.cancel(str(data.get("id") or ""))}) + return JSONResponse({"cancelled": await app.control(lambda: jobs.cancel(str(data.get("id") or "")))}) app.autopilot = autopilot diff --git a/src/decompnet/app/server.py b/src/decompnet/app/server.py index 2147c398d..314dd37c7 100644 --- a/src/decompnet/app/server.py +++ b/src/decompnet/app/server.py @@ -295,6 +295,7 @@ async def body(request: Request) -> dict: ui_limiter = anyio.CapacityLimiter(4) fast_limiter = anyio.CapacityLimiter(2) + control_limiter = anyio.CapacityLimiter(4) async def ui_call(fn, *args): def snapshot(): @@ -309,6 +310,9 @@ def snapshot(): return fn(*args) return await anyio.to_thread.run_sync(snapshot, limiter=fast_limiter) + async def control_call(fn, *args): + return await anyio.to_thread.run_sync(fn, *args, limiter=control_limiter) + async def call(fn, *args): return await anyio.to_thread.run_sync(fn, *args) @@ -328,7 +332,7 @@ def before_restart() -> None: # the app's parts add their routes (routes/__init__.py says which is where); forks come after state and projects, # whose autopilot and with_project they use app = routes.App(net, mcp, home, port, secret, mcp_url, build_id, manager, jobs, settings, allowed_hosts, - guard, body, call, fail, ui_call=ui_call, fast_call=fast_call, updater=updates.Updater(before_restart, port, secret), + guard, body, call, fail, ui_call=ui_call, fast_call=fast_call, control_call=control_call, updater=updates.Updater(before_restart, port, secret), busy=busy, quit_app=quit_app, web_login=net.web_login, headless=headless) from . import web_login web_login.register(app) diff --git a/src/decompnet/app/view_snapshots.py b/src/decompnet/app/view_snapshots.py index 1b240277a..e84600a15 100644 --- a/src/decompnet/app/view_snapshots.py +++ b/src/decompnet/app/view_snapshots.py @@ -6,6 +6,8 @@ from collections import OrderedDict from concurrent.futures import Future +SNAPSHOT_WAIT_SECONDS = 10 + class MapPayload(dict): """A cached immutable-by-convention map has a unique wire-cache identity.""" @@ -33,8 +35,14 @@ def get(self, key, build, weight=lambda value: 1, ttl=30): owner = future is None if owner: future = self.pending[key] = Future() + future.builder_thread = threading.get_ident() if not owner: - return future.result() + if future.builder_thread == threading.get_ident(): + raise RuntimeError('A project snapshot cannot wait for its own builder.') + try: + return future.result(timeout=SNAPSHOT_WAIT_SECONDS) + except TimeoutError: + raise TimeoutError('The project view is still being prepared. Try again shortly.') from None try: value = build() cost = max(1, weight(value)) diff --git a/src/decompnet/app/web_github.py b/src/decompnet/app/web_github.py index 0d2f1fc59..9396acdd3 100644 --- a/src/decompnet/app/web_github.py +++ b/src/decompnet/app/web_github.py @@ -105,7 +105,7 @@ async def settings(request): config.update(client_id=client_id,client_secret=secret,login=user['login'],user_id=user['id']) config['enabled'] = data['enabled'] app.settings.data['web_github'] = config - app.settings.save() + await app.control(app.settings.save) # Revoke previous sessions and pending authorizations after an identity policy change. if app.web_login: app.web_login = app.net.web_login = WebLogin(app.port,app.settings.data.get('web_remote_url','')) @@ -125,12 +125,15 @@ async def start(request): login,config = app.web_login,dict(configuration(app)) state,binding,verifier = secrets.token_urlsafe(32),secrets.token_urlsafe(32),secrets.token_urlsafe(48) now = time.monotonic() - with login.lock: - login.oauth_states = {key:value for key,value in login.oauth_states.items() if value['expires']>now} - if len(login.oauth_states)>=MAX_PENDING: - return failure('Too many pending sign-ins. Wait a few minutes and try again.',429) - login.oauth_states[state] = {'binding':hashlib.sha256(binding.encode()).hexdigest(), - 'verifier':verifier,'callback':callback,'expires':now+OAUTH_SECONDS,'config':config} + def reserve_state(): + with login.lock: + login.oauth_states = {key:value for key,value in login.oauth_states.items() if value['expires']>now} + if len(login.oauth_states)>=MAX_PENDING: + return failure('Too many pending sign-ins. Wait a few minutes and try again.',429) + login.oauth_states[state] = {'binding':hashlib.sha256(binding.encode()).hexdigest(), + 'verifier':verifier,'callback':callback,'expires':now+OAUTH_SECONDS,'config':config} + if (failure_response := await app.control(reserve_state)) is not None: + return failure_response challenge = base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).decode().rstrip('=') query = urlencode({'client_id':config['client_id'],'redirect_uri':callback,'scope':'', 'state':state,'code_challenge':challenge,'code_challenge_method':'S256', @@ -147,8 +150,10 @@ async def callback(request): login = app.web_login state = request.query_params.get('state','') binding = request.cookies.get(f'decomp_oauth_{app.port}','') - with login.lock: - pending = login.oauth_states.pop(state,None) + def consume_state(): + with login.lock: + return login.oauth_states.pop(state,None) + pending = await app.control(consume_state) if (not pending or pending['expires']<=time.monotonic() or not binding or not secrets.compare_digest(pending['binding'],hashlib.sha256(binding.encode()).hexdigest()) or pending['config']!=configuration(app) or pending['callback']!=callback_url(app)): @@ -164,7 +169,7 @@ async def callback(request): return failure('The login settings changed. Start again.',403) if not isinstance(user,dict) or type(user.get('id')) is not int or user['id']!=pending['config']['user_id']: return failure('This GitHub account is not permitted to access this app.',403) - token = login.issue_session() + token = await app.control(login.issue_session) response = RedirectResponse('/',status_code=303,headers=OAUTH_HEADERS) response.set_cookie(login.cookie,token,max_age=SESSION_SECONDS,httponly=True,secure=True,samesite='lax',path='/') response.delete_cookie(f'decomp_oauth_{app.port}',path='/login/github',secure=True,httponly=True,samesite='lax') diff --git a/src/decompnet/app/web_login.py b/src/decompnet/app/web_login.py index 9b9d456fe..0a3989a0b 100644 --- a/src/decompnet/app/web_login.py +++ b/src/decompnet/app/web_login.py @@ -51,10 +51,10 @@ def same_origin(self, request): def authorized(self, request): token = request.cookies.get(self.cookie, '') - with self.lock: - now = time.monotonic() - self.sessions = {key:end for key,end in self.sessions.items() if end > now} - return bool(token and self.sessions.get(hashlib.sha256(token.encode()).hexdigest())) + # Authorization is a read-only snapshot on the ASGI thread. Expired entries + # are pruned when issuing a session; it must never wait for a login writer. + until = self.sessions.get(hashlib.sha256(token.encode()).hexdigest()) if token else None + return bool(until and until > time.monotonic()) def sign_in(self, code): with self.lock: @@ -137,7 +137,7 @@ async def interface(request): if app.web_login: app.web_login = app.net.web_login = WebLogin(app.port, remote) app.settings.data['web_interface_enabled'] = enabled - app.settings.save() + await app.control(app.settings.save) if enabled and app.web_login is None: app.web_login = app.net.web_login = WebLogin(app.port, app.settings.data.get('web_remote_url', '')) elif not enabled: @@ -167,7 +167,7 @@ async def sign_in(request): return HTMLResponse(login_page('The sign-in request is too large.'), status_code=413, headers=LOGIN_HEADERS) from urllib.parse import parse_qs code = parse_qs(raw.decode('utf-8', errors='replace')).get('code', [''])[0] - token, status = login.sign_in(code) + token, status = await app.control(lambda: login.sign_in(code)) if token is None: message = 'Too many attempts. Wait one minute and try again.' if status == 429 else 'That code was not accepted. Check the terminal and try again.' return HTMLResponse(login_page(message), status_code=status, headers=LOGIN_HEADERS) @@ -182,7 +182,7 @@ async def sign_out(request): return RedirectResponse('/login', status_code=303, headers=LOGIN_HEADERS) if request.headers.get('host') not in app.allowed_hosts or not login.same_origin(request): return HTMLResponse('Sign out from this server’s own page.', status_code=403, headers=LOGIN_HEADERS) - login.sign_out(request) + await app.control(lambda: login.sign_out(request)) response = RedirectResponse('/login', status_code=303, headers=LOGIN_HEADERS) response.delete_cookie(login.cookie, path='/', httponly=True, samesite='strict') return response diff --git a/src/decompnet/hang_diagnostics.py b/src/decompnet/hang_diagnostics.py new file mode 100644 index 000000000..01100814a --- /dev/null +++ b/src/decompnet/hang_diagnostics.py @@ -0,0 +1,55 @@ +"""Capture a stalled event loop from another thread without logging/database locks.""" +import asyncio +import os +import sys +import threading +import time +from pathlib import Path + + +class Watchdog: + def __init__(self, path, *, threshold=5, interval=1): + self.path=Path(path) + self.threshold,self.interval=threshold,interval + self.last_beat=time.monotonic() + self.last_dump=float('-inf') + self.stop=threading.Event() + self.thread=threading.Thread(target=self.watch,daemon=True,name='event-loop-watchdog') + + async def pulse(self): + while True: + self.last_beat=time.monotonic() + await asyncio.sleep(.25) + + def capture(self, elapsed): + # Code locations only: never frame locals, arguments, source lines, SQL, + # request URLs, tokens or game data. Replace one bounded local report. + names={t.ident:t.name for t in threading.enumerate()} + lines=[f'Event loop unresponsive for {elapsed:.1f}s; captured at {time.strftime("%Y-%m-%d %H:%M:%S %z")}\n'] + frames=sorted(sys._current_frames().items(),key=lambda item:item[0]!=threading.main_thread().ident) + for ident,frame in frames[:256]: + lines.append(f'\nThread {ident}: {names.get(ident,"unknown")}\n') + for _ in range(24): + if frame is None:break + lines.append(f' {frame.f_code.co_filename}:{frame.f_lineno} in {frame.f_code.co_name}\n') + frame=frame.f_back + self.path.parent.mkdir(parents=True,exist_ok=True) + temp=self.path.with_suffix('.tmp') + temp.write_text(''.join(lines)[:512_000],encoding='utf-8') + os.replace(temp,self.path) + try:os.write(2,f'[decompdecomp] Server stalled for {elapsed:.0f}s. Thread locations saved to {self.path}\n'.encode()) + except OSError:pass + + def watch(self): + while not self.stop.wait(self.interval): + now=time.monotonic();elapsed=now-self.last_beat + if elapsed>=self.threshold and now-self.last_dump>=30: + self.last_dump=now + try:self.capture(elapsed) + except OSError:pass # Disk/terminal failures cannot interfere with shutdown. + + def start(self):self.thread.start() + + def close(self): + self.stop.set() + self.thread.join(timeout=.2) diff --git a/src/decompnet/ui_diagnostics.py b/src/decompnet/ui_diagnostics.py index 16d89e7cd..f8004aa0d 100644 --- a/src/decompnet/ui_diagnostics.py +++ b/src/decompnet/ui_diagnostics.py @@ -32,18 +32,29 @@ async def watch_loop(): while True: began=time.monotonic() await asyncio.sleep(1) - record('Event loop stall',max(0,time.monotonic()-began-1)) + await asyncio.to_thread(record,'Event loop stall',max(0,time.monotonic()-began-1)) class Diagnostics: """ASGI wrapper: monitor lifetime follows the server, without another polling thread.""" - def __init__(self, app):self.app=app + def __init__(self, app, *, stack_path=None):self.app,self.stack_path=app,stack_path async def __call__(self, scope, receive, send): if scope['type']=='lifespan': + watchdog=pulse=None + if self.stack_path is not None: + from .hang_diagnostics import Watchdog + watchdog=Watchdog(self.stack_path) + pulse=asyncio.create_task(watchdog.pulse()) + watchdog.start() task=asyncio.create_task(watch_loop()) try:await self.app(scope,receive,send) finally: + if watchdog:watchdog.close() + if pulse: + pulse.cancel() + try:await pulse + except asyncio.CancelledError:pass task.cancel() try:await task except asyncio.CancelledError:pass @@ -65,4 +76,4 @@ async def __call__(self, scope, receive, send): route=getattr(candidate,'path','') break label=route or getattr(scope.get('endpoint'),'__name__','API request') - record('Slow UI request',elapsed,label) + await asyncio.to_thread(record,'Slow UI request',elapsed,label) diff --git a/tests/test_server_lock_contention.py b/tests/test_server_lock_contention.py new file mode 100644 index 000000000..1bdf761e3 --- /dev/null +++ b/tests/test_server_lock_contention.py @@ -0,0 +1,238 @@ +"""Held backend locks must delay only their request, never the event loop or health.""" +import ast +import hashlib +import inspect +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import anyio +import pytest + +from decompnet.app import routes, support +from decompnet.app.web_login import WebLogin +from tests.test_app_server import app as app_fixture + + +@pytest.fixture +def app(tmp_path_factory): + client,net,manager=app_fixture.__wrapped__(tmp_path_factory) + try:yield client,net,manager + finally: + net.startup.close() + net.autosave.close() + net.deep_sweeps.shutdown() + net.permuter.close() + manager.request_shutdown() + + +def check_responsive(client,request,entered,release): + # One TestClient portal is essential: separate portals would hide ASGI stalls. + with client, ThreadPoolExecutor(2) as pool: + blocked=pool.submit(request) + try: + assert entered.wait(3) + health=pool.submit(client.get,'/health') + assert health.result(timeout=1).status_code==200 + assert not blocked.done() + finally:release.set() + return blocked.result(timeout=5) + + +def test_job_start_contention_does_not_block_health(app,monkeypatch): + client,net,manager=app + entered=threading.Event();release=threading.Event() + def start(*a,**kw): + entered.set();assert release.wait(5) + return 'synthetic-job' + monkeypatch.setattr(support.Jobs,'start',start) + response=check_responsive(client,lambda:client.post('/api/match-library',headers={'X-Decomp':'1'},json={}),entered,release) + assert response.json()=={'job':'synthetic-job'} + + +def test_repository_job_lock_contention_does_not_block_health(app,monkeypatch): + client,net,manager=app + entered=threading.Event();release=threading.Event() + class HeldLock: + def __enter__(self):entered.set();assert release.wait(5) + def __exit__(self,*a):pass + # GET repository-setup waits for its job reservation before reading Git. + jobs=[] + # The module fixture's Jobs is captured by the registered route. + for route in client.app.routes: + endpoint=getattr(route,'endpoint',None) + if endpoint and getattr(route,'path',None)=='/api/repository-setup': + owner=inspect.getclosurevars(endpoint).nonlocals.get('app') + assert isinstance(owner,routes.App) + jobs.append(owner.jobs) + assert jobs + monkeypatch.setattr(jobs[0],'_lock',HeldLock()) + response=check_responsive(client,lambda:client.get('/api/repository-setup?slug=missing'),entered,release) + assert response.status_code==400 + + +def test_progress_activity_lock_does_not_block_health(app,monkeypatch): + client,net,manager=app + entered=threading.Event();release=threading.Event() + def status(slug):entered.set();assert release.wait(5);return None + monkeypatch.setattr(net.progress,'status',status) + response=check_responsive(client,lambda:client.get('/api/project-loading?slug=game'),entered,release) + assert response.json()=={'verification':None} + + +def test_control_requests_have_capacity_when_normal_workers_are_full(): + from decompnet.app.routes import App + async def run(): + normal=anyio.CapacityLimiter(1);control=anyio.CapacityLimiter(1) + entered=threading.Event();release=threading.Event();seen=[] + async def call(fn,*args):return await anyio.to_thread.run_sync(fn,*args,limiter=normal) + async def controls(fn,*args):return await anyio.to_thread.run_sync(fn,*args,limiter=control) + owner=object.__new__(App);owner.call=call;owner.control_call=controls + def blocked():entered.set();release.wait(5) + async with anyio.create_task_group() as group: + group.start_soon(call,blocked) + try: + while not entered.is_set():await anyio.sleep(.01) + with anyio.fail_after(1):await owner.control(lambda:seen.append('cancel')) + finally:release.set() + assert seen==['cancel'] + anyio.run(run) + + +def test_cookie_authorization_does_not_wait_for_login_writer(): + from starlette.requests import Request + login=WebLogin(6492);token='test-token' + login.sessions[hashlib.sha256(token.encode()).hexdigest()]=time.monotonic()+60 + request=Request({'type':'http','headers':[(b'cookie',f'{login.cookie}={token}'.encode())]}) + with ThreadPoolExecutor(1) as pool: + with login.lock: + result=pool.submit(login.authorized,request) + assert result.result(timeout=1) + login.sessions[hashlib.sha256(token.encode()).hexdigest()]=0 + assert not login.authorized(request) + + +def test_async_routes_do_not_acquire_thread_locks_or_start_jobs_directly(): + root=Path(__file__).resolve().parents[1]/'src/decompnet/app' + violations=[] + for path in [*sorted((root/'routes').glob('*.py')),root/'web_github.py',root/'web_login.py']: + tree=ast.parse(path.read_text()) + for function in ast.walk(tree): + if not isinstance(function,ast.AsyncFunctionDef):continue + def visit(node,function=function,path=path): + if isinstance(node,(ast.FunctionDef,ast.AsyncFunctionDef,ast.Lambda)) and node is not function:return + if isinstance(node,ast.With): + for item in node.items: + name=ast.unparse(item.context_expr) + if 'lock' in name or '.tx(' in name:violations.append((path.name,node.lineno,name)) + if isinstance(node,ast.Call): + name=ast.unparse(node.func) + if name.endswith('.acquire') and not any(k.arg=='blocking' and isinstance(k.value,ast.Constant) and k.value.value is False for k in node.keywords) and not (node.args and isinstance(node.args[0],ast.Constant) and node.args[0].value is False): + violations.append((path.name,node.lineno,name)) + if name in {'jobs.start','app.jobs.start','jobs.active','jobs.cancel','jobs.check_start', + 'app.net.project_row','app.settings.save','settings.save','net.progress.status', + 'bridge.allowed','bridge.auth','bridge.pair','bridge.revoke','bridge.consume', + 'page_log.warning','jobs.logger.warning'}: + violations.append((path.name,node.lineno,name)) + for child in ast.iter_child_nodes(node):visit(child) + visit(function) + assert not violations + + +def test_repository_validation_does_not_block_health(app,monkeypatch,tmp_path): + client,net,manager=app + from decompnet.network import NetworkError + entered=threading.Event();release=threading.Event() + def project(slug): + entered.set();assert release.wait(5) + raise NetworkError('Synthetic unavailable project') + monkeypatch.setattr(net,'project_row',project) + response=check_responsive(client,lambda:client.post('/api/repository-sync',headers={'X-Decomp':'1'}, + json={'slug':'game','folder':str(tmp_path)}),entered,release) + assert response.status_code==400 and 'Synthetic unavailable' in response.text + + +def test_snapshot_recursive_wait_fails_instead_of_deadlocking(): + from decompnet.app.view_snapshots import Snapshots + cache=Snapshots() + with pytest.raises(RuntimeError,match='own builder'): + cache.get('recursive',lambda:cache.get('recursive',lambda:None)) + assert cache.get('recursive',lambda:'recovered')=='recovered' + + +def test_snapshot_wait_is_bounded_without_cancelling_builder(monkeypatch): + from decompnet.app import view_snapshots + cache=view_snapshots.Snapshots();entered=threading.Event();release=threading.Event() + monkeypatch.setattr(view_snapshots,'SNAPSHOT_WAIT_SECONDS',.03) + def build():entered.set();release.wait(3);return 'finished' + with ThreadPoolExecutor(1) as pool: + future=pool.submit(cache.get,'slow',build) + try: + assert entered.wait(1) + with pytest.raises(TimeoutError,match='still being prepared'):cache.get('slow',lambda:pytest.fail('Duplicate builder')) + finally:release.set() + assert future.result(timeout=1)=='finished' + assert cache.get('slow',lambda:pytest.fail('Lost completed snapshot'))=='finished' + + +def test_watchdog_captures_stall_without_frame_locals(tmp_path): + from decompnet.hang_diagnostics import Watchdog + report=tmp_path/'event-loop-stacks.log' + private_token='do-not-copy-private-local-values' + watchdog=Watchdog(report,threshold=.01,interval=.01) + watchdog.last_beat=time.monotonic()-1 + watchdog.start() + try: + deadline=time.monotonic()+2 + while not report.exists() and time.monotonic()