From 7682d3cf39818dee918781e1f74cff723b6029bd Mon Sep 17 00:00:00 2001 From: jasper blues Date: Fri, 6 Nov 2026 10:02:55 +1000 Subject: [PATCH] Vacuum the write-loop timing test's tables without parallel workers, so a pooled session's shared memory doesn't starve a later vector-index build test_compiled_write_loop_time ran VACUUM ANALYZE drivine_edge over 20k dead edges, which Postgres did in parallel, and left drivine_node at some 8 MB of dead rows, so a later index on it was built in parallel. Each pooled session that ran something in parallel kept a dynamic shared memory segment until it ended, and with two of them a parallel HNSW build couldn't get its 64 MB of a 64 MB /dev/shm: main failed on Python 3.14. The VACUUM now runs with PARALLEL 0, and the test vacuums its emptied tables after it. --- .../sql/test_compiled_write_loop_time.py | 13 +++++++++++-- .../sql/test_compiled_write_loop_time.py | 13 +++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/integration/sql/test_compiled_write_loop_time.py b/tests/integration/sql/test_compiled_write_loop_time.py index 313e365..fc00e0e 100644 --- a/tests/integration/sql/test_compiled_write_loop_time.py +++ b/tests/integration/sql/test_compiled_write_loop_time.py @@ -91,8 +91,11 @@ async def timed(pm: PersistenceManager, n: int) -> list[int]: times: list[int] = [] for write in WRITES: if vacuumed_empty(pm, write): - # Statistics that say the table is empty: autocommitted, as VACUUM runs outside a transaction. - await pm.execute(QuerySpecification("VACUUM ANALYZE drivine_edge")) + # Statistics that say the table is empty: autocommitted, as VACUUM runs outside a transaction. Not in + # parallel: a VACUUM of many dead relationships otherwise vacuums the edge indexes in parallel, and the + # pooled session that ran it keeps some 1.2 MB of /dev/shm until it ends, which a later parallel build of + # a vector index, asking for nearly all of a container's 64 MB, then fails for want of. + await pm.execute(QuerySpecification("VACUUM (PARALLEL 0, ANALYZE) drivine_edge")) start = time.perf_counter() rows = await query(pm, write.cypher, {"n": n}) times.append(round((time.perf_counter() - start) * 1000)) @@ -117,6 +120,12 @@ async def test_a_write_over_ten_times_the_rows_takes_about_ten_times_as_long_not TIMES[store] = (await timed(pm, SMALL), await timed(pm, LARGE)) finally: await pm.execute(QuerySpecification("DELETE FROM drivine_node")) + if pm.type == DatabaseType.POSTGRES: + # The tables are left as small as they were found, which a VACUUM of the empty tables truncates them + # to. The node table, left at some 8 MB of dead rows, was over min_parallel_table_scan_size, so a later + # index on it was built in parallel: a vector index asks for nearly all of a container's 64 MB of + # /dev/shm, and a pooled session that built any index in parallel keeps some of it until it ends. + await pm.execute(QuerySpecification("VACUUM (PARALLEL 0) drivine_node, drivine_edge")) small, large = TIMES[store] report = "; ".join( f"{it.name} {small[i]} ms over {SMALL} rows, {large[i]} ms over {LARGE}" for i, it in enumerate(WRITES) diff --git a/tests/sync_integration/sql/test_compiled_write_loop_time.py b/tests/sync_integration/sql/test_compiled_write_loop_time.py index 1f33714..a368023 100644 --- a/tests/sync_integration/sql/test_compiled_write_loop_time.py +++ b/tests/sync_integration/sql/test_compiled_write_loop_time.py @@ -93,8 +93,11 @@ def timed(pm: PersistenceManager, n: int) -> list[int]: times: list[int] = [] for write in WRITES: if vacuumed_empty(pm, write): - # Statistics that say the table is empty: autocommitted, as VACUUM runs outside a transaction. - pm.execute(QuerySpecification("VACUUM ANALYZE drivine_edge")) + # Statistics that say the table is empty: autocommitted, as VACUUM runs outside a transaction. Not in + # parallel: a VACUUM of many dead relationships otherwise vacuums the edge indexes in parallel, and the + # pooled session that ran it keeps some 1.2 MB of /dev/shm until it ends, which a later parallel build of + # a vector index, asking for nearly all of a container's 64 MB, then fails for want of. + pm.execute(QuerySpecification("VACUUM (PARALLEL 0, ANALYZE) drivine_edge")) start = time.perf_counter() rows = query(pm, write.cypher, {"n": n}) times.append(round((time.perf_counter() - start) * 1000)) @@ -119,6 +122,12 @@ def test_a_write_over_ten_times_the_rows_takes_about_ten_times_as_long_not_a_hun TIMES[store] = (timed(pm, SMALL), timed(pm, LARGE)) finally: pm.execute(QuerySpecification("DELETE FROM drivine_node")) + if pm.type == DatabaseType.POSTGRES: + # The tables are left as small as they were found, which a VACUUM of the empty tables truncates them + # to. The node table, left at some 8 MB of dead rows, was over min_parallel_table_scan_size, so a later + # index on it was built in parallel: a vector index asks for nearly all of a container's 64 MB of + # /dev/shm, and a pooled session that built any index in parallel keeps some of it until it ends. + pm.execute(QuerySpecification("VACUUM (PARALLEL 0) drivine_node, drivine_edge")) small, large = TIMES[store] report = "; ".join( f"{it.name} {small[i]} ms over {SMALL} rows, {large[i]} ms over {LARGE}" for i, it in enumerate(WRITES)