From 1d4d369d83012ce2855336841bc87a2c59bbdbd4 Mon Sep 17 00:00:00 2001 From: Jarred Stelfox Date: Tue, 20 Oct 2026 14:19:19 -0700 Subject: [PATCH 1/3] Drop idle DB connections before MySQL does Prod MySQL closes a connection after 60 seconds without a query (wait_timeout = 60), and mysql2 drops an idle pool connection only after its own idleTimeout, which also defaults to 60 seconds. On that tie the pool hands out a connection the server already closed, and the query fails with errno 4031. Between Oct 16 and 20 prod logged it 6 times, twice on PR saves (ifixit#63906 and an ops PR) that only landed on a later update. Dropping idle connections at 30 seconds, half the server's limit, means the pool always lets go first. TCP keepalive wouldn't help: MySQL counts time without queries, not time without packets. Co-Authored-By: Claude Opus 5.5 --- lib/db.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/db.js b/lib/db.js index 7a103482..57d6f1cf 100644 --- a/lib/db.js +++ b/lib/db.js @@ -12,12 +12,21 @@ import EncodingToCharset from "../node_modules/mysql2/lib/constants/encoding_cha // Add support for utf8mb3 charset EncodingToCharset.utf8mb3 = 192; +// MySQL closes a connection after wait_timeout seconds of no queries (60 on +// prod), and mysql2 only drops an idle pool connection after its own +// idleTimeout, which also defaults to 60s. On that tie the pool hands out a +// connection the server already closed, and the query fails with errno 4031 +// ("disconnected by the server because of inactivity"). Drop idle connections +// at half the server's limit so the pool always gets there first. +const IDLE_TIMEOUT_MS = 30 * 1000; + const pool = mysql.createPool({ host: config.mysql.host, database: config.mysql.db, user: config.mysql.user, password: config.mysql.pass, charset: "utf8mb4", + idleTimeout: IDLE_TIMEOUT_MS, }); export default { From c8208a0382246874776dd5ec90e05d602f7606ba Mon Sep 17 00:00:00 2001 From: Jarred Stelfox Date: Tue, 20 Oct 2026 14:29:35 -0700 Subject: [PATCH 2/3] Retry a query once when MySQL closed it for idling Review found the idleTimeout change did nothing: mysql2 runs its idle check only while maxIdle is below connectionLimit, and both default to 10. Turning the check on keeps a timer alive, so every test and bin that imports the DB without db.end() hangs. Instead, retry once when a query fails with errno 4031. The server closed the connection for idling before the query arrived, so the query never ran, and the pool discards that connection on the error: the retry gets a fresh one, writes included. Nothing else is retried; a connection lost mid-query may have run its write. Co-Authored-By: Claude Opus 5.5 --- lib/db.js | 19 ++++++++++++++++++- test/db-retry.test.js | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 test/db-retry.test.js diff --git a/lib/db.js b/lib/db.js index 57d6f1cf..d2a81605 100644 --- a/lib/db.js +++ b/lib/db.js @@ -29,9 +29,26 @@ const pool = mysql.createPool({ idleTimeout: IDLE_TIMEOUT_MS, }); +// MySQL closes a connection after wait_timeout seconds without a query (60 on +// prod), and the pool only finds out when the next query on it fails with +// errno 4031 ("disconnected by the server because of inactivity"). The +// server closed the connection before that query arrived, so the query never +// ran and one retry on a fresh connection is safe, writes included. Any other +// error, a connection lost mid-query among them, is passed on unchanged. +const SERVER_CLOSED_IDLE = 4031; + +export async function retryIfIdleClosed(run) { + try { + return await run(); + } catch (err) { + if (err?.errno !== SERVER_CLOSED_IDLE) throw err; + return run(); + } +} + export default { query: function (query, params) { - return pool.query(query, params) + return retryIfIdleClosed(() => pool.query(query, params)) .then((result) => result[0]); // [rows, fields] }, // The refresh-* bins call this to close the pool so the process can exit diff --git a/test/db-retry.test.js b/test/db-retry.test.js new file mode 100644 index 00000000..32a482d6 --- /dev/null +++ b/test/db-retry.test.js @@ -0,0 +1,41 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { retryIfIdleClosed } from '../lib/db.js'; + +const idleClosed = Object.assign(new Error('disconnected by the server'), { errno: 4031 }); + +test('a query on a connection MySQL closed for idling runs once more', async () => { + let calls = 0; + const rows = await retryIfIdleClosed(async () => { + calls++; + if (calls === 1) throw idleClosed; + return 'rows'; + }); + assert.equal(rows, 'rows'); + assert.equal(calls, 2); +}); + +test('any other error is not retried', async () => { + let calls = 0; + const lost = Object.assign(new Error('lost'), { code: 'PROTOCOL_CONNECTION_LOST' }); + await assert.rejects( + retryIfIdleClosed(async () => { + calls++; + throw lost; + }), + lost + ); + assert.equal(calls, 1); +}); + +test('it retries only once', async () => { + let calls = 0; + await assert.rejects( + retryIfIdleClosed(async () => { + calls++; + throw idleClosed; + }), + idleClosed + ); + assert.equal(calls, 2); +}); From fd72ec257348f26c7e71e7c4f6689b6df6722219 Mon Sep 17 00:00:00 2001 From: Jarred Stelfox Date: Tue, 20 Oct 2026 14:30:03 -0700 Subject: [PATCH 3/3] Drop the idleTimeout that never ran mysql2 ignores idleTimeout while maxIdle equals connectionLimit, so the setting and its comment described something that didn't happen. The retry on errno 4031 is the fix. Co-Authored-By: Claude Opus 5.5 --- lib/db.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/db.js b/lib/db.js index d2a81605..dd7bd571 100644 --- a/lib/db.js +++ b/lib/db.js @@ -12,21 +12,12 @@ import EncodingToCharset from "../node_modules/mysql2/lib/constants/encoding_cha // Add support for utf8mb3 charset EncodingToCharset.utf8mb3 = 192; -// MySQL closes a connection after wait_timeout seconds of no queries (60 on -// prod), and mysql2 only drops an idle pool connection after its own -// idleTimeout, which also defaults to 60s. On that tie the pool hands out a -// connection the server already closed, and the query fails with errno 4031 -// ("disconnected by the server because of inactivity"). Drop idle connections -// at half the server's limit so the pool always gets there first. -const IDLE_TIMEOUT_MS = 30 * 1000; - const pool = mysql.createPool({ host: config.mysql.host, database: config.mysql.db, user: config.mysql.user, password: config.mysql.pass, charset: "utf8mb4", - idleTimeout: IDLE_TIMEOUT_MS, }); // MySQL closes a connection after wait_timeout seconds without a query (60 on