From c64e94374d77c6de35bfe8b60c99efd461bd425a Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 10:42:20 +0000 Subject: [PATCH 01/20] feat(driver): engines with one writer at a time, and with migration scripts of their own driver.Capabilities.SingleWriter describes an engine that runs one writing transaction at a time, as SQLite does: claims need no SKIP LOCKED, the finalizing transaction settles its batch and promotes dependents itself, and SkipLocked and NoWait wait only briefly for another transaction's writes. A MigrationExecutor that implements driver.MigrationScripts gets its engine's own script for each schema version from hoppermigrate, in place of the embedded Postgres one. --- driver/driver.go | 28 +++++++++- hoppermigrate/hoppermigrate.go | 33 +++++++++++- hoppermigrate/scripts_test.go | 96 ++++++++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 hoppermigrate/scripts_test.go diff --git a/driver/driver.go b/driver/driver.go index c74fe76..9912667 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -71,6 +71,16 @@ type Capabilities struct { // Listen reports whether Listener is implemented (LISTEN/NOTIFY on // Postgres). Listen bool + // SingleWriter reports that the engine runs one transaction that writes + // at a time, as SQLite does, rather than locking rows: a transaction + // that writes holds up every other write until it ends. A claim needs no + // SKIP LOCKED to be exclusive, and the transaction that finalizes a job + // settles its batch and promotes its dependents with no other to race + // it, so BatchSettle and WorkflowPromoteReady find nothing a race left. + // With JobFinalizeParams.SkipLocked or JobRecordProgressParams.NoWait, + // a statement waits only briefly for another transaction's writes to + // end, and then skips every job, or returns ErrJobLocked. + SingleWriter bool } // Notification channels. Payloads are documented on each. @@ -367,6 +377,18 @@ type MigrationExecutor interface { Close(ctx context.Context) error } +// MigrationScripts is implemented by a MigrationExecutor whose engine is not +// Postgres, whose scripts hoppermigrate embeds. hoppermigrate then gives +// Apply the engine's own script for each schema version instead. An engine +// has a script, up and down, for every schema version hoppermigrate knows, +// so that a version names the same schema on every engine: the tables and +// columns a release's statements need. +type MigrationScripts interface { + // Script returns the engine's script for a schema version: the one that + // applies it (up) or the one that reverts it. + Script(version int, up bool) (string, error) +} + // JobState is a job's position in the state machine. type JobState string @@ -497,7 +519,8 @@ type JobRecordProgressParams struct { // Cursor is a JSON value, or nil for none. Cursor json.RawMessage // NoWait returns ErrJobLocked at once, instead of waiting, if another - // transaction holds the job's row. + // transaction holds the job's row; on a SingleWriter engine, if another + // transaction writes. NoWait bool } @@ -807,7 +830,8 @@ type JobFinalizeParams struct { NoBatches bool // SkipLocked skips the jobs whose rows another transaction holds, such // as a worker's CompleteTx that has not ended, instead of waiting for - // it. A skipped job is not among the applied. + // it. A skipped job is not among the applied. On a SingleWriter engine + // every job is skipped while another transaction writes. SkipLocked bool // KeepLogs is the most attempt logs a job keeps, the latest; a job given // another drops its oldest. Zero keeps them all. diff --git a/hoppermigrate/hoppermigrate.go b/hoppermigrate/hoppermigrate.go index de60242..3ca34a9 100644 --- a/hoppermigrate/hoppermigrate.go +++ b/hoppermigrate/hoppermigrate.go @@ -5,6 +5,10 @@ // so any number of replicas can call Up at startup. The raw files are also // available through Migrations for teams that run migrations with their own // tool; hopper_schema records the applied versions either way. +// +// The embedded scripts are Postgres's. A driver for another engine supplies +// its own script for each version (driver.MigrationScripts), and Up and Down +// apply those, at the same versions. package hoppermigrate import ( @@ -122,7 +126,11 @@ func run[TTx any](ctx context.Context, d driver.Driver[TTx], opts *Options, up b if m.Version > target || slices.Contains(applied, m.Version) { continue } - if err := exec.Apply(ctx, m.Version, m.Up, true); err != nil { + script, err := scriptFor(exec, m, true) + if err != nil { + return nil, err + } + if err := exec.Apply(ctx, m.Version, script, true); err != nil { return nil, fmt.Errorf("hoppermigrate: apply %03d_%s: %w", m.Version, m.Name, err) } logger.InfoContext(ctx, "hopper: applied migration", "version", m.Version, "name", m.Name) @@ -136,7 +144,11 @@ func run[TTx any](ctx context.Context, d driver.Driver[TTx], opts *Options, up b if m.Version <= target || !slices.Contains(applied, m.Version) { continue } - if err := exec.Apply(ctx, m.Version, m.Down, false); err != nil { + script, err := scriptFor(exec, m, false) + if err != nil { + return nil, err + } + if err := exec.Apply(ctx, m.Version, script, false); err != nil { return nil, fmt.Errorf("hoppermigrate: revert %03d_%s: %w", m.Version, m.Name, err) } logger.InfoContext(ctx, "hopper: reverted migration", "version", m.Version, "name", m.Name) @@ -146,6 +158,23 @@ func run[TTx any](ctx context.Context, d driver.Driver[TTx], opts *Options, up b return res, nil } +// scriptFor returns m's up or down script for the engine exec migrates: its +// own, if it has scripts of its own, or the embedded Postgres one. +func scriptFor(exec driver.MigrationExecutor, m Migration, up bool) (string, error) { + scripts, ok := exec.(driver.MigrationScripts) + if !ok { + if up { + return m.Up, nil + } + return m.Down, nil + } + script, err := scripts.Script(m.Version, up) + if err != nil { + return "", fmt.Errorf("hoppermigrate: the engine's script for %03d_%s: %w", m.Version, m.Name, err) + } + return script, nil +} + func current(applied []int) int { if len(applied) == 0 { return 0 diff --git a/hoppermigrate/scripts_test.go b/hoppermigrate/scripts_test.go new file mode 100644 index 0000000..635e39f --- /dev/null +++ b/hoppermigrate/scripts_test.go @@ -0,0 +1,96 @@ +package hoppermigrate_test + +import ( + "context" + "errors" + "fmt" + "log/slog" + "slices" + "strings" + "testing" + + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/hoppermigrate" +) + +// engineDriver is a driver whose engine has migration scripts of its own, +// which its executor records as they are applied. +type engineDriver struct{ exec *engineExec } + +func (engineDriver) Executor() driver.Executor { return nil } +func (engineDriver) UnwrapTx(struct{}) driver.Executor { return nil } +func (engineDriver) Capabilities() driver.Capabilities { return driver.Capabilities{} } +func (d engineDriver) Migrator() driver.Migrator { return d } +func (engineDriver) Listener(context.Context) (driver.Listener, error) { + return nil, driver.ErrNotSupported +} + +func (d engineDriver) Lock(context.Context) (driver.MigrationExecutor, error) { return d.exec, nil } + +type engineExec struct { + versions []int + applied []string + // missing is a version the engine has no script for. + missing int +} + +func (e *engineExec) Versions(context.Context) ([]int, error) { return slices.Clone(e.versions), nil } + +func (e *engineExec) Script(version int, up bool) (string, error) { + if version == e.missing { + return "", errors.New("no script") + } + if up { + return fmt.Sprintf("engine up %d", version), nil + } + return fmt.Sprintf("engine down %d", version), nil +} + +func (e *engineExec) Apply(_ context.Context, version int, script string, up bool) error { + e.applied = append(e.applied, script) + if up { + e.versions = append(e.versions, version) + } else { + e.versions = slices.DeleteFunc(e.versions, func(v int) bool { return v == version }) + } + return nil +} + +func (*engineExec) Close(context.Context) error { return nil } + +// TestEngineScripts migrates an engine that has scripts of its own: Up and +// Down apply its scripts, not the embedded Postgres ones, at hoppermigrate's +// versions, and an engine without a version's script fails the run, naming +// the migration. +func TestEngineScripts(t *testing.T) { + t.Parallel() + ctx := context.Background() + opts := &hoppermigrate.Options{Logger: slog.New(slog.DiscardHandler)} + exec := &engineExec{} + d := engineDriver{exec} + res, err := hoppermigrate.Up(ctx, d, opts) + if err != nil || res.Version != hoppermigrate.Latest() { + t.Fatalf("Up = %+v, %v", res, err) + } + for i, script := range exec.applied { + if want := fmt.Sprintf("engine up %d", i+1); script != want { + t.Fatalf("script %d = %q, want %q", i, script, want) + } + } + exec.applied = nil + if _, err := hoppermigrate.Down(ctx, d, &hoppermigrate.Options{Logger: opts.Logger, Target: hoppermigrate.Latest() - 2}); err != nil { + t.Fatal(err) + } + if want := []string{fmt.Sprintf("engine down %d", hoppermigrate.Latest()), fmt.Sprintf("engine down %d", hoppermigrate.Latest()-1)}; !slices.Equal(exec.applied, want) { + t.Errorf("Down applied %q, want %q", exec.applied, want) + } + + exec = &engineExec{missing: 3} + _, err = hoppermigrate.Up(ctx, engineDriver{exec}, opts) + if err == nil || !strings.Contains(err.Error(), "003_"+hoppermigrate.Migrations()[2].Name) { + t.Errorf("Up without a script for version 3 = %v", err) + } + if len(exec.applied) != 2 { + t.Errorf("applied %q before the missing script, want versions 1 and 2", exec.applied) + } +} From b0a8d1762bef672c840fb51d162f41c4e0941d05 Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 10:42:20 +0000 Subject: [PATCH 02/20] test(drivertest): split the engine-neutral suite from what the Postgres drivers share Run's tests now set up what no queue operation produces through Fixture hooks (ExpireJob, StaleWake, MakePending, SnoozeReason, SetSnoozeReason, SetProgress, CountEdges, WaitForLockWait) instead of Postgres SQL, compare JSON values rather than jsonb's text, and test what depends on row locks either way by Capabilities.SingleWriter. Tests of partitions, their indexes, the tags tables, progress rows and planner statistics move to RunPostgres, which takes a PostgresFixture (Fixture plus Exec and QueryText). drivertest.Postgres implements the SQL hooks for the Postgres drivers' fixtures. --- driver/hopperpgx/conformance_test.go | 76 ++---- driver/hoppersql/conformance_test.go | 57 ++--- drivertest/cluster.go | 10 +- drivertest/drivertest.go | 282 ++++++++++++++++++++-- drivertest/flow.go | 349 ++++++++++++++++----------- drivertest/jobs.go | 115 ++++++--- drivertest/logs.go | 46 ++-- drivertest/messaging.go | 2 +- drivertest/progress.go | 13 +- drivertest/snooze.go | 21 +- drivertest/stream.go | 40 ++- drivertest/tags.go | 195 ++++++++------- drivertest/unique.go | 2 +- drivertest/workflow.go | 218 ++++++++++++----- 14 files changed, 926 insertions(+), 500 deletions(-) diff --git a/driver/hopperpgx/conformance_test.go b/driver/hopperpgx/conformance_test.go index 68d5fd1..2ef5561 100644 --- a/driver/hopperpgx/conformance_test.go +++ b/driver/hopperpgx/conformance_test.go @@ -3,7 +3,6 @@ package hopperpgx_test import ( "context" "testing" - "time" "github.com/jackc/pgx/v5" @@ -13,20 +12,12 @@ import ( "github.com/parallelworks/hopper/internal/testdb" ) -// pgxFixture runs the conformance suite against hopperpgx. -type pgxFixture struct{} +// pgxFixture runs the conformance suite against hopperpgx. Its hooks are +// statements on hopper's schema (drivertest.Postgres), which pgxSQL runs on +// the driver's pool. +type pgxFixture struct{ drivertest.Postgres[pgx.Tx] } -func pool(d driver.Driver[pgx.Tx]) interface { - Exec(ctx context.Context, sql string, args ...any) (interface{ RowsAffected() int64 }, error) -} { - return execAdapter{d.(*hopperpgx.Driver)} -} - -type execAdapter struct{ d *hopperpgx.Driver } - -func (a execAdapter) Exec(ctx context.Context, sql string, args ...any) (interface{ RowsAffected() int64 }, error) { - return a.d.Pool().Exec(ctx, sql, args...) -} +func newFixture() pgxFixture { return pgxFixture{drivertest.Postgres[pgx.Tx]{SQL: pgxSQL{}}} } func (pgxFixture) NewDriver(t *testing.T) driver.Driver[pgx.Tx] { return testdb.Driver(testdb.Pool(t)) @@ -41,50 +32,17 @@ func (pgxFixture) Begin(ctx context.Context, t *testing.T, d driver.Driver[pgx.T return tx, func() error { return tx.Commit(ctx) }, func() error { return tx.Rollback(ctx) } } -func (pgxFixture) ExpireLease(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], clientID int64) { - t.Helper() - if _, err := pool(d).Exec(ctx, "UPDATE hopper_clients SET expires_at = now() - interval '1s' WHERE id = $1", clientID); err != nil { - t.Fatal(err) - } -} +// pgxSQL runs the suite's statements on a driver's pool. +type pgxSQL struct{} -func (pgxFixture) ExpireLeader(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx]) { +func (pgxSQL) Exec(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], query string, args ...any) { t.Helper() - if _, err := pool(d).Exec(ctx, "UPDATE hopper_leader SET expires_at = now() - interval '1s'"); err != nil { - t.Fatal(err) - } -} - -func (pgxFixture) MakeClaimable(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], id driver.JobID) { - t.Helper() - if _, err := pool(d).Exec(ctx, "UPDATE hopper_jobs SET scheduled_at = now() WHERE id = $1", [16]byte(id)); err != nil { - t.Fatal(err) - } -} - -func (pgxFixture) AgeAttempt(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], id driver.JobID, age time.Duration) { - t.Helper() - if _, err := pool(d).Exec(ctx, "UPDATE hopper_jobs SET attempted_at = now() - make_interval(secs => $2) WHERE id = $1", [16]byte(id), age.Seconds()); err != nil { - t.Fatal(err) - } -} - -func (pgxFixture) CountLive(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx]) int { - return count(ctx, t, d, "hopper_jobs") -} - -func (pgxFixture) CountHistory(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx]) int { - return count(ctx, t, d, "hopper_job_history") -} - -func (pgxFixture) Exec(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], query string, args ...any) { - t.Helper() - if _, err := pool(d).Exec(ctx, query, args...); err != nil { + if _, err := d.(*hopperpgx.Driver).Pool().Exec(ctx, query, args...); err != nil { t.Error(err) } } -func (pgxFixture) QueryText(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], query string, args ...any) string { +func (pgxSQL) QueryText(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], query string, args ...any) string { t.Helper() var s string if err := d.(*hopperpgx.Driver).Pool().QueryRow(ctx, query, args...).Scan(&s); err != nil { @@ -93,16 +51,12 @@ func (pgxFixture) QueryText(ctx context.Context, t *testing.T, d driver.Driver[p return s } -func count(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], table string) int { - t.Helper() - var n int - if err := d.(*hopperpgx.Driver).Pool().QueryRow(ctx, "SELECT count(*) FROM "+table).Scan(&n); err != nil { - t.Fatal(err) - } - return n +func TestConformance(t *testing.T) { + t.Parallel() + drivertest.Run(t, newFixture()) } -func TestConformance(t *testing.T) { +func TestConformancePostgres(t *testing.T) { t.Parallel() - drivertest.Run(t, pgxFixture{}) + drivertest.RunPostgres(t, newFixture()) } diff --git a/driver/hoppersql/conformance_test.go b/driver/hoppersql/conformance_test.go index 212e987..cd18ba7 100644 --- a/driver/hoppersql/conformance_test.go +++ b/driver/hoppersql/conformance_test.go @@ -22,8 +22,11 @@ import ( ) // sqlFixture runs the conformance suite against hoppersql, through pgx's -// database/sql adapter. -type sqlFixture struct{} +// database/sql adapter. Its hooks are statements on hopper's schema +// (drivertest.Postgres), which sqlSQL runs on the driver's database handle. +type sqlFixture struct{ drivertest.Postgres[*sql.Tx] } + +func newFixture() sqlFixture { return sqlFixture{drivertest.Postgres[*sql.Tx]{SQL: sqlSQL{}}} } // open returns a database/sql handle on the pool's schema. func open(t *testing.T, pool *pgxpool.Pool) *sql.DB { @@ -47,45 +50,17 @@ func (sqlFixture) Begin(ctx context.Context, t *testing.T, d driver.Driver[*sql. return tx, tx.Commit, tx.Rollback } -func exec(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) { - t.Helper() - if _, err := d.(*hoppersql.Driver).DB().ExecContext(ctx, query, args...); err != nil { - t.Fatal(err) - } -} - -func (sqlFixture) ExpireLease(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], clientID int64) { - exec(ctx, t, d, "UPDATE hopper_clients SET expires_at = now() - interval '1s' WHERE id = $1", clientID) -} - -func (sqlFixture) ExpireLeader(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) { - exec(ctx, t, d, "UPDATE hopper_leader SET expires_at = now() - interval '1s'") -} - -func (sqlFixture) MakeClaimable(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { - exec(ctx, t, d, "UPDATE hopper_jobs SET scheduled_at = now() WHERE id = $1", id) -} - -func (sqlFixture) AgeAttempt(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID, age time.Duration) { - exec(ctx, t, d, "UPDATE hopper_jobs SET attempted_at = now() - make_interval(secs => $2) WHERE id = $1", id, age.Seconds()) -} - -func (sqlFixture) CountLive(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) int { - return count(ctx, t, d, "hopper_jobs") -} - -func (sqlFixture) CountHistory(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) int { - return count(ctx, t, d, "hopper_job_history") -} +// sqlSQL runs the suite's statements on a driver's database handle. +type sqlSQL struct{} -func (sqlFixture) Exec(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) { +func (sqlSQL) Exec(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) { t.Helper() if _, err := d.(*hoppersql.Driver).DB().ExecContext(ctx, query, args...); err != nil { t.Error(err) } } -func (sqlFixture) QueryText(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) string { +func (sqlSQL) QueryText(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) string { t.Helper() var s string if err := d.(*hoppersql.Driver).DB().QueryRowContext(ctx, query, args...).Scan(&s); err != nil { @@ -94,18 +69,14 @@ func (sqlFixture) QueryText(ctx context.Context, t *testing.T, d driver.Driver[* return s } -func count(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], table string) int { - t.Helper() - var n int - if err := d.(*hoppersql.Driver).DB().QueryRowContext(ctx, "SELECT count(*) FROM "+table).Scan(&n); err != nil { - t.Fatal(err) - } - return n +func TestConformance(t *testing.T) { + t.Parallel() + drivertest.Run(t, newFixture()) } -func TestConformance(t *testing.T) { +func TestConformancePostgres(t *testing.T) { t.Parallel() - drivertest.Run(t, sqlFixture{}) + drivertest.RunPostgres(t, newFixture()) } // TestMigrate applies and reverts the schema through the database/sql diff --git a/drivertest/cluster.go b/drivertest/cluster.go index c11a82a..023cec8 100644 --- a/drivertest/cluster.go +++ b/drivertest/cluster.go @@ -380,7 +380,7 @@ func indexOf(s, sub string) int { // partition, including ones a canceled build left invalid, while leaders // overlap and history is read throughout: no attach may deadlock with a // reader. -func testHistoryIndexes[TTx any](t *testing.T, f Fixture[TTx]) { +func testHistoryIndexes[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -601,7 +601,7 @@ var onceAgain = driver.JobInsertParams{ } // reattachRevalidates runs ReattachRevalidates on the fixture's database. -func reattachRevalidates[TTx any](ctx context.Context, t *testing.T, f Fixture[TTx], d driver.Driver[TTx]) bool { +func reattachRevalidates[TTx any](ctx context.Context, t *testing.T, f PostgresFixture[TTx], d driver.Driver[TTx]) bool { t.Helper() return ReattachRevalidates( func(query string) { f.Exec(ctx, t, d, query) }, @@ -634,7 +634,7 @@ func ReattachRevalidates(exec func(query string), queryText func(query string) s // partitions an outcome table's index lacks, after a schema upgrade // declared it: the index must end valid, which nothing can do afterwards on // releases before May 2026. -func testHistoryRetentionCompletesIndex[TTx any](t *testing.T, f Fixture[TTx]) { +func testHistoryRetentionCompletesIndex[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -1116,7 +1116,7 @@ func testMaintain[TTx any](t *testing.T, f Fixture[TTx]) { // the table has statistics, the planner takes each lookup of a job's edges // for a two-hundredth of the table, and finalizes and the promotion sweep // scan it instead. -func testDepsTableMaintained[TTx any](t *testing.T, f Fixture[TTx]) { +func testDepsTableMaintained[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -1209,7 +1209,7 @@ func waitMaintain(ctx context.Context, t *testing.T, exec driver.Executor, done // past retention needs that transaction to end; maintenance puts the drop // off instead of waiting for it, which would queue every other history // write behind the drop, and drops it once the transaction is over. -func testHistoryMaintainWhileInUse[TTx any](t *testing.T, f Fixture[TTx]) { +func testHistoryMaintainWhileInUse[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() diff --git a/drivertest/drivertest.go b/drivertest/drivertest.go index baa7f43..e8f37c4 100644 --- a/drivertest/drivertest.go +++ b/drivertest/drivertest.go @@ -13,8 +13,11 @@ // (order, ownership, expiry, SKIP LOCKED), every finalize transition and its // fencing to the claim (also inside a caller's transaction), snooze reasons, leases, leader election, rescue candidates, // cancel, retry, TTL expiry, listing, queues, periodic slots, stats, -// notifications and history maintenance. Engine-specific behavior, such as how partitions are managed, -// belongs in the driver's own tests. +// notifications and history maintenance. What depends on whether the engine +// locks rows or runs one writer at a time (driver.Capabilities.SingleWriter) +// is tested either way. Engine-specific behavior, such as how partitions are +// managed, belongs in the driver's own tests; RunPostgres has the tests of +// what hopper's Postgres drivers share. package drivertest import ( @@ -23,6 +26,7 @@ import ( "errors" "fmt" "slices" + "strconv" "strings" "testing" "time" @@ -30,7 +34,9 @@ import ( "github.com/parallelworks/hopper/driver" ) -// Fixture gives the suite a database per test and the hooks it needs. +// Fixture gives the suite a database per test and the hooks it needs. The +// hooks report a failure with t.Error rather than t.Fatal when the suite may +// call them from another goroutine, as SnoozeReason says. type Fixture[TTx any] interface { // NewDriver returns a driver on a fresh database with hopper's schema // installed, torn down when the test ends. @@ -49,14 +55,31 @@ type Fixture[TTx any] interface { // CountLive and CountHistory count jobs in the live table and in history. CountLive(ctx context.Context, t *testing.T, d driver.Driver[TTx]) int CountHistory(ctx context.Context, t *testing.T, d driver.Driver[TTx]) int - // Exec runs a SQL statement with hopper's tables on the search path, for - // rows the suite writes directly, such as those a client of the previous - // release leaves behind. QueryText returns the single text value a query - // selects, for what the driver's reads do not show. Both report a failure - // with t.Error rather than t.Fatal, so the suite may call them from any - // goroutine. - Exec(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) - QueryText(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) string + // ExpireJob moves a live job's TTL deadline a second into the past. + ExpireJob(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) + // StaleWake marks a running job as woken by JobRetryOrderingKey a second + // before its attempt began, as a wake during an earlier attempt would. + StaleWake(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) + // MakePending puts a live job back in the pending state, as a client + // that stopped before promoting it would leave it. + MakePending(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) + // SnoozeReason reads a live job's stored snooze reason, or "" for none, + // which the driver's reads show only for a scheduled job. The suite + // calls it from any goroutine. SetSnoozeReason stores one, as a client + // of an earlier release, which cleared it neither on claim nor on + // finalize, can leave it on a job that is not snoozed. + SnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) string + SetSnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID, reason string) + // SetProgress stores a JSON value as a live job's progress as it is, as + // a direct SQL write could. + SetProgress(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID, value string) + // CountEdges counts the dependency edges the engine keeps for workflow + // jobs that have not finished. + CountEdges(ctx context.Context, t *testing.T, d driver.Driver[TTx]) int + // WaitForLockWait waits until a statement on the driver's database + // waits for a lock another transaction holds, so that a test knows its + // competing statement is blocked rather than not yet started. + WaitForLockWait(ctx context.Context, t *testing.T, d driver.Driver[TTx]) } // Run runs the suite. @@ -73,17 +96,6 @@ func Run[TTx any](t *testing.T, f Fixture[TTx]) { {"InsertCopy", testInsertCopy[TTx]}, {"JobTags", testJobTags[TTx]}, {"JobListFilters", testJobListFilters[TTx]}, - {"TagsRowsFollowJobs", testTagsRowsFollowJobs[TTx]}, - {"TagsTableMaintained", testTagsTableMaintained[TTx]}, - {"TagsTableMaintainedPastJobsError", testTagsTableMaintainedPastJobsError[TTx]}, - {"TagsFollowHistoryRetention", testTagsFollowHistoryRetention[TTx]}, - {"TagsPartitionWaitsForHistory", testTagsPartitionWaitsForHistory[TTx]}, - {"TagsLeaveWithTheirHour", testTagsLeaveWithTheirHour[TTx]}, - {"TagsHourOutlastsItsDelete", testTagsHourOutlastsItsDelete[TTx]}, - {"TagsHourDroppedMeanwhile", testTagsHourDroppedMeanwhile[TTx]}, - {"TagsDayGoesWithItsLastHour", testTagsDayGoesWithItsLastHour[TTx]}, - {"TagsFollowDirectUpdates", testTagsFollowDirectUpdates[TTx]}, - {"TagsFollowTruncate", testTagsFollowTruncate[TTx]}, {"TagsPagePastFilteredHistory", testTagsPagePastFilteredHistory[TTx]}, {"InsertTx", testInsertTx[TTx]}, {"Claim", testClaim[TTx]}, @@ -128,9 +140,6 @@ func Run[TTx any](t *testing.T, f Fixture[TTx]) { {"Stats", testStats[TTx]}, {"Notify", testNotify[TTx]}, {"HistoryMaintain", testHistoryMaintain[TTx]}, - {"HistoryMaintainWhileInUse", testHistoryMaintainWhileInUse[TTx]}, - {"HistoryIndexes", testHistoryIndexes[TTx]}, - {"HistoryRetentionCompletesIndex", testHistoryRetentionCompletesIndex[TTx]}, {"Now", testNow[TTx]}, {"Subscriptions", testSubscriptions[TTx]}, {"Publish", testPublish[TTx]}, @@ -146,7 +155,62 @@ func Run[TTx any](t *testing.T, f Fixture[TTx]) { {"WorkflowFanInConcurrent", testWorkflowFanInConcurrent[TTx]}, {"WorkflowsConcurrent", testWorkflowsConcurrent[TTx]}, {"Streams", testStreams[TTx]}, + {"StreamsRetention", testStreamsRetention[TTx]}, {"Maintain", testMaintain[TTx]}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.fn(t, f) + }) + } +} + +// PostgresFixture is the Fixture of one of hopper's Postgres drivers, which +// can also run SQL on hopper's schema. RunPostgres runs, with it, the tests +// of what those drivers share beyond the driver interface: history's +// partitions and their indexes, the tags tables and their triggers, the rows +// clients of earlier releases leave, and the planner statistics the leader +// keeps. +type PostgresFixture[TTx any] interface { + Fixture[TTx] + // Exec runs a SQL statement with hopper's tables on the search path, for + // rows the suite writes directly, such as those a client of the previous + // release leaves behind. QueryText returns the single text value a query + // selects, for what the driver's reads do not show. Both report a + // failure with t.Error rather than t.Fatal, so the suite may call them + // from any goroutine. + Exec(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) + QueryText(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) string +} + +// RunPostgres runs the tests of what hopper's Postgres drivers share, which +// a Postgres driver passes besides Run's. +func RunPostgres[TTx any](t *testing.T, f PostgresFixture[TTx]) { + t.Helper() + tests := []struct { + name string + fn func(*testing.T, PostgresFixture[TTx]) + }{ + {"JobTagsStored", testJobTagsStored[TTx]}, + {"TagsRowsFollowJobs", testTagsRowsFollowJobs[TTx]}, + {"TagsTableMaintained", testTagsTableMaintained[TTx]}, + {"TagsTableMaintainedPastJobsError", testTagsTableMaintainedPastJobsError[TTx]}, + {"TagsFollowHistoryRetention", testTagsFollowHistoryRetention[TTx]}, + {"TagsPartitionWaitsForHistory", testTagsPartitionWaitsForHistory[TTx]}, + {"TagsLeaveWithTheirHour", testTagsLeaveWithTheirHour[TTx]}, + {"TagsHourOutlastsItsDelete", testTagsHourOutlastsItsDelete[TTx]}, + {"TagsHourDroppedMeanwhile", testTagsHourDroppedMeanwhile[TTx]}, + {"TagsDayGoesWithItsLastHour", testTagsDayGoesWithItsLastHour[TTx]}, + {"TagsFollowDirectUpdates", testTagsFollowDirectUpdates[TTx]}, + {"TagsFollowTruncate", testTagsFollowTruncate[TTx]}, + {"JobLogsStored", testJobLogsStored[TTx]}, + {"RetryOrderingKeyAgingOrder", testRetryOrderingKeyAgingOrder[TTx]}, + {"BatchProgressRaces", testBatchProgressRaces[TTx]}, + {"StreamsPartitioned", testStreamsPartitioned[TTx]}, + {"HistoryMaintainWhileInUse", testHistoryMaintainWhileInUse[TTx]}, + {"HistoryIndexes", testHistoryIndexes[TTx]}, + {"HistoryRetentionCompletesIndex", testHistoryRetentionCompletesIndex[TTx]}, {"DepsTableMaintained", testDepsTableMaintained[TTx]}, } for _, tc := range tests { @@ -157,6 +221,174 @@ func Run[TTx any](t *testing.T, f Fixture[TTx]) { } } +// Postgres implements a PostgresFixture's hooks that are statements on +// hopper's Postgres schema, for the fixtures of hopper's Postgres drivers to +// embed, which then implement only NewDriver, Begin and SQL. +type Postgres[TTx any] struct { + // SQL runs statements on a driver's database, as PostgresFixture's Exec + // and QueryText describe. + SQL interface { + Exec(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) + QueryText(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) string + } +} + +// Exec implements PostgresFixture. +func (p Postgres[TTx]) Exec(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) { + t.Helper() + p.SQL.Exec(ctx, t, d, query, args...) +} + +// QueryText implements PostgresFixture. +func (p Postgres[TTx]) QueryText(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) string { + t.Helper() + return p.SQL.QueryText(ctx, t, d, query, args...) +} + +// ExpireLease implements Fixture. +func (p Postgres[TTx]) ExpireLease(ctx context.Context, t *testing.T, d driver.Driver[TTx], clientID int64) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_clients SET expires_at = now() - interval '1s' WHERE id = $1", clientID) +} + +// ExpireLeader implements Fixture. +func (p Postgres[TTx]) ExpireLeader(ctx context.Context, t *testing.T, d driver.Driver[TTx]) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_leader SET expires_at = now() - interval '1s'") +} + +// MakeClaimable implements Fixture. +func (p Postgres[TTx]) MakeClaimable(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET scheduled_at = now() WHERE id = $1::uuid", id.String()) +} + +// AgeAttempt implements Fixture. +func (p Postgres[TTx]) AgeAttempt(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID, age time.Duration) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET attempted_at = now() - make_interval(secs => $2::float8) WHERE id = $1::uuid", id.String(), age.Seconds()) +} + +// CountLive implements Fixture. +func (p Postgres[TTx]) CountLive(ctx context.Context, t *testing.T, d driver.Driver[TTx]) int { + t.Helper() + return p.count(ctx, t, d, "SELECT count(*)::text FROM hopper_jobs") +} + +// CountHistory implements Fixture. +func (p Postgres[TTx]) CountHistory(ctx context.Context, t *testing.T, d driver.Driver[TTx]) int { + t.Helper() + return p.count(ctx, t, d, "SELECT count(*)::text FROM hopper_job_history") +} + +// ExpireJob implements Fixture. +func (p Postgres[TTx]) ExpireJob(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET expires_at = now() - interval '1s' WHERE id = $1::uuid", id.String()) +} + +// StaleWake implements Fixture. +func (p Postgres[TTx]) StaleWake(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET wake_requested_at = attempted_at - interval '1s' WHERE id = $1::uuid", id.String()) +} + +// MakePending implements Fixture. +func (p Postgres[TTx]) MakePending(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET state = 'pending' WHERE id = $1::uuid", id.String()) +} + +// SnoozeReason implements Fixture. +func (p Postgres[TTx]) SnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) string { + t.Helper() + return p.SQL.QueryText(ctx, t, d, "SELECT coalesce(snooze_reason, '') FROM hopper_jobs WHERE id = $1::uuid", id.String()) +} + +// SetSnoozeReason implements Fixture. +func (p Postgres[TTx]) SetSnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID, reason string) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET snooze_reason = $2 WHERE id = $1::uuid", id.String(), reason) +} + +// SetProgress implements Fixture. +func (p Postgres[TTx]) SetProgress(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID, value string) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET progress = $2::jsonb WHERE id = $1::uuid", id.String(), value) +} + +// CountEdges implements Fixture. +func (p Postgres[TTx]) CountEdges(ctx context.Context, t *testing.T, d driver.Driver[TTx]) int { + t.Helper() + return p.count(ctx, t, d, "SELECT count(*)::text FROM hopper_job_deps") +} + +// WaitForLockWait implements Fixture: a statement on the driver's schema, +// whose name is in every statement's text, waits for a lock. +func (p Postgres[TTx]) WaitForLockWait(ctx context.Context, t *testing.T, d driver.Driver[TTx]) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for p.SQL.QueryText(ctx, t, d, `SELECT count(*)::text FROM pg_stat_activity + WHERE wait_event_type = 'Lock' AND position(current_schema() IN query) > 0 AND pid <> pg_backend_pid()`) == "0" { + if time.Now().After(deadline) { + t.Fatal("no statement started waiting for a lock") + } + time.Sleep(5 * time.Millisecond) + } +} + +func (p Postgres[TTx]) count(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string) int { + t.Helper() + n, err := strconv.Atoi(p.SQL.QueryText(ctx, t, d, query)) + if err != nil { + t.Error(err) + } + return n +} + +// singleWriter reports whether d's engine runs one writing transaction at a +// time. +func singleWriter[TTx any](d driver.Driver[TTx]) bool { return d.Capabilities().SingleWriter } + +// jsonEqual reports whether two JSON texts hold the same value: an engine +// need not keep the text as given (Postgres's jsonb orders an object's keys +// and spaces them), only the value. +func jsonEqual(got []byte, want string) bool { + var a, b any + if json.Unmarshal(got, &a) != nil || json.Unmarshal([]byte(want), &b) != nil { + return false + } + return fmt.Sprint(normalizeJSON(a)) == fmt.Sprint(normalizeJSON(b)) +} + +// normalizeJSON gives a decoded JSON value a printed form that does not +// depend on its object's key order. +func normalizeJSON(v any) any { + switch v := v.(type) { + case map[string]any: + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + slices.Sort(keys) + out := make([]string, len(keys)) + for i, k := range keys { + out[i] = strconv.Quote(k) + ":" + fmt.Sprint(normalizeJSON(v[k])) + } + return "{" + strings.Join(out, ",") + "}" + case []any: + out := make([]string, len(v)) + for i, e := range v { + out[i] = fmt.Sprint(normalizeJSON(e)) + } + return "[" + strings.Join(out, ",") + "]" + case string: + return strconv.Quote(v) + default: + return v + } +} + // noBatchesExec finalizes with JobFinalizeParams.NoBatches set, which a // caller may when none of the jobs belongs to a batch: the tests of jobs // outside batches see the same results through it. diff --git a/drivertest/flow.go b/drivertest/flow.go index 5ab1980..42aac9e 100644 --- a/drivertest/flow.go +++ b/drivertest/flow.go @@ -243,7 +243,7 @@ func testBatches[TTx any](t *testing.T, f Fixture[TTx]) { if err := json.Unmarshal(kinds["alert"].Metadata, &meta); err != nil || meta.BatchID != id.String() || meta.BatchFailed != 1 { t.Errorf("callback metadata = %s (%v)", kinds["alert"].Metadata, err) } - if j := kinds["alert"]; j.Queue != "callbacks" || j.Priority != 1 || j.MaxAttempts != 3 || string(j.Args) != `{"run": 9}` { + if j := kinds["alert"]; j.Queue != "callbacks" || j.Priority != 1 || j.MaxAttempts != 3 || !jsonEqual(j.Args, `{"run": 9}`) { t.Errorf("callback job = %+v", j) } @@ -273,22 +273,23 @@ func testBatches[TTx any](t *testing.T, f Fixture[TTx]) { } } -// testBatchesConcurrent finalizes a batch's jobs in concurrent statements. -// Each records its progress without waiting on the others, so a -// transaction that finalized a job and is still open holds up none of its -// siblings' finalizes. The last two, each missing the other's uncommitted -// progress, leave the batch open, and BatchSettle, run by each finalizing -// client after its statement commits, completes it once; the sweep finds -// what a client could not settle. -func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { - ctx := context.Background() - d := f.NewDriver(t) - exec := d.Executor() - clientID := register(ctx, t, exec) +// batchHelpers holds what the batch concurrency tests share: inserting a +// batch of n jobs of kind with callbacks on the callbacks queue, settling +// batches, and counting a batch's callbacks by kind. +type batchHelpers struct { + newBatch func(kind string, n int) driver.JobID + settle func(ids ...driver.JobID) driver.BatchSettleResult + sweep func() driver.BatchSettleResult + callbacks func(batch driver.JobID) map[string]int + completed func(id driver.JobID) driver.JobFinalize +} + +func newBatchHelpers(ctx context.Context, t *testing.T, exec driver.Executor, clientID int64) batchHelpers { cb := func(kind string) *driver.JobInsertParams { return &driver.JobInsertParams{Kind: kind, Queue: "callbacks", Priority: 2, MaxAttempts: 3, Args: json.RawMessage(`{}`)} } - callbacks := func(batch driver.JobID) map[string]int { + h := batchHelpers{} + h.callbacks = func(batch driver.JobID) map[string]int { t.Helper() jobs, err := exec.JobList(ctx, driver.JobListParams{Queue: "callbacks", Limit: 1000}) if err != nil { @@ -305,7 +306,7 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } return kinds } - newBatch := func(kind string, n int) driver.JobID { + h.newBatch = func(kind string, n int) driver.JobID { t.Helper() id, err := exec.BatchInsert(ctx, driver.BatchInsertParams{Total: n, OnSuccess: cb("ok"), OnFailure: cb("failed"), OnComplete: cb("always")}) if err != nil { @@ -318,7 +319,7 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { insert(ctx, t, exec, p) return id } - settle := func(ids ...driver.JobID) driver.BatchSettleResult { + h.settle = func(ids ...driver.JobID) driver.BatchSettleResult { t.Helper() p := driver.BatchSettleParams{IDs: ids, Limit: 100} res, err := exec.BatchSettle(ctx, p) @@ -327,62 +328,209 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } return res } - sweep := func() driver.BatchSettleResult { t.Helper(); return settle() } - completed := func(id driver.JobID) driver.JobFinalize { + h.sweep = func() driver.BatchSettleResult { t.Helper(); return h.settle() } + h.completed = func(id driver.JobID) driver.JobFinalize { return driver.JobFinalize{ID: id, AttemptedBy: clientID, Fence: 1, State: driver.JobStateCompleted, Archive: true} } + return h +} - batch := newBatch("pair", 2) +// testBatchesConcurrent finalizes a batch's jobs in concurrent statements. +// On an engine that locks rows, each records its progress without waiting +// on the others, so a transaction that finalized a job and is still open +// holds up none of its siblings' finalizes. The last two, each missing the +// other's uncommitted progress, leave the batch open, and BatchSettle, run +// by each finalizing client after its statement commits, completes it once; +// the sweep finds what a client could not settle. On an engine that runs +// one writer at a time, the sibling's finalize waits for the transaction, +// and completes the batch itself. +func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { + ctx := context.Background() + d := f.NewDriver(t) + exec := d.Executor() + clientID := register(ctx, t, exec) + h := newBatchHelpers(ctx, t, exec, clientID) + + batch := h.newBatch("pair", 2) jobs := claim(ctx, t, exec, clientID, 2) if len(jobs) != 2 { t.Fatalf("claimed %d", len(jobs)) } tx, commit, rollback := f.Begin(ctx, t, d) defer rollback() //nolint:errcheck // already committed on the happy path - finalizeCurrent(ctx, t, d.UnwrapTx(tx), completed(jobs[0].ID)) - sctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - if applied, err := exec.JobFinalizeMany(sctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{completed(jobs[1].ID)}}); err != nil || len(applied) != 1 { - t.Fatalf("a sibling's finalize while the transaction is open = %v, %v; it must not wait for it", applied, err) - } - if res := settle(batch); res.Completed != 0 { - t.Errorf("BatchSettle with a job's finalize uncommitted = %+v", res) - } - if err := commit(); err != nil { - t.Fatal(err) - } - if b, err := exec.BatchGet(ctx, batch); err != nil || b.Pending != 0 || !b.CompletedAt.IsZero() { - t.Fatalf("batch after both finalizes = %+v, %v; the race should leave it open", b, err) - } - if res := settle(batch); res.Completed != 1 { - t.Errorf("BatchSettle = %+v; want 1 completed", res) + finalizeCurrent(ctx, t, d.UnwrapTx(tx), h.completed(jobs[0].ID)) + if singleWriter(d) { + done := make(chan []driver.JobID, 1) + go func() { + applied, err := exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{h.completed(jobs[1].ID)}}) + if err != nil { + t.Error(err) + } + done <- applied + }() + f.WaitForLockWait(ctx, t, d) + if err := commit(); err != nil { + t.Fatal(err) + } + if applied := <-done; len(applied) != 1 { + t.Fatalf("the sibling's finalize after the transaction = %v", applied) + } + if res := h.settle(batch); res.Completed != 0 { + t.Errorf("BatchSettle of a batch its last finalize completed = %+v", res) + } + } else { + sctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if applied, err := exec.JobFinalizeMany(sctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{h.completed(jobs[1].ID)}}); err != nil || len(applied) != 1 { + t.Fatalf("a sibling's finalize while the transaction is open = %v, %v; it must not wait for it", applied, err) + } + if res := h.settle(batch); res.Completed != 0 { + t.Errorf("BatchSettle with a job's finalize uncommitted = %+v", res) + } + if err := commit(); err != nil { + t.Fatal(err) + } + if b, err := exec.BatchGet(ctx, batch); err != nil || b.Pending != 0 || !b.CompletedAt.IsZero() { + t.Fatalf("batch after both finalizes = %+v, %v; the race should leave it open", b, err) + } + if res := h.settle(batch); res.Completed != 1 { + t.Errorf("BatchSettle = %+v; want 1 completed", res) + } } - if res := sweep(); res != (driver.BatchSettleResult{}) { + if res := h.sweep(); res != (driver.BatchSettleResult{}) { t.Errorf("sweep after the batch was settled = %+v", res) } if b, err := exec.BatchGet(ctx, batch); err != nil || b.Pending != 0 || b.Failed != 0 || b.CompletedAt.IsZero() { t.Errorf("settled batch = %+v, %v", b, err) } - if kinds := callbacks(batch); len(kinds) != 2 || kinds["ok"] != 1 || kinds["always"] != 1 { + if kinds := h.callbacks(batch); len(kinds) != 2 || kinds["ok"] != 1 || kinds["always"] != 1 { t.Errorf("callbacks = %v, want ok and always once", kinds) } + // Many batches, their jobs finalized by concurrent clients in mixed + // sets of results, each client settling its batches after each + // statement as the finalizer does: every batch completes exactly once, + // with no sweep, its failures counted and the callbacks its outcome + // calls for. Every third batch has its first two jobs discarded, which + // may land in different statements. + const batches, size, clients, discards = 12, 6, 4, 2 + ids := make([]driver.JobID, batches) + wantFailed := map[string]int{} + for i := range ids { + kind := fmt.Sprintf("many%d", i) + ids[i] = h.newBatch(kind, size) + if i%3 == 0 { + wantFailed[kind] = discards + } + } + var wg sync.WaitGroup + for range clients { + owner := register(ctx, t, exec) + wg.Go(func() { + for { + res, err := exec.JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: owner, Limit: 3}) + if err != nil { + t.Error(err) + return + } + if len(res.Jobs) == 0 { + return + } + results := make([]driver.JobFinalize, len(res.Jobs)) + for i, j := range res.Jobs { + results[i] = driver.JobFinalize{ID: j.ID, AttemptedBy: owner, Fence: j.Fence, State: driver.JobStateCompleted, Archive: true} + if argsN(j) < wantFailed[j.Kind] { + results[i].State, results[i].Error = driver.JobStateDiscarded, &driver.AttemptError{Attempt: 1, Error: "x"} + } + } + if _, err := exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: results}); err != nil { + t.Error(err) + return + } + var batches []driver.JobID + for _, j := range res.Jobs { + if !slices.Contains(batches, j.BatchID) { + batches = append(batches, j.BatchID) + } + } + if _, err := exec.BatchSettle(ctx, driver.BatchSettleParams{IDs: batches}); err != nil { + t.Error(err) + return + } + } + }) + } + wg.Wait() + if res := h.sweep(); res.Completed != 0 { + t.Errorf("the sweep completed %d batches the clients' settles left open", res.Completed) + } + for i, id := range ids { + want := wantFailed[fmt.Sprintf("many%d", i)] + b, err := exec.BatchGet(ctx, id) + if err != nil || b.CompletedAt.IsZero() || b.Pending != 0 || b.Failed != want { + t.Errorf("batch %d = %+v, %v; want %d failed", i, b, err, want) + continue + } + outcome := "ok" + if want > 0 { + outcome = "failed" + } + if kinds := h.callbacks(id); len(kinds) != 2 || kinds["always"] != 1 || kinds[outcome] != 1 { + t.Errorf("batch %d (%d failed): callbacks %v", i, want, kinds) + } + } + + // A failure is counted where retention does not reach: a batch whose + // failed job's history is gone before the batch completes still fails. + pruned := h.newBatch("pruned", 2) + jobs = claim(ctx, t, exec, clientID, 2) + if len(jobs) != 2 { + t.Fatalf("claimed %d", len(jobs)) + } + finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: jobs[0].ID, AttemptedBy: clientID, State: driver.JobStateDiscarded, Error: &driver.AttemptError{Attempt: 1, Error: "x"}}) + h.sweep() + if _, err := exec.JobDelete(ctx, jobs[0].ID); err != nil { + t.Fatal(err) + } + finalizeCurrent(ctx, t, exec, h.completed(jobs[1].ID)) + if b, err := exec.BatchGet(ctx, pruned); err != nil || b.CompletedAt.IsZero() || b.Failed != 1 { + t.Errorf("batch whose failure's history was pruned = %+v, %v", b, err) + } + if kinds := h.callbacks(pruned); kinds["failed"] != 1 || kinds["ok"] != 0 { + t.Errorf("callbacks of the pruned batch = %v", kinds) + } +} + +// testBatchProgressRaces covers the progress rows Postgres's finalizes +// record a batch's progress in: a settle that cannot get a batch row in +// time leaves the batch to the sweep; of two clients that finalized a +// batch's last jobs at the same time, the one settling second waits for the +// first's settle and completes the batch; the sweep walks batches in ID +// order; a batch row from before callbacks were checked still completes; +// and no progress rows are left once every batch is settled. +func testBatchProgressRaces[TTx any](t *testing.T, f PostgresFixture[TTx]) { + ctx := context.Background() + d := f.NewDriver(t) + exec := d.Executor() + clientID := register(ctx, t, exec) + h := newBatchHelpers(ctx, t, exec, clientID) + // A settle that cannot get a batch row in time leaves the batch to the // sweep, without an error. - held := newBatch("held", 3) - jobs = claim(ctx, t, exec, clientID, 3) + held := h.newBatch("held", 3) + jobs := claim(ctx, t, exec, clientID, 3) if len(jobs) != 3 { t.Fatalf("claimed %d", len(jobs)) } - finalizeCurrent(ctx, t, exec, completed(jobs[0].ID)) - tx, commit, rollback = f.Begin(ctx, t, d) + finalizeCurrent(ctx, t, exec, h.completed(jobs[0].ID)) + tx, commit, rollback := f.Begin(ctx, t, d) defer rollback() //nolint:errcheck // already committed on the happy path if res, err := d.UnwrapTx(tx).BatchSettle(ctx, driver.BatchSettleParams{IDs: []driver.JobID{held}}); err != nil || res.Folded != 1 { t.Fatalf("BatchSettle in a transaction = %+v, %v", res, err) } - finalizeCurrent(ctx, t, exec, completed(jobs[1].ID)) + finalizeCurrent(ctx, t, exec, h.completed(jobs[1].ID)) start := time.Now() - if res := settle(held); res != (driver.BatchSettleResult{}) { + if res := h.settle(held); res != (driver.BatchSettleResult{}) { t.Errorf("BatchSettle of a held batch row = %+v", res) } if took := time.Since(start); took > 3*time.Second { @@ -391,26 +539,26 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { if err := commit(); err != nil { t.Fatal(err) } - finalizeCurrent(ctx, t, exec, completed(jobs[2].ID)) + finalizeCurrent(ctx, t, exec, h.completed(jobs[2].ID)) if b, err := exec.BatchGet(ctx, held); err != nil || b.Pending != 0 || b.CompletedAt.IsZero() { t.Errorf("held batch = %+v, %v", b, err) } - if kinds := callbacks(held); len(kinds) != 2 || kinds["ok"] != 1 || kinds["always"] != 1 { + if kinds := h.callbacks(held); len(kinds) != 2 || kinds["ok"] != 1 || kinds["always"] != 1 { t.Errorf("callbacks of the held batch = %v", kinds) } // Of two clients that finalized a batch's last jobs at the same time, // the one settling second waits for the first's settle, which could not // see its progress, and completes the batch. - raced := newBatch("raced", 2) + raced := h.newBatch("raced", 2) jobs = claim(ctx, t, exec, clientID, 2) if len(jobs) != 2 { t.Fatalf("claimed %d", len(jobs)) } txB, commitB, rollbackB := f.Begin(ctx, t, d) defer rollbackB() //nolint:errcheck // already committed on the happy path - finalizeCurrent(ctx, t, d.UnwrapTx(txB), completed(jobs[1].ID)) - finalizeCurrent(ctx, t, exec, completed(jobs[0].ID)) + finalizeCurrent(ctx, t, d.UnwrapTx(txB), h.completed(jobs[1].ID)) + finalizeCurrent(ctx, t, exec, h.completed(jobs[0].ID)) txA, commitA, rollbackA := f.Begin(ctx, t, d) defer rollbackA() //nolint:errcheck // already committed on the happy path if res, err := d.UnwrapTx(txA).BatchSettle(ctx, driver.BatchSettleParams{IDs: []driver.JobID{raced}}); err != nil || res.Completed != 0 { @@ -446,14 +594,14 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { // The sweep walks batches in ID order from After. var open [2]driver.JobID for i := range open { - open[i] = newBatch(fmt.Sprintf("open%d", i), 2) + open[i] = h.newBatch(fmt.Sprintf("open%d", i), 2) } var rest []driver.JobFinalize for _, j := range claim(ctx, t, exec, clientID, 4) { if argsN(j) == 0 { - finalizeCurrent(ctx, t, exec, completed(j.ID)) + finalizeCurrent(ctx, t, exec, h.completed(j.ID)) } else { - rest = append(rest, completed(j.ID)) + rest = append(rest, h.completed(j.ID)) } } slices.SortFunc(open[:], func(a, b driver.JobID) int { return bytes.Compare(a[:], b[:]) }) @@ -465,7 +613,7 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { if err != nil || second.Folded != 1 || second.Last != open[1] { t.Fatalf("second chunk = %+v, %v; want %s", second, err, open[1]) } - if res := sweep(); res.Folded != 0 { + if res := h.sweep(); res.Folded != 0 { t.Errorf("sweep after both chunks = %+v", res) } for _, id := range open { @@ -485,7 +633,7 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { VALUES (1, 1, '{"kind": "legacy", "queue": "callbacks", "priority": 0, "max_attempts": 99999}') RETURNING id::text`) f.Exec(ctx, t, d, `INSERT INTO hopper_batch_progress (batch_id, done, failed) VALUES ($1::uuid, 1, 0), ('00000000-0000-7000-8000-000000000001', 1, 0)`, legacy) - if res := sweep(); res.Completed != 1 { + if res := h.sweep(); res.Completed != 1 { t.Errorf("sweep with a legacy batch = %+v; want it completed", res) } cbs, err := exec.JobList(ctx, driver.JobListParams{Queue: "callbacks", Kinds: []string{"legacy"}, Limit: 10}) @@ -498,98 +646,7 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { t.Errorf("batch finished after its sweep = %+v, %v", b, err) } } - - // Many batches, their jobs finalized by concurrent clients in mixed - // sets of results, each client settling its batches after each - // statement as the finalizer does: every batch completes exactly once, - // with no sweep, its failures counted and the callbacks its outcome - // calls for. Every third batch has its first two jobs discarded, which - // may land in different statements. - const batches, size, clients, discards = 12, 6, 4, 2 - ids := make([]driver.JobID, batches) - wantFailed := map[string]int{} - for i := range ids { - kind := fmt.Sprintf("many%d", i) - ids[i] = newBatch(kind, size) - if i%3 == 0 { - wantFailed[kind] = discards - } - } - var wg sync.WaitGroup - for range clients { - owner := register(ctx, t, exec) - wg.Go(func() { - for { - res, err := exec.JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: owner, Limit: 3}) - if err != nil { - t.Error(err) - return - } - if len(res.Jobs) == 0 { - return - } - results := make([]driver.JobFinalize, len(res.Jobs)) - for i, j := range res.Jobs { - results[i] = driver.JobFinalize{ID: j.ID, AttemptedBy: owner, Fence: j.Fence, State: driver.JobStateCompleted, Archive: true} - if argsN(j) < wantFailed[j.Kind] { - results[i].State, results[i].Error = driver.JobStateDiscarded, &driver.AttemptError{Attempt: 1, Error: "x"} - } - } - if _, err := exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: results}); err != nil { - t.Error(err) - return - } - var batches []driver.JobID - for _, j := range res.Jobs { - if !slices.Contains(batches, j.BatchID) { - batches = append(batches, j.BatchID) - } - } - if _, err := exec.BatchSettle(ctx, driver.BatchSettleParams{IDs: batches}); err != nil { - t.Error(err) - return - } - } - }) - } - wg.Wait() - if res := sweep(); res.Completed != 0 { - t.Errorf("the sweep completed %d batches the clients' settles left open", res.Completed) - } - for i, id := range ids { - want := wantFailed[fmt.Sprintf("many%d", i)] - b, err := exec.BatchGet(ctx, id) - if err != nil || b.CompletedAt.IsZero() || b.Pending != 0 || b.Failed != want { - t.Errorf("batch %d = %+v, %v; want %d failed", i, b, err, want) - continue - } - outcome := "ok" - if want > 0 { - outcome = "failed" - } - if kinds := callbacks(id); len(kinds) != 2 || kinds["always"] != 1 || kinds[outcome] != 1 { - t.Errorf("batch %d (%d failed): callbacks %v", i, want, kinds) - } - } - - // A failure is counted where retention does not reach: a batch whose - // failed job's history is gone before the batch completes still fails. - pruned := newBatch("pruned", 2) - jobs = claim(ctx, t, exec, clientID, 2) - if len(jobs) != 2 { - t.Fatalf("claimed %d", len(jobs)) - } - finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: jobs[0].ID, AttemptedBy: clientID, State: driver.JobStateDiscarded, Error: &driver.AttemptError{Attempt: 1, Error: "x"}}) - sweep() - f.Exec(ctx, t, d, "DELETE FROM hopper_job_history WHERE kind = 'pruned'") - finalizeCurrent(ctx, t, exec, completed(jobs[1].ID)) - if b, err := exec.BatchGet(ctx, pruned); err != nil || b.CompletedAt.IsZero() || b.Failed != 1 { - t.Errorf("batch whose failure's history was pruned = %+v, %v", b, err) - } - if kinds := callbacks(pruned); kinds["failed"] != 1 || kinds["ok"] != 0 { - t.Errorf("callbacks of the pruned batch = %v", kinds) - } - + h.sweep() if left := f.QueryText(ctx, t, d, "SELECT count(*)::text FROM hopper_batch_progress"); left != "0" { t.Errorf("%s progress rows left after every batch completed and settled", left) } diff --git a/drivertest/jobs.go b/drivertest/jobs.go index 197b67b..688e0e6 100644 --- a/drivertest/jobs.go +++ b/drivertest/jobs.go @@ -129,6 +129,10 @@ func testClaim[TTx any](t *testing.T, f Fixture[TTx]) { } } +// testClaimSkipsLocked claims while another client's claim is held open in +// a transaction. An engine that locks rows skips the held jobs and claims +// the rest at once; one that runs one writer at a time waits for the +// transaction, and then claims the rest. func testClaimSkipsLocked[TTx any](t *testing.T, f Fixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) @@ -136,7 +140,7 @@ func testClaimSkipsLocked[TTx any](t *testing.T, f Fixture[TTx]) { insert(ctx, t, exec, params("k", 4)) a, b := register(ctx, t, exec), register(ctx, t, exec) - tx, _, rollback := f.Begin(ctx, t, d) + tx, commit, rollback := f.Begin(ctx, t, d) defer rollback() //nolint:errcheck // cleanup heldRes, err := d.UnwrapTx(tx).JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: a, Limit: 2}) held := heldRes.Jobs @@ -151,6 +155,17 @@ func testClaimSkipsLocked[TTx any](t *testing.T, f Fixture[TTx]) { } done <- res.Jobs }() + if singleWriter(d) { + f.WaitForLockWait(ctx, t, d) + select { + case jobs := <-done: + t.Fatalf("B claimed %d jobs while A's transaction wrote", len(jobs)) + case <-time.After(100 * time.Millisecond): + } + if err := commit(); err != nil { + t.Fatal(err) + } + } select { case jobs := <-done: if len(jobs) != 2 { @@ -247,8 +262,9 @@ func testFinalize[TTx any](t *testing.T, f Fixture[TTx], noBatches bool) { // testFinalizeSkipLocked finalizes with SkipLocked while another // transaction holds one of the rows, as a worker's CompleteTx that has not -// ended does: the statement returns at once, without that job, and applies -// it once the row is free. +// ended does: the statement returns at once, without that job (on an engine +// with one writer at a time, without any job), and applies it once the row +// is free. func testFinalizeSkipLocked[TTx any](t *testing.T, f Fixture[TTx], noBatches bool) { ctx := context.Background() d := f.NewDriver(t) @@ -269,14 +285,25 @@ func testFinalizeSkipLocked[TTx any](t *testing.T, f Fixture[TTx], noBatches boo applied, err := exec.JobFinalizeMany(sctx, driver.JobFinalizeParams{SkipLocked: true, Jobs: []driver.JobFinalize{ failed, {ID: free, AttemptedBy: clientID, Fence: 1, State: driver.JobStateCompleted, Archive: true}, }}) - if err != nil || !slices.Equal(applied, []driver.JobID{free}) { - t.Fatalf("finalize with a held row = %v, %v; want only %s", applied, err, free) + want := []driver.JobID{free} + if singleWriter(d) { + want = nil + } + if err != nil || !slices.Equal(applied, want) { + t.Fatalf("finalize with a held row = %v, %v; want %v", applied, err, want) } if err := rollback(); err != nil { t.Fatal(err) } - applied, err = exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{SkipLocked: true, Jobs: []driver.JobFinalize{failed}}) - if err != nil || !slices.Equal(applied, []driver.JobID{held}) { + results := []driver.JobFinalize{failed} + want = []driver.JobID{held} + if singleWriter(d) { + results = append(results, driver.JobFinalize{ID: free, AttemptedBy: clientID, Fence: 1, State: driver.JobStateCompleted, Archive: true}) + want = append(want, free) + } + applied, err = exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{SkipLocked: true, Jobs: results}) + slices.SortFunc(applied, func(a, b driver.JobID) int { return slices.Index(want, a) - slices.Index(want, b) }) + if err != nil || !slices.Equal(applied, want) { t.Fatalf("finalize once the row is free = %v, %v", applied, err) } if j, err := exec.JobGet(ctx, held); err != nil || j.State != driver.JobStateRetryable { @@ -365,7 +392,7 @@ func testFinalizeTx[TTx any](t *testing.T, f Fixture[TTx]) { t.Fatal(err) } j, err := exec.JobGet(ctx, id) - if err != nil || j.State != driver.JobStateCompleted || j.FinalizedAt.IsZero() || string(j.Output) != `{"ok": true}` { + if err != nil || j.State != driver.JobStateCompleted || j.FinalizedAt.IsZero() || !jsonEqual(j.Output, `{"ok": true}`) { t.Errorf("job after commit = %+v, %v", j, err) } if live, hist := f.CountLive(ctx, t, d), f.CountHistory(ctx, t, d); live != 0 || hist != 1 { @@ -406,7 +433,7 @@ func testFinalizeTxConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } done <- applied }() - waitForLockWait(ctx, t, f, d) + f.WaitForLockWait(ctx, t, d) end := rollback if commitTx { end = commit @@ -429,21 +456,6 @@ func testFinalizeTxConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } } -// waitForLockWait waits until a statement on the driver's schema is waiting -// for a lock, so a test knows its competing statement is blocked rather -// than not yet started. The schema's name is in every statement's text. -func waitForLockWait[TTx any](ctx context.Context, t *testing.T, f Fixture[TTx], d driver.Driver[TTx]) { - t.Helper() - deadline := time.Now().Add(10 * time.Second) - for f.QueryText(ctx, t, d, `SELECT count(*)::text FROM pg_stat_activity - WHERE wait_event_type = 'Lock' AND position(current_schema() IN query) > 0 AND pid <> pg_backend_pid()`) == "0" { - if time.Now().After(deadline) { - t.Fatal("no statement started waiting for a lock") - } - time.Sleep(5 * time.Millisecond) - } -} - func testCancel[TTx any](t *testing.T, f Fixture[TTx]) { ctx := context.Background() exec := f.NewDriver(t).Executor() @@ -555,7 +567,7 @@ func testRetryOrderingKey[TTx any](t *testing.T, f Fixture[TTx]) { if _, err := exec.JobCancel(ctx, id(4)); err != nil { t.Fatal(err) } - f.Exec(ctx, t, d, "UPDATE hopper_jobs SET expires_at = now() - interval '1s' WHERE id = $1::uuid", id(6).String()) + f.ExpireJob(ctx, t, d, id(6)) wf, err := exec.WorkflowInsert(ctx, driver.WorkflowInsertParams{ Name: "w", Jobs: []driver.JobInsertParams{{Kind: "w", Queue: "wf", Priority: 2, MaxAttempts: 1}, {Kind: "w", Queue: "wf", Priority: 2, MaxAttempts: 1, OrderingKey: "a"}}, Deps: []driver.JobDependency{{Job: 1, DependsOn: 0}}, @@ -620,7 +632,7 @@ func testRetryOrderingKey[TTx any](t *testing.T, f Fixture[TTx]) { finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: id(0), AttemptedBy: clientID, State: driver.JobStateScheduled, Delay: time.Hour, Snooze: true, SnoozeReason: "waiting"}) claimed() wake("default", 10, 1) - if r := storedReason(ctx, t, f, d, id(0)); r != "" { + if r := f.SnoozeReason(ctx, t, d, id(0)); r != "" { t.Errorf("woken job kept reason %q", r) } claimed(id(0)) @@ -628,13 +640,13 @@ func testRetryOrderingKey[TTx any](t *testing.T, f Fixture[TTx]) { // A wake during the attempt makes its snooze run at once, without a reason. wake("default", 10, 0) finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: id(0), AttemptedBy: clientID, State: driver.JobStateScheduled, Delay: time.Hour, Snooze: true, SnoozeReason: "waiting"}) - if r := storedReason(ctx, t, f, d, id(0)); r != "" { + if r := f.SnoozeReason(ctx, t, d, id(0)); r != "" { t.Errorf("snooze after a wake kept reason %q", r) } claimed(id(0)) // A mark from before the attempt started does not count. - f.Exec(ctx, t, d, "UPDATE hopper_jobs SET wake_requested_at = attempted_at - interval '1s' WHERE id = $1::uuid", id(0).String()) + f.StaleWake(ctx, t, d, id(0)) finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: id(0), AttemptedBy: clientID, State: driver.JobStateRetryable, Delay: time.Hour, Error: &driver.AttemptError{Attempt: 1, Error: "x"}}) claimed() if n, err := exec.JobRetryOrderingKey(ctx, driver.JobRetryOrderingKeyParams{Queue: "default", OrderingKey: "none", Limit: 10}); err != nil || n != 0 { @@ -643,8 +655,7 @@ func testRetryOrderingKey[TTx any](t *testing.T, f Fixture[TTx]) { } // testRetryOrderingKeyConcurrent checks that a wake racing the end of an -// attempt is never lost, and that a wake and the leader's aging, which lock -// the same rows, never deadlock. +// attempt is never lost. func testRetryOrderingKeyConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) @@ -673,16 +684,44 @@ func testRetryOrderingKeyConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } }) wg.Wait() - if due := f.QueryText(ctx, t, d, "SELECT (scheduled_at <= now())::text FROM hopper_jobs WHERE id = $1::uuid", job.String()); due != "true" { + if !due(ctx, t, exec, job) { t.Fatalf("round %d: a wake racing the snooze was lost", round) } - f.Exec(ctx, t, d, "DELETE FROM hopper_jobs WHERE id = $1::uuid", job.String()) + if _, err := exec.JobDelete(ctx, job); err != nil { + t.Fatal(err) + } } +} - // Rows laid out in the heap in the reverse of their seq order: the - // aging pass meets them in one order and the wake locks them in the - // other. Each round rewrites them one by one, newest first, which - // appends their new versions in that order. +// due reports whether a live job's scheduled time has come, by the +// database's clock. +func due(ctx context.Context, t *testing.T, exec driver.Executor, id driver.JobID) bool { + t.Helper() + now, err := exec.Now(ctx) + if err != nil { + t.Fatal(err) + } + j, err := exec.JobGet(ctx, id) + if err != nil { + t.Fatal(err) + } + return !j.ScheduledAt.After(now) +} + +// testRetryOrderingKeyAgingOrder checks that a wake and the leader's aging, +// which lock the same rows, never deadlock, with rows laid out in the heap +// in the reverse of their seq order: the aging pass meets them in one order +// and the wake locks them in the other. Each round rewrites them one by +// one, newest first, which appends their new versions in that order. +func testRetryOrderingKeyAgingOrder[TTx any](t *testing.T, f PostgresFixture[TTx]) { + ctx := context.Background() + d := f.NewDriver(t) + exec := d.Executor() + wakeKey := func(key string) { + if _, err := exec.JobRetryOrderingKey(ctx, driver.JobRetryOrderingKeyParams{Queue: "default", OrderingKey: key, Limit: 1000}); err != nil { + t.Error(err) + } + } p := params("k", 300) for i := range p { p[i].OrderingKey, p[i].Priority, p[i].ScheduledAt = "aging", 3, time.Now().Add(time.Hour) @@ -1032,7 +1071,7 @@ func testJobDeleteRacesRetry[TTx any](t *testing.T, f Fixture[TTx]) { deleted, err = exec.JobDelete(ctx, id) done <- err }() - waitForLockWait(ctx, t, f, d) + f.WaitForLockWait(ctx, t, d) if err := commit(); err != nil { t.Fatal(err) } @@ -1071,7 +1110,7 @@ func testRetryOrderingKeyTx[TTx any](t *testing.T, f Fixture[TTx]) { t.Fatal(err) } finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: job, AttemptedBy: clientID, State: driver.JobStateScheduled, Delay: time.Hour, Snooze: true}) - if due := f.QueryText(ctx, t, d, "SELECT (scheduled_at <= now())::text FROM hopper_jobs WHERE id = $1::uuid", job.String()); due != "true" { + if !due(ctx, t, exec, job) { t.Error("a wake in a transaction older than the attempt was lost") } } diff --git a/drivertest/logs.go b/drivertest/logs.go index 26f4547..e9f2ee0 100644 --- a/drivertest/logs.go +++ b/drivertest/logs.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "slices" - "strconv" "sync" "testing" "time" @@ -111,25 +110,16 @@ func testJobLogs[TTx any](t *testing.T, f Fixture[TTx], noBatches bool) { // A snooze that logs nothing leaves a job without logs as it is, and // takes one whose only log is a snooze's back to none. - noLogs := func(what string) { - t.Helper() - if got := f.QueryText(ctx, t, d, "SELECT coalesce(logs::text, 'NULL') FROM hopper_jobs WHERE id = $1::uuid", other.String()); got != "NULL" { - t.Errorf("logs %s = %s, want NULL", what, got) - } - } run(other, driver.JobFinalize{State: driver.JobStateScheduled, Snooze: true}, 2, false) - noLogs("after a snooze that logged nothing") - attempt, err := strconv.Atoi(f.QueryText(ctx, t, d, "SELECT attempt::text FROM hopper_jobs WHERE id = $1::uuid", other.String())) - if err != nil { - t.Fatal(err) - } + check("after a snooze that logged nothing", logs(other)) + attempt := get(ctx, t, exec, other).Attempt run(other, driver.JobFinalize{State: driver.JobStateScheduled, Snooze: true, Log: &driver.AttemptLog{Attempt: attempt + 1, Log: "x\n"}}, 2, false) run(other, driver.JobFinalize{State: driver.JobStateScheduled, Snooze: true}, 2, false) - noLogs("after a snooze that logged nothing removed the only log") + check("after a snooze that logged nothing removed the only log", logs(other)) // An expired job takes its logs to history too. run(other, driver.JobFinalize{State: driver.JobStateRetryable, Error: &driver.AttemptError{Attempt: 1, Error: "x"}, Log: &a1}, 2, false) - f.Exec(ctx, t, d, "UPDATE hopper_jobs SET expires_at = now() - interval '1s' WHERE id = $1::uuid", other.String()) + f.ExpireJob(ctx, t, d, other) if expired, err := exec.JobDiscardExpired(ctx, 10); err != nil || len(expired) != 1 || expired[0].ID != other { t.Fatalf("expired = %v, %v", expired, err) } @@ -140,6 +130,34 @@ func testJobLogs[TTx any](t *testing.T, f Fixture[TTx], noBatches bool) { } } +// testJobLogsStored checks how the logs column is kept: a snooze that logs +// nothing leaves a job without logs as it is, stored as NULL rather than +// rewritten, and takes one whose only log is a snooze's back to NULL. +func testJobLogsStored[TTx any](t *testing.T, f PostgresFixture[TTx]) { + ctx := context.Background() + d := f.NewDriver(t) + exec := d.Executor() + id := insert(ctx, t, exec, params("k", 1))[0].Job.ID + clientID := register(ctx, t, exec) + snooze := func(log *driver.AttemptLog) { + t.Helper() + f.MakeClaimable(ctx, t, d, id) + j := claimOne(ctx, t, exec, clientID, id) + finalize(ctx, t, exec, driver.JobFinalize{ID: id, AttemptedBy: clientID, Fence: j.Fence, State: driver.JobStateScheduled, Snooze: true, Log: log}) + } + noLogs := func(what string) { + t.Helper() + if got := f.QueryText(ctx, t, d, "SELECT coalesce(logs::text, 'NULL') FROM hopper_jobs WHERE id = $1::uuid", id.String()); got != "NULL" { + t.Errorf("logs %s = %s, want NULL", what, got) + } + } + snooze(nil) + noLogs("after a snooze that logged nothing") + snooze(&driver.AttemptLog{Attempt: 1, Log: "x\n"}) + snooze(nil) + noLogs("after a snooze that logged nothing removed the only log") +} + // testJobLogsFenced finalizes a job from its attempt and, concurrently, // from clients that do not own it, such as one whose lease lapsed: only the // owner's log is appended, once, and only the owner's snooze that logged diff --git a/drivertest/messaging.go b/drivertest/messaging.go index 5b81df3..be093f5 100644 --- a/drivertest/messaging.go +++ b/drivertest/messaging.go @@ -81,7 +81,7 @@ func testPublish[TTx any](t *testing.T, f Fixture[TTx]) { } else if meta.MessageID != messageID { t.Error("deliveries of one message have different IDs") } - if j.OrderingKey != "alloc:42" || j.UniqueKey != "msg:evt-1" || j.Priority != 1 || j.ExpiresAt.IsZero() || !j.Await || string(j.Args) != `{"id": 42}` { + if j.OrderingKey != "alloc:42" || j.UniqueKey != "msg:evt-1" || j.Priority != 1 || j.ExpiresAt.IsZero() || !j.Await || !jsonEqual(j.Args, `{"id": 42}`) { t.Errorf("delivery = %+v", j) } switch j.Kind { diff --git a/drivertest/progress.go b/drivertest/progress.go index 18c11f5..4e5b09d 100644 --- a/drivertest/progress.go +++ b/drivertest/progress.go @@ -48,7 +48,7 @@ func testJobRecordProgress[TTx any](t *testing.T, f Fixture[TTx]) { } // A cursor is stored as given, a null within it included. got := record(exec, driver.JobRecordProgressParams{Fence: 1, Attempt: 1, Step: "download", Cursor: json.RawMessage(`{"page": 2, "etag": null}`)}) - if got == nil || got.Step != "download" || string(got.Cursor) != `{"etag": null, "page": 2}` || got.RecordedAt.IsZero() { + if got == nil || got.Step != "download" || !jsonEqual(got.Cursor, `{"etag": null, "page": 2}`) || got.RecordedAt.IsZero() { t.Fatalf("recorded = %+v", got) } if p := progress(); p == nil || p.Step != "download" || string(p.Cursor) != string(got.Cursor) || !p.RecordedAt.Equal(got.RecordedAt) { @@ -150,7 +150,7 @@ func testJobRecordProgressRacesFinalize[TTx any](t *testing.T, f Fixture[TTx]) { } done <- applied }() - waitForLockWait(ctx, t, f, d) + f.WaitForLockWait(ctx, t, d) if err := commit(); err != nil { t.Fatal(err) } @@ -178,7 +178,7 @@ func testJobRecordProgressRacesFinalize[TTx any](t *testing.T, f Fixture[TTx]) { } recorded <- p }() - waitForLockWait(ctx, t, f, d) + f.WaitForLockWait(ctx, t, d) if err := commit(); err != nil { t.Fatal(err) } @@ -254,7 +254,7 @@ func testJobProgressReset[TTx any](t *testing.T, f Fixture[TTx]) { if j, err := exec.JobRetry(ctx, driver.JobRetryParams{ID: inserted[1].Job.ID, FromStart: true}); err != nil || j.Progress != nil { t.Errorf("JobRetry FromStart from history = %+v, %v", j, err) } - f.Exec(ctx, t, d, `UPDATE hopper_jobs SET progress = '{"step": "e"}' WHERE id = $1`, inserted[0].Job.ID.String()) + f.SetProgress(ctx, t, d, inserted[0].Job.ID, `{"step": "e"}`) if j, err := exec.JobRetry(ctx, driver.JobRetryParams{ID: inserted[0].Job.ID}); err != nil || j.Progress == nil || j.Progress.Step != "e" { t.Errorf("JobRetry of a waiting job = %+v, %v", j, err) } @@ -271,9 +271,10 @@ func testJobProgressReset[TTx any](t *testing.T, f Fixture[TTx]) { `{"step": 3, "cursor": null, "at": "yesterday"}`: {}, `{"step": "s", "cursor": {"p": 1}, "at": "2026-01-02T03:04:05Z"}`: {Step: "s", Cursor: json.RawMessage(`{"p": 1}`), RecordedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)}, } { - f.Exec(ctx, t, d, `UPDATE hopper_jobs SET progress = $1::jsonb WHERE id = $2`, value, inserted[0].Job.ID.String()) + f.SetProgress(ctx, t, d, inserted[0].Job.ID, value) got := get(inserted[0].Job.ID).Progress - if (got == nil) != (want == nil) || (got != nil && (got.Step != want.Step || string(got.Cursor) != string(want.Cursor) || !got.RecordedAt.Equal(want.RecordedAt))) { + if (got == nil) != (want == nil) || (got != nil && (got.Step != want.Step || (got.Cursor != nil) != (want.Cursor != nil) || + (got.Cursor != nil && !jsonEqual(got.Cursor, string(want.Cursor))) || !got.RecordedAt.Equal(want.RecordedAt))) { t.Errorf("progress %s reads as %+v, want %+v", value, got, want) } } diff --git a/drivertest/snooze.go b/drivertest/snooze.go index d7ce0f6..b627f78 100644 --- a/drivertest/snooze.go +++ b/drivertest/snooze.go @@ -12,11 +12,11 @@ import ( "github.com/parallelworks/hopper/driver" ) -// storedReason reads the snooze_reason column itself, which the driver's +// storedReason reads the stored snooze reason itself, which the driver's // reads show only for a scheduled job. func storedReason[TTx any](ctx context.Context, t *testing.T, f Fixture[TTx], d driver.Driver[TTx], id driver.JobID) string { t.Helper() - return f.QueryText(ctx, t, d, "SELECT coalesce(snooze_reason, '') FROM hopper_jobs WHERE id = $1::uuid", id.String()) + return f.SnoozeReason(ctx, t, d, id) } // leaveReason writes a reason on a job directly, as a client of the @@ -24,7 +24,7 @@ func storedReason[TTx any](ctx context.Context, t *testing.T, f Fixture[TTx], d // leaves one on a running or retrying job. func leaveReason[TTx any](ctx context.Context, t *testing.T, f Fixture[TTx], d driver.Driver[TTx], id driver.JobID) { t.Helper() - f.Exec(ctx, t, d, "UPDATE hopper_jobs SET snooze_reason = 'left behind' WHERE id = $1::uuid", id.String()) + f.SetSnoozeReason(ctx, t, d, id, "left behind") } func testSnoozeReason[TTx any](t *testing.T, f Fixture[TTx], noBatches bool) { @@ -155,7 +155,7 @@ func testSnoozeReason[TTx any](t *testing.T, f Fixture[TTx], noBatches bool) { t.Fatal(err) } snooze("fifth") - f.Exec(ctx, t, d, "UPDATE hopper_jobs SET expires_at = now() - interval '1s' WHERE id = $1::uuid", id.String()) + f.ExpireJob(ctx, t, d, id) expired, err := exec.JobDiscardExpired(ctx, 10) if err != nil || len(expired) != 1 || expired[0].Errors[len(expired[0].Errors)-1].Error != "hopper: expired while snoozed: fifth" { t.Errorf("expired = %+v, %v", expired, err) @@ -200,13 +200,14 @@ func testSnoozeReasonConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } // Only this client finalizes the jobs it holds, so what the // claim cleared stays cleared until then. - ids := make([]string, len(res.Jobs)) - for i, j := range res.Jobs { - ids[i] = j.ID.String() + left := 0 + for _, j := range res.Jobs { + if f.SnoozeReason(ctx, t, d, j.ID) != "" { + left++ + } } - if left := f.QueryText(ctx, t, d, "SELECT count(snooze_reason)::text FROM hopper_jobs WHERE id = ANY($1::text[]::uuid[])", - "{"+strings.Join(ids, ",")+"}"); left != "0" { - t.Errorf("claim left snooze reasons on %s of the jobs it took", left) + if left != 0 { + t.Errorf("claim left snooze reasons on %d of the jobs it took", left) } results := make([]driver.JobFinalize, 0, 2*len(res.Jobs)) for i, j := range res.Jobs { diff --git a/drivertest/stream.go b/drivertest/stream.go index 34d2822..4272fd2 100644 --- a/drivertest/stream.go +++ b/drivertest/stream.go @@ -59,7 +59,7 @@ func testStreams[TTx any](t *testing.T, f Fixture[TTx]) { if e1.Position.IsZero() || !e1.Position.Less(e2.Position) || !e2.Position.Less(e3.Position) { t.Errorf("positions not ascending: %v %v %v", e1.Position, e2.Position, e3.Position) } - if e1.Key != "o1" || e1.Headers["n"] != "1" || e1.MessageID == "" || e1.CreatedAt.IsZero() || string(e1.Payload) != `{"i": 1}` { + if e1.Key != "o1" || e1.Headers["n"] != "1" || e1.MessageID == "" || e1.CreatedAt.IsZero() || !jsonEqual(e1.Payload, `{"i": 1}`) { t.Errorf("event = %+v", e1) } if got := read("", driver.StreamPosition{}); len(got) != 3 || got[0].Position != e1.Position || got[2].Position != e3.Position { @@ -89,7 +89,7 @@ func testStreams[TTx any](t *testing.T, f Fixture[TTx]) { if err != nil || len(consumers) != 2 || consumers[0].Name != "audit" || consumers[1].Name != "orders" { t.Fatalf("consumers = %v, %v", consumers, err) } - if c := consumers[1]; !c.Position.IsZero() || c.Queue != "orders" || string(c.Metadata) != `{"c": 1}` || c.CreatedAt.IsZero() || c.Snapshot == "" { + if c := consumers[1]; !c.Position.IsZero() || c.Queue != "orders" || !jsonEqual(c.Metadata, `{"c": 1}`) || c.CreatedAt.IsZero() || c.Snapshot == "" { t.Errorf("orders consumer = %+v", c) } if c := consumers[0]; c.MaxAttempts != 3 || c.Snapshot == consumers[1].Snapshot { @@ -107,7 +107,7 @@ func testStreams[TTx any](t *testing.T, f Fixture[TTx]) { if argsN(j) != 1 { j = jobs[1] } - if j.Kind != "stream:orders" || j.OrderingKey != "o1" || j.MaxAttempts != 10 || string(j.Args) != `{"i": 1}` { + if j.Kind != "stream:orders" || j.OrderingKey != "o1" || j.MaxAttempts != 10 || !jsonEqual(j.Args, `{"i": 1}`) { t.Errorf("delivery = %+v", j) } var meta struct { @@ -194,7 +194,12 @@ func testStreams[TTx any](t *testing.T, f Fixture[TTx]) { } // A transaction that commits late is delivered when it commits, and - // does not hold up the events of younger transactions meanwhile. + // does not hold up the events of younger transactions meanwhile. An + // engine that runs one writer at a time has no younger transaction to + // commit first: events commit in the order they are appended. + if singleWriter(d) { + return + } tx, commit, rollback := f.Begin(ctx, t, d) defer rollback() //nolint:errcheck // already committed on the happy path old, err := d.UnwrapTx(tx).StreamAppend(ctx, driver.StreamAppendParams{Topic: "order.held", Payload: json.RawMessage(`{}`)}) @@ -220,8 +225,30 @@ func testStreams[TTx any](t *testing.T, f Fixture[TTx]) { if res := pump("orders"); res.Delivered != 0 { t.Errorf("late commit delivered twice: %+v", res) } +} - // Retention keeps a partition ahead and drops nothing young. +// testStreamsRetention checks that retention drops no young event. +func testStreamsRetention[TTx any](t *testing.T, f Fixture[TTx]) { + ctx := context.Background() + exec := f.NewDriver(t).Executor() + for i := range 3 { + if _, err := exec.StreamAppend(ctx, driver.StreamAppendParams{Topic: "t", Payload: json.RawMessage(fmt.Sprintf(`{"i":%d}`, i))}); err != nil { + t.Fatal(err) + } + } + if _, err := exec.HistoryMaintain(ctx, driver.HistoryMaintainParams{CompletedRetention: time.Hour, FailedRetention: time.Hour, StreamRetention: time.Hour}); err != nil { + t.Fatal(err) + } + if got, err := exec.StreamRead(ctx, driver.StreamReadParams{Limit: 10}); err != nil || len(got) != 3 { + t.Errorf("events after maintenance = %d, %v; want 3", len(got), err) + } +} + +// testStreamsPartitioned checks that retention keeps a partition of the +// stream log ahead. +func testStreamsPartitioned[TTx any](t *testing.T, f PostgresFixture[TTx]) { + ctx := context.Background() + exec := f.NewDriver(t).Executor() m, err := exec.HistoryMaintain(ctx, driver.HistoryMaintainParams{CompletedRetention: time.Hour, FailedRetention: time.Hour, StreamRetention: time.Hour}) if err != nil { t.Fatal(err) @@ -229,9 +256,6 @@ func testStreams[TTx any](t *testing.T, f Fixture[TTx]) { if !containsPrefix(m.Created, "hopper_stream_events_") { t.Errorf("no stream partition created: %v", m.Created) } - if got := read("", driver.StreamPosition{}); len(got) != 7 { - t.Errorf("events after maintenance = %d, want 7", len(got)) - } } func containsPrefix(names []string, prefix string) bool { diff --git a/drivertest/tags.go b/drivertest/tags.go index 7963168..cccc7ea 100644 --- a/drivertest/tags.go +++ b/drivertest/tags.go @@ -18,7 +18,7 @@ import ( // testJobTags checks that a job's tags are stored with it on every insert // path, returned wherever the job is, kept through history and a retry // from it, and replaced by a replacing insert, and that a job without tags -// stores none. +// has none. func testJobTags[TTx any](t *testing.T, f Fixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) @@ -32,6 +32,102 @@ func testJobTags[TTx any](t *testing.T, f Fixture[TTx]) { t.Errorf("inserted job %d tags = %q, want %q", i, r.Job.Tags, p[i].Tags) } } + if j := get(ctx, t, exec, res[2].Job.ID); j.Tags != nil { + t.Errorf("untagged job's tags = %q, want none", j.Tags) + } + + clientID := register(ctx, t, exec) + running := claim(ctx, t, exec, clientID, 3) + for _, j := range running { + if want := p[argsN(j)].Tags; !slices.Equal(j.Tags, want) { + t.Errorf("claimed job %d tags = %q, want %q", argsN(j), j.Tags, want) + } + finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: j.ID, AttemptedBy: clientID, State: driver.JobStateCompleted, Archive: true}) + } + got, err := exec.JobGet(ctx, res[1].Job.ID) + if err != nil || got.State != driver.JobStateCompleted || !slices.Equal(got.Tags, p[1].Tags) { + t.Fatalf("job in history = %+v, %v", got, err) + } + retried, err := exec.JobRetry(ctx, driver.JobRetryParams{ID: res[0].Job.ID}) + if err != nil || !slices.Equal(retried.Tags, p[0].Tags) { + t.Errorf("job retried from history = %+v, %v", retried, err) + } + + u := params("u", 1)[0] + u.UniqueKey, u.Tags = "one", []string{"old"} + insert(ctx, t, exec, []driver.JobInsertParams{u}) + u.Tags = []string{"new"} + rep, err := exec.JobInsertMany(ctx, []driver.JobInsertParams{u}, driver.JobInsertOpts{OnConflict: driver.ConflictReplace}) + if err != nil || !rep[0].Duplicate || !slices.Equal(rep[0].Job.Tags, []string{"new"}) { + t.Errorf("replaced job = %+v, %v", rep, err) + } + if byTag, err := exec.JobList(ctx, driver.JobListParams{TagsAny: []string{"old", "new"}, Limit: 10}); err != nil || len(byTag) != 1 || !slices.Equal(byTag[0].Tags, []string{"new"}) { + t.Errorf("replaced job listed by its tags = %+v, %v", byTag, err) + } + u.Tags = nil + rep, err = exec.JobInsertMany(ctx, []driver.JobInsertParams{u}, driver.JobInsertOpts{OnConflict: driver.ConflictReplace}) + if err != nil || rep[0].Job.Tags != nil { + t.Errorf("job replaced without tags = %+v, %v", rep, err) + } + + if d.Capabilities().Copy { + c := params("copy", 2) + c[1].Tags = []string{"copied"} + copied, err := exec.JobInsertCopy(ctx, c, driver.JobInsertOpts{}) + if err != nil { + t.Fatal(err) + } + for i, r := range copied { + stored, err := exec.JobGet(ctx, r.Job.ID) + if err != nil || !slices.Equal(stored.Tags, c[i].Tags) || !slices.Equal(r.Job.Tags, c[i].Tags) { + t.Errorf("copied job %d tags = %q (result %q), want %q, %v", i, stored.Tags, r.Job.Tags, c[i].Tags, err) + } + } + } + + w := params("wf", 2) + w[0].Tags = []string{"flow"} + wf, err := exec.WorkflowInsert(ctx, driver.WorkflowInsertParams{Jobs: w, Deps: []driver.JobDependency{{Job: 1, DependsOn: 0}}}) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(wf.Jobs[0].Job.Tags, []string{"flow"}) || wf.Jobs[1].Job.Tags != nil { + t.Errorf("workflow job tags = %q, %q", wf.Jobs[0].Job.Tags, wf.Jobs[1].Job.Tags) + } + + // A batch's callback keeps its tags until the batch completes. + cb := &driver.JobInsertParams{Kind: "done", Queue: "callbacks", Priority: 1, MaxAttempts: 3, Tags: []string{"callback"}} + batch, err := exec.BatchInsert(ctx, driver.BatchInsertParams{Total: 1, OnComplete: cb}) + if err != nil { + t.Fatal(err) + } + b := params("shard", 1) + b[0].Queue, b[0].BatchID = "shards", batch + insert(ctx, t, exec, b) + shards, err := exec.JobClaim(ctx, driver.JobClaimParams{Queue: "shards", ClientID: clientID, Limit: 1}) + if err != nil || len(shards.Jobs) != 1 { + t.Fatalf("claim = %+v, %v", shards, err) + } + finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: shards.Jobs[0].ID, AttemptedBy: clientID, State: driver.JobStateCompleted, Archive: true}) + callbacks, err := exec.JobList(ctx, driver.JobListParams{Queue: "callbacks", Limit: 10}) + if err != nil || len(callbacks) != 1 || !slices.Equal(callbacks[0].Tags, []string{"callback"}) { + t.Errorf("callbacks = %+v, %v", callbacks, err) + } +} + +// testJobTagsStored checks how tags are stored on Postgres: a job without +// tags stores NULL; both tables' CHECKs refuse tags of another shape; tags a +// direct SQL write made odd read as what they hold, in listings, a retry and +// a batch's callback; and a job finished on a new installation keeps its +// tags rows out of the DEFAULT partition. +func testJobTagsStored[TTx any](t *testing.T, f PostgresFixture[TTx]) { + ctx := context.Background() + d := f.NewDriver(t) + exec := d.Executor() + p := params("k", 3) + p[0].Tags = []string{"tenant:acme", "nightly"} + p[1].Tags = []string{"a"} + insert(ctx, t, exec, p) if n := f.QueryText(ctx, t, d, "SELECT count(*)::text FROM hopper_jobs WHERE tags IS NULL"); n != "1" { t.Errorf("%s jobs store NULL tags, want 1 (the untagged one)", n) } @@ -117,82 +213,13 @@ func testJobTags[TTx any](t *testing.T, f Fixture[TTx]) { f.Exec(ctx, t, d, "DELETE FROM hopper_jobs WHERE queue = 'odd-tags'") clientID := register(ctx, t, exec) - running := claim(ctx, t, exec, clientID, 3) - for _, j := range running { - if want := p[argsN(j)].Tags; !slices.Equal(j.Tags, want) { - t.Errorf("claimed job %d tags = %q, want %q", argsN(j), j.Tags, want) - } + for _, j := range claim(ctx, t, exec, clientID, 3) { finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: j.ID, AttemptedBy: clientID, State: driver.JobStateCompleted, Archive: true}) } if n := f.QueryText(ctx, t, d, "SELECT count(*)::text FROM hopper_job_history_tags_completed_default"); n != "0" { t.Errorf("%s tags rows of jobs finished on a new installation in the DEFAULT partition, want 0", n) } - got, err := exec.JobGet(ctx, res[1].Job.ID) - if err != nil || got.State != driver.JobStateCompleted || !slices.Equal(got.Tags, p[1].Tags) { - t.Fatalf("job in history = %+v, %v", got, err) - } - retried, err := exec.JobRetry(ctx, driver.JobRetryParams{ID: res[0].Job.ID}) - if err != nil || !slices.Equal(retried.Tags, p[0].Tags) { - t.Errorf("job retried from history = %+v, %v", retried, err) - } - u := params("u", 1)[0] - u.UniqueKey, u.Tags = "one", []string{"old"} - insert(ctx, t, exec, []driver.JobInsertParams{u}) - u.Tags = []string{"new"} - rep, err := exec.JobInsertMany(ctx, []driver.JobInsertParams{u}, driver.JobInsertOpts{OnConflict: driver.ConflictReplace}) - if err != nil || !rep[0].Duplicate || !slices.Equal(rep[0].Job.Tags, []string{"new"}) { - t.Errorf("replaced job = %+v, %v", rep, err) - } - u.Tags = nil - rep, err = exec.JobInsertMany(ctx, []driver.JobInsertParams{u}, driver.JobInsertOpts{OnConflict: driver.ConflictReplace}) - if err != nil || rep[0].Job.Tags != nil { - t.Errorf("job replaced without tags = %+v, %v", rep, err) - } - - if d.Capabilities().Copy { - c := params("copy", 2) - c[1].Tags = []string{"copied"} - copied, err := exec.JobInsertCopy(ctx, c, driver.JobInsertOpts{}) - if err != nil { - t.Fatal(err) - } - for i, r := range copied { - stored, err := exec.JobGet(ctx, r.Job.ID) - if err != nil || !slices.Equal(stored.Tags, c[i].Tags) || !slices.Equal(r.Job.Tags, c[i].Tags) { - t.Errorf("copied job %d tags = %q (result %q), want %q, %v", i, stored.Tags, r.Job.Tags, c[i].Tags, err) - } - } - } - - w := params("wf", 2) - w[0].Tags = []string{"flow"} - wf, err := exec.WorkflowInsert(ctx, driver.WorkflowInsertParams{Jobs: w, Deps: []driver.JobDependency{{Job: 1, DependsOn: 0}}}) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(wf.Jobs[0].Job.Tags, []string{"flow"}) || wf.Jobs[1].Job.Tags != nil { - t.Errorf("workflow job tags = %q, %q", wf.Jobs[0].Job.Tags, wf.Jobs[1].Job.Tags) - } - - // A batch's callback keeps its tags until the batch completes. - cb := &driver.JobInsertParams{Kind: "done", Queue: "callbacks", Priority: 1, MaxAttempts: 3, Tags: []string{"callback"}} - batch, err := exec.BatchInsert(ctx, driver.BatchInsertParams{Total: 1, OnComplete: cb}) - if err != nil { - t.Fatal(err) - } - b := params("shard", 1) - b[0].Queue, b[0].BatchID = "shards", batch - insert(ctx, t, exec, b) - shards, err := exec.JobClaim(ctx, driver.JobClaimParams{Queue: "shards", ClientID: clientID, Limit: 1}) - if err != nil || len(shards.Jobs) != 1 { - t.Fatalf("claim = %+v, %v", shards, err) - } - finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: shards.Jobs[0].ID, AttemptedBy: clientID, State: driver.JobStateCompleted, Archive: true}) - callbacks, err := exec.JobList(ctx, driver.JobListParams{Queue: "callbacks", Limit: 10}) - if err != nil || len(callbacks) != 1 || !slices.Equal(callbacks[0].Tags, []string{"callback"}) { - t.Errorf("callbacks = %+v, %v", callbacks, err) - } // A callback whose tags a direct SQL write made odd gets the tags they // read as, rather than failing the finalize that completes its batch. for _, odd := range []struct { @@ -372,7 +399,7 @@ func testJobListFilters[TTx any](t *testing.T, f Fixture[TTx]) { // tags partition beside it is kept, so that a job still in history is still // listed by its tags. A read of the tags partition defers only the tags // partition's drop, which is dropped once the read ends. -func testTagsPartitionWaitsForHistory[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsPartitionWaitsForHistory[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -446,7 +473,7 @@ func testTagsPartitionWaitsForHistory[TTx any](t *testing.T, f Fixture[TTx]) { // hourly history partitions. Its job's tags rows go with it, though the // day's tags partition stays for the second hour: left there, a listing by // tag walked them before the day's jobs still in history. -func testTagsLeaveWithTheirHour[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsLeaveWithTheirHour[TTx any](t *testing.T, f PostgresFixture[TTx]) { dropTaggedHour(t, f, false) } @@ -455,14 +482,14 @@ func testTagsLeaveWithTheirHour[TTx any](t *testing.T, f Fixture[TTx]) { // a busy hour's delete does. The hour goes without deleting them, rather // than staying, with every later hour, for each pass to try again, and its // rows stay in the day's tags partition, which a listing by tag reads past. -func testTagsHourOutlastsItsDelete[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsHourOutlastsItsDelete[TTx any](t *testing.T, f PostgresFixture[TTx]) { dropTaggedHour(t, f, true) } // dropTaggedHour drops the first of a day's two hourly history partitions, // each holding a tagged job, by retention, its tags rows' delete canceled // if canceled, and checks what stays. -func dropTaggedHour[TTx any](t *testing.T, f Fixture[TTx], canceled bool) { +func dropTaggedHour[TTx any](t *testing.T, f PostgresFixture[TTx], canceled bool) { t.Helper() ctx := context.Background() d := f.NewDriver(t) @@ -512,7 +539,7 @@ func dropTaggedHour[TTx any](t *testing.T, f Fixture[TTx], canceled bool) { // The hour's tags rows go with that partition, which the same pass drops // after the hour, rather than be deleted first: a trigger fails any delete // from it. -func testTagsDayGoesWithItsLastHour[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsDayGoesWithItsLastHour[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -542,7 +569,7 @@ func testTagsDayGoesWithItsLastHour[TTx any](t *testing.T, f Fixture[TTx]) { // partition that another session drops first, as an overlapping leader's // pass does, while this pass's delete of its tags rows waits for it. The // pass reports the hour dropped, as it does one gone before its DROP. -func testTagsHourDroppedMeanwhile[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsHourDroppedMeanwhile[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -587,7 +614,7 @@ func testTagsHourDroppedMeanwhile[TTx any](t *testing.T, f Fixture[TTx]) { // jobs, then checks that the tags tables hold exactly the tags of the jobs // beside them, and that a listing by each tag finds exactly the jobs that // have it, in order, across pages. -func testTagsRowsFollowJobs[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsRowsFollowJobs[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -758,7 +785,7 @@ func testTagsRowsFollowJobs[TTx any](t *testing.T, f Fixture[TTx]) { // testTagsTableMaintained checks that the leader's live table maintenance // vacuums the live tags table, whose index a listing by tag walks, once its // dead rows number as many as the live table's would. -func testTagsTableMaintained[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsTableMaintained[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -792,7 +819,7 @@ func testTagsTableMaintained[TTx any](t *testing.T, f Fixture[TTx]) { // testTagsTableMaintainedPastJobsError checks that the live tags table is // maintained even when hopper_jobs' maintenance fails, here by an extended // statistic whose expression its ANALYZE cannot compute. -func testTagsTableMaintainedPastJobsError[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsTableMaintainedPastJobsError[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -829,7 +856,7 @@ func testTagsTableMaintainedPastJobsError[TTx any](t *testing.T, f Fixture[TTx]) // history's DEFAULT partition takes its tags rows along, wherever they are, // and a tags row in the tags' DEFAULT partition stays while its job is in one // of history's time partitions. -func testTagsFollowHistoryRetention[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsFollowHistoryRetention[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -922,7 +949,7 @@ func testTagsFollowHistoryRetention[TTx any](t *testing.T, f Fixture[TTx]) { // and a partition, which fire none of a partitioned table's statement // triggers. Each tags table then holds exactly its jobs' tags, and a listing // by tag finds each job where it now is. -func testTagsFollowDirectUpdates[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsFollowDirectUpdates[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -1005,7 +1032,7 @@ func testTagsFollowDirectUpdates[TTx any](t *testing.T, f Fixture[TTx]) { // checkTagsRows checks that each tags table holds exactly the tags of the // jobs beside it, comparing the two with except: "EXCEPT", or "EXCEPT ALL" // to count a duplicate tags row as a mismatch too. -func checkTagsRows[TTx any](ctx context.Context, t *testing.T, f Fixture[TTx], d driver.Driver[TTx], except string) { +func checkTagsRows[TTx any](ctx context.Context, t *testing.T, f PostgresFixture[TTx], d driver.Driver[TTx], except string) { t.Helper() const liveExpanded = `SELECT e.tag, j.id FROM hopper_jobs j, LATERAL jsonb_array_elements_text(j.tags) e(tag)` const historyExpanded = `SELECT h.state, h.finalized_at, e.tag, h.id FROM hopper_job_history h, LATERAL jsonb_array_elements_text(h.tags) e(tag)` @@ -1050,7 +1077,7 @@ func mustID(t *testing.T, s string) driver.JobID { // testTagsFollowTruncate truncates hopper_jobs, then history, each alone and // then together with every tags table, as a reset of every table does. The // tags tables follow, though TRUNCATE fires no delete trigger. -func testTagsFollowTruncate[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsFollowTruncate[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() diff --git a/drivertest/unique.go b/drivertest/unique.go index 6f1878e..7d3add4 100644 --- a/drivertest/unique.go +++ b/drivertest/unique.go @@ -178,7 +178,7 @@ func testInsertUniqueFinishedRacesFinalize[TTx any](t *testing.T, f Fixture[TTx] } done <- res[0] }() - waitForLockWait(ctx, t, f, d) + f.WaitForLockWait(ctx, t, d) if err := commit(); err != nil { t.Fatal(err) } diff --git a/drivertest/workflow.go b/drivertest/workflow.go index 714cf5b..6cdf69f 100644 --- a/drivertest/workflow.go +++ b/drivertest/workflow.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "strconv" "strings" "sync" "testing" @@ -168,9 +167,12 @@ func testWorkflows[TTx any](t *testing.T, f Fixture[TTx]) { t.Errorf("chain completed = %+v", row.Batch) } cbs := callbacks() + var alert struct { + BatchFailed int `json:"batch_failed"` + } if cbs["wf-alert2"] == nil || cbs["wf-done2"] != nil { t.Errorf("chain callbacks = %v", cbs) - } else if !strings.Contains(string(cbs["wf-alert2"].Metadata), `"batch_failed": 3`) { + } else if err := json.Unmarshal(cbs["wf-alert2"].Metadata, &alert); err != nil || alert.BatchFailed != 3 { t.Errorf("alert metadata = %s", cbs["wf-alert2"].Metadata) } @@ -212,10 +214,12 @@ func testWorkflows[TTx any](t *testing.T, f Fixture[TTx]) { } // testWorkflowFanInConcurrent finalizes a job's two dependencies in -// concurrent transactions. Each statement still sees the other dependency -// live, so neither promotes the dependent; WorkflowPromoteReady, run by -// each finalizing client after its statement commits, does once both have -// committed, and the leader's sweep finds what clients left. +// concurrent transactions. On an engine that locks rows, each statement +// still sees the other dependency live, so neither promotes the dependent; +// WorkflowPromoteReady, run by each finalizing client after its statement +// commits, does once both have committed. On an engine that runs one writer +// at a time, the second finalize waits for the first and promotes the +// dependent itself. Either way, the leader's sweep finds what clients left. func testWorkflowFanInConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) @@ -249,23 +253,44 @@ func testWorkflowFanInConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { if applied := finalizeCurrent(ctx, t, d.UnwrapTx(tx), completed(x)); len(applied) != 1 { t.Fatalf("finalize in tx applied %v", applied) } - // The other dependency's finalize does not wait for the transaction. - fctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - if applied, err := exec.JobFinalizeMany(fctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{completed(y)}}); err != nil || len(applied) != 1 { - t.Fatalf("concurrent finalize = %v, %v", applied, err) - } - if n := promote(y); n != 0 { - t.Errorf("promoted %d while a dependency's finalize was uncommitted", n) - } - if err := commit(); err != nil { - t.Fatal(err) - } - if j, err := exec.JobGet(ctx, dep); err != nil || j.State != driver.JobStatePending { - t.Fatalf("dependent after both finalizes = %+v, %v; the race should leave it pending", j, err) - } - if n := promote(x); n != 1 { - t.Errorf("promoted %d, want 1", n) + if singleWriter(d) { + done := make(chan []driver.JobID, 1) + go func() { + applied, err := exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{completed(y)}}) + if err != nil { + t.Error(err) + } + done <- applied + }() + f.WaitForLockWait(ctx, t, d) + if err := commit(); err != nil { + t.Fatal(err) + } + if applied := <-done; len(applied) != 1 { + t.Fatalf("finalize after the transaction applied %v", applied) + } + if j, err := exec.JobGet(ctx, dep); err != nil || j.State != driver.JobStateAvailable { + t.Fatalf("dependent after both finalizes = %+v, %v; the second should promote it", j, err) + } + } else { + // The other dependency's finalize does not wait for the transaction. + fctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if applied, err := exec.JobFinalizeMany(fctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{completed(y)}}); err != nil || len(applied) != 1 { + t.Fatalf("concurrent finalize = %v, %v", applied, err) + } + if n := promote(y); n != 0 { + t.Errorf("promoted %d while a dependency's finalize was uncommitted", n) + } + if err := commit(); err != nil { + t.Fatal(err) + } + if j, err := exec.JobGet(ctx, dep); err != nil || j.State != driver.JobStatePending { + t.Fatalf("dependent after both finalizes = %+v, %v; the race should leave it pending", j, err) + } + if n := promote(x); n != 1 { + t.Errorf("promoted %d, want 1", n) + } } if j, err := exec.JobGet(ctx, dep); err != nil || j.State != driver.JobStateAvailable { t.Errorf("dependent after promote = %+v, %v", j, err) @@ -298,7 +323,7 @@ func testWorkflowFanInConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { t.Fatalf("claimed %+v", jobs) } finalizeCurrent(ctx, t, exec, completed(jobs[0].ID)) - f.Exec(ctx, t, d, `UPDATE hopper_jobs SET state = 'pending' WHERE id = $1::uuid`, wf.Jobs[1].Job.ID.String()) + f.MakePending(ctx, t, d, wf.Jobs[1].Job.ID) steps = append(steps, wf.Jobs[1].Job.ID) } sweep := func(after driver.JobID) driver.WorkflowPromoteResult { @@ -340,6 +365,10 @@ func testWorkflowsConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { exec := d.Executor() const workflows, middles = 100, 4 cb := &driver.JobInsertParams{Kind: "done", Queue: "callbacks", Priority: 2, MaxAttempts: 3, Args: json.RawMessage(`{}`)} + var ( + batchIDs []driver.JobID + dependsOn = map[driver.JobID][]driver.JobID{} + ) for w := range workflows { jobs := params("root", 1) mids := params("mid", middles) @@ -351,10 +380,38 @@ func testWorkflowsConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { for m := 1; m <= middles; m++ { deps = append(deps, driver.JobDependency{Job: m, DependsOn: 0}, driver.JobDependency{Job: middles + 1, DependsOn: m}) } - if _, err := exec.WorkflowInsert(ctx, driver.WorkflowInsertParams{Name: "chaos", Jobs: jobs, Deps: deps, OnComplete: cb}); err != nil { + wf, err := exec.WorkflowInsert(ctx, driver.WorkflowInsertParams{Name: "chaos", Jobs: jobs, Deps: deps, OnComplete: cb}) + if err != nil { t.Fatal(err) } + batchIDs = append(batchIDs, wf.ID) + for _, dep := range deps { + job := wf.Jobs[dep.Job].Job.ID + dependsOn[job] = append(dependsOn[job], wf.Jobs[dep.DependsOn].Job.ID) + } } + // live reports whether a job is still in the live table. + live := func(id driver.JobID) (bool, error) { + j, err := exec.JobGet(ctx, id) + if errors.Is(err, driver.ErrNotFound) { + return false, nil + } + if err != nil { + return false, err + } + return j.FinalizedAt.IsZero(), nil + } + list := func(p driver.JobListParams) []*driver.JobRow { + t.Helper() + p.Limit = 10000 + jobs, err := exec.JobList(ctx, p) + if err != nil { + t.Fatal(err) + } + return jobs + } + liveStates := []driver.JobState{driver.JobStatePending, driver.JobStateAvailable, driver.JobStateScheduled, driver.JobStateRunning, driver.JobStateRetryable} + finished := []driver.JobState{driver.JobStateCompleted, driver.JobStateCanceled, driver.JobStateDiscarded} wctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() @@ -418,22 +475,26 @@ func testWorkflowsConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } fin := make([]driver.JobFinalize, len(res.Jobs)) batches := make([]driver.JobID, len(res.Jobs)) - ids := make([]string, len(res.Jobs)) for i, j := range res.Jobs { fin[i] = driver.JobFinalize{ID: j.ID, AttemptedBy: clientID, Fence: j.Fence, State: driver.JobStateCompleted, Archive: true} if j.Kind == "fail" { fin[i].State = driver.JobStateDiscarded } batches[i] = j.BatchID - ids[i] = j.ID.String() - } - // A dependency leaves the live table when its finalize - // commits, and never returns: one still there ran after - // its dependent was claimed. - if n := f.QueryText(ctx, t, d, `SELECT count(*)::text FROM hopper_job_deps e - JOIN hopper_jobs x ON x.id = e.depends_on WHERE e.job_id = ANY($1::uuid[])`, "{"+strings.Join(ids, ",")+"}"); n != "0" { - fail(fmt.Errorf("claimed %d jobs with %s dependencies still live", len(ids), n)) - return + // A dependency leaves the live table when its finalize + // commits, and never returns: one still there ran after + // its dependent was claimed. + for _, dep := range dependsOn[j.ID] { + isLive, err := live(dep) + if err != nil { + fail(err) + return + } + if isLive { + fail(fmt.Errorf("claimed job %s with dependency %s still live", j.ID, dep)) + return + } + } } finished := finalizeRetrying(fin) record(finished) @@ -482,8 +543,22 @@ func testWorkflowsConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { time.Sleep(5 * time.Millisecond) } }) - for wctx.Err() == nil && f.QueryText(ctx, t, d, `SELECT ((SELECT count(*) FROM hopper_jobs WHERE queue = 'default') = 0 - AND (SELECT count(*) FROM hopper_batches WHERE completed_at IS NULL) = 0)::text`) != "true" { + done := func() bool { + if len(list(driver.JobListParams{Queue: "default", States: liveStates})) > 0 { + return false + } + for _, id := range batchIDs { + b, err := exec.BatchGet(ctx, id) + if err != nil { + t.Fatal(err) + } + if b.CompletedAt.IsZero() { + return false + } + } + return true + } + for wctx.Err() == nil && !done() { time.Sleep(20 * time.Millisecond) } timedOut := wctx.Err() != nil @@ -494,8 +569,11 @@ func testWorkflowsConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { t.Fatal(err) } if timedOut { - t.Fatalf("not finished in a minute (last finalize error: %v): %s", lastErr, f.QueryText(ctx, t, d, - "SELECT coalesce(string_agg(state || ' ' || n, ', '), 'none') FROM (SELECT state::text, count(*) AS n FROM hopper_jobs GROUP BY state) s")) + states := map[driver.JobState]int{} + for _, j := range list(driver.JobListParams{Queue: "default", States: liveStates}) { + states[j.State]++ + } + t.Fatalf("not finished in a minute (last finalize error: %v): %v", lastErr, states) } failing := workflows / 5 @@ -509,26 +587,50 @@ func testWorkflowsConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { if len(applied) != want || twice != 0 { t.Errorf("results applied for %d jobs, %d of them more than once; want %d jobs, once each", len(applied), twice, want) } - checks := []struct{ what, query, want string }{ - { - "history rows, distinct jobs", `SELECT count(*) || ' ' || count(DISTINCT id) FROM hopper_job_history WHERE queue = 'default'`, - fmt.Sprintf("%d %d", workflows*(middles+2), workflows*(middles+2)), - }, - { - "last steps by state", `SELECT string_agg(state || ' ' || n, ', ' ORDER BY state) FROM ( - SELECT state::text, count(*) AS n FROM hopper_job_history WHERE kind = 'last' GROUP BY state) s`, - fmt.Sprintf("canceled %d, completed %d", failing, workflows-failing), - }, - {"canceled last steps that ran", `SELECT count(*) FROM hopper_job_history WHERE kind = 'last' AND state = 'canceled' AND attempted_at IS NOT NULL`, "0"}, - {"workflows whose failures miscount", `SELECT count(*) FROM hopper_batches b - WHERE b.failed <> (SELECT count(*) FROM hopper_job_history h WHERE h.batch_id = b.id AND h.state <> 'completed')`, "0"}, - {"failures", `SELECT sum(failed) FROM hopper_batches`, strconv.Itoa(2 * failing)}, - {"completion callbacks", `SELECT count(*) FROM hopper_jobs WHERE queue = 'callbacks'`, strconv.Itoa(workflows)}, - {"edges left", `SELECT count(*) FROM hopper_job_deps`, "0"}, + history := list(driver.JobListParams{Queue: "default", States: finished}) + distinct := map[driver.JobID]bool{} + lastByState := map[driver.JobState]int{} + ranCanceled := 0 + failedByBatch := map[driver.JobID]int{} + for _, j := range history { + distinct[j.ID] = true + if j.Kind == "last" { + lastByState[j.State]++ + if j.State == driver.JobStateCanceled && !j.AttemptedAt.IsZero() { + ranCanceled++ + } + } + if j.State != driver.JobStateCompleted { + failedByBatch[j.BatchID]++ + } + } + if len(history) != workflows*(middles+2) || len(distinct) != len(history) { + t.Errorf("history has %d rows of %d distinct jobs, want %d of as many", len(history), len(distinct), workflows*(middles+2)) + } + if lastByState[driver.JobStateCanceled] != failing || lastByState[driver.JobStateCompleted] != workflows-failing || len(lastByState) != 2 { + t.Errorf("last steps by state = %v, want %d canceled and %d completed", lastByState, failing, workflows-failing) + } + if ranCanceled != 0 { + t.Errorf("%d canceled last steps ran", ranCanceled) } - for _, c := range checks { - if got := f.QueryText(ctx, t, d, "SELECT ("+c.query+")::text"); got != c.want { - t.Errorf("%s = %s, want %s", c.what, got, c.want) + failures := 0 + for _, id := range batchIDs { + b, err := exec.BatchGet(ctx, id) + if err != nil { + t.Fatal(err) + } + failures += b.Failed + if b.Failed != failedByBatch[id] { + t.Errorf("workflow %s counts %d failures, its history %d", id, b.Failed, failedByBatch[id]) } } + if failures != 2*failing { + t.Errorf("failures = %d, want %d", failures, 2*failing) + } + if n := len(list(driver.JobListParams{Queue: "callbacks"})); n != workflows { + t.Errorf("completion callbacks = %d, want %d", n, workflows) + } + if n := f.CountEdges(ctx, t, d); n != 0 { + t.Errorf("edges left = %d, want 0", n) + } } From a2d63620d9e1a4110bca20e58ad5633ec45308c4 Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 11:05:36 +0000 Subject: [PATCH 03/20] refactor(driver): move the codecs of a job's errors, progress, logs and tags to driver/internal/jobcodec They read a value a direct write left in another form as what can be made of it, and every engine needs them so; hoppersqlite will share them. --- driver/internal/jobcodec/jobcodec.go | 110 +++++++++++++++++++++++++++ driver/internal/pgsql/jobs.go | 46 +---------- driver/internal/pgsql/values.go | 30 ++------ 3 files changed, 122 insertions(+), 64 deletions(-) create mode 100644 driver/internal/jobcodec/jobcodec.go diff --git a/driver/internal/jobcodec/jobcodec.go b/driver/internal/jobcodec/jobcodec.go new file mode 100644 index 0000000..e5fa694 --- /dev/null +++ b/driver/internal/jobcodec/jobcodec.go @@ -0,0 +1,110 @@ +// Package jobcodec encodes and decodes the JSON values a job row keeps +// beside its args and metadata: its errors, progress, logs and tags. Every +// engine stores them the same way, and reads them the same way: a value +// that a direct SQL write left in another form reads as what can be made of +// it, rather than failing the read, so that one such row fails neither the +// claim that takes it (whose update has committed by the time it is read) +// nor a listing or the rescuer. +package jobcodec + +import ( + "encoding/json" + "strings" + "time" + + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/internal/jobtags" +) + +// StorableText replaces what a database may refuse in text and JSON +// strings, NUL and invalid UTF-8, so that one worker's text cannot fail the +// statement that finalizes a whole batch. +func StorableText(s string) string { + return strings.ReplaceAll(strings.ToValidUTF8(s, "�"), "\x00", "�") +} + +// MarshalAttemptError encodes an error record without its timestamp, which +// the engine fills in from database time, its text made storable. +func MarshalAttemptError(e *driver.AttemptError) ([]byte, error) { + return json.Marshal(struct { + Attempt int `json:"attempt"` + Fence int64 `json:"fence,omitempty"` + Error string `json:"error"` + Trace string `json:"trace,omitempty"` + }{e.Attempt, e.Fence, StorableText(e.Error), StorableText(e.Trace)}) +} + +// DecodeErrors reads a job's errors, a JSON array of error records. An +// empty value reads as none. +func DecodeErrors(stored string) ([]driver.AttemptError, error) { + errs := []driver.AttemptError{} + if stored == "" { + return errs, nil + } + if err := json.Unmarshal([]byte(stored), &errs); err != nil { + return nil, err + } + return errs, nil +} + +// DecodeProgress reads a progress value, keeping what it can of one that is +// not in the form an engine writes: a value that is not an object reads as +// none. +func DecodeProgress(stored string) *driver.Progress { + var v struct { + Step any `json:"step"` + Cursor json.RawMessage `json:"cursor"` + At any `json:"at"` + } + if !strings.HasPrefix(strings.TrimSpace(stored), "{") || json.Unmarshal([]byte(stored), &v) != nil { + return nil + } + p := &driver.Progress{Cursor: v.Cursor} + if s, ok := v.Step.(string); ok { + p.Step = s + } + if s, ok := v.At.(string); ok { + p.RecordedAt, _ = time.Parse(time.RFC3339Nano, s) + } + if string(p.Cursor) == "null" { + p.Cursor = nil + } + return p +} + +// DecodeLogs reads a job's logs, skipping entries that are not in the form +// a finalize writes. +func DecodeLogs(stored string) []driver.AttemptLog { + // Anything but an array (or null) fails to decode into entries. + var entries []json.RawMessage + if json.Unmarshal([]byte(stored), &entries) != nil { + return nil + } + var logs []driver.AttemptLog + for _, raw := range entries { + var l driver.AttemptLog + if json.Unmarshal(raw, &l) == nil && l.Log != "" { + logs = append(logs, l) + } + } + return logs +} + +// DecodeTags reads a tags value, a JSON array of strings: a value that is +// not an array reads as no tags, and elements that are not strings are +// skipped, as are empty strings and repeats. +func DecodeTags(stored string) []string { + if stored == "" { + return nil + } + var elems []json.RawMessage + _ = json.Unmarshal([]byte(stored), &elems) // anything but an array reads as no tags + var tags []string + for _, e := range elems { + var s string + if len(e) > 0 && e[0] == '"' && json.Unmarshal(e, &s) == nil && s != "" { + tags = append(tags, s) + } + } + return jobtags.Distinct(tags) +} diff --git a/driver/internal/pgsql/jobs.go b/driver/internal/pgsql/jobs.go index f1e7000..8829763 100644 --- a/driver/internal/pgsql/jobs.go +++ b/driver/internal/pgsql/jobs.go @@ -17,6 +17,7 @@ import ( "unicode/utf8" "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/internal/jobcodec" "github.com/parallelworks/hopper/internal/jobtags" ) @@ -748,12 +749,7 @@ func keepLogs(keep int) int32 { // marshalAttemptError encodes an error record without its timestamp, which // the finalize statement fills in from database time. func marshalAttemptError(e *driver.AttemptError) ([]byte, error) { - return json.Marshal(struct { - Attempt int `json:"attempt"` - Fence int64 `json:"fence,omitempty"` - Error string `json:"error"` - Trace string `json:"trace,omitempty"` - }{e.Attempt, e.Fence, storableText(e.Error), storableText(e.Trace)}) + return jobcodec.MarshalAttemptError(e) } var jobGetQuery = fmt.Sprintf(` @@ -818,27 +814,7 @@ func (e *Executor) JobRecordProgress(ctx context.Context, params driver.JobRecor // is not in the form JobRecordProgress writes, such as one written by hand: // a row must still scan, or a claim that took it would fail with the claim // already applied. A value that is not an object reads as none. -func decodeProgress(stored string) *driver.Progress { - var v struct { - Step any `json:"step"` - Cursor json.RawMessage `json:"cursor"` - At any `json:"at"` - } - if !strings.HasPrefix(strings.TrimSpace(stored), "{") || json.Unmarshal([]byte(stored), &v) != nil { - return nil - } - p := &driver.Progress{Cursor: v.Cursor} - if s, ok := v.Step.(string); ok { - p.Step = s - } - if s, ok := v.At.(string); ok { - p.RecordedAt, _ = time.Parse(time.RFC3339Nano, s) - } - if string(p.Cursor) == "null" { - p.Cursor = nil - } - return p -} +func decodeProgress(stored string) *driver.Progress { return jobcodec.DecodeProgress(stored) } // JobGet implements driver.Executor. func (e *Executor) JobGet(ctx context.Context, id driver.JobID) (*driver.JobRow, error) { @@ -873,21 +849,7 @@ func (e *Executor) JobLogs(ctx context.Context, id driver.JobID) ([]driver.Attem // decodeLogs reads a job's logs, skipping entries that are not in the form // the finalize writes, such as ones written by hand. -func decodeLogs(stored string) []driver.AttemptLog { - // Anything but an array (or null) fails to decode into entries. - var entries []json.RawMessage - if json.Unmarshal([]byte(stored), &entries) != nil { - return nil - } - var logs []driver.AttemptLog - for _, raw := range entries { - var l driver.AttemptLog - if json.Unmarshal(raw, &l) == nil && l.Log != "" { - logs = append(logs, l) - } - } - return logs -} +func decodeLogs(stored string) []driver.AttemptLog { return jobcodec.DecodeLogs(stored) } var ( jobListLiveSQL = fmt.Sprintf(`SELECT %s, NULL::timestamptz, NULL::jsonb FROM {{schema}}.hopper_jobs`, JobColumns("")) diff --git a/driver/internal/pgsql/values.go b/driver/internal/pgsql/values.go index 7a64567..f1d74a3 100644 --- a/driver/internal/pgsql/values.go +++ b/driver/internal/pgsql/values.go @@ -9,7 +9,7 @@ import ( "time" "github.com/parallelworks/hopper/driver" - "github.com/parallelworks/hopper/internal/jobtags" + "github.com/parallelworks/hopper/driver/internal/jobcodec" ) // Array parameters cross the transport as plain slices, which pgx encodes @@ -23,9 +23,7 @@ func textArray(values []string) []string { return orEmpty(values) } // storableText replaces what PostgreSQL refuses in text and jsonb strings, // NUL and invalid UTF-8, so that one worker's text cannot fail the // statement that finalizes a whole batch. -func storableText(s string) string { - return strings.ReplaceAll(strings.ToValidUTF8(s, "�"), "\x00", "�") -} +func storableText(s string) string { return jobcodec.StorableText(s) } // nullableTextArray sends empty strings as NULL. func nullableTextArray(values []string) []*string { @@ -261,11 +259,12 @@ func (a *attemptErrors) Scan(src any) error { if err := text.Scan(src); err != nil { return err } - *a = attemptErrors{} - if text == "" { - return nil + errs, err := jobcodec.DecodeErrors(string(text)) + if err != nil { + return err } - return json.Unmarshal([]byte(text), (*[]driver.AttemptError)(a)) + *a = errs + return nil } // tagList scans a tags column, a JSON array of strings or NULL. hopper @@ -281,20 +280,7 @@ func (l *tagList) Scan(src any) error { if err := text.Scan(src); err != nil { return err } - *l = nil - if text == "" { - return nil - } - var elems []json.RawMessage - _ = json.Unmarshal([]byte(text), &elems) // anything but an array reads as no tags - var tags []string - for _, e := range elems { - var s string - if len(e) > 0 && e[0] == '"' && json.Unmarshal(e, &s) == nil && s != "" { - tags = append(tags, s) - } - } - *l = jobtags.Distinct(tags) + *l = jobcodec.DecodeTags(string(text)) return nil } From 4dcafd4de9131c8fc6c21e4c4907a4e4a0885f8e Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 11:05:36 +0000 Subject: [PATCH 04/20] feat(hoppersqlite): a SQLite driver, in a module of its own hoppersqlite runs hopper on a SQLite database through modernc.org/sqlite, in pure Go. Its connections take the write lock on a transaction's first statement (BEGIN IMMEDIATE), queued in turn within a process and through SQLite's busy handler across processes, so claims are exclusive without SKIP LOCKED and the finalizing transaction settles batches and promotes dependents itself. Notifications are rows delivered on commit, at once in the process and by polling in others. Its migrations follow hoppermigrate's versions one for one. Open refuses a database on a network filesystem. --- driver/hoppersqlite/cluster.go | 586 ++++++++++++ driver/hoppersqlite/conformance_test.go | 142 +++ driver/hoppersqlite/conn.go | 474 ++++++++++ driver/hoppersqlite/executor.go | 375 ++++++++ driver/hoppersqlite/export_test.go | 5 + driver/hoppersqlite/finalize.go | 570 ++++++++++++ driver/hoppersqlite/flow.go | 203 +++++ driver/hoppersqlite/funcs.go | 347 +++++++ driver/hoppersqlite/go.mod | 28 + driver/hoppersqlite/go.sum | 50 + driver/hoppersqlite/hoppersqlite.go | 316 +++++++ driver/hoppersqlite/id.go | 26 + driver/hoppersqlite/jobs.go | 856 ++++++++++++++++++ driver/hoppersqlite/listener.go | 133 +++ driver/hoppersqlite/localfs.go | 31 + driver/hoppersqlite/localfs_bsd.go | 18 + driver/hoppersqlite/localfs_darwin.go | 7 + driver/hoppersqlite/localfs_freebsd.go | 7 + driver/hoppersqlite/localfs_linux.go | 45 + driver/hoppersqlite/localfs_netbsd.go | 16 + driver/hoppersqlite/localfs_openbsd.go | 7 + driver/hoppersqlite/localfs_other.go | 11 + driver/hoppersqlite/localfs_windows.go | 32 + driver/hoppersqlite/messaging.go | 476 ++++++++++ .../migrations/001_schema.down.sql | 9 + .../hoppersqlite/migrations/001_schema.up.sql | 145 +++ .../migrations/002_messaging.down.sql | 4 + .../migrations/002_messaging.up.sql | 17 + .../migrations/003_flow_control.down.sql | 5 + .../migrations/003_flow_control.up.sql | 26 + .../migrations/004_workflows.down.sql | 4 + .../migrations/004_workflows.up.sql | 23 + .../migrations/005_streams.down.sql | 2 + .../migrations/005_streams.up.sql | 31 + .../migrations/006_autovacuum.down.sql | 1 + .../migrations/006_autovacuum.up.sql | 3 + .../migrations/007_vacuum_truncate.down.sql | 1 + .../migrations/007_vacuum_truncate.up.sql | 2 + .../migrations/008_function_schema.down.sql | 1 + .../migrations/008_function_schema.up.sql | 2 + .../migrations/009_snooze_reason.down.sql | 1 + .../migrations/009_snooze_reason.up.sql | 8 + .../migrations/010_wake_requested.down.sql | 1 + .../migrations/010_wake_requested.up.sql | 9 + .../migrations/011_batch_completion.down.sql | 1 + .../migrations/011_batch_completion.up.sql | 5 + .../migrations/012_canceled_state.down.sql | 1 + .../migrations/012_canceled_state.up.sql | 2 + .../migrations/013_history_ordering.down.sql | 1 + .../migrations/013_history_ordering.up.sql | 4 + .../migrations/014_history_unique.down.sql | 1 + .../migrations/014_history_unique.up.sql | 5 + .../migrations/015_job_progress.down.sql | 2 + .../migrations/015_job_progress.up.sql | 8 + .../migrations/016_job_logs.down.sql | 2 + .../migrations/016_job_logs.up.sql | 6 + .../migrations/017_job_tags.down.sql | 10 + .../migrations/017_job_tags.up.sql | 72 ++ .../migrations/018_claim_fence.down.sql | 2 + .../migrations/018_claim_fence.up.sql | 11 + driver/hoppersqlite/workflow.go | 190 ++++ 61 files changed, 5379 insertions(+) create mode 100644 driver/hoppersqlite/cluster.go create mode 100644 driver/hoppersqlite/conformance_test.go create mode 100644 driver/hoppersqlite/conn.go create mode 100644 driver/hoppersqlite/executor.go create mode 100644 driver/hoppersqlite/export_test.go create mode 100644 driver/hoppersqlite/finalize.go create mode 100644 driver/hoppersqlite/flow.go create mode 100644 driver/hoppersqlite/funcs.go create mode 100644 driver/hoppersqlite/go.mod create mode 100644 driver/hoppersqlite/go.sum create mode 100644 driver/hoppersqlite/hoppersqlite.go create mode 100644 driver/hoppersqlite/id.go create mode 100644 driver/hoppersqlite/jobs.go create mode 100644 driver/hoppersqlite/listener.go create mode 100644 driver/hoppersqlite/localfs.go create mode 100644 driver/hoppersqlite/localfs_bsd.go create mode 100644 driver/hoppersqlite/localfs_darwin.go create mode 100644 driver/hoppersqlite/localfs_freebsd.go create mode 100644 driver/hoppersqlite/localfs_linux.go create mode 100644 driver/hoppersqlite/localfs_netbsd.go create mode 100644 driver/hoppersqlite/localfs_openbsd.go create mode 100644 driver/hoppersqlite/localfs_other.go create mode 100644 driver/hoppersqlite/localfs_windows.go create mode 100644 driver/hoppersqlite/messaging.go create mode 100644 driver/hoppersqlite/migrations/001_schema.down.sql create mode 100644 driver/hoppersqlite/migrations/001_schema.up.sql create mode 100644 driver/hoppersqlite/migrations/002_messaging.down.sql create mode 100644 driver/hoppersqlite/migrations/002_messaging.up.sql create mode 100644 driver/hoppersqlite/migrations/003_flow_control.down.sql create mode 100644 driver/hoppersqlite/migrations/003_flow_control.up.sql create mode 100644 driver/hoppersqlite/migrations/004_workflows.down.sql create mode 100644 driver/hoppersqlite/migrations/004_workflows.up.sql create mode 100644 driver/hoppersqlite/migrations/005_streams.down.sql create mode 100644 driver/hoppersqlite/migrations/005_streams.up.sql create mode 100644 driver/hoppersqlite/migrations/006_autovacuum.down.sql create mode 100644 driver/hoppersqlite/migrations/006_autovacuum.up.sql create mode 100644 driver/hoppersqlite/migrations/007_vacuum_truncate.down.sql create mode 100644 driver/hoppersqlite/migrations/007_vacuum_truncate.up.sql create mode 100644 driver/hoppersqlite/migrations/008_function_schema.down.sql create mode 100644 driver/hoppersqlite/migrations/008_function_schema.up.sql create mode 100644 driver/hoppersqlite/migrations/009_snooze_reason.down.sql create mode 100644 driver/hoppersqlite/migrations/009_snooze_reason.up.sql create mode 100644 driver/hoppersqlite/migrations/010_wake_requested.down.sql create mode 100644 driver/hoppersqlite/migrations/010_wake_requested.up.sql create mode 100644 driver/hoppersqlite/migrations/011_batch_completion.down.sql create mode 100644 driver/hoppersqlite/migrations/011_batch_completion.up.sql create mode 100644 driver/hoppersqlite/migrations/012_canceled_state.down.sql create mode 100644 driver/hoppersqlite/migrations/012_canceled_state.up.sql create mode 100644 driver/hoppersqlite/migrations/013_history_ordering.down.sql create mode 100644 driver/hoppersqlite/migrations/013_history_ordering.up.sql create mode 100644 driver/hoppersqlite/migrations/014_history_unique.down.sql create mode 100644 driver/hoppersqlite/migrations/014_history_unique.up.sql create mode 100644 driver/hoppersqlite/migrations/015_job_progress.down.sql create mode 100644 driver/hoppersqlite/migrations/015_job_progress.up.sql create mode 100644 driver/hoppersqlite/migrations/016_job_logs.down.sql create mode 100644 driver/hoppersqlite/migrations/016_job_logs.up.sql create mode 100644 driver/hoppersqlite/migrations/017_job_tags.down.sql create mode 100644 driver/hoppersqlite/migrations/017_job_tags.up.sql create mode 100644 driver/hoppersqlite/migrations/018_claim_fence.down.sql create mode 100644 driver/hoppersqlite/migrations/018_claim_fence.up.sql create mode 100644 driver/hoppersqlite/workflow.go diff --git a/driver/hoppersqlite/cluster.go b/driver/hoppersqlite/cluster.go new file mode 100644 index 0000000..2a11e9b --- /dev/null +++ b/driver/hoppersqlite/cluster.go @@ -0,0 +1,586 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/parallelworks/hopper/driver" +) + +// ClientRegister implements driver.Executor. +func (e *executor) ClientRegister(ctx context.Context, params driver.ClientRegisterParams) (int64, error) { + info, err := jsonValue(params.Info, "{}", "client info") + if err != nil { + return 0, err + } + var id int64 + err = e.write(ctx, func(x *w) error { + return x.QueryRowContext(ctx, `INSERT INTO hopper_clients (hostname, started_at, expires_at, info) VALUES (?, ?, ?, ?) RETURNING id`, + params.Hostname, x.us(), x.at(params.TTL), info).Scan(&id) + }) + if err != nil { + return 0, fmt.Errorf("hopper: register client: %w", err) + } + return id, nil +} + +// strings reads the text values a query selects. +func stringsOf(ctx context.Context, q querier, query string, args ...any) ([]string, error) { + return queryAll(ctx, q, func(r scanner) (string, error) { + var s string + err := r.Scan(&s) + return s, err + }, query, args...) +} + +// ClientRenew implements driver.Executor: it extends an unexpired lease and +// reads the control state the client applies. +func (e *executor) ClientRenew(ctx context.Context, params driver.ClientRenewParams) (driver.ClientRenewResult, error) { + var res driver.ClientRenewResult + err := e.write(ctx, func(x *w) error { + renewed, err := x.ExecContext(ctx, `UPDATE hopper_clients SET expires_at = ? WHERE id = ? AND expires_at > ?`, + x.at(params.TTL), params.ClientID, x.us()) + if err != nil { + return err + } + n, err := renewed.RowsAffected() + if err != nil { + return err + } + res.Renewed = n > 0 + if res.CancelRequested, err = x.ids(ctx, `SELECT id FROM hopper_jobs + WHERE state = 'running' AND attempted_by = ? AND cancel_requested_at IS NOT NULL ORDER BY id`, params.ClientID); err != nil { + return err + } + if res.PausedQueues, err = stringsOf(ctx, x, `SELECT name FROM hopper_queues WHERE paused_at IS NOT NULL ORDER BY name`); err != nil { + return err + } + res.LimitedQueues, err = stringsOf(ctx, x, `SELECT name FROM hopper_queues + WHERE global_limit IS NOT NULL OR rate_per_sec IS NOT NULL OR partition_limit IS NOT NULL ORDER BY name`) + return err + }) + if err != nil { + return driver.ClientRenewResult{}, fmt.Errorf("hopper: renew client lease: %w", err) + } + return res, nil +} + +// ClientDelete implements driver.Executor. +func (e *executor) ClientDelete(ctx context.Context, clientID int64) error { + err := e.write(ctx, func(x *w) error { + _, err := x.ExecContext(ctx, `DELETE FROM hopper_clients WHERE id = ?`, clientID) + return err + }) + if err != nil { + return fmt.Errorf("hopper: delete client: %w", err) + } + return nil +} + +// ClientPruneExpired implements driver.Executor. +func (e *executor) ClientPruneExpired(ctx context.Context) (int64, error) { + var n int64 + err := e.write(ctx, func(x *w) error { + res, err := x.ExecContext(ctx, `DELETE FROM hopper_clients WHERE expires_at < ?`, x.us()) + if err != nil { + return err + } + n, err = res.RowsAffected() + return err + }) + if err != nil { + return 0, fmt.Errorf("hopper: prune clients: %w", err) + } + return n, nil +} + +// ClientList implements driver.Executor. +func (e *executor) ClientList(ctx context.Context) ([]*driver.ClientRow, error) { + clients, err := queryAll(ctx, e.read(), func(r scanner) (*driver.ClientRow, error) { + var ( + c driver.ClientRow + started, expires int64 + info string + ) + if err := r.Scan(&c.ID, &c.Hostname, &started, &expires, &info); err != nil { + return nil, err + } + c.StartedAt, c.ExpiresAt, c.Info = time.UnixMicro(started).UTC(), time.UnixMicro(expires).UTC(), json.RawMessage(info) + return &c, nil + }, `SELECT id, hostname, started_at, expires_at, info FROM hopper_clients ORDER BY id`) + if err != nil { + return nil, fmt.Errorf("hopper: list clients: %w", err) + } + return clients, nil +} + +// LeaderAttempt implements driver.Executor: it takes the lease if it is +// free or expired, or renews it for the holder. +func (e *executor) LeaderAttempt(ctx context.Context, params driver.LeaderParams) (bool, error) { + var leader bool + err := e.write(ctx, func(x *w) error { + var id int64 + err := x.QueryRowContext(ctx, ` + INSERT INTO hopper_leader (name, client_id, elected_at, expires_at) VALUES ('default', ?1, ?2, ?3) + ON CONFLICT (name) DO UPDATE SET + client_id = excluded.client_id, + elected_at = CASE WHEN hopper_leader.client_id = excluded.client_id THEN hopper_leader.elected_at ELSE excluded.elected_at END, + expires_at = excluded.expires_at + WHERE hopper_leader.expires_at < ?2 OR hopper_leader.client_id = excluded.client_id + RETURNING client_id`, params.ClientID, x.us(), x.at(params.TTL)).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + leader = err == nil && id == params.ClientID + return err + }) + if err != nil { + return false, fmt.Errorf("hopper: leader attempt: %w", err) + } + return leader, nil +} + +// LeaderResign implements driver.Executor: it gives the lease up if this +// client holds it and, only then, signals other clients to elect a +// replacement. +func (e *executor) LeaderResign(ctx context.Context, clientID int64) error { + err := e.write(ctx, func(x *w) error { + res, err := x.ExecContext(ctx, `DELETE FROM hopper_leader WHERE name = 'default' AND client_id = ?`, clientID) + if err != nil { + return err + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return err + } + return x.notify(ctx, driver.ChannelLeader, "resigned") + }) + if err != nil { + return fmt.Errorf("hopper: leader resign: %w", err) + } + return nil +} + +// QueueEnsure implements driver.Executor. +func (e *executor) QueueEnsure(ctx context.Context, names []string) error { + if len(names) == 0 { + return nil + } + err := e.write(ctx, func(x *w) error { + for _, name := range names { + if _, err := x.ExecContext(ctx, `INSERT INTO hopper_queues (name, updated_at) VALUES (?, ?) ON CONFLICT (name) DO NOTHING`, name, x.us()); err != nil { + return err + } + } + return nil + }) + if err != nil { + return fmt.Errorf("hopper: ensure queues: %w", err) + } + return nil +} + +// QueuePause implements driver.Executor. +func (e *executor) QueuePause(ctx context.Context, name string) error { + return e.setPaused(ctx, name, true) +} + +// QueueResume implements driver.Executor. +func (e *executor) QueueResume(ctx context.Context, name string) error { + return e.setPaused(ctx, name, false) +} + +func (e *executor) setPaused(ctx context.Context, name string, paused bool) error { + action, stmt := "resume", `INSERT INTO hopper_queues (name, updated_at) VALUES (?1, ?2) + ON CONFLICT (name) DO UPDATE SET paused_at = NULL, updated_at = ?2` + if paused { + action, stmt = "pause", `INSERT INTO hopper_queues (name, paused_at, updated_at) VALUES (?1, ?2, ?2) + ON CONFLICT (name) DO UPDATE SET paused_at = coalesce(hopper_queues.paused_at, ?2), updated_at = ?2` + } + err := e.write(ctx, func(x *w) error { + if _, err := x.ExecContext(ctx, stmt, name, x.us()); err != nil { + return err + } + return x.notify(ctx, driver.ChannelControl, action+":"+name) + }) + if err != nil { + return fmt.Errorf("hopper: %s queue: %w", action, err) + } + return nil +} + +// QueueList implements driver.Executor. +func (e *executor) QueueList(ctx context.Context) ([]*driver.QueueRow, error) { + queues, err := queryAll(ctx, e.read(), func(r scanner) (*driver.QueueRow, error) { + var ( + q driver.QueueRow + paused micros + updated int64 + global, burst, partition, aging sql.NullInt64 + rate sql.NullFloat64 + ) + if err := r.Scan(&q.Name, &paused, &updated, &global, &rate, &burst, &partition, &aging); err != nil { + return nil, err + } + q.PausedAt, q.UpdatedAt = paused.time(), time.UnixMicro(updated).UTC() + q.Limits = driver.QueueLimits{ + Name: q.Name, GlobalLimit: int(global.Int64), RatePerSec: rate.Float64, RateBurst: int(burst.Int64), + PartitionLimit: int(partition.Int64), Aging: time.Duration(aging.Int64) * time.Second, + } + return &q, nil + }, `SELECT name, paused_at, updated_at, global_limit, rate_per_sec, rate_burst, partition_limit, aging_seconds + FROM hopper_queues ORDER BY name`) + if err != nil { + return nil, fmt.Errorf("hopper: list queues: %w", err) + } + return queues, nil +} + +func nullablePositive(v int) any { + if v <= 0 { + return nil + } + return v +} + +// QueueSetLimits implements driver.Executor. A new rate starts with a full +// bucket. +func (e *executor) QueueSetLimits(ctx context.Context, l driver.QueueLimits) error { + var rate any + if l.RatePerSec > 0 { + rate = l.RatePerSec + } + err := e.write(ctx, func(x *w) error { + if _, err := x.ExecContext(ctx, ` + INSERT INTO hopper_queues (name, global_limit, rate_per_sec, rate_burst, partition_limit, aging_seconds, tokens, refilled_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, NULL, NULL, ?7) + ON CONFLICT (name) DO UPDATE SET global_limit = excluded.global_limit, rate_per_sec = excluded.rate_per_sec, + rate_burst = excluded.rate_burst, partition_limit = excluded.partition_limit, aging_seconds = excluded.aging_seconds, + tokens = CASE WHEN hopper_queues.rate_per_sec IS NOT excluded.rate_per_sec THEN NULL ELSE hopper_queues.tokens END, + updated_at = ?7`, + l.Name, nullablePositive(l.GlobalLimit), rate, nullablePositive(l.RateBurst), nullablePositive(l.PartitionLimit), + nullablePositive(int(l.Aging/time.Second)), x.us()); err != nil { + return err + } + return x.notify(ctx, driver.ChannelControl, "limit:"+l.Name) + }) + if err != nil { + return fmt.Errorf("hopper: set queue limits: %w", err) + } + return nil +} + +// PeriodicInsert implements driver.Executor: the job is inserted only by +// the write that advances its slot, so a slot produces at most one job. +func (e *executor) PeriodicInsert(ctx context.Context, params driver.PeriodicInsertParams) (*driver.JobRow, bool, error) { + j, err := encodeInsert(params.Job) + if err != nil { + return nil, false, err + } + var ( + job *driver.JobRow + inserted bool + ) + err = e.write(ctx, func(x *w) error { + var name string + err := x.QueryRowContext(ctx, ` + INSERT INTO hopper_periodic (name, last_slot) VALUES (?, ?) + ON CONFLICT (name) DO UPDATE SET last_slot = excluded.last_slot WHERE hopper_periodic.last_slot < excluded.last_slot + RETURNING name`, params.Name, params.Slot.UnixMicro()).Scan(&name) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return err + } + results, err := x.insertJobs(ctx, []jobInsert{j}, driver.ConflictSkip) + if err != nil { + return err + } + job, inserted = results[0].Job, !results[0].Duplicate + return x.notify(ctx, driver.ChannelInsert, j.queue) + }) + if err != nil { + return nil, false, fmt.Errorf("hopper: insert periodic job %s: %w", params.Name, err) + } + return job, inserted, nil +} + +// PeriodicLastSlots implements driver.Executor. +func (e *executor) PeriodicLastSlots(ctx context.Context) (map[string]time.Time, error) { + type slot struct { + name string + at int64 + } + slots, err := queryAll(ctx, e.read(), func(r scanner) (slot, error) { + var s slot + err := r.Scan(&s.name, &s.at) + return s, err + }, `SELECT name, last_slot FROM hopper_periodic`) + if err != nil { + return nil, fmt.Errorf("hopper: periodic slots: %w", err) + } + out := make(map[string]time.Time, len(slots)) + for _, s := range slots { + out[s.name] = time.UnixMicro(s.at).UTC() + } + return out, nil +} + +// Stats implements driver.Executor, from one snapshot of the database. +func (e *executor) Stats(ctx context.Context) (*driver.Stats, error) { + stats := &driver.Stats{Queues: map[string]*driver.QueueStats{}, RunningByClient: map[int64]int{}} + queue := func(name string) *driver.QueueStats { + q := stats.Queues[name] + if q == nil { + q = &driver.QueueStats{} + stats.Queues[name] = q + } + return q + } + now := clock() + err := e.readTx(ctx, func(q querier) error { + rows, err := q.QueryContext(ctx, ` + SELECT queue, state, count(*), + min(CASE WHEN state IN ('available', 'scheduled', 'retryable') AND scheduled_at <= ?1 THEN scheduled_at END) + FROM hopper_jobs GROUP BY queue, state`, now.UnixMicro()) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var ( + name, state string + n int + oldest micros + ) + if err := rows.Scan(&name, &state, &n, &oldest); err != nil { + return err + } + qs := queue(name) + switch driver.JobState(state) { + case driver.JobStateAvailable: + qs.Available = n + case driver.JobStateScheduled: + qs.Scheduled = n + case driver.JobStateRetryable: + qs.Retryable = n + case driver.JobStateRunning: + qs.Running = n + case driver.JobStatePending, driver.JobStateCompleted, driver.JobStateCanceled, driver.JobStateDiscarded: + } + if oldest.Valid { + qs.OldestAvailable = max(qs.OldestAvailable, now.Sub(oldest.time())) + } + } + if err := rows.Err(); err != nil { + return err + } + rows, err = q.QueryContext(ctx, ` + SELECT q.name, q.paused_at IS NOT NULL, + (SELECT count(*) FROM hopper_job_history h WHERE h.queue = q.name AND h.state = 'completed' AND h.finalized_at > ?1) + FROM hopper_queues q`, now.Add(-time.Minute).UnixMicro()) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var ( + name string + paused bool + completed int + ) + if err := rows.Scan(&name, &paused, &completed); err != nil { + return err + } + qs := queue(name) + qs.Paused, qs.CompletedLastMinute = paused, completed + } + if err := rows.Err(); err != nil { + return err + } + if err := q.QueryRowContext(ctx, `SELECT count(*) FROM hopper_clients WHERE expires_at > ?`, now.UnixMicro()).Scan(&stats.LiveClients); err != nil { + return err + } + if err := q.QueryRowContext(ctx, `SELECT coalesce((SELECT client_id FROM hopper_leader WHERE name = 'default' AND expires_at > ?), 0)`, + now.UnixMicro()).Scan(&stats.Leader); err != nil { + return err + } + rows, err = q.QueryContext(ctx, `SELECT attempted_by, count(*) FROM hopper_jobs + WHERE state = 'running' AND attempted_by IS NOT NULL GROUP BY attempted_by`) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var id int64 + var n int + if err := rows.Scan(&id, &n); err != nil { + return err + } + stats.RunningByClient[id] = n + } + return rows.Err() + }) + if err != nil { + return nil, fmt.Errorf("hopper: stats: %w", err) + } + return stats, nil +} + +// retentionBatch is how many rows a retention pass deletes per +// transaction, each of which holds every other write up for as long as it +// runs. +const retentionBatch = 1000 + +// HistoryMaintain implements driver.Executor: retention deletes finished +// jobs and stream events past their retention, a batch per transaction. +// Each batch waits at most params.LockTimeout (lockWaitBrief if zero) for +// the write lock; a group whose batch gives up is reported deferred, and a +// later pass deletes the rest. History has no storage units to create or +// drop. +func (e *executor) HistoryMaintain(ctx context.Context, params driver.HistoryMaintainParams) (driver.HistoryMaintainResult, error) { + var res driver.HistoryMaintainResult + if e.tx != nil { + return res, errors.New("hopper: history maintenance must run on the pool, not in a transaction") + } + wait := params.LockTimeout + if wait <= 0 { + wait = lockWaitBrief + } + groups := []struct { + name, table, where string + keep time.Duration + }{ + {"hopper_job_history_completed", "hopper_job_history", "state = 'completed' AND finalized_at < ?", params.CompletedRetention}, + {"hopper_job_history_failed", "hopper_job_history", "state IN ('canceled', 'discarded') AND finalized_at < ?", params.FailedRetention}, + {"hopper_stream_events", "hopper_stream_events", "created_at < ?", params.StreamRetention}, + } + var errs []error + for _, g := range groups { + if g.keep <= 0 { + continue + } + for { + var n int64 + err := e.writeLocking(ctx, wait, func(x *w) error { + r, err := x.ExecContext(ctx, `DELETE FROM `+g.table+` WHERE rowid IN ( + SELECT rowid FROM `+g.table+` WHERE `+g.where+` LIMIT ?)`, x.at(-g.keep), retentionBatch) + if err != nil { + return err + } + n, err = r.RowsAffected() + return err + }) + if errors.Is(err, errLockBusy) { + res.Deferred = append(res.Deferred, g.name) + break + } + if err != nil { + errs = append(errs, fmt.Errorf("hopper: prune %s: %w", g.name, err)) + break + } + res.Pruned += n + if n < retentionBatch { + break + } + } + } + return res, errors.Join(errs...) +} + +// HistoryIndex implements driver.Executor: history's indexes are built +// with it, so a pass finds nothing to do. +func (e *executor) HistoryIndex(context.Context, driver.HistoryIndexParams) (driver.HistoryIndexResult, error) { + return driver.HistoryIndexResult{}, nil +} + +// maintainedTables are the tables JobsMaintain keeps the planner's +// statistics of: the live table, and the live tags and dependency tables, +// which churn with it. +var maintainedTables = []string{"hopper_jobs", "hopper_job_tags", "hopper_job_deps"} + +// JobsMaintain implements driver.Executor. SQLite reuses the pages of +// deleted rows and needs no vacuum, but its planner, like Postgres's, +// chooses its plans by the row counts its statistics hold: a table +// analyzed while it was empty is planned as if it still were. A table is +// analyzed when the count the statistics hold is off by an order of +// magnitude, or it has none and has grown to a thousand rows. An analyze +// reads a sample of each index (analysis_limit), and waits at most +// lockWaitBrief for the write lock; one that cannot get it waits for the +// next pass. +func (e *executor) JobsMaintain(ctx context.Context) (driver.JobsMaintainResult, error) { + var res driver.JobsMaintainResult + if e.tx != nil { + return res, errors.New("hopper: live table maintenance must run on the pool, not in a transaction") + } + var errs []error + for _, table := range maintainedTables { + believed, err := e.believedRows(ctx, table) + if err != nil { + errs = append(errs, err) + continue + } + floor := max(believed, 100) + // Count up to the point where the counts are an order of magnitude + // apart, which bounds the count's cost by the table's believed size. + var live int64 + if err := e.d.sqlDB.QueryRowContext(ctx, `SELECT count(*) FROM (SELECT 1 FROM `+table+` LIMIT ?)`, 10*floor).Scan(&live); err != nil { + errs = append(errs, fmt.Errorf("hopper: count %s: %w", table, err)) + continue + } + stale := live >= 10*floor || (believed >= 0 && believed >= 10*max(live, 100)) + if believed < 0 { + stale = live >= 1000 + } + if !stale { + continue + } + err = e.writeBrief(ctx, func(x *w) error { + _, err := x.ExecContext(ctx, "PRAGMA analysis_limit = 1000; ANALYZE "+table) + return err + }) + switch { + case errors.Is(err, errLockBusy): + case err != nil: + errs = append(errs, fmt.Errorf("hopper: analyze %s: %w", table, err)) + default: + res.Analyzed = true + } + } + return res, errors.Join(errs...) +} + +// believedRows returns the row count the planner's statistics hold for a +// table, or -1 if it has none. +func (e *executor) believedRows(ctx context.Context, table string) (int64, error) { + var hasStats bool + if err := e.d.sqlDB.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'sqlite_stat1')`).Scan(&hasStats); err != nil { + return 0, err + } + if !hasStats { + return -1, nil + } + // Each index's row starts with its count of rows; a partial index + // counts only its own, and the table's other indexes all of them. + var n sql.NullInt64 + if err := e.d.sqlDB.QueryRowContext(ctx, `SELECT max(CAST(stat AS INTEGER)) FROM sqlite_stat1 WHERE tbl = ?`, table).Scan(&n); err != nil { + return 0, err + } + if !n.Valid { + return -1, nil + } + return n.Int64, nil +} + +// AnalyzeForTest analyzes the live table as it is, for the conformance +// suite to give the planner statistics of a known state. +func (e *executor) AnalyzeForTest(ctx context.Context) error { + return e.write(ctx, func(x *w) error { + _, err := x.ExecContext(ctx, "ANALYZE hopper_jobs") + return err + }) +} diff --git a/driver/hoppersqlite/conformance_test.go b/driver/hoppersqlite/conformance_test.go new file mode 100644 index 0000000..9232bfc --- /dev/null +++ b/driver/hoppersqlite/conformance_test.go @@ -0,0 +1,142 @@ +package hoppersqlite_test + +import ( + "context" + "database/sql" + "log/slog" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/hoppersqlite" + "github.com/parallelworks/hopper/drivertest" + "github.com/parallelworks/hopper/hoppermigrate" +) + +// open returns a driver on a new database in the test's temporary +// directory, migrated to the latest schema. +func open(t testing.TB) *hoppersqlite.Driver { + t.Helper() + d, err := hoppersqlite.Open(filepath.Join(t.TempDir(), "hopper.db"), nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = d.Close() }) + if _, err := hoppermigrate.Up(context.Background(), d, &hoppermigrate.Options{Logger: slog.New(slog.DiscardHandler)}); err != nil { + t.Fatal(err) + } + return d +} + +// fixture runs the conformance suite against hoppersqlite. Its hooks are +// statements on the driver's database handle, with the host's clock as the +// database's. +type fixture struct{} + +func db(d driver.Driver[*sql.Tx]) *sql.DB { return d.(*hoppersqlite.Driver).DB() } + +func now() int64 { return time.Now().UnixMicro() } + +func exec(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) { + t.Helper() + if _, err := db(d).ExecContext(ctx, query, args...); err != nil { + t.Error(err) + } +} + +func queryText(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) string { + t.Helper() + var s string + if err := db(d).QueryRowContext(ctx, query, args...).Scan(&s); err != nil { + t.Error(err) + } + return s +} + +func count(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string) int { + t.Helper() + n, err := strconv.Atoi(queryText(ctx, t, d, query)) + if err != nil { + t.Error(err) + } + return n +} + +func (fixture) NewDriver(t *testing.T) driver.Driver[*sql.Tx] { return open(t) } + +func (fixture) Begin(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) (*sql.Tx, func() error, func() error) { + t.Helper() + tx, err := db(d).BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + return tx, tx.Commit, tx.Rollback +} + +func (fixture) ExpireLease(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], clientID int64) { + exec(ctx, t, d, "UPDATE hopper_clients SET expires_at = ? WHERE id = ?", now()-1e6, clientID) +} + +func (fixture) ExpireLeader(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) { + exec(ctx, t, d, "UPDATE hopper_leader SET expires_at = ?", now()-1e6) +} + +func (fixture) MakeClaimable(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { + exec(ctx, t, d, "UPDATE hopper_jobs SET scheduled_at = ? WHERE id = ?", now(), id[:]) +} + +func (fixture) AgeAttempt(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID, age time.Duration) { + exec(ctx, t, d, "UPDATE hopper_jobs SET attempted_at = ? WHERE id = ? AND attempted_at IS NOT NULL", now()-age.Microseconds(), id[:]) +} + +func (fixture) CountLive(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) int { + return count(ctx, t, d, "SELECT count(*) FROM hopper_jobs") +} + +func (fixture) CountHistory(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) int { + return count(ctx, t, d, "SELECT count(*) FROM hopper_job_history") +} + +func (fixture) ExpireJob(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { + exec(ctx, t, d, "UPDATE hopper_jobs SET expires_at = ? WHERE id = ?", now()-1e6, id[:]) +} + +func (fixture) StaleWake(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { + exec(ctx, t, d, "UPDATE hopper_jobs SET wake_requested_at = attempted_at - 1000000 WHERE id = ?", id[:]) +} + +func (fixture) MakePending(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { + exec(ctx, t, d, "UPDATE hopper_jobs SET state = 'pending' WHERE id = ?", id[:]) +} + +func (fixture) SnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) string { + return queryText(ctx, t, d, "SELECT coalesce(snooze_reason, '') FROM hopper_jobs WHERE id = ?", id[:]) +} + +func (fixture) SetSnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID, reason string) { + exec(ctx, t, d, "UPDATE hopper_jobs SET snooze_reason = ? WHERE id = ?", reason, id[:]) +} + +func (fixture) SetProgress(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID, value string) { + exec(ctx, t, d, "UPDATE hopper_jobs SET progress = ? WHERE id = ?", value, id[:]) +} + +func (fixture) CountEdges(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) int { + return count(ctx, t, d, "SELECT count(*) FROM hopper_job_deps") +} + +func (fixture) WaitForLockWait(_ context.Context, t *testing.T, d driver.Driver[*sql.Tx]) { + t.Helper() + for deadline := time.Now().Add(10 * time.Second); hoppersqlite.LockWaiters(d.(*hoppersqlite.Driver)) == 0; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("no statement started waiting for the write lock") + } + } +} + +func TestConformance(t *testing.T) { + t.Parallel() + drivertest.Run(t, fixture{}) +} diff --git a/driver/hoppersqlite/conn.go b/driver/hoppersqlite/conn.go new file mode 100644 index 0000000..e2fe072 --- /dev/null +++ b/driver/hoppersqlite/conn.go @@ -0,0 +1,474 @@ +package hoppersqlite + +import ( + "context" + "database/sql/driver" + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "time" + + "modernc.org/sqlite" + sqlite3 "modernc.org/sqlite/lib" +) + +// SQLite runs one writing transaction at a time, which it holds the +// database's write lock for. A transaction that reads first and writes +// later can find, once it gets the lock, that another wrote in between, and +// must fail: what it read is stale. So every transaction that may write +// takes the lock first, before its first statement reads anything: +// BEGIN IMMEDIATE. +// +// The connections of this package's database/sql handle do that for every +// transaction not begun read-only, but lazily: BeginTx sends nothing, and +// the transaction's first statement, whatever it is, takes the lock. A +// caller that begins a transaction while another holds the lock, and only +// then hands the transaction to a goroutine of its own, therefore never +// blocks in BeginTx. Within this process, transactions queue for the lock +// on a channel, in turn and for as long as their statement's context +// allows; across processes, SQLite's busy handler polls for it, in +// quanta of busyQuantum, between which the context is checked. + +// busyQuantum is the busy_timeout of every connection: how long SQLite's +// busy handler waits for a lock another process holds before the statement +// fails with SQLITE_BUSY, which this package retries while the statement's +// context allows. +const busyQuantum = 50 * time.Millisecond + +// database is what every connection to one database file in this process +// shares: the write lock and the listeners to wake when a transaction that +// notified commits. +type database struct { + path string + // lock holds a token while a connection of this process holds + // SQLite's write lock or waits for another process's to end. + lock chan struct{} + // waiting counts the statements waiting for the write lock, here or + // in another process. + waiting atomic.Int64 + // refs counts the Drivers open on the database (registry.mu). + refs int + + listenMu sync.Mutex + listeners map[chan struct{}]struct{} +} + +// registry holds the databases open in this process by path, so that two +// Drivers on one file share its write lock. +var registry = struct { + mu sync.Mutex + dbs map[string]*database +}{dbs: map[string]*database{}} + +func openDatabase(path string) *database { + registry.mu.Lock() + defer registry.mu.Unlock() + db := registry.dbs[path] + if db == nil { + db = &database{path: path, lock: make(chan struct{}, 1), listeners: map[chan struct{}]struct{}{}} + registry.dbs[path] = db + } + db.refs++ + return db +} + +func (db *database) close() { + registry.mu.Lock() + defer registry.mu.Unlock() + if db.refs--; db.refs == 0 { + delete(registry.dbs, db.path) + } +} + +// acquire takes the write lock for this process, waiting in turn for as +// long as ctx allows. +func (db *database) acquire(ctx context.Context) error { + select { + case db.lock <- struct{}{}: + return nil + default: + } + db.waiting.Add(1) + defer db.waiting.Add(-1) + select { + case db.lock <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (db *database) release() { <-db.lock } + +// listen registers a channel to be signaled when a transaction that wrote +// notifications commits; the returned function unregisters it. +func (db *database) listen(ch chan struct{}) func() { + db.listenMu.Lock() + defer db.listenMu.Unlock() + db.listeners[ch] = struct{}{} + return func() { + db.listenMu.Lock() + defer db.listenMu.Unlock() + delete(db.listeners, ch) + } +} + +// poke signals every listener of this process. +func (db *database) poke() { + db.listenMu.Lock() + defer db.listenMu.Unlock() + for ch := range db.listeners { + select { + case ch <- struct{}{}: + default: + } + } +} + +// busy reports an error of SQLite's busy family, which a statement that +// waited busyQuantum for another process's lock returns. +func busy(err error) bool { + var e *sqlite.Error + return errors.As(err, &e) && e.Code()&0xff == sqlite3.SQLITE_BUSY +} + +// retryBusy runs fn again while it fails with SQLITE_BUSY and ctx allows, +// counting itself among the database's waiters meanwhile. +func retryBusy[T any](ctx context.Context, db *database, fn func() (T, error)) (T, error) { + v, err := fn() + if !busy(err) { + return v, err + } + db.waiting.Add(1) + defer db.waiting.Add(-1) + for busy(err) && ctx.Err() == nil { + v, err = fn() + } + if busy(err) { + return v, errors.Join(ctx.Err(), err) + } + return v, err +} + +// innerConn is what a connection of modernc's driver implements. +type innerConn interface { + driver.Conn + driver.ConnPrepareContext + driver.ExecerContext + driver.QueryerContext + driver.Pinger + driver.SessionResetter + driver.Validator +} + +// connector opens the connections of a Driver's database/sql handle. +type connector struct { + db *database + dsn string + synchronous string +} + +var _ driver.Connector = (*connector)(nil) + +// pragmas sets up a connection: the busy handler's quantum, the +// durability of its commits, and foreign keys off, as SQLite has them by +// default and hopper's schema has none. +func (c *connector) pragmas() []string { + return []string{ + fmt.Sprintf("PRAGMA busy_timeout = %d", busyQuantum.Milliseconds()), + "PRAGMA synchronous = " + c.synchronous, + "PRAGMA foreign_keys = OFF", + } +} + +func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { + opened, err := sqliteDriver.Open(c.dsn) + if err != nil { + return nil, err + } + inner, ok := opened.(innerConn) + if !ok { + _ = opened.Close() + return nil, fmt.Errorf("hoppersqlite: modernc.org/sqlite's connection is a %T, which lacks a method this driver needs", opened) + } + for _, p := range c.pragmas() { + if _, err := inner.ExecContext(ctx, p, nil); err != nil { + _ = inner.Close() + return nil, fmt.Errorf("hoppersqlite: %s: %w", p, err) + } + } + return &conn{inner: inner, db: c.db}, nil +} + +func (c *connector) Driver() driver.Driver { return connectorDriver{c} } + +// connectorDriver is the driver.Driver a connector reports, which opens +// what the connector does whatever the name. +type connectorDriver struct{ c *connector } + +func (d connectorDriver) Open(string) (driver.Conn, error) { return d.c.Connect(context.Background()) } + +// conn is a connection whose transactions take the write lock on their +// first statement. +type conn struct { + inner innerConn + db *database + tx *tx +} + +var ( + _ driver.ConnBeginTx = (*conn)(nil) + _ driver.ConnPrepareContext = (*conn)(nil) + _ driver.ExecerContext = (*conn)(nil) + _ driver.QueryerContext = (*conn)(nil) + _ driver.Pinger = (*conn)(nil) + _ driver.SessionResetter = (*conn)(nil) + _ driver.Validator = (*conn)(nil) +) + +// tx is a transaction on a conn. +type tx struct { + c *conn + readOnly bool + // begun reports that BEGIN has run, and locked that the transaction + // holds this process's write lock. + begun, locked bool + // notified reports that the transaction wrote notifications, so that + // its commit wakes this process's listeners. + notified bool +} + +func (c *conn) exec(ctx context.Context, query string) error { + _, err := c.inner.ExecContext(ctx, query, nil) + return err +} + +func (c *conn) Prepare(query string) (driver.Stmt, error) { + return c.PrepareContext(context.Background(), query) +} + +func (c *conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + s, err := c.inner.PrepareContext(ctx, query) + if err != nil { + return nil, err + } + inner, ok := s.(innerStmt) + if !ok { + _ = s.Close() + return nil, fmt.Errorf("hoppersqlite: modernc.org/sqlite's statement is a %T, which lacks a method this driver needs", s) + } + return &stmt{inner: inner, c: c, query: query}, nil +} + +func (c *conn) Close() error { + if c.tx != nil { + _ = c.tx.Rollback() + } + return c.inner.Close() +} + +func (c *conn) Begin() (driver.Tx, error) { return c.BeginTx(context.Background(), driver.TxOptions{}) } + +// BeginTx begins a transaction. One begun read-only reads a snapshot of +// the database and may not write; any other takes the write lock on its +// first statement. Every isolation level is SQLite's, serializable, which +// is at least as strong as any asked for. +func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if c.tx != nil { + return nil, errors.New("hoppersqlite: a transaction is already open on this connection") + } + t := &tx{c: c, readOnly: opts.ReadOnly} + if opts.ReadOnly { + if err := c.exec(ctx, "PRAGMA query_only = ON"); err != nil { + return nil, err + } + if _, err := retryBusy(ctx, c.db, func() (struct{}, error) { return struct{}{}, c.exec(ctx, "BEGIN") }); err != nil { + _ = c.exec(context.Background(), "PRAGMA query_only = OFF") + return nil, err + } + t.begun = true + } + c.tx = t + return t, nil +} + +// beginWrite takes the write lock for the open transaction and begins it. +func (c *conn) beginWrite(ctx context.Context) error { + if err := c.db.acquire(ctx); err != nil { + return err + } + if _, err := retryBusy(ctx, c.db, func() (struct{}, error) { return struct{}{}, c.exec(ctx, "BEGIN IMMEDIATE") }); err != nil { + c.db.release() + return err + } + c.tx.begun, c.tx.locked = true, true + return nil +} + +// statement prepares the connection to run query: inside a transaction, +// it begins the transaction if this is its first statement, and notes a +// notification written; outside one, a statement that may write (exec) +// takes the write lock for its run, which done releases. +func (c *conn) statement(ctx context.Context, query string, write bool) (done func(), err error) { + if c.tx == nil { + if !write { + return func() {}, nil + } + if err := c.db.acquire(ctx); err != nil { + return nil, err + } + return c.db.release, nil + } + if !c.tx.begun { + if err := c.beginWrite(ctx); err != nil { + return nil, err + } + } + if query == notifySQL { + c.tx.notified = true + } + return func() {}, nil +} + +func (c *conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + done, err := c.statement(ctx, query, true) + if err != nil { + return nil, err + } + defer done() + if c.tx != nil { + return c.inner.ExecContext(ctx, query, args) + } + return retryBusy(ctx, c.db, func() (driver.Result, error) { return c.inner.ExecContext(ctx, query, args) }) +} + +func (c *conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + if _, err := c.statement(ctx, query, false); err != nil { + return nil, err + } + if c.tx != nil { + return c.inner.QueryContext(ctx, query, args) + } + return retryBusy(ctx, c.db, func() (driver.Rows, error) { return c.inner.QueryContext(ctx, query, args) }) +} + +func (c *conn) Ping(ctx context.Context) error { return c.inner.Ping(ctx) } + +func (c *conn) ResetSession(ctx context.Context) error { + if c.tx != nil { + _ = c.tx.Rollback() + } + return c.inner.ResetSession(ctx) +} + +func (c *conn) IsValid() bool { return c.inner.IsValid() } + +// end ends the transaction's hold on the connection and the lock. +func (t *tx) end() { + if t.locked { + t.c.db.release() + } + if t.readOnly { + _ = t.c.exec(context.Background(), "PRAGMA query_only = OFF") + } + t.c.tx = nil +} + +func (t *tx) Commit() error { + defer t.end() + if !t.begun { + return nil + } + if err := t.c.exec(context.Background(), "COMMIT"); err != nil { + // A COMMIT that failed can leave the transaction open, which + // database/sql does not expect of a connection it reuses. + _ = t.rollback() + return err + } + if t.notified { + t.c.db.poke() + } + return nil +} + +func (t *tx) Rollback() error { + defer t.end() + if !t.begun { + return nil + } + return t.rollback() +} + +// rollback rolls the transaction back. SQLite rolls a transaction back +// itself after some errors (a full disk, an I/O error), which leaves +// nothing to roll back here. +func (t *tx) rollback() error { + err := t.c.exec(context.Background(), "ROLLBACK") + if err != nil && strings.Contains(err.Error(), "no transaction is active") { + return nil + } + return err +} + +// innerStmt is what a statement of modernc's driver implements. +type innerStmt interface { + driver.Stmt + driver.StmtExecContext + driver.StmtQueryContext +} + +// stmt is a prepared statement whose runs take part in their connection's +// transaction as its statements do. +type stmt struct { + inner innerStmt + c *conn + query string +} + +var ( + _ driver.StmtExecContext = (*stmt)(nil) + _ driver.StmtQueryContext = (*stmt)(nil) +) + +func (s *stmt) Close() error { return s.inner.Close() } +func (s *stmt) NumInput() int { return s.inner.NumInput() } + +func (s *stmt) Exec(args []driver.Value) (driver.Result, error) { + return s.ExecContext(context.Background(), named(args)) +} + +func (s *stmt) Query(args []driver.Value) (driver.Rows, error) { + return s.QueryContext(context.Background(), named(args)) +} + +func (s *stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + done, err := s.c.statement(ctx, s.query, true) + if err != nil { + return nil, err + } + defer done() + if s.c.tx != nil { + return s.inner.ExecContext(ctx, args) + } + return retryBusy(ctx, s.c.db, func() (driver.Result, error) { return s.inner.ExecContext(ctx, args) }) +} + +func (s *stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + if _, err := s.c.statement(ctx, s.query, false); err != nil { + return nil, err + } + if s.c.tx != nil { + return s.inner.QueryContext(ctx, args) + } + return retryBusy(ctx, s.c.db, func() (driver.Rows, error) { return s.inner.QueryContext(ctx, args) }) +} + +func named(args []driver.Value) []driver.NamedValue { + out := make([]driver.NamedValue, len(args)) + for i, v := range args { + out[i] = driver.NamedValue{Ordinal: i + 1, Value: v} + } + return out +} diff --git a/driver/hoppersqlite/executor.go b/driver/hoppersqlite/executor.go new file mode 100644 index 0000000..ec44974 --- /dev/null +++ b/driver/hoppersqlite/executor.go @@ -0,0 +1,375 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "time" + + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/internal/jobcodec" +) + +// executor implements driver.Executor on the pool, or inside a caller's +// transaction. +type executor struct { + d *Driver + tx *sql.Tx +} + +var _ driver.Executor = (*executor)(nil) + +// querier is what *sql.DB, *sql.Tx and *sql.Conn share. +type querier interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// errLockBusy reports that a write gave up waiting briefly for the write +// lock (writeBrief). +var errLockBusy = errors.New("hoppersqlite: another transaction holds the database's write lock") + +// clock returns the database's time, which is the host's (see the package +// documentation), in Postgres's precision. +func clock() time.Time { return time.Now().UTC().Truncate(time.Microsecond) } + +// w is a write in progress: a transaction that holds the write lock, and +// its time, read once the lock was taken, so that every write sees a time +// no earlier than the writes before it. +type w struct { + querier + now time.Time +} + +// us is the write's time in the unit the schema keeps times in. +func (x *w) us() int64 { return x.now.UnixMicro() } + +// at returns the write's time plus d, in the schema's unit. +func (x *w) at(d time.Duration) int64 { return x.now.Add(d).UnixMicro() } + +// write runs fn in a transaction that holds the write lock: the caller's, +// inside a savepoint so that an error leaves it as it was, or a new one. +func (e *executor) write(ctx context.Context, fn func(*w) error) error { + return e.writeLocking(ctx, 0, fn) +} + +// writeBrief is write that waits at most lockWaitBrief for a write lock +// another transaction holds, and then returns errLockBusy. +func (e *executor) writeBrief(ctx context.Context, fn func(*w) error) error { + return e.writeLocking(ctx, lockWaitBrief, fn) +} + +func (e *executor) writeLocking(ctx context.Context, wait time.Duration, fn func(*w) error) (err error) { + // The first statement takes the lock, under its own deadline when the + // wait is bounded. + lock := func(q querier, statement string) error { + lctx := ctx + if wait > 0 { + var cancel context.CancelFunc + lctx, cancel = context.WithTimeout(ctx, wait) + defer cancel() + } + _, err := q.ExecContext(lctx, statement) + if err != nil && wait > 0 && ctx.Err() == nil && errors.Is(err, context.DeadlineExceeded) { + return errLockBusy + } + return err + } + if e.tx != nil { + if err := lock(e.tx, "SAVEPOINT hopper"); err != nil { + return err + } + defer func() { + if err != nil { + _, rerr := e.tx.ExecContext(context.WithoutCancel(ctx), "ROLLBACK TO hopper; RELEASE hopper") + err = errors.Join(err, rerr) + } + }() + if err := fn(&w{querier: e.tx, now: clock()}); err != nil { + return err + } + _, err := e.tx.ExecContext(ctx, "RELEASE hopper") + return err + } + tx, err := e.d.sqlDB.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { + if err != nil { + err = errors.Join(err, ignoreDone(tx.Rollback())) + } + }() + if err := lock(tx, "SELECT 1"); err != nil { + return err + } + if err := fn(&w{querier: tx, now: clock()}); err != nil { + return err + } + return tx.Commit() +} + +func ignoreDone(err error) error { + if errors.Is(err, sql.ErrTxDone) { + return nil + } + return err +} + +// read returns what a read runs on: the caller's transaction, or the pool, +// on which each statement reads the latest committed state. +func (e *executor) read() querier { + if e.tx != nil { + return e.tx + } + return e.d.sqlDB +} + +// readTx runs fn on one snapshot of the database: the caller's +// transaction, or a read-only one. +func (e *executor) readTx(ctx context.Context, fn func(querier) error) (err error) { + if e.tx != nil { + return fn(e.tx) + } + tx, err := e.d.sqlDB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return err + } + defer func() { err = errors.Join(err, ignoreDone(tx.Rollback())) }() + return fn(tx) +} + +// notifySQL writes a notification. conn.go recognizes the statement by its +// text, to wake this process's listeners when the transaction commits. +const notifySQL = "INSERT INTO hopper_notifications (channel, payload, created_at) VALUES (?, ?, ?)" + +// notificationRetention is how long notifications are kept for listeners +// that fall behind; one that falls further behind misses some, and its +// client polls as it would without them. +const notificationRetention = time.Minute + +// notify writes notifications, one per payload, delivered when the write +// commits, and deletes those past notificationRetention. +func (x *w) notify(ctx context.Context, channel string, payloads ...string) error { + if len(payloads) == 0 { + return nil + } + for _, p := range payloads { + if _, err := x.ExecContext(ctx, notifySQL, channel, p, x.us()); err != nil { + return fmt.Errorf("hopper: notify: %w", err) + } + } + if _, err := x.ExecContext(ctx, "DELETE FROM hopper_notifications WHERE created_at < ?", x.at(-notificationRetention)); err != nil { + return fmt.Errorf("hopper: notify: %w", err) + } + return nil +} + +// Notify implements driver.Executor. +func (e *executor) Notify(ctx context.Context, channel string, payloads []string) error { + if len(payloads) == 0 { + return nil + } + return e.write(ctx, func(x *w) error { return x.notify(ctx, channel, payloads...) }) +} + +// Now implements driver.Executor. +func (e *executor) Now(context.Context) (time.Time, error) { return clock(), nil } + +// SchemaVersion implements driver.Executor. +func (e *executor) SchemaVersion(ctx context.Context) (int, error) { + versions, err := schemaVersions(ctx, e.read()) + if err != nil { + return 0, fmt.Errorf("hopper: read schema version: %w", err) + } + if len(versions) == 0 { + return 0, nil + } + return slices.Max(versions), nil +} + +// Rows. + +// jobColumns are hopper_jobs' columns in scan order; historyColumns are +// history's, in the same order, with no snooze reason (a finalized job is +// not waiting), followed by finalized_at and output. +const ( + jobColumns = `id, kind, queue, state, priority, attempt, max_attempts, scheduled_at, attempted_at, attempted_by, + args, metadata, errors, unique_key, ordering_key, partition_key, batch_id, expires_at, cancel_requested_at, await, + created_at, snooze_reason, progress, tags, fence` + historyColumns = `id, kind, queue, state, priority, attempt, max_attempts, scheduled_at, attempted_at, attempted_by, + args, metadata, errors, unique_key, ordering_key, partition_key, batch_id, expires_at, cancel_requested_at, await, + created_at, NULL AS snooze_reason, progress, tags, fence, finalized_at, output` + // liveAsHistory reads live rows in historyColumns' shape. + liveAsHistory = jobColumns + `, NULL AS finalized_at, NULL AS output` +) + +// prefixed qualifies a column list with a table alias. +func prefixed(columns, alias string) string { + parts := strings.Split(columns, ",") + for i, p := range parts { + p = strings.TrimSpace(p) + if !strings.HasPrefix(p, "NULL") { + p = alias + "." + p + } + parts[i] = p + } + return strings.Join(parts, ", ") +} + +type scanner interface{ Scan(dest ...any) error } + +// micros scans a nullable time column. +type micros struct{ sql.NullInt64 } + +func (m micros) time() time.Time { + if !m.Valid { + return time.Time{} + } + return time.UnixMicro(m.Int64).UTC() +} + +// scanJob scans a row of jobColumns, plus finalized_at and output when +// history is true, plus trailing columns into extra. +func scanJob(row scanner, history bool, extra ...any) (*driver.JobRow, error) { + var ( + j driver.JobRow + id, batchID []byte + state string + scheduledAt int64 + attemptedAt, expiresAt, cancelAt, createdAt, final micros + attemptedBy sql.NullInt64 + args, metadata, errs string + uniqueKey, orderingKey, partKey, snoozeReason sql.NullString + progress, tags, output sql.NullString + await bool + ) + dest := []any{ + &id, &j.Kind, &j.Queue, &state, &j.Priority, &j.Attempt, &j.MaxAttempts, &scheduledAt, &attemptedAt, &attemptedBy, + &args, &metadata, &errs, &uniqueKey, &orderingKey, &partKey, &batchID, &expiresAt, &cancelAt, &await, + &createdAt, &snoozeReason, &progress, &tags, &j.Fence, + } + if history { + dest = append(dest, &final, &output) + } + if err := row.Scan(append(dest, extra...)...); err != nil { + return nil, err + } + if err := j.ID.Scan(id); err != nil { + return nil, err + } + if len(batchID) > 0 { + if err := j.BatchID.Scan(batchID); err != nil { + return nil, err + } + } + var err error + if j.Errors, err = jobcodec.DecodeErrors(errs); err != nil { + return nil, fmt.Errorf("errors of job %s: %w", j.ID, err) + } + j.State = driver.JobState(state) + j.ScheduledAt = time.UnixMicro(scheduledAt).UTC() + j.AttemptedAt, j.ExpiresAt, j.CancelRequestedAt = attemptedAt.time(), expiresAt.time(), cancelAt.time() + j.CreatedAt, j.FinalizedAt = createdAt.time(), final.time() + j.AttemptedBy = attemptedBy.Int64 + j.Args, j.Metadata = json.RawMessage(args), json.RawMessage(metadata) + if output.Valid { + j.Output = json.RawMessage(output.String) + } + j.UniqueKey, j.OrderingKey, j.PartitionKey = uniqueKey.String, orderingKey.String, partKey.String + j.Await = await + j.Tags = jobcodec.DecodeTags(tags.String) + // Only a snoozed job is waiting for a reason; one left on a job in + // another state by a direct write is not reported. + if j.State == driver.JobStateScheduled { + j.SnoozeReason = snoozeReason.String + } + if progress.Valid { + j.Progress = jobcodec.DecodeProgress(progress.String) + } + return &j, nil +} + +// queryAll runs a query and reads every row it returns through scan. +func queryAll[T any](ctx context.Context, q querier, scan func(scanner) (T, error), query string, args ...any) ([]T, error) { + rows, err := q.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []T + for rows.Next() { + v, err := scan(rows) + if err != nil { + return nil, err + } + out = append(out, v) + } + return out, rows.Err() +} + +// scanHistoryJob scans a row in historyColumns' shape. +func scanHistoryJob(r scanner) (*driver.JobRow, error) { return scanJob(r, true) } + +// Parameters. + +func idParam(id driver.JobID) []byte { return id[:] } + +// nullableID is a BLOB parameter for an ID, NULL for the zero ID. +func nullableID(id driver.JobID) any { + if id.IsZero() { + return nil + } + return id[:] +} + +// nullable is a TEXT parameter, NULL for the empty string. +func nullable(s string) any { + if s == "" { + return nil + } + return s +} + +// smallint checks a value bound for what Postgres keeps in a smallint, so +// that a value means the same on either engine. +func smallint(v int, column string) (int, error) { + if v < -32768 || v > 32767 { + return 0, fmt.Errorf("hopper: %s %d is out of range", column, v) + } + return v, nil +} + +// jsonValue is a JSON parameter: raw, or def when empty. It refuses text +// that is not JSON, as Postgres's jsonb does. +func jsonValue(raw []byte, def, what string) (string, error) { + if len(raw) == 0 { + return def, nil + } + if !json.Valid(raw) { + return "", fmt.Errorf("hopper: %s is not JSON: %.40q", what, raw) + } + return string(raw), nil +} + +// tagsJSON is a job's tags column: a JSON array, or NULL for none. +func tagsJSON(tags []string) any { + if len(tags) == 0 { + return nil + } + b, _ := json.Marshal(tags) + return string(b) +} + +// placeholders returns n comma-separated parameter markers. +func placeholders(n int) string { + if n <= 0 { + return "" + } + return strings.Repeat("?, ", n-1) + "?" +} diff --git a/driver/hoppersqlite/export_test.go b/driver/hoppersqlite/export_test.go new file mode 100644 index 0000000..54b2e8c --- /dev/null +++ b/driver/hoppersqlite/export_test.go @@ -0,0 +1,5 @@ +package hoppersqlite + +// LockWaiters counts the statements waiting for d's database's write lock, +// in this process or for another process's. +func LockWaiters(d *Driver) int64 { return d.db.waiting.Load() } diff --git a/driver/hoppersqlite/finalize.go b/driver/hoppersqlite/finalize.go new file mode 100644 index 0000000..c96632a --- /dev/null +++ b/driver/hoppersqlite/finalize.go @@ -0,0 +1,570 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "math" + "time" + + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/internal/jobcodec" + "github.com/parallelworks/hopper/internal/jobtags" +) + +// Every write that finalizes jobs (the batched finalize, canceling or +// deleting a waiting job, expiring) runs in a transaction that holds the +// write lock, so it reads each job, checks it in Go and writes it back with +// no other writer in between: what Postgres's statements do in one +// statement, with row locks and a recheck of every condition, is here a +// sequence of plain statements. The tail does what finalized jobs entail +// in the same transaction: canceling the dependents of a failed job, +// archiving, counting the jobs down in their batches and completing those +// they finish, promoting dependents whose dependencies are all finished, +// and dropping satisfied edges. No other transaction can finalize a +// batch's or a dependent's last jobs at the same time, so nothing is left +// for BatchSettle or WorkflowPromoteReady to find. + +// liveRow is a live job's row with the columns its finalize reads beside +// the JobRow's. +type liveRow struct { + job *driver.JobRow + logs sql.NullString + wakeRequestedAt micros + snoozeReason sql.NullString + // errors is the errors column as stored, which a finalize appends to. + errors string +} + +const liveRowColumns = jobColumns + ", logs, wake_requested_at, snooze_reason, errors" + +func scanLive(row scanner) (*liveRow, error) { + var r liveRow + var err error + r.job, err = scanJob(row, false, &r.logs, &r.wakeRequestedAt, &r.snoozeReason, &r.errors) + return &r, err +} + +// liveRow returns a live job's row, or nil if the job is not live. +func (x *w) liveRow(ctx context.Context, id driver.JobID) (*liveRow, error) { + r, err := scanLive(x.QueryRowContext(ctx, "SELECT "+liveRowColumns+" FROM hopper_jobs WHERE id = ?", idParam(id))) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return r, err +} + +// finished is a job leaving the live table: its final state and what its +// history row keeps. +type finished struct { + row *liveRow + state driver.JobState + errors string + logs sql.NullString + output any + archive bool +} + +func (f finished) failed() bool { return f.state != driver.JobStateCompleted } + +// jobRow returns the job as it is once finished: in its final state, with +// its errors, logs and output, finalized at now, and waiting for nothing. +func (f finished) jobRow(now time.Time) (*driver.JobRow, error) { + j := *f.row.job + j.State, j.FinalizedAt, j.SnoozeReason = f.state, now, "" + var err error + if j.Errors, err = jobcodec.DecodeErrors(f.errors); err != nil { + return nil, err + } + j.Output = nil + if s, ok := f.output.(string); ok { + j.Output = json.RawMessage(s) + } + return &j, nil +} + +// archiveSQL copies a live job to history, with its final state, errors, +// logs, finalization time and output. +const archiveSQL = ` +INSERT INTO hopper_job_history (id, seq, kind, queue, state, priority, attempt, max_attempts, scheduled_at, attempted_at, + attempted_by, args, metadata, errors, unique_key, ordering_key, partition_key, batch_id, expires_at, cancel_requested_at, + await, created_at, progress, tags, fence, logs, finalized_at, output) +SELECT id, seq, kind, queue, ?, priority, attempt, max_attempts, scheduled_at, attempted_at, + attempted_by, args, metadata, ?, unique_key, ordering_key, partition_key, batch_id, expires_at, cancel_requested_at, + await, created_at, progress, tags, fence, ?, ?, ? +FROM hopper_jobs WHERE id = ?` + +// finish moves a job out of the live table, to history if it is archived, +// and announces it to those awaiting it. +func (x *w) finish(ctx context.Context, f finished) error { + id := idParam(f.row.job.ID) + if f.archive { + if _, err := x.ExecContext(ctx, archiveSQL, string(f.state), f.errors, f.logs, x.us(), f.output, id); err != nil { + return fmt.Errorf("archive job %s: %w", f.row.job.ID, err) + } + } + if _, err := x.ExecContext(ctx, "DELETE FROM hopper_jobs WHERE id = ?", id); err != nil { + return fmt.Errorf("delete job %s: %w", f.row.job.ID, err) + } + if f.row.job.Await { + return x.notify(ctx, driver.ChannelDone, f.row.job.ID.String()) + } + return nil +} + +// appendJSON appends an element to a JSON array as stored, keeping the +// elements there as they are. A value that is not an array starts a new one. +func appendJSON(stored string, elem []byte) (string, error) { + var elems []json.RawMessage + _ = json.Unmarshal([]byte(stored), &elems) + b, err := json.Marshal(append(elems, elem)) + return string(b), err +} + +// withErrorAt appends an error record, timestamped at now, to stored. +func withErrorAt(stored string, e *driver.AttemptError, now time.Time) (string, error) { + b, err := jobcodec.MarshalAttemptError(e) + if err != nil { + return "", err + } + var rec map[string]json.RawMessage + if err := json.Unmarshal(b, &rec); err != nil { + return "", err + } + rec["at"], _ = json.Marshal(now) + b, err = json.Marshal(rec) + if err != nil { + return "", err + } + return appendJSON(stored, b) +} + +// hopperError is the errors column of a job that hopper itself fails, such +// as one canceled or expired while waiting: a snoozed job's reason is kept +// in the error, since the row that held it is leaving the live table. +func hopperError(r *liveRow, message string, now time.Time) (string, error) { + if r.job.State == driver.JobStateScheduled && r.snoozeReason.Valid { + message += " while snoozed: " + r.snoozeReason.String + } + return withErrorAt(r.errors, &driver.AttemptError{Attempt: r.job.Attempt, Error: message}, now) +} + +// keepLogs is how many attempt logs a finalize keeps for keep. +func keepLogs(keep int) int { + if keep <= 0 { + return math.MaxInt32 + } + return keep +} + +// appendLog appends an attempt's log, if it has one, to a job's logs as +// stored, keeping the latest keep. A snooze gives its attempt back, so a job +// that snoozes again and again would fill its logs with them: a snooze's +// log, or its lack of one, replaces the log of the job's previous snooze of +// the same attempt number, attempt. Logs that do not change are returned +// as stored; none at all, as NULL. +func appendLog(stored sql.NullString, attempt int, snooze bool, log []byte, keep int) (sql.NullString, error) { + var entries []json.RawMessage + if stored.Valid { + _ = json.Unmarshal([]byte(stored.String), &entries) + } + sameSnooze := func(raw json.RawMessage) bool { + var e map[string]json.RawMessage + if json.Unmarshal(raw, &e) != nil { + return false + } + _, snoozed := e["snoozed"] + var a json.Number + return snoozed && json.Unmarshal(e["attempt"], &a) == nil && a.String() == fmt.Sprint(attempt) + } + replaces := false + if snooze { + for _, e := range entries { + replaces = replaces || sameSnooze(e) + } + } + if log == nil && !replaces { + return stored, nil + } + kept := entries[:0:0] + for _, e := range entries { + if !snooze || !sameSnooze(e) { + kept = append(kept, e) + } + } + if log != nil { + kept = append(kept, log) + } + if len(kept) > keep { + kept = kept[len(kept)-keep:] + } + if len(kept) == 0 { + return sql.NullString{}, nil + } + b, err := json.Marshal(kept) + return sql.NullString{String: string(b), Valid: true}, err +} + +// result is a JobFinalize with its values encoded. +type result struct { + driver.JobFinalize + err *driver.AttemptError + log []byte + output any + reason any +} + +func encodeResults(jobs []driver.JobFinalize) ([]result, error) { + out := make([]result, len(jobs)) + for i, f := range jobs { + switch f.State { + case driver.JobStateCompleted, driver.JobStateCanceled, driver.JobStateDiscarded, + driver.JobStateRetryable, driver.JobStateScheduled: + default: + return nil, fmt.Errorf("hopper: finalize jobs: %s is not a finalize state", f.State) + } + r := result{JobFinalize: f, err: f.Error, reason: nullable(jobcodec.StorableText(f.SnoozeReason))} + if f.Log != nil && f.KeepsLog() { + b, err := json.Marshal(driver.AttemptLog{Attempt: f.Log.Attempt, Snoozed: f.Snooze, Log: jobcodec.StorableText(f.Log.Log)}) + if err != nil { + return nil, fmt.Errorf("hopper: finalize jobs: %w", err) + } + r.log = b + } + if len(f.Output) > 0 { + out, err := jsonValue(f.Output, "", "output") + if err != nil { + return nil, fmt.Errorf("hopper: finalize jobs: %w", err) + } + r.output = out + } + out[i] = r + } + return out, nil +} + +// JobFinalizeMany implements driver.Executor. A result applies only to a +// job running in the claim that produced it, by the same client at the same +// fence, so a stale result never overwrites a newer attempt. With +// SkipLocked, a write lock another transaction holds is waited for only +// briefly, after which no result is applied. +func (e *executor) JobFinalizeMany(ctx context.Context, params driver.JobFinalizeParams) ([]driver.JobID, error) { + if len(params.Jobs) == 0 { + return nil, nil + } + results, err := encodeResults(params.Jobs) + if err != nil { + return nil, err + } + keep := keepLogs(params.KeepLogs) + var applied []driver.JobID + write := e.write + if params.SkipLocked { + write = e.writeBrief + } + err = write(ctx, func(x *w) error { + applied = applied[:0] + var done []finished + for _, r := range results { + row, err := x.liveRow(ctx, r.ID) + if err != nil { + return err + } + if row == nil { + continue + } + j := row.job + if j.State != driver.JobStateRunning || j.AttemptedBy == 0 || j.AttemptedBy != r.AttemptedBy || j.Fence != r.Fence { + continue + } + errs := row.errors + if r.err != nil { + if errs, err = withErrorAt(errs, r.err, x.now); err != nil { + return err + } + } + logs, err := appendLog(row.logs, j.Attempt, r.Snooze, r.log, keep) + if err != nil { + return err + } + if r.State.Terminal() { + f := finished{row: row, state: r.State, errors: errs, logs: logs, output: r.output, archive: r.Archived()} + if err := x.finish(ctx, f); err != nil { + return err + } + done = append(done, f) + applied = append(applied, r.ID) + continue + } + // A wake since the attempt began (JobRetryOrderingKey) makes the + // job runnable at once, and takes the place of a snooze's reason. + woken := row.wakeRequestedAt.Valid && !j.AttemptedAt.IsZero() && !row.wakeRequestedAt.time().Before(j.AttemptedAt) + scheduledAt := x.at(max(r.Delay, 0)) + if woken { + scheduledAt = x.us() + } + attempt := j.Attempt + var reason any + if r.Snooze { + attempt-- + if !woken { + reason = r.reason + } + } + if _, err := x.ExecContext(ctx, ` + UPDATE hopper_jobs SET state = ?, scheduled_at = ?, attempt = ?, errors = ?, logs = ?, attempted_by = NULL, + snooze_reason = ?, wake_requested_at = NULL + WHERE id = ?`, string(r.State), scheduledAt, attempt, errs, logs, reason, idParam(r.ID)); err != nil { + return err + } + applied = append(applied, r.ID) + } + if params.NoBatches { + return nil + } + _, err := x.tail(ctx, done) + return err + }) + if errors.Is(err, errLockBusy) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("hopper: finalize jobs: %w", err) + } + return applied, nil +} + +// tail does what the jobs in done, which have just left the live table, +// entail, and returns done with the dependents it canceled. +func (x *w) tail(ctx context.Context, done []finished) ([]finished, error) { + if len(done) == 0 { + return done, nil + } + // Cancel the dependents of failed jobs, down the graph by cancel edges. + cascaded := map[driver.JobID]bool{} + var queue []driver.JobID + for _, f := range done { + if f.failed() { + queue = append(queue, f.row.job.ID) + } + } + seen := map[driver.JobID]bool{} + for len(queue) > 0 { + id := queue[0] + queue = queue[1:] + deps, err := x.ids(ctx, "SELECT job_id FROM hopper_job_deps WHERE depends_on = ? AND on_failure = 'cancel'", idParam(id)) + if err != nil { + return nil, err + } + for _, dep := range deps { + if seen[dep] { + continue + } + seen[dep] = true + queue = append(queue, dep) + row, err := x.liveRow(ctx, dep) + if err != nil { + return nil, err + } + if row == nil || row.job.State != driver.JobStatePending { + continue + } + errs, err := hopperError(row, "hopper: dependency failed", x.now) + if err != nil { + return nil, err + } + f := finished{row: row, state: driver.JobStateCanceled, errors: errs, logs: row.logs, archive: true} + if err := x.finish(ctx, f); err != nil { + return nil, err + } + done = append(done, f) + cascaded[dep] = true + } + } + + // Count the jobs down in their batches. + type count struct{ n, failed int } + var batches []driver.JobID + counts := map[driver.JobID]*count{} + for _, f := range done { + b := f.row.job.BatchID + if b.IsZero() { + continue + } + c := counts[b] + if c == nil { + c = &count{} + counts[b] = c + batches = append(batches, b) + } + c.n++ + if f.failed() { + c.failed++ + } + } + for _, b := range batches { + if err := x.countBatch(ctx, b, counts[b].n, counts[b].failed); err != nil { + return nil, err + } + } + + // Promote the dependents whose dependencies have all finished, and drop + // the edges of the jobs that finished. + considered := map[driver.JobID]bool{} + for _, f := range done { + deps, err := x.ids(ctx, "SELECT job_id FROM hopper_job_deps WHERE depends_on = ?", idParam(f.row.job.ID)) + if err != nil { + return nil, err + } + for _, dep := range deps { + if considered[dep] || cascaded[dep] { + continue + } + considered[dep] = true + if _, err := x.promoteIfReady(ctx, dep); err != nil { + return nil, err + } + } + } + for _, f := range done { + if _, err := x.ExecContext(ctx, "DELETE FROM hopper_job_deps WHERE job_id = ?", idParam(f.row.job.ID)); err != nil { + return nil, err + } + } + return done, nil +} + +// ids returns the IDs a query selects. +func (x *w) ids(ctx context.Context, query string, args ...any) ([]driver.JobID, error) { + return queryAll(ctx, x, func(r scanner) (driver.JobID, error) { + var b []byte + if err := r.Scan(&b); err != nil { + return driver.JobID{}, err + } + var id driver.JobID + err := id.Scan(b) + return id, err + }, query, args...) +} + +// promoteIfReady makes a pending job claimable if none of its dependencies +// is still live, and reports whether it did. +func (x *w) promoteIfReady(ctx context.Context, id driver.JobID) (bool, error) { + var queue string + err := x.QueryRowContext(ctx, ` + UPDATE hopper_jobs SET state = CASE WHEN scheduled_at > ? THEN 'scheduled' ELSE 'available' END + WHERE id = ? AND state = 'pending' + AND NOT EXISTS (SELECT 1 FROM hopper_job_deps d JOIN hopper_jobs x ON x.id = d.depends_on WHERE d.job_id = ?) + RETURNING queue`, x.us(), idParam(id), idParam(id)).Scan(&queue) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, err + } + return true, x.notify(ctx, driver.ChannelInsert, queue) +} + +// countBatch counts n jobs of a batch, failed of them canceled or +// discarded, as finished, and completes the batch, inserting its +// callbacks, if nothing remains. A batch that has completed already counts +// them too, as a Postgres batch's settle does. +func (x *w) countBatch(ctx context.Context, id driver.JobID, n, failed int) error { + var ( + pending, wasFailed int + completed micros + onSuccess, onFailure, onComplete sql.NullString + ) + err := x.QueryRowContext(ctx, `SELECT pending, failed, completed_at, on_success, on_failure, on_complete FROM hopper_batches WHERE id = ?`, + idParam(id)).Scan(&pending, &wasFailed, &completed, &onSuccess, &onFailure, &onComplete) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("batch %s: %w", id, err) + } + pending -= n + failed += wasFailed + if completed.Valid || pending > 0 { + _, err := x.ExecContext(ctx, `UPDATE hopper_batches SET pending = ?, failed = ? WHERE id = ?`, max(pending, 0), failed, idParam(id)) + return err + } + if _, err := x.ExecContext(ctx, `UPDATE hopper_batches SET pending = 0, failed = ?, completed_at = ? WHERE id = ?`, failed, x.us(), idParam(id)); err != nil { + return err + } + outcome := onSuccess + if failed > 0 { + outcome = onFailure + } + for _, cb := range []sql.NullString{outcome, onComplete} { + if cb.Valid { + if err := x.insertCallback(ctx, cb.String, id, failed); err != nil { + return fmt.Errorf("batch %s: callback: %w", id, err) + } + } + } + return nil +} + +// insertCallback inserts a completed batch's callback. Its values are read +// as Postgres reads them: brought into range, tags as a job's tags column +// reads them, and the batch's ID and failures added to its metadata, so +// that one batch row a direct write made odd cannot fail the finalize that +// completes it. +func (x *w) insertCallback(ctx context.Context, stored string, batch driver.JobID, failed int) error { + var cb struct { + Kind string `json:"kind"` + Queue string `json:"queue"` + Priority json.Number `json:"priority"` + MaxAttempts json.Number `json:"max_attempts"` + Args json.RawMessage `json:"args"` + Metadata json.RawMessage `json:"metadata"` + Tags json.RawMessage `json:"tags"` + } + dec := json.NewDecoder(bytesReader(stored)) + dec.UseNumber() + if err := dec.Decode(&cb); err != nil { + return err + } + clamp := func(n json.Number, def, lo, hi float64) int { + v, err := n.Float64() + if err != nil { + v = def + } + return int(math.Min(math.Max(math.Round(v), lo), hi)) + } + queue := cb.Queue + if queue == "" { + queue = "default" + } + args := "{}" + if len(cb.Args) > 0 && string(cb.Args) != "null" { + args = string(cb.Args) + } + meta := map[string]json.RawMessage{} + if len(cb.Metadata) > 0 { + _ = json.Unmarshal(cb.Metadata, &meta) + } + meta["batch_id"], _ = json.Marshal(batch.String()) + meta["batch_failed"], _ = json.Marshal(failed) + metadata, err := json.Marshal(meta) + if err != nil { + return err + } + var tags []string + for _, t := range jobcodec.DecodeTags(string(cb.Tags)) { + if len(t) <= jobtags.MaxBytes { + tags = append(tags, t) + } + } + _, err = x.insertJob(ctx, jobInsert{ + kind: cb.Kind, queue: queue, priority: clamp(cb.Priority, 2, 1, 4), maxAttempts: clamp(cb.MaxAttempts, 25, 1, 32767), + args: args, metadata: string(metadata), tags: tags, + }) + if err != nil { + return err + } + return x.notify(ctx, driver.ChannelInsert, queue) +} diff --git a/driver/hoppersqlite/flow.go b/driver/hoppersqlite/flow.go new file mode 100644 index 0000000..dac369b --- /dev/null +++ b/driver/hoppersqlite/flow.go @@ -0,0 +1,203 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "math" + "time" + + "github.com/parallelworks/hopper/driver" +) + +// claimLimited claims within a queue's limits: the budget is the smallest +// of the claim's limit, what the global limit leaves, and the whole tokens +// in the rate bucket. The write lock serializes every client's claims on +// the queue, so the limits hold exactly. +func (e *executor) claimLimited(ctx context.Context, params driver.JobClaimParams) (driver.JobClaimResult, error) { + var res driver.JobClaimResult + err := e.write(ctx, func(x *w) error { + var ( + global, burst, partition sql.NullInt64 + rate, tokens sql.NullFloat64 + refilled micros + running int + ) + err := x.QueryRowContext(ctx, `SELECT global_limit, rate_per_sec, rate_burst, tokens, refilled_at, partition_limit + FROM hopper_queues WHERE name = ?`, params.Queue).Scan(&global, &rate, &burst, &tokens, &refilled, &partition) + if errors.Is(err, sql.ErrNoRows) { + // No row, no limits. + res.Jobs, err = x.claim(ctx, params.Queue, params.ClientID, params.Limit, 0) + return err + } + if err != nil { + return fmt.Errorf("hopper: read queue limits: %w", err) + } + if err := x.QueryRowContext(ctx, `SELECT count(*) FROM hopper_jobs WHERE queue = ? AND state = 'running'`, params.Queue).Scan(&running); err != nil { + return fmt.Errorf("hopper: count running: %w", err) + } + budget := params.Limit + if global.Valid { + budget = min(budget, int(global.Int64)-running) + } + var available float64 + if rate.Valid && rate.Float64 > 0 { + capacity := float64(burst.Int64) + if !burst.Valid || capacity <= 0 { + capacity = math.Ceil(rate.Float64) + } + available = capacity + if tokens.Valid { + elapsed := 0.0 + if refilled.Valid { + elapsed = x.now.Sub(refilled.time()).Seconds() + } + available = min(capacity, tokens.Float64+rate.Float64*elapsed) + } + budget = min(budget, int(math.Floor(available))) + if budget <= 0 { + res.Wait = time.Duration((1 - available) / rate.Float64 * float64(time.Second)) + } + } + if budget <= 0 { + return nil + } + if res.Jobs, err = x.claim(ctx, params.Queue, params.ClientID, budget, partition.Int64); err != nil { + return err + } + if rate.Valid && rate.Float64 > 0 { + if _, err := x.ExecContext(ctx, `UPDATE hopper_queues SET tokens = ?, refilled_at = ? WHERE name = ?`, + available-float64(len(res.Jobs)), x.us(), params.Queue); err != nil { + return fmt.Errorf("hopper: update rate bucket: %w", err) + } + } + return nil + }) + if err != nil { + return driver.JobClaimResult{}, err + } + return res, nil +} + +// callbackJSON encodes a batch callback for the batch row, refusing values +// a job could not be inserted with, which would fail the transaction that +// completes the batch. +func callbackJSON(p *driver.JobInsertParams) (any, error) { + if p == nil { + return nil, nil + } + if p.Priority < 1 || p.Priority > 4 { + return nil, fmt.Errorf("hopper: callback priority %d is out of range 1..4", p.Priority) + } + if _, err := smallint(p.MaxAttempts, "callback max_attempts"); err != nil { + return nil, err + } + args, err := jsonValue(p.Args, "{}", "callback args") + if err != nil { + return nil, err + } + metadata, err := jsonValue(p.Metadata, "{}", "callback metadata") + if err != nil { + return nil, err + } + cb := map[string]any{ + "kind": p.Kind, "queue": p.Queue, "priority": p.Priority, "max_attempts": p.MaxAttempts, + "args": json.RawMessage(args), "metadata": json.RawMessage(metadata), + } + if len(p.Tags) > 0 { + cb["tags"] = p.Tags + } + b, err := json.Marshal(cb) + if err != nil { + return nil, err + } + return string(b), nil +} + +// batchRow is a batch's callbacks and metadata, encoded for its row. +type batchRow struct { + onSuccess, onFailure, onComplete any + metadata string +} + +func encodeBatch(onSuccess, onFailure, onComplete *driver.JobInsertParams, metadata []byte) (batchRow, error) { + var b batchRow + var err error + if b.onSuccess, err = callbackJSON(onSuccess); err != nil { + return b, err + } + if b.onFailure, err = callbackJSON(onFailure); err != nil { + return b, err + } + if b.onComplete, err = callbackJSON(onComplete); err != nil { + return b, err + } + b.metadata, err = jsonValue(metadata, "{}", "batch metadata") + return b, err +} + +// insertBatch inserts a batch row expecting total jobs. +func (x *w) insertBatch(ctx context.Context, id driver.JobID, name string, total int, b batchRow, edges any) error { + _, err := x.ExecContext(ctx, `INSERT INTO hopper_batches (id, name, pending, total, on_success, on_failure, on_complete, metadata, edges, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + idParam(id), nullable(name), total, total, b.onSuccess, b.onFailure, b.onComplete, b.metadata, edges, x.us()) + return err +} + +// BatchInsert implements driver.Executor. +func (e *executor) BatchInsert(ctx context.Context, params driver.BatchInsertParams) (driver.JobID, error) { + b, err := encodeBatch(params.OnSuccess, params.OnFailure, params.OnComplete, params.Metadata) + if err != nil { + return driver.JobID{}, err + } + var id driver.JobID + err = e.write(ctx, func(x *w) error { + id = newID(x.now) + return x.insertBatch(ctx, id, "", params.Total, b, nil) + }) + if err != nil { + return driver.JobID{}, fmt.Errorf("hopper: insert batch: %w", err) + } + return id, nil +} + +// BatchSettle implements driver.Executor. The transaction that finalizes a +// batch's jobs counts them down on its row and completes it, so there is no +// progress to fold, and no batch a race left open. +func (e *executor) BatchSettle(context.Context, driver.BatchSettleParams) (driver.BatchSettleResult, error) { + return driver.BatchSettleResult{}, nil +} + +// batchColumns read a batch's progress as BatchRow has it. +const batchColumns = `id, CASE WHEN completed_at IS NULL THEN max(pending, 0) ELSE 0 END, failed, total, created_at, completed_at, metadata` + +func scanBatch(row scanner, extra ...any) (*driver.BatchRow, error) { + var ( + b driver.BatchRow + id []byte + created, completed micros + metadata string + ) + if err := row.Scan(append([]any{&id, &b.Pending, &b.Failed, &b.Total, &created, &completed, &metadata}, extra...)...); err != nil { + return nil, err + } + if err := b.ID.Scan(id); err != nil { + return nil, err + } + b.CreatedAt, b.CompletedAt, b.Metadata = created.time(), completed.time(), json.RawMessage(metadata) + return &b, nil +} + +// BatchGet implements driver.Executor. +func (e *executor) BatchGet(ctx context.Context, id driver.JobID) (*driver.BatchRow, error) { + b, err := scanBatch(e.read().QueryRowContext(ctx, `SELECT `+batchColumns+` FROM hopper_batches WHERE id = ?`, idParam(id))) + if errors.Is(err, sql.ErrNoRows) { + return nil, driver.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("hopper: get batch: %w", err) + } + return b, nil +} diff --git a/driver/hoppersqlite/funcs.go b/driver/hoppersqlite/funcs.go new file mode 100644 index 0000000..da81b09 --- /dev/null +++ b/driver/hoppersqlite/funcs.go @@ -0,0 +1,347 @@ +package hoppersqlite + +import ( + "bytes" + sqldriver "database/sql/driver" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + "sync" + "unicode/utf16" + "unicode/utf8" + + "modernc.org/sqlite" +) + +// sqliteDriver is modernc's driver with hopper's SQL functions, which +// only this package's connections have: registered on the package-level +// driver, they would reach every connection the process opens. +var sqliteDriver = func() *sqlite.Driver { + d := &sqlite.Driver{} + d.MustRegisterDeterministicScalarFunction("hopper_topic_matches", 2, topicMatchesSQL) + d.MustRegisterDeterministicScalarFunction("hopper_json_contains", 2, jsonContainsSQL) + return d +}() + +// textArg reads a TEXT (or BLOB) argument of a SQL function, and reports +// false for NULL. +func textArg(v sqldriver.Value) (string, bool) { + switch v := v.(type) { + case string: + return v, true + case []byte: + return string(v), true + } + return "", false +} + +// topicMatchesSQL is hopper_topic_matches(pattern, topic): whether an AMQP +// topic pattern matches a topic, NULL if either is. +func topicMatchesSQL(_ *sqlite.FunctionContext, args []sqldriver.Value) (sqldriver.Value, error) { + pattern, ok := textArg(args[0]) + topic, ok2 := textArg(args[1]) + if !ok || !ok2 { + return nil, nil + } + re, err := topicRegexp(pattern) + if err != nil { + return nil, err + } + return re.MatchString(topic), nil +} + +// topicPatterns caches compiled patterns, which subscriptions, consumers +// and stream reads repeat. It is emptied once it holds maxTopicPatterns, so +// that reads with ever new patterns cannot grow it without bound. +var topicPatterns = struct { + mu sync.Mutex + m map[string]*regexp.Regexp +}{m: map[string]*regexp.Regexp{}} + +const maxTopicPatterns = 1024 + +// topicSpecial are the characters Postgres's hopper_topic_regex escapes +// before it turns the wildcards into regular expressions. +var topicSpecial = regexp.MustCompile(`([.+?^${}()|\[\]\\])`) + +// topicRegexp builds the regular expression an AMQP pattern means, as +// Postgres's hopper_topic_regex does (schema v2): words are separated by +// dots, '*' matches one word and '#' zero or more. The steps are the same, +// in the same order, so that a pattern, however odd, matches the same +// topics on either engine; (?s) gives '.' Postgres's meaning, which takes a +// newline too. +func topicRegexp(pattern string) (*regexp.Regexp, error) { + topicPatterns.mu.Lock() + defer topicPatterns.mu.Unlock() + if re, ok := topicPatterns.m[pattern]; ok { + return re, nil + } + expr := topicSpecial.ReplaceAllString(pattern, `\$1`) + for _, r := range [][2]string{ + {`*`, `[^.]+`}, // one word (before '#', whose rules introduce '.*') + {`\.#\.`, `(\..*)?\.`}, // a.#.b: zero or more words between + {`\.#`, `(\..*)?`}, // a.#: the word and anything under it + {`#\.`, `(.*\.)?`}, // #.b: anything above the word + {`#`, `.*`}, // #: everything + } { + expr = strings.ReplaceAll(expr, r[0], r[1]) + } + re, err := regexp.Compile(`(?s)^` + expr + `$`) + if err != nil { + return nil, fmt.Errorf("hoppersqlite: topic pattern %q: %w", pattern, err) + } + if len(topicPatterns.m) >= maxTopicPatterns { + clear(topicPatterns.m) + } + topicPatterns.m[pattern] = re + return re, nil +} + +// jsonContainsSQL is hopper_json_contains(doc, filter): whether the JSON +// document doc contains filter as Postgres's jsonb @> has it, NULL if +// either is. A document that is not JSON contains nothing. +func jsonContainsSQL(_ *sqlite.FunctionContext, args []sqldriver.Value) (sqldriver.Value, error) { + doc, ok := textArg(args[0]) + filter, ok2 := textArg(args[1]) + if !ok || !ok2 { + return nil, nil + } + want, err := decodeJSON(filter) + if err != nil { + return nil, fmt.Errorf("hoppersqlite: metadata filter: %w", err) + } + if have, err := decodeJSON(doc); err == nil { + return jsonContains(have, want, true), nil + } + return false, nil +} + +// decodeJSON decodes a JSON value, keeping numbers as their text. +func decodeJSON(s string) (any, error) { + dec := json.NewDecoder(strings.NewReader(s)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, err + } + if dec.More() { + return nil, fmt.Errorf("text after the JSON value") + } + return v, nil +} + +// jsonContains reports whether have contains want as jsonb's @> has it: an +// object contains another whose every key it has with a value that +// contains the other's; an array contains another each of whose elements +// some element of it contains, whatever their order and repeats; scalars +// contain the equal scalar, numbers by value. At the top level, an array +// also contains a scalar it has as an element. +func jsonContains(have, want any, top bool) bool { + switch w := want.(type) { + case map[string]any: + h, ok := have.(map[string]any) + if !ok { + return false + } + for k, wv := range w { + hv, ok := h[k] + if !ok || !jsonContains(hv, wv, false) { + return false + } + } + return true + case []any: + h, ok := have.([]any) + if !ok { + return false + } + for _, wv := range w { + found := false + for _, hv := range h { + if jsonContains(hv, wv, false) { + found = true + break + } + } + if !found { + return false + } + } + return true + } + if h, ok := have.([]any); ok && top { + for _, hv := range h { + if jsonScalarEqual(hv, want) { + return true + } + } + return false + } + return jsonScalarEqual(have, want) +} + +func jsonScalarEqual(a, b any) bool { + switch a := a.(type) { + case json.Number: + b, ok := b.(json.Number) + if !ok { + return false + } + ca, err := canonicalNumber(string(a)) + cb, err2 := canonicalNumber(string(b)) + return err == nil && err2 == nil && ca == cb + case string: + b, ok := b.(string) + return ok && a == b + case bool: + b, ok := b.(bool) + return ok && a == b + case nil: + return b == nil + } + return false +} + +// decimal is a JSON number as digits and a power of ten: the number is +// digits × 10^exp, with no leading or trailing zero in digits, which is +// empty for zero. +type decimal struct { + negative bool + digits string + exp int64 +} + +func (d decimal) String() string { + if d.digits == "" { + return "0" + } + sign := "" + if d.negative { + sign = "-" + } + return sign + d.digits + "e" + strconv.FormatInt(d.exp, 10) +} + +// parseDecimal parses a JSON number. Its exponent may be as large as an +// int64 holds, beyond which it is refused. +func parseDecimal(s string) (decimal, error) { + var d decimal + if strings.HasPrefix(s, "-") { + d.negative, s = true, s[1:] + } + mantissa, exponent, hasExp := strings.Cut(strings.ToLower(s), "e") + if hasExp { + e, err := strconv.ParseInt(strings.TrimPrefix(exponent, "+"), 10, 64) + if err != nil { + return decimal{}, fmt.Errorf("number %s: exponent out of range", s) + } + d.exp = e + } + whole, frac, _ := strings.Cut(mantissa, ".") + digits := whole + frac + if d.exp-int64(len(frac)) > d.exp { + return decimal{}, fmt.Errorf("number %s: exponent out of range", s) + } + d.exp -= int64(len(frac)) + digits = strings.TrimLeft(digits, "0") + trimmed := strings.TrimRight(digits, "0") + d.exp += int64(len(digits) - len(trimmed)) + d.digits = trimmed + if d.digits == "" { + d.negative, d.exp = false, 0 + } + return d, nil +} + +// canonicalNumber is a number's text that equal numbers share. +func canonicalNumber(s string) (string, error) { + d, err := parseDecimal(s) + if err != nil { + return "", err + } + return d.String(), nil +} + +// Postgres's numeric, which jsonb keeps numbers in, holds up to 131072 +// digits before the decimal point and 16383 after it. +const ( + maxNumericWhole = 131072 + maxNumericFraction = 16383 +) + +// checkJSONFilter refuses a metadata filter Postgres's jsonb input would, +// so that a listing's filter means the same on either engine: text that is +// not UTF-8, a \u0000 escape, an unpaired surrogate escape, or a number +// beyond numeric's range. +func checkJSONFilter(s string) error { + if !utf8.ValidString(s) { + return fmt.Errorf("the metadata filter is not UTF-8") + } + if err := checkEscapes(s); err != nil { + return err + } + dec := json.NewDecoder(strings.NewReader(s)) + dec.UseNumber() + for { + tok, err := dec.Token() + if err != nil { + if err.Error() == "EOF" { + return nil + } + return fmt.Errorf("the metadata filter is not JSON: %w", err) + } + n, ok := tok.(json.Number) + if !ok { + continue + } + d, err := parseDecimal(string(n)) + if err != nil { + return fmt.Errorf("the metadata filter's %w", err) + } + if whole := int64(len(d.digits)) + d.exp; d.digits != "" && whole > maxNumericWhole { + return fmt.Errorf("the metadata filter's number %.20s... is beyond the range of Postgres's numeric", n) + } + if d.exp < -maxNumericFraction { + return fmt.Errorf("the metadata filter's number %.20s... has more decimals than Postgres's numeric holds", n) + } + } +} + +// checkEscapes refuses a \u0000 escape and an unpaired surrogate escape in +// a JSON text's strings, which decoding would turn into a NUL and U+FFFD. +func checkEscapes(s string) error { + b := []byte(s) + for i := 0; i < len(b); i++ { + if b[i] != '\\' { + continue + } + if i+1 < len(b) && b[i+1] != 'u' { + i++ + continue + } + if i+6 > len(b) { + return nil // the decoder reports the truncated escape + } + r, err := strconv.ParseUint(string(b[i+2:i+6]), 16, 16) + switch { + case err != nil: + // The decoder reports the malformed escape. + case r == 0: + return fmt.Errorf("the metadata filter has a \\u0000 escape") + case utf16.IsSurrogate(rune(r)): + if r >= 0xdc00 || i+12 > len(b) || !bytes.HasPrefix(b[i+6:], []byte(`\u`)) { + return fmt.Errorf("the metadata filter has an unpaired surrogate escape") + } + low, err := strconv.ParseUint(string(b[i+8:i+12]), 16, 16) + if err != nil || low < 0xdc00 || low > 0xdfff { + return fmt.Errorf("the metadata filter has an unpaired surrogate escape") + } + i += 11 + default: + i += 5 + } + } + return nil +} diff --git a/driver/hoppersqlite/go.mod b/driver/hoppersqlite/go.mod new file mode 100644 index 0000000..8a0d0e3 --- /dev/null +++ b/driver/hoppersqlite/go.mod @@ -0,0 +1,28 @@ +module github.com/parallelworks/hopper/driver/hoppersqlite + +go 1.27.0 + +toolchain go1.27.3 + +// hopper does no cryptography of its own, and neither does SQLite. Running +// tests in fips140=only mode proves that nothing non-approved sneaks in. +godebug fips140=only + +require ( + github.com/parallelworks/hopper v0.7.0 // x-release-please-version + golang.org/x/sys v0.48.0 + modernc.org/sqlite v1.60.1 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + modernc.org/libc v1.77.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect +) + +replace github.com/parallelworks/hopper => ../../ diff --git a/driver/hoppersqlite/go.sum b/driver/hoppersqlite/go.sum new file mode 100644 index 0000000..8b50aa6 --- /dev/null +++ b/driver/hoppersqlite/go.sum @@ -0,0 +1,50 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= +golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= +modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg= +modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE= +modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs= +modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw= +modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/driver/hoppersqlite/hoppersqlite.go b/driver/hoppersqlite/hoppersqlite.go new file mode 100644 index 0000000..b0f53d5 --- /dev/null +++ b/driver/hoppersqlite/hoppersqlite.go @@ -0,0 +1,316 @@ +// Package hoppersqlite is hopper's SQLite driver, for an application that +// runs on one machine and wants hopper without a database server: a +// command-line tool that runs a workflow, a single-node service, local +// development. It is pure Go (modernc.org/sqlite, SQLite transpiled to Go), +// so a program that uses it builds with CGO_ENABLED=0. +// +// d, err := hoppersqlite.Open("/var/lib/app/hopper.db", nil) +// if _, err := hoppermigrate.Up(ctx, d, nil); err != nil { ... } +// client, err := hopper.NewClient(d, cfg) +// +// It passes the drivertest conformance suite, as the Postgres drivers do, +// and several processes may share one database. What SQLite's design makes +// different from Postgres: +// +// - One transaction writes at a time (driver.Capabilities.SingleWriter). +// A transaction that writes holds the database's write lock until it +// ends, and every other write waits for it: within this process in +// turn, for as long as its context allows, and across processes +// through SQLite's busy handler. A claim is exclusive without SKIP +// LOCKED, and the transaction that finalizes a job settles its batch +// and promotes its dependents itself. A finalize with SkipLocked, or a +// progress write with NoWait, waits briefly (lockWaitBrief) for a write +// lock held elsewhere, and then skips every job, or returns +// driver.ErrJobLocked. +// - Transactions come from DB, whose connections take the write lock +// when a transaction runs its first statement, not when it begins, so +// that what it reads is never stale by the time it writes (see +// conn.go). A transaction begun read-only takes no lock and may not +// write. Within a transaction, write through it (InsertTx and the +// like), not through the client: a write on the pool would wait for +// the transaction's own lock. +// - The database must be on a local filesystem: Open refuses one on a +// network filesystem with ErrNetworkFilesystem. Its clock is therefore +// the host's, which every process that opens it shares, and the driver +// reads it as database time. +// - Notifications are rows of a table, delivered on commit to listeners +// in every process that opens the database: at once in this process, +// and within ListenInterval in others. +// - History is one table, and retention deletes its rows in batches; a +// stream's events commit in the order they are appended, so a +// position is an event's sequence number (Xid is always 0). +// - Args, metadata and other JSON values are stored as given, not +// normalized as Postgres's jsonb normalizes them. A metadata filter +// matches as jsonb's @> does, and is refused for what jsonb refuses. +// - There is no COPY: bulk inserts go through the one insert path. +package hoppersqlite + +import ( + "context" + "database/sql" + "embed" + "errors" + "fmt" + "io/fs" + "net/url" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/parallelworks/hopper/driver" +) + +// Config tunes the SQLite driver. A nil *Config means the defaults. +type Config struct { + // Synchronous is SQLite's synchronous setting for the database's + // connections: "FULL" (the default), which makes every commit durable + // before it returns, as Postgres's default does, or "NORMAL", with which + // a power failure, not a crash of the process, can lose the last commits + // before a checkpoint, and commits take less time. + Synchronous string + // ListenInterval is how often a listener looks for notifications + // committed by other processes. Those of this process wake it at once. + // Zero means 50ms. + ListenInterval time.Duration +} + +// DefaultListenInterval is ListenInterval's default. +const DefaultListenInterval = 50 * time.Millisecond + +// lockWaitBrief is how long a finalize with SkipLocked, a progress write +// with NoWait, and the leader's maintenance wait for a write lock another +// transaction holds before they give up: long enough for a statement of +// another client to end, short enough not to hold the caller up behind a +// transaction that stays open. +const lockWaitBrief = 100 * time.Millisecond + +// Driver implements driver.Driver[*sql.Tx] on a SQLite database. +type Driver struct { + sqlDB *sql.DB + db *database + listenInterval time.Duration +} + +var _ driver.Driver[*sql.Tx] = (*Driver)(nil) + +// Open opens the SQLite database at path, creating it if needed, in +// write-ahead log mode. It returns ErrNetworkFilesystem for a path on a +// network filesystem. Close the Driver to close the database. +func Open(path string, cfg *Config) (*Driver, error) { + var c Config + if cfg != nil { + c = *cfg + } + sync := strings.ToUpper(c.Synchronous) + switch sync { + case "": + sync = "FULL" + case "FULL", "NORMAL": + default: + return nil, fmt.Errorf("hoppersqlite: Synchronous is %q, not FULL or NORMAL", c.Synchronous) + } + if c.ListenInterval < 0 { + return nil, errors.New("hoppersqlite: ListenInterval is negative") + } + if c.ListenInterval == 0 { + c.ListenInterval = DefaultListenInterval + } + if path == "" || path == ":memory:" || strings.HasPrefix(path, "file:") { + return nil, fmt.Errorf("hoppersqlite: %q is not a file path; the database must be a file, which every connection and process opens", path) + } + abs, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("hoppersqlite: %w", err) + } + if err := checkLocal(abs); err != nil { + return nil, err + } + db := openDatabase(abs) + sqlDB := sql.OpenDB(&connector{db: db, dsn: dsn(abs), synchronous: sync}) + sqlDB.SetMaxIdleConns(16) + d := &Driver{sqlDB: sqlDB, db: db, listenInterval: c.ListenInterval} + var mode string + if err := sqlDB.QueryRowContext(context.Background(), "PRAGMA journal_mode = WAL").Scan(&mode); err != nil { + _ = d.Close() + return nil, fmt.Errorf("hoppersqlite: open %s: %w", abs, err) + } + if !strings.EqualFold(mode, "wal") { + _ = d.Close() + return nil, fmt.Errorf("hoppersqlite: open %s: the database is in journal mode %s and cannot use a write-ahead log", abs, mode) + } + return d, nil +} + +// dsn is the URI of the database file at an absolute path, for modernc's +// driver, which takes its query parameters out of a plain file name. +func dsn(abs string) string { + p := filepath.ToSlash(abs) + if !strings.HasPrefix(p, "/") { + p = "/" + p // a Windows drive letter + } + return (&url.URL{Scheme: "file", OmitHost: true, Path: p}).String() +} + +// DB returns the database/sql handle the driver uses. Begin the +// transactions you pass to InsertTx and the like on it: its connections +// take the database's write lock as a transaction's first statement runs. +func (d *Driver) DB() *sql.DB { return d.sqlDB } + +// Close closes the database. +func (d *Driver) Close() error { + err := d.sqlDB.Close() + d.db.close() + return err +} + +// Executor implements driver.Driver. +func (d *Driver) Executor() driver.Executor { return &executor{d: d} } + +// UnwrapTx implements driver.Driver. The transaction must come from DB. +func (d *Driver) UnwrapTx(tx *sql.Tx) driver.Executor { return &executor{d: d, tx: tx} } + +// Capabilities implements driver.Driver. +func (d *Driver) Capabilities() driver.Capabilities { + return driver.Capabilities{Listen: true, SingleWriter: true} +} + +// Migrator implements driver.Driver. +func (d *Driver) Migrator() driver.Migrator { return migrator{d} } + +// Listener implements driver.Driver. +func (d *Driver) Listener(ctx context.Context) (driver.Listener, error) { return d.newListener(ctx) } + +//go:embed migrations/*.sql +var migrationFiles embed.FS + +var migrationName = regexp.MustCompile(`^(\d+)_[a-z0-9_]+\.(up|down)\.sql$`) + +// scripts are SQLite's migration scripts by version and direction. +var scripts = func() map[int]map[bool]string { + out := map[int]map[bool]string{} + entries, err := fs.ReadDir(migrationFiles, "migrations") + if err != nil { + panic(err) + } + for _, e := range entries { + m := migrationName.FindStringSubmatch(e.Name()) + if m == nil { + panic("hoppersqlite: malformed migration file name " + e.Name()) + } + version, _ := strconv.Atoi(m[1]) + body, err := fs.ReadFile(migrationFiles, "migrations/"+e.Name()) + if err != nil { + panic(err) + } + if out[version] == nil { + out[version] = map[bool]string{} + } + out[version][m[2] == "up"] = string(body) + } + return out +}() + +// migrator implements driver.Migrator. Its lock is SQLite's write lock, +// held from Lock to Close on a connection of its own: each version applies +// in a savepoint of that transaction, and Close commits the versions that +// applied. +type migrator struct{ d *Driver } + +func (m migrator) Lock(ctx context.Context) (driver.MigrationExecutor, error) { + conn, err := m.d.sqlDB.Conn(ctx) + if err != nil { + return nil, err + } + tx, err := conn.BeginTx(ctx, nil) + if err == nil { + _, err = tx.ExecContext(ctx, "SELECT 1") // takes the write lock + } + if err != nil { + if tx != nil { + _ = tx.Rollback() + } + _ = conn.Close() + return nil, err + } + return &migration{conn: conn, tx: tx}, nil +} + +// migration implements driver.MigrationExecutor and driver.MigrationScripts. +type migration struct { + conn *sql.Conn + tx *sql.Tx +} + +var _ driver.MigrationScripts = (*migration)(nil) + +func (m *migration) Versions(ctx context.Context) ([]int, error) { return schemaVersions(ctx, m.tx) } + +// Script implements driver.MigrationScripts. +func (m *migration) Script(version int, up bool) (string, error) { + script, ok := scripts[version][up] + if !ok { + return "", fmt.Errorf("hoppersqlite has no script for schema version %d", version) + } + return script, nil +} + +func (m *migration) Apply(ctx context.Context, version int, script string, up bool) (err error) { + if _, err := m.tx.ExecContext(ctx, "SAVEPOINT migration"); err != nil { + return err + } + defer func() { + if err != nil { + _, rerr := m.tx.ExecContext(context.WithoutCancel(ctx), "ROLLBACK TO migration; RELEASE migration") + err = errors.Join(err, rerr) + } + }() + if !up { + // The script of version 1 drops hopper_schema with the rest. + if _, err := m.tx.ExecContext(ctx, "DELETE FROM hopper_schema WHERE version = ?", version); err != nil { + return err + } + } + if _, err := m.tx.ExecContext(ctx, script); err != nil { + return err + } + if up { + if _, err := m.tx.ExecContext(ctx, "INSERT INTO hopper_schema (version, applied_at) VALUES (?, ?)", version, time.Now().UnixMicro()); err != nil { + return err + } + } + _, err = m.tx.ExecContext(ctx, "RELEASE migration") + return err +} + +func (m *migration) Close(context.Context) error { + err := m.tx.Commit() + return errors.Join(err, m.conn.Close()) +} + +// schemaVersions returns the applied schema versions in ascending order, or +// nil if the schema has never been installed. +func schemaVersions(ctx context.Context, q querier) ([]int, error) { + var exists bool + if err := q.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'hopper_schema')").Scan(&exists); err != nil { + return nil, err + } + if !exists { + return nil, nil + } + rows, err := q.QueryContext(ctx, "SELECT version FROM hopper_schema ORDER BY version") + if err != nil { + return nil, err + } + defer rows.Close() + var versions []int + for rows.Next() { + var v int + if err := rows.Scan(&v); err != nil { + return nil, err + } + versions = append(versions, v) + } + return versions, rows.Err() +} diff --git a/driver/hoppersqlite/id.go b/driver/hoppersqlite/id.go new file mode 100644 index 0000000..7261ce8 --- /dev/null +++ b/driver/hoppersqlite/id.go @@ -0,0 +1,26 @@ +package hoppersqlite + +import ( + "crypto/rand" + "encoding/binary" + "time" + + "github.com/parallelworks/hopper/driver" +) + +// newID returns a UUIDv7 for a job, batch or message: 48 bits of Unix +// milliseconds at now, 12 bits of the millisecond's fraction, as Postgres +// 18's uuidv7() fills them, so that IDs minted in one process in one +// millisecond mostly sort in order, and 62 random bits. SQLite has no +// generator of its own, so the driver mints IDs where Postgres's does in the +// database. crypto/rand reads the operating system's generator through Go's +// FIPS 140-3 DRBG, as fips140=only requires. +func newID(now time.Time) driver.JobID { + var id driver.JobID + ms := uint64(max(now.UnixMilli(), 0)) + fraction := uint64(now.Nanosecond()%int(time.Millisecond)) * 4096 / uint64(time.Millisecond) //nolint:gosec // Nanosecond is in [0, 1e9) + binary.BigEndian.PutUint64(id[0:8], ms<<16|0x7000|fraction) + _, _ = rand.Read(id[8:]) // never fails (crypto/rand.Read) + id[8] = id[8]&0x3f | 0x80 + return id +} diff --git a/driver/hoppersqlite/jobs.go b/driver/hoppersqlite/jobs.go new file mode 100644 index 0000000..f96d74c --- /dev/null +++ b/driver/hoppersqlite/jobs.go @@ -0,0 +1,856 @@ +package hoppersqlite + +import ( + "bytes" + "cmp" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "math" + "slices" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/internal/jobcodec" + "github.com/parallelworks/hopper/internal/jobtags" +) + +func bytesReader(s string) *strings.Reader { return strings.NewReader(s) } + +// jobInsert is a job to insert, its values encoded. +type jobInsert struct { + kind, queue string + priority, maxAttempts int + scheduledAt time.Time + args, metadata string + tags []string + uniqueKey string + uniqueFinished []driver.JobState + orderingKey, partitionKey string + batchID driver.JobID + ttl time.Duration + await bool + // pending starts the job waiting on workflow dependencies, and id + // gives it its ID; a zero id has one minted. + pending bool + id driver.JobID +} + +func encodeInsert(p driver.JobInsertParams) (jobInsert, error) { + priority, err := smallint(p.Priority, "priority") + if err != nil { + return jobInsert{}, err + } + maxAttempts, err := smallint(p.MaxAttempts, "max_attempts") + if err != nil { + return jobInsert{}, err + } + args, err := jsonValue(p.Args, "{}", "args") + if err != nil { + return jobInsert{}, err + } + metadata, err := jsonValue(p.Metadata, "{}", "metadata") + if err != nil { + return jobInsert{}, err + } + return jobInsert{ + kind: p.Kind, queue: p.Queue, priority: priority, maxAttempts: maxAttempts, scheduledAt: p.ScheduledAt, + args: args, metadata: metadata, tags: p.Tags, uniqueKey: p.UniqueKey, uniqueFinished: p.UniqueFinished, + orderingKey: p.OrderingKey, partitionKey: p.PartitionKey, batchID: p.BatchID, ttl: p.TTL, await: p.Await, + }, nil +} + +// insertJobSQL inserts a job and returns it as stored. +const insertJobSQL = ` +INSERT INTO hopper_jobs (id, kind, queue, state, priority, max_attempts, scheduled_at, args, metadata, unique_key, + expires_at, await, ordering_key, partition_key, batch_id, tags, created_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +RETURNING ` + jobColumns + +// insertJob inserts a job: pending, or scheduled if its time is ahead, or +// available. +func (x *w) insertJob(ctx context.Context, j jobInsert) (*driver.JobRow, error) { + scheduledAt := j.scheduledAt + if scheduledAt.IsZero() { + scheduledAt = x.now + } + state := driver.JobStateAvailable + switch { + case j.pending: + state = driver.JobStatePending + case scheduledAt.After(x.now): + state = driver.JobStateScheduled + } + var expires any + if j.ttl > 0 { + expires = x.at(j.ttl) + } + id := j.id + if id.IsZero() { + id = newID(x.now) + } + return scanJob(x.QueryRowContext(ctx, insertJobSQL, + idParam(id), j.kind, j.queue, string(state), j.priority, j.maxAttempts, scheduledAt.UnixMicro(), j.args, j.metadata, + nullable(j.uniqueKey), expires, j.await, nullable(j.orderingKey), nullable(j.partitionKey), nullableID(j.batchID), + tagsJSON(j.tags), x.us()), false) +} + +// JobInsertMany implements driver.Executor. An input with a unique key is +// first looked for among live jobs, which with ConflictReplace it replaces +// unless running, and then, with UniqueFinished, in history: a job found +// in either is the input's duplicate, and nothing is inserted for it. The +// write lock keeps both looks and the insert free of any other writer. +func (e *executor) JobInsertMany(ctx context.Context, params []driver.JobInsertParams, opts driver.JobInsertOpts) ([]driver.JobInsertResult, error) { + if len(params) == 0 { + return nil, nil + } + jobs := make([]jobInsert, len(params)) + for i, p := range params { + var err error + if jobs[i], err = encodeInsert(p); err != nil { + return nil, err + } + } + var results []driver.JobInsertResult + err := e.write(ctx, func(x *w) error { + var err error + if results, err = x.insertJobs(ctx, jobs, opts.OnConflict); err != nil { + return err + } + if opts.Notify { + return x.notify(ctx, driver.ChannelInsert, distinctQueues(jobs)...) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("hopper: insert jobs: %w", err) + } + return results, nil +} + +func distinctQueues(jobs []jobInsert) []string { + var out []string + for _, j := range jobs { + if !slices.Contains(out, j.queue) { + out = append(out, j.queue) + } + } + return out +} + +// replaceSQL replaces a waiting job's args, metadata, tags, priority, max +// attempts and schedule, as a replacing insert does, and clears its +// progress and snooze reason. +const replaceSQL = ` +UPDATE hopper_jobs SET args = ?, metadata = ?, tags = ?, priority = ?, max_attempts = ?, scheduled_at = ?, state = ?, + progress = NULL, snooze_reason = NULL +WHERE id = ? +RETURNING ` + jobColumns + +func (x *w) insertJobs(ctx context.Context, jobs []jobInsert, onConflict driver.ConflictAction) ([]driver.JobInsertResult, error) { + results := make([]driver.JobInsertResult, len(jobs)) + for i, j := range jobs { + if j.uniqueKey != "" { + dup, err := scanJob(x.QueryRowContext(ctx, "SELECT "+jobColumns+" FROM hopper_jobs WHERE kind = ? AND unique_key = ?", j.kind, j.uniqueKey), false) + switch { + case errors.Is(err, sql.ErrNoRows): + case err != nil: + return nil, err + default: + if onConflict == driver.ConflictReplace && dup.State != driver.JobStateRunning { + scheduledAt := cmp.Or(j.scheduledAt, x.now) + state := driver.JobStateAvailable + if scheduledAt.After(x.now) { + state = driver.JobStateScheduled + } + if dup, err = scanJob(x.QueryRowContext(ctx, replaceSQL, j.args, j.metadata, tagsJSON(j.tags), j.priority, j.maxAttempts, + scheduledAt.UnixMicro(), string(state), idParam(dup.ID)), false); err != nil { + return nil, err + } + } + results[i] = driver.JobInsertResult{Job: dup, Duplicate: true} + continue + } + if len(j.uniqueFinished) > 0 { + found, err := x.finishedDuplicate(ctx, j) + if err != nil { + return nil, err + } + if found != nil { + results[i] = driver.JobInsertResult{Job: found, Duplicate: true} + // The job never joins its batch, which counts it as + // the outcome it was found in, as its finalize would. + if !j.batchID.IsZero() { + failed := 0 + if found.State != driver.JobStateCompleted { + failed = 1 + } + if err := x.countBatch(ctx, j.batchID, 1, failed); err != nil { + return nil, err + } + } + continue + } + } + } + job, err := x.insertJob(ctx, j) + if err != nil { + return nil, err + } + results[i] = driver.JobInsertResult{Job: job} + } + return results, nil +} + +// finishedDuplicate returns the latest job in history with an insert's +// kind and unique key in one of its UniqueFinished states, or nil. +func (x *w) finishedDuplicate(ctx context.Context, j jobInsert) (*driver.JobRow, error) { + args := []any{j.kind, j.uniqueKey} + for _, s := range j.uniqueFinished { + args = append(args, string(s)) + } + found, err := scanJob(x.QueryRowContext(ctx, "SELECT "+historyColumns+` FROM hopper_job_history + WHERE kind = ? AND unique_key = ? AND state IN (`+placeholders(len(j.uniqueFinished))+`) + ORDER BY finalized_at DESC LIMIT 1`, args...), true) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return found, err +} + +// JobInsertCopy implements driver.Executor: SQLite has no COPY. +func (e *executor) JobInsertCopy(context.Context, []driver.JobInsertParams, driver.JobInsertOpts) ([]driver.JobInsertResult, error) { + return nil, driver.ErrNotSupported +} + +// claimMax is the most jobs one claim returns, as on Postgres. +const claimMax = 512 + +// claimEligibleSQL is the WHERE clause of a claim, over an alias cand, with +// the time as ?1 and the queue as ?3. A job with an ordering key is +// claimable only if it is the oldest waiting job of its key and no job of +// that key is running. +const claimEligibleSQL = ` + cand.queue = ?3 AND cand.state IN ('available', 'scheduled', 'retryable') + AND cand.scheduled_at <= ?1 + AND (cand.expires_at IS NULL OR cand.expires_at > ?1) + AND (cand.ordering_key IS NULL OR ( + NOT EXISTS (SELECT 1 FROM hopper_jobs r + WHERE r.queue = cand.queue AND r.ordering_key = cand.ordering_key AND r.state = 'running') + AND cand.seq = (SELECT min(o.seq) FROM hopper_jobs o + WHERE o.queue = cand.queue AND o.ordering_key = cand.ordering_key + AND o.state IN ('available', 'scheduled', 'retryable'))))` + +// claimSetSQL moves claimed jobs to running, owned by the client ?2 at ?1, +// with their fences incremented. +const claimSetSQL = ` +UPDATE hopper_jobs SET state = 'running', attempt = attempt + 1, fence = fence + 1, attempted_at = ?1, attempted_by = ?2, + snooze_reason = NULL +WHERE seq IN (%s) +RETURNING ` + jobColumns + `, seq` + +// claimSQL claims up to ?4 jobs in claim order. +var claimSQL = fmt.Sprintf(claimSetSQL, `SELECT cand.seq FROM hopper_jobs cand WHERE `+claimEligibleSQL+` + ORDER BY cand.priority, cand.scheduled_at, cand.seq LIMIT ?4`) + +// claimPartitionedSQL claims up to ?4 jobs in claim order, admitting only +// those whose rank within their partition key, counting the key's running +// jobs, fits under the partition limit ?5. +var claimPartitionedSQL = fmt.Sprintf(claimSetSQL, `SELECT seq FROM ( + SELECT cand.seq, cand.priority, cand.scheduled_at, cand.partition_key, + row_number() OVER (PARTITION BY cand.partition_key ORDER BY cand.priority, cand.scheduled_at, cand.seq) + + (SELECT count(*) FROM hopper_jobs pr + WHERE pr.queue = cand.queue AND pr.partition_key = cand.partition_key AND pr.state = 'running') AS slot + FROM hopper_jobs cand WHERE `+claimEligibleSQL+` +) WHERE partition_key IS NULL OR slot <= ?5 +ORDER BY priority, scheduled_at, seq LIMIT ?4`) + +// claim claims up to limit jobs of a queue for a client, with a partition +// limit if partition is positive, and returns them in claim order. +func (x *w) claim(ctx context.Context, queue string, clientID int64, limit int, partition int64) ([]*driver.JobRow, error) { + limit = min(limit, claimMax) + if limit <= 0 { + return nil, nil + } + type claimed struct { + job *driver.JobRow + seq int64 + } + query, args := claimSQL, []any{x.us(), clientID, queue, limit} + if partition > 0 { + query, args = claimPartitionedSQL, append(args, partition) + } + rows, err := queryAll(ctx, x, func(r scanner) (claimed, error) { + var c claimed + var err error + c.job, err = scanJob(r, false, &c.seq) + return c, err + }, query, args...) + if err != nil { + return nil, fmt.Errorf("hopper: claim jobs: %w", err) + } + slices.SortFunc(rows, func(a, b claimed) int { + return cmp.Or(cmp.Compare(a.job.Priority, b.job.Priority), a.job.ScheduledAt.Compare(b.job.ScheduledAt), cmp.Compare(a.seq, b.seq)) + }) + jobs := make([]*driver.JobRow, len(rows)) + for i, r := range rows { + jobs[i] = r.job + } + return jobs, nil +} + +// JobClaim implements driver.Executor. The write lock makes the claim +// exclusive: no other transaction claims, or changes, a job between the +// statement's read of it and its update. +func (e *executor) JobClaim(ctx context.Context, params driver.JobClaimParams) (driver.JobClaimResult, error) { + if params.Limit <= 0 { + return driver.JobClaimResult{}, nil + } + if params.Limited { + return e.claimLimited(ctx, params) + } + var jobs []*driver.JobRow + err := e.write(ctx, func(x *w) error { + var err error + jobs, err = x.claim(ctx, params.Queue, params.ClientID, params.Limit, 0) + return err + }) + if err != nil { + return driver.JobClaimResult{}, err + } + return driver.JobClaimResult{Jobs: jobs}, nil +} + +// JobAge implements driver.Executor. Bumped jobs restart their wait, so a +// job climbs one level per period. +func (e *executor) JobAge(ctx context.Context, queue string, after time.Duration) (int64, error) { + if after <= 0 { + return 0, nil + } + var n int64 + err := e.write(ctx, func(x *w) error { + res, err := x.ExecContext(ctx, ` + UPDATE hopper_jobs SET priority = priority - 1, scheduled_at = ? + WHERE queue = ? AND state IN ('available', 'scheduled', 'retryable') AND priority > 1 AND scheduled_at <= ?`, + x.us(), queue, x.at(-after)) + if err != nil { + return err + } + n, err = res.RowsAffected() + return err + }) + if err != nil { + return 0, fmt.Errorf("hopper: age jobs: %w", err) + } + return n, nil +} + +// JobRecordProgress implements driver.Executor. With NoWait, a write lock +// another transaction holds is waited for only briefly, after which it +// returns ErrJobLocked. +func (e *executor) JobRecordProgress(ctx context.Context, params driver.JobRecordProgressParams) (*driver.Progress, error) { + var cursor json.RawMessage + if len(params.Cursor) > 0 { + c, err := jsonValue(params.Cursor, "", "the progress cursor") + if err != nil { + return nil, err + } + cursor = json.RawMessage(c) + } + var progress *driver.Progress + write := e.write + if params.NoWait { + write = e.writeBrief + } + err := write(ctx, func(x *w) error { + var stored sql.NullString + err := x.QueryRowContext(ctx, `SELECT progress FROM hopper_jobs + WHERE id = ? AND state = 'running' AND attempted_by = ? AND fence = ? AND attempt = ?`, + idParam(params.ID), params.AttemptedBy, params.Fence, params.Attempt).Scan(&stored) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return err + } + step := params.Step + if params.KeepStep { + step = "" + if p := jobcodec.DecodeProgress(stored.String); p != nil { + step = p.Step + } + } + b, err := json.Marshal(struct { + Step string `json:"step"` + At time.Time `json:"at"` + Cursor json.RawMessage `json:"cursor,omitempty"` + }{step, x.now, cursor}) + if err != nil { + return err + } + if _, err := x.ExecContext(ctx, "UPDATE hopper_jobs SET progress = ? WHERE id = ?", string(b), idParam(params.ID)); err != nil { + return err + } + progress = jobcodec.DecodeProgress(string(b)) + return nil + }) + if errors.Is(err, errLockBusy) { + return nil, driver.ErrJobLocked + } + if err != nil { + return nil, fmt.Errorf("hopper: record job progress: %w", err) + } + return progress, nil +} + +// JobGet implements driver.Executor. +func (e *executor) JobGet(ctx context.Context, id driver.JobID) (*driver.JobRow, error) { + job, err := scanJob(e.read().QueryRowContext(ctx, ` + SELECT `+liveAsHistory+` FROM hopper_jobs WHERE id = ?1 + UNION ALL + SELECT `+historyColumns+` FROM hopper_job_history WHERE id = ?1 + LIMIT 1`, idParam(id)), true) + if errors.Is(err, sql.ErrNoRows) { + return nil, driver.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("hopper: get job: %w", err) + } + return job, nil +} + +// JobLogs implements driver.Executor. +func (e *executor) JobLogs(ctx context.Context, id driver.JobID) ([]driver.AttemptLog, error) { + var logs sql.NullString + err := e.read().QueryRowContext(ctx, ` + SELECT logs FROM hopper_jobs WHERE id = ?1 + UNION ALL + SELECT logs FROM hopper_job_history WHERE id = ?1 + LIMIT 1`, idParam(id)).Scan(&logs) + if errors.Is(err, sql.ErrNoRows) { + return nil, driver.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("hopper: get job logs: %w", err) + } + return jobcodec.DecodeLogs(logs.String), nil +} + +// errOrderingKeyNeedsQueue refuses a key listing across queues: a key is a +// lane only within its queue. +var errOrderingKeyNeedsQueue = fmt.Errorf("%w: an ordering key is listed within a queue; set the queue", driver.ErrInvalidFilter) + +// checkFilterText refuses filter text Postgres would refuse: a NUL, or +// bytes that are not UTF-8. +func checkFilterText(texts ...string) error { + for _, s := range texts { + if !utf8.ValidString(s) || strings.ContainsRune(s, 0) { + return fmt.Errorf("%w: %q is not UTF-8 text without NUL", driver.ErrInvalidFilter, s) + } + } + return nil +} + +// JobList implements driver.Executor. Its filters are refused where +// Postgres's are, so that a listing means the same on either engine. +func (e *executor) JobList(ctx context.Context, params driver.JobListParams) ([]*driver.JobRow, error) { + if params.OrderingKey != "" && params.Queue == "" { + return nil, errOrderingKeyNeedsQueue + } + for _, p := range params.Priorities { + if _, err := smallint(p, "Priorities"); err != nil { + return nil, fmt.Errorf("%w: %w", driver.ErrInvalidFilter, err) + } + } + states := make([]string, len(params.States)) + for i, s := range params.States { + states[i] = string(s) + } + tagsAll, tagsAny := jobtags.Distinct(params.TagsAll), jobtags.Distinct(params.TagsAny) + for _, texts := range [][]string{{params.Queue, params.OrderingKey}, params.Kinds, states, tagsAll, tagsAny} { + if err := checkFilterText(texts...); err != nil { + return nil, err + } + } + if len(params.Metadata) > 0 { + if err := checkJSONFilter(string(params.Metadata)); err != nil { + return nil, fmt.Errorf("%w: %w", driver.ErrInvalidFilter, err) + } + } + live, history := len(params.States) == 0, len(params.States) == 0 + for _, s := range params.States { + if s.Terminal() { + history = true + } else { + live = true + } + } + + var ( + where []string + args []any + ) + in := func(column string, values []any) { + where = append(where, column+" IN ("+placeholders(len(values))+")") + args = append(args, values...) + } + anyOf := func(values []string) []any { + out := make([]any, len(values)) + for i, v := range values { + out[i] = v + } + return out + } + if params.Queue != "" { + where, args = append(where, "queue = ?"), append(args, params.Queue) + } + if len(params.Kinds) > 0 { + in("kind", anyOf(params.Kinds)) + } + if len(states) > 0 { + in("state", anyOf(states)) + } + order := "id" + switch { + case params.NewestFirst: + order = "id DESC" + if !params.After.IsZero() { + where, args = append(where, "id < ?"), append(args, idParam(params.After)) + } + default: + where, args = append(where, "id > ?"), append(args, idParam(params.After)) + } + if params.OrderingKey != "" { + where, args = append(where, "ordering_key = ?"), append(args, params.OrderingKey) + } + if len(params.IDs) > 0 { + ids := make([]any, len(params.IDs)) + for i, id := range params.IDs { + ids[i] = idParam(id) + } + in("id", ids) + } + if len(params.Priorities) > 0 { + ps := make([]any, len(params.Priorities)) + for i, p := range params.Priorities { + ps[i] = p + } + in("priority", ps) + } + if len(params.Metadata) > 0 { + where, args = append(where, "hopper_json_contains(metadata, ?)"), append(args, string(params.Metadata)) + } + // A listing by tags walks the tags table's rows of a tag, and checks + // the job's own tags, which those rows follow, for every tag asked for. + tagged := func(table string) (string, []any) { + var conds []string + var targs []any + walked := tagsAny + if len(tagsAll) > 0 { + walked = tagsAll[:1] + for _, t := range tagsAll { + conds = append(conds, "EXISTS (SELECT 1 FROM json_each(j.tags) WHERE type = 'text' AND value = ?)") + targs = append(targs, t) + } + } + if len(tagsAny) > 0 { + conds = append(conds, "EXISTS (SELECT 1 FROM json_each(j.tags) WHERE type = 'text' AND value IN ("+placeholders(len(tagsAny))+"))") + targs = append(targs, anyOf(tagsAny)...) + } + if len(walked) == 0 { + return "", nil + } + walk := "j.id IN (SELECT t.id FROM " + table + " t WHERE t.tag IN (" + placeholders(len(walked)) + "))" + return " AND " + strings.Join(append([]string{walk}, conds...), " AND "), append(anyOf(walked), targs...) + } + limit := " ORDER BY " + order + " LIMIT " + strconv.Itoa(max(params.Limit, 1)) + filter := " WHERE " + strings.Join(where, " AND ") + if len(where) == 0 { + filter = " WHERE true" + } + var ( + arms []string + armArgs []any + ) + if live { + tags, targs := tagged("hopper_job_tags") + arms = append(arms, "SELECT * FROM (SELECT "+liveAsHistory+" FROM hopper_jobs j"+filter+tags+limit+")") + armArgs = append(append(armArgs, args...), targs...) + } + if history { + tags, targs := tagged("hopper_job_history_tags") + arms = append(arms, "SELECT * FROM (SELECT "+historyColumns+" FROM hopper_job_history j"+filter+tags+limit+")") + armArgs = append(append(armArgs, args...), targs...) + } + query := strings.Join(arms, " UNION ALL ") + limit + jobs, err := queryAll(ctx, e.read(), scanHistoryJob, query, armArgs...) + if err != nil { + return nil, fmt.Errorf("hopper: list jobs: %w", err) + } + return jobs, nil +} + +// lockState returns a live job's row, or nil if it is not live. +func (x *w) lockState(ctx context.Context, id driver.JobID) (*liveRow, error) { + return x.liveRow(ctx, id) +} + +// cancelWaiting finalizes a waiting job as canceled, archived or not, with +// everything that entails for its batch and its dependents, and returns it +// as it is afterwards. +func (x *w) cancelWaiting(ctx context.Context, row *liveRow, archive bool) (*driver.JobRow, error) { + errs, err := hopperError(row, "hopper: canceled", x.now) + if err != nil { + return nil, err + } + f := finished{row: row, state: driver.JobStateCanceled, errors: errs, logs: row.logs, archive: archive} + if err := x.finish(ctx, f); err != nil { + return nil, err + } + if _, err := x.tail(ctx, []finished{f}); err != nil { + return nil, err + } + return f.jobRow(x.now) +} + +// JobCancel implements driver.Executor. +func (e *executor) JobCancel(ctx context.Context, id driver.JobID) (*driver.JobRow, error) { + var job *driver.JobRow + err := e.write(ctx, func(x *w) error { + row, err := x.lockState(ctx, id) + switch { + case err != nil: + return err + case row == nil: + // Not live: finalized already, or unknown. + job, err = (&executor{d: e.d, tx: txOf(x)}).JobGet(ctx, id) + return err + case row.job.State == driver.JobStateRunning: + if job, err = scanJob(x.QueryRowContext(ctx, `UPDATE hopper_jobs SET cancel_requested_at = coalesce(cancel_requested_at, ?) + WHERE id = ? RETURNING `+jobColumns, x.us(), idParam(id)), false); err != nil { + return err + } + return x.notify(ctx, driver.ChannelControl, "cancel:"+id.String()) + } + job, err = x.cancelWaiting(ctx, row, true) + return err + }) + if err != nil { + if errors.Is(err, driver.ErrNotFound) { + return nil, driver.ErrNotFound + } + return nil, fmt.Errorf("hopper: cancel job: %w", err) + } + return job, nil +} + +// txOf returns the transaction a write runs in, for a read within it. +func txOf(x *w) *sql.Tx { + tx, _ := x.querier.(*sql.Tx) + return tx +} + +// JobDelete implements driver.Executor. +func (e *executor) JobDelete(ctx context.Context, id driver.JobID) (*driver.JobRow, error) { + var job *driver.JobRow + err := e.write(ctx, func(x *w) error { + row, err := x.lockState(ctx, id) + switch { + case err != nil: + return err + case row == nil: + job, err = scanJob(x.QueryRowContext(ctx, `DELETE FROM hopper_job_history WHERE id = ? RETURNING `+historyColumns, idParam(id)), true) + if errors.Is(err, sql.ErrNoRows) { + return driver.ErrNotFound + } + return err + case row.job.State == driver.JobStateRunning: + return driver.ErrJobRunning + } + job, err = x.cancelWaiting(ctx, row, false) + return err + }) + if err != nil { + if errors.Is(err, driver.ErrNotFound) || errors.Is(err, driver.ErrJobRunning) { + return nil, err + } + return nil, fmt.Errorf("hopper: delete job: %w", err) + } + return job, nil +} + +// runNowSQL is the SET list that makes a waiting job runnable now and drops +// the reason it was waiting. A job that was already due keeps its +// scheduled_at, so it keeps its place in claim order and its aging. +const runNowSQL = `state = 'available', scheduled_at = min(scheduled_at, ?), snooze_reason = NULL, wake_requested_at = NULL` + +// retryHistorySQL moves a finalized job back to the live table with its +// history (attempts, errors, progress unless from the start, logs, fence) +// intact, a fresh seq, and one more attempt if it had run out. Its tags are +// what its tags column reads as. +const retryHistorySQL = ` +INSERT INTO hopper_jobs (id, kind, queue, state, priority, attempt, max_attempts, scheduled_at, args, metadata, + errors, unique_key, ordering_key, partition_key, batch_id, await, created_at, progress, logs, tags, fence) +SELECT id, kind, queue, 'available', priority, attempt, + CASE WHEN attempt >= max_attempts THEN attempt + 1 ELSE max_attempts END, ?, args, metadata, + errors, unique_key, ordering_key, partition_key, batch_id, await, created_at, CASE WHEN ? THEN NULL ELSE progress END, logs, + ?, fence +FROM hopper_job_history WHERE rowid = ? +RETURNING ` + jobColumns + +// JobRetry implements driver.Executor. +func (e *executor) JobRetry(ctx context.Context, params driver.JobRetryParams) (*driver.JobRow, error) { + id := params.ID + var job *driver.JobRow + err := e.write(ctx, func(x *w) error { + row, err := x.lockState(ctx, id) + switch { + case err != nil: + return err + case row == nil: + var ( + rowid int64 + kind, tags sql.NullString + uniqueKey sql.NullString + ) + err := x.QueryRowContext(ctx, `SELECT rowid, kind, unique_key, tags FROM hopper_job_history WHERE id = ? LIMIT 1`, idParam(id)). + Scan(&rowid, &kind, &uniqueKey, &tags) + if errors.Is(err, sql.ErrNoRows) { + return driver.ErrNotFound + } + if err != nil { + return err + } + if uniqueKey.Valid { + var held bool + if err := x.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM hopper_jobs WHERE kind = ? AND unique_key = ?)`, + kind.String, uniqueKey.String).Scan(&held); err != nil { + return err + } + if held { + return driver.ErrUniqueConflict + } + } + if job, err = scanJob(x.QueryRowContext(ctx, retryHistorySQL, x.us(), params.FromStart, + tagsJSON(jobcodec.DecodeTags(tags.String)), rowid), false); err != nil { + return err + } + _, err = x.ExecContext(ctx, `DELETE FROM hopper_job_history WHERE rowid = ?`, rowid) + return err + case row.job.State == driver.JobStateRunning: + return driver.ErrJobRunning + } + job, err = scanJob(x.QueryRowContext(ctx, `UPDATE hopper_jobs SET `+runNowSQL+`, expires_at = NULL, + progress = CASE WHEN ? THEN NULL ELSE progress END + WHERE id = ? RETURNING `+jobColumns, x.us(), params.FromStart, idParam(id)), false) + return err + }) + if err != nil { + if errors.Is(err, driver.ErrNotFound) || errors.Is(err, driver.ErrJobRunning) || errors.Is(err, driver.ErrUniqueConflict) { + return nil, err + } + return nil, fmt.Errorf("hopper: retry job: %w", err) + } + return job, nil +} + +// JobRetryOrderingKey implements driver.Executor: it marks the key's +// running job, so that its finalize honors the wake, and wakes the key's +// waiting jobs, oldest first, leaving those past their TTL to the expiry +// sweep. +func (e *executor) JobRetryOrderingKey(ctx context.Context, params driver.JobRetryOrderingKeyParams) (int, error) { + var n int64 + err := e.write(ctx, func(x *w) error { + if _, err := x.ExecContext(ctx, `UPDATE hopper_jobs SET wake_requested_at = ? + WHERE queue = ? AND ordering_key = ? AND state = 'running'`, x.us(), params.Queue, params.OrderingKey); err != nil { + return err + } + res, err := x.ExecContext(ctx, `UPDATE hopper_jobs SET `+runNowSQL+` + WHERE seq IN ( + SELECT seq FROM hopper_jobs + WHERE queue = ? AND ordering_key = ? AND state IN ('scheduled', 'retryable') AND (expires_at IS NULL OR expires_at > ?) + ORDER BY seq LIMIT ?)`, x.us(), params.Queue, params.OrderingKey, x.us(), max(params.Limit, 1)) + if err != nil { + return err + } + n, err = res.RowsAffected() + return err + }) + if err != nil { + return 0, fmt.Errorf("hopper: retry ordering key: %w", err) + } + return int(n), nil +} + +// JobDiscardExpired implements driver.Executor. +func (e *executor) JobDiscardExpired(ctx context.Context, limit int) ([]*driver.JobRow, error) { + var jobs []*driver.JobRow + err := e.write(ctx, func(x *w) error { + jobs = jobs[:0] + rows, err := queryAll(ctx, x, scanLive, `SELECT `+liveRowColumns+` FROM hopper_jobs + WHERE expires_at <= ? AND state IN ('available', 'scheduled', 'retryable') + ORDER BY expires_at LIMIT ?`, x.us(), max(limit, 1)) + if err != nil { + return err + } + done := make([]finished, 0, len(rows)) + for _, row := range rows { + errs, err := hopperError(row, "hopper: expired", x.now) + if err != nil { + return err + } + f := finished{row: row, state: driver.JobStateDiscarded, errors: errs, logs: row.logs, archive: true} + if err := x.finish(ctx, f); err != nil { + return err + } + done = append(done, f) + job, err := f.jobRow(x.now) + if err != nil { + return err + } + jobs = append(jobs, job) + } + _, err = x.tail(ctx, done) + return err + }) + if err != nil { + return nil, fmt.Errorf("hopper: discard expired jobs: %w", err) + } + return jobs, nil +} + +// JobRescueCandidates implements driver.Executor: running jobs whose +// client has no live lease, and, with StuckAfter, those claimed longer ago +// than that, oldest first. +func (e *executor) JobRescueCandidates(ctx context.Context, params driver.JobRescueParams) ([]driver.RescueCandidate, error) { + now := clock() + stuckBefore := int64(math.MinInt64) + if params.StuckAfter > 0 { + stuckBefore = now.Add(-params.StuckAfter).UnixMicro() + } + candidates, err := queryAll(ctx, e.read(), func(r scanner) (driver.RescueCandidate, error) { + var c driver.RescueCandidate + var err error + c.Job, err = scanJob(r, false, &c.Stuck) + return c, err + }, `SELECT * FROM ( + SELECT `+prefixed(jobColumns, "j")+`, coalesce(j.attempted_at < ?1, false) AS stuck + FROM hopper_jobs j WHERE j.state = 'running' + ) j + WHERE stuck OR NOT EXISTS (SELECT 1 FROM hopper_clients c WHERE c.id = j.attempted_by AND c.expires_at > ?2) + ORDER BY j.attempted_at LIMIT ?3`, stuckBefore, now.UnixMicro(), params.Limit) + if err != nil { + return nil, fmt.Errorf("hopper: rescue candidates: %w", err) + } + return candidates, nil +} + +// sortIDs orders IDs as they sort in the database. +func sortIDs(ids []driver.JobID) { + slices.SortFunc(ids, func(a, b driver.JobID) int { return bytes.Compare(a[:], b[:]) }) +} diff --git a/driver/hoppersqlite/listener.go b/driver/hoppersqlite/listener.go new file mode 100644 index 0000000..04d9a7a --- /dev/null +++ b/driver/hoppersqlite/listener.go @@ -0,0 +1,133 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "errors" + "time" + + "github.com/parallelworks/hopper/driver" +) + +// listener implements driver.Listener on hopper_notifications. It reads +// the rows committed after the last it read, on a connection of its own: +// at once when a transaction of this process that wrote some commits, and +// every listenInterval otherwise, for those of other processes, which it +// reads only when PRAGMA data_version says another connection has +// committed since its last look. +type listener struct { + d *Driver + conn *sql.Conn + channels map[string]bool + poke chan struct{} + unlisten func() + // last is the seq of the last notification read, and version the + // connection's data_version when it was read. + last, version int64 + pending []driver.Notification +} + +// listenPage is how many notifications a listener reads at a time. +const listenPage = 1000 + +func (d *Driver) newListener(ctx context.Context) (*listener, error) { + conn, err := d.sqlDB.Conn(ctx) + if err != nil { + return nil, err + } + l := &listener{d: d, conn: conn, channels: map[string]bool{}, poke: make(chan struct{}, 1), version: -1} + l.unlisten = d.db.listen(l.poke) + // Notifications committed from now on are delivered. + if err := conn.QueryRowContext(ctx, `SELECT coalesce(max(seq), 0) FROM hopper_notifications`).Scan(&l.last); err != nil { + _ = l.Close(ctx) + return nil, err + } + return l, nil +} + +// Listen implements driver.Listener. +func (l *listener) Listen(_ context.Context, channels ...string) error { + for _, c := range channels { + l.channels[c] = true + } + return nil +} + +// Next implements driver.Listener. +func (l *listener) Next(ctx context.Context) (driver.Notification, error) { + poked := true + for { + if len(l.pending) > 0 { + n := l.pending[0] + l.pending = l.pending[1:] + return n, nil + } + if err := l.read(ctx, poked); err != nil { + return driver.Notification{}, err + } + if len(l.pending) > 0 { + continue + } + timer := time.NewTimer(l.d.listenInterval) + select { + case <-l.poke: + poked = true + case <-timer.C: + poked = false + case <-ctx.Done(): + timer.Stop() + return driver.Notification{}, ctx.Err() + } + timer.Stop() + } +} + +// read reads the notifications committed since the last read, if this +// process poked the listener or another connection has committed since. +func (l *listener) read(ctx context.Context, poked bool) error { + var version int64 + if err := l.conn.QueryRowContext(ctx, "PRAGMA data_version").Scan(&version); err != nil { + return err + } + if !poked && version == l.version { + return nil + } + l.version = version + rows, err := l.conn.QueryContext(ctx, `SELECT seq, channel, payload FROM hopper_notifications WHERE seq > ? ORDER BY seq LIMIT ?`, l.last, listenPage) + if err != nil { + return err + } + defer rows.Close() + read := 0 + for rows.Next() { + var n driver.Notification + if err := rows.Scan(&l.last, &n.Channel, &n.Payload); err != nil { + return err + } + read++ + if l.channels[n.Channel] { + l.pending = append(l.pending, n) + } + } + if err := rows.Err(); err != nil { + return err + } + // A full page leaves more to read: look again without waiting. + if read == listenPage { + select { + case l.poke <- struct{}{}: + default: + } + } + return nil +} + +// Close implements driver.Listener. +func (l *listener) Close(context.Context) error { + l.unlisten() + err := l.conn.Close() + if errors.Is(err, sql.ErrConnDone) { + return nil + } + return err +} diff --git a/driver/hoppersqlite/localfs.go b/driver/hoppersqlite/localfs.go new file mode 100644 index 0000000..6bacaca --- /dev/null +++ b/driver/hoppersqlite/localfs.go @@ -0,0 +1,31 @@ +package hoppersqlite + +import ( + "errors" + "fmt" + "path/filepath" +) + +// ErrNetworkFilesystem is returned by Open for a database on a network or +// cluster filesystem (NFS, SMB, Lustre, GPFS and the like, or FUSE, which +// such filesystems often are), where SQLite's file locks and the shared +// memory its write-ahead log needs do not work reliably: two processes +// could both believe they hold the write lock and corrupt the database. +// Home directories on clusters are often on one. Choose a path on a local +// disk instead. +var ErrNetworkFilesystem = errors.New("hoppersqlite: the database is on a network filesystem, where SQLite cannot lock it reliably; choose a path on a local disk") + +// checkLocal returns ErrNetworkFilesystem if the directory the database +// file at path is in, where SQLite also keeps its log and shared memory, +// is on a network filesystem. +func checkLocal(path string) error { + dir := filepath.Dir(path) + kind, err := networkFilesystem(dir) + if err != nil { + return fmt.Errorf("hoppersqlite: tell whether %s is on a local filesystem: %w", dir, err) + } + if kind != "" { + return fmt.Errorf("%w: %s is on %s", ErrNetworkFilesystem, dir, kind) + } + return nil +} diff --git a/driver/hoppersqlite/localfs_bsd.go b/driver/hoppersqlite/localfs_bsd.go new file mode 100644 index 0000000..8db8295 --- /dev/null +++ b/driver/hoppersqlite/localfs_bsd.go @@ -0,0 +1,18 @@ +//go:build darwin || freebsd || openbsd + +package hoppersqlite + +import "golang.org/x/sys/unix" + +// networkFilesystem reports a filesystem the kernel does not mark local +// (MNT_LOCAL), as NFS, SMB, AFP, WebDAV and FUSE mounts are not. +func networkFilesystem(dir string) (string, error) { + var st unix.Statfs_t + if err := unix.Statfs(dir, &st); err != nil { + return "", err + } + if statfsFlags(&st)&unix.MNT_LOCAL == 0 { + return "a filesystem not marked local (" + statfsType(&st) + ")", nil + } + return "", nil +} diff --git a/driver/hoppersqlite/localfs_darwin.go b/driver/hoppersqlite/localfs_darwin.go new file mode 100644 index 0000000..945ff68 --- /dev/null +++ b/driver/hoppersqlite/localfs_darwin.go @@ -0,0 +1,7 @@ +package hoppersqlite + +import "golang.org/x/sys/unix" + +func statfsFlags(st *unix.Statfs_t) uint64 { return uint64(st.Flags) } + +func statfsType(st *unix.Statfs_t) string { return unix.ByteSliceToString(st.Fstypename[:]) } diff --git a/driver/hoppersqlite/localfs_freebsd.go b/driver/hoppersqlite/localfs_freebsd.go new file mode 100644 index 0000000..9be34e8 --- /dev/null +++ b/driver/hoppersqlite/localfs_freebsd.go @@ -0,0 +1,7 @@ +package hoppersqlite + +import "golang.org/x/sys/unix" + +func statfsFlags(st *unix.Statfs_t) uint64 { return st.Flags } + +func statfsType(st *unix.Statfs_t) string { return unix.ByteSliceToString(st.Fstypename[:]) } diff --git a/driver/hoppersqlite/localfs_linux.go b/driver/hoppersqlite/localfs_linux.go new file mode 100644 index 0000000..5d9cdc0 --- /dev/null +++ b/driver/hoppersqlite/localfs_linux.go @@ -0,0 +1,45 @@ +package hoppersqlite + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +// networkMagic names the filesystems, by statfs(2)'s magic number, whose +// locks SQLite cannot rely on: network and cluster filesystems, the shared +// folders of virtual machines, and FUSE, which sshfs, s3fs, GlusterFS and +// many others are. +var networkMagic = map[int64]string{ + 0x6969: "NFS", + 0x517b: "SMB", + 0xfe534d42: "SMB2", + 0xff534d42: "CIFS", + 0x73757245: "Coda", + 0x5346414f: "AFS", + 0x6b414653: "AFS", + 0x01021997: "9P", + 0x564c: "NCP", + 0x00c36400: "CephFS", + 0x01161970: "GFS2", + 0x7461636f: "OCFS2", + 0x0bd00bd0: "Lustre", + 0x47504653: "GPFS", + 0x19830326: "BeeGFS", + 0xaad7aaea: "PanFS", + 0x20030528: "OrangeFS", + 0x6a656a63: "virtiofs", + 0x786f4256: "VirtualBox shared folders", + 0x65735546: "FUSE", +} + +func networkFilesystem(dir string) (string, error) { + var st unix.Statfs_t + if err := unix.Statfs(dir, &st); err != nil { + return "", err + } + if name, ok := networkMagic[int64(st.Type)&0xffffffff]; ok { //nolint:unconvert // st.Type is an int32 or a uint32 on some architectures + return fmt.Sprintf("%s (filesystem type %#x)", name, st.Type), nil + } + return "", nil +} diff --git a/driver/hoppersqlite/localfs_netbsd.go b/driver/hoppersqlite/localfs_netbsd.go new file mode 100644 index 0000000..4202a8b --- /dev/null +++ b/driver/hoppersqlite/localfs_netbsd.go @@ -0,0 +1,16 @@ +package hoppersqlite + +import "golang.org/x/sys/unix" + +// networkFilesystem reports a filesystem the kernel does not mark local +// (MNT_LOCAL), as NFS, SMB and FUSE mounts are not. +func networkFilesystem(dir string) (string, error) { + var st unix.Statvfs_t + if err := unix.Statvfs(dir, &st); err != nil { + return "", err + } + if uint64(st.Flag)&unix.MNT_LOCAL == 0 { + return "a filesystem not marked local (" + unix.ByteSliceToString(st.Fstypename[:]) + ")", nil + } + return "", nil +} diff --git a/driver/hoppersqlite/localfs_openbsd.go b/driver/hoppersqlite/localfs_openbsd.go new file mode 100644 index 0000000..e1e7f4e --- /dev/null +++ b/driver/hoppersqlite/localfs_openbsd.go @@ -0,0 +1,7 @@ +package hoppersqlite + +import "golang.org/x/sys/unix" + +func statfsFlags(st *unix.Statfs_t) uint64 { return uint64(st.F_flags) } + +func statfsType(st *unix.Statfs_t) string { return unix.ByteSliceToString(st.F_fstypename[:]) } diff --git a/driver/hoppersqlite/localfs_other.go b/driver/hoppersqlite/localfs_other.go new file mode 100644 index 0000000..c1fdf89 --- /dev/null +++ b/driver/hoppersqlite/localfs_other.go @@ -0,0 +1,11 @@ +//go:build !linux && !darwin && !freebsd && !openbsd && !netbsd && !windows + +package hoppersqlite + +import "errors" + +// networkFilesystem cannot tell on this system, and refuses rather than +// risk a database on a filesystem whose locks do not hold. +func networkFilesystem(string) (string, error) { + return "", errors.New("this system has no way to tell a local filesystem from a network one") +} diff --git a/driver/hoppersqlite/localfs_windows.go b/driver/hoppersqlite/localfs_windows.go new file mode 100644 index 0000000..fd35dcf --- /dev/null +++ b/driver/hoppersqlite/localfs_windows.go @@ -0,0 +1,32 @@ +package hoppersqlite + +import ( + "path/filepath" + "strings" + + "golang.org/x/sys/windows" +) + +// networkFilesystem reports a UNC path or a drive Windows reports as remote, +// such as a mapped network drive. +func networkFilesystem(dir string) (string, error) { + abs, err := filepath.Abs(dir) + if err != nil { + return "", err + } + if strings.HasPrefix(abs, `\\`) && !strings.HasPrefix(abs, `\\?\`) { + return "a network share", nil + } + path, err := windows.UTF16PtrFromString(abs) + if err != nil { + return "", err + } + volume := make([]uint16, windows.MAX_PATH+1) + if err := windows.GetVolumePathName(path, &volume[0], uint32(len(volume))); err != nil { + return "", err + } + if windows.GetDriveType(&volume[0]) == windows.DRIVE_REMOTE { + return "a network drive", nil + } + return "", nil +} diff --git a/driver/hoppersqlite/messaging.go b/driver/hoppersqlite/messaging.go new file mode 100644 index 0000000..4d56d9e --- /dev/null +++ b/driver/hoppersqlite/messaging.go @@ -0,0 +1,476 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "slices" + "strconv" + "time" + + "github.com/parallelworks/hopper/driver" +) + +// SubscriptionUpsert implements driver.Executor. +func (e *executor) SubscriptionUpsert(ctx context.Context, subs []driver.SubscriptionRow) error { + if len(subs) == 0 { + return nil + } + type row struct { + maxAttempts any + metadata string + } + rows := make([]row, len(subs)) + for i, s := range subs { + if s.MaxAttempts > 0 { + v, err := smallint(s.MaxAttempts, "max_attempts") + if err != nil { + return err + } + rows[i].maxAttempts = v + } + var err error + if rows[i].metadata, err = jsonValue(s.Metadata, "{}", "subscription metadata"); err != nil { + return err + } + } + err := e.write(ctx, func(x *w) error { + for i, s := range subs { + if _, err := x.ExecContext(ctx, ` + INSERT INTO hopper_subscriptions (name, pattern, kind, queue, max_attempts, metadata, created_at) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (name) DO UPDATE SET pattern = excluded.pattern, kind = excluded.kind, queue = excluded.queue, + max_attempts = excluded.max_attempts, metadata = excluded.metadata`, + s.Name, s.Pattern, s.Kind, s.Queue, rows[i].maxAttempts, rows[i].metadata, x.us()); err != nil { + return err + } + } + return nil + }) + if err != nil { + return fmt.Errorf("hopper: upsert subscriptions: %w", err) + } + return nil +} + +// SubscriptionList implements driver.Executor. +func (e *executor) SubscriptionList(ctx context.Context) ([]*driver.SubscriptionRow, error) { + subs, err := queryAll(ctx, e.read(), func(r scanner) (*driver.SubscriptionRow, error) { + var ( + s driver.SubscriptionRow + maxAttempts sql.NullInt64 + metadata string + created int64 + ) + if err := r.Scan(&s.Name, &s.Pattern, &s.Kind, &s.Queue, &maxAttempts, &metadata, &created); err != nil { + return nil, err + } + s.MaxAttempts, s.Metadata, s.CreatedAt = int(maxAttempts.Int64), json.RawMessage(metadata), time.UnixMicro(created).UTC() + return &s, nil + }, `SELECT name, pattern, kind, queue, max_attempts, metadata, created_at FROM hopper_subscriptions ORDER BY name`) + if err != nil { + return nil, fmt.Errorf("hopper: list subscriptions: %w", err) + } + return subs, nil +} + +// mergeObjects returns a JSON object with base's members and over's, over +// winning, as jsonb's || has it for two objects. A value that is not an +// object adds nothing. +func mergeObjects(base, over string) (string, error) { + m := map[string]json.RawMessage{} + _ = json.Unmarshal([]byte(base), &m) + var o map[string]json.RawMessage + _ = json.Unmarshal([]byte(over), &o) + for k, v := range o { + m[k] = v + } + b, err := json.Marshal(m) + return string(b), err +} + +// MessagePublish implements driver.Executor: it inserts one delivery per +// subscription whose pattern matches the topic. Each delivery's metadata +// records the topic, the message ID and the headers, merged with the +// subscription's own; a dedup key becomes the deliveries' unique key, +// scoped by kind, so each subscription deduplicates on its own. +func (e *executor) MessagePublish(ctx context.Context, p driver.MessagePublishParams) ([]driver.JobInsertResult, error) { + if p.Priority == 0 { + p.Priority = 2 + } + priority, err := smallint(p.Priority, "priority") + if err != nil { + return nil, err + } + headers := json.RawMessage("{}") + if p.Headers != nil { + if headers, err = json.Marshal(p.Headers); err != nil { + return nil, fmt.Errorf("hopper: publish: encode headers: %w", err) + } + } + payload, err := jsonValue(p.Payload, "null", "payload") + if err != nil { + return nil, err + } + var results []driver.JobInsertResult + err = e.write(ctx, func(x *w) error { + results = results[:0] + type sub struct { + kind, queue string + maxAttempts sql.NullInt64 + metadata string + } + subs, err := queryAll(ctx, x, func(r scanner) (sub, error) { + var s sub + err := r.Scan(&s.kind, &s.queue, &s.maxAttempts, &s.metadata) + return s, err + }, `SELECT kind, queue, max_attempts, metadata FROM hopper_subscriptions + WHERE hopper_topic_matches(pattern, ?) ORDER BY name`, p.Topic) + if err != nil { + return err + } + messageID := newID(x.now).String() + base, err := json.Marshal(map[string]any{"topic": p.Topic, "message_id": messageID, "headers": headers}) + if err != nil { + return err + } + var queues []string + for _, s := range subs { + metadata, err := mergeObjects(string(base), s.metadata) + if err != nil { + return err + } + attempts := 25 + switch { + case s.maxAttempts.Valid: + attempts = int(s.maxAttempts.Int64) + case p.OrderingKey != "": + attempts = 10 + } + j := jobInsert{ + kind: s.kind, queue: s.queue, priority: priority, maxAttempts: attempts, scheduledAt: x.now.Add(max(p.Delay, 0)), + args: payload, metadata: metadata, orderingKey: p.OrderingKey, ttl: p.TTL, await: p.Await, + } + if p.DedupKey != "" { + j.uniqueKey = "msg:" + p.DedupKey + } + r, err := x.insertJobs(ctx, []jobInsert{j}, driver.ConflictSkip) + if err != nil { + return err + } + results = append(results, r[0]) + if !slices.Contains(queues, s.queue) { + queues = append(queues, s.queue) + } + } + if p.Notify { + return x.notify(ctx, driver.ChannelInsert, queues...) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("hopper: publish: %w", err) + } + return results, nil +} + +// Streams. SQLite runs one writer at a time, so events commit in the order +// they are appended: an event's position is its seq, and a consumer has +// read every event up to the seq it has seen, with nothing committed later +// to sort before it. Positions carry no transaction ID (Xid is 0). + +func position(seq int64) driver.StreamPosition { return driver.StreamPosition{Seq: seq} } + +const streamEventColumns = `seq, topic, key, payload, headers, message_id, created_at` + +func scanEvent(r scanner) (*driver.StreamEvent, error) { + var ( + ev driver.StreamEvent + seq, created int64 + key sql.NullString + payload, headers string + messageID []byte + ) + if err := r.Scan(&seq, &ev.Topic, &key, &payload, &headers, &messageID, &created); err != nil { + return nil, err + } + var id driver.JobID + if err := id.Scan(messageID); err != nil { + return nil, err + } + ev.Position, ev.Key, ev.Payload, ev.MessageID, ev.CreatedAt = position(seq), key.String, json.RawMessage(payload), id.String(), time.UnixMicro(created).UTC() + if headers != "{}" { + if err := json.Unmarshal([]byte(headers), &ev.Headers); err != nil { + return nil, fmt.Errorf("headers: %w", err) + } + } + return &ev, nil +} + +// StreamAppend implements driver.Executor. +func (e *executor) StreamAppend(ctx context.Context, params driver.StreamAppendParams) (*driver.StreamEvent, error) { + headers := []byte("{}") + if params.Headers != nil { + var err error + if headers, err = json.Marshal(params.Headers); err != nil { + return nil, fmt.Errorf("hopper: append: encode headers: %w", err) + } + } + payload, err := jsonValue(params.Payload, "null", "payload") + if err != nil { + return nil, err + } + var ev *driver.StreamEvent + err = e.write(ctx, func(x *w) error { + var err error + if ev, err = scanEvent(x.QueryRowContext(ctx, `INSERT INTO hopper_stream_events (topic, key, payload, headers, message_id, created_at) + VALUES (?, ?, ?, ?, ?, ?) RETURNING `+streamEventColumns, + params.Topic, nullable(params.Key), payload, string(headers), idParam(newID(x.now)), x.us())); err != nil { + return err + } + if params.Notify { + return x.notify(ctx, driver.ChannelStream, params.Topic) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("hopper: append: %w", err) + } + return ev, nil +} + +// StreamRead implements driver.Executor. +func (e *executor) StreamRead(ctx context.Context, params driver.StreamReadParams) ([]*driver.StreamEvent, error) { + events, err := queryAll(ctx, e.read(), scanEvent, `SELECT `+streamEventColumns+` FROM hopper_stream_events + WHERE seq > ? AND (? = '' OR hopper_topic_matches(?, topic)) ORDER BY seq LIMIT ?`, + params.After.Seq, params.Pattern, params.Pattern, max(params.Limit, 1)) + if err != nil { + return nil, fmt.Errorf("hopper: read stream: %w", err) + } + return events, nil +} + +// StreamConsumerUpsert implements driver.Executor. A consumer starting at +// latest has seen every event appended so far. +func (e *executor) StreamConsumerUpsert(ctx context.Context, consumers []driver.StreamConsumerRow) error { + if len(consumers) == 0 { + return nil + } + err := e.write(ctx, func(x *w) error { + var latest int64 + if err := x.QueryRowContext(ctx, `SELECT coalesce(max(seq), 0) FROM hopper_stream_events`).Scan(&latest); err != nil { + return err + } + for _, c := range consumers { + var maxAttempts any + if c.MaxAttempts > 0 { + v, err := smallint(c.MaxAttempts, "max_attempts") + if err != nil { + return err + } + maxAttempts = v + } + metadata, err := jsonValue(c.Metadata, "{}", "consumer metadata") + if err != nil { + return err + } + seen := latest + if c.Start == driver.StreamStartEarliest { + seen = 0 + } + if _, err := x.ExecContext(ctx, ` + INSERT INTO hopper_stream_consumers (name, pattern, kind, queue, max_attempts, metadata, seen, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (name) DO UPDATE SET pattern = excluded.pattern, kind = excluded.kind, queue = excluded.queue, + max_attempts = excluded.max_attempts, metadata = excluded.metadata`, + c.Name, c.Pattern, c.Kind, c.Queue, maxAttempts, metadata, seen, x.us()); err != nil { + return err + } + } + return nil + }) + if err != nil { + return fmt.Errorf("hopper: upsert stream consumers: %w", err) + } + return nil +} + +// StreamConsumerList implements driver.Executor. A consumer's Snapshot is +// the seq it has read up to, and its Position the last event it delivered. +func (e *executor) StreamConsumerList(ctx context.Context) ([]*driver.StreamConsumerRow, error) { + consumers, err := queryAll(ctx, e.read(), func(r scanner) (*driver.StreamConsumerRow, error) { + var ( + c driver.StreamConsumerRow + maxAttempts sql.NullInt64 + metadata string + seen, pos, created int64 + delivered micros + ) + if err := r.Scan(&c.Name, &c.Pattern, &c.Kind, &c.Queue, &maxAttempts, &metadata, &seen, &pos, &delivered, &created); err != nil { + return nil, err + } + c.MaxAttempts, c.Metadata = int(maxAttempts.Int64), json.RawMessage(metadata) + c.Snapshot, c.Position = strconv.FormatInt(seen, 10), position(pos) + c.DeliveredAt, c.CreatedAt = delivered.time(), time.UnixMicro(created).UTC() + return &c, nil + }, `SELECT name, pattern, kind, queue, max_attempts, metadata, seen, position, delivered_at, created_at + FROM hopper_stream_consumers ORDER BY name`) + if err != nil { + return nil, fmt.Errorf("hopper: list stream consumers: %w", err) + } + return consumers, nil +} + +// StreamConsumerSeek implements driver.Executor. +func (e *executor) StreamConsumerSeek(ctx context.Context, name string, params driver.StreamSeekParams) error { + err := e.write(ctx, func(x *w) error { + var ( + seen int64 + latest bool + ) + switch { + case !params.Position.IsZero(): + seen = params.Position.Seq + case !params.Time.IsZero(): + // Just before the first event appended at or after the time. + err := x.QueryRowContext(ctx, `SELECT seq - 1 FROM hopper_stream_events WHERE created_at >= ? ORDER BY seq LIMIT 1`, + params.Time.UnixMicro()).Scan(&seen) + if errors.Is(err, sql.ErrNoRows) { + latest = true + break + } + if err != nil { + return err + } + case params.Earliest: + case params.Latest: + latest = true + default: + return errors.New("nothing to seek to") + } + if latest { + if err := x.QueryRowContext(ctx, `SELECT coalesce(max(seq), 0) FROM hopper_stream_events`).Scan(&seen); err != nil { + return err + } + } + res, err := x.ExecContext(ctx, `UPDATE hopper_stream_consumers SET seen = ?, position = ? WHERE name = ?`, seen, seen, name) + if err != nil { + return err + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return cmpErr(err, driver.ErrNotFound) + } + return nil + }) + if err != nil { + if errors.Is(err, driver.ErrNotFound) { + return err + } + return fmt.Errorf("hopper: seek stream consumer %s: %w", name, err) + } + return nil +} + +// cmpErr returns err, or def if err is nil. +func cmpErr(err, def error) error { + if err != nil { + return err + } + return def +} + +// StreamPump implements driver.Executor: it turns up to limit events past +// a consumer's seen seq that match its pattern into delivery jobs, and +// advances it past them, or past every event if fewer matched. Deliveries +// carry the topic, message ID, headers, consumer name and position in +// their metadata, over the consumer's own, and the event's key as their +// ordering key. +func (e *executor) StreamPump(ctx context.Context, name string, limit int) (driver.StreamPumpResult, error) { + var res driver.StreamPumpResult + limit = max(limit, 1) + err := e.write(ctx, func(x *w) error { + res = driver.StreamPumpResult{} + var ( + pattern, kind, queue, metadata string + maxAttempts sql.NullInt64 + seen, pos int64 + ) + err := x.QueryRowContext(ctx, `SELECT pattern, kind, queue, max_attempts, metadata, seen, position FROM hopper_stream_consumers WHERE name = ?`, + name).Scan(&pattern, &kind, &queue, &maxAttempts, &metadata, &seen, &pos) + if errors.Is(err, sql.ErrNoRows) { + return driver.ErrNotFound + } + if err != nil { + return err + } + res.Position = position(pos) + events, err := queryAll(ctx, x, scanEvent, `SELECT `+streamEventColumns+` FROM hopper_stream_events + WHERE seq > ? AND hopper_topic_matches(?, topic) ORDER BY seq LIMIT ?`, seen, pattern, limit) + if err != nil { + return err + } + for _, ev := range events { + headers, err := json.Marshal(ev.Headers) + if err != nil { + return err + } + if ev.Headers == nil { + headers = []byte("{}") + } + over, err := json.Marshal(map[string]any{ + "topic": ev.Topic, "message_id": ev.MessageID, "headers": json.RawMessage(headers), + "stream": name, "position": ev.Position.String(), + }) + if err != nil { + return err + } + meta, err := mergeObjects(metadata, string(over)) + if err != nil { + return err + } + attempts := 25 + switch { + case maxAttempts.Valid: + attempts = int(maxAttempts.Int64) + case ev.Key != "": + attempts = 10 + } + if _, err := x.insertJob(ctx, jobInsert{ + kind: kind, queue: queue, priority: 2, maxAttempts: attempts, args: string(ev.Payload), metadata: meta, orderingKey: ev.Key, + }); err != nil { + return err + } + res.Position = ev.Position + seen = ev.Position.Seq + } + res.Delivered = len(events) + if res.Delivered < limit { + // Every event matching the pattern has been delivered: the + // consumer has seen every event appended so far. + if err := x.QueryRowContext(ctx, `SELECT max(?, coalesce(max(seq), 0)) FROM hopper_stream_events`, seen).Scan(&seen); err != nil { + return err + } + } + var delivered any + if res.Delivered > 0 { + delivered = x.us() + res.Queues = []string{queue} + } + if _, err := x.ExecContext(ctx, `UPDATE hopper_stream_consumers SET seen = ?, position = ?, delivered_at = coalesce(?, delivered_at) WHERE name = ?`, + seen, res.Position.Seq, delivered, name); err != nil { + return err + } + if res.Delivered > 0 { + return x.notify(ctx, driver.ChannelInsert, queue) + } + return nil + }) + if err != nil { + if errors.Is(err, driver.ErrNotFound) { + return res, err + } + return res, fmt.Errorf("hopper: pump stream consumer %s: %w", name, err) + } + return res, nil +} diff --git a/driver/hoppersqlite/migrations/001_schema.down.sql b/driver/hoppersqlite/migrations/001_schema.down.sql new file mode 100644 index 0000000..5391ab5 --- /dev/null +++ b/driver/hoppersqlite/migrations/001_schema.down.sql @@ -0,0 +1,9 @@ +DROP TABLE hopper_schema; +DROP TABLE hopper_notifications; +DROP TABLE hopper_subscriptions; +DROP TABLE hopper_periodic; +DROP TABLE hopper_queues; +DROP TABLE hopper_leader; +DROP TABLE hopper_clients; +DROP TABLE hopper_job_history; +DROP TABLE hopper_jobs; diff --git a/driver/hoppersqlite/migrations/001_schema.up.sql b/driver/hoppersqlite/migrations/001_schema.up.sql new file mode 100644 index 0000000..f1cdee7 --- /dev/null +++ b/driver/hoppersqlite/migrations/001_schema.up.sql @@ -0,0 +1,145 @@ +-- hopper schema v1 on SQLite. +-- +-- SQLite's scripts follow hoppermigrate's Postgres scripts version for +-- version, so that a version names the same tables and columns on both. +-- What only Postgres has (partitions, enum types, autovacuum settings, SQL +-- functions) is left out, and a version that changes only those is a script +-- of comments here. +-- +-- Times are INTEGER microseconds since the Unix epoch, which is Postgres's +-- timestamptz precision; IDs are 16-byte BLOBs, which sort as UUIDs do; JSON +-- values are TEXT. + +-- Live jobs only. Finalized jobs move to hopper_job_history, so this table and +-- its indexes stay sized to the backlog, not to all-time volume. seq is the +-- rowid, which AUTOINCREMENT never reuses: history keeps it, and it orders a +-- workflow's jobs. +CREATE TABLE hopper_jobs ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + id BLOB NOT NULL UNIQUE CHECK (length(id) = 16), + kind TEXT NOT NULL, + queue TEXT NOT NULL DEFAULT 'default', + state TEXT NOT NULL DEFAULT 'available' + CHECK (state IN ('pending', 'available', 'scheduled', 'running', 'retryable')), + priority INTEGER NOT NULL DEFAULT 2 CHECK (priority BETWEEN 1 AND 4), + attempt INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 25, + scheduled_at INTEGER NOT NULL, + attempted_at INTEGER, + attempted_by INTEGER, -- hopper_clients.id holding the claim + args TEXT NOT NULL DEFAULT '{}', + metadata TEXT NOT NULL DEFAULT '{}', + errors TEXT NOT NULL DEFAULT '[]', + unique_key TEXT, + ordering_key TEXT, + partition_key TEXT, + batch_id BLOB, + expires_at INTEGER, -- TTL: discarded if not started by then + cancel_requested_at INTEGER, + await INTEGER NOT NULL DEFAULT 0, -- notify waiters on finalize + created_at INTEGER NOT NULL +); + +-- Claim path: the only index the hot query touches. +CREATE INDEX hopper_jobs_claim ON hopper_jobs (queue, priority, scheduled_at, seq) + WHERE state IN ('available', 'scheduled', 'retryable'); + +-- Running set: rescue, cancel delivery, global limits. Bounded by total concurrency. +CREATE INDEX hopper_jobs_running ON hopper_jobs (queue, attempted_by) WHERE state = 'running'; + +-- Uniqueness applies only while a job is live. +CREATE UNIQUE INDEX hopper_jobs_unique ON hopper_jobs (kind, unique_key) WHERE unique_key IS NOT NULL; + +-- The expiry sweep's walk. +CREATE INDEX hopper_jobs_expires ON hopper_jobs (expires_at) WHERE expires_at IS NOT NULL; + +-- Finalized jobs, in one table: retention deletes them in batches. +CREATE TABLE hopper_job_history ( + id BLOB NOT NULL, + seq INTEGER NOT NULL, + kind TEXT NOT NULL, + queue TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('completed', 'canceled', 'discarded')), + priority INTEGER NOT NULL, + attempt INTEGER NOT NULL, + max_attempts INTEGER NOT NULL, + scheduled_at INTEGER NOT NULL, + attempted_at INTEGER, + attempted_by INTEGER, + args TEXT NOT NULL DEFAULT '{}', + metadata TEXT NOT NULL DEFAULT '{}', + errors TEXT NOT NULL DEFAULT '[]', + unique_key TEXT, + ordering_key TEXT, + partition_key TEXT, + batch_id BLOB, + expires_at INTEGER, + cancel_requested_at INTEGER, + await INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + finalized_at INTEGER NOT NULL, + output TEXT +); + +CREATE INDEX hopper_job_history_id ON hopper_job_history (id); +CREATE INDEX hopper_job_history_queue ON hopper_job_history (queue, finalized_at); +-- Retention's walk, by outcome. +CREATE INDEX hopper_job_history_retention ON hopper_job_history (state, finalized_at); + +-- Liveness: one lease row per running client process. +CREATE TABLE hopper_clients ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hostname TEXT NOT NULL, + started_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + info TEXT NOT NULL DEFAULT '{}' -- version, queues, pid +); + +CREATE TABLE hopper_leader ( + name TEXT PRIMARY KEY DEFAULT 'default', + client_id INTEGER NOT NULL, + elected_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL +); + +CREATE TABLE hopper_queues ( + name TEXT PRIMARY KEY, + paused_at INTEGER, + global_limit INTEGER, -- NULL = unlimited + rate_per_sec REAL, -- NULL = unlimited + rate_burst INTEGER, + tokens REAL, + refilled_at INTEGER, + metadata TEXT NOT NULL DEFAULT '{}', + updated_at INTEGER NOT NULL +); + +CREATE TABLE hopper_periodic ( -- last inserted slot per periodic job + name TEXT PRIMARY KEY, + last_slot INTEGER NOT NULL +); + +CREATE TABLE hopper_subscriptions ( + name TEXT PRIMARY KEY, + pattern TEXT NOT NULL, -- AMQP topic syntax + kind TEXT NOT NULL, + queue TEXT NOT NULL, + created_at INTEGER NOT NULL +); + +-- Notifications, which SQLite has no LISTEN for: a notifying transaction +-- writes its rows, which commit with it, and listeners in every process that +-- opens the database read the ones after the last they saw. Rows a minute +-- old are deleted as new ones are written. +CREATE TABLE hopper_notifications ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + channel TEXT NOT NULL, + payload TEXT NOT NULL, + created_at INTEGER NOT NULL +); +CREATE INDEX hopper_notifications_created ON hopper_notifications (created_at); + +CREATE TABLE hopper_schema ( + version INTEGER PRIMARY KEY, + applied_at INTEGER NOT NULL +); diff --git a/driver/hoppersqlite/migrations/002_messaging.down.sql b/driver/hoppersqlite/migrations/002_messaging.down.sql new file mode 100644 index 0000000..9f8038a --- /dev/null +++ b/driver/hoppersqlite/migrations/002_messaging.down.sql @@ -0,0 +1,4 @@ +DROP INDEX hopper_jobs_ordering; +DROP INDEX hopper_jobs_ordering_running; +ALTER TABLE hopper_subscriptions DROP COLUMN metadata; +ALTER TABLE hopper_subscriptions DROP COLUMN max_attempts; diff --git a/driver/hoppersqlite/migrations/002_messaging.up.sql b/driver/hoppersqlite/migrations/002_messaging.up.sql new file mode 100644 index 0000000..8a4996d --- /dev/null +++ b/driver/hoppersqlite/migrations/002_messaging.up.sql @@ -0,0 +1,17 @@ +-- hopper schema v2: messaging. +-- +-- Subscriptions carry their own retry budget, and deliveries with an +-- ordering key run one at a time per key. Postgres's SQL insert and publish +-- functions have no SQLite counterpart. + +ALTER TABLE hopper_subscriptions ADD COLUMN max_attempts INTEGER; -- NULL: the client default (10 for ordered deliveries) +ALTER TABLE hopper_subscriptions ADD COLUMN metadata TEXT NOT NULL DEFAULT '{}'; + +-- Backstop for ordering keys: the claim selects at most one delivery per +-- (queue, ordering_key), and this makes running two of one key impossible. +CREATE UNIQUE INDEX hopper_jobs_ordering_running ON hopper_jobs (queue, ordering_key) + WHERE state = 'running' AND ordering_key IS NOT NULL; + +-- Oldest-first lookup per ordering key for the claim. +CREATE INDEX hopper_jobs_ordering ON hopper_jobs (queue, ordering_key, seq) + WHERE ordering_key IS NOT NULL AND state IN ('available', 'scheduled', 'retryable'); diff --git a/driver/hoppersqlite/migrations/003_flow_control.down.sql b/driver/hoppersqlite/migrations/003_flow_control.down.sql new file mode 100644 index 0000000..b6b7fa9 --- /dev/null +++ b/driver/hoppersqlite/migrations/003_flow_control.down.sql @@ -0,0 +1,5 @@ +DROP INDEX hopper_jobs_batch; +DROP TABLE hopper_batches; +DROP INDEX hopper_jobs_partition_running; +ALTER TABLE hopper_queues DROP COLUMN aging_seconds; +ALTER TABLE hopper_queues DROP COLUMN partition_limit; diff --git a/driver/hoppersqlite/migrations/003_flow_control.up.sql b/driver/hoppersqlite/migrations/003_flow_control.up.sql new file mode 100644 index 0000000..85e9991 --- /dev/null +++ b/driver/hoppersqlite/migrations/003_flow_control.up.sql @@ -0,0 +1,26 @@ +-- hopper schema v3: flow control and batches. + +ALTER TABLE hopper_queues ADD COLUMN partition_limit INTEGER; -- max running jobs per partition_key, NULL = unlimited +ALTER TABLE hopper_queues ADD COLUMN aging_seconds INTEGER; -- raise a waiting job's priority after this long, NULL = off + +-- Partitioned limits count running jobs per partition key. +CREATE INDEX hopper_jobs_partition_running ON hopper_jobs (queue, partition_key) + WHERE state = 'running' AND partition_key IS NOT NULL; + +-- A batch counts its pending jobs down in the transaction that finalizes +-- them, which no other writes at the same time, and enqueues its callbacks +-- when the count reaches zero. +CREATE TABLE hopper_batches ( + id BLOB PRIMARY KEY CHECK (length(id) = 16), + pending INTEGER NOT NULL, + failed INTEGER NOT NULL DEFAULT 0, + total INTEGER NOT NULL, + on_success TEXT, -- {kind, queue, args, priority, max_attempts, metadata, tags} + on_failure TEXT, + on_complete TEXT, + metadata TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL, + completed_at INTEGER +); + +CREATE INDEX hopper_jobs_batch ON hopper_jobs (batch_id) WHERE batch_id IS NOT NULL; diff --git a/driver/hoppersqlite/migrations/004_workflows.down.sql b/driver/hoppersqlite/migrations/004_workflows.down.sql new file mode 100644 index 0000000..96874b6 --- /dev/null +++ b/driver/hoppersqlite/migrations/004_workflows.down.sql @@ -0,0 +1,4 @@ +DROP INDEX hopper_job_history_batch; +DROP TABLE hopper_job_deps; +ALTER TABLE hopper_batches DROP COLUMN edges; +ALTER TABLE hopper_batches DROP COLUMN name; diff --git a/driver/hoppersqlite/migrations/004_workflows.up.sql b/driver/hoppersqlite/migrations/004_workflows.up.sql new file mode 100644 index 0000000..2bc2499 --- /dev/null +++ b/driver/hoppersqlite/migrations/004_workflows.up.sql @@ -0,0 +1,23 @@ +-- hopper schema v4: workflows. +-- +-- A workflow is a batch whose jobs depend on each other. Dependents wait in +-- the 'pending' state; the transaction that finalizes a job promotes the +-- dependents whose dependencies are all finalized, and cancels the +-- dependents of a failed job (transitively) unless their edge says to +-- ignore the failure. hopper_job_deps is the working set of edges, deleted +-- as dependents finalize; the batch row keeps the whole graph. + +ALTER TABLE hopper_batches ADD COLUMN name TEXT; +ALTER TABLE hopper_batches ADD COLUMN edges TEXT; -- [[job_id, depends_on], ...] for workflows + +CREATE TABLE hopper_job_deps ( + job_id BLOB NOT NULL, -- the dependent, pending until depends_on is finalized + depends_on BLOB NOT NULL, + on_failure TEXT NOT NULL DEFAULT 'cancel', -- 'cancel' the dependent when depends_on fails, or 'ignore' + PRIMARY KEY (job_id, depends_on) +) WITHOUT ROWID; + +CREATE INDEX hopper_job_deps_depends_on ON hopper_job_deps (depends_on); + +-- Workflow inspection reads a batch's finished jobs from history. +CREATE INDEX hopper_job_history_batch ON hopper_job_history (batch_id) WHERE batch_id IS NOT NULL; diff --git a/driver/hoppersqlite/migrations/005_streams.down.sql b/driver/hoppersqlite/migrations/005_streams.down.sql new file mode 100644 index 0000000..c8d47fa --- /dev/null +++ b/driver/hoppersqlite/migrations/005_streams.down.sql @@ -0,0 +1,2 @@ +DROP TABLE hopper_stream_consumers; +DROP TABLE hopper_stream_events; diff --git a/driver/hoppersqlite/migrations/005_streams.up.sql b/driver/hoppersqlite/migrations/005_streams.up.sql new file mode 100644 index 0000000..7a5cf5b --- /dev/null +++ b/driver/hoppersqlite/migrations/005_streams.up.sql @@ -0,0 +1,31 @@ +-- hopper schema v5: streams. +-- +-- A stream is an append-only log of events. SQLite runs one writer at a +-- time, so events commit in the order they are appended, and an event's +-- position is its seq alone: a consumer reads the log past the last seq it +-- has seen, and nothing committed later can sort before it. + +CREATE TABLE hopper_stream_events ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + topic TEXT NOT NULL, + key TEXT, -- becomes the delivery's ordering key + payload TEXT NOT NULL DEFAULT 'null', + headers TEXT NOT NULL DEFAULT '{}', + message_id BLOB NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE INDEX hopper_stream_events_created ON hopper_stream_events (created_at); + +CREATE TABLE hopper_stream_consumers ( + name TEXT PRIMARY KEY, + pattern TEXT NOT NULL, -- AMQP topic syntax + kind TEXT NOT NULL, -- the delivery job kind + queue TEXT NOT NULL, + max_attempts INTEGER, + metadata TEXT NOT NULL DEFAULT '{}', + seen INTEGER NOT NULL DEFAULT 0, -- every event up to this seq has been read + position INTEGER NOT NULL DEFAULT 0, -- the last event delivered + delivered_at INTEGER, + created_at INTEGER NOT NULL +); diff --git a/driver/hoppersqlite/migrations/006_autovacuum.down.sql b/driver/hoppersqlite/migrations/006_autovacuum.down.sql new file mode 100644 index 0000000..59ffeec --- /dev/null +++ b/driver/hoppersqlite/migrations/006_autovacuum.down.sql @@ -0,0 +1 @@ +-- Nothing to revert: v6 changes nothing on SQLite. diff --git a/driver/hoppersqlite/migrations/006_autovacuum.up.sql b/driver/hoppersqlite/migrations/006_autovacuum.up.sql new file mode 100644 index 0000000..0564685 --- /dev/null +++ b/driver/hoppersqlite/migrations/006_autovacuum.up.sql @@ -0,0 +1,3 @@ +-- hopper schema v6: autovacuum settings for Postgres's live table. SQLite +-- has no autovacuum to tune: the leader refreshes the planner's statistics +-- when the live table's size changes (JobsMaintain). diff --git a/driver/hoppersqlite/migrations/007_vacuum_truncate.down.sql b/driver/hoppersqlite/migrations/007_vacuum_truncate.down.sql new file mode 100644 index 0000000..9e5e853 --- /dev/null +++ b/driver/hoppersqlite/migrations/007_vacuum_truncate.down.sql @@ -0,0 +1 @@ +-- Nothing to revert: v7 changes nothing on SQLite. diff --git a/driver/hoppersqlite/migrations/007_vacuum_truncate.up.sql b/driver/hoppersqlite/migrations/007_vacuum_truncate.up.sql new file mode 100644 index 0000000..b33d791 --- /dev/null +++ b/driver/hoppersqlite/migrations/007_vacuum_truncate.up.sql @@ -0,0 +1,2 @@ +-- hopper schema v7: no heap truncation on Postgres's live table. SQLite +-- reuses the pages of deleted rows, and never runs a VACUUM on its own. diff --git a/driver/hoppersqlite/migrations/008_function_schema.down.sql b/driver/hoppersqlite/migrations/008_function_schema.down.sql new file mode 100644 index 0000000..0bb276e --- /dev/null +++ b/driver/hoppersqlite/migrations/008_function_schema.down.sql @@ -0,0 +1 @@ +-- Nothing to revert: v8 changes nothing on SQLite. diff --git a/driver/hoppersqlite/migrations/008_function_schema.up.sql b/driver/hoppersqlite/migrations/008_function_schema.up.sql new file mode 100644 index 0000000..15e9dc7 --- /dev/null +++ b/driver/hoppersqlite/migrations/008_function_schema.up.sql @@ -0,0 +1,2 @@ +-- hopper schema v8: binds Postgres's SQL producer functions to their +-- schema. SQLite has no such functions. diff --git a/driver/hoppersqlite/migrations/009_snooze_reason.down.sql b/driver/hoppersqlite/migrations/009_snooze_reason.down.sql new file mode 100644 index 0000000..c76ec6b --- /dev/null +++ b/driver/hoppersqlite/migrations/009_snooze_reason.down.sql @@ -0,0 +1 @@ +ALTER TABLE hopper_jobs DROP COLUMN snooze_reason; diff --git a/driver/hoppersqlite/migrations/009_snooze_reason.up.sql b/driver/hoppersqlite/migrations/009_snooze_reason.up.sql new file mode 100644 index 0000000..9fb0965 --- /dev/null +++ b/driver/hoppersqlite/migrations/009_snooze_reason.up.sql @@ -0,0 +1,8 @@ +-- hopper schema v9: why a snoozed job is waiting. +-- +-- A worker can give a reason when it snoozes a job. The finalize stores it +-- with the snooze, and the claim, every other finalize, a retry and a +-- replacing insert clear it, so it only ever describes the current wait. +-- History has no such column: a finalized job is not waiting. + +ALTER TABLE hopper_jobs ADD COLUMN snooze_reason TEXT; diff --git a/driver/hoppersqlite/migrations/010_wake_requested.down.sql b/driver/hoppersqlite/migrations/010_wake_requested.down.sql new file mode 100644 index 0000000..6e0e6cb --- /dev/null +++ b/driver/hoppersqlite/migrations/010_wake_requested.down.sql @@ -0,0 +1 @@ +ALTER TABLE hopper_jobs DROP COLUMN wake_requested_at; diff --git a/driver/hoppersqlite/migrations/010_wake_requested.up.sql b/driver/hoppersqlite/migrations/010_wake_requested.up.sql new file mode 100644 index 0000000..bba92de --- /dev/null +++ b/driver/hoppersqlite/migrations/010_wake_requested.up.sql @@ -0,0 +1,9 @@ +-- hopper schema v10: a wake that finds its key's job running. +-- +-- JobRetryOrderingKey brings a key's waiting jobs forward, and marks a job +-- of the key that is running at the time. If that job then snoozes or +-- fails, the finalize makes it runnable at once instead of after its delay, +-- and clears the mark. Only a mark set since the job's current attempt +-- started counts. History has no such column. + +ALTER TABLE hopper_jobs ADD COLUMN wake_requested_at INTEGER; diff --git a/driver/hoppersqlite/migrations/011_batch_completion.down.sql b/driver/hoppersqlite/migrations/011_batch_completion.down.sql new file mode 100644 index 0000000..ff8892f --- /dev/null +++ b/driver/hoppersqlite/migrations/011_batch_completion.down.sql @@ -0,0 +1 @@ +-- Nothing to revert: v11 changes nothing on SQLite. diff --git a/driver/hoppersqlite/migrations/011_batch_completion.up.sql b/driver/hoppersqlite/migrations/011_batch_completion.up.sql new file mode 100644 index 0000000..88d95dd --- /dev/null +++ b/driver/hoppersqlite/migrations/011_batch_completion.up.sql @@ -0,0 +1,5 @@ +-- hopper schema v11: on Postgres, batch progress in rows of its own, so +-- that the concurrent finalizes of a batch's jobs do not queue on its row. +-- SQLite runs one writer at a time: the transaction that finalizes a +-- batch's jobs counts them down on the batch row, and nothing queues there +-- that would not queue anyway. diff --git a/driver/hoppersqlite/migrations/012_canceled_state.down.sql b/driver/hoppersqlite/migrations/012_canceled_state.down.sql new file mode 100644 index 0000000..b244346 --- /dev/null +++ b/driver/hoppersqlite/migrations/012_canceled_state.down.sql @@ -0,0 +1 @@ +-- Nothing to revert: v12 changes nothing on SQLite. diff --git a/driver/hoppersqlite/migrations/012_canceled_state.up.sql b/driver/hoppersqlite/migrations/012_canceled_state.up.sql new file mode 100644 index 0000000..6f112fe --- /dev/null +++ b/driver/hoppersqlite/migrations/012_canceled_state.up.sql @@ -0,0 +1,2 @@ +-- hopper schema v12: Postgres's canceled state is spelled canceled. SQLite's +-- schema has spelled it so from v1. diff --git a/driver/hoppersqlite/migrations/013_history_ordering.down.sql b/driver/hoppersqlite/migrations/013_history_ordering.down.sql new file mode 100644 index 0000000..4215c5e --- /dev/null +++ b/driver/hoppersqlite/migrations/013_history_ordering.down.sql @@ -0,0 +1 @@ +DROP INDEX hopper_job_history_ordering; diff --git a/driver/hoppersqlite/migrations/013_history_ordering.up.sql b/driver/hoppersqlite/migrations/013_history_ordering.up.sql new file mode 100644 index 0000000..c011e61 --- /dev/null +++ b/driver/hoppersqlite/migrations/013_history_ordering.up.sql @@ -0,0 +1,4 @@ +-- hopper schema v13: list a key's finished jobs without reading all history. + +CREATE INDEX hopper_job_history_ordering ON hopper_job_history (ordering_key, id) + WHERE ordering_key IS NOT NULL; diff --git a/driver/hoppersqlite/migrations/014_history_unique.down.sql b/driver/hoppersqlite/migrations/014_history_unique.down.sql new file mode 100644 index 0000000..8a29894 --- /dev/null +++ b/driver/hoppersqlite/migrations/014_history_unique.down.sql @@ -0,0 +1 @@ +DROP INDEX hopper_job_history_unique; diff --git a/driver/hoppersqlite/migrations/014_history_unique.up.sql b/driver/hoppersqlite/migrations/014_history_unique.up.sql new file mode 100644 index 0000000..2565c55 --- /dev/null +++ b/driver/hoppersqlite/migrations/014_history_unique.up.sql @@ -0,0 +1,5 @@ +-- hopper schema v14: find a unique key's finished jobs without reading all +-- history, for an insert whose UniqueOpts.Finished names finished states. + +CREATE INDEX hopper_job_history_unique ON hopper_job_history (kind, unique_key, finalized_at) + WHERE unique_key IS NOT NULL; diff --git a/driver/hoppersqlite/migrations/015_job_progress.down.sql b/driver/hoppersqlite/migrations/015_job_progress.down.sql new file mode 100644 index 0000000..96c86a1 --- /dev/null +++ b/driver/hoppersqlite/migrations/015_job_progress.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE hopper_jobs DROP COLUMN progress; +ALTER TABLE hopper_job_history DROP COLUMN progress; diff --git a/driver/hoppersqlite/migrations/015_job_progress.up.sql b/driver/hoppersqlite/migrations/015_job_progress.up.sql new file mode 100644 index 0000000..32aa871 --- /dev/null +++ b/driver/hoppersqlite/migrations/015_job_progress.up.sql @@ -0,0 +1,8 @@ +-- hopper schema v15: a job's progress through its steps. +-- +-- A worker records the last step it finished, and optionally a cursor within +-- the next, so that a later attempt resumes there. The progress survives +-- retries, snoozes and a JobRetry from history, so history keeps it too. + +ALTER TABLE hopper_job_history ADD COLUMN progress TEXT; +ALTER TABLE hopper_jobs ADD COLUMN progress TEXT; diff --git a/driver/hoppersqlite/migrations/016_job_logs.down.sql b/driver/hoppersqlite/migrations/016_job_logs.down.sql new file mode 100644 index 0000000..73cd280 --- /dev/null +++ b/driver/hoppersqlite/migrations/016_job_logs.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE hopper_jobs DROP COLUMN logs; +ALTER TABLE hopper_job_history DROP COLUMN logs; diff --git a/driver/hoppersqlite/migrations/016_job_logs.up.sql b/driver/hoppersqlite/migrations/016_job_logs.up.sql new file mode 100644 index 0000000..293b6e7 --- /dev/null +++ b/driver/hoppersqlite/migrations/016_job_logs.up.sql @@ -0,0 +1,6 @@ +-- hopper schema v16: the log lines of a job's latest attempts, written by +-- the transaction that finalizes each attempt and kept through retries and +-- a JobRetry from history, so history keeps them too. + +ALTER TABLE hopper_job_history ADD COLUMN logs TEXT; +ALTER TABLE hopper_jobs ADD COLUMN logs TEXT; diff --git a/driver/hoppersqlite/migrations/017_job_tags.down.sql b/driver/hoppersqlite/migrations/017_job_tags.down.sql new file mode 100644 index 0000000..eb3ef1f --- /dev/null +++ b/driver/hoppersqlite/migrations/017_job_tags.down.sql @@ -0,0 +1,10 @@ +DROP TRIGGER hopper_job_history_tags_update; +DROP TRIGGER hopper_job_history_tags_delete; +DROP TRIGGER hopper_job_history_tags_insert; +DROP TRIGGER hopper_jobs_tags_update; +DROP TRIGGER hopper_jobs_tags_delete; +DROP TRIGGER hopper_jobs_tags_insert; +DROP TABLE hopper_job_history_tags; +DROP TABLE hopper_job_tags; +ALTER TABLE hopper_jobs DROP COLUMN tags; +ALTER TABLE hopper_job_history DROP COLUMN tags; diff --git a/driver/hoppersqlite/migrations/017_job_tags.up.sql b/driver/hoppersqlite/migrations/017_job_tags.up.sql new file mode 100644 index 0000000..4c1db13 --- /dev/null +++ b/driver/hoppersqlite/migrations/017_job_tags.up.sql @@ -0,0 +1,72 @@ +-- hopper schema v17: tags, for finding jobs by label. +-- +-- A job's tags are a JSON array of strings on its row, NULL when it has +-- none, and they are kept in history like the rest of the job. A CHECK keeps +-- a direct SQL write from storing anything but an array; a read skips what +-- in one is not a non-empty string, once each. +-- +-- JobList finds a tag's jobs through tables of (tag, id) rows, one for live +-- jobs and one for history, whose primary keys walk a tag's jobs in ID +-- order. Triggers keep them in step with the tags column on every write, +-- whoever makes it. + +ALTER TABLE hopper_job_history ADD COLUMN tags TEXT CHECK (tags IS NULL OR json_type(tags) = 'array'); +ALTER TABLE hopper_jobs ADD COLUMN tags TEXT CHECK (tags IS NULL OR json_type(tags) = 'array'); + +CREATE TABLE hopper_job_tags ( + tag TEXT NOT NULL, + id BLOB NOT NULL, + PRIMARY KEY (tag, id) +) WITHOUT ROWID; + +CREATE TABLE hopper_job_history_tags ( + tag TEXT NOT NULL, + id BLOB NOT NULL, + PRIMARY KEY (tag, id) +) WITHOUT ROWID; + +CREATE TRIGGER hopper_jobs_tags_insert AFTER INSERT ON hopper_jobs +WHEN NEW.tags IS NOT NULL +BEGIN + INSERT OR IGNORE INTO hopper_job_tags (tag, id) + SELECT value, NEW.id FROM json_each(NEW.tags) WHERE type = 'text' AND value <> ''; +END; + +CREATE TRIGGER hopper_jobs_tags_delete AFTER DELETE ON hopper_jobs +WHEN OLD.tags IS NOT NULL +BEGIN + DELETE FROM hopper_job_tags + WHERE id = OLD.id AND tag IN (SELECT value FROM json_each(OLD.tags) WHERE type = 'text'); +END; + +CREATE TRIGGER hopper_jobs_tags_update AFTER UPDATE OF tags, id ON hopper_jobs +WHEN OLD.tags IS NOT NEW.tags OR OLD.id IS NOT NEW.id +BEGIN + DELETE FROM hopper_job_tags + WHERE OLD.tags IS NOT NULL AND id = OLD.id AND tag IN (SELECT value FROM json_each(OLD.tags) WHERE type = 'text'); + INSERT OR IGNORE INTO hopper_job_tags (tag, id) + SELECT value, NEW.id FROM json_each(coalesce(NEW.tags, '[]')) WHERE type = 'text' AND value <> ''; +END; + +CREATE TRIGGER hopper_job_history_tags_insert AFTER INSERT ON hopper_job_history +WHEN NEW.tags IS NOT NULL +BEGIN + INSERT OR IGNORE INTO hopper_job_history_tags (tag, id) + SELECT value, NEW.id FROM json_each(NEW.tags) WHERE type = 'text' AND value <> ''; +END; + +CREATE TRIGGER hopper_job_history_tags_delete AFTER DELETE ON hopper_job_history +WHEN OLD.tags IS NOT NULL +BEGIN + DELETE FROM hopper_job_history_tags + WHERE id = OLD.id AND tag IN (SELECT value FROM json_each(OLD.tags) WHERE type = 'text'); +END; + +CREATE TRIGGER hopper_job_history_tags_update AFTER UPDATE OF tags, id ON hopper_job_history +WHEN OLD.tags IS NOT NEW.tags OR OLD.id IS NOT NEW.id +BEGIN + DELETE FROM hopper_job_history_tags + WHERE OLD.tags IS NOT NULL AND id = OLD.id AND tag IN (SELECT value FROM json_each(OLD.tags) WHERE type = 'text'); + INSERT OR IGNORE INTO hopper_job_history_tags (tag, id) + SELECT value, NEW.id FROM json_each(coalesce(NEW.tags, '[]')) WHERE type = 'text' AND value <> ''; +END; diff --git a/driver/hoppersqlite/migrations/018_claim_fence.down.sql b/driver/hoppersqlite/migrations/018_claim_fence.down.sql new file mode 100644 index 0000000..7c68de9 --- /dev/null +++ b/driver/hoppersqlite/migrations/018_claim_fence.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE hopper_job_history DROP COLUMN fence; +ALTER TABLE hopper_jobs DROP COLUMN fence; diff --git a/driver/hoppersqlite/migrations/018_claim_fence.up.sql b/driver/hoppersqlite/migrations/018_claim_fence.up.sql new file mode 100644 index 0000000..51afb18 --- /dev/null +++ b/driver/hoppersqlite/migrations/018_claim_fence.up.sql @@ -0,0 +1,11 @@ +-- hopper schema v18: a fencing number per claim. +-- +-- Every claim increments a job's fence, so that a result, a progress write +-- or a rescue applies only to the claim it belongs to: a job rescued and +-- claimed again by the same client, or snoozed and claimed again at the +-- same attempt number, has a new fence. Snoozes, retries and the archive +-- keep it, and a JobRetry from history carries it back, so for a given job +-- it never repeats. History has the column for that reason. + +ALTER TABLE hopper_jobs ADD COLUMN fence INTEGER NOT NULL DEFAULT 0; +ALTER TABLE hopper_job_history ADD COLUMN fence INTEGER NOT NULL DEFAULT 0; diff --git a/driver/hoppersqlite/workflow.go b/driver/hoppersqlite/workflow.go new file mode 100644 index 0000000..429ab6a --- /dev/null +++ b/driver/hoppersqlite/workflow.go @@ -0,0 +1,190 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "slices" + + "github.com/parallelworks/hopper/driver" +) + +// WorkflowInsert implements driver.Executor. +func (e *executor) WorkflowInsert(ctx context.Context, params driver.WorkflowInsertParams) (*driver.WorkflowInsertResult, error) { + n := len(params.Jobs) + if n == 0 { + return nil, errors.New("hopper: insert workflow: no jobs") + } + pending := make([]bool, n) + for _, d := range params.Deps { + if d.Job < 0 || d.Job >= n || d.DependsOn < 0 || d.DependsOn >= n || d.Job == d.DependsOn { + return nil, fmt.Errorf("hopper: insert workflow: dependency %d -> %d is out of range", d.Job, d.DependsOn) + } + pending[d.Job] = true + } + jobs := make([]jobInsert, n) + var notifyQueues []string + for i, p := range params.Jobs { + if p.UniqueKey != "" { + return nil, fmt.Errorf("hopper: insert workflow: job %d has a unique key", i) + } + var err error + if jobs[i], err = encodeInsert(p); err != nil { + return nil, err + } + jobs[i].pending = pending[i] + if !pending[i] && !slices.Contains(notifyQueues, p.Queue) { + notifyQueues = append(notifyQueues, p.Queue) + } + } + batch, err := encodeBatch(params.OnSuccess, params.OnFailure, params.OnComplete, params.Metadata) + if err != nil { + return nil, err + } + res := &driver.WorkflowInsertResult{} + err = e.write(ctx, func(x *w) error { + for i := range jobs { + jobs[i].id = newID(x.now) + } + // The graph, for inspection, and the working set of edges. + edges := make([][2]string, len(params.Deps)) + for i, d := range params.Deps { + edges[i] = [2]string{jobs[d.Job].id.String(), jobs[d.DependsOn].id.String()} + } + edgesJSON, err := json.Marshal(edges) + if err != nil { + return err + } + res.ID = newID(x.now) + for i := range jobs { + jobs[i].batchID = res.ID + } + if err := x.insertBatch(ctx, res.ID, params.Name, n, batch, string(edgesJSON)); err != nil { + return fmt.Errorf("insert batch: %w", err) + } + res.Jobs = make([]driver.JobInsertResult, n) + for i, j := range jobs { + job, err := x.insertJob(ctx, j) + if err != nil { + return fmt.Errorf("insert jobs: %w", err) + } + res.Jobs[i] = driver.JobInsertResult{Job: job} + } + for _, d := range params.Deps { + policy := d.OnFailure + if policy == "" { + policy = driver.DependencyCancel + } + if _, err := x.ExecContext(ctx, `INSERT OR IGNORE INTO hopper_job_deps (job_id, depends_on, on_failure) VALUES (?, ?, ?)`, + idParam(jobs[d.Job].id), idParam(jobs[d.DependsOn].id), string(policy)); err != nil { + return fmt.Errorf("insert dependencies: %w", err) + } + } + if params.Notify { + return x.notify(ctx, driver.ChannelInsert, notifyQueues...) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("hopper: insert workflow: %w", err) + } + return res, nil +} + +// WorkflowGet implements driver.Executor. +func (e *executor) WorkflowGet(ctx context.Context, id driver.JobID) (*driver.WorkflowRow, error) { + var w driver.WorkflowRow + err := e.readTx(ctx, func(q querier) error { + var ( + name sql.NullString + edges sql.NullString + ) + b, err := scanBatch(q.QueryRowContext(ctx, `SELECT `+batchColumns+`, name, edges FROM hopper_batches WHERE id = ?`, idParam(id)), &name, &edges) + if errors.Is(err, sql.ErrNoRows) { + return driver.ErrNotFound + } + if err != nil { + return err + } + w.Batch = *b + w.Batch.Name = name.String + if edges.Valid { + var pairs [][2]driver.JobID + if err := json.Unmarshal([]byte(edges.String), &pairs); err != nil { + return fmt.Errorf("edges: %w", err) + } + w.Edges = make([]driver.JobEdge, len(pairs)) + for i, p := range pairs { + w.Edges[i] = driver.JobEdge{Job: p[0], DependsOn: p[1]} + } + } + // Jobs in insertion order, live and finished alike. + w.Jobs, err = queryAll(ctx, q, func(r scanner) (*driver.JobRow, error) { + var seq int64 + return scanJob(r, true, &seq) + }, `SELECT * FROM ( + SELECT `+liveAsHistory+`, seq FROM hopper_jobs WHERE batch_id = ?1 + UNION ALL + SELECT `+historyColumns+`, seq FROM hopper_job_history WHERE batch_id = ?1 + ) ORDER BY seq`, idParam(id)) + return err + }) + if err != nil { + if errors.Is(err, driver.ErrNotFound) { + return nil, err + } + return nil, fmt.Errorf("hopper: get workflow: %w", err) + } + return &w, nil +} + +// WorkflowPromoteReady implements driver.Executor. The transaction that +// finalizes a job promotes its dependents itself, so this finds only what +// a direct write left pending. +func (e *executor) WorkflowPromoteReady(ctx context.Context, params driver.WorkflowPromoteParams) (driver.WorkflowPromoteResult, error) { + if params.DependsOn != nil && len(params.DependsOn) == 0 { + return driver.WorkflowPromoteResult{}, nil + } + var res driver.WorkflowPromoteResult + err := e.write(ctx, func(x *w) error { + res = driver.WorkflowPromoteResult{} + var ( + candidates []driver.JobID + err error + ) + if params.DependsOn == nil { + candidates, err = x.ids(ctx, `SELECT DISTINCT job_id FROM hopper_job_deps WHERE job_id > ? ORDER BY job_id LIMIT ?`, + idParam(params.After), max(params.Limit, 1)) + } else { + ids := make([]any, len(params.DependsOn)) + for i, id := range params.DependsOn { + ids[i] = idParam(id) + } + candidates, err = x.ids(ctx, `SELECT DISTINCT job_id FROM hopper_job_deps WHERE depends_on IN (`+placeholders(len(ids))+`)`, ids...) + } + if err != nil { + return err + } + sortIDs(candidates) + res.Scanned = int64(len(candidates)) + if len(candidates) > 0 { + res.Last = candidates[len(candidates)-1] + } + for _, id := range candidates { + promoted, err := x.promoteIfReady(ctx, id) + if err != nil { + return err + } + if promoted { + res.Promoted++ + } + } + return nil + }) + if err != nil { + return driver.WorkflowPromoteResult{}, fmt.Errorf("hopper: promote ready workflow jobs: %w", err) + } + return res, nil +} From 04478d36ab892b6d7560b3158000979128f3858a Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 15:15:29 +0000 Subject: [PATCH 05/20] perf(hoppersqlite): keep statements prepared, and promote dependents without the write lock unless one is ready modernc.org/sqlite parses every statement it is given anew, which took almost half of a finalize's time: statements are now prepared once on the pool and bound to a transaction's connection with Tx.StmtContext, which took throughput from 4.1k to 11.3k jobs/s. WorkflowPromoteReady, which the finalizer calls after every flush of workflow jobs, looks for ready dependents first, and takes the write lock only to promote them. A metadata filter that is not JSON is refused as Postgres refuses it. --- driver/hoppersqlite/executor.go | 132 ++++++++++++++++++++++++--- driver/hoppersqlite/finalize.go | 6 +- driver/hoppersqlite/funcs.go | 11 ++- driver/hoppersqlite/go.sum | 10 ++ driver/hoppersqlite/hoppersqlite.go | 18 ++-- driver/hoppersqlite/jobs.go | 2 +- driver/hoppersqlite/localfs_linux.go | 5 +- driver/hoppersqlite/workflow.go | 72 +++++++++------ 8 files changed, 202 insertions(+), 54 deletions(-) diff --git a/driver/hoppersqlite/executor.go b/driver/hoppersqlite/executor.go index ec44974..799ea00 100644 --- a/driver/hoppersqlite/executor.go +++ b/driver/hoppersqlite/executor.go @@ -8,6 +8,7 @@ import ( "fmt" "slices" "strings" + "sync" "time" "github.com/parallelworks/hopper/driver" @@ -42,10 +43,117 @@ func clock() time.Time { return time.Now().UTC().Truncate(time.Microsecond) } // its time, read once the lock was taken, so that every write sees a time // no earlier than the writes before it. type w struct { - querier + cached now time.Time } +// maxStatements bounds the statements a Driver keeps prepared. Listings, +// whose text varies with their filters, may otherwise add one for each +// shape of filter. +const maxStatements = 256 + +// statements holds a Driver's prepared statements by text. database/sql +// keeps each prepared on every connection it has run on, and binds it to a +// transaction's connection with Tx.StmtContext, so SQLite parses each text +// once a connection, rather than once a run, which cost most of a +// finalize's time. +type statements struct { + mu sync.Mutex + m map[string]*sql.Stmt +} + +// prepared returns the prepared statement of a query, preparing it on db +// the first time, or nil for a script of several statements, or once +// maxStatements are kept. +func (s *statements) prepared(ctx context.Context, db *sql.DB, query string) (*sql.Stmt, error) { + if strings.Contains(query, ";") { + return nil, nil + } + s.mu.Lock() + defer s.mu.Unlock() + if st, ok := s.m[query]; ok { + return st, nil + } + if len(s.m) >= maxStatements { + return nil, nil + } + st, err := db.PrepareContext(ctx, query) //nolint:sqlclosecheck // kept for the Driver's life; Close closes it + if err != nil { + return nil, err + } + if s.m == nil { + s.m = map[string]*sql.Stmt{} + } + s.m[query] = st + return st, nil +} + +func (s *statements) close() error { + s.mu.Lock() + defer s.mu.Unlock() + var errs []error + for _, st := range s.m { + errs = append(errs, st.Close()) + } + s.m = nil + return errors.Join(errs...) +} + +// cached runs statements on a querier, the pool or a transaction, through +// the Driver's prepared statements. +type cached struct { + q querier + d *Driver +} + +var _ querier = cached{} + +// stmt returns the prepared statement of query bound to the querier, or +// nil to run the text as it is. +func (c cached) stmt(ctx context.Context, query string) (*sql.Stmt, error) { + st, err := c.d.statements.prepared(ctx, c.d.sqlDB, query) + if err != nil || st == nil { + return nil, err + } + if tx, ok := c.q.(*sql.Tx); ok { + return tx.StmtContext(ctx, st), nil + } + return st, nil +} + +func (c cached) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) { + st, err := c.stmt(ctx, query) //nolint:sqlclosecheck // the cache's, or bound to a transaction that closes it + if err != nil { + return nil, err + } + if st == nil { + return c.q.ExecContext(ctx, query, args...) + } + return st.ExecContext(ctx, args...) +} + +func (c cached) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + st, err := c.stmt(ctx, query) //nolint:sqlclosecheck // the cache's, or bound to a transaction that closes it + if err != nil { + return nil, err + } + if st == nil { + return c.q.QueryContext(ctx, query, args...) + } + return st.QueryContext(ctx, args...) +} + +// QueryRowContext runs a query that returns at most one row. A statement +// that fails to prepare reports its error through Scan, as the query's +// would. +func (c cached) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row { + st, err := c.stmt(ctx, query) //nolint:sqlclosecheck // the cache's, or bound to a transaction that closes it + if err != nil || st == nil { + return c.q.QueryRowContext(ctx, query, args...) + } + return st.QueryRowContext(ctx, args...) +} + // us is the write's time in the unit the schema keeps times in. func (x *w) us() int64 { return x.now.UnixMicro() } @@ -67,7 +175,7 @@ func (e *executor) writeBrief(ctx context.Context, fn func(*w) error) error { func (e *executor) writeLocking(ctx context.Context, wait time.Duration, fn func(*w) error) (err error) { // The first statement takes the lock, under its own deadline when the // wait is bounded. - lock := func(q querier, statement string) error { + lock := func(q cached, statement string) error { lctx := ctx if wait > 0 { var cancel context.CancelFunc @@ -81,7 +189,8 @@ func (e *executor) writeLocking(ctx context.Context, wait time.Duration, fn func return err } if e.tx != nil { - if err := lock(e.tx, "SAVEPOINT hopper"); err != nil { + q := cached{q: e.tx, d: e.d} + if err := lock(q, "SAVEPOINT hopper"); err != nil { return err } defer func() { @@ -90,10 +199,10 @@ func (e *executor) writeLocking(ctx context.Context, wait time.Duration, fn func err = errors.Join(err, rerr) } }() - if err := fn(&w{querier: e.tx, now: clock()}); err != nil { + if err := fn(&w{cached: q, now: clock()}); err != nil { return err } - _, err := e.tx.ExecContext(ctx, "RELEASE hopper") + _, err := q.ExecContext(ctx, "RELEASE hopper") return err } tx, err := e.d.sqlDB.BeginTx(ctx, nil) @@ -105,10 +214,11 @@ func (e *executor) writeLocking(ctx context.Context, wait time.Duration, fn func err = errors.Join(err, ignoreDone(tx.Rollback())) } }() - if err := lock(tx, "SELECT 1"); err != nil { + q := cached{q: tx, d: e.d} + if err := lock(q, "SELECT 1"); err != nil { return err } - if err := fn(&w{querier: tx, now: clock()}); err != nil { + if err := fn(&w{cached: q, now: clock()}); err != nil { return err } return tx.Commit() @@ -125,23 +235,23 @@ func ignoreDone(err error) error { // on which each statement reads the latest committed state. func (e *executor) read() querier { if e.tx != nil { - return e.tx + return cached{q: e.tx, d: e.d} } - return e.d.sqlDB + return cached{q: e.d.sqlDB, d: e.d} } // readTx runs fn on one snapshot of the database: the caller's // transaction, or a read-only one. func (e *executor) readTx(ctx context.Context, fn func(querier) error) (err error) { if e.tx != nil { - return fn(e.tx) + return fn(cached{q: e.tx, d: e.d}) } tx, err := e.d.sqlDB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) if err != nil { return err } defer func() { err = errors.Join(err, ignoreDone(tx.Rollback())) }() - return fn(tx) + return fn(cached{q: tx, d: e.d}) } // notifySQL writes a notification. conn.go recognizes the statement by its diff --git a/driver/hoppersqlite/finalize.go b/driver/hoppersqlite/finalize.go index c96632a..2d3573f 100644 --- a/driver/hoppersqlite/finalize.go +++ b/driver/hoppersqlite/finalize.go @@ -438,7 +438,11 @@ func (x *w) tail(ctx context.Context, done []finished) ([]finished, error) { // ids returns the IDs a query selects. func (x *w) ids(ctx context.Context, query string, args ...any) ([]driver.JobID, error) { - return queryAll(ctx, x, func(r scanner) (driver.JobID, error) { + return queryIDs(ctx, x, query, args...) +} + +func queryIDs(ctx context.Context, q querier, query string, args ...any) ([]driver.JobID, error) { + return queryAll(ctx, q, func(r scanner) (driver.JobID, error) { var b []byte if err := r.Scan(&b); err != nil { return driver.JobID{}, err diff --git a/driver/hoppersqlite/funcs.go b/driver/hoppersqlite/funcs.go index da81b09..0800684 100644 --- a/driver/hoppersqlite/funcs.go +++ b/driver/hoppersqlite/funcs.go @@ -4,7 +4,9 @@ import ( "bytes" sqldriver "database/sql/driver" "encoding/json" + "errors" "fmt" + "io" "regexp" "strconv" "strings" @@ -279,6 +281,9 @@ func checkJSONFilter(s string) error { if !utf8.ValidString(s) { return fmt.Errorf("the metadata filter is not UTF-8") } + if !json.Valid([]byte(s)) { + return fmt.Errorf("the metadata filter is not JSON") + } if err := checkEscapes(s); err != nil { return err } @@ -286,10 +291,10 @@ func checkJSONFilter(s string) error { dec.UseNumber() for { tok, err := dec.Token() + if errors.Is(err, io.EOF) { + return nil + } if err != nil { - if err.Error() == "EOF" { - return nil - } return fmt.Errorf("the metadata filter is not JSON: %w", err) } n, ok := tok.(json.Number) diff --git a/driver/hoppersqlite/go.sum b/driver/hoppersqlite/go.sum index 8b50aa6..9e6b7d9 100644 --- a/driver/hoppersqlite/go.sum +++ b/driver/hoppersqlite/go.sum @@ -6,6 +6,14 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= @@ -18,6 +26,8 @@ golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg= diff --git a/driver/hoppersqlite/hoppersqlite.go b/driver/hoppersqlite/hoppersqlite.go index b0f53d5..dc3d88b 100644 --- a/driver/hoppersqlite/hoppersqlite.go +++ b/driver/hoppersqlite/hoppersqlite.go @@ -19,16 +19,15 @@ // through SQLite's busy handler. A claim is exclusive without SKIP // LOCKED, and the transaction that finalizes a job settles its batch // and promotes its dependents itself. A finalize with SkipLocked, or a -// progress write with NoWait, waits briefly (lockWaitBrief) for a write -// lock held elsewhere, and then skips every job, or returns -// driver.ErrJobLocked. +// progress write with NoWait, waits 100ms for a write lock held +// elsewhere, and then skips every job, or returns driver.ErrJobLocked. // - Transactions come from DB, whose connections take the write lock // when a transaction runs its first statement, not when it begins, so -// that what it reads is never stale by the time it writes (see -// conn.go). A transaction begun read-only takes no lock and may not -// write. Within a transaction, write through it (InsertTx and the -// like), not through the client: a write on the pool would wait for -// the transaction's own lock. +// that what it reads is never stale by the time it writes, and so that +// beginning one never blocks. A transaction begun read-only takes no +// lock and may not write. Within a transaction, write through it +// (InsertTx and the like), not through the client: a write on the pool +// would wait for the transaction's own lock. // - The database must be on a local filesystem: Open refuses one on a // network filesystem with ErrNetworkFilesystem. Its clock is therefore // the host's, which every process that opens it shares, and the driver @@ -91,6 +90,7 @@ type Driver struct { sqlDB *sql.DB db *database listenInterval time.Duration + statements statements } var _ driver.Driver[*sql.Tx] = (*Driver)(nil) @@ -160,7 +160,7 @@ func (d *Driver) DB() *sql.DB { return d.sqlDB } // Close closes the database. func (d *Driver) Close() error { - err := d.sqlDB.Close() + err := errors.Join(d.statements.close(), d.sqlDB.Close()) d.db.close() return err } diff --git a/driver/hoppersqlite/jobs.go b/driver/hoppersqlite/jobs.go index f96d74c..fe72a73 100644 --- a/driver/hoppersqlite/jobs.go +++ b/driver/hoppersqlite/jobs.go @@ -651,7 +651,7 @@ func (e *executor) JobCancel(ctx context.Context, id driver.JobID) (*driver.JobR // txOf returns the transaction a write runs in, for a read within it. func txOf(x *w) *sql.Tx { - tx, _ := x.querier.(*sql.Tx) + tx, _ := x.q.(*sql.Tx) return tx } diff --git a/driver/hoppersqlite/localfs_linux.go b/driver/hoppersqlite/localfs_linux.go index 5d9cdc0..3b8fd38 100644 --- a/driver/hoppersqlite/localfs_linux.go +++ b/driver/hoppersqlite/localfs_linux.go @@ -33,9 +33,12 @@ var networkMagic = map[int64]string{ 0x65735546: "FUSE", } +// statfs is unix.Statfs, which tests replace. +var statfs = unix.Statfs + func networkFilesystem(dir string) (string, error) { var st unix.Statfs_t - if err := unix.Statfs(dir, &st); err != nil { + if err := statfs(dir, &st); err != nil { return "", err } if name, ok := networkMagic[int64(st.Type)&0xffffffff]; ok { //nolint:unconvert // st.Type is an int32 or a uint32 on some architectures diff --git a/driver/hoppersqlite/workflow.go b/driver/hoppersqlite/workflow.go index 429ab6a..80811a0 100644 --- a/driver/hoppersqlite/workflow.go +++ b/driver/hoppersqlite/workflow.go @@ -140,39 +140,55 @@ func (e *executor) WorkflowGet(ctx context.Context, id driver.JobID) (*driver.Wo return &w, nil } +// readySQL selects, of the IDs it is given, the pending jobs none of whose +// dependencies is still live. +const readySQL = `SELECT j.id FROM hopper_jobs j +WHERE j.id IN (%s) AND j.state = 'pending' + AND NOT EXISTS (SELECT 1 FROM hopper_job_deps d JOIN hopper_jobs x ON x.id = d.depends_on WHERE d.job_id = j.id)` + // WorkflowPromoteReady implements driver.Executor. The transaction that -// finalizes a job promotes its dependents itself, so this finds only what -// a direct write left pending. +// finalizes a job promotes its dependents itself, so a pass finds only +// what a direct write left pending: it looks without the write lock, and +// takes the lock only to promote what it found. func (e *executor) WorkflowPromoteReady(ctx context.Context, params driver.WorkflowPromoteParams) (driver.WorkflowPromoteResult, error) { if params.DependsOn != nil && len(params.DependsOn) == 0 { return driver.WorkflowPromoteResult{}, nil } - var res driver.WorkflowPromoteResult - err := e.write(ctx, func(x *w) error { - res = driver.WorkflowPromoteResult{} - var ( - candidates []driver.JobID - err error - ) - if params.DependsOn == nil { - candidates, err = x.ids(ctx, `SELECT DISTINCT job_id FROM hopper_job_deps WHERE job_id > ? ORDER BY job_id LIMIT ?`, - idParam(params.After), max(params.Limit, 1)) - } else { - ids := make([]any, len(params.DependsOn)) - for i, id := range params.DependsOn { - ids[i] = idParam(id) - } - candidates, err = x.ids(ctx, `SELECT DISTINCT job_id FROM hopper_job_deps WHERE depends_on IN (`+placeholders(len(ids))+`)`, ids...) - } - if err != nil { - return err - } - sortIDs(candidates) - res.Scanned = int64(len(candidates)) - if len(candidates) > 0 { - res.Last = candidates[len(candidates)-1] - } - for _, id := range candidates { + var ( + res driver.WorkflowPromoteResult + candidates []driver.JobID + err error + ) + if params.DependsOn == nil { + candidates, err = queryIDs(ctx, e.read(), `SELECT DISTINCT job_id FROM hopper_job_deps WHERE job_id > ? ORDER BY job_id LIMIT ?`, + idParam(params.After), max(params.Limit, 1)) + } else { + ids := make([]any, len(params.DependsOn)) + for i, id := range params.DependsOn { + ids[i] = idParam(id) + } + candidates, err = queryIDs(ctx, e.read(), `SELECT DISTINCT job_id FROM hopper_job_deps WHERE depends_on IN (`+placeholders(len(ids))+`)`, ids...) + } + if err != nil { + return res, fmt.Errorf("hopper: promote ready workflow jobs: %w", err) + } + sortIDs(candidates) + res.Scanned = int64(len(candidates)) + if len(candidates) == 0 { + return res, nil + } + res.Last = candidates[len(candidates)-1] + ids := make([]any, len(candidates)) + for i, id := range candidates { + ids[i] = idParam(id) + } + ready, err := queryIDs(ctx, e.read(), fmt.Sprintf(readySQL, placeholders(len(ids))), ids...) + if err != nil || len(ready) == 0 { + return res, err + } + err = e.write(ctx, func(x *w) error { + res.Promoted = 0 + for _, id := range ready { promoted, err := x.promoteIfReady(ctx, id) if err != nil { return err From e0ed93575ab49f1df65bf7bfa5968c70ad516958 Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 15:15:29 +0000 Subject: [PATCH 06/20] test(hoppersqlite): the conformance suite, transactions and the write lock, claims and notifications across processes, and upgrades from every schema version under traffic --- driver/hoppersqlite/conformance_test.go | 28 +- driver/hoppersqlite/funcs_test.go | 145 ++++++ driver/hoppersqlite/localfs_linux_test.go | 33 ++ driver/hoppersqlite/migrate_test.go | 349 ++++++++++++++ driver/hoppersqlite/sqlite_test.go | 539 ++++++++++++++++++++++ 5 files changed, 1083 insertions(+), 11 deletions(-) create mode 100644 driver/hoppersqlite/funcs_test.go create mode 100644 driver/hoppersqlite/localfs_linux_test.go create mode 100644 driver/hoppersqlite/migrate_test.go create mode 100644 driver/hoppersqlite/sqlite_test.go diff --git a/driver/hoppersqlite/conformance_test.go b/driver/hoppersqlite/conformance_test.go index 9232bfc..6659b42 100644 --- a/driver/hoppersqlite/conformance_test.go +++ b/driver/hoppersqlite/conformance_test.go @@ -19,7 +19,13 @@ import ( // directory, migrated to the latest schema. func open(t testing.TB) *hoppersqlite.Driver { t.Helper() - d, err := hoppersqlite.Open(filepath.Join(t.TempDir(), "hopper.db"), nil) + return openAt(t, filepath.Join(t.TempDir(), "hopper.db")) +} + +// openAt is open on the database at path. +func openAt(t testing.TB, path string) *hoppersqlite.Driver { + t.Helper() + d, err := hoppersqlite.Open(path, nil) if err != nil { t.Fatal(err) } @@ -39,7 +45,7 @@ func db(d driver.Driver[*sql.Tx]) *sql.DB { return d.(*hoppersqlite.Driver).DB() func now() int64 { return time.Now().UnixMicro() } -func exec(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) { +func execSQL(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) { t.Helper() if _, err := db(d).ExecContext(ctx, query, args...); err != nil { t.Error(err) @@ -76,19 +82,19 @@ func (fixture) Begin(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx] } func (fixture) ExpireLease(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], clientID int64) { - exec(ctx, t, d, "UPDATE hopper_clients SET expires_at = ? WHERE id = ?", now()-1e6, clientID) + execSQL(ctx, t, d, "UPDATE hopper_clients SET expires_at = ? WHERE id = ?", now()-1e6, clientID) } func (fixture) ExpireLeader(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) { - exec(ctx, t, d, "UPDATE hopper_leader SET expires_at = ?", now()-1e6) + execSQL(ctx, t, d, "UPDATE hopper_leader SET expires_at = ?", now()-1e6) } func (fixture) MakeClaimable(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { - exec(ctx, t, d, "UPDATE hopper_jobs SET scheduled_at = ? WHERE id = ?", now(), id[:]) + execSQL(ctx, t, d, "UPDATE hopper_jobs SET scheduled_at = ? WHERE id = ?", now(), id[:]) } func (fixture) AgeAttempt(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID, age time.Duration) { - exec(ctx, t, d, "UPDATE hopper_jobs SET attempted_at = ? WHERE id = ? AND attempted_at IS NOT NULL", now()-age.Microseconds(), id[:]) + execSQL(ctx, t, d, "UPDATE hopper_jobs SET attempted_at = ? WHERE id = ? AND attempted_at IS NOT NULL", now()-age.Microseconds(), id[:]) } func (fixture) CountLive(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) int { @@ -100,15 +106,15 @@ func (fixture) CountHistory(ctx context.Context, t *testing.T, d driver.Driver[* } func (fixture) ExpireJob(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { - exec(ctx, t, d, "UPDATE hopper_jobs SET expires_at = ? WHERE id = ?", now()-1e6, id[:]) + execSQL(ctx, t, d, "UPDATE hopper_jobs SET expires_at = ? WHERE id = ?", now()-1e6, id[:]) } func (fixture) StaleWake(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { - exec(ctx, t, d, "UPDATE hopper_jobs SET wake_requested_at = attempted_at - 1000000 WHERE id = ?", id[:]) + execSQL(ctx, t, d, "UPDATE hopper_jobs SET wake_requested_at = attempted_at - 1000000 WHERE id = ?", id[:]) } func (fixture) MakePending(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { - exec(ctx, t, d, "UPDATE hopper_jobs SET state = 'pending' WHERE id = ?", id[:]) + execSQL(ctx, t, d, "UPDATE hopper_jobs SET state = 'pending' WHERE id = ?", id[:]) } func (fixture) SnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) string { @@ -116,11 +122,11 @@ func (fixture) SnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[* } func (fixture) SetSnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID, reason string) { - exec(ctx, t, d, "UPDATE hopper_jobs SET snooze_reason = ? WHERE id = ?", reason, id[:]) + execSQL(ctx, t, d, "UPDATE hopper_jobs SET snooze_reason = ? WHERE id = ?", reason, id[:]) } func (fixture) SetProgress(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID, value string) { - exec(ctx, t, d, "UPDATE hopper_jobs SET progress = ? WHERE id = ?", value, id[:]) + execSQL(ctx, t, d, "UPDATE hopper_jobs SET progress = ? WHERE id = ?", value, id[:]) } func (fixture) CountEdges(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) int { diff --git a/driver/hoppersqlite/funcs_test.go b/driver/hoppersqlite/funcs_test.go new file mode 100644 index 0000000..3253b67 --- /dev/null +++ b/driver/hoppersqlite/funcs_test.go @@ -0,0 +1,145 @@ +package hoppersqlite + +import ( + "encoding/binary" + "strings" + "testing" + "time" +) + +// TestTopicPatterns checks AMQP patterns against topics, with the results +// Postgres's hopper_topic_regex gives, odd patterns included. +func TestTopicPatterns(t *testing.T) { + for _, c := range []struct { + pattern, topic string + want bool + }{ + {"#", "", true}, + {"#", "a.b.c", true}, + {"a.*", "a.b", true}, + {"a.*", "a", false}, + {"a.*", "a.b.c", false}, + {"a.*", "a.", false}, + {"a.#", "a", true}, + {"a.#", "a.b.c", true}, + {"a.#", "ab", false}, + {"#.b", "b", true}, + {"#.b", "a.x.b", true}, + {"#.b", "ab", false}, + {"a.#.b", "a.b", true}, + {"a.#.b", "a.x.y.b", true}, + {"a.#.b", "a.xb", false}, + {"*.created", "order.created", true}, + {"*.created", "order.paid", false}, + {"a#b", "axyzb", true}, // '#' between letters is '.*', as on Postgres + {"a+b", "a+b", true}, + {"a+b", "aab", false}, + {"(x)", "(x)", true}, + {"a.b", "a.b", true}, + {"a.b", "axb", false}, + {"#", "line\nbreak", true}, + } { + re, err := topicRegexp(c.pattern) + if err != nil { + t.Fatalf("%q: %v", c.pattern, err) + } + if got := re.MatchString(c.topic); got != c.want { + t.Errorf("%q matches %q = %v, want %v", c.pattern, c.topic, got, c.want) + } + } +} + +// TestJSONContains checks containment as jsonb's @> has it. +func TestJSONContains(t *testing.T) { + for _, c := range []struct { + have, want string + contains bool + }{ + {`{"a": 1, "b": 2}`, `{"a": 1}`, true}, + {`{"a": 1}`, `{"a": 1, "b": 2}`, false}, + {`{"a": 1.0}`, `{"a": 1}`, true}, + {`{"a": 100}`, `{"a": 1e2}`, true}, + {`{"a": 0.10}`, `{"a": 1e-1}`, true}, + {`{"a": -0}`, `{"a": 0}`, true}, + {`{"a": "1"}`, `{"a": 1}`, false}, + {`{"a": {"b": [1, 2, 3]}}`, `{"a": {"b": [3, 1]}}`, true}, + {`{"a": [1, 2]}`, `{"a": [1, 1, 1]}`, true}, + {`{"a": [1, 2]}`, `{"a": [4]}`, false}, + {`{"a": [[1, 3], 2]}`, `{"a": [[3]]}`, true}, + {`{"a": [[1, 3], 2]}`, `{"a": [3]}`, false}, + {`{"a": null}`, `{"a": null}`, true}, + {`{"a": false}`, `{"a": null}`, false}, + {`{}`, `{}`, true}, + {`["x", "y"]`, `"x"`, true}, + {`{"a": ["x"]}`, `{"a": "x"}`, false}, + {`[1]`, `{}`, false}, + } { + have, err := decodeJSON(c.have) + if err != nil { + t.Fatal(err) + } + want, err := decodeJSON(c.want) + if err != nil { + t.Fatal(err) + } + if got := jsonContains(have, want, true); got != c.contains { + t.Errorf("%s @> %s = %v, want %v", c.have, c.want, got, c.contains) + } + } +} + +// TestJSONFilterChecks checks that a metadata filter is refused where +// Postgres's jsonb input refuses it, and only there. +func TestJSONFilterChecks(t *testing.T) { + for filter, refused := range map[string]bool{ + `{"a": 1}`: false, + `{"a": "é😀"}`: false, + `{"a": 1e131071}`: false, + `{"a": 1e131072}`: true, + `{"a": 1e1000000}`: true, + `{"a": 1e-16383}`: false, + `{"a": 1e-16384}`: true, + `{"a": 12.5e-16382}`: false, + `{"a": 12.5e-16383}`: true, + `{"a": "\u0000"}`: true, + `{"a": "x\\u0000"}`: false, + `{"a": "\ud800x"}`: true, + `{"a": "\udc00\ud800"}`: true, + "{\"a\": \"\xff\"}": true, + `{"a": 1`: true, + `{"a": 99999999999999999999e-20}`: false, + } { + err := checkJSONFilter(filter) + if (err != nil) != refused { + t.Errorf("checkJSONFilter(%s) = %v, want refused %v", filter, err, refused) + } + if err != nil && strings.Contains(err.Error(), "Metadata") { + t.Errorf("checkJSONFilter(%s) names the field: %v", filter, err) + } + } +} + +// TestNewID checks that IDs are UUIDv7s of the time they were minted, in +// order across milliseconds. +func TestNewID(t *testing.T) { + now := time.Date(2026, 10, 22, 10, 0, 0, 500_000, time.UTC) + id := newID(now) + if v := id[6] >> 4; v != 7 { + t.Errorf("version %d", v) + } + if v := id[8] >> 6; v != 2 { + t.Errorf("variant %b", v) + } + if ms := int64(binary.BigEndian.Uint64(id[0:8]) >> 16); ms != now.UnixMilli() { + t.Errorf("time %d, want %d", ms, now.UnixMilli()) + } + if fraction := binary.BigEndian.Uint16(id[6:8]) & 0x0fff; fraction != 2048 { + t.Errorf("fraction %d, want 2048 (half a millisecond)", fraction) + } + if later := newID(now.Add(time.Millisecond)); string(later[:]) <= string(id[:]) { + t.Errorf("%s does not sort after %s", later, id) + } + if again := newID(now); again == id { + t.Error("two IDs minted at one time are equal") + } +} diff --git a/driver/hoppersqlite/localfs_linux_test.go b/driver/hoppersqlite/localfs_linux_test.go new file mode 100644 index 0000000..2c57b97 --- /dev/null +++ b/driver/hoppersqlite/localfs_linux_test.go @@ -0,0 +1,33 @@ +package hoppersqlite + +import ( + "errors" + "strings" + "testing" + + "golang.org/x/sys/unix" +) + +// TestNetworkFilesystem checks that a directory on NFS, Lustre, GPFS or +// FUSE is refused, naming the filesystem, and one on a local filesystem is +// not. +func TestNetworkFilesystem(t *testing.T) { + if err := checkLocal(t.TempDir() + "/hopper.db"); err != nil { + t.Fatalf("a temporary directory: %v", err) + } + defer func(real func(string, *unix.Statfs_t) error) { statfs = real }(statfs) + for magic, name := range map[int64]string{0x6969: "NFS", 0x0bd00bd0: "Lustre", 0x47504653: "GPFS", 0x65735546: "FUSE", 0xff534d42: "CIFS"} { + statfs = func(_ string, st *unix.Statfs_t) error { + st.Type = magic + return nil + } + err := checkLocal("/home/someone/hopper.db") + if !errors.Is(err, ErrNetworkFilesystem) || !strings.Contains(err.Error(), name) || !strings.Contains(err.Error(), "/home/someone") { + t.Errorf("a directory on %s: %v", name, err) + } + } + statfs = func(string, *unix.Statfs_t) error { return unix.ENOENT } + if err := checkLocal("/nowhere/hopper.db"); err == nil || errors.Is(err, ErrNetworkFilesystem) { + t.Errorf("a directory that cannot be read: %v", err) + } +} diff --git a/driver/hoppersqlite/migrate_test.go b/driver/hoppersqlite/migrate_test.go new file mode 100644 index 0000000..420f403 --- /dev/null +++ b/driver/hoppersqlite/migrate_test.go @@ -0,0 +1,349 @@ +package hoppersqlite_test + +import ( + "context" + "crypto/rand" + "encoding/json" + "fmt" + "log/slog" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/parallelworks/hopper" + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/hoppersqlite" + "github.com/parallelworks/hopper/hoppermigrate" +) + +var quiet = &hoppermigrate.Options{Logger: slog.New(slog.DiscardHandler)} + +// openEmpty returns a driver on a new database without hopper's schema. +func openEmpty(t *testing.T) *hoppersqlite.Driver { + t.Helper() + d, err := hoppersqlite.Open(filepath.Join(t.TempDir(), "hopper.db"), nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = d.Close() }) + return d +} + +// objects counts hopper's tables, indexes and triggers in the database. +func objects(t *testing.T, d *hoppersqlite.Driver) int { + t.Helper() + var n int + if err := d.DB().QueryRow(`SELECT count(*) FROM sqlite_schema WHERE name LIKE 'hopper%'`).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +// TestMigrate applies every version, reverts them all and applies them +// again: each script, up and down, runs, and down to zero leaves nothing of +// hopper's behind. +func TestMigrate(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := openEmpty(t) + if v, err := hoppermigrate.Version(ctx, d); err != nil || v != 0 { + t.Fatalf("version of an empty database = %d, %v", v, err) + } + if v, err := d.Executor().SchemaVersion(ctx); err != nil || v != 0 { + t.Fatalf("schema version of an empty database = %d, %v", v, err) + } + for range 2 { + res, err := hoppermigrate.Up(ctx, d, quiet) + if err != nil || res.Version != hoppermigrate.Latest() || len(res.Applied) != hoppermigrate.Latest() { + t.Fatalf("Up = %+v, %v", res, err) + } + if res, err := hoppermigrate.Up(ctx, d, quiet); err != nil || len(res.Applied) != 0 { + t.Fatalf("a second Up = %+v, %v", res, err) + } + if v, err := d.Executor().SchemaVersion(ctx); err != nil || v != hoppermigrate.Latest() { + t.Fatalf("schema version = %d, %v", v, err) + } + if _, err := hoppermigrate.Down(ctx, d, quiet); err != nil { + t.Fatalf("Down: %v", err) + } + if n := objects(t, d); n != 0 { + t.Fatalf("%d of hopper's objects left after Down to zero", n) + } + } + // Down and up one version at a time. + if _, err := hoppermigrate.Up(ctx, d, quiet); err != nil { + t.Fatal(err) + } + for v := hoppermigrate.Latest() - 1; v >= 0; v-- { + if _, err := hoppermigrate.Down(ctx, d, &hoppermigrate.Options{Logger: quiet.Logger, Target: v}); err != nil { + t.Fatalf("Down to %d: %v", v, err) + } + } + for v := 1; v <= hoppermigrate.Latest(); v++ { + if _, err := hoppermigrate.Up(ctx, d, &hoppermigrate.Options{Logger: quiet.Logger, Target: v}); err != nil { + t.Fatalf("Up to %d: %v", v, err) + } + } +} + +// TestUpConcurrent runs Up from several Drivers on one database at once: +// one applies the migrations, and the others find them applied. +func TestUpConcurrent(t *testing.T) { + t.Parallel() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "hopper.db") + var applied atomic.Int64 + var wg sync.WaitGroup + for range 4 { + d, err := hoppersqlite.Open(path, nil) + if err != nil { + t.Fatal(err) + } + defer d.Close() + wg.Go(func() { + res, err := hoppermigrate.Up(ctx, d, quiet) + if err != nil { + t.Error(err) + return + } + applied.Add(int64(len(res.Applied))) + }) + } + wg.Wait() + if n := applied.Load(); n != int64(hoppermigrate.Latest()) { + t.Errorf("applied %d migrations in all, want %d", n, hoppermigrate.Latest()) + } +} + +type upgradeJob struct { + N int `json:"n"` +} + +func (upgradeJob) Kind() string { return "upgrade" } + +// TestUpgradeUnderTraffic migrates to the latest version from every +// earlier one, the upgrades a release can need, while jobs wait in every +// state and inserts, claims and finalizes keep running, and then has a +// client work every job. Those statements are written as a client of an +// earlier release would write them, with the columns of the first version, +// which every later one accepts. The migration holds the write lock while +// it runs, as every transaction that writes does, so they wait for it; none +// of them may fail, and no job may be lost. +func TestUpgradeUnderTraffic(t *testing.T) { + t.Parallel() + for from := 1; from < hoppermigrate.Latest(); from++ { + t.Run(fmt.Sprintf("from_v%d", from), func(t *testing.T) { + t.Parallel() + upgradeUnderTraffic(t, from) + }) + } +} + +func upgradeUnderTraffic(t *testing.T, from int) { + ctx := context.Background() + d := openEmpty(t) + if _, err := hoppermigrate.Up(ctx, d, &hoppermigrate.Options{Logger: quiet.Logger, Target: from}); err != nil { + t.Fatal(err) + } + db := d.DB() + now := time.Now().UnixMicro() + var next atomic.Int64 + insert := func(state string, scheduledAt int64) (driver.JobID, error) { + var id driver.JobID + _, _ = rand.Read(id[:]) + args, _ := json.Marshal(upgradeJob{N: int(next.Add(1))}) + _, err := db.ExecContext(ctx, `INSERT INTO hopper_jobs (id, kind, queue, state, priority, max_attempts, scheduled_at, args, created_at) + VALUES (?, 'upgrade', 'default', ?, 2, 5, ?, ?, ?)`, id[:], state, scheduledAt, string(args), time.Now().UnixMicro()) + return id, err + } + // Jobs in every state, as an earlier release leaves them: available, + // scheduled, retryable, running on a client that has since died, and + // finished in history. + for _, state := range []string{"available", "scheduled", "retryable"} { + at := now + if state == "scheduled" { + at = now + time.Second.Microseconds() + } + for range 5 { + if _, err := insert(state, at); err != nil { + t.Fatal(err) + } + } + } + var dead int64 + if err := db.QueryRowContext(ctx, `INSERT INTO hopper_clients (hostname, started_at, expires_at, info) VALUES ('gone', ?, ?, '{}') RETURNING id`, + now-time.Hour.Microseconds(), now-time.Minute.Microseconds()).Scan(&dead); err != nil { + t.Fatal(err) + } + for range 5 { + id, err := insert("available", now) + if err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, `UPDATE hopper_jobs SET state = 'running', attempt = 1, attempted_at = ?, attempted_by = ? WHERE id = ?`, + now, dead, id[:]); err != nil { + t.Fatal(err) + } + } + finished := func(id driver.JobID) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck // committed below + if _, err := tx.ExecContext(ctx, `INSERT INTO hopper_job_history (id, seq, kind, queue, state, priority, attempt, max_attempts, + scheduled_at, args, metadata, errors, await, created_at, finalized_at) + SELECT id, seq, kind, queue, 'completed', priority, attempt, max_attempts, scheduled_at, args, metadata, errors, await, + created_at, ? FROM hopper_jobs WHERE id = ?`, time.Now().UnixMicro(), id[:]); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM hopper_jobs WHERE id = ?`, id[:]); err != nil { + return err + } + return tx.Commit() + } + for range 3 { + id, err := insert("available", now) + if err != nil { + t.Fatal(err) + } + if err := finished(id); err != nil { + t.Fatal(err) + } + } + + // Traffic while the migration runs: a producer inserting, and a client + // of the earlier release claiming and finishing jobs with statements + // every version accepts. A claimed job's number is recorded as worked. + var ( + mu sync.Mutex + worked = map[int]int{} + longest time.Duration + stop = make(chan struct{}) + wg sync.WaitGroup + ) + timed := func(fn func() error) error { + start := time.Now() + err := fn() + mu.Lock() + longest = max(longest, time.Since(start)) + mu.Unlock() + return err + } + wg.Go(func() { + for { + select { + case <-stop: + return + default: + } + if err := timed(func() error { _, err := insert("available", time.Now().UnixMicro()); return err }); err != nil { + t.Error(err) + return + } + time.Sleep(time.Millisecond) + } + }) + wg.Go(func() { + for { + select { + case <-stop: + return + default: + } + err := timed(func() error { + var ( + id []byte + args string + ) + err := db.QueryRowContext(ctx, `UPDATE hopper_jobs SET state = 'running', attempt = attempt + 1, attempted_at = ?, attempted_by = ? + WHERE seq = (SELECT seq FROM hopper_jobs WHERE state = 'available' AND scheduled_at <= ? ORDER BY seq LIMIT 1) + RETURNING id, args`, time.Now().UnixMicro(), dead, time.Now().UnixMicro()).Scan(&id, &args) + if err != nil { + return nil //nolint:nilerr // nothing to claim + } + var job upgradeJob + if err := json.Unmarshal([]byte(args), &job); err != nil { + return err + } + mu.Lock() + worked[job.N]++ + mu.Unlock() + return finished(driver.JobID(id)) + }) + if err != nil { + t.Error(err) + return + } + time.Sleep(time.Millisecond) + } + }) + time.Sleep(20 * time.Millisecond) + res, err := hoppermigrate.Up(ctx, d, quiet) + if err != nil { + t.Fatal(err) + } + if res.Version != hoppermigrate.Latest() { + t.Fatalf("Up = %+v", res) + } + time.Sleep(20 * time.Millisecond) + close(stop) + wg.Wait() + if longest > 5*time.Second { + t.Errorf("a statement waited %s for the migration", longest) + } + + // The release's client works every job left, the running ones once + // rescued from the dead client. + workers := hopper.NewWorkers() + hopper.AddWorkFunc(workers, func(_ context.Context, job *hopper.Job[upgradeJob]) error { + mu.Lock() + worked[job.Args.N]++ + mu.Unlock() + return nil + }) + c, err := hopper.NewClient(d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: 10}}, + Workers: workers, + PollInterval: 20 * time.Millisecond, + Logger: slog.New(slog.DiscardHandler), + }) + if err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + defer c.Stop(ctx) //nolint:errcheck // stopped below + total := int(next.Load()) + for deadline := time.Now().Add(30 * time.Second); ; time.Sleep(20 * time.Millisecond) { + var live int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM hopper_jobs`).Scan(&live); err != nil { + t.Fatal(err) + } + if live == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("%d jobs still live", live) + } + } + if err := c.Stop(ctx); err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + // The finished jobs inserted before the migration were never worked. + if len(worked) != total-3 { + t.Errorf("%d jobs worked, want %d", len(worked), total-3) + } + for n, times := range worked { + // A job running on the dead client was worked by it before (not + // recorded) and by the rescue after. + if times != 1 { + t.Errorf("job %d worked %d times", n, times) + } + } +} diff --git a/driver/hoppersqlite/sqlite_test.go b/driver/hoppersqlite/sqlite_test.go new file mode 100644 index 0000000..bd18cb1 --- /dev/null +++ b/driver/hoppersqlite/sqlite_test.go @@ -0,0 +1,539 @@ +package hoppersqlite_test + +import ( + "bufio" + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/parallelworks/hopper" + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/hoppersqlite" +) + +func TestOpenRefuses(t *testing.T) { + t.Parallel() + dir := t.TempDir() + for _, c := range []struct { + path string + cfg *hoppersqlite.Config + }{ + {"", nil}, + {":memory:", nil}, + {"file:" + filepath.Join(dir, "x.db"), nil}, + {filepath.Join(dir, "x.db"), &hoppersqlite.Config{Synchronous: "OFF"}}, + {filepath.Join(dir, "x.db"), &hoppersqlite.Config{ListenInterval: -time.Second}}, + {filepath.Join(dir, "missing", "x.db"), nil}, + } { + if d, err := hoppersqlite.Open(c.path, c.cfg); err == nil { + _ = d.Close() + t.Errorf("Open(%q, %+v) succeeded", c.path, c.cfg) + } + } +} + +// TestOpen opens one file twice in a process, as two clients of one +// application could: both see each other's writes, share one write lock, +// and keep the database in write-ahead log mode. +func TestOpen(t *testing.T) { + t.Parallel() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "hopper db?.sqlite") + first, err := hoppersqlite.Open(path, &hoppersqlite.Config{Synchronous: "normal"}) + if err != nil { + t.Fatal(err) + } + defer first.Close() + second, err := hoppersqlite.Open(path, nil) + if err != nil { + t.Fatal(err) + } + defer second.Close() + var mode string + if err := second.DB().QueryRowContext(ctx, "PRAGMA journal_mode").Scan(&mode); err != nil || mode != "wal" { + t.Errorf("journal mode = %q, %v", mode, err) + } + if _, err := first.DB().ExecContext(ctx, "CREATE TABLE t (x INTEGER)"); err != nil { + t.Fatal(err) + } + tx, err := first.DB().BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err != nil { + t.Fatal(err) + } + wctx, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + if _, err := second.DB().ExecContext(wctx, "INSERT INTO t VALUES (2)"); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("a write through the other Driver while the first's transaction is open = %v, want it to wait", err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + if _, err := second.DB().ExecContext(ctx, "INSERT INTO t VALUES (2)"); err != nil { + t.Fatal(err) + } + var n int + if err := first.DB().QueryRowContext(ctx, "SELECT count(*) FROM t").Scan(&n); err != nil || n != 2 { + t.Errorf("rows = %d, %v", n, err) + } +} + +// TestTransactions checks how the driver's transactions take the write +// lock: not when they begin but on their first statement, which therefore +// never reads what another transaction changes before it writes; and a +// read-only one not at all. +func TestTransactions(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + db := d.DB() + if _, err := db.ExecContext(ctx, "CREATE TABLE counter (n INTEGER); INSERT INTO counter VALUES (0)"); err != nil { + t.Fatal(err) + } + holder, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := holder.ExecContext(ctx, "UPDATE counter SET n = n + 1"); err != nil { + t.Fatal(err) + } + + // Begin returns at once while another transaction writes. + start := time.Now() + waiter, err := db.BeginTx(ctx, nil) + if err != nil || time.Since(start) > time.Second { + t.Fatalf("BeginTx while another transaction writes = %v after %s", err, time.Since(start)) + } + // A read-only transaction reads the last committed state meanwhile, and + // may not write. + ro, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + t.Fatal(err) + } + var n int + if err := ro.QueryRowContext(ctx, "SELECT n FROM counter").Scan(&n); err != nil || n != 0 { + t.Errorf("read-only read = %d, %v", n, err) + } + if _, err := ro.ExecContext(ctx, "UPDATE counter SET n = 100"); err == nil { + t.Error("a read-only transaction wrote") + } + if err := ro.Rollback(); err != nil { + t.Fatal(err) + } + // The waiter's first statement, a read, waits for the lock as long as + // its context allows. + short, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + if err := waiter.QueryRowContext(short, "SELECT n FROM counter").Scan(&n); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("first statement while another transaction writes = %v, want it to wait", err) + } + // Once the holder commits, the read sees its write, and a write after + // it succeeds: what it read cannot have gone stale. + done := make(chan error, 1) + go func() { + var seen int + if err := waiter.QueryRowContext(ctx, "SELECT n FROM counter").Scan(&seen); err != nil { + done <- err + return + } + if seen != 1 { + done <- fmt.Errorf("read %d after the holder committed, want 1", seen) + return + } + _, err := waiter.ExecContext(ctx, "UPDATE counter SET n = ?", seen+1) + done <- err + }() + for hoppersqlite.LockWaiters(d) == 0 { + time.Sleep(time.Millisecond) + } + if err := holder.Commit(); err != nil { + t.Fatal(err) + } + if err := <-done; err != nil { + t.Fatal(err) + } + if err := waiter.Commit(); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, "SELECT n FROM counter").Scan(&n); err != nil || n != 2 { + t.Errorf("counter = %d, %v; want 2", n, err) + } +} + +type countArgs struct { + N int `json:"n"` +} + +func (countArgs) Kind() string { return "count" } + +// TestClient runs hopper's client on the driver: two clients work jobs, +// one with an awaited result and one completed in the worker's own +// transaction, and a workflow, whose steps run after their dependencies. +func TestClient(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + var ( + mu sync.Mutex + done = map[int]int{} + ) + workers := hopper.NewWorkers() + hopper.AddWorkFunc(workers, func(ctx context.Context, job *hopper.Job[countArgs]) error { + mu.Lock() + done[job.Args.N]++ + mu.Unlock() + if job.Args.N == -1 { + return hopper.SetOutput(ctx, map[string]int{"answer": 42}) + } + if job.Args.N == -2 { + client, ok := hopper.ClientFromContext[*sql.Tx](ctx) + if !ok { + return errors.New("no client in the worker's context") + } + tx, err := d.DB().BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck // committed below + if _, err := client.InsertTx(ctx, tx, countArgs{N: -3}, nil); err != nil { + return err + } + if err := hopper.CompleteTx(ctx, tx, job); err != nil { + return err + } + return tx.Commit() + } + return nil + }) + clients := make([]*hopper.Client[*sql.Tx], 2) + for i := range clients { + c, err := hopper.NewClient(d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: 10}}, + Workers: workers, + PollInterval: 50 * time.Millisecond, + }) + if err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + defer c.Stop(ctx) //nolint:errcheck // cleanup + clients[i] = c + } + params := make([]hopper.InsertParams, 500) + for i := range params { + params[i] = hopper.InsertParams{Args: countArgs{N: i}} + } + if _, err := clients[0].InsertMany(ctx, params); err != nil { + t.Fatal(err) + } + awaited, err := clients[1].Insert(ctx, countArgs{N: -1}, &hopper.InsertOpts{Await: true}) + if err != nil { + t.Fatal(err) + } + if _, err := clients[1].Insert(ctx, countArgs{N: -2}, nil); err != nil { + t.Fatal(err) + } + wf := hopper.NewWorkflow("chain", nil) + first := wf.Add(countArgs{N: 1000}, nil) + second := wf.Add(countArgs{N: 1001}, hopper.After(first)) + wf.Add(countArgs{N: 1002}, hopper.After(second)) + flow, err := clients[0].InsertWorkflow(ctx, wf) + if err != nil { + t.Fatal(err) + } + + actx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + answer, err := hopper.Await[map[string]int](actx, clients[0], awaited.Job.ID) + if err != nil || answer["answer"] != 42 { + t.Fatalf("awaited job = %+v, %v", answer, err) + } + for deadline := time.Now().Add(30 * time.Second); ; time.Sleep(20 * time.Millisecond) { + mu.Lock() + finished := len(done) + mu.Unlock() + if finished == 504+1 { // 500, -1, -2, -3 and the workflow's three + break + } + if time.Now().After(deadline) { + t.Fatalf("%d jobs worked, want %d", finished, 505) + } + } + mu.Lock() + for n, times := range done { + if times != 1 { + t.Errorf("job %d worked %d times", n, times) + } + } + mu.Unlock() + for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(20 * time.Millisecond) { + w, err := clients[1].WorkflowGet(ctx, flow.ID) + if err != nil { + t.Fatal(err) + } + if !w.Batch.CompletedAt.IsZero() { + var order []int + for _, j := range w.Jobs { + var args countArgs + if err := json.Unmarshal(j.Args, &args); err != nil { + t.Fatal(err) + } + order = append(order, args.N) + if j.State != hopper.JobStateCompleted { + t.Errorf("workflow job %d is %s", args.N, j.State) + } + } + if fmt.Sprint(order) != "[1000 1001 1002]" { + t.Errorf("workflow jobs = %v", order) + } + break + } + if time.Now().After(deadline) { + t.Fatalf("workflow not completed: %+v", w) + } + } +} + +// helperEnv names the role a re-executed test binary plays: one process of +// several that share a database. +const helperEnv = "HOPPERSQLITE_TEST_HELPER" + +// TestHelperProcess is not a test: TestAcrossProcesses runs the test binary +// again with helperEnv set, and this plays the role it names. +func TestHelperProcess(t *testing.T) { + role := os.Getenv(helperEnv) + if role == "" { + t.Skip("run by TestAcrossProcesses") + } + ctx := context.Background() + d, err := hoppersqlite.Open(os.Getenv("HOPPERSQLITE_TEST_DB"), nil) + if err != nil { + t.Fatal(err) + } + defer d.Close() + x := d.Executor() + switch role { + case "claim": + if err := claimAll(ctx, x, os.Stdout); err != nil { + t.Fatal(err) + } + case "notify": + if _, err := x.JobInsertMany(ctx, []driver.JobInsertParams{{Kind: "k", Queue: "from-another-process", Priority: 2, MaxAttempts: 1}}, + driver.JobInsertOpts{Notify: true}); err != nil { + t.Fatal(err) + } + case "hold": + tx, err := d.DB().BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "UPDATE hopper_queues SET updated_at = updated_at"); err != nil { + t.Fatal(err) + } + fmt.Println("holding") + time.Sleep(time.Second) + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + } +} + +// claimAll claims and completes jobs of the default queue until none is +// left, writing the ID and fence of each claim to out. +func claimAll(ctx context.Context, exec driver.Executor, out *os.File) error { + clientID, err := exec.ClientRegister(ctx, driver.ClientRegisterParams{Hostname: "test", TTL: time.Minute}) + if err != nil { + return err + } + w := bufio.NewWriter(out) + defer w.Flush() + for { + res, err := exec.JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: clientID, Limit: 7}) + if err != nil { + return err + } + if len(res.Jobs) == 0 { + return nil + } + fin := make([]driver.JobFinalize, len(res.Jobs)) + for i, j := range res.Jobs { + fmt.Fprintf(w, "claimed %s %d\n", j.ID, j.Fence) + fin[i] = driver.JobFinalize{ID: j.ID, AttemptedBy: clientID, Fence: j.Fence, State: driver.JobStateCompleted, Archive: true} + } + applied, err := exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: fin}) + if err != nil { + return err + } + if len(applied) != len(fin) { + return fmt.Errorf("applied %d of %d results", len(applied), len(fin)) + } + } +} + +// output collects a process's output, which the test reads while the +// process writes it. +type output struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (o *output) Write(p []byte) (int, error) { + o.mu.Lock() + defer o.mu.Unlock() + return o.buf.Write(p) +} + +func (o *output) String() string { + o.mu.Lock() + defer o.mu.Unlock() + return o.buf.String() +} + +// helper starts the test binary as a process playing role on the database +// at path. +func helper(t *testing.T, path, role string) (*exec.Cmd, *output) { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^TestHelperProcess$", "-test.v") + cmd.Env = append(os.Environ(), helperEnv+"="+role, "HOPPERSQLITE_TEST_DB="+path) + out := &output{} + cmd.Stdout, cmd.Stderr = out, out + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + return cmd, out +} + +// TestAcrossProcesses shares one database among processes: claims by four +// processes and by four goroutines of this one take every job exactly +// once; a notification another process commits reaches this one's +// listener; and a write waits for another process's transaction. +func TestAcrossProcesses(t *testing.T) { + t.Parallel() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "hopper.db") + d := openAt(t, path) + x := d.Executor() + + const jobs = 1200 + p := make([]driver.JobInsertParams, jobs) + for i := range p { + p[i] = driver.JobInsertParams{Kind: "k", Queue: "default", Priority: 1 + i%4, MaxAttempts: 1} + } + if _, err := x.JobInsertMany(ctx, p, driver.JobInsertOpts{}); err != nil { + t.Fatal(err) + } + type proc struct { + cmd *exec.Cmd + out *output + } + var procs []proc + for range 4 { + cmd, out := helper(t, path, "claim") + procs = append(procs, proc{cmd, out}) + } + outs := make([]*os.File, 4) + var wg sync.WaitGroup + for i := range outs { + f, err := os.Create(filepath.Join(t.TempDir(), "claims")) + if err != nil { + t.Fatal(err) + } + outs[i] = f + wg.Go(func() { + if err := claimAll(ctx, x, f); err != nil { + t.Error(err) + } + }) + } + wg.Wait() + claims := map[string]int{} + count := func(text string) { + for _, line := range strings.Split(text, "\n") { + if id, ok := strings.CutPrefix(line, "claimed "); ok { + claims[id]++ + } + } + } + for _, p := range procs { + if err := p.cmd.Wait(); err != nil { + t.Fatalf("claiming process: %v\n%s", err, p.out) + } + count(p.out.String()) + } + for _, f := range outs { + b, err := os.ReadFile(f.Name()) + if err != nil { + t.Fatal(err) + } + _ = f.Close() + count(string(b)) + } + twice := 0 + for _, n := range claims { + if n != 1 { + twice++ + } + } + if len(claims) != jobs || twice != 0 { + t.Errorf("%d jobs claimed, %d of them more than once; want %d, once each", len(claims), twice, jobs) + } + var history, fenced int + if err := d.DB().QueryRowContext(ctx, `SELECT count(*), count(*) FILTER (WHERE fence <> 1 OR attempt <> 1 OR state <> 'completed') FROM hopper_job_history`). + Scan(&history, &fenced); err != nil { + t.Fatal(err) + } + if history != jobs || fenced != 0 { + t.Errorf("history has %d jobs, %d not completed in one claim; want %d, 0", history, fenced, jobs) + } + + // A notification committed by another process. + l, err := d.Listener(ctx) + if err != nil { + t.Fatal(err) + } + defer l.Close(ctx) //nolint:errcheck // cleanup + if err := l.Listen(ctx, driver.ChannelInsert); err != nil { + t.Fatal(err) + } + cmd, out := helper(t, path, "notify") + if err := cmd.Wait(); err != nil { + t.Fatalf("notifying process: %v\n%s", err, out) + } + nctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if n, err := l.Next(nctx); err != nil || n.Payload != "from-another-process" { + t.Errorf("notification from another process = %+v, %v", n, err) + } + + // A write waits for another process's transaction, as long as its + // context allows. + cmd, out = helper(t, path, "hold") + for !strings.Contains(out.String(), "holding") { + time.Sleep(5 * time.Millisecond) + } + short, cancelShort := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancelShort() + if err := x.QueueEnsure(short, []string{"q"}); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("a write while another process's transaction is open = %v, want it to wait", err) + } + if err := x.QueueEnsure(ctx, []string{"q"}); err != nil { + t.Errorf("a write once it ends = %v", err) + } + if err := cmd.Wait(); err != nil { + t.Fatalf("holding process: %v\n%s", err, out) + } +} From 97bdb9d4f006133737cab737c65fdce7ed41289a Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 15:15:29 +0000 Subject: [PATCH 07/20] feat(hoppersqlite): hopperbench-sqlite, hopperbench's scenarios on SQLite in hopperbench's format --- Makefile | 14 + .../cmd/hopperbench-sqlite/main.go | 432 ++++++++++++++++++ 2 files changed, 446 insertions(+) create mode 100644 driver/hoppersqlite/cmd/hopperbench-sqlite/main.go diff --git a/Makefile b/Makefile index 81c0eb6..221c9aa 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,7 @@ lint: ## Lint Go code $(GOTOOL) golangci-lint run ./... cd hopperotel && go tool -modfile=../tools/go.mod golangci-lint run ./... cd hopperui && go tool -modfile=../tools/go.mod golangci-lint run ./... + cd driver/hoppersqlite && go tool -modfile=../../tools/go.mod golangci-lint run ./... .PHONY: fmt fmt: ## Format Go code @@ -58,7 +59,20 @@ test-modules: go test -race -cover -timeout 10m ./... cd hopperotel && go test -race -cover -timeout 10m ./... cd hopperui && go test -race -cover -timeout 10m ./... + cd driver/hoppersqlite && go test -race -cover -timeout 10m ./... .PHONY: bench bench: ## Run the benchmark harness against the test database go run ./cmd/hopperbench -database-url "$(HOPPER_TEST_DATABASE_URL)" + +.PHONY: bench-sqlite +bench-sqlite: ## Run the benchmark harness on a SQLite database in a temporary directory + cd driver/hoppersqlite && go run ./cmd/hopperbench-sqlite + +.PHONY: cgo-check +cgo-check: ## Check that hoppersqlite, its tests and every module they import use no cgo + @cd driver/hoppersqlite && for os in linux darwin windows freebsd openbsd netbsd; do \ + GOOS=$$os CGO_ENABLED=0 go vet ./... || exit 1; \ + pkgs=$$(GOOS=$$os CGO_ENABLED=1 go list -deps -test -f '{{if and .CgoFiles (not .Standard)}}{{.ImportPath}}{{end}}' ./...) || exit 1; \ + if [ -n "$$pkgs" ]; then echo "cgo is not allowed, but these packages use it on $$os:"; echo "$$pkgs"; exit 1; fi; \ + done diff --git a/driver/hoppersqlite/cmd/hopperbench-sqlite/main.go b/driver/hoppersqlite/cmd/hopperbench-sqlite/main.go new file mode 100644 index 0000000..87a19b0 --- /dev/null +++ b/driver/hoppersqlite/cmd/hopperbench-sqlite/main.go @@ -0,0 +1,432 @@ +// Command hopperbench-sqlite measures hopper's hot paths on SQLite, as +// hopperbench does on Postgres, and prints its results in hopperbench's +// format, one JSON object per scenario, which hopperbench -compare reads. +// +// hopperbench-sqlite [-dir /tmp] [-scenarios throughput,workflow,insert,latency] [-jobs 50000] [-clients 4] +// +// Each scenario runs on a database file of its own in a temporary directory +// under -dir, removed afterwards. -dir must be on a local disk. +// +// Scenarios: +// +// throughput no-op jobs preloaded, then drained by -clients clients +// retry as throughput, with every job failing its first attempt +// workflow as throughput, in workflows of ten steps (one, then eight after it, then one after those), +// each step recording a small output; -jobs is rounded down to a multiple of ten +// insert InsertMany in batches of 100 +// latency commit-to-Work on an idle queue, local wake-up +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "flag" + "fmt" + "log/slog" + "os" + "os/signal" + "path/filepath" + "runtime/pprof" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/parallelworks/hopper" + "github.com/parallelworks/hopper/driver/hoppersqlite" + "github.com/parallelworks/hopper/hoppermigrate" +) + +type benchArgs struct { + N int `json:"n"` +} + +func (benchArgs) Kind() string { return "bench" } + +// result is hopperbench's. +type result struct { + Scenario string `json:"scenario"` + Jobs int `json:"jobs"` + Clients int `json:"clients,omitempty"` + Workers int `json:"workers_per_client,omitempty"` + Seconds float64 `json:"seconds"` + JobsPerSec float64 `json:"jobs_per_sec,omitempty"` + P50Millis float64 `json:"p50_ms,omitempty"` + P99Millis float64 `json:"p99_ms,omitempty"` + MaxMillis float64 `json:"max_ms,omitempty"` +} + +type options struct { + jobs, clients, workers, samples int + synchronous string +} + +func main() { os.Exit(benchMain()) } + +// benchMain runs the command and returns its exit status, after its +// deferred calls, which os.Exit would skip. +func benchMain() int { + var ( + dir = flag.String("dir", os.TempDir(), "directory for the scenarios' databases, on a local disk") + scenarios = flag.String("scenarios", "throughput,workflow,insert,latency", "comma-separated scenarios to run") + jobs = flag.Int("jobs", 50000, "jobs per throughput scenario") + clients = flag.Int("clients", 4, "clients working the throughput scenarios") + workers = flag.Int("workers", 100, "MaxWorkers per client") + samples = flag.Int("samples", 200, "latency samples") + synchronous = flag.String("synchronous", "FULL", "SQLite's synchronous setting: FULL or NORMAL") + cpuProfile = flag.String("cpuprofile", "", "write a CPU profile of the run to this file") + ) + flag.Parse() + if *cpuProfile != "" { + f, err := os.Create(*cpuProfile) + if err != nil { + fmt.Fprintln(os.Stderr, "hopperbench-sqlite:", err) + return 1 + } + if err := pprof.StartCPUProfile(f); err != nil { + fmt.Fprintln(os.Stderr, "hopperbench-sqlite:", err) + return 1 + } + defer pprof.StopCPUProfile() + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + err := run(ctx, *dir, strings.Split(*scenarios, ","), options{jobs: *jobs, clients: *clients, workers: *workers, samples: *samples, synchronous: *synchronous}) + stop() + if err != nil { + fmt.Fprintln(os.Stderr, "hopperbench-sqlite:", err) + return 1 + } + return 0 +} + +func run(ctx context.Context, dir string, scenarios []string, opts options) error { + for _, s := range scenarios { + b, cleanup, err := newBench(ctx, dir, opts) + if err != nil { + return err + } + var r result + switch s = strings.TrimSpace(s); s { + case "throughput": + r, err = b.drain(ctx, "throughput", b.params(opts.jobs), noopWorker) + case "retry": + r, err = b.drain(ctx, "retry", b.params(opts.jobs), failOnceWorker) + case "workflow": + r, err = b.workflows(ctx, opts.jobs) + case "insert": + r, err = b.insert(ctx, opts.jobs, 100) + case "latency": + r, err = b.latency(ctx, opts.samples) + default: + err = fmt.Errorf("unknown scenario %q", s) + } + cleanup() + if err != nil { + return fmt.Errorf("%s: %w", s, err) + } + if err := json.NewEncoder(os.Stdout).Encode(r); err != nil { + return err + } + } + return nil +} + +type bench struct { + d *hoppersqlite.Driver + opts options + logger *slog.Logger +} + +// newBench opens a migrated database in a new directory under dir. +func newBench(ctx context.Context, dir string, opts options) (*bench, func(), error) { + tmp, err := os.MkdirTemp(dir, "hopperbench-sqlite") + if err != nil { + return nil, nil, err + } + d, err := hoppersqlite.Open(filepath.Join(tmp, "hopper.db"), &hoppersqlite.Config{Synchronous: opts.synchronous}) + if err != nil { + _ = os.RemoveAll(tmp) + return nil, nil, err + } + cleanup := func() { + _ = d.Close() + _ = os.RemoveAll(tmp) + } + if _, err := hoppermigrate.Up(ctx, d, &hoppermigrate.Options{Logger: slog.New(slog.DiscardHandler)}); err != nil { + cleanup() + return nil, nil, err + } + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})) + return &bench{d: d, opts: opts, logger: logger}, cleanup, nil +} + +func (b *bench) params(n int) []hopper.InsertParams { + params := make([]hopper.InsertParams, n) + for i := range params { + params[i] = hopper.InsertParams{Args: benchArgs{N: i}} + } + return params +} + +func noopWorker(context.Context, *hopper.Job[benchArgs]) error { return nil } + +var errFirstAttempt = errors.New("first attempt") + +// failOnceWorker fails every first attempt, so each job is finalized twice. +func failOnceWorker(_ context.Context, job *hopper.Job[benchArgs]) error { + if job.Attempt == 1 { + return errFirstAttempt + } + return nil +} + +// immediateRetry is a worker whose failed attempts retry at once. +type immediateRetry struct { + hopper.WorkerDefaults[benchArgs] + fn func(context.Context, *hopper.Job[benchArgs]) error +} + +func (w immediateRetry) Work(ctx context.Context, job *hopper.Job[benchArgs]) error { + return w.fn(ctx, job) +} + +func (immediateRetry) NextRetry(*hopper.Job[benchArgs]) time.Time { return time.Now() } + +const workflowSteps = 10 + +func (b *bench) drain(ctx context.Context, name string, params []hopper.InsertParams, work func(context.Context, *hopper.Job[benchArgs]) error) (result, error) { + return b.drainLoaded(ctx, name, len(params), func(inserter *hopper.Client[*sql.Tx]) error { + for i := 0; i < len(params); i += 10000 { + if _, err := inserter.InsertMany(ctx, params[i:min(i+10000, len(params))]); err != nil { + return err + } + } + return nil + }, work) +} + +func (b *bench) workflows(ctx context.Context, jobs int) (result, error) { + n := jobs / workflowSteps + if n < 1 { + return result{}, fmt.Errorf("-jobs must be at least %d", workflowSteps) + } + load := func(inserter *hopper.Client[*sql.Tx]) error { + return fanOut(ctx, n, 100, func(ctx context.Context, lo, hi int) error { + tx, err := b.d.DB().BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck // committed below + for i := lo; i < hi; i++ { + if _, err := inserter.InsertWorkflowTx(ctx, tx, benchWorkflow(i)); err != nil { + return err + } + } + return tx.Commit() + }) + } + r, err := b.drainLoaded(ctx, "workflow", n*workflowSteps, load, outputWorker) + if err != nil { + return result{}, err + } + var outputs, open int + if err := b.d.DB().QueryRowContext(ctx, `SELECT + (SELECT count(*) FROM hopper_job_history WHERE batch_id IS NOT NULL AND kind = 'bench' AND output IS NOT NULL), + (SELECT count(*) FROM hopper_batches WHERE completed_at IS NULL)`).Scan(&outputs, &open); err != nil { + return result{}, err + } + if outputs != r.Jobs { + return result{}, fmt.Errorf("%d of %d jobs stored an output", outputs, r.Jobs) + } + if open != 0 { + return result{}, fmt.Errorf("%d of %d workflows not completed", open, n) + } + return r, nil +} + +func benchWorkflow(i int) *hopper.Workflow { + wf := hopper.NewWorkflow("bench-"+strconv.Itoa(i), nil) + n := i * workflowSteps + root := wf.Add(benchArgs{N: n}, nil) + middle := make([]hopper.Step, workflowSteps-2) + for j := range middle { + middle[j] = wf.Add(benchArgs{N: n + 1 + j}, hopper.After(root)) + } + wf.Add(benchArgs{N: n + workflowSteps - 1}, hopper.After(middle...)) + return wf +} + +type benchOutput struct { + N int `json:"n"` + Artifact string `json:"artifact"` +} + +func outputWorker(ctx context.Context, job *hopper.Job[benchArgs]) error { + return hopper.SetOutput(ctx, benchOutput{N: job.Args.N, Artifact: "bench/" + strconv.Itoa(job.Args.N)}) +} + +// fanOut runs fn over [0, n) in chunks from eight inserters, as an +// application with several goroutines inserting would; the first error +// cancels the rest. +func fanOut(ctx context.Context, n, chunk int, fn func(ctx context.Context, lo, hi int) error) error { + const inserters = 8 + ctx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + var wg sync.WaitGroup + for w := range inserters { + wg.Go(func() { + for lo := w * chunk; lo < n && ctx.Err() == nil; lo += inserters * chunk { + if err := fn(ctx, lo, min(lo+chunk, n)); err != nil { + cancel(err) + return + } + } + }) + } + wg.Wait() + return context.Cause(ctx) +} + +func (b *bench) drainLoaded(ctx context.Context, name string, jobs int, load func(*hopper.Client[*sql.Tx]) error, work func(context.Context, *hopper.Job[benchArgs]) error) (result, error) { + inserter, err := hopper.NewClient(b.d, &hopper.Config{Logger: b.logger}) + if err != nil { + return result{}, err + } + defer inserter.Stop(ctx) //nolint:errcheck // flushes pending notifications + if err := load(inserter); err != nil { + return result{}, err + } + ws := hopper.NewWorkers() + hopper.AddWorker(ws, immediateRetry{fn: work}) + cs := make([]*hopper.Client[*sql.Tx], b.opts.clients) + for i := range cs { + c, err := hopper.NewClient(b.d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: b.opts.workers}}, + Workers: ws, + Logger: b.logger, + }) + if err != nil { + return result{}, err + } + cs[i] = c + } + defer func() { + sctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + for _, c := range cs { + _ = c.Stop(sctx) + } + }() + start := time.Now() + for _, c := range cs { + if err := c.Start(ctx); err != nil { + return result{}, err + } + } + if err := b.waitDrained(ctx); err != nil { + return result{}, err + } + elapsed := time.Since(start) + for _, c := range cs { + if err := c.Stop(ctx); err != nil { + return result{}, err + } + } + var done int + if err := b.d.DB().QueryRowContext(ctx, "SELECT count(*) FROM hopper_job_history WHERE state = 'completed' AND kind = 'bench'").Scan(&done); err != nil { + return result{}, err + } + if done != jobs { + return result{}, fmt.Errorf("%d of %d jobs completed", done, jobs) + } + return result{ + Scenario: name, Jobs: jobs, Clients: b.opts.clients, Workers: b.opts.workers, + Seconds: elapsed.Seconds(), JobsPerSec: float64(jobs) / elapsed.Seconds(), + }, nil +} + +func (b *bench) waitDrained(ctx context.Context) error { + last, changed := -1, time.Now() + for { + var live int + if err := b.d.DB().QueryRowContext(ctx, "SELECT count(*) FROM hopper_jobs").Scan(&live); err != nil { + return err + } + if live == 0 { + return nil + } + if live != last { + last, changed = live, time.Now() + } else if time.Since(changed) > 2*time.Minute { + return fmt.Errorf("no progress for two minutes with %d jobs left", live) + } + time.Sleep(20 * time.Millisecond) + } +} + +// insert measures InsertMany in batches of the given size. +func (b *bench) insert(ctx context.Context, jobs, batch int) (result, error) { + c, err := hopper.NewClient(b.d, &hopper.Config{Logger: b.logger}) + if err != nil { + return result{}, err + } + defer c.Stop(ctx) //nolint:errcheck // flushes pending notifications + params := b.params(jobs) + start := time.Now() + err = fanOut(ctx, len(params), batch, func(ctx context.Context, lo, hi int) error { + _, err := c.InsertMany(ctx, params[lo:hi]) + return err + }) + elapsed := time.Since(start) + if err != nil { + return result{}, err + } + return result{Scenario: "insert", Jobs: jobs, Seconds: elapsed.Seconds(), JobsPerSec: float64(jobs) / elapsed.Seconds()}, nil +} + +// latency measures commit-to-Work on an idle queue, with the insert coming +// from the working client itself (local wake-up). +func (b *bench) latency(ctx context.Context, samples int) (result, error) { + started := make(chan time.Time, 1) + ws := hopper.NewWorkers() + hopper.AddWorkFunc(ws, func(context.Context, *hopper.Job[benchArgs]) error { + started <- time.Now() + return nil + }) + c, err := hopper.NewClient(b.d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: 10}}, + Workers: ws, + Logger: b.logger, + }) + if err != nil { + return result{}, err + } + if err := c.Start(ctx); err != nil { + return result{}, err + } + defer c.Stop(ctx) //nolint:errcheck // best effort + lat := make([]time.Duration, 0, samples) + begin := time.Now() + for i := range samples { + t0 := time.Now() + if _, err := c.Insert(ctx, benchArgs{N: i}, nil); err != nil { + return result{}, err + } + select { + case t1 := <-started: + lat = append(lat, t1.Sub(t0)) + case <-time.After(10 * time.Second): + return result{}, errors.New("job not picked up within 10s") + } + time.Sleep(5 * time.Millisecond) + } + slices.Sort(lat) + ms := func(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 } + return result{ + Scenario: "latency", Jobs: samples, Seconds: time.Since(begin).Seconds(), + P50Millis: ms(lat[len(lat)/2]), P99Millis: ms(lat[len(lat)*99/100]), MaxMillis: ms(lat[len(lat)-1]), + }, nil +} From 7c3933d29c97877dc773941cd04501d0cd9106d2 Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 15:15:29 +0000 Subject: [PATCH 08/20] docs: the SQLite driver in the plan, the README and getting started; build, lint and test it in CI without cgo --- .github/dependabot.yml | 2 +- .github/workflows/ci.yml | 22 ++++++++++++++++++ README.md | 6 +++-- docs/PLAN.md | 47 +++++++++++++++++++++++++++++++------- docs/getting-started.md | 29 +++++++++++++++++++++++ release-please-config.json | 4 ++++ 6 files changed, 99 insertions(+), 11 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fa39351..17f8047 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,7 +3,7 @@ version: 2 # compromised release is likely to have been pulled before it is proposed here. updates: - package-ecosystem: gomod - directories: ["/", "/tools", "/tools/dev", "/hopperotel", "/hopperui"] + directories: ["/", "/tools", "/tools/dev", "/hopperotel", "/hopperui", "/driver/hoppersqlite"] schedule: { interval: weekly } cooldown: { default-days: 7 } commit-message: { prefix: "build(deps)" } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed86d88..df71218 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,11 +41,13 @@ jobs: (cd tools/dev && GOTOOLCHAIN=auto go mod tidy -diff) (cd hopperotel && go mod tidy -diff) (cd hopperui && go mod tidy -diff) + (cd driver/hoppersqlite && go mod tidy -diff) - name: Lint run: | go tool -modfile=tools/go.mod golangci-lint run ./... (cd hopperotel && go tool -modfile=../tools/go.mod golangci-lint run ./...) (cd hopperui && go tool -modfile=../tools/go.mod golangci-lint run ./...) + (cd driver/hoppersqlite && go tool -modfile=../../tools/go.mod golangci-lint run ./...) # Audits the workflows themselves: unpinned actions, script injection, # over-broad tokens, leaked credentials. Any finding fails the job. @@ -97,3 +99,23 @@ jobs: go test -race -cover -timeout 10m ./... (cd hopperotel && go test -race -cover -timeout 10m ./...) (cd hopperui && go test -race -cover -timeout 10m ./...) + + # hoppersqlite needs no server, so it runs once rather than per Postgres + # version, and proves that it and everything it imports build without cgo. + test-sqlite: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Build without cgo + run: make cgo-check + - name: Test + env: + GODEBUG: fips140=only + working-directory: driver/hoppersqlite + run: go test -race -cover -timeout 10m ./... diff --git a/README.md b/README.md index 5f5d33c..d09ef0c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,8 @@ hopper runs background jobs and delivers messages using the PostgreSQL database your application already has. There's no broker to operate and no separate worker service: it is a library that runs inside your process and scales out with it. -Postgres is the first engine, behind a driver interface designed for more. +Postgres is the first engine, behind a driver interface designed for more; SQLite +is the second, for an application that runs on one machine. - **Fast:** set-based claim, finalize and insert (including `COPY`), a live table that stays small however much history accumulates, and retention by @@ -36,7 +37,8 @@ Postgres is the first engine, behind a driver interface designed for more. > ordering keys, streams with consumers, a SQL contract for producers in other > languages), flow control, batches and workflows, plus the `hopper` CLI, > `hoppertest`, `hopperotel`, the `hopperui` web UI and the `drivertest` -> conformance suite. [docs/PLAN.md](docs/PLAN.md) is the plan of record and +> conformance suite, and a SQLite driver (`driver/hoppersqlite`, pure Go, a module +> of its own) that passes it. [docs/PLAN.md](docs/PLAN.md) is the plan of record and > [docs/getting-started.md](docs/getting-started.md) the place to start. > Until v1.0.0, minor versions may change the API. Feedback is welcome > through issues and PRs. diff --git a/docs/PLAN.md b/docs/PLAN.md index 5818c26..f530cfd 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -416,7 +416,7 @@ the way that performs best on that engine. | `hopper/driver/hopperpgx` | Postgres via pgx v5 (the reference driver): COPY, LISTEN, pipelined statements | core | | `hopper/driver/hoppersql` | Postgres via `database/sql` (lib/pq, pgx stdlib); polling only | core | | `hopper/drivertest` | Conformance, concurrency and chaos suite every driver must pass | core | -| `hoppersqlite` (later) | SQLite: embedded, single-node and local development | separate module | +| `hopper/driver/hoppersqlite` | SQLite (modernc.org/sqlite, pure Go): embedded, single-node and local development | separate module | | `hoppermongo` (later) | MongoDB (replica set, for transactions) | separate module | The two Postgres drivers share one implementation of the SQL and the logic around it @@ -434,10 +434,25 @@ clock. Push notifications are optional, because polling is always available. The planned engines meet these requirements differently, which is why the interface is defined as operations rather than SQL: -- **SQLite** has a single writer, so a claim is an `UPDATE … RETURNING` inside a - `BEGIN IMMEDIATE` transaction, and SKIP LOCKED is unnecessary. There is no LISTEN, - so clients in the same process wake each other directly and other processes poll. - Retention is a batched `DELETE`. +- **SQLite** has a single writer (`Capabilities.SingleWriter`), so a claim is an + `UPDATE … RETURNING` inside a transaction that holds the write lock, and SKIP LOCKED + is unnecessary. Every transaction that may write takes the lock before its first + statement reads (`BEGIN IMMEDIATE`), so what it reads cannot go stale before it + writes; the driver's `database/sql` connections do that lazily, on a transaction's + first statement, so that beginning one never blocks. Within a process, writers queue + for the lock in turn, for as long as their context allows; across processes, SQLite's + busy handler polls for it. With no other writer in between, the finalizing transaction + reads each job, checks it in Go and writes it back, settles the job's batch and promotes + its dependents itself, so `BatchSettle` and `WorkflowPromoteReady` find nothing a race + left; `SkipLocked` and `NoWait` wait briefly for another transaction's writes and then + skip every job or return `ErrJobLocked`. Notifications are rows of a table, written in + the notifying transaction: a listener in the same process is woken when it commits, + and listeners in other processes look every 50ms, when `PRAGMA data_version` says + another connection committed. Retention is a batched `DELETE`; stream events commit + in append order, so a position is an event's sequence number. The database must be on + a local filesystem, where SQLite's locks hold: `Open` refuses one on NFS, SMB, Lustre, + GPFS, FUSE and the like. Its migrations follow `hoppermigrate`'s versions one for one + (`driver.MigrationScripts`). - **MongoDB** claims with atomic `findOneAndUpdate` or `updateMany` (tagging a batch with a claim token, then reading it back). It uses `$$NOW` for server time, change streams for notifications, and TTL indexes or dropped collections for retention. @@ -673,7 +688,10 @@ there is **no scheduler loop** moving jobs between states. ### 7.1 Clock All time comparisons use the database's `now()`, never the application clock, so clock -skew between pods cannot cause early or duplicate execution. +skew between pods cannot cause early or duplicate execution. SQLite's clock is the host's, +which every process that opens the database shares, since it must be on a local +filesystem: hoppersqlite reads it once a transaction holds the write lock, so each write's +time is no earlier than the writes before it. ### 7.2 Insert - **Single and small batches:** one `INSERT … SELECT FROM unnest($kinds, $args, …) @@ -1758,6 +1776,10 @@ rates: connection is named `hopper-listener:` in `pg_stat_activity`. - **Channels are per database**, not per schema. Two hopper schemas in one database see each other's notifications, which only causes spurious wake-ups. +- **SQLite** has no LISTEN: hoppersqlite writes notifications to a table in the + notifying transaction, wakes the listeners of its own process when it commits, and + has listeners in other processes look for new rows every 50ms, when another + connection has committed. Rows are kept for a minute. ### 7.12 Shutdown `Stop(ctx)` and `Run` (when its context is canceled) shut down in three steps: @@ -2105,6 +2127,15 @@ observability without new machinery. - SQL files are embedded in the `hoppermigrate` package, versioned, and forward-only by default. Down migrations exist for development. +- The embedded scripts are Postgres's. Another engine's driver has its own script, up + and down, for every version (`driver.MigrationScripts`), and `hoppermigrate` applies + those, so that a version names the same tables and columns on every engine and + `Client.Start`'s check holds on each. hoppersqlite's follow Postgres's version for + version; one that changes only what SQLite lacks (partitions, autovacuum settings, SQL + functions) is a script of comments. Its migrator holds SQLite's write lock from `Lock` + to `Close` and applies each version in a savepoint, so writers wait for the migration, + which takes milliseconds; its upgrade test (`TestUpgradeUnderTraffic`) starts from every + earlier version. - `hoppermigrate.Up(ctx, driver)` applies them under an advisory lock taken on a dedicated connection, not a pooled one. This avoids a pool-starvation deadlock: processes waiting for the lock must not hold the connections the migration needs. @@ -2223,7 +2254,7 @@ observability without new machinery. | --- | --- | | Unit | Backoff, cron parsing and time zones, pattern matching, unique-key construction, the state transition table, option validation. Timers are tested with `testing/synctest`. | | Integration | Against real Postgres 14–18 in CI (a service container per version), always with `-race` and `GODEBUG=fips140=only`. Each test gets its own schema, so tests run in parallel. | -| Driver conformance | `drivertest.Run(t, fixture)` runs the behavioral suite (inserting, unique keys, claiming, every finalize transition and fence, leases, leader, rescue, cancel, retry, TTL, listing, queues, periodic slots, stats, notifications, history maintenance) against every driver. A `Fixture` supplies isolated databases and the few engine-specific hooks (expire a lease, backdate an attempt, count rows). Engine internals such as partition management stay in the driver's own tests. | +| Driver conformance | `drivertest.Run(t, fixture)` runs the behavioral suite (inserting, unique keys, claiming, every finalize transition and fence, leases, leader, rescue, cancel, retry, TTL, listing, queues, periodic slots, stats, notifications, history maintenance) against every driver. A `Fixture` supplies isolated databases and the few engine-specific hooks (expire a lease or a job, backdate an attempt, store a value as a direct write would, wait until a statement waits for a lock, count rows); what depends on row locks is tested either way by `Capabilities.SingleWriter`. `drivertest.RunPostgres` adds the tests of what the Postgres drivers share beyond the interface (partitions and their indexes, the tags tables, progress rows, planner statistics); other engine internals stay in the driver's own tests. | | Concurrency | N clients by M jobs: every job is finalized exactly once when nothing crashes. A finalize held open in a caller's transaction (`CompleteTx`) against a competing fenced finalize: exactly one applies, whichever way the transaction ends. Uniqueness under concurrent inserts. Periodic slots unique across two leaders. An insert counting finished duplicates against its duplicate's finalize held open: the insert reports the finished job and leaves no live one. A progress write held open in a caller's transaction against the attempt's competing finalize: a write that commits first is kept, one that comes second is fenced. A stuck attempt rescued and claimed again by the same client: its late result does not finalize the new claim, and a cancel still reaches the new claim. Global and rate limits never exceeded. Ordering keys never run two at once. | | Chaos | `pg_terminate_backend` on a client mid-job, followed by rescue and re-run. A client that crashes after its workers committed `CompleteTx` transactions but before its finalizer flushed: none of those jobs runs again. A frozen client that loses its lease and must fence itself. Leader killed, new leader within the TTL. Stop with an expired context. Listener dropped, polling continues and the listener reconnects. | | Upgrade | Migrate from release N-1 with jobs in every state and live traffic. | @@ -2259,7 +2290,7 @@ The estimates assume one engineer. Each milestone is one or more PRs. | M6 | Messaging | Subscriptions, AMQP topic patterns, typed `Message[T]`, PublishTx fan-out, dedup, ordering keys, request/reply, SQL publish contract, `ReplayDiscarded`, the upgrade test. **Done.** | 5d | | M7 | Flow control and batches | Global limits, rate limits, partitioned limits, priority aging, batches with callbacks, `hoppersql` driver. **Done.** | 5d | | M8 | Workflows, streams, UI | Job dependencies and DAG workflows, streams with consumers, `hopperui`. **Done**, one PR each. | 2–3w | -| M9 | More engines (later) | `hoppersqlite`, then `hoppermongo`, each in its own module and passing `drivertest`. Not scheduled yet. | per engine | +| M9 | More engines | `hoppersqlite`, in its own module and passing `drivertest`. **Done.** `hoppermongo` later, not scheduled yet. | per engine | M0–M4 take roughly four weeks to a production-ready job queue that meets its performance targets. v1.0.0 follows M8, once the API has been proven in production. diff --git a/docs/getting-started.md b/docs/getting-started.md index 1bd7921..7f5cf46 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -68,6 +68,35 @@ application's `search_path` is unchanged, so the pool can be shared. **Existing installations:** set `Schema` to the schema where your Hopper tables already live (for example `public`) before upgrading. Changing this setting selects another installation; it does not move existing jobs or tables. + +### SQLite + +An application that runs on one machine, such as a command-line tool or a +single-node service, can keep hopper in a SQLite file instead, with no server +to run. The driver is pure Go (`modernc.org/sqlite`), so the program still +builds with `CGO_ENABLED=0`, and it is a module of its own, +`github.com/parallelworks/hopper/driver/hoppersqlite`: + +```go +driver, err := hoppersqlite.Open(filepath.Join(stateDir, "hopper.db"), nil) +if err != nil { + return err // hoppersqlite.ErrNetworkFilesystem for a file on NFS and the like +} +defer driver.Close() +if _, err := hoppermigrate.Up(ctx, driver, nil); err != nil { + return err +} +client, err := hopper.NewClient(driver, cfg) +``` + +Its transactions are `*sql.Tx`; begin them on `driver.DB()`, whose +connections take the database's write lock when a transaction runs its first +statement. SQLite runs one writing transaction at a time, so keep them short, +and write through the transaction (`InsertTx`, `CompleteTx`) rather than the +pool while one is open. Several processes can share the file: clients in +other processes are woken within 50ms of an insert. The file must be on a +local disk; `Open` refuses one on a network filesystem, where SQLite's locks +do not hold. Migration credentials need permission to create the schema, or you can create it beforehand. Migrating down to zero removes Hopper objects and leaves the namespace itself intact. diff --git a/release-please-config.json b/release-please-config.json index b0ac00b..2acdae5 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -66,6 +66,10 @@ { "type": "generic", "path": "hopperui/go.mod" + }, + { + "type": "generic", + "path": "driver/hoppersqlite/go.mod" } ] } From e62a077ca4cf9ede9f28a347ab2def11799177ff Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 19:57:46 +0000 Subject: [PATCH 09/20] fix(release): tag driver/hoppersqlite at each release, so that go get finds it at the release's version The release workflow tagged only hopperotel and hopperui, so go get of driver/hoppersqlite at a release version found no such version. It now tags every sub-module, driver/hoppersqlite among them, and pushes the tags together. releasing.md and the plan's status line count four modules, and releasing.md says why hoppersqlite is released with the core module and upgraded with it: it imports the core module's internal packages. --- .github/workflows/release.yml | 12 ++++++++---- docs/PLAN.md | 4 ++-- docs/releasing.md | 25 +++++++++++++++++-------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 87217ff..94a2d25 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,17 +67,21 @@ jobs: - name: Tag the sub-modules at the same version # release-please manages one package, the core module, and rewrites # the sub-modules' requirement on it (x-release-please-version in - # their go.mod). Their tags are added here so that all three modules - # resolve at one version. + # their go.mod). Their tags are added here so that every module + # resolves at one version. A module in a subdirectory is tagged with + # its path, as the go command looks it up. env: TAG: ${{ needs.release-please.outputs.tag_name }} run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - for m in hopperotel hopperui; do + modules="hopperotel hopperui driver/hoppersqlite" + tags=() + for m in $modules; do git tag -a "$m/$TAG" "$TAG" -m "$m $TAG" + tags+=("$m/$TAG") done - git push origin "hopperotel/$TAG" "hopperui/$TAG" + git push origin "${tags[@]}" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod diff --git a/docs/PLAN.md b/docs/PLAN.md index f530cfd..f5552e0 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -9,7 +9,7 @@ in-process job framework and a separate message broker. - **Module:** `github.com/parallelworks/hopper` - **License:** Apache-2.0 - **Dependencies:** the Go standard library and `github.com/jackc/pgx/v5`. Nothing else in the core module. -- **Status:** M0 through M4 and M6 through M8 are implemented (§16), the §8.2 targets have been run on the reference hardware, and everything ships together as one version across the core module, `hopperotel` and `hopperui` ([docs/releasing.md](releasing.md)); **v0.1.1** is current. This document is the plan of record, and changes to it go through PRs. +- **Status:** M0 through M4 and M6 through M9 are implemented (§16), the §8.2 targets have been run on the reference hardware, and everything ships together as one version across the core module, `hopperotel`, `hopperui` and `driver/hoppersqlite` ([docs/releasing.md](releasing.md)); **v0.1.1** is current. This document is the plan of record, and changes to it go through PRs. The name refers to a feed hopper, which releases work into a machine one piece at a time, and to RADM Grace Hopper. It is also a fitting name for something @@ -2271,7 +2271,7 @@ observability without new machinery. connection (§7.11). Otherwise the client uses adaptive polling and logs that once. - **Stability:** API stability starts at v1.0.0. Until then, minor versions may break the API, with changes noted in the CHANGELOG. Releases are cut by release-please from - the Conventional Commit titles of merged PRs, one version across the three modules, + the Conventional Commit titles of merged PRs, one version across every module, with goreleaser attaching the CLI binaries ([docs/releasing.md](releasing.md)). The SQL insert contract (§10) is versioned separately and changes only with a deprecation window. diff --git a/docs/releasing.md b/docs/releasing.md index 55079b1..eb0d41e 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,10 +1,17 @@ # Releasing hopper -hopper is three Go modules in one repository: the core module at the root, -`hopperotel` and `hopperui`. A release is one version across all three, cut -as three tags on the same commit of `canary`: `vX.Y.Z`, `hopperotel/vX.Y.Z` -and `hopperui/vX.Y.Z`. Until v1.0.0, minor versions may change the API; -patch versions do not. +hopper is four Go modules in one repository: the core module at the root, +`hopperotel`, `hopperui` and the SQLite driver, `driver/hoppersqlite`. A +release is one version across all four, cut as four tags on the same commit +of `canary`: `vX.Y.Z`, `hopperotel/vX.Y.Z`, `hopperui/vX.Y.Z` and +`driver/hoppersqlite/vX.Y.Z`. Until v1.0.0, minor versions may change the +API; patch versions do not. + +The sub-modules are released with the core module, not on their own, and +depend on that version alone: `hoppersqlite` imports the core module's +internal packages (`internal/jobtags`, `driver/internal/jobcodec`), which +have no compatibility promise from one version to the next, so an +application upgrades hopper and hoppersqlite together. Releases are automated with [release-please](https://github.com/googleapis/release-please) and [goreleaser](https://goreleaser.com) (`.github/workflows/release.yml`). @@ -19,7 +26,7 @@ and [goreleaser](https://goreleaser.com) (`.github/workflows/release.yml`). bumps the minor version before v1.0.0 (`bump-minor-pre-major`). release-please manages one package, the core module; the sub-modules are versioned with it rather than on their own, so a change anywhere in the - repository releases all three. + repository releases all four. 2. The release PR updates `CHANGELOG.md`, the manifest (`.release-please-manifest.json`), and the sub-modules' `go.mod` requirement on the core module, through the `// x-release-please-version` @@ -29,7 +36,8 @@ and [goreleaser](https://goreleaser.com) (`.github/workflows/release.yml`). 3. Merging the release PR (with `gh pr merge --squash --admin`, since the `canary` ruleset asks for two reviews) creates the `vX.Y.Z` tag and the GitHub release with the changelog entry. The workflow then tags - `hopperotel/vX.Y.Z` and `hopperui/vX.Y.Z` on the same commit, and + `hopperotel/vX.Y.Z`, `hopperui/vX.Y.Z` and `driver/hoppersqlite/vX.Y.Z` + on the same commit, and goreleaser builds `hopper`, `hopperbench` and `hopperui` for Linux and macOS (amd64, arm64) and attaches the archives and checksums to the release. @@ -82,7 +90,8 @@ After the tags exist, from an empty module: ```sh cd "$(mktemp -d)" && go mod init check && go get github.com/parallelworks/hopper@vX.Y.Z \ - github.com/parallelworks/hopper/hopperotel@vX.Y.Z github.com/parallelworks/hopper/hopperui@vX.Y.Z + github.com/parallelworks/hopper/hopperotel@vX.Y.Z github.com/parallelworks/hopper/hopperui@vX.Y.Z \ + github.com/parallelworks/hopper/driver/hoppersqlite@vX.Y.Z ``` ## History From a86776074247e7257f705ff304012e801dbae766 Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 19:57:51 +0000 Subject: [PATCH 10/20] refactor(driver): encode a finalize's attempt log in jobcodec, and call jobcodec directly from pgsql Both engines marshalled the attempt log a finalize keeps with the same code; jobcodec.MarshalAttemptLog now does it once. pgsql's one-line wrappers around jobcodec (storableText, decodeProgress, marshalAttemptError) are gone. The bytes each engine writes are unchanged. --- driver/hoppersqlite/finalize.go | 10 +++----- driver/internal/jobcodec/jobcodec.go | 10 ++++++++ driver/internal/pgsql/jobs.go | 37 +++++++--------------------- driver/internal/pgsql/values.go | 5 ---- 4 files changed, 23 insertions(+), 39 deletions(-) diff --git a/driver/hoppersqlite/finalize.go b/driver/hoppersqlite/finalize.go index 2d3573f..7f4f6c4 100644 --- a/driver/hoppersqlite/finalize.go +++ b/driver/hoppersqlite/finalize.go @@ -226,13 +226,11 @@ func encodeResults(jobs []driver.JobFinalize) ([]result, error) { return nil, fmt.Errorf("hopper: finalize jobs: %s is not a finalize state", f.State) } r := result{JobFinalize: f, err: f.Error, reason: nullable(jobcodec.StorableText(f.SnoozeReason))} - if f.Log != nil && f.KeepsLog() { - b, err := json.Marshal(driver.AttemptLog{Attempt: f.Log.Attempt, Snoozed: f.Snooze, Log: jobcodec.StorableText(f.Log.Log)}) - if err != nil { - return nil, fmt.Errorf("hopper: finalize jobs: %w", err) - } - r.log = b + log, err := jobcodec.MarshalAttemptLog(f) + if err != nil { + return nil, fmt.Errorf("hopper: finalize jobs: %w", err) } + r.log = log if len(f.Output) > 0 { out, err := jsonValue(f.Output, "", "output") if err != nil { diff --git a/driver/internal/jobcodec/jobcodec.go b/driver/internal/jobcodec/jobcodec.go index e5fa694..a94493e 100644 --- a/driver/internal/jobcodec/jobcodec.go +++ b/driver/internal/jobcodec/jobcodec.go @@ -34,6 +34,16 @@ func MarshalAttemptError(e *driver.AttemptError) ([]byte, error) { }{e.Attempt, e.Fence, StorableText(e.Error), StorableText(e.Trace)}) } +// MarshalAttemptLog encodes the log a finalize appends for result f, which +// a snooze marks Snoozed, its text made storable, or returns nil when there +// is none to keep (JobFinalize.KeepsLog). +func MarshalAttemptLog(f driver.JobFinalize) ([]byte, error) { + if f.Log == nil || !f.KeepsLog() { + return nil, nil + } + return json.Marshal(driver.AttemptLog{Attempt: f.Log.Attempt, Snoozed: f.Snooze, Log: StorableText(f.Log.Log)}) +} + // DecodeErrors reads a job's errors, a JSON array of error records. An // empty value reads as none. func DecodeErrors(stored string) ([]driver.AttemptError, error) { diff --git a/driver/internal/pgsql/jobs.go b/driver/internal/pgsql/jobs.go index 8829763..143864f 100644 --- a/driver/internal/pgsql/jobs.go +++ b/driver/internal/pgsql/jobs.go @@ -5,7 +5,6 @@ import ( "cmp" "context" "database/sql" - "encoding/json" "errors" "fmt" "math" @@ -91,7 +90,7 @@ func scanJob(row Row, history bool, extra ...any) (*driver.JobRow, error) { j.BatchID = batchID.ID } if progress != "" { - j.Progress = decodeProgress(string(progress)) + j.Progress = jobcodec.DecodeProgress(string(progress)) } return &j, nil } @@ -696,9 +695,9 @@ func (e *Executor) JobFinalizeMany(ctx context.Context, params driver.JobFinaliz return nil, fmt.Errorf("hopper: finalize jobs: %s is not a finalize state", f.State) } ids[i], attemptedBy[i], fences[i], states[i] = f.ID, f.AttemptedBy, f.Fence, string(f.State) - delays[i], snoozes[i], reasons[i] = max(f.Delay, 0).Seconds(), f.Snooze, storableText(f.SnoozeReason) + delays[i], snoozes[i], reasons[i] = max(f.Delay, 0).Seconds(), f.Snooze, jobcodec.StorableText(f.SnoozeReason) if f.Error != nil { - b, err := marshalAttemptError(f.Error) + b, err := jobcodec.MarshalAttemptError(f.Error) if err != nil { return nil, fmt.Errorf("hopper: finalize jobs: %w", err) } @@ -708,13 +707,11 @@ func (e *Executor) JobFinalizeMany(ctx context.Context, params driver.JobFinaliz // Only completed jobs may skip history; the failed partitions are the // dead-letter queue. archives[i] = f.Archived() - if f.Log != nil && f.KeepsLog() { - b, err := json.Marshal(driver.AttemptLog{Attempt: f.Log.Attempt, Snoozed: f.Snooze, Log: storableText(f.Log.Log)}) - if err != nil { - return nil, fmt.Errorf("hopper: finalize jobs: %w", err) - } - logs[i] = b + b, err := jobcodec.MarshalAttemptLog(f) + if err != nil { + return nil, fmt.Errorf("hopper: finalize jobs: %w", err) } + logs[i] = b } args := []any{ uuidArray(ids), int64Array(attemptedBy), textArray(states), floatArray(delays), boolArray(snoozes), @@ -746,12 +743,6 @@ func keepLogs(keep int) int32 { return int32(keep) } -// marshalAttemptError encodes an error record without its timestamp, which -// the finalize statement fills in from database time. -func marshalAttemptError(e *driver.AttemptError) ([]byte, error) { - return jobcodec.MarshalAttemptError(e) -} - var jobGetQuery = fmt.Sprintf(` SELECT %s, NULL::timestamptz AS finalized_at, NULL::jsonb AS output FROM {{schema}}.hopper_jobs WHERE id = $1 UNION ALL @@ -807,15 +798,9 @@ func (e *Executor) JobRecordProgress(ctx context.Context, params driver.JobRecor if err != nil { return nil, fmt.Errorf("hopper: record job progress: %w", err) } - return decodeProgress(stored), nil + return jobcodec.DecodeProgress(stored), nil } -// decodeProgress reads a progress value, keeping what it can of one that -// is not in the form JobRecordProgress writes, such as one written by hand: -// a row must still scan, or a claim that took it would fail with the claim -// already applied. A value that is not an object reads as none. -func decodeProgress(stored string) *driver.Progress { return jobcodec.DecodeProgress(stored) } - // JobGet implements driver.Executor. func (e *Executor) JobGet(ctx context.Context, id driver.JobID) (*driver.JobRow, error) { job, err := scanJob(e.Conn.QueryRow(ctx, jobGetQuery, uuidParam(id)), true) @@ -844,13 +829,9 @@ func (e *Executor) JobLogs(ctx context.Context, id driver.JobID) ([]driver.Attem if err != nil { return nil, fmt.Errorf("hopper: get job logs: %w", err) } - return decodeLogs(string(logs)), nil + return jobcodec.DecodeLogs(string(logs)), nil } -// decodeLogs reads a job's logs, skipping entries that are not in the form -// the finalize writes, such as ones written by hand. -func decodeLogs(stored string) []driver.AttemptLog { return jobcodec.DecodeLogs(stored) } - var ( jobListLiveSQL = fmt.Sprintf(`SELECT %s, NULL::timestamptz, NULL::jsonb FROM {{schema}}.hopper_jobs`, JobColumns("")) jobListHistorySQL = fmt.Sprintf(`SELECT %s, finalized_at, output FROM {{schema}}.hopper_job_history`, historyColumns("")) diff --git a/driver/internal/pgsql/values.go b/driver/internal/pgsql/values.go index f1d74a3..39b4f0d 100644 --- a/driver/internal/pgsql/values.go +++ b/driver/internal/pgsql/values.go @@ -20,11 +20,6 @@ import ( func textArray(values []string) []string { return orEmpty(values) } -// storableText replaces what PostgreSQL refuses in text and jsonb strings, -// NUL and invalid UTF-8, so that one worker's text cannot fail the -// statement that finalizes a whole batch. -func storableText(s string) string { return jobcodec.StorableText(s) } - // nullableTextArray sends empty strings as NULL. func nullableTextArray(values []string) []*string { elems := make([]*string, len(values)) From f034c5315c29264969069e701535c991f30c1c8f Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 19:57:57 +0000 Subject: [PATCH 11/20] test(drivertest): restore the Postgres checks the split into Run and RunPostgres narrowed testBatchesConcurrent checked, on Postgres, that its races left no hopper_batch_progress rows; moving it into the neutral suite dropped the check, and testBatchProgressRaces' extra sweep did not stand in for it. The check is back at the end of testBatchesConcurrent, for a PostgresFixture, and the extra sweep is gone. JobLogsStored runs under both finalize statements again, keeping two logs as it did, and jsonEqual compares numbers by their exact value rather than as float64s. --- drivertest/drivertest.go | 27 ++++++++++++++++++++------- drivertest/flow.go | 9 ++++++++- drivertest/logs.go | 14 ++++++++++---- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/drivertest/drivertest.go b/drivertest/drivertest.go index e8f37c4..cb36d63 100644 --- a/drivertest/drivertest.go +++ b/drivertest/drivertest.go @@ -21,10 +21,12 @@ package drivertest import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "math/big" "slices" "strconv" "strings" @@ -204,7 +206,8 @@ func RunPostgres[TTx any](t *testing.T, f PostgresFixture[TTx]) { {"TagsDayGoesWithItsLastHour", testTagsDayGoesWithItsLastHour[TTx]}, {"TagsFollowDirectUpdates", testTagsFollowDirectUpdates[TTx]}, {"TagsFollowTruncate", testTagsFollowTruncate[TTx]}, - {"JobLogsStored", testJobLogsStored[TTx]}, + {"JobLogsStored", func(t *testing.T, f PostgresFixture[TTx]) { testJobLogsStored(t, f, false) }}, + {"JobLogsStoredNoBatches", func(t *testing.T, f PostgresFixture[TTx]) { testJobLogsStored(t, f, true) }}, {"RetryOrderingKeyAgingOrder", testRetryOrderingKeyAgingOrder[TTx]}, {"BatchProgressRaces", testBatchProgressRaces[TTx]}, {"StreamsPartitioned", testStreamsPartitioned[TTx]}, @@ -352,17 +355,22 @@ func singleWriter[TTx any](d driver.Driver[TTx]) bool { return d.Capabilities(). // jsonEqual reports whether two JSON texts hold the same value: an engine // need not keep the text as given (Postgres's jsonb orders an object's keys -// and spaces them), only the value. +// and spaces them), only the value. Numbers compare by their exact value, +// not as float64s. func jsonEqual(got []byte, want string) bool { - var a, b any - if json.Unmarshal(got, &a) != nil || json.Unmarshal([]byte(want), &b) != nil { - return false + decode := func(b []byte) (any, bool) { + dec := json.NewDecoder(bytes.NewReader(b)) + dec.UseNumber() + var v any + return v, dec.Decode(&v) == nil } - return fmt.Sprint(normalizeJSON(a)) == fmt.Sprint(normalizeJSON(b)) + a, ok := decode(got) + b, ok2 := decode([]byte(want)) + return ok && ok2 && fmt.Sprint(normalizeJSON(a)) == fmt.Sprint(normalizeJSON(b)) } // normalizeJSON gives a decoded JSON value a printed form that does not -// depend on its object's key order. +// depend on its object's key order, nor on how its numbers are written. func normalizeJSON(v any) any { switch v := v.(type) { case map[string]any: @@ -384,6 +392,11 @@ func normalizeJSON(v any) any { return "[" + strings.Join(out, ",") + "]" case string: return strconv.Quote(v) + case json.Number: + if r, ok := new(big.Rat).SetString(string(v)); ok { + return r.RatString() + } + return string(v) default: return v } diff --git a/drivertest/flow.go b/drivertest/flow.go index 42aac9e..f97b030 100644 --- a/drivertest/flow.go +++ b/drivertest/flow.go @@ -499,6 +499,14 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { if kinds := h.callbacks(pruned); kinds["failed"] != 1 || kinds["ok"] != 0 { t.Errorf("callbacks of the pruned batch = %v", kinds) } + + // Postgres records a batch's progress in rows of its own, which the + // settles above leave none of once every batch has completed. + if pf, ok := any(f).(PostgresFixture[TTx]); ok { + if left := pf.QueryText(ctx, t, d, "SELECT count(*)::text FROM hopper_batch_progress"); left != "0" { + t.Errorf("%s progress rows left after every batch completed and settled", left) + } + } } // testBatchProgressRaces covers the progress rows Postgres's finalizes @@ -646,7 +654,6 @@ func testBatchProgressRaces[TTx any](t *testing.T, f PostgresFixture[TTx]) { t.Errorf("batch finished after its sweep = %+v, %v", b, err) } } - h.sweep() if left := f.QueryText(ctx, t, d, "SELECT count(*)::text FROM hopper_batch_progress"); left != "0" { t.Errorf("%s progress rows left after every batch completed and settled", left) } diff --git a/drivertest/logs.go b/drivertest/logs.go index e9f2ee0..656d4b4 100644 --- a/drivertest/logs.go +++ b/drivertest/logs.go @@ -130,20 +130,26 @@ func testJobLogs[TTx any](t *testing.T, f Fixture[TTx], noBatches bool) { } } -// testJobLogsStored checks how the logs column is kept: a snooze that logs +// testJobLogsStored checks how the logs column is kept, by both finalize +// statements, keeping two logs as testJobLogs does: a snooze that logs // nothing leaves a job without logs as it is, stored as NULL rather than // rewritten, and takes one whose only log is a snooze's back to NULL. -func testJobLogsStored[TTx any](t *testing.T, f PostgresFixture[TTx]) { +func testJobLogsStored[TTx any](t *testing.T, f PostgresFixture[TTx], noBatches bool) { ctx := context.Background() d := f.NewDriver(t) - exec := d.Executor() + exec := executor(d, noBatches) id := insert(ctx, t, exec, params("k", 1))[0].Job.ID clientID := register(ctx, t, exec) snooze := func(log *driver.AttemptLog) { t.Helper() f.MakeClaimable(ctx, t, d, id) j := claimOne(ctx, t, exec, clientID, id) - finalize(ctx, t, exec, driver.JobFinalize{ID: id, AttemptedBy: clientID, Fence: j.Fence, State: driver.JobStateScheduled, Snooze: true, Log: log}) + applied, err := exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{KeepLogs: 2, Jobs: []driver.JobFinalize{ + {ID: id, AttemptedBy: clientID, Fence: j.Fence, State: driver.JobStateScheduled, Snooze: true, Log: log}, + }}) + if err != nil || len(applied) != 1 { + t.Fatalf("snooze = %v, %v", applied, err) + } } noLogs := func(what string) { t.Helper() From 714f43abc6394bf4fcf1897ea885c6861e4f126a Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 19:58:25 +0000 Subject: [PATCH 12/20] feat(driver): mark a job attempt's context with its owner, and bound the lease renewal's wait by the lease On an engine that runs one writer at a time, a worker that writes on the pool while its own transaction holds the write lock waits for itself until its context ends, and every write of the process waits with it, lease renewals included, until the client loses its lease and is fenced. The core now marks each attempt's context with the attempt (driver.WithOwner), which names it, so that such an engine can refuse that write at once and say who holds a lock a write waits for. A lease renewal now waits at most until the lease it renews would expire, then reports what the engine says of who holds the lock and tries again, rather than wait on without a word. On Postgres a renewal takes a row lock only on its own client's row and is not held up. CompleteTx, RecordStepTx, JobCancelTx, JobDeleteTx and JobRetryOrderingKeyTx say what their transaction holds on such an engine: every write in the database, not one job's row. --- await.go | 6 ++++++ client.go | 15 ++++++++++++++- complete.go | 9 ++++++++- control.go | 9 ++++++--- driver/driver.go | 33 +++++++++++++++++++++++++++------ driver/owner_test.go | 27 +++++++++++++++++++++++++++ execute.go | 1 + steps.go | 5 +++++ 8 files changed, 94 insertions(+), 11 deletions(-) create mode 100644 driver/owner_test.go diff --git a/await.go b/await.go index dd4d14c..c368067 100644 --- a/await.go +++ b/await.go @@ -50,6 +50,12 @@ type attempt struct { logEnded bool } +// String names the attempt, as the owner of its worker's writes +// (driver.WithOwner). +func (a *attempt) String() string { + return fmt.Sprintf("job %s (%s), attempt %d", a.row.ID, a.row.Kind, a.attempt) +} + func (a *attempt) markHeldByTx() { a.mu.Lock() a.heldByTx = true diff --git a/client.go b/client.go index 1000710..9b0725e 100644 --- a/client.go +++ b/client.go @@ -494,9 +494,17 @@ func (c *Client[TTx]) wakeQueue(queue string) { // ID. Results still in flight are submitted anyway; the finalize statement // rejects any whose job was rescued meanwhile and applies the rest, which is // equivalent to a rescue. +// +// A renewal waits until the lease it renews would expire, and at least one +// renewal interval: on an engine that runs one writer at a time, a +// transaction that holds the database holds every renewal up, and a renewal +// that gives up is reported, with what the engine says of who holds the +// lock, and tried again, rather than left waiting past the lease without a +// word. func (c *Client[TTx]) leaseLoop(ctx context.Context) { ticker := time.NewTicker(c.tuning.leaseRenew) defer ticker.Stop() + renewed := time.Now() // the lease was taken just before the loop starts for { select { case <-ctx.Done(): @@ -504,7 +512,10 @@ func (c *Client[TTx]) leaseLoop(ctx context.Context) { case <-ticker.C: } id := c.clientID.Load() - res, err := c.exec.ClientRenew(ctx, driver.ClientRenewParams{ClientID: id, TTL: c.tuning.leaseTTL}) + start := time.Now() + rctx, cancel := context.WithTimeout(ctx, max(time.Until(renewed.Add(c.tuning.leaseTTL)), c.tuning.leaseRenew)) + res, err := c.exec.ClientRenew(rctx, driver.ClientRenewParams{ClientID: id, TTL: c.tuning.leaseTTL}) + cancel() if err != nil { if ctx.Err() == nil { c.logger.WarnContext(ctx, "hopper: renew client lease", "client_id", id, "error", err) @@ -513,8 +524,10 @@ func (c *Client[TTx]) leaseLoop(ctx context.Context) { } if !res.Renewed { c.fence(ctx, id) + renewed = time.Now() continue } + renewed = start // Control state rides along with every renewal, in case a // notification was missed. for _, jobID := range res.CancelRequested { diff --git a/complete.go b/complete.go index e143651..5040e61 100644 --- a/complete.go +++ b/complete.go @@ -30,13 +30,20 @@ var ErrJobFenced = errors.New("hopper: job is no longer running in this attempt" // whatever the worker returned, and the job.completed event fires. If it did // not, the attempt fails with the worker's error, or with a failure if the // worker returned nil, and the job is retried. A worker that returns before -// tx ends holds up only its own job's result, until tx ends. +// tx ends holds up only its own job's result, until tx ends, on an engine +// that locks rows. // // Call it last, just before the commit. From CompleteTx until tx ends, tx // holds the job's row and those of workflow dependents it promotes or // cancels, and holds off the creation and drop of history partitions. The // job's siblings in its batch finalize without waiting for it. // +// On an engine that runs one writer at a time +// (driver.Capabilities.SingleWriter, SQLite), tx holds the whole database +// for writes from its first statement until it ends: every other write +// waits for it, the results of other jobs and lease renewals included, and +// a write of the worker's own on the pool fails rather than wait for tx. +// // func (w *Worker) Work(ctx context.Context, job *hopper.Job[Args]) error { // tx, err := w.pool.Begin(ctx) // if err != nil { diff --git a/control.go b/control.go index 8953ba7..fc19bee 100644 --- a/control.go +++ b/control.go @@ -47,7 +47,8 @@ func (c *Client[TTx]) JobCancel(ctx context.Context, id JobID) (*JobRow, error) // // Until tx ends it holds the job's row, so a running job that finishes // meanwhile has its result wait for tx; its client's other results do not. -// Call it late in tx, as CompleteTx. +// Call it late in tx, as CompleteTx. On an engine that runs one writer at a +// time, tx holds every write in the database up until it ends. func (c *Client[TTx]) JobCancelTx(ctx context.Context, tx TTx, id JobID) (*JobRow, error) { return c.driver.UnwrapTx(tx).JobCancel(ctx, id) } @@ -114,7 +115,8 @@ func (c *Client[TTx]) JobDelete(ctx context.Context, id JobID) (*JobRow, error) // JobDeleteTx is JobDelete inside the caller's transaction. Until tx ends // it holds the job's row and, for a waiting job, those of the workflow -// dependents it cancels or promotes. +// dependents it cancels or promotes; on an engine that runs one writer at a +// time, every write in the database. func (c *Client[TTx]) JobDeleteTx(ctx context.Context, tx TTx, id JobID) (*JobRow, error) { return c.driver.UnwrapTx(tx).JobDelete(ctx, id) } @@ -144,7 +146,8 @@ func (c *Client[TTx]) JobRetryOrderingKey(ctx context.Context, queue, orderingKe // // Until tx ends it holds the rows of the jobs it woke and of the key's // running job, whose result waits for tx; its client's other results do -// not. Call it late in tx, as CompleteTx. +// not. Call it late in tx, as CompleteTx. On an engine that runs one writer +// at a time, tx holds every write in the database up until it ends. func (c *Client[TTx]) JobRetryOrderingKeyTx(ctx context.Context, tx TTx, queue, orderingKey string) (int, error) { queue = cmp.Or(queue, QueueDefault) exec := c.driver.UnwrapTx(tx) diff --git a/driver/driver.go b/driver/driver.go index 9912667..64f9ed5 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -83,6 +83,26 @@ type Capabilities struct { SingleWriter bool } +// ownerKey is the context key of a context's Owner. +type ownerKey struct{} + +// WithOwner returns ctx marked as running for owner: a job's attempt, whose +// worker's statements, in its own transactions and on the pool, all carry +// it. An engine that runs one writer at a time (Capabilities.SingleWriter) +// uses it to refuse at once a write that would wait for a transaction the +// same owner holds, which would wait until its context ended, holding up +// every other write meanwhile, and to say who holds the lock a write waits +// for. Owners compare by identity, so owner should be a pointer. +func WithOwner(ctx context.Context, owner fmt.Stringer) context.Context { + return context.WithValue(ctx, ownerKey{}, owner) +} + +// Owner returns the owner ctx runs for, or nil. +func Owner(ctx context.Context) fmt.Stringer { + o, _ := ctx.Value(ownerKey{}).(fmt.Stringer) + return o +} + // Notification channels. Payloads are documented on each. const ( // ChannelInsert carries the queue name of a committed insert. @@ -377,12 +397,13 @@ type MigrationExecutor interface { Close(ctx context.Context) error } -// MigrationScripts is implemented by a MigrationExecutor whose engine is not -// Postgres, whose scripts hoppermigrate embeds. hoppermigrate then gives -// Apply the engine's own script for each schema version instead. An engine -// has a script, up and down, for every schema version hoppermigrate knows, -// so that a version names the same schema on every engine: the tables and -// columns a release's statements need. +// MigrationScripts is implemented by the MigrationExecutor of an engine +// other than Postgres. hoppermigrate embeds Postgres's scripts; for an +// executor that implements MigrationScripts, it gives Apply the engine's +// own script for each schema version instead. Such an engine has a script, +// up and down, for every schema version hoppermigrate knows, so that a +// version names the same schema on every engine: the tables and columns a +// release's statements need. type MigrationScripts interface { // Script returns the engine's script for a schema version: the one that // applies it (up) or the one that reverts it. diff --git a/driver/owner_test.go b/driver/owner_test.go new file mode 100644 index 0000000..ceb4c05 --- /dev/null +++ b/driver/owner_test.go @@ -0,0 +1,27 @@ +package driver + +import ( + "context" + "testing" +) + +type attempt struct{ name string } + +func (a *attempt) String() string { return a.name } + +// TestOwner checks that a context carries the owner WithOwner gives it, +// and that one without an owner, or given a nil one, has none. +func TestOwner(t *testing.T) { + t.Parallel() + ctx := context.Background() + if Owner(ctx) != nil { + t.Error("a context without an owner has one") + } + a := &attempt{"job 1"} + if got := Owner(WithOwner(ctx, a)); got != a { + t.Errorf("Owner = %v, want %v", got, a) + } + if got := Owner(WithOwner(WithOwner(ctx, a), nil)); got != nil { + t.Errorf("Owner after WithOwner(nil) = %v, want none", got) + } +} diff --git a/execute.go b/execute.go index 04eae96..603c3d7 100644 --- a/execute.go +++ b/execute.go @@ -42,6 +42,7 @@ func (c *Client[TTx]) execute(ctx context.Context, row *driver.JobRow, qcfg Queu defer func() { a.closeLog(c.logCloseTimeout(ctx)) }() jobCtx = context.WithValue(jobCtx, attemptKey{}, a) jobCtx = context.WithValue(jobCtx, clientKey{}, c) + jobCtx = driver.WithOwner(jobCtx, a) claim := c.trackRunning(row.ID, cancel) defer c.untrackRunning(row.ID, claim) diff --git a/steps.go b/steps.go index a850450..56d2487 100644 --- a/steps.go +++ b/steps.go @@ -52,6 +52,11 @@ func RecordStep[T JobArgs](ctx context.Context, job *Job[T], step string, cursor // before the step returns, since RunSteps records the next finished step on // the pool, and a record on the pool while tx is open returns ErrJobLocked. // A job's finalize skips a row tx still holds, and is retried once tx ends. +// On an engine that runs one writer at a time +// (driver.Capabilities.SingleWriter, SQLite), tx holds the whole database +// for writes from its first statement until it ends: every other write +// waits for it, lease renewals included, and a write of the worker's own +// on the pool fails rather than wait for tx. func RecordStepTx[TTx any, T JobArgs](ctx context.Context, tx TTx, job *Job[T], step string, cursor any) error { return record(ctx, job, func() (driver.Executor, error) { return txExecutor(ctx, tx, job.ID) }, step, false, cursor) } From 731ae04a9a2d678bebffac61a540d39f1107797f Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 19:58:37 +0000 Subject: [PATCH 13/20] fix(hoppersqlite): refuse what a transaction SQLite rolled back runs next, a worker's write that would wait for itself, and a database a link puts on a network filesystem Found by review, each with a test that failed before: - After an interrupted write, a full disk or an I/O error, SQLite rolls the whole transaction back itself; the driver went on running the transaction's later statements, each committing on its own, and Rollback reported success, so CompleteTx was not all or nothing. The connection now learns of the rollback from SQLite's rollback hook and refuses the transaction's later statements and its Commit with ErrTransactionAborted, as Postgres does those of an aborted transaction. - A worker that wrote on the pool while its own transaction held the write lock waited for itself until its context ended. A write whose context names the owner of the transaction that holds the lock now fails at once with ErrLockHeldByOwner, and a wait that its context ends says who held the lock and for how long. - Open checked the directory named in the path, while SQLite follows a symbolic link and keeps its log beside the target, so a link to a file on NFS was opened. Open now resolves links, a dangling one too, checks the file as well as its directory, opens the resolved path, checks the file SQLite reports having opened, and keys the database by it. Parallels and VMware shared folders are refused too. - A statement that writes, run on its own through Query (INSERT ... RETURNING), skipped the in-process queue for the write lock; it now waits its turn. - JSON that jsonb refuses (a \u0000 escape, text that is not UTF-8, a number beyond numeric's range) was stored; args, metadata and output are now checked as metadata filters were. And: - The database's clock never returns a time earlier than one it returned before in the process. - Each connection cuts the write-ahead log back to 32MiB once a checkpoint empties it, and JobsMaintain truncates a log past 8MiB, which a long reader leaves behind. - On macOS, Synchronous FULL also sets fullfsync, without which a commit may sit in the disk's cache. - A read-only transaction's first statement retries SQLITE_BUSY. OpenContext opens under a context; Open waits at most 30 seconds. - Migrations returns SQLite's scripts for other migration tools, as hoppermigrate.Migrations returns Postgres's. - TestClient waited for 505 jobs of 506; TestChaos kills worker processes mid-transaction, and TestFairness measures writes against another process that writes without pause. --- driver/hoppersqlite/chaos_test.go | 192 +++++++++ driver/hoppersqlite/cluster.go | 29 +- driver/hoppersqlite/conn.go | 428 ++++++++++++++++--- driver/hoppersqlite/conn_internal_test.go | 87 ++++ driver/hoppersqlite/executor.go | 19 +- driver/hoppersqlite/funcs.go | 144 +++---- driver/hoppersqlite/hoppersqlite.go | 146 +++++-- driver/hoppersqlite/jobs.go | 2 +- driver/hoppersqlite/localfs.go | 72 +++- driver/hoppersqlite/localfs_bsd.go | 4 +- driver/hoppersqlite/localfs_linux.go | 9 +- driver/hoppersqlite/localfs_linux_test.go | 96 ++++- driver/hoppersqlite/localfs_netbsd.go | 4 +- driver/hoppersqlite/localfs_windows.go | 8 +- driver/hoppersqlite/migrate_test.go | 32 ++ driver/hoppersqlite/robustness_test.go | 475 ++++++++++++++++++++++ driver/hoppersqlite/sqlite_test.go | 11 +- hoppermigrate/hoppermigrate.go | 8 +- 18 files changed, 1563 insertions(+), 203 deletions(-) create mode 100644 driver/hoppersqlite/chaos_test.go create mode 100644 driver/hoppersqlite/conn_internal_test.go create mode 100644 driver/hoppersqlite/robustness_test.go diff --git a/driver/hoppersqlite/chaos_test.go b/driver/hoppersqlite/chaos_test.go new file mode 100644 index 0000000..64f47b5 --- /dev/null +++ b/driver/hoppersqlite/chaos_test.go @@ -0,0 +1,192 @@ +package hoppersqlite_test + +import ( + "context" + "fmt" + "log/slog" + "math/rand/v2" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/parallelworks/hopper" + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/hoppersqlite" +) + +type chaosArgs struct { + N int `json:"n"` +} + +func (chaosArgs) Kind() string { return "chaos" } + +// chaosWorker is the "worker" role of TestHelperProcess: a client whose +// worker writes the job's effect and completes the job in one +// transaction, until the test kills it. +func chaosWorker(ctx context.Context, t *testing.T, d *hoppersqlite.Driver) { + workers := hopper.NewWorkers() + hopper.AddWorkFunc(workers, func(ctx context.Context, job *hopper.Job[chaosArgs]) error { + time.Sleep(time.Duration(rand.IntN(20)) * time.Millisecond) + tx, err := d.DB().BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck // committed below + if _, err := tx.ExecContext(ctx, "INSERT INTO effects (n, attempt) VALUES (?, ?)", job.Args.N, job.Attempt); err != nil { + return err + } + if err := hopper.CompleteTx(ctx, tx, job); err != nil { + return err + } + return tx.Commit() + }) + c, err := hopper.NewClient(d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: 8}}, + Workers: workers, + PollInterval: 20 * time.Millisecond, + LeaseTTL: time.Second, + Logger: slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})), + }) + if err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + fmt.Println("started") + time.Sleep(time.Minute) +} + +// hog is the "hog" role of TestHelperProcess: eight goroutines that write +// as fast as they can for three seconds. +func hog(ctx context.Context, t *testing.T, d *hoppersqlite.Driver) { + x := d.Executor() + deadline := time.Now().Add(3 * time.Second) + fmt.Println("hogging") + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + for time.Now().Before(deadline) { + if _, err := x.JobInsertMany(ctx, []driver.JobInsertParams{{Kind: "hog", Queue: "hog", Priority: 2, MaxAttempts: 1}}, driver.JobInsertOpts{}); err != nil { + t.Error(err) + return + } + } + }) + } + wg.Wait() +} + +// TestChaos runs jobs in worker processes that write each job's effect and +// complete it in one transaction, and kills a process at random every few +// hundred milliseconds, often inside a transaction, starting another in +// its place. Every job ends completed with exactly one effect, and the +// database passes its integrity check. +func TestChaos(t *testing.T) { + t.Parallel() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "hopper.db") + d := openAt(t, path) + if _, err := d.DB().ExecContext(ctx, "CREATE TABLE effects (n INTEGER, attempt INTEGER)"); err != nil { + t.Fatal(err) + } + const jobs, procs = 600, 4 + c, err := hopper.NewClient(d, &hopper.Config{Logger: slog.New(slog.DiscardHandler)}) + if err != nil { + t.Fatal(err) + } + params := make([]hopper.InsertParams, jobs) + for i := range params { + params[i] = hopper.InsertParams{Args: chaosArgs{N: i}} + } + if _, err := c.InsertMany(ctx, params); err != nil { + t.Fatal(err) + } + var running []*exec.Cmd + defer func() { + for _, cmd := range running { + _ = cmd.Process.Kill() + _ = cmd.Wait() + } + }() + for range procs { + cmd, _ := helper(t, path, "worker") + running = append(running, cmd) + } + kills := 0 + live := func() int { + var n int + if err := d.DB().QueryRowContext(ctx, "SELECT count(*) FROM hopper_jobs").Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + for start := time.Now(); time.Since(start) < 5*time.Second && live() > 0; { + time.Sleep(time.Duration(100+rand.IntN(200)) * time.Millisecond) + i := rand.IntN(len(running)) + _ = running[i].Process.Signal(syscall.SIGKILL) + _ = running[i].Wait() + kills++ + running = slices.Delete(running, i, i+1) + cmd, _ := helper(t, path, "worker") + running = append(running, cmd) + } + left := live() + for deadline := time.Now().Add(time.Minute); left > 0 && time.Now().Before(deadline); left = live() { + time.Sleep(100 * time.Millisecond) + } + var completed, effects, distinct int + if err := d.DB().QueryRowContext(ctx, `SELECT + (SELECT count(*) FROM hopper_job_history WHERE state = 'completed'), + (SELECT count(*) FROM effects), (SELECT count(DISTINCT n) FROM effects)`).Scan(&completed, &effects, &distinct); err != nil { + t.Fatal(err) + } + t.Logf("killed %d processes; %d jobs left, %d completed, %d effects", kills, left, completed, effects) + if left != 0 || completed != jobs || effects != jobs || distinct != jobs { + t.Errorf("%d jobs: %d left, %d completed, %d effects of %d jobs; want every job completed with one effect", jobs, left, completed, effects, distinct) + } + var check string + if err := d.DB().QueryRowContext(ctx, "PRAGMA integrity_check").Scan(&check); err != nil || check != "ok" { + t.Errorf("integrity_check = %q, %v", check, err) + } +} + +// TestFairness writes in this process while another writes as fast as it +// can, from eight goroutines, for three seconds: writes here are held up, +// since SQLite's busy handler polls for the lock rather than queueing for +// it, but not for as long as the other process writes. It runs alone, not +// in parallel, for its waits to be the lock's. +func TestFairness(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "hopper.db") + d := openAt(t, path) + x := d.Executor() + cmd, out := helper(t, path, "hog") + for !strings.Contains(out.String(), "hogging") { + time.Sleep(5 * time.Millisecond) + } + var waits []time.Duration + for end := time.Now().Add(2 * time.Second); time.Now().Before(end); time.Sleep(20 * time.Millisecond) { + start := time.Now() + if err := x.QueueEnsure(ctx, []string{"q" + strconv.Itoa(len(waits))}); err != nil { + t.Fatal(err) + } + waits = append(waits, time.Since(start)) + } + if err := cmd.Wait(); err != nil { + t.Fatalf("hogging process: %v\n%s", err, out) + } + slices.Sort(waits) + p50, p90, worst := waits[len(waits)/2], waits[len(waits)*9/10], waits[len(waits)-1] + t.Logf("%d writes: p50 %s, p90 %s, max %s", len(waits), p50, p90, worst) + if worst > 2*time.Second { + t.Errorf("a write waited %s while another process wrote", worst) + } +} diff --git a/driver/hoppersqlite/cluster.go b/driver/hoppersqlite/cluster.go index 2a11e9b..db0401f 100644 --- a/driver/hoppersqlite/cluster.go +++ b/driver/hoppersqlite/cluster.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "os" "time" "github.com/parallelworks/hopper/driver" @@ -341,7 +342,7 @@ func (e *executor) Stats(ctx context.Context) (*driver.Stats, error) { } return q } - now := clock() + now := e.clock() err := e.readTx(ctx, func(q querier) error { rows, err := q.QueryContext(ctx, ` SELECT queue, state, count(*), @@ -551,9 +552,35 @@ func (e *executor) JobsMaintain(ctx context.Context) (driver.JobsMaintainResult, res.Analyzed = true } } + if err := e.truncateLog(ctx); err != nil { + errs = append(errs, err) + } return res, errors.Join(errs...) } +// walTruncateSize is the size of write-ahead log past which JobsMaintain +// checkpoints it and truncates it: twice what SQLite's automatic +// checkpoints, every thousand pages, let it reach. +const walTruncateSize = 8 << 20 + +// truncateLog checkpoints the write-ahead log and truncates it to nothing +// once it is past walTruncateSize. The log grows past that while a reader +// keeps a snapshot from before the checkpoints, which then cannot start it +// over, and SQLite cuts it back only to journalSizeLimit once one can. The +// checkpoint waits for writers and readers at most busyQuantum each, and +// gives up, to be tried at the next pass, if they are still there. +func (e *executor) truncateLog(ctx context.Context) error { + st, err := os.Stat(e.d.db.path + "-wal") + if err != nil || st.Size() <= walTruncateSize { + return nil //nolint:nilerr // no log, or none to speak of + } + var busy, logPages, checkpointed int64 + if err := e.d.sqlDB.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logPages, &checkpointed); err != nil { + return fmt.Errorf("hopper: checkpoint the write-ahead log: %w", err) + } + return nil +} + // believedRows returns the row count the planner's statistics hold for a // table, or -1 if it has none. func (e *executor) believedRows(ctx context.Context, table string) (int64, error) { diff --git a/driver/hoppersqlite/conn.go b/driver/hoppersqlite/conn.go index e2fe072..7bf09b2 100644 --- a/driver/hoppersqlite/conn.go +++ b/driver/hoppersqlite/conn.go @@ -5,6 +5,9 @@ import ( "database/sql/driver" "errors" "fmt" + "reflect" + "runtime" + "slices" "strings" "sync" "sync/atomic" @@ -12,6 +15,8 @@ import ( "modernc.org/sqlite" sqlite3 "modernc.org/sqlite/lib" + + hopperdriver "github.com/parallelworks/hopper/driver" ) // SQLite runs one writing transaction at a time, which it holds the @@ -30,6 +35,13 @@ import ( // on a channel, in turn and for as long as their statement's context // allows; across processes, SQLite's busy handler polls for it, in // quanta of busyQuantum, between which the context is checked. +// +// After some errors (an interrupted write, a full disk, an I/O error) +// SQLite rolls the whole transaction back itself, and would run the +// transaction's later statements on their own, each committing at once. +// SQLite's rollback hook tells the connection, which then refuses every +// later statement of the transaction (ErrTransactionAborted), as Postgres +// does those of a transaction an error aborted. // busyQuantum is the busy_timeout of every connection: how long SQLite's // busy handler waits for a lock another process holds before the statement @@ -37,9 +49,34 @@ import ( // context allows. const busyQuantum = 50 * time.Millisecond +// journalSizeLimit is the size SQLite truncates the write-ahead log to +// when a checkpoint has copied it all back and it starts over, so that a +// burst of writes, or a reader that held up checkpoints for a while, does +// not leave a large log behind for good. +const journalSizeLimit = 32 << 20 + +// ErrTransactionAborted is returned for a statement of a transaction that +// SQLite rolled back itself after an error, such as an interrupted write, a +// full disk or an I/O error: the transaction's earlier statements are +// undone, and its later ones are refused rather than run on their own, as +// Postgres refuses those of a transaction an error aborted. Roll it back; +// Commit rolls it back and returns this error too. +var ErrTransactionAborted = errors.New("hoppersqlite: the transaction was rolled back after an error and runs no more statements; roll it back") + +// ErrLockHeldByOwner is returned for a write that would wait for the +// database's write lock while a transaction of the same owner holds it (a +// job's attempt: driver.WithOwner), which would wait until its context +// ended and hold every other write up meanwhile. Write through that +// transaction instead, as InsertTx and the other Tx methods do, or end it +// first. A worker that writes from two goroutines at once, one of them in +// a transaction, gets it too, where Postgres would run both; give the +// other goroutine a context without the owner (driver.WithOwner(ctx, nil)) +// to have it wait its turn instead. +var ErrLockHeldByOwner = errors.New("hoppersqlite: the write would wait for the database's write lock, which a transaction of the same job attempt holds; write through that transaction, or end it first") + // database is what every connection to one database file in this process -// shares: the write lock and the listeners to wake when a transaction that -// notified commits. +// shares: the write lock, the clock, and the listeners to wake when a +// transaction that notified commits. type database struct { path string // lock holds a token while a connection of this process holds @@ -50,11 +87,34 @@ type database struct { waiting atomic.Int64 // refs counts the Drivers open on the database (registry.mu). refs int + // last is the latest time the clock read, in microseconds. + last atomic.Int64 + + holderMu sync.Mutex + holder holder listenMu sync.Mutex listeners map[chan struct{}]struct{} } +// holder is who holds the write lock: the owner its transaction runs for, +// or none, since when. +type holder struct { + owner fmt.Stringer + since time.Time +} + +func (h holder) String() string { + if h.since.IsZero() { + return "a transaction that has just ended" + } + who := "a transaction of this process" + if h.owner != nil { + who = "a transaction of " + h.owner.String() + } + return fmt.Sprintf("%s for %s", who, time.Since(h.since).Round(time.Millisecond)) +} + // registry holds the databases open in this process by path, so that two // Drivers on one file share its write lock. var registry = struct { @@ -82,25 +142,79 @@ func (db *database) close() { } } -// acquire takes the write lock for this process, waiting in turn for as -// long as ctx allows. -func (db *database) acquire(ctx context.Context) error { +// clock returns the database's time: the host's, in Postgres's precision, +// never earlier than a time it returned before in this process, so that a +// wall clock stepped back (by NTP, say) cannot date a write before the +// writes it follows. +func (db *database) clock() time.Time { + now := time.Now().UTC().Truncate(time.Microsecond).UnixMicro() + for { + last := db.last.Load() + if now <= last { + return time.UnixMicro(last).UTC() + } + if db.last.CompareAndSwap(last, now) { + return time.UnixMicro(now).UTC() + } + } +} + +// acquire takes the write lock for this process for owner, waiting in +// turn for as long as ctx allows. It refuses at once to wait for a lock a +// transaction of the same owner holds (ErrLockHeldByOwner), and a wait +// that ctx ends says who held the lock. +func (db *database) acquire(ctx context.Context, owner fmt.Stringer) error { select { case db.lock <- struct{}{}: + db.held(owner) return nil default: } + db.holderMu.Lock() + h := db.holder + db.holderMu.Unlock() + if sameOwner(h.owner, owner) { + return fmt.Errorf("%w (%s)", ErrLockHeldByOwner, owner) + } db.waiting.Add(1) defer db.waiting.Add(-1) + start := time.Now() select { case db.lock <- struct{}{}: + db.held(owner) return nil case <-ctx.Done(): - return ctx.Err() + db.holderMu.Lock() + h := db.holder + db.holderMu.Unlock() + return fmt.Errorf("hoppersqlite: waited %s for the database's write lock, held by %s: %w", + time.Since(start).Round(time.Millisecond), h, ctx.Err()) } } -func (db *database) release() { <-db.lock } +// sameOwner reports whether a and b are one owner. Owners compare by +// identity; one whose type cannot be compared is never the same as +// another. +func sameOwner(a, b fmt.Stringer) bool { + if a == nil || b == nil { + return false + } + t := reflect.TypeOf(a) + return t == reflect.TypeOf(b) && t.Comparable() && a == b +} + +func (db *database) held(owner fmt.Stringer) { + db.holderMu.Lock() + db.holder = holder{owner: owner, since: time.Now()} + db.holderMu.Unlock() +} + +func (db *database) release() { + db.holderMu.Lock() + db.holder = holder{} + db.holderMu.Unlock() + <-db.lock +} // listen registers a channel to be signaled when a transaction that wrote // notifications commits; the returned function unregisters it. @@ -143,11 +257,13 @@ func retryBusy[T any](ctx context.Context, db *database, fn func() (T, error)) ( } db.waiting.Add(1) defer db.waiting.Add(-1) + start := time.Now() for busy(err) && ctx.Err() == nil { v, err = fn() } if busy(err) { - return v, errors.Join(ctx.Err(), err) + return v, fmt.Errorf("hoppersqlite: waited %s for a lock another process holds on the database: %w", + time.Since(start).Round(time.Millisecond), errors.Join(ctx.Err(), err)) } return v, err } @@ -161,6 +277,7 @@ type innerConn interface { driver.Pinger driver.SessionResetter driver.Validator + sqlite.HookRegisterer } // connector opens the connections of a Driver's database/sql handle. @@ -172,15 +289,22 @@ type connector struct { var _ driver.Connector = (*connector)(nil) -// pragmas sets up a connection: the busy handler's quantum, the -// durability of its commits, and foreign keys off, as SQLite has them by -// default and hopper's schema has none. +// pragmas sets up a connection: the busy handler's quantum; the +// durability of its commits, on macOS with F_FULLFSYNC, without which fsync +// leaves the commit in the disk's cache; the size the write-ahead log is +// cut back to; and foreign keys off, as SQLite has them by default and +// hopper's schema has none. func (c *connector) pragmas() []string { - return []string{ + p := []string{ fmt.Sprintf("PRAGMA busy_timeout = %d", busyQuantum.Milliseconds()), "PRAGMA synchronous = " + c.synchronous, + fmt.Sprintf("PRAGMA journal_size_limit = %d", journalSizeLimit), "PRAGMA foreign_keys = OFF", } + if runtime.GOOS == "darwin" && c.synchronous == "FULL" { + p = append(p, "PRAGMA fullfsync = ON", "PRAGMA checkpoint_fullfsync = ON") + } + return p } func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { @@ -199,7 +323,11 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { return nil, fmt.Errorf("hoppersqlite: %s: %w", p, err) } } - return &conn{inner: inner, db: c.db}, nil + cn := &conn{inner: inner, db: c.db} + // SQLite calls the hook on the goroutine running the statement that + // rolls a transaction back, whether by ROLLBACK or after an error. + inner.RegisterRollbackHook(func() { cn.rolledBack = true }) + return cn, nil } func (c *connector) Driver() driver.Driver { return connectorDriver{c} } @@ -216,6 +344,9 @@ type conn struct { inner innerConn db *database tx *tx + // rolledBack is set by SQLite's rollback hook when a transaction rolls + // back; a statement resets it before it runs. + rolledBack bool } var ( @@ -232,12 +363,19 @@ var ( type tx struct { c *conn readOnly bool + // owner is who the transaction runs for (driver.Owner of the context + // it began with). + owner fmt.Stringer // begun reports that BEGIN has run, and locked that the transaction // holds this process's write lock. begun, locked bool + // read reports that a read-only transaction has run a statement. + read bool // notified reports that the transaction wrote notifications, so that // its commit wakes this process's listeners. notified bool + // aborted is the error after which SQLite rolled the transaction back. + aborted error } func (c *conn) exec(ctx context.Context, query string) error { @@ -279,12 +417,12 @@ func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, e if c.tx != nil { return nil, errors.New("hoppersqlite: a transaction is already open on this connection") } - t := &tx{c: c, readOnly: opts.ReadOnly} + t := &tx{c: c, readOnly: opts.ReadOnly, owner: hopperdriver.Owner(ctx)} if opts.ReadOnly { if err := c.exec(ctx, "PRAGMA query_only = ON"); err != nil { return nil, err } - if _, err := retryBusy(ctx, c.db, func() (struct{}, error) { return struct{}{}, c.exec(ctx, "BEGIN") }); err != nil { + if err := c.exec(ctx, "BEGIN"); err != nil { _ = c.exec(context.Background(), "PRAGMA query_only = OFF") return nil, err } @@ -296,7 +434,7 @@ func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, e // beginWrite takes the write lock for the open transaction and begins it. func (c *conn) beginWrite(ctx context.Context) error { - if err := c.db.acquire(ctx); err != nil { + if err := c.db.acquire(ctx, c.tx.owner); err != nil { return err } if _, err := retryBusy(ctx, c.db, func() (struct{}, error) { return struct{}{}, c.exec(ctx, "BEGIN IMMEDIATE") }); err != nil { @@ -307,51 +445,145 @@ func (c *conn) beginWrite(ctx context.Context) error { return nil } -// statement prepares the connection to run query: inside a transaction, -// it begins the transaction if this is its first statement, and notes a -// notification written; outside one, a statement that may write (exec) -// takes the write lock for its run, which done releases. -func (c *conn) statement(ctx context.Context, query string, write bool) (done func(), err error) { - if c.tx == nil { - if !write { - return func() {}, nil +// abortedError is the error of a statement refused in an aborted +// transaction. It names the cause but does not wrap it: a statement +// refused after an earlier one was interrupted did not see its own context +// end. +func abortedError(cause error) error { + return fmt.Errorf("%w (after: %v)", ErrTransactionAborted, cause) //nolint:errorlint // see above +} + +// enter prepares the open transaction for a statement: it refuses one +// after SQLite rolled the transaction back, begins the transaction on its +// first statement, notes a notification written, and resets the rollback +// hook's flag. It reports whether the statement may meet SQLITE_BUSY that +// can be retried: the first of a read-only transaction, which has read no +// snapshot yet. +func (c *conn) enter(ctx context.Context, query string) (retry bool, err error) { + t := c.tx + if t.aborted != nil { + return false, abortedError(t.aborted) + } + if !t.begun { + if err := c.beginWrite(ctx); err != nil { + return false, err } - if err := c.db.acquire(ctx); err != nil { - return nil, err + } + if query == notifySQL { + t.notified = true + } + retry = t.readOnly && !t.read + t.read = true + c.rolledBack = false + return retry, nil +} + +// leave notes whether a statement of the open transaction, ending with +// err, made SQLite roll the transaction back. +func (c *conn) leave(err error) { + if c.tx != nil && c.rolledBack && c.tx.aborted == nil { + if err == nil { + err = errors.New("SQLite rolled the transaction back") } - return c.db.release, nil + c.tx.aborted = err } - if !c.tx.begun { - if err := c.beginWrite(ctx); err != nil { +} + +// runExec runs a statement that may write: in the open transaction, or on +// its own, holding the write lock while it runs. +func (c *conn) runExec(ctx context.Context, query string, run func() (driver.Result, error)) (driver.Result, error) { + if c.tx == nil { + if err := c.db.acquire(ctx, hopperdriver.Owner(ctx)); err != nil { return nil, err } + defer c.db.release() + return retryBusy(ctx, c.db, run) } - if query == notifySQL { - c.tx.notified = true + if _, err := c.enter(ctx, query); err != nil { + return nil, err } - return func() {}, nil + res, err := run() + c.leave(err) + return res, err } -func (c *conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { - done, err := c.statement(ctx, query, true) +// runQuery runs a query: in the open transaction, or on its own. On its +// own, one that may write (mayWrite) holds the write lock until its rows +// are closed, as a statement that writes in autocommit holds SQLite's +// until it is done. +func (c *conn) runQuery(ctx context.Context, query string, run func() (driver.Rows, error)) (driver.Rows, error) { + if c.tx == nil { + if !mayWrite(query) { + return retryBusy(ctx, c.db, run) + } + if err := c.db.acquire(ctx, hopperdriver.Owner(ctx)); err != nil { + return nil, err + } + r, err := retryBusy(ctx, c.db, run) + if err != nil { + c.db.release() + return nil, err + } + return &rows{inner: r, release: c.db.release}, nil + } + retry, err := c.enter(ctx, query) if err != nil { return nil, err } - defer done() - if c.tx != nil { - return c.inner.ExecContext(ctx, query, args) + var r driver.Rows + if retry { + r, err = retryBusy(ctx, c.db, run) + } else { + r, err = run() } - return retryBusy(ctx, c.db, func() (driver.Result, error) { return c.inner.ExecContext(ctx, query, args) }) -} - -func (c *conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { - if _, err := c.statement(ctx, query, false); err != nil { + c.leave(err) + if err != nil { return nil, err } - if c.tx != nil { - return c.inner.QueryContext(ctx, query, args) + return &rows{inner: r, c: c, t: c.tx}, nil +} + +// writeKeywords are the first words of the statements that may write. +var writeKeywords = []string{"INSERT", "UPDATE", "DELETE", "REPLACE", "CREATE", "DROP", "ALTER", "VACUUM", "REINDEX", "ANALYZE"} + +// mayWrite reports whether a query, run on its own, may write: by its +// first word after any comments, for a WITH statement by whether INSERT, +// UPDATE, DELETE or REPLACE appears in it at all, and for a PRAGMA by +// whether it sets a value. +func mayWrite(query string) bool { + q := strings.TrimSpace(query) + for strings.HasPrefix(q, "--") || strings.HasPrefix(q, "/*") { + end := "\n" + if q[1] == '*' { + end = "*/" + } + _, rest, ok := strings.Cut(q, end) + if !ok { + return false + } + q = strings.TrimSpace(rest) + } + n := strings.IndexFunc(q, func(r rune) bool { return (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') }) + if n < 0 { + n = len(q) + } + switch word := strings.ToUpper(q[:n]); word { + case "WITH": + upper := strings.ToUpper(q) + return slices.ContainsFunc(writeKeywords[:4], func(k string) bool { return strings.Contains(upper, k) }) + case "PRAGMA": + return strings.Contains(q, "=") + default: + return slices.Contains(writeKeywords, word) } - return retryBusy(ctx, c.db, func() (driver.Rows, error) { return c.inner.QueryContext(ctx, query, args) }) +} + +func (c *conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + return c.runExec(ctx, query, func() (driver.Result, error) { return c.inner.ExecContext(ctx, query, args) }) +} + +func (c *conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + return c.runQuery(ctx, query, func() (driver.Rows, error) { return c.inner.QueryContext(ctx, query, args) }) } func (c *conn) Ping(ctx context.Context) error { return c.inner.Ping(ctx) } @@ -376,11 +608,17 @@ func (t *tx) end() { t.c.tx = nil } +// Commit commits the transaction, or, if SQLite rolled it back after an +// error, rolls back what is left of it and returns ErrTransactionAborted. func (t *tx) Commit() error { defer t.end() if !t.begun { return nil } + if t.aborted != nil { + _ = t.rollback() + return abortedError(t.aborted) + } if err := t.c.exec(context.Background(), "COMMIT"); err != nil { // A COMMIT that failed can leave the transaction open, which // database/sql does not expect of a connection it reuses. @@ -402,8 +640,8 @@ func (t *tx) Rollback() error { } // rollback rolls the transaction back. SQLite rolls a transaction back -// itself after some errors (a full disk, an I/O error), which leaves -// nothing to roll back here. +// itself after some errors (a full disk, an I/O error, an interrupted +// write), which leaves nothing to roll back here. func (t *tx) rollback() error { err := t.c.exec(context.Background(), "ROLLBACK") if err != nil && strings.Contains(err.Error(), "no transaction is active") { @@ -412,6 +650,84 @@ func (t *tx) rollback() error { return err } +// rows reads a query's rows. In a transaction, it notes a rollback SQLite +// makes while it steps; on its own, a query that may write holds the write +// lock until it is closed. +type rows struct { + inner driver.Rows + c *conn + t *tx + release func() +} + +var ( + _ driver.RowsColumnTypeDatabaseTypeName = (*rows)(nil) + _ driver.RowsColumnTypeLength = (*rows)(nil) + _ driver.RowsColumnTypeNullable = (*rows)(nil) + _ driver.RowsColumnTypePrecisionScale = (*rows)(nil) + _ driver.RowsColumnTypeScanType = (*rows)(nil) +) + +func (r *rows) Columns() []string { return r.inner.Columns() } + +func (r *rows) Close() error { + err := r.inner.Close() + if r.release != nil { + r.release() + r.release = nil + } + return err +} + +func (r *rows) Next(dest []driver.Value) error { + if r.t == nil { + return r.inner.Next(dest) + } + if r.t.aborted != nil { + return abortedError(r.t.aborted) + } + err := r.inner.Next(dest) + if r.c.tx == r.t && r.c.rolledBack && r.t.aborted == nil { + r.c.leave(err) + } + return err +} + +func (r *rows) ColumnTypeDatabaseTypeName(index int) string { + if t, ok := r.inner.(driver.RowsColumnTypeDatabaseTypeName); ok { + return t.ColumnTypeDatabaseTypeName(index) + } + return "" +} + +func (r *rows) ColumnTypeLength(index int) (int64, bool) { + if t, ok := r.inner.(driver.RowsColumnTypeLength); ok { + return t.ColumnTypeLength(index) + } + return 0, false +} + +func (r *rows) ColumnTypeNullable(index int) (bool, bool) { + if t, ok := r.inner.(driver.RowsColumnTypeNullable); ok { + return t.ColumnTypeNullable(index) + } + return false, false +} + +func (r *rows) ColumnTypePrecisionScale(index int) (int64, int64, bool) { + if t, ok := r.inner.(driver.RowsColumnTypePrecisionScale); ok { + return t.ColumnTypePrecisionScale(index) + } + return 0, 0, false +} + +func (r *rows) ColumnTypeScanType(index int) reflect.Type { + if t, ok := r.inner.(driver.RowsColumnTypeScanType); ok { + return t.ColumnTypeScanType(index) + } + return reflect.TypeFor[any]() +} + // innerStmt is what a statement of modernc's driver implements. type innerStmt interface { driver.Stmt @@ -444,25 +760,11 @@ func (s *stmt) Query(args []driver.Value) (driver.Rows, error) { } func (s *stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { - done, err := s.c.statement(ctx, s.query, true) - if err != nil { - return nil, err - } - defer done() - if s.c.tx != nil { - return s.inner.ExecContext(ctx, args) - } - return retryBusy(ctx, s.c.db, func() (driver.Result, error) { return s.inner.ExecContext(ctx, args) }) + return s.c.runExec(ctx, s.query, func() (driver.Result, error) { return s.inner.ExecContext(ctx, args) }) } func (s *stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { - if _, err := s.c.statement(ctx, s.query, false); err != nil { - return nil, err - } - if s.c.tx != nil { - return s.inner.QueryContext(ctx, args) - } - return retryBusy(ctx, s.c.db, func() (driver.Rows, error) { return s.inner.QueryContext(ctx, args) }) + return s.c.runQuery(ctx, s.query, func() (driver.Rows, error) { return s.inner.QueryContext(ctx, args) }) } func named(args []driver.Value) []driver.NamedValue { diff --git a/driver/hoppersqlite/conn_internal_test.go b/driver/hoppersqlite/conn_internal_test.go new file mode 100644 index 0000000..90783e4 --- /dev/null +++ b/driver/hoppersqlite/conn_internal_test.go @@ -0,0 +1,87 @@ +package hoppersqlite + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +// TestMayWrite checks which statements run on their own take the write +// lock. +func TestMayWrite(t *testing.T) { + for query, want := range map[string]bool{ + "SELECT 1": false, + "select * from t where x = 'INSERT'": false, + "INSERT INTO t VALUES (1) RETURNING x": true, + " -- note\n/* more */ update t SET x = 1": true, + "DELETE FROM t": true, + "REPLACE INTO t VALUES (1)": true, + "WITH c AS (SELECT 1) INSERT INTO t SELECT * FROM c": true, + "WITH c AS (SELECT 1) SELECT * FROM c": false, + "PRAGMA journal_mode": false, + "PRAGMA journal_mode = WAL": true, + "CREATE TABLE t (x)": true, + "ANALYZE t": true, + "-- only a comment": false, + "INSERT\nINTO t VALUES (1)": true, + "delete\tFROM t": true, + } { + if got := mayWrite(query); got != want { + t.Errorf("mayWrite(%q) = %v, want %v", query, got, want) + } + } +} + +// TestClock checks that the database's clock never goes back, whatever +// the host's does. +func TestClock(t *testing.T) { + db := &database{} + first := db.clock() + if second := db.clock(); second.Before(first) { + t.Errorf("the clock went back from %s to %s", first, second) + } + ahead := time.Now().Add(time.Hour).UnixMicro() + db.last.Store(ahead) // as if the host's clock was stepped back an hour + if got := db.clock(); got.UnixMicro() != ahead { + t.Errorf("clock = %s, want the latest time it read, %s", got, time.UnixMicro(ahead)) + } +} + +type owner struct{ name string } + +func (o *owner) String() string { return o.name } + +type value []int + +func (value) String() string { return "value" } + +// TestOwnerWaits checks that a write fails at once, naming the owner, when +// a transaction of its own owner holds the lock, and that one of another +// owner waits, saying who holds the lock when its context ends. +func TestOwnerWaits(t *testing.T) { + db := &database{lock: make(chan struct{}, 1)} + a, b := &owner{"job a"}, &owner{"job b"} + if err := db.acquire(context.Background(), a); err != nil { + t.Fatal(err) + } + err := db.acquire(context.Background(), a) + if !errors.Is(err, ErrLockHeldByOwner) || !strings.Contains(err.Error(), "job a") { + t.Errorf("acquire by the holder's owner = %v, want ErrLockHeldByOwner", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + err = db.acquire(ctx, b) + if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), "job a") { + t.Errorf("acquire by another owner = %v, want it to wait and name the holder", err) + } + if sameOwner(value{1}, value{1}) || sameOwner(nil, nil) || !sameOwner(a, a) || sameOwner(a, b) { + t.Error("sameOwner compares owners other than by identity, or panics") + } + db.release() + if err := db.acquire(context.Background(), b); err != nil { + t.Fatal(err) + } + db.release() +} diff --git a/driver/hoppersqlite/executor.go b/driver/hoppersqlite/executor.go index 799ea00..78b601b 100644 --- a/driver/hoppersqlite/executor.go +++ b/driver/hoppersqlite/executor.go @@ -35,9 +35,8 @@ type querier interface { // lock (writeBrief). var errLockBusy = errors.New("hoppersqlite: another transaction holds the database's write lock") -// clock returns the database's time, which is the host's (see the package -// documentation), in Postgres's precision. -func clock() time.Time { return time.Now().UTC().Truncate(time.Microsecond) } +// clock returns the database's time (database.clock). +func (e *executor) clock() time.Time { return e.d.db.clock() } // w is a write in progress: a transaction that holds the write lock, and // its time, read once the lock was taken, so that every write sees a time @@ -199,7 +198,7 @@ func (e *executor) writeLocking(ctx context.Context, wait time.Duration, fn func err = errors.Join(err, rerr) } }() - if err := fn(&w{cached: q, now: clock()}); err != nil { + if err := fn(&w{cached: q, now: e.clock()}); err != nil { return err } _, err := q.ExecContext(ctx, "RELEASE hopper") @@ -218,7 +217,7 @@ func (e *executor) writeLocking(ctx context.Context, wait time.Duration, fn func if err := lock(q, "SELECT 1"); err != nil { return err } - if err := fn(&w{cached: q, now: clock()}); err != nil { + if err := fn(&w{cached: q, now: e.clock()}); err != nil { return err } return tx.Commit() @@ -289,7 +288,7 @@ func (e *executor) Notify(ctx context.Context, channel string, payloads []string } // Now implements driver.Executor. -func (e *executor) Now(context.Context) (time.Time, error) { return clock(), nil } +func (e *executor) Now(context.Context) (time.Time, error) { return e.clock(), nil } // SchemaVersion implements driver.Executor. func (e *executor) SchemaVersion(ctx context.Context) (int, error) { @@ -455,14 +454,14 @@ func smallint(v int, column string) (int, error) { return v, nil } -// jsonValue is a JSON parameter: raw, or def when empty. It refuses text -// that is not JSON, as Postgres's jsonb does. +// jsonValue is a JSON parameter: raw, or def when empty. It refuses what +// Postgres's jsonb refuses (checkJSON). func jsonValue(raw []byte, def, what string) (string, error) { if len(raw) == 0 { return def, nil } - if !json.Valid(raw) { - return "", fmt.Errorf("hopper: %s is not JSON: %.40q", what, raw) + if err := checkJSON(raw); err != nil { + return "", fmt.Errorf("hopper: %s %w: %.40q", what, err, raw) } return string(raw), nil } diff --git a/driver/hoppersqlite/funcs.go b/driver/hoppersqlite/funcs.go index 0800684..af45021 100644 --- a/driver/hoppersqlite/funcs.go +++ b/driver/hoppersqlite/funcs.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "regexp" "strconv" "strings" @@ -273,80 +272,89 @@ const ( maxNumericFraction = 16383 ) -// checkJSONFilter refuses a metadata filter Postgres's jsonb input would, -// so that a listing's filter means the same on either engine: text that is -// not UTF-8, a \u0000 escape, an unpaired surrogate escape, or a number -// beyond numeric's range. -func checkJSONFilter(s string) error { - if !utf8.ValidString(s) { - return fmt.Errorf("the metadata filter is not UTF-8") - } - if !json.Valid([]byte(s)) { - return fmt.Errorf("the metadata filter is not JSON") +// checkJSON refuses JSON text Postgres's jsonb input refuses, so that a +// value means the same on either engine and a job stored on one could be +// stored on the other: text that is not JSON or not UTF-8, a \u0000 escape +// (text cannot hold NUL), an unpaired surrogate escape, or a number beyond +// the range of Postgres's numeric. It reads the text once, outside its +// strings for numbers and inside them for escapes. +func checkJSON(b []byte) error { + if !utf8.Valid(b) { + return errors.New("is not UTF-8") } - if err := checkEscapes(s); err != nil { - return err - } - dec := json.NewDecoder(strings.NewReader(s)) - dec.UseNumber() - for { - tok, err := dec.Token() - if errors.Is(err, io.EOF) { - return nil - } - if err != nil { - return fmt.Errorf("the metadata filter is not JSON: %w", err) - } - n, ok := tok.(json.Number) - if !ok { - continue - } - d, err := parseDecimal(string(n)) - if err != nil { - return fmt.Errorf("the metadata filter's %w", err) - } - if whole := int64(len(d.digits)) + d.exp; d.digits != "" && whole > maxNumericWhole { - return fmt.Errorf("the metadata filter's number %.20s... is beyond the range of Postgres's numeric", n) - } - if d.exp < -maxNumericFraction { - return fmt.Errorf("the metadata filter's number %.20s... has more decimals than Postgres's numeric holds", n) - } + if !json.Valid(b) { + return errors.New("is not JSON") } -} - -// checkEscapes refuses a \u0000 escape and an unpaired surrogate escape in -// a JSON text's strings, which decoding would turn into a NUL and U+FFFD. -func checkEscapes(s string) error { - b := []byte(s) + inString := false for i := 0; i < len(b); i++ { - if b[i] != '\\' { - continue - } - if i+1 < len(b) && b[i+1] != 'u' { - i++ - continue - } - if i+6 > len(b) { - return nil // the decoder reports the truncated escape - } - r, err := strconv.ParseUint(string(b[i+2:i+6]), 16, 16) + c := b[i] switch { - case err != nil: - // The decoder reports the malformed escape. - case r == 0: - return fmt.Errorf("the metadata filter has a \\u0000 escape") - case utf16.IsSurrogate(rune(r)): - if r >= 0xdc00 || i+12 > len(b) || !bytes.HasPrefix(b[i+6:], []byte(`\u`)) { - return fmt.Errorf("the metadata filter has an unpaired surrogate escape") + case inString && c == '"': + inString = false + case inString && c == '\\': + n, err := checkEscape(b[i:]) + if err != nil { + return err } - low, err := strconv.ParseUint(string(b[i+8:i+12]), 16, 16) - if err != nil || low < 0xdc00 || low > 0xdfff { - return fmt.Errorf("the metadata filter has an unpaired surrogate escape") + i += n - 1 + case inString: + case c == '"': + inString = true + case c == '-' || c >= '0' && c <= '9': + end := i + for end < len(b) && strings.IndexByte("+-.eE0123456789", b[end]) >= 0 { + end++ } - i += 11 - default: - i += 5 + if err := checkNumber(string(b[i:end])); err != nil { + return err + } + i = end - 1 } } return nil } + +// checkEscape checks the escape at the start of b, inside a string of valid +// JSON, and returns its length. +func checkEscape(b []byte) (int, error) { + if b[1] != 'u' { + return 2, nil + } + r, _ := strconv.ParseUint(string(b[2:6]), 16, 16) + switch { + case r == 0: + return 0, errors.New("has a \\u0000 escape") + case !utf16.IsSurrogate(rune(r)): + return 6, nil + case r >= 0xdc00 || len(b) < 12 || !bytes.HasPrefix(b[6:], []byte(`\u`)): + return 0, errors.New("has an unpaired surrogate escape") + } + if low, err := strconv.ParseUint(string(b[8:12]), 16, 16); err != nil || low < 0xdc00 || low > 0xdfff { + return 0, errors.New("has an unpaired surrogate escape") + } + return 12, nil +} + +// checkNumber refuses a number beyond the range of Postgres's numeric. +func checkNumber(n string) error { + d, err := parseDecimal(n) + if err != nil { + return err + } + if whole := int64(len(d.digits)) + d.exp; d.digits != "" && whole > maxNumericWhole { + return fmt.Errorf("has a number, %.20s..., beyond the range of Postgres's numeric", n) + } + if d.exp < -maxNumericFraction { + return fmt.Errorf("has a number, %.20s..., with more decimals than Postgres's numeric holds", n) + } + return nil +} + +// checkJSONFilter refuses a metadata filter Postgres's jsonb input would +// (checkJSON), so that a listing's filter means the same on either engine. +func checkJSONFilter(s string) error { + if err := checkJSON([]byte(s)); err != nil { + return fmt.Errorf("the metadata filter %w", err) + } + return nil +} diff --git a/driver/hoppersqlite/hoppersqlite.go b/driver/hoppersqlite/hoppersqlite.go index dc3d88b..65d6fdf 100644 --- a/driver/hoppersqlite/hoppersqlite.go +++ b/driver/hoppersqlite/hoppersqlite.go @@ -9,39 +9,69 @@ // client, err := hopper.NewClient(d, cfg) // // It passes the drivertest conformance suite, as the Postgres drivers do, -// and several processes may share one database. What SQLite's design makes -// different from Postgres: +// and several processes may share one database. It uses internal packages +// of hopper's module, so it works with the hopper release of its own +// version: upgrade the two together. What SQLite's design makes different +// from Postgres: // // - One transaction writes at a time (driver.Capabilities.SingleWriter). // A transaction that writes holds the database's write lock until it // ends, and every other write waits for it: within this process in // turn, for as long as its context allows, and across processes -// through SQLite's busy handler. A claim is exclusive without SKIP -// LOCKED, and the transaction that finalizes a job settles its batch -// and promotes its dependents itself. A finalize with SkipLocked, or a -// progress write with NoWait, waits 100ms for a write lock held -// elsewhere, and then skips every job, or returns driver.ErrJobLocked. +// through SQLite's busy handler, which polls rather than queues, so a +// process that writes without pause holds another's writes up for +// longer than its own. A claim is exclusive without SKIP LOCKED, and +// the transaction that finalizes a job settles its batch and promotes +// its dependents itself. A finalize with SkipLocked, or a progress +// write with NoWait, waits 100ms for a write lock held elsewhere, and +// then skips every job, or returns driver.ErrJobLocked. +// - So a transaction held open holds up every client that shares the +// database, their lease renewals included: one held for longer than +// about the lease (hopper.Config.LeaseTTL) costs them their leases, +// and their running jobs are rescued. Keep transactions short. // - Transactions come from DB, whose connections take the write lock // when a transaction runs its first statement, not when it begins, so // that what it reads is never stale by the time it writes, and so that // beginning one never blocks. A transaction begun read-only takes no // lock and may not write. Within a transaction, write through it // (InsertTx and the like), not through the client: a write on the pool -// would wait for the transaction's own lock. +// would wait for the transaction's own lock. In a job's worker, whose +// context names the job's attempt, such a write fails at once with +// ErrLockHeldByOwner instead. +// - After some errors (an interrupted write, a full disk, an I/O error) +// SQLite rolls the whole transaction back itself. The transaction's +// later statements, and its Commit, then fail with +// ErrTransactionAborted, as those of a transaction an error aborted do +// on Postgres. An error that SQLite undoes only the statement of, such +// as a constraint's, leaves the transaction open, where Postgres would +// abort it. // - The database must be on a local filesystem: Open refuses one on a -// network filesystem with ErrNetworkFilesystem. Its clock is therefore -// the host's, which every process that opens it shares, and the driver -// reads it as database time. +// network filesystem, or a virtual machine's shared folder, with +// ErrNetworkFilesystem, checking where symbolic links in the path +// lead. Its clock is therefore the host's, which every process that +// opens it shares, and the driver reads it as database time, never +// earlier than a time it read before in the process. A host that +// sleeps and wakes finds its clock moved on, as a database server's +// would, and may rescue the jobs of its own clients, whose leases +// expired while it slept. +// - Commits are durable once they return (Config.Synchronous), on macOS +// with F_FULLFSYNC. The write-ahead log is cut back to 32MiB once a +// checkpoint empties it, and JobsMaintain truncates it once a reader +// that held up checkpoints is done. // - Notifications are rows of a table, delivered on commit to listeners // in every process that opens the database: at once in this process, // and within ListenInterval in others. -// - History is one table, and retention deletes its rows in batches; a -// stream's events commit in the order they are appended, so a -// position is an event's sequence number (Xid is always 0). +// - History is one table, and retention deletes its rows in batches, +// without returning the space to the filesystem; JobsMaintain +// refreshes statistics but never vacuums. A stream's events commit in +// the order they are appended, so a position is an event's sequence +// number (Xid is always 0). // - Args, metadata and other JSON values are stored as given, not -// normalized as Postgres's jsonb normalizes them. A metadata filter -// matches as jsonb's @> does, and is refused for what jsonb refuses. -// - There is no COPY: bulk inserts go through the one insert path. +// normalized as Postgres's jsonb normalizes them, and refused for what +// jsonb refuses. A metadata filter matches as jsonb's @> does. +// - Job IDs are minted in Go, from crypto/rand, rather than by the +// database. There is no COPY: bulk inserts go through the one insert +// path. Nor are there SQL functions to insert jobs from SQL. package hoppersqlite import ( @@ -59,15 +89,18 @@ import ( "time" "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/hoppermigrate" ) // Config tunes the SQLite driver. A nil *Config means the defaults. type Config struct { // Synchronous is SQLite's synchronous setting for the database's // connections: "FULL" (the default), which makes every commit durable - // before it returns, as Postgres's default does, or "NORMAL", with which - // a power failure, not a crash of the process, can lose the last commits - // before a checkpoint, and commits take less time. + // before it returns, as Postgres's default does (on macOS with + // F_FULLFSYNC, without which the disk may hold the commit in its + // cache), or "NORMAL", with which a power failure, not a crash of the + // process, can lose the last commits before a checkpoint, and commits + // take less time. Synchronous string // ListenInterval is how often a listener looks for notifications // committed by other processes. Those of this process wake it at once. @@ -95,10 +128,25 @@ type Driver struct { var _ driver.Driver[*sql.Tx] = (*Driver)(nil) -// Open opens the SQLite database at path, creating it if needed, in -// write-ahead log mode. It returns ErrNetworkFilesystem for a path on a -// network filesystem. Close the Driver to close the database. +// DefaultOpenTimeout is how long Open waits for the database: for another +// process that holds it while it creates it or recovers its log, say. +const DefaultOpenTimeout = 30 * time.Second + +// Open opens the SQLite database at path as OpenContext does, waiting at +// most DefaultOpenTimeout. func Open(path string, cfg *Config) (*Driver, error) { + ctx, cancel := context.WithTimeout(context.Background(), DefaultOpenTimeout) + defer cancel() + return OpenContext(ctx, path, cfg) +} + +// OpenContext opens the SQLite database at path, creating it if needed, in +// write-ahead log mode, waiting for as long as ctx allows for another +// process that holds it. A symbolic link in the path is resolved, as +// SQLite resolves it, and the database opened where it leads. OpenContext +// returns ErrNetworkFilesystem for a database on a network filesystem. +// Close the Driver to close the database. +func OpenContext(ctx context.Context, path string, cfg *Config) (*Driver, error) { var c Config if cfg != nil { c = *cfg @@ -120,27 +168,43 @@ func Open(path string, cfg *Config) (*Driver, error) { if path == "" || path == ":memory:" || strings.HasPrefix(path, "file:") { return nil, fmt.Errorf("hoppersqlite: %q is not a file path; the database must be a file, which every connection and process opens", path) } - abs, err := filepath.Abs(path) + resolved, err := resolve(path) if err != nil { return nil, fmt.Errorf("hoppersqlite: %w", err) } - if err := checkLocal(abs); err != nil { + if err := checkLocal(resolved); err != nil { return nil, err } - db := openDatabase(abs) - sqlDB := sql.OpenDB(&connector{db: db, dsn: dsn(abs), synchronous: sync}) + db := openDatabase(resolved) + sqlDB := sql.OpenDB(&connector{db: db, dsn: dsn(resolved), synchronous: sync}) sqlDB.SetMaxIdleConns(16) d := &Driver{sqlDB: sqlDB, db: db, listenInterval: c.ListenInterval} - var mode string - if err := sqlDB.QueryRowContext(context.Background(), "PRAGMA journal_mode = WAL").Scan(&mode); err != nil { + if err := d.setUp(ctx, resolved); err != nil { _ = d.Close() - return nil, fmt.Errorf("hoppersqlite: open %s: %w", abs, err) + return nil, fmt.Errorf("hoppersqlite: open %s: %w", resolved, err) + } + return d, nil +} + +// setUp puts a database just opened in write-ahead log mode, and checks +// that the file SQLite opened is where Open checked: on a local +// filesystem. +func (d *Driver) setUp(ctx context.Context, resolved string) error { + var mode string + if err := d.sqlDB.QueryRowContext(ctx, "PRAGMA journal_mode = WAL").Scan(&mode); err != nil { + return err } if !strings.EqualFold(mode, "wal") { - _ = d.Close() - return nil, fmt.Errorf("hoppersqlite: open %s: the database is in journal mode %s and cannot use a write-ahead log", abs, mode) + return fmt.Errorf("the database is in journal mode %s and cannot use a write-ahead log", mode) } - return d, nil + var file string + if err := d.sqlDB.QueryRowContext(ctx, "SELECT file FROM pragma_database_list WHERE name = 'main'").Scan(&file); err != nil { + return err + } + if file != resolved { + return checkLocal(file) + } + return nil } // dsn is the URI of the database file at an absolute path, for modernc's @@ -156,6 +220,9 @@ func dsn(abs string) string { // DB returns the database/sql handle the driver uses. Begin the // transactions you pass to InsertTx and the like on it: its connections // take the database's write lock as a transaction's first statement runs. +// Leave its limit on open connections alone: each listener, and the +// migrator, holds a connection of its own for as long as it runs, and a +// limit that left a transaction without one would wait for them. func (d *Driver) DB() *sql.DB { return d.sqlDB } // Close closes the database. @@ -212,6 +279,19 @@ var scripts = func() map[int]map[bool]string { return out }() +// Migrations returns SQLite's migration scripts, one for every schema +// version hoppermigrate knows, in ascending order, for an application that +// applies its schema with a tool of its own rather than hoppermigrate.Up, +// whose Migrations returns Postgres's. Record each version applied in +// hopper_schema, as Up does. +func Migrations() []hoppermigrate.Migration { + ms := hoppermigrate.Migrations() + for i := range ms { + ms[i].Up, ms[i].Down = scripts[ms[i].Version][true], scripts[ms[i].Version][false] + } + return ms +} + // migrator implements driver.Migrator. Its lock is SQLite's write lock, // held from Lock to Close on a connection of its own: each version applies // in a savepoint of that transaction, and Close commits the versions that diff --git a/driver/hoppersqlite/jobs.go b/driver/hoppersqlite/jobs.go index fe72a73..fc43bd0 100644 --- a/driver/hoppersqlite/jobs.go +++ b/driver/hoppersqlite/jobs.go @@ -828,7 +828,7 @@ func (e *executor) JobDiscardExpired(ctx context.Context, limit int) ([]*driver. // client has no live lease, and, with StuckAfter, those claimed longer ago // than that, oldest first. func (e *executor) JobRescueCandidates(ctx context.Context, params driver.JobRescueParams) ([]driver.RescueCandidate, error) { - now := clock() + now := e.clock() stuckBefore := int64(math.MinInt64) if params.StuckAfter > 0 { stuckBefore = now.Add(-params.StuckAfter).UnixMicro() diff --git a/driver/hoppersqlite/localfs.go b/driver/hoppersqlite/localfs.go index 6bacaca..b6e4426 100644 --- a/driver/hoppersqlite/localfs.go +++ b/driver/hoppersqlite/localfs.go @@ -3,6 +3,8 @@ package hoppersqlite import ( "errors" "fmt" + "io/fs" + "os" "path/filepath" ) @@ -11,21 +13,71 @@ import ( // such filesystems often are), where SQLite's file locks and the shared // memory its write-ahead log needs do not work reliably: two processes // could both believe they hold the write lock and corrupt the database. -// Home directories on clusters are often on one. Choose a path on a local -// disk instead. +// Home directories on clusters are often on one, and so are the folders a +// virtual machine or container shares with its host, Docker Desktop's bind +// mounts among them. Choose a path on a local disk instead. var ErrNetworkFilesystem = errors.New("hoppersqlite: the database is on a network filesystem, where SQLite cannot lock it reliably; choose a path on a local disk") -// checkLocal returns ErrNetworkFilesystem if the directory the database -// file at path is in, where SQLite also keeps its log and shared memory, -// is on a network filesystem. +// maxLinks bounds the symbolic links resolve follows, as the kernel's +// ELOOP does. +const maxLinks = 40 + +// resolve returns the absolute path with every symbolic link in it +// resolved, a last link to a file that does not exist yet included. SQLite +// opens the file a link names and keeps its write-ahead log and shared +// memory beside it, not beside the link, so the path resolved is the one +// to check and to open. +func resolve(path string) (string, error) { + p, err := filepath.Abs(path) + if err != nil { + return "", err + } + for range maxLinks { + dir, err := filepath.EvalSymlinks(filepath.Dir(p)) + if err != nil { + return "", err + } + p = filepath.Join(dir, filepath.Base(p)) + fi, err := os.Lstat(p) + if errors.Is(err, fs.ErrNotExist) { + return p, nil + } + if err != nil { + return "", err + } + if fi.Mode()&fs.ModeSymlink == 0 { + return p, nil + } + target, err := os.Readlink(p) + if err != nil { + return "", err + } + if !filepath.IsAbs(target) { + target = filepath.Join(dir, target) + } + p = target + } + return "", fmt.Errorf("%s: more than %d symbolic links", path, maxLinks) +} + +// checkLocal returns ErrNetworkFilesystem if the database file at path, or +// the directory it is in, where SQLite also keeps its log and shared +// memory, is on a network filesystem. path has its links resolved +// (resolve). func checkLocal(path string) error { dir := filepath.Dir(path) - kind, err := networkFilesystem(dir) - if err != nil { - return fmt.Errorf("hoppersqlite: tell whether %s is on a local filesystem: %w", dir, err) + check := []string{dir} + if _, err := os.Stat(path); err == nil { + check = append(check, path) // a file mounted on its own } - if kind != "" { - return fmt.Errorf("%w: %s is on %s", ErrNetworkFilesystem, dir, kind) + for _, p := range check { + kind, err := networkFilesystem(p) + if err != nil { + return fmt.Errorf("hoppersqlite: tell whether %s is on a local filesystem: %w", p, err) + } + if kind != "" { + return fmt.Errorf("%w: %s is on %s", ErrNetworkFilesystem, p, kind) + } } return nil } diff --git a/driver/hoppersqlite/localfs_bsd.go b/driver/hoppersqlite/localfs_bsd.go index 8db8295..9d76bb3 100644 --- a/driver/hoppersqlite/localfs_bsd.go +++ b/driver/hoppersqlite/localfs_bsd.go @@ -6,9 +6,9 @@ import "golang.org/x/sys/unix" // networkFilesystem reports a filesystem the kernel does not mark local // (MNT_LOCAL), as NFS, SMB, AFP, WebDAV and FUSE mounts are not. -func networkFilesystem(dir string) (string, error) { +func networkFilesystem(path string) (string, error) { var st unix.Statfs_t - if err := unix.Statfs(dir, &st); err != nil { + if err := unix.Statfs(path, &st); err != nil { return "", err } if statfsFlags(&st)&unix.MNT_LOCAL == 0 { diff --git a/driver/hoppersqlite/localfs_linux.go b/driver/hoppersqlite/localfs_linux.go index 3b8fd38..eb7ea54 100644 --- a/driver/hoppersqlite/localfs_linux.go +++ b/driver/hoppersqlite/localfs_linux.go @@ -9,7 +9,8 @@ import ( // networkMagic names the filesystems, by statfs(2)'s magic number, whose // locks SQLite cannot rely on: network and cluster filesystems, the shared // folders of virtual machines, and FUSE, which sshfs, s3fs, GlusterFS and -// many others are. +// many others are, as are the bind mounts of Docker Desktop and of other +// tools that run Linux in a virtual machine. var networkMagic = map[int64]string{ 0x6969: "NFS", 0x517b: "SMB", @@ -30,15 +31,17 @@ var networkMagic = map[int64]string{ 0x20030528: "OrangeFS", 0x6a656a63: "virtiofs", 0x786f4256: "VirtualBox shared folders", + 0x7c7c6673: "Parallels shared folders", + 0xbacbacbc: "VMware shared folders", 0x65735546: "FUSE", } // statfs is unix.Statfs, which tests replace. var statfs = unix.Statfs -func networkFilesystem(dir string) (string, error) { +func networkFilesystem(path string) (string, error) { var st unix.Statfs_t - if err := statfs(dir, &st); err != nil { + if err := statfs(path, &st); err != nil { return "", err } if name, ok := networkMagic[int64(st.Type)&0xffffffff]; ok { //nolint:unconvert // st.Type is an int32 or a uint32 on some architectures diff --git a/driver/hoppersqlite/localfs_linux_test.go b/driver/hoppersqlite/localfs_linux_test.go index 2c57b97..55050f4 100644 --- a/driver/hoppersqlite/localfs_linux_test.go +++ b/driver/hoppersqlite/localfs_linux_test.go @@ -2,12 +2,18 @@ package hoppersqlite import ( "errors" + "os" + "path/filepath" "strings" "testing" "golang.org/x/sys/unix" ) +// setType sets a statfs type, which is an int32, an int64 or a uint32 by +// architecture, to a magic number. +func setType[T int32 | int64 | uint32](t *T, v int64) { *t = T(v) } + // TestNetworkFilesystem checks that a directory on NFS, Lustre, GPFS or // FUSE is refused, naming the filesystem, and one on a local filesystem is // not. @@ -16,9 +22,9 @@ func TestNetworkFilesystem(t *testing.T) { t.Fatalf("a temporary directory: %v", err) } defer func(real func(string, *unix.Statfs_t) error) { statfs = real }(statfs) - for magic, name := range map[int64]string{0x6969: "NFS", 0x0bd00bd0: "Lustre", 0x47504653: "GPFS", 0x65735546: "FUSE", 0xff534d42: "CIFS"} { + for magic, name := range map[int64]string{0x6969: "NFS", 0x0bd00bd0: "Lustre", 0x47504653: "GPFS", 0x65735546: "FUSE", 0xff534d42: "CIFS", 0x7c7c6673: "Parallels", 0xbacbacbc: "VMware"} { statfs = func(_ string, st *unix.Statfs_t) error { - st.Type = magic + setType(&st.Type, magic) return nil } err := checkLocal("/home/someone/hopper.db") @@ -31,3 +37,89 @@ func TestNetworkFilesystem(t *testing.T) { t.Errorf("a directory that cannot be read: %v", err) } } + +// TestNetworkFilesystemThroughLink checks that a database reached through +// a symbolic link is checked, and opened, where the link leads: SQLite +// opens the file a link names and keeps its log and shared memory beside +// it, so a link in a local directory to a database on NFS is refused, as +// one whose directory is a link to NFS is, and a dangling link too, which +// SQLite would follow to create the database. +func TestNetworkFilesystemThroughLink(t *testing.T) { + local, remote := t.TempDir(), t.TempDir() // remote stands for a directory on NFS + defer func(real func(string, *unix.Statfs_t) error) { statfs = real }(statfs) + statfs = func(path string, st *unix.Statfs_t) error { + if err := unix.Statfs(path, st); err != nil { + return err + } + if path == remote || strings.HasPrefix(path, remote+"/") { + setType(&st.Type, 0x6969) + } + return nil + } + refused := func(what, path string) { + t.Helper() + d, err := Open(path, nil) + if err == nil { + _ = d.Close() + t.Errorf("%s: opened", what) + return + } + if !errors.Is(err, ErrNetworkFilesystem) || !strings.Contains(err.Error(), remote) { + t.Errorf("%s: %v", what, err) + } + } + refused("a database on NFS", filepath.Join(remote, "direct.db")) + + link := filepath.Join(local, "dangling.db") + if err := os.Symlink(filepath.Join(remote, "dangling.db"), link); err != nil { + t.Fatal(err) + } + refused("a dangling link to NFS", link) + + if err := os.WriteFile(filepath.Join(remote, "existing.db"), nil, 0o600); err != nil { + t.Fatal(err) + } + link = filepath.Join(local, "existing.db") + if err := os.Symlink(filepath.Join("..", filepath.Base(remote), "existing.db"), link); err != nil { + // remote and local share a parent only on some systems + if err := os.Symlink(filepath.Join(remote, "existing.db"), link); err != nil { + t.Fatal(err) + } + } + refused("a link to a database on NFS", link) + + dir := filepath.Join(local, "dir") + if err := os.Symlink(remote, dir); err != nil { + t.Fatal(err) + } + refused("a database in a linked directory on NFS", filepath.Join(dir, "linked.db")) + + if entries, _ := os.ReadDir(remote); len(entries) != 1 { + t.Errorf("the NFS directory holds %d files, want only the one the test made", len(entries)) + } + + // A link to a local database opens it where the link leads, sharing + // the write lock of the Driver that opens it by its own path. + target := filepath.Join(local, "target.db") + link = filepath.Join(local, "link.db") + if err := os.Symlink("target.db", link); err != nil { + t.Fatal(err) + } + statfs = unix.Statfs + d1, err := Open(target, nil) + if err != nil { + t.Fatal(err) + } + defer d1.Close() + d2, err := Open(link, nil) + if err != nil { + t.Fatal(err) + } + defer d2.Close() + if d1.db != d2.db { + t.Error("a link and its target are two databases to the registry") + } + if _, err := os.Stat(target + "-wal"); err != nil { + t.Errorf("the log is not beside the target: %v", err) + } +} diff --git a/driver/hoppersqlite/localfs_netbsd.go b/driver/hoppersqlite/localfs_netbsd.go index 4202a8b..921f838 100644 --- a/driver/hoppersqlite/localfs_netbsd.go +++ b/driver/hoppersqlite/localfs_netbsd.go @@ -4,9 +4,9 @@ import "golang.org/x/sys/unix" // networkFilesystem reports a filesystem the kernel does not mark local // (MNT_LOCAL), as NFS, SMB and FUSE mounts are not. -func networkFilesystem(dir string) (string, error) { +func networkFilesystem(path string) (string, error) { var st unix.Statvfs_t - if err := unix.Statvfs(dir, &st); err != nil { + if err := unix.Statvfs(path, &st); err != nil { return "", err } if uint64(st.Flag)&unix.MNT_LOCAL == 0 { diff --git a/driver/hoppersqlite/localfs_windows.go b/driver/hoppersqlite/localfs_windows.go index fd35dcf..aad0c0c 100644 --- a/driver/hoppersqlite/localfs_windows.go +++ b/driver/hoppersqlite/localfs_windows.go @@ -9,20 +9,20 @@ import ( // networkFilesystem reports a UNC path or a drive Windows reports as remote, // such as a mapped network drive. -func networkFilesystem(dir string) (string, error) { - abs, err := filepath.Abs(dir) +func networkFilesystem(path string) (string, error) { + abs, err := filepath.Abs(path) if err != nil { return "", err } if strings.HasPrefix(abs, `\\`) && !strings.HasPrefix(abs, `\\?\`) { return "a network share", nil } - path, err := windows.UTF16PtrFromString(abs) + p, err := windows.UTF16PtrFromString(abs) if err != nil { return "", err } volume := make([]uint16, windows.MAX_PATH+1) - if err := windows.GetVolumePathName(path, &volume[0], uint32(len(volume))); err != nil { + if err := windows.GetVolumePathName(p, &volume[0], uint32(len(volume))); err != nil { return "", err } if windows.GetDriveType(&volume[0]) == windows.DRIVE_REMOTE { diff --git a/driver/hoppersqlite/migrate_test.go b/driver/hoppersqlite/migrate_test.go index 420f403..21c1837 100644 --- a/driver/hoppersqlite/migrate_test.go +++ b/driver/hoppersqlite/migrate_test.go @@ -88,6 +88,38 @@ func TestMigrate(t *testing.T) { } } +// TestMigrations checks the scripts Migrations returns: SQLite's, one up +// and one down for every version hoppermigrate knows, under its names, +// which applied in order by hand build the schema Up builds. +func TestMigrations(t *testing.T) { + t.Parallel() + ctx := context.Background() + pg := hoppermigrate.Migrations() + ms := hoppersqlite.Migrations() + if len(ms) != len(pg) { + t.Fatalf("%d migrations, want %d", len(ms), len(pg)) + } + d := openEmpty(t) + for i, m := range ms { + if m.Version != pg[i].Version || m.Name != pg[i].Name || m.Up == "" || m.Down == "" || m.Up == pg[i].Up { + t.Fatalf("migration %d = version %d %q, up %d bytes, down %d bytes; want SQLite's scripts of version %d %q", + i, m.Version, m.Name, len(m.Up), len(m.Down), pg[i].Version, pg[i].Name) + } + if _, err := d.DB().ExecContext(ctx, m.Up); err != nil { + t.Fatalf("version %d: %v", m.Version, err) + } + if _, err := d.DB().ExecContext(ctx, "INSERT INTO hopper_schema (version, applied_at) VALUES (?, 0)", m.Version); err != nil { + t.Fatal(err) + } + } + if v, err := hoppermigrate.Version(ctx, d); err != nil || v != hoppermigrate.Latest() { + t.Errorf("version after applying the scripts by hand = %d, %v", v, err) + } + if res, err := hoppermigrate.Up(ctx, d, quiet); err != nil || len(res.Applied) != 0 { + t.Errorf("Up after applying the scripts by hand = %+v, %v", res, err) + } +} + // TestUpConcurrent runs Up from several Drivers on one database at once: // one applies the migrations, and the others find them applied. func TestUpConcurrent(t *testing.T) { diff --git a/driver/hoppersqlite/robustness_test.go b/driver/hoppersqlite/robustness_test.go new file mode 100644 index 0000000..bc20a56 --- /dev/null +++ b/driver/hoppersqlite/robustness_test.go @@ -0,0 +1,475 @@ +package hoppersqlite_test + +import ( + "bytes" + "context" + "database/sql" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/parallelworks/hopper" + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/hoppersqlite" +) + +// longWrite is a write that runs for seconds, for a short context to +// interrupt. +const longWrite = `WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c WHERE x < 100000000) + INSERT INTO t SELECT x + 100 FROM c` + +// interrupt runs longWrite in tx under a context that ends while it runs, +// which makes SQLite roll the whole transaction back. +func interrupt(t *testing.T, tx *sql.Tx) { + t.Helper() + short, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + if _, err := tx.ExecContext(short, longWrite); err == nil { + t.Fatal("the long write was not interrupted") + } +} + +// TestInterruptedTransaction checks a transaction that SQLite rolled back +// itself, after a write its context interrupted: its later statements, the +// executor's included, are refused rather than run on their own; Commit +// returns ErrTransactionAborted and Rollback nothing; nothing it wrote is +// left; and the write lock is free again. +func TestInterruptedTransaction(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + db := d.DB() + if _, err := db.ExecContext(ctx, "CREATE TABLE t (x INTEGER)"); err != nil { + t.Fatal(err) + } + count := func() (rows, jobs int) { + t.Helper() + if err := db.QueryRowContext(ctx, "SELECT (SELECT count(*) FROM t), (SELECT count(*) FROM hopper_jobs)").Scan(&rows, &jobs); err != nil { + t.Fatal(err) + } + return rows, jobs + } + for _, end := range []string{"commit", "rollback"} { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err != nil { + t.Fatal(err) + } + interrupt(t, tx) + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (2)"); !errors.Is(err, hoppersqlite.ErrTransactionAborted) { + t.Errorf("a statement after the interrupted one = %v, want ErrTransactionAborted", err) + } + if err := tx.QueryRowContext(ctx, "SELECT count(*) FROM t").Scan(new(int)); !errors.Is(err, hoppersqlite.ErrTransactionAborted) { + t.Errorf("a query after the interrupted statement = %v, want ErrTransactionAborted", err) + } + _, err = d.UnwrapTx(tx).JobInsertMany(ctx, []driver.JobInsertParams{{Kind: "k", Queue: "default", Priority: 2, MaxAttempts: 1}}, driver.JobInsertOpts{}) + if !errors.Is(err, hoppersqlite.ErrTransactionAborted) { + t.Errorf("an insert through the transaction = %v, want ErrTransactionAborted", err) + } + if end == "commit" { + if err := tx.Commit(); !errors.Is(err, hoppersqlite.ErrTransactionAborted) { + t.Errorf("Commit = %v, want ErrTransactionAborted", err) + } + } else if err := tx.Rollback(); err != nil { + t.Errorf("Rollback = %v", err) + } + if rows, jobs := count(); rows != 0 || jobs != 0 { + t.Errorf("after %s: %d rows and %d jobs left of the transaction", end, rows, jobs) + } + wctx, cancel := context.WithTimeout(ctx, time.Second) + if _, err := db.ExecContext(wctx, "INSERT INTO t VALUES (3); DELETE FROM t"); err != nil { + t.Errorf("a write after %s: %v", end, err) + } + cancel() + } + + // An error that SQLite undoes only the statement of, such as a + // constraint's, leaves the transaction open, as on Postgres it would + // not; and so does a read that is interrupted. + if _, err := db.ExecContext(ctx, "CREATE UNIQUE INDEX t_x ON t (x)"); err != nil { + t.Fatal(err) + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() //nolint:errcheck // committed below + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err == nil || errors.Is(err, hoppersqlite.ErrTransactionAborted) { + t.Errorf("a duplicate = %v, want a constraint error", err) + } + short, cancel := context.WithTimeout(ctx, 30*time.Millisecond) + defer cancel() + if err := tx.QueryRowContext(short, `WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c WHERE x < 100000000) SELECT count(*) FROM c`).Scan(new(int)); err == nil { + t.Error("the long read was not interrupted") + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (2)"); err != nil { + t.Errorf("a statement after a constraint error and an interrupted read = %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + if rows, _ := count(); rows != 2 { + t.Errorf("t has %d rows, want 2", rows) + } +} + +// TestInterruptedFinalizeInTx finalizes a job in a worker's transaction +// after a statement of the transaction was interrupted: the finalize is +// refused, and the job stays running once the transaction rolls back, +// rather than completing in a write of its own. +func TestInterruptedFinalizeInTx(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + x := d.Executor() + clientID, err := x.ClientRegister(ctx, driver.ClientRegisterParams{Hostname: "test", TTL: time.Minute}) + if err != nil { + t.Fatal(err) + } + if _, err := x.JobInsertMany(ctx, []driver.JobInsertParams{{Kind: "k", Queue: "default", Priority: 2, MaxAttempts: 3}}, driver.JobInsertOpts{}); err != nil { + t.Fatal(err) + } + claimed, err := x.JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: clientID, Limit: 1}) + if err != nil || len(claimed.Jobs) != 1 { + t.Fatalf("claim = %+v, %v", claimed, err) + } + j := claimed.Jobs[0] + db := d.DB() + if _, err := db.ExecContext(ctx, "CREATE TABLE t (x INTEGER)"); err != nil { + t.Fatal(err) + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + interrupt(t, tx) + applied, err := d.UnwrapTx(tx).JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{{ + ID: j.ID, AttemptedBy: clientID, Fence: j.Fence, State: driver.JobStateCompleted, Archive: true, + }}}) + if !errors.Is(err, hoppersqlite.ErrTransactionAborted) || len(applied) != 0 { + t.Errorf("finalize in the transaction = %d applied, %v; want ErrTransactionAborted", len(applied), err) + } + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + got, err := x.JobGet(ctx, j.ID) + if err != nil { + t.Fatal(err) + } + if got.State != driver.JobStateRunning { + t.Errorf("the job is %s after its transaction rolled back, want running", got.State) + } +} + +// TestStaleFinalizeAfterRescue checks fencing once a job is rescued: the +// result of the client that lost it applies neither before the job is +// claimed again nor after, nor does a result with the old fence from the +// client that claimed it again; that client's own result does. +func TestStaleFinalizeAfterRescue(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + x := d.Executor() + register := func(host string) int64 { + t.Helper() + id, err := x.ClientRegister(ctx, driver.ClientRegisterParams{Hostname: host, TTL: time.Minute}) + if err != nil { + t.Fatal(err) + } + return id + } + a, b := register("a"), register("b") + if _, err := x.JobInsertMany(ctx, []driver.JobInsertParams{{Kind: "k", Queue: "default", Priority: 2, MaxAttempts: 5}}, driver.JobInsertOpts{}); err != nil { + t.Fatal(err) + } + ca, err := x.JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: a, Limit: 1}) + if err != nil || len(ca.Jobs) != 1 { + t.Fatalf("claim by a = %+v, %v", ca, err) + } + ja := ca.Jobs[0] + if _, err := d.DB().ExecContext(ctx, "UPDATE hopper_clients SET expires_at = 0 WHERE id = ?", a); err != nil { + t.Fatal(err) + } + candidates, err := x.JobRescueCandidates(ctx, driver.JobRescueParams{Limit: 10}) + if err != nil || len(candidates) != 1 { + t.Fatalf("rescue candidates = %+v, %v", candidates, err) + } + c := candidates[0].Job + applied, err := x.JobFinalizeMany(ctx, driver.JobFinalizeParams{SkipLocked: true, Jobs: []driver.JobFinalize{{ + ID: c.ID, AttemptedBy: c.AttemptedBy, Fence: c.Fence, State: driver.JobStateRetryable, Archive: true, + Error: &driver.AttemptError{Attempt: c.Attempt, Fence: c.Fence, Error: "client lost"}, + }}}) + if err != nil || len(applied) != 1 { + t.Fatalf("rescue = %d applied, %v", len(applied), err) + } + finalize := func(f driver.JobFinalize) int { + t.Helper() + applied, err := x.JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{f}}) + if err != nil { + t.Fatal(err) + } + return len(applied) + } + stale := driver.JobFinalize{ID: ja.ID, AttemptedBy: a, Fence: ja.Fence, State: driver.JobStateCompleted, Archive: true} + if n := finalize(stale); n != 0 { + t.Error("the lost client's result applied before the job was claimed again") + } + if _, err := d.DB().ExecContext(ctx, "UPDATE hopper_jobs SET scheduled_at = 0"); err != nil { + t.Fatal(err) + } + cb, err := x.JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: b, Limit: 1}) + if err != nil || len(cb.Jobs) != 1 || cb.Jobs[0].Fence != ja.Fence+1 { + t.Fatalf("claim by b = %+v, %v", cb, err) + } + if n := finalize(stale); n != 0 { + t.Error("the lost client's result applied after the job was claimed again") + } + stale.AttemptedBy = b + if n := finalize(stale); n != 0 { + t.Error("a result of b with the old fence applied") + } + if n := finalize(driver.JobFinalize{ID: ja.ID, AttemptedBy: b, Fence: cb.Jobs[0].Fence, State: driver.JobStateCompleted, Archive: true}); n != 1 { + t.Error("b's result did not apply") + } +} + +type selfArgs struct{} + +func (selfArgs) Kind() string { return "self" } + +// syncBuffer is a bytes.Buffer that a client's logger and the test share. +type syncBuffer struct { + mu sync.Mutex + b bytes.Buffer +} + +func (s *syncBuffer) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.Write(p) +} + +func (s *syncBuffer) String() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.String() +} + +// TestSelfWait runs a worker that writes on the pool while its own +// transaction holds the write lock, which would wait for itself until its +// context ended, and every write of the process with it: the write fails +// at once with ErrLockHeldByOwner, naming the job, and the client keeps +// its lease. +func TestSelfWait(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + var ( + took time.Duration + recErr error + done = make(chan struct{}) + ) + workers := hopper.NewWorkers() + hopper.AddWorkFunc(workers, func(ctx context.Context, job *hopper.Job[selfArgs]) error { + defer close(done) + tx, err := d.DB().BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck // committed below + if _, err := tx.ExecContext(ctx, "UPDATE hopper_queues SET updated_at = updated_at"); err != nil { + return err + } + start := time.Now() + recErr = hopper.RecordStep(ctx, job, "a", nil) + took = time.Since(start) + return tx.Commit() + }) + logs := &syncBuffer{} + c, err := hopper.NewClient(d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: 1}}, + Workers: workers, + JobTimeout: 10 * time.Second, // what a write that waited for itself would wait + LeaseTTL: 3 * time.Second, + Logger: slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelWarn})), + }) + if err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + defer c.Stop(ctx) //nolint:errcheck // cleanup + res, err := c.Insert(ctx, selfArgs{}, nil) + if err != nil { + t.Fatal(err) + } + <-done + if !errors.Is(recErr, hoppersqlite.ErrLockHeldByOwner) || !strings.Contains(recErr.Error(), res.Job.ID.String()) || took > time.Second { + t.Errorf("RecordStep in the worker's transaction = %v after %s, want ErrLockHeldByOwner naming the job at once", recErr, took) + } + for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(20 * time.Millisecond) { + j, err := c.JobGet(ctx, res.Job.ID) + if err != nil { + t.Fatal(err) + } + if j.State == hopper.JobStateCompleted { + break + } + if time.Now().After(deadline) { + t.Fatalf("the job is %s, want completed", j.State) + } + } + if strings.Contains(logs.String(), "lease") { + t.Errorf("the client's lease suffered:\n%s", logs) + } +} + +// TestJSONRefused checks that JSON Postgres's jsonb refuses (a \u0000 +// escape, text that is not UTF-8, a number beyond numeric's range) is +// refused on insert too, as it would be on Postgres. +func TestJSONRefused(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + for _, args := range []string{`{"a":"\u0000"}`, "{\"a\":\"\xff\"}", `{"a":"\ud800"}`, `{"a":1e200000}`} { + _, err := d.Executor().JobInsertMany(ctx, []driver.JobInsertParams{{Kind: "k", Queue: "default", Priority: 2, MaxAttempts: 1, Args: []byte(args)}}, driver.JobInsertOpts{}) + if err == nil { + t.Errorf("args %q inserted", args) + } + } + var n int + if err := d.DB().QueryRowContext(ctx, "SELECT count(*) FROM hopper_jobs").Scan(&n); err != nil || n != 0 { + t.Errorf("jobs = %d, %v", n, err) + } +} + +// TestLogTruncated checks that JobsMaintain cuts back a write-ahead log +// that grew while a reader held up checkpoints, once the reader is done. +func TestLogTruncated(t *testing.T) { + t.Parallel() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "hopper.db") + d := openAt(t, path) + x := d.Executor() + size := func() int64 { + st, err := os.Stat(path + "-wal") + if err != nil { + t.Fatal(err) + } + return st.Size() + } + args := []byte(`{"pad":"` + strings.Repeat("x", 4000) + `"}`) + p := make([]driver.JobInsertParams, 100) + for i := range p { + p[i] = driver.JobInsertParams{Kind: "k", Queue: "default", Priority: 2, MaxAttempts: 1, Args: args} + } + ro, err := d.DB().BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + t.Fatal(err) + } + if err := ro.QueryRowContext(ctx, "SELECT count(*) FROM hopper_jobs").Scan(new(int)); err != nil { + t.Fatal(err) + } + for range 40 { + if _, err := x.JobInsertMany(ctx, p, driver.JobInsertOpts{}); err != nil { + t.Fatal(err) + } + } + if err := ro.Rollback(); err != nil { + t.Fatal(err) + } + grown := size() + if grown < 16<<20 { + t.Fatalf("the log grew to %d bytes only", grown) + } + if _, err := x.JobsMaintain(ctx); err != nil { + t.Fatal(err) + } + if after := size(); after > 1<<20 { + t.Errorf("the log is %d bytes after JobsMaintain, from %d", after, grown) + } +} + +// TestQueryThatWrites checks that a statement that writes, run on its own +// with Query (INSERT ... RETURNING), waits for the write lock in turn, as +// one run with Exec does, saying who holds it when its context ends. +func TestQueryThatWrites(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + db := d.DB() + if _, err := db.ExecContext(ctx, "CREATE TABLE t (x INTEGER)"); err != nil { + t.Fatal(err) + } + // Open the connections the test needs beforehand, for its short + // context to be spent waiting for the lock. + conns := make([]*sql.Conn, 3) + for i := range conns { + c, err := db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + conns[i] = c + } + for _, c := range conns { + _ = c.Close() + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err != nil { + t.Fatal(err) + } + short, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + err = db.QueryRowContext(short, "INSERT INTO t VALUES (2) RETURNING x").Scan(new(int)) + if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), "write lock") { + t.Errorf("INSERT ... RETURNING while a transaction writes = %v, want it to wait for the write lock", err) + } + done := make(chan error, 1) + go func() { + var x int + err := db.QueryRowContext(ctx, "-- a comment\n insert INTO t VALUES (3) RETURNING x").Scan(&x) + done <- err + }() + for hoppersqlite.LockWaiters(d) == 0 { + time.Sleep(time.Millisecond) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + if err := <-done; err != nil { + t.Fatal(err) + } + var n int + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM t").Scan(&n); err != nil || n != 2 { + t.Errorf("rows = %d, %v; want 2", n, err) + } +} + +// TestOpenContext checks that OpenContext gives up when its context ends. +func TestOpenContext(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + d, err := hoppersqlite.OpenContext(ctx, filepath.Join(t.TempDir(), "hopper.db"), nil) + if err == nil { + _ = d.Close() + t.Fatal("OpenContext with an ended context succeeded") + } + if !errors.Is(err, context.Canceled) { + t.Errorf("OpenContext with an ended context = %v", err) + } +} diff --git a/driver/hoppersqlite/sqlite_test.go b/driver/hoppersqlite/sqlite_test.go index bd18cb1..d84c2aa 100644 --- a/driver/hoppersqlite/sqlite_test.go +++ b/driver/hoppersqlite/sqlite_test.go @@ -266,14 +266,17 @@ func TestClient(t *testing.T) { mu.Lock() finished := len(done) mu.Unlock() - if finished == 504+1 { // 500, -1, -2, -3 and the workflow's three + if finished >= 506 { // 500, -1, -2, -3 and the workflow's three break } if time.Now().After(deadline) { - t.Fatalf("%d jobs worked, want %d", finished, 505) + t.Fatalf("%d jobs worked, want %d", finished, 506) } } mu.Lock() + if len(done) != 506 { + t.Errorf("%d jobs worked, want 506", len(done)) + } for n, times := range done { if times != 1 { t.Errorf("job %d worked %d times", n, times) @@ -336,6 +339,10 @@ func TestHelperProcess(t *testing.T) { driver.JobInsertOpts{Notify: true}); err != nil { t.Fatal(err) } + case "worker": + chaosWorker(ctx, t, d) + case "hog": + hog(ctx, t, d) case "hold": tx, err := d.DB().BeginTx(ctx, nil) if err != nil { diff --git a/hoppermigrate/hoppermigrate.go b/hoppermigrate/hoppermigrate.go index 3ca34a9..3c8f533 100644 --- a/hoppermigrate/hoppermigrate.go +++ b/hoppermigrate/hoppermigrate.go @@ -8,7 +8,9 @@ // // The embedded scripts are Postgres's. A driver for another engine supplies // its own script for each version (driver.MigrationScripts), and Up and Down -// apply those, at the same versions. +// apply those, at the same versions; such a driver also returns its scripts +// for other tools, as hoppersqlite.Migrations does. A new schema version +// therefore needs a script for each engine. package hoppermigrate import ( @@ -37,7 +39,9 @@ type Migration struct { var migrations = mustLoad() -// Migrations returns every known migration in ascending version order. +// Migrations returns every known migration in ascending version order, +// with Postgres's scripts. A driver for another engine has its own +// (hoppersqlite.Migrations, for SQLite). func Migrations() []Migration { return slices.Clone(migrations) } // Latest returns the newest schema version this package knows. From 11f93e20c4699ca411d412cb77f21ef4189c5b24 Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 19:58:45 +0000 Subject: [PATCH 14/20] build: vet hoppersqlite on 32-bit Linux in cgo-check localfs_linux_test.go set statfs's type as an int64, which is an int32 on 386 and arm, so the test did not build there, and no target the check vetted was 32-bit. cgo-check now vets linux/386 and linux/arm among its targets, each named with its architecture. --- Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 221c9aa..e84543d 100644 --- a/Makefile +++ b/Makefile @@ -71,8 +71,9 @@ bench-sqlite: ## Run the benchmark harness on a SQLite database in a temporary d .PHONY: cgo-check cgo-check: ## Check that hoppersqlite, its tests and every module they import use no cgo - @cd driver/hoppersqlite && for os in linux darwin windows freebsd openbsd netbsd; do \ - GOOS=$$os CGO_ENABLED=0 go vet ./... || exit 1; \ - pkgs=$$(GOOS=$$os CGO_ENABLED=1 go list -deps -test -f '{{if and .CgoFiles (not .Standard)}}{{.ImportPath}}{{end}}' ./...) || exit 1; \ - if [ -n "$$pkgs" ]; then echo "cgo is not allowed, but these packages use it on $$os:"; echo "$$pkgs"; exit 1; fi; \ + @cd driver/hoppersqlite && for target in linux/amd64 linux/386 linux/arm darwin/arm64 windows/amd64 freebsd/amd64 openbsd/amd64 netbsd/amd64; do \ + export GOOS=$${target%/*} GOARCH=$${target#*/}; \ + CGO_ENABLED=0 go vet ./... || exit 1; \ + pkgs=$$(CGO_ENABLED=1 go list -deps -test -f '{{if and .CgoFiles (not .Standard)}}{{.ImportPath}}{{end}}' ./...) || exit 1; \ + if [ -n "$$pkgs" ]; then echo "cgo is not allowed, but these packages use it on $$target:"; echo "$$pkgs"; exit 1; fi; \ done From 71af3a84599c9bfefca49fa16ae835d6186c655b Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 19:58:45 +0000 Subject: [PATCH 15/20] docs: what a single writer costs and how hoppersqlite guards against it, in the plan and getting started; a schema change needs a SQLite script The plan's SQLite section says what the reviews found missing: a transaction held open holds up every client sharing the database, lease renewals included; the owner token and ErrLockHeldByOwner; aborted transactions; JSON stored as given and the @> emulation; no COPY or SQL producer functions; IDs minted in Go (6.1, with crypto/rand); ANALYZE without VACUUM and the log's truncation; links followed by the filesystem check; Open's bound. 7.1 says the clock never goes back. getting-started moves the SQLite section out of the Postgres text and says the same in short. AGENTS.md: a schema change is a version in both migration directories. --- AGENTS.md | 3 +++ docs/PLAN.md | 46 ++++++++++++++++++++++++++++++++++------- docs/getting-started.md | 37 +++++++++++++++++++++------------ 3 files changed, 66 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 91bbbee..f200116 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,9 @@ owning client. Changes to the SQL in the claim, finalize, rescue or leader paths need a concurrency or chaos test. - Always use database time (`now()`), never application time, when comparing job times. +- A schema change is a new version in `hoppermigrate/migrations` and the same version in + `driver/hoppersqlite/migrations`, up and down; for a change SQLite has no use for, its + script is a comment saying why. hoppersqlite's tests fail for a version it lacks. - Changes to the insert, claim or finalize paths include `hopperbench` results in the PR. A regression of more than 10% against `canary` blocks the merge. - [docs/PLAN.md](docs/PLAN.md) is the design of record. Change it in the same PR as a diff --git a/docs/PLAN.md b/docs/PLAN.md index f5552e0..d4bf1a9 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -451,8 +451,36 @@ defined as operations rather than SQL: another connection committed. Retention is a batched `DELETE`; stream events commit in append order, so a position is an event's sequence number. The database must be on a local filesystem, where SQLite's locks hold: `Open` refuses one on NFS, SMB, Lustre, - GPFS, FUSE and the like. Its migrations follow `hoppermigrate`'s versions one for one - (`driver.MigrationScripts`). + GPFS, FUSE, a virtual machine's shared folders and the like, checking where symbolic + links in the path lead, since SQLite follows them. Its migrations follow + `hoppermigrate`'s versions one for one (`driver.MigrationScripts`), and + `hoppersqlite.Migrations` returns them for other tools. + + What a single writer costs, and how the driver keeps it from failing silently: + - A transaction held open holds up every client that shares the database, their + lease renewals included. A renewal waits until the lease it renews would expire, + then logs who holds the lock and tries again; a transaction held for longer than + about the lease therefore costs every client its lease, and their running jobs are + rescued. + - A worker that writes on the pool while its own transaction holds the lock would + wait for itself until its context ended. The core marks each attempt's context with + its owner (`driver.WithOwner`), and the driver refuses at once such a write + (`ErrLockHeldByOwner`). + - After some errors (an interrupted write, a full disk, an I/O error) SQLite rolls the + whole transaction back itself and would run its later statements on their own. The + driver learns of it from SQLite's rollback hook and refuses the transaction's later + statements and its commit (`ErrTransactionAborted`), as Postgres refuses those of an + aborted transaction, so `CompleteTx` stays all or nothing. + - JSON is stored as given rather than normalized as jsonb is, and refused for what + jsonb refuses (a `\u0000` escape, text that is not UTF-8, a number beyond numeric's + range); a metadata filter's `@>` is a SQL function that matches as jsonb's does. + - There is no `COPY`, so bulk inserts take the one insert path, and there are no SQL + functions for producers outside Go. Job IDs are minted in Go (§6.1). + - `JobsMaintain` refreshes statistics but never vacuums, so the file keeps the space + of deleted rows for SQLite to reuse; it truncates the write-ahead log once a reader + that held up checkpoints is done, and each connection cuts the log back to 32MiB + whenever a checkpoint empties it. + - `Open` waits for the database at most 30 seconds; `OpenContext` takes a context. - **MongoDB** claims with atomic `findOneAndUpdate` or `updateMany` (tagging a batch with a claim token, then reading it back). It uses `$$NOW` for server time, change streams for notifications, and TTL indexes or dropped collections for retention. @@ -653,9 +681,10 @@ index inserts stay local like a sequence. On Postgres 18 and later it calls the built-in `uuidv7()`. On 14–17 it is a SQL function that builds a v7 UUID from `clock_timestamp()` and the built-in `gen_random_uuid()`. Generation therefore works the same for Go inserts, `COPY` - and the SQL contract, and hopper's Go code needs no random number generator or - cryptography. Engines without a database-side generator (MongoDB) generate IDs in - the driver. + and the SQL contract, and the core's Go code needs no random number generator or + cryptography. Engines without a database-side generator generate IDs in the driver: + hoppersqlite mints UUIDv7s in Go, from the database's clock and `crypto/rand`, and + MongoDB's driver will too. - **Ordering uses `seq`, not the ID.** Several IDs created in the same millisecond are not guaranteed to sort in insertion order on every Postgres version. An internal identity column, `seq`, provides the tie-break for claim order and for @@ -690,8 +719,11 @@ there is **no scheduler loop** moving jobs between states. All time comparisons use the database's `now()`, never the application clock, so clock skew between pods cannot cause early or duplicate execution. SQLite's clock is the host's, which every process that opens the database shares, since it must be on a local -filesystem: hoppersqlite reads it once a transaction holds the write lock, so each write's -time is no earlier than the writes before it. +filesystem: hoppersqlite reads it once a transaction holds the write lock, and never +returns a time earlier than one it returned before in the process, so each write's time +is no earlier than the writes before it even when the host's clock is stepped back. A +host that sleeps finds its clock moved on when it wakes, as a server's would, and its +clients' leases expired. ### 7.2 Insert - **Single and small batches:** one `INSERT … SELECT FROM unnest($kinds, $args, …) diff --git a/docs/getting-started.md b/docs/getting-started.md index 7f5cf46..d789887 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -69,6 +69,14 @@ application's `search_path` is unchanged, so the pool can be shared. already live (for example `public`) before upgrading. Changing this setting selects another installation; it does not move existing jobs or tables. +Migration credentials need permission to create the schema, or you can create +it beforehand. Migrating down to zero removes Hopper objects and leaves the +namespace itself intact. + +The SQL files are in `hoppermigrate/migrations` for teams that run migrations +with their own tool. Set a transaction-local `search_path` to the target schema +when running the raw SQL. `hopper_schema` records the applied versions either way. + ### SQLite An application that runs on one machine, such as a command-line tool or a @@ -91,19 +99,22 @@ client, err := hopper.NewClient(driver, cfg) Its transactions are `*sql.Tx`; begin them on `driver.DB()`, whose connections take the database's write lock when a transaction runs its first -statement. SQLite runs one writing transaction at a time, so keep them short, -and write through the transaction (`InsertTx`, `CompleteTx`) rather than the -pool while one is open. Several processes can share the file: clients in -other processes are woken within 50ms of an insert. The file must be on a -local disk; `Open` refuses one on a network filesystem, where SQLite's locks -do not hold. -Migration credentials need permission to create the schema, or you can create -it beforehand. Migrating down to zero removes Hopper objects and leaves the -namespace itself intact. - -The SQL files are in `hoppermigrate/migrations` for teams that run migrations -with their own tool. Set a transaction-local `search_path` to the target schema -when running the raw SQL. `hopper_schema` records the applied versions either way. +statement. SQLite runs one writing transaction at a time, so keep them short: +one held open holds up every client sharing the file, lease renewals +included, and one held longer than the lease costs them their running jobs. +Write through the transaction (`InsertTx`, `CompleteTx`) rather than the pool +while one is open; in a worker, a write on the pool then fails at once with +`hoppersqlite.ErrLockHeldByOwner`. After an error that makes SQLite roll a +transaction back (an interrupted write, a full disk), its later statements +fail with `hoppersqlite.ErrTransactionAborted`; roll it back. + +Several processes can share the file: clients in other processes are woken +within 50ms of an insert. The file must be on a local disk; `Open` refuses +one on a network filesystem or a virtual machine's shared folder (Docker +Desktop's bind mounts among them), where SQLite's locks do not hold. +`hoppersqlite.Migrations` returns the SQLite scripts for teams that run +migrations with their own tool. Upgrade hopper and hoppersqlite together: +the driver uses hopper's internal packages. ## Define a job and its worker From 62b9268dc3fc09d1c7c080cc1a495c0a5936d115 Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 20:55:55 +0000 Subject: [PATCH 16/20] fix(client): bound a renewal by the lease register took too, and test the bound A renewal waits until the lease it renews would expire. The lease loop counted that from its own renewals only, so after the leader's rescue fenced the client and registered it again, the first renewal of the new lease was given just the renewal interval. register now records when the lease was taken, and the loop reads it. A synctest test checks the bound with an executor whose renewals block: from the last renewal, from a registration in between, and the interval once the lease has expired. CompleteTx and RecordStepTx say the single-writer self-wait check needs tx begun with the worker's ctx. --- client.go | 24 +++++++++--- complete.go | 4 +- lease_internal_test.go | 83 ++++++++++++++++++++++++++++++++++++++++++ steps.go | 3 +- 4 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 lease_internal_test.go diff --git a/client.go b/client.go index 9b0725e..389a9a2 100644 --- a/client.go +++ b/client.go @@ -35,6 +35,10 @@ type Client[TTx any] struct { // clientID is the lease row ID while running. It changes if the lease is // lost and re-registered. clientID atomic.Int64 + // leaseFrom is when the current lease was last taken or renewed, as + // register and the lease loop saw it: its expiry bounds a renewal's + // wait. + leaseFrom atomic.Pointer[time.Time] mu sync.Mutex state clientState @@ -330,6 +334,7 @@ func (c *Client[TTx]) declare(ctx context.Context) error { } func (c *Client[TTx]) register(ctx context.Context) (int64, error) { + start := time.Now() info, _ := json.Marshal(map[string]any{ "pid": os.Getpid(), "queues": slices.Sorted(maps.Keys(c.cfg.Queues)), @@ -342,6 +347,7 @@ func (c *Client[TTx]) register(ctx context.Context) (int64, error) { if err != nil { return 0, fmt.Errorf("hopper: register client: %w", err) } + c.leaseFrom.Store(&start) return id, nil } @@ -504,7 +510,6 @@ func (c *Client[TTx]) wakeQueue(queue string) { func (c *Client[TTx]) leaseLoop(ctx context.Context) { ticker := time.NewTicker(c.tuning.leaseRenew) defer ticker.Stop() - renewed := time.Now() // the lease was taken just before the loop starts for { select { case <-ctx.Done(): @@ -513,7 +518,7 @@ func (c *Client[TTx]) leaseLoop(ctx context.Context) { } id := c.clientID.Load() start := time.Now() - rctx, cancel := context.WithTimeout(ctx, max(time.Until(renewed.Add(c.tuning.leaseTTL)), c.tuning.leaseRenew)) + rctx, cancel := context.WithTimeout(ctx, c.renewalWait()) res, err := c.exec.ClientRenew(rctx, driver.ClientRenewParams{ClientID: id, TTL: c.tuning.leaseTTL}) cancel() if err != nil { @@ -523,11 +528,10 @@ func (c *Client[TTx]) leaseLoop(ctx context.Context) { continue } if !res.Renewed { - c.fence(ctx, id) - renewed = time.Now() + c.fence(ctx, id) // which registers again, and so records the new lease continue } - renewed = start + c.leaseFrom.Store(&start) // Control state rides along with every renewal, in case a // notification was missed. for _, jobID := range res.CancelRequested { @@ -538,6 +542,16 @@ func (c *Client[TTx]) leaseLoop(ctx context.Context) { } } +// renewalWait is how long a renewal may wait: until the lease would expire, +// and at least one renewal interval. +func (c *Client[TTx]) renewalWait() time.Duration { + wait := c.tuning.leaseRenew + if from := c.leaseFrom.Load(); from != nil { + wait = max(wait, time.Until(from.Add(c.tuning.leaseTTL))) + } + return wait +} + // limitedQueues reads which queues have limits. func (c *Client[TTx]) limitedQueues(ctx context.Context) ([]string, error) { queues, err := c.exec.QueueList(ctx) diff --git a/complete.go b/complete.go index 5040e61..ac900ee 100644 --- a/complete.go +++ b/complete.go @@ -42,7 +42,9 @@ var ErrJobFenced = errors.New("hopper: job is no longer running in this attempt" // (driver.Capabilities.SingleWriter, SQLite), tx holds the whole database // for writes from its first statement until it ends: every other write // waits for it, the results of other jobs and lease renewals included, and -// a write of the worker's own on the pool fails rather than wait for tx. +// a write of the worker's own on the pool fails rather than wait for tx, +// provided tx was begun with the worker's ctx (or one derived from it), +// which names the job's attempt. // // func (w *Worker) Work(ctx context.Context, job *hopper.Job[Args]) error { // tx, err := w.pool.Begin(ctx) diff --git a/lease_internal_test.go b/lease_internal_test.go new file mode 100644 index 0000000..1fd29fd --- /dev/null +++ b/lease_internal_test.go @@ -0,0 +1,83 @@ +package hopper + +import ( + "context" + "log/slog" + "slices" + "sync" + "testing" + "testing/synctest" + "time" + + "github.com/parallelworks/hopper/driver" +) + +// leaseExec registers clients and answers renewals: it renews at once +// while renew is set, and otherwise holds each renewal until its context +// ends, as a renewal behind a write lock another transaction holds waits, +// recording how long each was given. +type leaseExec struct { + driver.Executor + mu sync.Mutex + renew bool + waits []time.Duration +} + +func (e *leaseExec) ClientRegister(context.Context, driver.ClientRegisterParams) (int64, error) { + return 1, nil +} + +func (e *leaseExec) ClientRenew(ctx context.Context, _ driver.ClientRenewParams) (driver.ClientRenewResult, error) { + deadline, _ := ctx.Deadline() + e.mu.Lock() + e.waits = append(e.waits, time.Until(deadline)) + renew := e.renew + e.mu.Unlock() + if renew { + return driver.ClientRenewResult{Renewed: true}, nil + } + <-ctx.Done() + return driver.ClientRenewResult{}, ctx.Err() +} + +func (e *leaseExec) setRenew(renew bool) { + e.mu.Lock() + e.renew = renew + e.mu.Unlock() +} + +// TestRenewalWaitsUntilTheLeaseExpires checks the lease renewal's bound: a +// renewal held up waits until the lease it renews would expire, counted +// from when it was last taken or renewed (by registering again too), and +// at least one renewal interval once it has. +func TestRenewalWaitsUntilTheLeaseExpires(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + exec := &leaseExec{renew: true} + tun := defaultTuning.withLease(15 * time.Second) // renewals every 5s + c := &Client[struct{}]{exec: exec, logger: slog.New(slog.DiscardHandler), tuning: tun} + if _, err := c.register(ctx); err != nil { // the lease from 0s + t.Fatal(err) + } + done := make(chan struct{}) + go func() { defer close(done); c.leaseLoop(ctx) }() + + time.Sleep(6 * time.Second) // renewed at 5s + exec.setRenew(false) + time.Sleep(2 * time.Second) + if _, err := c.register(ctx); err != nil { // registered again at 8s + t.Fatal(err) + } + // Renewals at 10s, held until 23s, the new lease's expiry; then + // at 23s, the expired lease's, held one interval, to 28s. + time.Sleep(25 * time.Second) + cancel() + <-done + exec.mu.Lock() + defer exec.mu.Unlock() + if want := []time.Duration{10 * time.Second, 13 * time.Second, 5 * time.Second}; !slices.Equal(exec.waits[:min(len(exec.waits), 3)], want) { + t.Errorf("renewals were given %v, want %v first", exec.waits, want) + } + }) +} diff --git a/steps.go b/steps.go index 56d2487..d6f1a90 100644 --- a/steps.go +++ b/steps.go @@ -56,7 +56,8 @@ func RecordStep[T JobArgs](ctx context.Context, job *Job[T], step string, cursor // (driver.Capabilities.SingleWriter, SQLite), tx holds the whole database // for writes from its first statement until it ends: every other write // waits for it, lease renewals included, and a write of the worker's own -// on the pool fails rather than wait for tx. +// on the pool fails rather than wait for tx, provided tx was begun with the +// worker's ctx. func RecordStepTx[TTx any, T JobArgs](ctx context.Context, tx TTx, job *Job[T], step string, cursor any) error { return record(ctx, job, func() (driver.Executor, error) { return txExecutor(ctx, tx, job.ID) }, step, false, cursor) } From edc09db31d884f3fb8d105709d831b950ba40e83 Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 20:55:56 +0000 Subject: [PATCH 17/20] test(drivertest): jsonEqual refuses a text with anything after its value It decoded one value and ignored the rest, so a text with trailing data matched. A unit test covers key order, spacing, numbers by exact value and trailing data. --- drivertest/drivertest.go | 7 ++++++- drivertest/json_internal_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 drivertest/json_internal_test.go diff --git a/drivertest/drivertest.go b/drivertest/drivertest.go index cb36d63..6bb8c65 100644 --- a/drivertest/drivertest.go +++ b/drivertest/drivertest.go @@ -26,6 +26,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "math/big" "slices" "strconv" @@ -362,7 +363,11 @@ func jsonEqual(got []byte, want string) bool { dec := json.NewDecoder(bytes.NewReader(b)) dec.UseNumber() var v any - return v, dec.Decode(&v) == nil + if dec.Decode(&v) != nil { + return nil, false + } + _, err := dec.Token() // one value, and nothing after it + return v, errors.Is(err, io.EOF) } a, ok := decode(got) b, ok2 := decode([]byte(want)) diff --git a/drivertest/json_internal_test.go b/drivertest/json_internal_test.go new file mode 100644 index 0000000..6b0f199 --- /dev/null +++ b/drivertest/json_internal_test.go @@ -0,0 +1,27 @@ +package drivertest + +import "testing" + +// TestJSONEqual checks that JSON texts compare by value: key order, +// spacing and how a number is written do not count, its exact value does, +// and a text with anything after its value matches nothing. +func TestJSONEqual(t *testing.T) { + t.Parallel() + for _, c := range []struct { + got, want string + equal bool + }{ + {`{"a":1,"b":[true,null]}`, `{ "b": [true, null], "a": 1 }`, true}, + {`{"n":1.50}`, `{"n":1.5}`, true}, + {`{"n":1e2}`, `{"n":100}`, true}, + {`{"n":9007199254740993}`, `{"n":9007199254740992}`, false}, + {`{"a":"x"}`, `{"a":"y"}`, false}, + {`{"a":1} {"b":2}`, `{"a":1}`, false}, + {`{"a":1}`, `{"a":1}x`, false}, + {`{"a":`, `{"a":1}`, false}, + } { + if got := jsonEqual([]byte(c.got), c.want); got != c.equal { + t.Errorf("jsonEqual(%s, %s) = %v, want %v", c.got, c.want, got, c.equal) + } + } +} From e45612178fdc33dc4b009ff294c479a55940d2a2 Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 20:56:06 +0000 Subject: [PATCH 18/20] fix(hoppersqlite): ErrLockHeldByOwner only for a transaction the worker began, and see COMMIT, links and writes as SQLite does From the second review, each with a test that failed before: - ErrLockHeldByOwner fired for writes of one attempt on the pool from several goroutines, with no transaction of the worker's open: the driver's own one-operation transaction recorded the attempt as the lock's holder, so the attempt's next write failed rather than wait (173 of 202 in TestConcurrentWritesOfOneAttempt). Only a transaction the caller began records its owner now; the driver's own, and statements on their own, still check the waiter's owner but record none, and say whose they were when a wait times out. - A COMMIT, END or ROLLBACK statement run in a transaction went unnoticed, and later statements ran on their own. The commit hook joins the rollback hook: later statements fail with sql.ErrTxDone, Commit returns nil after a COMMIT and sql.ErrTxDone after a ROLLBACK. TestHooksMatchSQLite checks fourteen statements against modernc's own connection: the driver ends the transaction exactly when SQLite did. - resolve cleaned ".." lexically, where the kernel resolves a linked directory first, so a link to sub/../x.db could be checked at the wrong place. It now resolves an existing path with filepath.EvalSymlinks and a missing one component by component, ".." from the directory resolved so far. - mayWrite matched INSERT, UPDATE, DELETE and REPLACE as substrings of a WITH statement (updated_at, replace(), 'deleted'). It now reads words, skipping literals, quoted identifiers, comments and parameters, and checks every statement of a query. TestUpgradeUnderTraffic's 5 s bound, and TestFairness's, scale under the race detector, beside which they were flaky. The package doc says the clock's floor holds within one process, and that a worker's transaction must be begun with the job's context for the owner check to apply. --- driver/hoppersqlite/chaos_test.go | 2 +- driver/hoppersqlite/conn.go | 309 ++++++++++++++----- driver/hoppersqlite/conn_internal_test.go | 39 ++- driver/hoppersqlite/executor.go | 5 +- driver/hoppersqlite/hooks_test.go | 176 +++++++++++ driver/hoppersqlite/hoppersqlite.go | 12 +- driver/hoppersqlite/localfs.go | 81 +++-- driver/hoppersqlite/migrate_test.go | 4 +- driver/hoppersqlite/norace_test.go | 6 + driver/hoppersqlite/race_test.go | 8 + driver/hoppersqlite/resolve_internal_test.go | 53 ++++ driver/hoppersqlite/robustness_test.go | 80 +++++ 12 files changed, 666 insertions(+), 109 deletions(-) create mode 100644 driver/hoppersqlite/hooks_test.go create mode 100644 driver/hoppersqlite/norace_test.go create mode 100644 driver/hoppersqlite/race_test.go create mode 100644 driver/hoppersqlite/resolve_internal_test.go diff --git a/driver/hoppersqlite/chaos_test.go b/driver/hoppersqlite/chaos_test.go index 64f47b5..8eea98e 100644 --- a/driver/hoppersqlite/chaos_test.go +++ b/driver/hoppersqlite/chaos_test.go @@ -186,7 +186,7 @@ func TestFairness(t *testing.T) { slices.Sort(waits) p50, p90, worst := waits[len(waits)/2], waits[len(waits)*9/10], waits[len(waits)-1] t.Logf("%d writes: p50 %s, p90 %s, max %s", len(waits), p50, p90, worst) - if worst > 2*time.Second { + if worst > slowdown*2*time.Second { t.Errorf("a write waited %s while another process wrote", worst) } } diff --git a/driver/hoppersqlite/conn.go b/driver/hoppersqlite/conn.go index 7bf09b2..19926a9 100644 --- a/driver/hoppersqlite/conn.go +++ b/driver/hoppersqlite/conn.go @@ -2,6 +2,7 @@ package hoppersqlite import ( "context" + "database/sql" "database/sql/driver" "errors" "fmt" @@ -41,7 +42,10 @@ import ( // transaction's later statements on their own, each committing at once. // SQLite's rollback hook tells the connection, which then refuses every // later statement of the transaction (ErrTransactionAborted), as Postgres -// does those of a transaction an error aborted. +// does those of a transaction an error aborted. A COMMIT, END or ROLLBACK +// statement run in the transaction ends it the same way, which the commit +// and rollback hooks tell too: its later statements are refused +// (sql.ErrTxDone) rather than run on their own. // busyQuantum is the busy_timeout of every connection: how long SQLite's // busy handler waits for a lock another process holds before the statement @@ -64,15 +68,20 @@ const journalSizeLimit = 32 << 20 var ErrTransactionAborted = errors.New("hoppersqlite: the transaction was rolled back after an error and runs no more statements; roll it back") // ErrLockHeldByOwner is returned for a write that would wait for the -// database's write lock while a transaction of the same owner holds it (a -// job's attempt: driver.WithOwner), which would wait until its context -// ended and hold every other write up meanwhile. Write through that -// transaction instead, as InsertTx and the other Tx methods do, or end it -// first. A worker that writes from two goroutines at once, one of them in -// a transaction, gets it too, where Postgres would run both; give the -// other goroutine a context without the owner (driver.WithOwner(ctx, nil)) -// to have it wait its turn instead. -var ErrLockHeldByOwner = errors.New("hoppersqlite: the write would wait for the database's write lock, which a transaction of the same job attempt holds; write through that transaction, or end it first") +// database's write lock while a transaction of the same owner holds it, +// one its caller began on DB and keeps open, which would wait until its +// context ended and hold every other write up meanwhile. The owner is a +// job's attempt (driver.WithOwner), which the worker's context carries: +// the check applies to a worker's transaction begun with that context, or +// one derived from it, and not to one begun with another. Write through +// that transaction instead, as InsertTx and the other Tx methods do, or end +// it first. Writes of one attempt that are not in a transaction of its own, +// from however many goroutines, wait their turn as any others do. A worker +// that writes from two goroutines at once, one of them in its own +// transaction, gets it, where Postgres would run both; give the other +// goroutine a context without the owner (driver.WithOwner(ctx, nil)) to +// have it wait its turn instead. +var ErrLockHeldByOwner = errors.New("hoppersqlite: the write would wait for the database's write lock, which a transaction the same job attempt began holds open; write through that transaction, or end it first") // database is what every connection to one database file in this process // shares: the write lock, the clock, and the listeners to wake when a @@ -98,10 +107,13 @@ type database struct { } // holder is who holds the write lock: the owner its transaction runs for, -// or none, since when. +// or none, since when, and whether the transaction is one a caller began +// and keeps open while its own code runs, rather than the driver's for one +// operation, or a statement on its own. type holder struct { - owner fmt.Stringer - since time.Time + owner fmt.Stringer + since time.Time + callers bool } func (h holder) String() string { @@ -159,21 +171,22 @@ func (db *database) clock() time.Time { } } -// acquire takes the write lock for this process for owner, waiting in -// turn for as long as ctx allows. It refuses at once to wait for a lock a -// transaction of the same owner holds (ErrLockHeldByOwner), and a wait -// that ctx ends says who held the lock. -func (db *database) acquire(ctx context.Context, owner fmt.Stringer) error { +// acquire takes the write lock for this process for owner, for a +// transaction its caller began (callers) or not, waiting in turn for as +// long as ctx allows. It refuses at once to wait for a lock a transaction +// that a caller of the same owner began holds (ErrLockHeldByOwner), and a +// wait that ctx ends says who held the lock. +func (db *database) acquire(ctx context.Context, owner fmt.Stringer, callers bool) error { select { case db.lock <- struct{}{}: - db.held(owner) + db.held(owner, callers) return nil default: } db.holderMu.Lock() h := db.holder db.holderMu.Unlock() - if sameOwner(h.owner, owner) { + if h.callers && sameOwner(h.owner, owner) { return fmt.Errorf("%w (%s)", ErrLockHeldByOwner, owner) } db.waiting.Add(1) @@ -181,7 +194,7 @@ func (db *database) acquire(ctx context.Context, owner fmt.Stringer) error { start := time.Now() select { case db.lock <- struct{}{}: - db.held(owner) + db.held(owner, callers) return nil case <-ctx.Done(): db.holderMu.Lock() @@ -203,12 +216,22 @@ func sameOwner(a, b fmt.Stringer) bool { return t == reflect.TypeOf(b) && t.Comparable() && a == b } -func (db *database) held(owner fmt.Stringer) { +func (db *database) held(owner fmt.Stringer, callers bool) { db.holderMu.Lock() - db.holder = holder{owner: owner, since: time.Now()} + db.holder = holder{owner: owner, since: time.Now(), callers: callers} db.holderMu.Unlock() } +// oneOperationKey marks the context of a transaction the driver begins for +// one operation of its own, which never stays open while a caller's code +// runs. +type oneOperationKey struct{} + +// oneOperation returns ctx marked as oneOperationKey says. +func oneOperation(ctx context.Context) context.Context { + return context.WithValue(ctx, oneOperationKey{}, true) +} + func (db *database) release() { db.holderMu.Lock() db.holder = holder{} @@ -324,9 +347,11 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { } } cn := &conn{inner: inner, db: c.db} - // SQLite calls the hook on the goroutine running the statement that - // rolls a transaction back, whether by ROLLBACK or after an error. + // SQLite calls the hooks on the goroutine running the statement that + // commits or rolls a transaction back, whether by COMMIT or ROLLBACK + // or, for a rollback, after an error. inner.RegisterRollbackHook(func() { cn.rolledBack = true }) + inner.RegisterCommitHook(func() int32 { cn.committed = true; return 0 }) return cn, nil } @@ -344,9 +369,10 @@ type conn struct { inner innerConn db *database tx *tx - // rolledBack is set by SQLite's rollback hook when a transaction rolls - // back; a statement resets it before it runs. - rolledBack bool + // rolledBack and committed are set by SQLite's rollback and commit + // hooks when a transaction rolls back or commits; a statement resets + // them before it runs. + rolledBack, committed bool } var ( @@ -364,8 +390,10 @@ type tx struct { c *conn readOnly bool // owner is who the transaction runs for (driver.Owner of the context - // it began with). - owner fmt.Stringer + // it began with), and oneOperation reports one the driver began for + // one operation of its own. + owner fmt.Stringer + oneOperation bool // begun reports that BEGIN has run, and locked that the transaction // holds this process's write lock. begun, locked bool @@ -376,6 +404,23 @@ type tx struct { notified bool // aborted is the error after which SQLite rolled the transaction back. aborted error + // endedBy names the statement of the caller's that ended the + // transaction (COMMIT, END or ROLLBACK), and committed reports that it + // committed it. + endedBy string + committed bool +} + +// refused returns the error of a statement of a transaction that has +// ended, or nil. +func (t *tx) refused() error { + switch { + case t.aborted != nil: + return abortedError(t.aborted) + case t.endedBy != "": + return fmt.Errorf("hoppersqlite: %s ended the transaction; end it with Commit or Rollback instead: %w", t.endedBy, sql.ErrTxDone) + } + return nil } func (c *conn) exec(ctx context.Context, query string) error { @@ -417,7 +462,7 @@ func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, e if c.tx != nil { return nil, errors.New("hoppersqlite: a transaction is already open on this connection") } - t := &tx{c: c, readOnly: opts.ReadOnly, owner: hopperdriver.Owner(ctx)} + t := &tx{c: c, readOnly: opts.ReadOnly, owner: hopperdriver.Owner(ctx), oneOperation: ctx.Value(oneOperationKey{}) != nil} if opts.ReadOnly { if err := c.exec(ctx, "PRAGMA query_only = ON"); err != nil { return nil, err @@ -434,7 +479,7 @@ func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, e // beginWrite takes the write lock for the open transaction and begins it. func (c *conn) beginWrite(ctx context.Context) error { - if err := c.db.acquire(ctx, c.tx.owner); err != nil { + if err := c.db.acquire(ctx, c.tx.owner, !c.tx.oneOperation); err != nil { return err } if _, err := retryBusy(ctx, c.db, func() (struct{}, error) { return struct{}{}, c.exec(ctx, "BEGIN IMMEDIATE") }); err != nil { @@ -454,15 +499,15 @@ func abortedError(cause error) error { } // enter prepares the open transaction for a statement: it refuses one -// after SQLite rolled the transaction back, begins the transaction on its -// first statement, notes a notification written, and resets the rollback -// hook's flag. It reports whether the statement may meet SQLITE_BUSY that -// can be retried: the first of a read-only transaction, which has read no +// after the transaction ended (refused), begins the transaction on its +// first statement, notes a notification written, and resets the hooks' +// flags. It reports whether the statement may meet SQLITE_BUSY that can be +// retried: the first of a read-only transaction, which has read no // snapshot yet. func (c *conn) enter(ctx context.Context, query string) (retry bool, err error) { t := c.tx - if t.aborted != nil { - return false, abortedError(t.aborted) + if err := t.refused(); err != nil { + return false, err } if !t.begun { if err := c.beginWrite(ctx); err != nil { @@ -474,18 +519,25 @@ func (c *conn) enter(ctx context.Context, query string) (retry bool, err error) } retry = t.readOnly && !t.read t.read = true - c.rolledBack = false + c.rolledBack, c.committed = false, false return retry, nil } // leave notes whether a statement of the open transaction, ending with -// err, made SQLite roll the transaction back. +// err, ended the transaction: SQLite rolled it back after the statement's +// error, or the statement was a ROLLBACK, a COMMIT or an END. func (c *conn) leave(err error) { - if c.tx != nil && c.rolledBack && c.tx.aborted == nil { - if err == nil { - err = errors.New("SQLite rolled the transaction back") - } - c.tx.aborted = err + t := c.tx + if t == nil || t.refused() != nil { + return + } + switch { + case c.rolledBack && err != nil: + t.aborted = err + case c.rolledBack: + t.endedBy = "a ROLLBACK statement" + case c.committed: + t.endedBy, t.committed = "a COMMIT statement", true } } @@ -493,7 +545,7 @@ func (c *conn) leave(err error) { // its own, holding the write lock while it runs. func (c *conn) runExec(ctx context.Context, query string, run func() (driver.Result, error)) (driver.Result, error) { if c.tx == nil { - if err := c.db.acquire(ctx, hopperdriver.Owner(ctx)); err != nil { + if err := c.db.acquire(ctx, hopperdriver.Owner(ctx), false); err != nil { return nil, err } defer c.db.release() @@ -516,7 +568,7 @@ func (c *conn) runQuery(ctx context.Context, query string, run func() (driver.Ro if !mayWrite(query) { return retryBusy(ctx, c.db, run) } - if err := c.db.acquire(ctx, hopperdriver.Owner(ctx)); err != nil { + if err := c.db.acquire(ctx, hopperdriver.Owner(ctx), false); err != nil { return nil, err } r, err := retryBusy(ctx, c.db, run) @@ -546,36 +598,125 @@ func (c *conn) runQuery(ctx context.Context, query string, run func() (driver.Ro // writeKeywords are the first words of the statements that may write. var writeKeywords = []string{"INSERT", "UPDATE", "DELETE", "REPLACE", "CREATE", "DROP", "ALTER", "VACUUM", "REINDEX", "ANALYZE"} -// mayWrite reports whether a query, run on its own, may write: by its -// first word after any comments, for a WITH statement by whether INSERT, -// UPDATE, DELETE or REPLACE appears in it at all, and for a PRAGMA by -// whether it sets a value. +// readPragmas are the pragmas that take an argument in parentheses and +// only read. wal_checkpoint is among them: it waits for writers through +// SQLite's busy handler rather than in turn, as a reader would. +var readPragmas = []string{ + "COLLATION_LIST", "DATABASE_LIST", "FOREIGN_KEY_CHECK", "FOREIGN_KEY_LIST", "FUNCTION_LIST", + "INDEX_INFO", "INDEX_LIST", "INDEX_XINFO", "INTEGRITY_CHECK", "MODULE_LIST", "PRAGMA_LIST", + "QUICK_CHECK", "TABLE_INFO", "TABLE_LIST", "TABLE_XINFO", "WAL_CHECKPOINT", +} + +// mayWrite reports whether a query, run on its own, may write: whether +// any of its statements does, by its first word; a WITH statement by +// whether INSERT, UPDATE, DELETE or REPLACE INTO is among its words; a +// PRAGMA by whether it sets a value, with "=" or in parentheses, or is +// optimize or incremental_vacuum. Words in string literals, quoted +// identifiers and comments do not count. func mayWrite(query string) bool { - q := strings.TrimSpace(query) - for strings.HasPrefix(q, "--") || strings.HasPrefix(q, "/*") { - end := "\n" - if q[1] == '*' { - end = "*/" - } - _, rest, ok := strings.Cut(q, end) - if !ok { - return false + for _, stmt := range sqlTokens(query) { + if statementMayWrite(stmt) { + return true } - q = strings.TrimSpace(rest) } - n := strings.IndexFunc(q, func(r rune) bool { return (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') }) - if n < 0 { - n = len(q) + return false +} + +func statementMayWrite(t []string) bool { + if len(t) == 0 { + return false } - switch word := strings.ToUpper(q[:n]); word { + switch t[0] { case "WITH": - upper := strings.ToUpper(q) - return slices.ContainsFunc(writeKeywords[:4], func(k string) bool { return strings.Contains(upper, k) }) + for i, w := range t { + switch { + case w == "INSERT" || w == "UPDATE" || w == "DELETE": + return true + case w == "REPLACE" && i+1 < len(t) && t[i+1] == "INTO": + return true + } + } + return false case "PRAGMA": - return strings.Contains(q, "=") + name := "" + for i, w := range t[1:] { + switch { + case w == "=": + return true + case w == "(": + return !slices.Contains(readPragmas, name) + case w != "." && i < 3: + name = w // the last of schema.name + } + } + return name == "OPTIMIZE" || name == "INCREMENTAL_VACUUM" default: - return slices.Contains(writeKeywords, word) + return slices.Contains(writeKeywords, t[0]) + } +} + +// sqlTokens splits SQL into statements, each a list of its words, +// upper-cased, and of its punctuation, one character each. String and blob +// literals, quoted identifiers, numbers, parameters and comments are left +// out. +func sqlTokens(q string) [][]string { + var ( + stmts [][]string + cur []string + ) + isWord := func(c byte) bool { + return c == '_' || c == '$' || c >= '0' && c <= '9' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= 0x80 + } + skipTo := func(i int, end string) int { + if j := strings.Index(q[i:], end); j >= 0 { + return i + j + len(end) + } + return len(q) + } + for i := 0; i < len(q); { + c := q[i] + switch { + case c == '\'' || c == '"' || c == '`': + // A quote doubled inside is the quote itself. + j := i + 1 + for j < len(q) && (q[j] != c || j+1 < len(q) && q[j+1] == c) { + if q[j] == c { + j++ + } + j++ + } + i = j + 1 + case c == '[': + i = skipTo(i, "]") + case strings.HasPrefix(q[i:], "--"): + i = skipTo(i, "\n") + case strings.HasPrefix(q[i:], "/*"): + i = skipTo(i, "*/") + case c == ';': + stmts, cur = append(stmts, cur), nil + i++ + case c == '?' || c == ':' || c == '@' || c >= '0' && c <= '9' || c == '$': + // A parameter or a number, which may run into letters. + j := i + 1 + for j < len(q) && (isWord(q[j]) || q[j] == '.') { + j++ + } + i = j + case isWord(c): + j := i + 1 + for j < len(q) && isWord(q[j]) { + j++ + } + cur = append(cur, strings.ToUpper(q[i:j])) + i = j + case c == ' ' || c == '\t' || c == '\n' || c == '\r': + i++ + default: + cur = append(cur, string(c)) + i++ + } } + return append(stmts, cur) } func (c *conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { @@ -608,16 +749,25 @@ func (t *tx) end() { t.c.tx = nil } -// Commit commits the transaction, or, if SQLite rolled it back after an -// error, rolls back what is left of it and returns ErrTransactionAborted. +// Commit commits the transaction. If SQLite rolled it back after an error, +// it rolls back what is left of it and returns ErrTransactionAborted; if a +// statement of the caller's ended it, Commit returns nil for a COMMIT and +// sql.ErrTxDone for a ROLLBACK. func (t *tx) Commit() error { defer t.end() - if !t.begun { + switch { + case !t.begun: return nil - } - if t.aborted != nil { + case t.aborted != nil: _ = t.rollback() return abortedError(t.aborted) + case t.endedBy != "" && !t.committed: + return t.refused() + case t.endedBy != "": + if t.notified { + t.c.db.poke() + } + return nil } if err := t.c.exec(context.Background(), "COMMIT"); err != nil { // A COMMIT that failed can leave the transaction open, which @@ -631,9 +781,10 @@ func (t *tx) Commit() error { return nil } +// Rollback rolls the transaction back, or does nothing if it has ended. func (t *tx) Rollback() error { defer t.end() - if !t.begun { + if !t.begun || t.endedBy != "" { return nil } return t.rollback() @@ -651,7 +802,7 @@ func (t *tx) rollback() error { } // rows reads a query's rows. In a transaction, it notes a rollback SQLite -// makes while it steps; on its own, a query that may write holds the write +// makes while it steps (leave); on its own, a query that may write holds the write // lock until it is closed. type rows struct { inner driver.Rows @@ -683,11 +834,11 @@ func (r *rows) Next(dest []driver.Value) error { if r.t == nil { return r.inner.Next(dest) } - if r.t.aborted != nil { - return abortedError(r.t.aborted) + if err := r.t.refused(); err != nil { + return err } err := r.inner.Next(dest) - if r.c.tx == r.t && r.c.rolledBack && r.t.aborted == nil { + if r.c.tx == r.t { r.c.leave(err) } return err diff --git a/driver/hoppersqlite/conn_internal_test.go b/driver/hoppersqlite/conn_internal_test.go index 90783e4..0da3883 100644 --- a/driver/hoppersqlite/conn_internal_test.go +++ b/driver/hoppersqlite/conn_internal_test.go @@ -27,6 +27,25 @@ func TestMayWrite(t *testing.T) { "-- only a comment": false, "INSERT\nINTO t VALUES (1)": true, "delete\tFROM t": true, + "WITH q AS (SELECT name, updated_at FROM hopper_queues) SELECT * FROM q": false, + "WITH j AS (SELECT replace(kind, 'a', 'b') AS k FROM hopper_jobs) SELECT k FROM j": false, + "WITH d AS (SELECT count(*) FROM h WHERE state = 'deleted') SELECT * FROM d": false, + "WITH d AS (SELECT 1 AS \"update\") SELECT * FROM d": false, + "WITH d AS (SELECT 1) SELECT * FROM d -- then DELETE": false, + "WITH d AS (SELECT 'it''s; DELETE') SELECT * FROM d": false, + "WITH d AS (SELECT 1) REPLACE INTO t SELECT * FROM d": true, + "SELECT * FROM t WHERE x = 1; DELETE FROM t": true, + "SELECT ';DELETE FROM t'": false, + "PRAGMA wal_checkpoint(TRUNCATE)": false, + "PRAGMA main.table_info(t)": false, + "PRAGMA user_version(5)": true, + "PRAGMA main.journal_mode = WAL": true, + "PRAGMA optimize": true, + "PRAGMA data_version": false, + "(SELECT 1)": false, + "VALUES (1)": false, + "EXPLAIN DELETE FROM t": false, + "SELECT :update, ?1, $delete": false, } { if got := mayWrite(query); got != want { t.Errorf("mayWrite(%q) = %v, want %v", query, got, want) @@ -63,16 +82,16 @@ func (value) String() string { return "value" } func TestOwnerWaits(t *testing.T) { db := &database{lock: make(chan struct{}, 1)} a, b := &owner{"job a"}, &owner{"job b"} - if err := db.acquire(context.Background(), a); err != nil { + if err := db.acquire(context.Background(), a, true); err != nil { t.Fatal(err) } - err := db.acquire(context.Background(), a) + err := db.acquire(context.Background(), a, true) if !errors.Is(err, ErrLockHeldByOwner) || !strings.Contains(err.Error(), "job a") { t.Errorf("acquire by the holder's owner = %v, want ErrLockHeldByOwner", err) } ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) defer cancel() - err = db.acquire(ctx, b) + err = db.acquire(ctx, b, true) if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), "job a") { t.Errorf("acquire by another owner = %v, want it to wait and name the holder", err) } @@ -80,8 +99,20 @@ func TestOwnerWaits(t *testing.T) { t.Error("sameOwner compares owners other than by identity, or panics") } db.release() - if err := db.acquire(context.Background(), b); err != nil { + if err := db.acquire(context.Background(), b, true); err != nil { t.Fatal(err) } db.release() + + // A lock held by a transaction the driver began for one operation, or + // by a statement on its own, is waited for by its owner's other writes. + if err := db.acquire(context.Background(), a, false); err != nil { + t.Fatal(err) + } + ctx, cancel = context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if err := db.acquire(ctx, a, false); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("acquire by the owner of a lock no caller's transaction holds = %v, want it to wait", err) + } + db.release() } diff --git a/driver/hoppersqlite/executor.go b/driver/hoppersqlite/executor.go index 78b601b..20c0703 100644 --- a/driver/hoppersqlite/executor.go +++ b/driver/hoppersqlite/executor.go @@ -204,7 +204,10 @@ func (e *executor) writeLocking(ctx context.Context, wait time.Duration, fn func _, err := q.ExecContext(ctx, "RELEASE hopper") return err } - tx, err := e.d.sqlDB.BeginTx(ctx, nil) + // A transaction of the driver's own, for this one operation, which may + // wait for a lock its owner's own transaction holds but never holds + // one that another write of the owner should not wait for. + tx, err := e.d.sqlDB.BeginTx(oneOperation(ctx), nil) if err != nil { return err } diff --git a/driver/hoppersqlite/hooks_test.go b/driver/hoppersqlite/hooks_test.go new file mode 100644 index 0000000..909ca51 --- /dev/null +++ b/driver/hoppersqlite/hooks_test.go @@ -0,0 +1,176 @@ +package hoppersqlite_test + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "strconv" + "strings" + "testing" + + _ "modernc.org/sqlite" // the "sqlite" driver, unwrapped, to compare with + + "github.com/parallelworks/hopper/driver/hoppersqlite" +) + +// hookCase is a statement run in a transaction after an insert, on a +// table made by table; full limits the database's size first, so that the +// statement fills it. +type hookCase struct { + name, table, stmt string + full bool +} + +// rawEndsTransaction runs c on modernc's own driver and reports whether +// SQLite still had the transaction open after the statement. +func rawEndsTransaction(t *testing.T, c hookCase) bool { + t.Helper() + ctx := context.Background() + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "raw.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + exec := func(q string) { + t.Helper() + if _, err := conn.ExecContext(ctx, q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + exec(c.table) + var pages int + if err := conn.QueryRowContext(ctx, "PRAGMA page_count").Scan(&pages); err != nil { + t.Fatal(err) + } + exec("BEGIN IMMEDIATE") + exec("INSERT INTO t VALUES (1)") + if c.full { + exec("PRAGMA max_page_count = " + strconv.Itoa(pages+3)) + } + _, _ = conn.ExecContext(ctx, c.stmt) + _, err = conn.ExecContext(ctx, "ROLLBACK") + if err != nil && !strings.Contains(err.Error(), "no transaction is active") { + t.Fatalf("ROLLBACK: %v", err) + } + return err != nil +} + +// TestHooksMatchSQLite runs statements that end a transaction, or leave it +// open, in one of the driver's transactions and on modernc's own +// connection: the driver refuses the transaction's next statement exactly +// when SQLite ended the transaction, with ErrTransactionAborted after an +// error and sql.ErrTxDone after a ROLLBACK, COMMIT or END, and nothing the +// refused statement would have written is left. +func TestHooksMatchSQLite(t *testing.T) { + t.Parallel() + ctx := context.Background() + big := "INSERT INTO t VALUES (randomblob(200000))" + rows := "INSERT INTO t SELECT randomblob(3000) FROM (WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM c WHERE x < 200) SELECT x FROM c)" + trigger := func(action string) string { + return "CREATE TABLE t (x); CREATE TRIGGER tr BEFORE INSERT ON t WHEN NEW.x = 9 BEGIN SELECT RAISE(" + action + ", 'no'); END" + } + for _, c := range []hookCase{ + {"a full disk, one row", "CREATE TABLE t (x)", big, true}, + {"a full disk, many rows", "CREATE TABLE t (x)", rows, true}, + {"a full disk, many rows, a CHECK", "CREATE TABLE t (x CHECK (x IS NOT NULL))", rows, true}, + {"a full disk, an UPDATE", "CREATE TABLE t (x)", "UPDATE t SET x = randomblob(300000)", true}, + {"INSERT OR ROLLBACK", "CREATE TABLE t (x UNIQUE)", "INSERT OR ROLLBACK INTO t VALUES (1)", false}, + {"RAISE(ROLLBACK)", trigger("ROLLBACK"), "INSERT INTO t VALUES (9)", false}, + {"RAISE(ABORT)", trigger("ABORT"), "INSERT INTO t VALUES (9)", false}, + {"RAISE(FAIL)", trigger("FAIL"), "INSERT INTO t VALUES (9)", false}, + {"a unique constraint", "CREATE TABLE t (x UNIQUE)", "INSERT INTO t VALUES (1)", false}, + {"a savepoint rolled back to and released", "CREATE TABLE t (x)", "SAVEPOINT a; INSERT INTO t VALUES (5); ROLLBACK TO a; RELEASE a", false}, + {"ROLLBACK TO no savepoint", "CREATE TABLE t (x)", "ROLLBACK TO nosuch", false}, + {"ROLLBACK", "CREATE TABLE t (x)", "ROLLBACK", false}, + {"COMMIT", "CREATE TABLE t (x)", "COMMIT", false}, + {"END", "CREATE TABLE t (x)", "END", false}, + } { + ended := rawEndsTransaction(t, c) + d := open(t) + db := d.DB() + if _, err := db.ExecContext(ctx, c.table+"; CREATE TABLE side (x)"); err != nil { + t.Fatal(err) + } + var pages int + if err := db.QueryRowContext(ctx, "PRAGMA page_count").Scan(&pages); err != nil { + t.Fatal(err) + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err != nil { + t.Fatal(err) + } + if c.full { + if _, err := tx.ExecContext(ctx, "PRAGMA max_page_count = "+strconv.Itoa(pages+3)); err != nil { + t.Fatal(err) + } + } + stmtErr := func() error { _, err := tx.ExecContext(ctx, c.stmt); return err }() + _, err = tx.ExecContext(ctx, "INSERT INTO side VALUES (1)") + aborted, done := errors.Is(err, hoppersqlite.ErrTransactionAborted), errors.Is(err, sql.ErrTxDone) + switch { + case ended && stmtErr != nil && !aborted: + t.Errorf("%s: SQLite rolled the transaction back after %v; the next statement = %v, want ErrTransactionAborted", c.name, stmtErr, err) + case ended && stmtErr == nil && !done: + t.Errorf("%s: the statement ended the transaction; the next statement = %v, want sql.ErrTxDone", c.name, err) + case !ended && err != nil: + t.Errorf("%s: SQLite kept the transaction open; the next statement = %v", c.name, err) + } + if c.full { + _, _ = tx.ExecContext(ctx, "PRAGMA max_page_count = 1073741823") + } + _ = tx.Rollback() + var side int + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM side").Scan(&side); err != nil || side != 0 { + t.Errorf("%s: %d rows of side left after Rollback, %v", c.name, side, err) + } + } +} + +// TestStatementEndsTransaction ends a transaction with a statement of its +// own: after COMMIT, what it wrote is committed and Commit returns nil; +// after ROLLBACK, nothing is and Commit returns sql.ErrTxDone. Either way +// the write lock is free once the transaction ends. +func TestStatementEndsTransaction(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + db := d.DB() + if _, err := db.ExecContext(ctx, "CREATE TABLE t (x)"); err != nil { + t.Fatal(err) + } + for _, c := range []struct { + stmt string + rows int + err error + }{{"COMMIT", 1, nil}, {"ROLLBACK", 0, sql.ErrTxDone}} { + if _, err := db.ExecContext(ctx, "DELETE FROM t"); err != nil { + t.Fatal(err) + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, c.stmt); err != nil { + t.Fatalf("%s: %v", c.stmt, err) + } + if err := tx.Commit(); !errors.Is(err, c.err) || (c.err == nil && err != nil) { + t.Errorf("Commit after %s = %v, want %v", c.stmt, err, c.err) + } + var n int + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM t").Scan(&n); err != nil || n != c.rows { + t.Errorf("after %s: %d rows, %v; want %d", c.stmt, n, err, c.rows) + } + } +} diff --git a/driver/hoppersqlite/hoppersqlite.go b/driver/hoppersqlite/hoppersqlite.go index 65d6fdf..89a1601 100644 --- a/driver/hoppersqlite/hoppersqlite.go +++ b/driver/hoppersqlite/hoppersqlite.go @@ -37,20 +37,26 @@ // (InsertTx and the like), not through the client: a write on the pool // would wait for the transaction's own lock. In a job's worker, whose // context names the job's attempt, such a write fails at once with -// ErrLockHeldByOwner instead. +// ErrLockHeldByOwner instead, if the worker began the transaction with +// that context or one derived from it; begin it so. Writes of the +// attempt with no transaction of its own open wait their turn, from +// however many goroutines. // - After some errors (an interrupted write, a full disk, an I/O error) // SQLite rolls the whole transaction back itself. The transaction's // later statements, and its Commit, then fail with // ErrTransactionAborted, as those of a transaction an error aborted do // on Postgres. An error that SQLite undoes only the statement of, such // as a constraint's, leaves the transaction open, where Postgres would -// abort it. +// abort it. A COMMIT, END or ROLLBACK statement run in a transaction +// ends it: its later statements fail with sql.ErrTxDone. // - The database must be on a local filesystem: Open refuses one on a // network filesystem, or a virtual machine's shared folder, with // ErrNetworkFilesystem, checking where symbolic links in the path // lead. Its clock is therefore the host's, which every process that // opens it shares, and the driver reads it as database time, never -// earlier than a time it read before in the process. A host that +// earlier than a time it read before in the same process (a host +// clock stepped back can still date one process's write before +// another's earlier one). A host that // sleeps and wakes finds its clock moved on, as a database server's // would, and may rescue the jobs of its own clients, whose leases // expired while it slept. diff --git a/driver/hoppersqlite/localfs.go b/driver/hoppersqlite/localfs.go index b6e4426..7b43007 100644 --- a/driver/hoppersqlite/localfs.go +++ b/driver/hoppersqlite/localfs.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" ) // ErrNetworkFilesystem is returned by Open for a database on a network or @@ -26,38 +27,80 @@ const maxLinks = 40 // resolved, a last link to a file that does not exist yet included. SQLite // opens the file a link names and keeps its write-ahead log and shared // memory beside it, not beside the link, so the path resolved is the one -// to check and to open. +// to check and to open. A path that exists is resolved as the kernel +// resolves it (filepath.EvalSymlinks); one that does not yet, component by +// component, as the kernel would to create it. func resolve(path string) (string, error) { - p, err := filepath.Abs(path) - if err != nil { - return "", err - } - for range maxLinks { - dir, err := filepath.EvalSymlinks(filepath.Dir(p)) + // Not filepath.Abs, which would clean ".." against what precedes it + // before the links in it are followed. + p := path + if !filepath.IsAbs(p) { + wd, err := os.Getwd() if err != nil { return "", err } - p = filepath.Join(dir, filepath.Base(p)) - fi, err := os.Lstat(p) - if errors.Is(err, fs.ErrNotExist) { - return p, nil + p = wd + string(filepath.Separator) + p + } + r, err := filepath.EvalSymlinks(p) + if err == nil { + return r, nil + } + if !errors.Is(err, fs.ErrNotExist) { + return "", err + } + return resolveMissing(p) +} + +// resolveMissing resolves the links of an absolute path whose last +// component, or the last a link in it leads to, does not exist: each +// component in turn, a link's target in its place, and ".." from the +// directory resolved so far, as the kernel does, not lexically. +func resolveMissing(p string) (string, error) { + vol := filepath.VolumeName(p) + cur := vol + string(filepath.Separator) + rest := split(p[len(vol):]) + links := 0 + for len(rest) > 0 { + name := rest[0] + rest = rest[1:] + switch name { + case ".": + continue + case "..": + cur = filepath.Dir(cur) + continue } - if err != nil { + next := filepath.Join(cur, name) + fi, err := os.Lstat(next) + switch { + case errors.Is(err, fs.ErrNotExist) && len(rest) == 0: + return next, nil // the file SQLite will create + case err != nil: return "", err + case fi.Mode()&fs.ModeSymlink == 0: + cur = next + continue } - if fi.Mode()&fs.ModeSymlink == 0 { - return p, nil + if links++; links > maxLinks { + return "", fmt.Errorf("%s: more than %d symbolic links", p, maxLinks) } - target, err := os.Readlink(p) + target, err := os.Readlink(next) if err != nil { return "", err } - if !filepath.IsAbs(target) { - target = filepath.Join(dir, target) + if filepath.IsAbs(target) { + vol = filepath.VolumeName(target) + cur = vol + string(filepath.Separator) + target = target[len(vol):] } - p = target + rest = append(split(target), rest...) } - return "", fmt.Errorf("%s: more than %d symbolic links", path, maxLinks) + return cur, nil +} + +// split returns the components of a path, without empty ones. +func split(p string) []string { + return strings.FieldsFunc(p, func(r rune) bool { return r < 0x80 && os.IsPathSeparator(byte(r)) }) //nolint:gosec // r < 0x80 } // checkLocal returns ErrNetworkFilesystem if the database file at path, or diff --git a/driver/hoppersqlite/migrate_test.go b/driver/hoppersqlite/migrate_test.go index 21c1837..d9b68be 100644 --- a/driver/hoppersqlite/migrate_test.go +++ b/driver/hoppersqlite/migrate_test.go @@ -323,8 +323,8 @@ func upgradeUnderTraffic(t *testing.T, from int) { time.Sleep(20 * time.Millisecond) close(stop) wg.Wait() - if longest > 5*time.Second { - t.Errorf("a statement waited %s for the migration", longest) + if bound := slowdown * 5 * time.Second; longest > bound { + t.Errorf("a statement waited %s for the migration, more than %s", longest, bound) } // The release's client works every job left, the running ones once diff --git a/driver/hoppersqlite/norace_test.go b/driver/hoppersqlite/norace_test.go new file mode 100644 index 0000000..983b5ce --- /dev/null +++ b/driver/hoppersqlite/norace_test.go @@ -0,0 +1,6 @@ +//go:build !race + +package hoppersqlite_test + +// slowdown is 1 without the race detector (race_test.go). +const slowdown = 1 diff --git a/driver/hoppersqlite/race_test.go b/driver/hoppersqlite/race_test.go new file mode 100644 index 0000000..a600377 --- /dev/null +++ b/driver/hoppersqlite/race_test.go @@ -0,0 +1,8 @@ +//go:build race + +package hoppersqlite_test + +// slowdown is how many times longer the race detector makes the driver's +// work, which is mostly SQLite's transpiled C: bounds on how long a +// statement may wait are multiplied by it. +const slowdown = 6 diff --git a/driver/hoppersqlite/resolve_internal_test.go b/driver/hoppersqlite/resolve_internal_test.go new file mode 100644 index 0000000..cd838ff --- /dev/null +++ b/driver/hoppersqlite/resolve_internal_test.go @@ -0,0 +1,53 @@ +package hoppersqlite + +import ( + "os" + "path/filepath" + "testing" +) + +// TestResolve checks that resolve follows links as the kernel does: ".." +// after a linked directory leads up from the link's target, not from the +// link, for a file that exists and one that does not yet; and a chain of +// links ends where the last leads. +func TestResolve(t *testing.T) { + t.Parallel() + local, remote := t.TempDir(), t.TempDir() + local, _ = filepath.EvalSymlinks(local) + remote, _ = filepath.EvalSymlinks(remote) + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(os.Mkdir(filepath.Join(remote, "inner"), 0o755)) + must(os.Symlink(filepath.Join(remote, "inner"), filepath.Join(local, "sub"))) + must(os.WriteFile(filepath.Join(remote, "other.db"), nil, 0o600)) + must(os.Symlink("sub/../other.db", filepath.Join(local, "existing.db"))) + must(os.Symlink("sub/../missing.db", filepath.Join(local, "dangling.db"))) + must(os.Symlink("dangling.db", filepath.Join(local, "chain.db"))) + for link, want := range map[string]string{ + "existing.db": filepath.Join(remote, "other.db"), + "dangling.db": filepath.Join(remote, "missing.db"), + "chain.db": filepath.Join(remote, "missing.db"), + "sub/../new.db": filepath.Join(remote, "new.db"), + "sub/./x/../../a.db": "", + "plain.db": filepath.Join(local, "plain.db"), + } { + got, err := resolve(local + string(filepath.Separator) + link) // not Join, which cleans ".." + if want == "" { + if err == nil { + t.Errorf("resolve(%s) = %s, want an error for a missing directory", link, got) + } + continue + } + if err != nil || got != want { + t.Errorf("resolve(%s) = %s, %v; want %s", link, got, err, want) + } + } + must(os.Symlink("loop.db", filepath.Join(local, "loop.db"))) + if got, err := resolve(filepath.Join(local, "loop.db")); err == nil { + t.Errorf("resolve of a link to itself = %s, want an error", got) + } +} diff --git a/driver/hoppersqlite/robustness_test.go b/driver/hoppersqlite/robustness_test.go index bc20a56..dd20aa0 100644 --- a/driver/hoppersqlite/robustness_test.go +++ b/driver/hoppersqlite/robustness_test.go @@ -473,3 +473,83 @@ func TestOpenContext(t *testing.T) { t.Errorf("OpenContext with an ended context = %v", err) } } + +type fanArgs struct{} + +func (fanArgs) Kind() string { return "fan" } + +type leafArgs struct{ N int } + +func (leafArgs) Kind() string { return "leaf" } + +// TestConcurrentWritesOfOneAttempt runs a worker that writes on the pool +// from eight goroutines at once, with no transaction of its own open, and +// then records a step while it inserts: every write waits its turn and +// succeeds, as on Postgres, rather than failing with ErrLockHeldByOwner +// because another write of the same attempt holds the lock. +func TestConcurrentWritesOfOneAttempt(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + var ( + mu sync.Mutex + errs []error + worked = make(chan struct{}) + ) + note := func(err error) { + if err != nil { + mu.Lock() + errs = append(errs, err) + mu.Unlock() + } + } + workers := hopper.NewWorkers() + hopper.AddWorkFunc(workers, func(ctx context.Context, job *hopper.Job[fanArgs]) error { + defer close(worked) + c, _ := hopper.ClientFromContext[*sql.Tx](ctx) + var wg sync.WaitGroup + for g := range 8 { + wg.Go(func() { + for i := range 25 { + _, err := c.Insert(ctx, leafArgs{N: g*100 + i}, nil) + note(err) + } + }) + } + wg.Wait() + wg.Go(func() { note(hopper.RecordStep(ctx, job, "a", nil)) }) + wg.Go(func() { _, err := c.Insert(ctx, leafArgs{N: -1}, nil); note(err) }) + wg.Wait() + return nil + }) + hopper.AddWorkFunc(workers, func(context.Context, *hopper.Job[leafArgs]) error { return nil }) + c, err := hopper.NewClient(d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: 1}}, + Workers: workers, + Logger: slog.New(slog.DiscardHandler), + }) + if err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + defer c.Stop(ctx) //nolint:errcheck // cleanup + if _, err := c.Insert(ctx, fanArgs{}, nil); err != nil { + t.Fatal(err) + } + select { + case <-worked: + case <-time.After(time.Minute): + t.Fatal("the worker did not finish") + } + mu.Lock() + defer mu.Unlock() + if len(errs) > 0 { + t.Errorf("%d of 202 writes of one attempt failed, the first: %v", len(errs), errs[0]) + } + var leaves int + if err := d.DB().QueryRowContext(ctx, "SELECT (SELECT count(*) FROM hopper_jobs WHERE kind = 'leaf') + (SELECT count(*) FROM hopper_job_history WHERE kind = 'leaf')").Scan(&leaves); err != nil || leaves != 201 { + t.Errorf("leaf jobs = %d, %v; want 201", leaves, err) + } +} From a748f524df72c81e5cb9883e520890a4cfe8bd8e Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 20:56:06 +0000 Subject: [PATCH 19/20] docs: the lease renewal's bound in PLAN 7.6; the owner check's scope, statement-ended transactions and the clock within one process; Release-As at merge time PLAN 7.6 describes the renewal's bound for every engine. 5.1 says the owner check applies to a transaction the worker began with its context, and that a COMMIT or ROLLBACK statement ends a transaction; 7.1 that the clock's floor holds within one process. getting-started says to begin a worker's transaction with the job's context. releasing.md: the repository squashes with a blank body, so a Release-As or BREAKING CHANGE footer is given at merge time with gh pr merge --body. --- docs/PLAN.md | 38 +++++++++++++++++++++++++++----------- docs/getting-started.md | 5 +++-- docs/releasing.md | 7 +++++-- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/docs/PLAN.md b/docs/PLAN.md index d4bf1a9..e70c17a 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -458,19 +458,26 @@ defined as operations rather than SQL: What a single writer costs, and how the driver keeps it from failing silently: - A transaction held open holds up every client that shares the database, their - lease renewals included. A renewal waits until the lease it renews would expire, - then logs who holds the lock and tries again; a transaction held for longer than - about the lease therefore costs every client its lease, and their running jobs are - rescued. + lease renewals included. A renewal waits until the lease it renews would expire + (§7.6), then logs who holds the lock and tries again; a transaction held for longer + than about the lease therefore costs every client its lease, and their running jobs + are rescued. - A worker that writes on the pool while its own transaction holds the lock would wait for itself until its context ended. The core marks each attempt's context with - its owner (`driver.WithOwner`), and the driver refuses at once such a write - (`ErrLockHeldByOwner`). + its owner (`driver.WithOwner`); the driver records the owner of a transaction its + caller began with that context, and refuses at once a write of the same owner that + would wait for it (`ErrLockHeldByOwner`). The driver's own transactions, each for one + operation, record none, so writes of one attempt from several goroutines, with no + transaction of its own open, wait their turn as on Postgres. A worker's transaction + begun with another context is not recognised, and its pool write waits as before. - After some errors (an interrupted write, a full disk, an I/O error) SQLite rolls the whole transaction back itself and would run its later statements on their own. The driver learns of it from SQLite's rollback hook and refuses the transaction's later statements and its commit (`ErrTransactionAborted`), as Postgres refuses those of an - aborted transaction, so `CompleteTx` stays all or nothing. + aborted transaction, so `CompleteTx` stays all or nothing. A `COMMIT`, `END` or + `ROLLBACK` statement the caller runs in the transaction ends it too, which the commit + and rollback hooks tell; its later statements fail with `sql.ErrTxDone` rather than + run on their own. - JSON is stored as given rather than normalized as jsonb is, and refused for what jsonb refuses (a `\u0000` escape, text that is not UTF-8, a number beyond numeric's range); a metadata filter's `@>` is a SQL function that matches as jsonb's does. @@ -720,10 +727,11 @@ All time comparisons use the database's `now()`, never the application clock, so skew between pods cannot cause early or duplicate execution. SQLite's clock is the host's, which every process that opens the database shares, since it must be on a local filesystem: hoppersqlite reads it once a transaction holds the write lock, and never -returns a time earlier than one it returned before in the process, so each write's time -is no earlier than the writes before it even when the host's clock is stepped back. A -host that sleeps finds its clock moved on when it wakes, as a server's would, and its -clients' leases expired. +returns a time earlier than one it returned before in the same process, so within one +process each write's time is no earlier than the writes before it even when the host's +clock is stepped back. Across processes the floor does not hold: a clock stepped back +can date one process's write before another's earlier one. A host that sleeps finds its +clock moved on when it wakes, as a server's would, and its clients' leases expired. ### 7.2 Insert - **Single and small batches:** one `INSERT … SELECT FROM unnest($kinds, $args, …) @@ -1397,6 +1405,14 @@ A heartbeat therefore costs one row per process regardless of how many jobs it i running. Running jobs are never rewritten just to prove they are alive, which removes the largest source of write amplification in heartbeat-based designs. +A renewal waits at most until the lease it renews would expire, counted from when the +lease was last taken or renewed (registering again after fencing takes a new one), and +at least one renewal interval. On Postgres a renewal locks only its own client's row and +is seldom held up; on an engine that runs one writer at a time, any open transaction +holds it up. A renewal that gives up is logged, with what the engine says of who holds +the lock, and tried again at once, rather than left waiting past the lease without a +word. + - **Rescue.** The leader's rescuer finds `running` jobs whose `attempted_by` has no unexpired lease and moves them to `retryable` with the error `client lost` (or to `discarded` if they are out of attempts). A crashed pod's jobs run again within about diff --git a/docs/getting-started.md b/docs/getting-started.md index d789887..b0c988d 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -103,8 +103,9 @@ statement. SQLite runs one writing transaction at a time, so keep them short: one held open holds up every client sharing the file, lease renewals included, and one held longer than the lease costs them their running jobs. Write through the transaction (`InsertTx`, `CompleteTx`) rather than the pool -while one is open; in a worker, a write on the pool then fails at once with -`hoppersqlite.ErrLockHeldByOwner`. After an error that makes SQLite roll a +while one is open. In a worker, begin it with the job's context: a write on +the pool then fails at once with `hoppersqlite.ErrLockHeldByOwner`, rather +than wait for the worker's own transaction until the context ends. After an error that makes SQLite roll a transaction back (an interrupted write, a full disk), its later statements fail with `hoppersqlite.ErrTransactionAborted`; roll it back. diff --git a/docs/releasing.md b/docs/releasing.md index eb0d41e..15d598a 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -76,8 +76,11 @@ the ruleset's commit-message pattern. ## Steering a release -- **Force a version:** add a footer `Release-As: X.Y.Z` to a squash commit's - body (the PR description), and the release PR proposes that version. +- **Force a version, or mark a breaking change:** the repository squashes a PR + with its title and a blank body, so nothing in the PR description reaches the + commit. Give the footer when merging, and the release PR reads it: + `gh pr merge N --squash --admin --body 'Release-As: X.Y.Z'`, or + `--body 'BREAKING CHANGE: …'` to say what a `!` in the title breaks. - **Hold a change out of the changelog:** only `feat`, `fix`, `perf` and `revert` appear; `docs`, `test`, `ci`, `build`, `chore` and `refactor` are hidden. From e066f05cdb9cd0fccf3da446062ed45eded0dfe4 Mon Sep 17 00:00:00 2001 From: Michael McQuade Date: Thu, 22 Oct 2026 23:13:59 +0000 Subject: [PATCH 20/20] fix(hoppersqlite): a form feed is whitespace to mayWrite, and a COMMIT counts only when it succeeds; test that a renewal moves the lease's bound From the third review: - sqlTokens took a form feed for punctuation, so a statement after one ("\fDELETE FROM t") read as one that does not write; every write that a 40,000-input fuzz against sqlite3_stmt_readonly found mayWrite missing began so. A form feed is whitespace, as it is to SQLite; a vertical tab is not, and fails to prepare. TestMayWrite has the cases. - leave took the commit hook as a COMMIT statement whatever the statement returned; it now needs the statement to have succeeded. The case of a rollback after an error still comes first. - TestRenewalBoundMovesWithRenewals: after renewals at 5s and 10s, the renewal held at 15s is given until 25s. Dropping the store after a successful renewal passed the earlier test; it fails this one. The package doc says SQLite reports no COMMIT in a read-only transaction, after which its reads run on their own. --- driver/hoppersqlite/conn.go | 5 ++-- driver/hoppersqlite/conn_internal_test.go | 4 +++ driver/hoppersqlite/hoppersqlite.go | 4 ++- lease_internal_test.go | 31 +++++++++++++++++++++++ 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/driver/hoppersqlite/conn.go b/driver/hoppersqlite/conn.go index 19926a9..a83e748 100644 --- a/driver/hoppersqlite/conn.go +++ b/driver/hoppersqlite/conn.go @@ -536,7 +536,7 @@ func (c *conn) leave(err error) { t.aborted = err case c.rolledBack: t.endedBy = "a ROLLBACK statement" - case c.committed: + case c.committed && err == nil: t.endedBy, t.committed = "a COMMIT statement", true } } @@ -709,7 +709,8 @@ func sqlTokens(q string) [][]string { } cur = append(cur, strings.ToUpper(q[i:j])) i = j - case c == ' ' || c == '\t' || c == '\n' || c == '\r': + case c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r': + // SQLite's whitespace; a vertical tab is not, and fails to prepare. i++ default: cur = append(cur, string(c)) diff --git a/driver/hoppersqlite/conn_internal_test.go b/driver/hoppersqlite/conn_internal_test.go index 0da3883..9a34a24 100644 --- a/driver/hoppersqlite/conn_internal_test.go +++ b/driver/hoppersqlite/conn_internal_test.go @@ -46,6 +46,10 @@ func TestMayWrite(t *testing.T) { "VALUES (1)": false, "EXPLAIN DELETE FROM t": false, "SELECT :update, ?1, $delete": false, + "\fDELETE FROM t": true, + "SELECT 1;\f\fUPDATE t SET x = 1": true, + "WITH d AS (SELECT 1)\fINSERT INTO t SELECT * FROM d": true, + "\f-- note\n\fREPLACE INTO t VALUES (1)": true, } { if got := mayWrite(query); got != want { t.Errorf("mayWrite(%q) = %v, want %v", query, got, want) diff --git a/driver/hoppersqlite/hoppersqlite.go b/driver/hoppersqlite/hoppersqlite.go index 89a1601..096910e 100644 --- a/driver/hoppersqlite/hoppersqlite.go +++ b/driver/hoppersqlite/hoppersqlite.go @@ -48,7 +48,9 @@ // on Postgres. An error that SQLite undoes only the statement of, such // as a constraint's, leaves the transaction open, where Postgres would // abort it. A COMMIT, END or ROLLBACK statement run in a transaction -// ends it: its later statements fail with sql.ErrTxDone. +// ends it: its later statements fail with sql.ErrTxDone. In a read-only +// transaction SQLite tells nothing of a COMMIT, and the statements +// after it, reads all, run on their own. // - The database must be on a local filesystem: Open refuses one on a // network filesystem, or a virtual machine's shared folder, with // ErrNetworkFilesystem, checking where symbolic links in the path diff --git a/lease_internal_test.go b/lease_internal_test.go index 1fd29fd..98876ee 100644 --- a/lease_internal_test.go +++ b/lease_internal_test.go @@ -81,3 +81,34 @@ func TestRenewalWaitsUntilTheLeaseExpires(t *testing.T) { } }) } + +// TestRenewalBoundMovesWithRenewals checks the common case: each renewal +// that succeeds moves the bound, so a renewal held up after two that +// succeeded is given until the lease the last of them took expires. +func TestRenewalBoundMovesWithRenewals(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + exec := &leaseExec{renew: true} + tun := defaultTuning.withLease(15 * time.Second) // renewals every 5s + c := &Client[struct{}]{exec: exec, logger: slog.New(slog.DiscardHandler), tuning: tun} + if _, err := c.register(ctx); err != nil { // the lease from 0s + t.Fatal(err) + } + done := make(chan struct{}) + go func() { defer close(done); c.leaseLoop(ctx) }() + + time.Sleep(12 * time.Second) // renewed at 5s and 10s + exec.setRenew(false) + // The renewal at 15s is held, and given until 25s, when the lease + // renewed at 10s expires. + time.Sleep(5 * time.Second) + cancel() + <-done + exec.mu.Lock() + defer exec.mu.Unlock() + if want := []time.Duration{10 * time.Second, 10 * time.Second, 10 * time.Second}; !slices.Equal(exec.waits, want) { + t.Errorf("renewals were given %v, want %v", exec.waits, want) + } + }) +}