diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fa39351..17f8047 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,7 +3,7 @@ version: 2 # compromised release is likely to have been pulled before it is proposed here. updates: - package-ecosystem: gomod - directories: ["/", "/tools", "/tools/dev", "/hopperotel", "/hopperui"] + directories: ["/", "/tools", "/tools/dev", "/hopperotel", "/hopperui", "/driver/hoppersqlite"] schedule: { interval: weekly } cooldown: { default-days: 7 } commit-message: { prefix: "build(deps)" } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed86d88..df71218 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,11 +41,13 @@ jobs: (cd tools/dev && GOTOOLCHAIN=auto go mod tidy -diff) (cd hopperotel && go mod tidy -diff) (cd hopperui && go mod tidy -diff) + (cd driver/hoppersqlite && go mod tidy -diff) - name: Lint run: | go tool -modfile=tools/go.mod golangci-lint run ./... (cd hopperotel && go tool -modfile=../tools/go.mod golangci-lint run ./...) (cd hopperui && go tool -modfile=../tools/go.mod golangci-lint run ./...) + (cd driver/hoppersqlite && go tool -modfile=../../tools/go.mod golangci-lint run ./...) # Audits the workflows themselves: unpinned actions, script injection, # over-broad tokens, leaked credentials. Any finding fails the job. @@ -97,3 +99,23 @@ jobs: go test -race -cover -timeout 10m ./... (cd hopperotel && go test -race -cover -timeout 10m ./...) (cd hopperui && go test -race -cover -timeout 10m ./...) + + # hoppersqlite needs no server, so it runs once rather than per Postgres + # version, and proves that it and everything it imports build without cgo. + test-sqlite: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Build without cgo + run: make cgo-check + - name: Test + env: + GODEBUG: fips140=only + working-directory: driver/hoppersqlite + run: go test -race -cover -timeout 10m ./... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 87217ff..94a2d25 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -67,17 +67,21 @@ jobs: - name: Tag the sub-modules at the same version # release-please manages one package, the core module, and rewrites # the sub-modules' requirement on it (x-release-please-version in - # their go.mod). Their tags are added here so that all three modules - # resolve at one version. + # their go.mod). Their tags are added here so that every module + # resolves at one version. A module in a subdirectory is tagged with + # its path, as the go command looks it up. env: TAG: ${{ needs.release-please.outputs.tag_name }} run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - for m in hopperotel hopperui; do + modules="hopperotel hopperui driver/hoppersqlite" + tags=() + for m in $modules; do git tag -a "$m/$TAG" "$TAG" -m "$m $TAG" + tags+=("$m/$TAG") done - git push origin "hopperotel/$TAG" "hopperui/$TAG" + git push origin "${tags[@]}" - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod diff --git a/AGENTS.md b/AGENTS.md index 91bbbee..f200116 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,9 @@ owning client. Changes to the SQL in the claim, finalize, rescue or leader paths need a concurrency or chaos test. - Always use database time (`now()`), never application time, when comparing job times. +- A schema change is a new version in `hoppermigrate/migrations` and the same version in + `driver/hoppersqlite/migrations`, up and down; for a change SQLite has no use for, its + script is a comment saying why. hoppersqlite's tests fail for a version it lacks. - Changes to the insert, claim or finalize paths include `hopperbench` results in the PR. A regression of more than 10% against `canary` blocks the merge. - [docs/PLAN.md](docs/PLAN.md) is the design of record. Change it in the same PR as a diff --git a/Makefile b/Makefile index 81c0eb6..e84543d 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,7 @@ lint: ## Lint Go code $(GOTOOL) golangci-lint run ./... cd hopperotel && go tool -modfile=../tools/go.mod golangci-lint run ./... cd hopperui && go tool -modfile=../tools/go.mod golangci-lint run ./... + cd driver/hoppersqlite && go tool -modfile=../../tools/go.mod golangci-lint run ./... .PHONY: fmt fmt: ## Format Go code @@ -58,7 +59,21 @@ test-modules: go test -race -cover -timeout 10m ./... cd hopperotel && go test -race -cover -timeout 10m ./... cd hopperui && go test -race -cover -timeout 10m ./... + cd driver/hoppersqlite && go test -race -cover -timeout 10m ./... .PHONY: bench bench: ## Run the benchmark harness against the test database go run ./cmd/hopperbench -database-url "$(HOPPER_TEST_DATABASE_URL)" + +.PHONY: bench-sqlite +bench-sqlite: ## Run the benchmark harness on a SQLite database in a temporary directory + cd driver/hoppersqlite && go run ./cmd/hopperbench-sqlite + +.PHONY: cgo-check +cgo-check: ## Check that hoppersqlite, its tests and every module they import use no cgo + @cd driver/hoppersqlite && for target in linux/amd64 linux/386 linux/arm darwin/arm64 windows/amd64 freebsd/amd64 openbsd/amd64 netbsd/amd64; do \ + export GOOS=$${target%/*} GOARCH=$${target#*/}; \ + CGO_ENABLED=0 go vet ./... || exit 1; \ + pkgs=$$(CGO_ENABLED=1 go list -deps -test -f '{{if and .CgoFiles (not .Standard)}}{{.ImportPath}}{{end}}' ./...) || exit 1; \ + if [ -n "$$pkgs" ]; then echo "cgo is not allowed, but these packages use it on $$target:"; echo "$$pkgs"; exit 1; fi; \ + done diff --git a/README.md b/README.md index 5f5d33c..d09ef0c 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,8 @@ hopper runs background jobs and delivers messages using the PostgreSQL database your application already has. There's no broker to operate and no separate worker service: it is a library that runs inside your process and scales out with it. -Postgres is the first engine, behind a driver interface designed for more. +Postgres is the first engine, behind a driver interface designed for more; SQLite +is the second, for an application that runs on one machine. - **Fast:** set-based claim, finalize and insert (including `COPY`), a live table that stays small however much history accumulates, and retention by @@ -36,7 +37,8 @@ Postgres is the first engine, behind a driver interface designed for more. > ordering keys, streams with consumers, a SQL contract for producers in other > languages), flow control, batches and workflows, plus the `hopper` CLI, > `hoppertest`, `hopperotel`, the `hopperui` web UI and the `drivertest` -> conformance suite. [docs/PLAN.md](docs/PLAN.md) is the plan of record and +> conformance suite, and a SQLite driver (`driver/hoppersqlite`, pure Go, a module +> of its own) that passes it. [docs/PLAN.md](docs/PLAN.md) is the plan of record and > [docs/getting-started.md](docs/getting-started.md) the place to start. > Until v1.0.0, minor versions may change the API. Feedback is welcome > through issues and PRs. diff --git a/await.go b/await.go index dd4d14c..c368067 100644 --- a/await.go +++ b/await.go @@ -50,6 +50,12 @@ type attempt struct { logEnded bool } +// String names the attempt, as the owner of its worker's writes +// (driver.WithOwner). +func (a *attempt) String() string { + return fmt.Sprintf("job %s (%s), attempt %d", a.row.ID, a.row.Kind, a.attempt) +} + func (a *attempt) markHeldByTx() { a.mu.Lock() a.heldByTx = true diff --git a/client.go b/client.go index 1000710..389a9a2 100644 --- a/client.go +++ b/client.go @@ -35,6 +35,10 @@ type Client[TTx any] struct { // clientID is the lease row ID while running. It changes if the lease is // lost and re-registered. clientID atomic.Int64 + // leaseFrom is when the current lease was last taken or renewed, as + // register and the lease loop saw it: its expiry bounds a renewal's + // wait. + leaseFrom atomic.Pointer[time.Time] mu sync.Mutex state clientState @@ -330,6 +334,7 @@ func (c *Client[TTx]) declare(ctx context.Context) error { } func (c *Client[TTx]) register(ctx context.Context) (int64, error) { + start := time.Now() info, _ := json.Marshal(map[string]any{ "pid": os.Getpid(), "queues": slices.Sorted(maps.Keys(c.cfg.Queues)), @@ -342,6 +347,7 @@ func (c *Client[TTx]) register(ctx context.Context) (int64, error) { if err != nil { return 0, fmt.Errorf("hopper: register client: %w", err) } + c.leaseFrom.Store(&start) return id, nil } @@ -494,6 +500,13 @@ func (c *Client[TTx]) wakeQueue(queue string) { // ID. Results still in flight are submitted anyway; the finalize statement // rejects any whose job was rescued meanwhile and applies the rest, which is // equivalent to a rescue. +// +// A renewal waits until the lease it renews would expire, and at least one +// renewal interval: on an engine that runs one writer at a time, a +// transaction that holds the database holds every renewal up, and a renewal +// that gives up is reported, with what the engine says of who holds the +// lock, and tried again, rather than left waiting past the lease without a +// word. func (c *Client[TTx]) leaseLoop(ctx context.Context) { ticker := time.NewTicker(c.tuning.leaseRenew) defer ticker.Stop() @@ -504,7 +517,10 @@ func (c *Client[TTx]) leaseLoop(ctx context.Context) { case <-ticker.C: } id := c.clientID.Load() - res, err := c.exec.ClientRenew(ctx, driver.ClientRenewParams{ClientID: id, TTL: c.tuning.leaseTTL}) + start := time.Now() + rctx, cancel := context.WithTimeout(ctx, c.renewalWait()) + res, err := c.exec.ClientRenew(rctx, driver.ClientRenewParams{ClientID: id, TTL: c.tuning.leaseTTL}) + cancel() if err != nil { if ctx.Err() == nil { c.logger.WarnContext(ctx, "hopper: renew client lease", "client_id", id, "error", err) @@ -512,9 +528,10 @@ func (c *Client[TTx]) leaseLoop(ctx context.Context) { continue } if !res.Renewed { - c.fence(ctx, id) + c.fence(ctx, id) // which registers again, and so records the new lease continue } + c.leaseFrom.Store(&start) // Control state rides along with every renewal, in case a // notification was missed. for _, jobID := range res.CancelRequested { @@ -525,6 +542,16 @@ func (c *Client[TTx]) leaseLoop(ctx context.Context) { } } +// renewalWait is how long a renewal may wait: until the lease would expire, +// and at least one renewal interval. +func (c *Client[TTx]) renewalWait() time.Duration { + wait := c.tuning.leaseRenew + if from := c.leaseFrom.Load(); from != nil { + wait = max(wait, time.Until(from.Add(c.tuning.leaseTTL))) + } + return wait +} + // limitedQueues reads which queues have limits. func (c *Client[TTx]) limitedQueues(ctx context.Context) ([]string, error) { queues, err := c.exec.QueueList(ctx) diff --git a/complete.go b/complete.go index e143651..ac900ee 100644 --- a/complete.go +++ b/complete.go @@ -30,13 +30,22 @@ var ErrJobFenced = errors.New("hopper: job is no longer running in this attempt" // whatever the worker returned, and the job.completed event fires. If it did // not, the attempt fails with the worker's error, or with a failure if the // worker returned nil, and the job is retried. A worker that returns before -// tx ends holds up only its own job's result, until tx ends. +// tx ends holds up only its own job's result, until tx ends, on an engine +// that locks rows. // // Call it last, just before the commit. From CompleteTx until tx ends, tx // holds the job's row and those of workflow dependents it promotes or // cancels, and holds off the creation and drop of history partitions. The // job's siblings in its batch finalize without waiting for it. // +// On an engine that runs one writer at a time +// (driver.Capabilities.SingleWriter, SQLite), tx holds the whole database +// for writes from its first statement until it ends: every other write +// waits for it, the results of other jobs and lease renewals included, and +// a write of the worker's own on the pool fails rather than wait for tx, +// provided tx was begun with the worker's ctx (or one derived from it), +// which names the job's attempt. +// // func (w *Worker) Work(ctx context.Context, job *hopper.Job[Args]) error { // tx, err := w.pool.Begin(ctx) // if err != nil { diff --git a/control.go b/control.go index 8953ba7..fc19bee 100644 --- a/control.go +++ b/control.go @@ -47,7 +47,8 @@ func (c *Client[TTx]) JobCancel(ctx context.Context, id JobID) (*JobRow, error) // // Until tx ends it holds the job's row, so a running job that finishes // meanwhile has its result wait for tx; its client's other results do not. -// Call it late in tx, as CompleteTx. +// Call it late in tx, as CompleteTx. On an engine that runs one writer at a +// time, tx holds every write in the database up until it ends. func (c *Client[TTx]) JobCancelTx(ctx context.Context, tx TTx, id JobID) (*JobRow, error) { return c.driver.UnwrapTx(tx).JobCancel(ctx, id) } @@ -114,7 +115,8 @@ func (c *Client[TTx]) JobDelete(ctx context.Context, id JobID) (*JobRow, error) // JobDeleteTx is JobDelete inside the caller's transaction. Until tx ends // it holds the job's row and, for a waiting job, those of the workflow -// dependents it cancels or promotes. +// dependents it cancels or promotes; on an engine that runs one writer at a +// time, every write in the database. func (c *Client[TTx]) JobDeleteTx(ctx context.Context, tx TTx, id JobID) (*JobRow, error) { return c.driver.UnwrapTx(tx).JobDelete(ctx, id) } @@ -144,7 +146,8 @@ func (c *Client[TTx]) JobRetryOrderingKey(ctx context.Context, queue, orderingKe // // Until tx ends it holds the rows of the jobs it woke and of the key's // running job, whose result waits for tx; its client's other results do -// not. Call it late in tx, as CompleteTx. +// not. Call it late in tx, as CompleteTx. On an engine that runs one writer +// at a time, tx holds every write in the database up until it ends. func (c *Client[TTx]) JobRetryOrderingKeyTx(ctx context.Context, tx TTx, queue, orderingKey string) (int, error) { queue = cmp.Or(queue, QueueDefault) exec := c.driver.UnwrapTx(tx) diff --git a/docs/PLAN.md b/docs/PLAN.md index 5818c26..e70c17a 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -9,7 +9,7 @@ in-process job framework and a separate message broker. - **Module:** `github.com/parallelworks/hopper` - **License:** Apache-2.0 - **Dependencies:** the Go standard library and `github.com/jackc/pgx/v5`. Nothing else in the core module. -- **Status:** M0 through M4 and M6 through M8 are implemented (§16), the §8.2 targets have been run on the reference hardware, and everything ships together as one version across the core module, `hopperotel` and `hopperui` ([docs/releasing.md](releasing.md)); **v0.1.1** is current. This document is the plan of record, and changes to it go through PRs. +- **Status:** M0 through M4 and M6 through M9 are implemented (§16), the §8.2 targets have been run on the reference hardware, and everything ships together as one version across the core module, `hopperotel`, `hopperui` and `driver/hoppersqlite` ([docs/releasing.md](releasing.md)); **v0.1.1** is current. This document is the plan of record, and changes to it go through PRs. The name refers to a feed hopper, which releases work into a machine one piece at a time, and to RADM Grace Hopper. It is also a fitting name for something @@ -416,7 +416,7 @@ the way that performs best on that engine. | `hopper/driver/hopperpgx` | Postgres via pgx v5 (the reference driver): COPY, LISTEN, pipelined statements | core | | `hopper/driver/hoppersql` | Postgres via `database/sql` (lib/pq, pgx stdlib); polling only | core | | `hopper/drivertest` | Conformance, concurrency and chaos suite every driver must pass | core | -| `hoppersqlite` (later) | SQLite: embedded, single-node and local development | separate module | +| `hopper/driver/hoppersqlite` | SQLite (modernc.org/sqlite, pure Go): embedded, single-node and local development | separate module | | `hoppermongo` (later) | MongoDB (replica set, for transactions) | separate module | The two Postgres drivers share one implementation of the SQL and the logic around it @@ -434,10 +434,60 @@ clock. Push notifications are optional, because polling is always available. The planned engines meet these requirements differently, which is why the interface is defined as operations rather than SQL: -- **SQLite** has a single writer, so a claim is an `UPDATE … RETURNING` inside a - `BEGIN IMMEDIATE` transaction, and SKIP LOCKED is unnecessary. There is no LISTEN, - so clients in the same process wake each other directly and other processes poll. - Retention is a batched `DELETE`. +- **SQLite** has a single writer (`Capabilities.SingleWriter`), so a claim is an + `UPDATE … RETURNING` inside a transaction that holds the write lock, and SKIP LOCKED + is unnecessary. Every transaction that may write takes the lock before its first + statement reads (`BEGIN IMMEDIATE`), so what it reads cannot go stale before it + writes; the driver's `database/sql` connections do that lazily, on a transaction's + first statement, so that beginning one never blocks. Within a process, writers queue + for the lock in turn, for as long as their context allows; across processes, SQLite's + busy handler polls for it. With no other writer in between, the finalizing transaction + reads each job, checks it in Go and writes it back, settles the job's batch and promotes + its dependents itself, so `BatchSettle` and `WorkflowPromoteReady` find nothing a race + left; `SkipLocked` and `NoWait` wait briefly for another transaction's writes and then + skip every job or return `ErrJobLocked`. Notifications are rows of a table, written in + the notifying transaction: a listener in the same process is woken when it commits, + and listeners in other processes look every 50ms, when `PRAGMA data_version` says + another connection committed. Retention is a batched `DELETE`; stream events commit + in append order, so a position is an event's sequence number. The database must be on + a local filesystem, where SQLite's locks hold: `Open` refuses one on NFS, SMB, Lustre, + GPFS, FUSE, a virtual machine's shared folders and the like, checking where symbolic + links in the path lead, since SQLite follows them. Its migrations follow + `hoppermigrate`'s versions one for one (`driver.MigrationScripts`), and + `hoppersqlite.Migrations` returns them for other tools. + + What a single writer costs, and how the driver keeps it from failing silently: + - A transaction held open holds up every client that shares the database, their + lease renewals included. A renewal waits until the lease it renews would expire + (§7.6), then logs who holds the lock and tries again; a transaction held for longer + than about the lease therefore costs every client its lease, and their running jobs + are rescued. + - A worker that writes on the pool while its own transaction holds the lock would + wait for itself until its context ended. The core marks each attempt's context with + its owner (`driver.WithOwner`); the driver records the owner of a transaction its + caller began with that context, and refuses at once a write of the same owner that + would wait for it (`ErrLockHeldByOwner`). The driver's own transactions, each for one + operation, record none, so writes of one attempt from several goroutines, with no + transaction of its own open, wait their turn as on Postgres. A worker's transaction + begun with another context is not recognised, and its pool write waits as before. + - After some errors (an interrupted write, a full disk, an I/O error) SQLite rolls the + whole transaction back itself and would run its later statements on their own. The + driver learns of it from SQLite's rollback hook and refuses the transaction's later + statements and its commit (`ErrTransactionAborted`), as Postgres refuses those of an + aborted transaction, so `CompleteTx` stays all or nothing. A `COMMIT`, `END` or + `ROLLBACK` statement the caller runs in the transaction ends it too, which the commit + and rollback hooks tell; its later statements fail with `sql.ErrTxDone` rather than + run on their own. + - JSON is stored as given rather than normalized as jsonb is, and refused for what + jsonb refuses (a `\u0000` escape, text that is not UTF-8, a number beyond numeric's + range); a metadata filter's `@>` is a SQL function that matches as jsonb's does. + - There is no `COPY`, so bulk inserts take the one insert path, and there are no SQL + functions for producers outside Go. Job IDs are minted in Go (§6.1). + - `JobsMaintain` refreshes statistics but never vacuums, so the file keeps the space + of deleted rows for SQLite to reuse; it truncates the write-ahead log once a reader + that held up checkpoints is done, and each connection cuts the log back to 32MiB + whenever a checkpoint empties it. + - `Open` waits for the database at most 30 seconds; `OpenContext` takes a context. - **MongoDB** claims with atomic `findOneAndUpdate` or `updateMany` (tagging a batch with a claim token, then reading it back). It uses `$$NOW` for server time, change streams for notifications, and TTL indexes or dropped collections for retention. @@ -638,9 +688,10 @@ index inserts stay local like a sequence. On Postgres 18 and later it calls the built-in `uuidv7()`. On 14–17 it is a SQL function that builds a v7 UUID from `clock_timestamp()` and the built-in `gen_random_uuid()`. Generation therefore works the same for Go inserts, `COPY` - and the SQL contract, and hopper's Go code needs no random number generator or - cryptography. Engines without a database-side generator (MongoDB) generate IDs in - the driver. + and the SQL contract, and the core's Go code needs no random number generator or + cryptography. Engines without a database-side generator generate IDs in the driver: + hoppersqlite mints UUIDv7s in Go, from the database's clock and `crypto/rand`, and + MongoDB's driver will too. - **Ordering uses `seq`, not the ID.** Several IDs created in the same millisecond are not guaranteed to sort in insertion order on every Postgres version. An internal identity column, `seq`, provides the tie-break for claim order and for @@ -673,7 +724,14 @@ there is **no scheduler loop** moving jobs between states. ### 7.1 Clock All time comparisons use the database's `now()`, never the application clock, so clock -skew between pods cannot cause early or duplicate execution. +skew between pods cannot cause early or duplicate execution. SQLite's clock is the host's, +which every process that opens the database shares, since it must be on a local +filesystem: hoppersqlite reads it once a transaction holds the write lock, and never +returns a time earlier than one it returned before in the same process, so within one +process each write's time is no earlier than the writes before it even when the host's +clock is stepped back. Across processes the floor does not hold: a clock stepped back +can date one process's write before another's earlier one. A host that sleeps finds its +clock moved on when it wakes, as a server's would, and its clients' leases expired. ### 7.2 Insert - **Single and small batches:** one `INSERT … SELECT FROM unnest($kinds, $args, …) @@ -1347,6 +1405,14 @@ A heartbeat therefore costs one row per process regardless of how many jobs it i running. Running jobs are never rewritten just to prove they are alive, which removes the largest source of write amplification in heartbeat-based designs. +A renewal waits at most until the lease it renews would expire, counted from when the +lease was last taken or renewed (registering again after fencing takes a new one), and +at least one renewal interval. On Postgres a renewal locks only its own client's row and +is seldom held up; on an engine that runs one writer at a time, any open transaction +holds it up. A renewal that gives up is logged, with what the engine says of who holds +the lock, and tried again at once, rather than left waiting past the lease without a +word. + - **Rescue.** The leader's rescuer finds `running` jobs whose `attempted_by` has no unexpired lease and moves them to `retryable` with the error `client lost` (or to `discarded` if they are out of attempts). A crashed pod's jobs run again within about @@ -1758,6 +1824,10 @@ rates: connection is named `hopper-listener:` in `pg_stat_activity`. - **Channels are per database**, not per schema. Two hopper schemas in one database see each other's notifications, which only causes spurious wake-ups. +- **SQLite** has no LISTEN: hoppersqlite writes notifications to a table in the + notifying transaction, wakes the listeners of its own process when it commits, and + has listeners in other processes look for new rows every 50ms, when another + connection has committed. Rows are kept for a minute. ### 7.12 Shutdown `Stop(ctx)` and `Run` (when its context is canceled) shut down in three steps: @@ -2105,6 +2175,15 @@ observability without new machinery. - SQL files are embedded in the `hoppermigrate` package, versioned, and forward-only by default. Down migrations exist for development. +- The embedded scripts are Postgres's. Another engine's driver has its own script, up + and down, for every version (`driver.MigrationScripts`), and `hoppermigrate` applies + those, so that a version names the same tables and columns on every engine and + `Client.Start`'s check holds on each. hoppersqlite's follow Postgres's version for + version; one that changes only what SQLite lacks (partitions, autovacuum settings, SQL + functions) is a script of comments. Its migrator holds SQLite's write lock from `Lock` + to `Close` and applies each version in a savepoint, so writers wait for the migration, + which takes milliseconds; its upgrade test (`TestUpgradeUnderTraffic`) starts from every + earlier version. - `hoppermigrate.Up(ctx, driver)` applies them under an advisory lock taken on a dedicated connection, not a pooled one. This avoids a pool-starvation deadlock: processes waiting for the lock must not hold the connections the migration needs. @@ -2223,7 +2302,7 @@ observability without new machinery. | --- | --- | | Unit | Backoff, cron parsing and time zones, pattern matching, unique-key construction, the state transition table, option validation. Timers are tested with `testing/synctest`. | | Integration | Against real Postgres 14–18 in CI (a service container per version), always with `-race` and `GODEBUG=fips140=only`. Each test gets its own schema, so tests run in parallel. | -| Driver conformance | `drivertest.Run(t, fixture)` runs the behavioral suite (inserting, unique keys, claiming, every finalize transition and fence, leases, leader, rescue, cancel, retry, TTL, listing, queues, periodic slots, stats, notifications, history maintenance) against every driver. A `Fixture` supplies isolated databases and the few engine-specific hooks (expire a lease, backdate an attempt, count rows). Engine internals such as partition management stay in the driver's own tests. | +| Driver conformance | `drivertest.Run(t, fixture)` runs the behavioral suite (inserting, unique keys, claiming, every finalize transition and fence, leases, leader, rescue, cancel, retry, TTL, listing, queues, periodic slots, stats, notifications, history maintenance) against every driver. A `Fixture` supplies isolated databases and the few engine-specific hooks (expire a lease or a job, backdate an attempt, store a value as a direct write would, wait until a statement waits for a lock, count rows); what depends on row locks is tested either way by `Capabilities.SingleWriter`. `drivertest.RunPostgres` adds the tests of what the Postgres drivers share beyond the interface (partitions and their indexes, the tags tables, progress rows, planner statistics); other engine internals stay in the driver's own tests. | | Concurrency | N clients by M jobs: every job is finalized exactly once when nothing crashes. A finalize held open in a caller's transaction (`CompleteTx`) against a competing fenced finalize: exactly one applies, whichever way the transaction ends. Uniqueness under concurrent inserts. Periodic slots unique across two leaders. An insert counting finished duplicates against its duplicate's finalize held open: the insert reports the finished job and leaves no live one. A progress write held open in a caller's transaction against the attempt's competing finalize: a write that commits first is kept, one that comes second is fenced. A stuck attempt rescued and claimed again by the same client: its late result does not finalize the new claim, and a cancel still reaches the new claim. Global and rate limits never exceeded. Ordering keys never run two at once. | | Chaos | `pg_terminate_backend` on a client mid-job, followed by rescue and re-run. A client that crashes after its workers committed `CompleteTx` transactions but before its finalizer flushed: none of those jobs runs again. A frozen client that loses its lease and must fence itself. Leader killed, new leader within the TTL. Stop with an expired context. Listener dropped, polling continues and the listener reconnects. | | Upgrade | Migrate from release N-1 with jobs in every state and live traffic. | @@ -2240,7 +2319,7 @@ observability without new machinery. connection (§7.11). Otherwise the client uses adaptive polling and logs that once. - **Stability:** API stability starts at v1.0.0. Until then, minor versions may break the API, with changes noted in the CHANGELOG. Releases are cut by release-please from - the Conventional Commit titles of merged PRs, one version across the three modules, + the Conventional Commit titles of merged PRs, one version across every module, with goreleaser attaching the CLI binaries ([docs/releasing.md](releasing.md)). The SQL insert contract (§10) is versioned separately and changes only with a deprecation window. @@ -2259,7 +2338,7 @@ The estimates assume one engineer. Each milestone is one or more PRs. | M6 | Messaging | Subscriptions, AMQP topic patterns, typed `Message[T]`, PublishTx fan-out, dedup, ordering keys, request/reply, SQL publish contract, `ReplayDiscarded`, the upgrade test. **Done.** | 5d | | M7 | Flow control and batches | Global limits, rate limits, partitioned limits, priority aging, batches with callbacks, `hoppersql` driver. **Done.** | 5d | | M8 | Workflows, streams, UI | Job dependencies and DAG workflows, streams with consumers, `hopperui`. **Done**, one PR each. | 2–3w | -| M9 | More engines (later) | `hoppersqlite`, then `hoppermongo`, each in its own module and passing `drivertest`. Not scheduled yet. | per engine | +| M9 | More engines | `hoppersqlite`, in its own module and passing `drivertest`. **Done.** `hoppermongo` later, not scheduled yet. | per engine | M0–M4 take roughly four weeks to a production-ready job queue that meets its performance targets. v1.0.0 follows M8, once the API has been proven in production. diff --git a/docs/getting-started.md b/docs/getting-started.md index 1bd7921..b0c988d 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -68,6 +68,7 @@ application's `search_path` is unchanged, so the pool can be shared. **Existing installations:** set `Schema` to the schema where your Hopper tables already live (for example `public`) before upgrading. Changing this setting selects another installation; it does not move existing jobs or tables. + Migration credentials need permission to create the schema, or you can create it beforehand. Migrating down to zero removes Hopper objects and leaves the namespace itself intact. @@ -76,6 +77,46 @@ The SQL files are in `hoppermigrate/migrations` for teams that run migrations with their own tool. Set a transaction-local `search_path` to the target schema when running the raw SQL. `hopper_schema` records the applied versions either way. +### SQLite + +An application that runs on one machine, such as a command-line tool or a +single-node service, can keep hopper in a SQLite file instead, with no server +to run. The driver is pure Go (`modernc.org/sqlite`), so the program still +builds with `CGO_ENABLED=0`, and it is a module of its own, +`github.com/parallelworks/hopper/driver/hoppersqlite`: + +```go +driver, err := hoppersqlite.Open(filepath.Join(stateDir, "hopper.db"), nil) +if err != nil { + return err // hoppersqlite.ErrNetworkFilesystem for a file on NFS and the like +} +defer driver.Close() +if _, err := hoppermigrate.Up(ctx, driver, nil); err != nil { + return err +} +client, err := hopper.NewClient(driver, cfg) +``` + +Its transactions are `*sql.Tx`; begin them on `driver.DB()`, whose +connections take the database's write lock when a transaction runs its first +statement. SQLite runs one writing transaction at a time, so keep them short: +one held open holds up every client sharing the file, lease renewals +included, and one held longer than the lease costs them their running jobs. +Write through the transaction (`InsertTx`, `CompleteTx`) rather than the pool +while one is open. In a worker, begin it with the job's context: a write on +the pool then fails at once with `hoppersqlite.ErrLockHeldByOwner`, rather +than wait for the worker's own transaction until the context ends. After an error that makes SQLite roll a +transaction back (an interrupted write, a full disk), its later statements +fail with `hoppersqlite.ErrTransactionAborted`; roll it back. + +Several processes can share the file: clients in other processes are woken +within 50ms of an insert. The file must be on a local disk; `Open` refuses +one on a network filesystem or a virtual machine's shared folder (Docker +Desktop's bind mounts among them), where SQLite's locks do not hold. +`hoppersqlite.Migrations` returns the SQLite scripts for teams that run +migrations with their own tool. Upgrade hopper and hoppersqlite together: +the driver uses hopper's internal packages. + ## Define a job and its worker Args are a struct with a `Kind` method. The kind names the job type and maps diff --git a/docs/releasing.md b/docs/releasing.md index 55079b1..15d598a 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,10 +1,17 @@ # Releasing hopper -hopper is three Go modules in one repository: the core module at the root, -`hopperotel` and `hopperui`. A release is one version across all three, cut -as three tags on the same commit of `canary`: `vX.Y.Z`, `hopperotel/vX.Y.Z` -and `hopperui/vX.Y.Z`. Until v1.0.0, minor versions may change the API; -patch versions do not. +hopper is four Go modules in one repository: the core module at the root, +`hopperotel`, `hopperui` and the SQLite driver, `driver/hoppersqlite`. A +release is one version across all four, cut as four tags on the same commit +of `canary`: `vX.Y.Z`, `hopperotel/vX.Y.Z`, `hopperui/vX.Y.Z` and +`driver/hoppersqlite/vX.Y.Z`. Until v1.0.0, minor versions may change the +API; patch versions do not. + +The sub-modules are released with the core module, not on their own, and +depend on that version alone: `hoppersqlite` imports the core module's +internal packages (`internal/jobtags`, `driver/internal/jobcodec`), which +have no compatibility promise from one version to the next, so an +application upgrades hopper and hoppersqlite together. Releases are automated with [release-please](https://github.com/googleapis/release-please) and [goreleaser](https://goreleaser.com) (`.github/workflows/release.yml`). @@ -19,7 +26,7 @@ and [goreleaser](https://goreleaser.com) (`.github/workflows/release.yml`). bumps the minor version before v1.0.0 (`bump-minor-pre-major`). release-please manages one package, the core module; the sub-modules are versioned with it rather than on their own, so a change anywhere in the - repository releases all three. + repository releases all four. 2. The release PR updates `CHANGELOG.md`, the manifest (`.release-please-manifest.json`), and the sub-modules' `go.mod` requirement on the core module, through the `// x-release-please-version` @@ -29,7 +36,8 @@ and [goreleaser](https://goreleaser.com) (`.github/workflows/release.yml`). 3. Merging the release PR (with `gh pr merge --squash --admin`, since the `canary` ruleset asks for two reviews) creates the `vX.Y.Z` tag and the GitHub release with the changelog entry. The workflow then tags - `hopperotel/vX.Y.Z` and `hopperui/vX.Y.Z` on the same commit, and + `hopperotel/vX.Y.Z`, `hopperui/vX.Y.Z` and `driver/hoppersqlite/vX.Y.Z` + on the same commit, and goreleaser builds `hopper`, `hopperbench` and `hopperui` for Linux and macOS (amd64, arm64) and attaches the archives and checksums to the release. @@ -68,8 +76,11 @@ the ruleset's commit-message pattern. ## Steering a release -- **Force a version:** add a footer `Release-As: X.Y.Z` to a squash commit's - body (the PR description), and the release PR proposes that version. +- **Force a version, or mark a breaking change:** the repository squashes a PR + with its title and a blank body, so nothing in the PR description reaches the + commit. Give the footer when merging, and the release PR reads it: + `gh pr merge N --squash --admin --body 'Release-As: X.Y.Z'`, or + `--body 'BREAKING CHANGE: …'` to say what a `!` in the title breaks. - **Hold a change out of the changelog:** only `feat`, `fix`, `perf` and `revert` appear; `docs`, `test`, `ci`, `build`, `chore` and `refactor` are hidden. @@ -82,7 +93,8 @@ After the tags exist, from an empty module: ```sh cd "$(mktemp -d)" && go mod init check && go get github.com/parallelworks/hopper@vX.Y.Z \ - github.com/parallelworks/hopper/hopperotel@vX.Y.Z github.com/parallelworks/hopper/hopperui@vX.Y.Z + github.com/parallelworks/hopper/hopperotel@vX.Y.Z github.com/parallelworks/hopper/hopperui@vX.Y.Z \ + github.com/parallelworks/hopper/driver/hoppersqlite@vX.Y.Z ``` ## History diff --git a/driver/driver.go b/driver/driver.go index c74fe76..64f9ed5 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -71,6 +71,36 @@ type Capabilities struct { // Listen reports whether Listener is implemented (LISTEN/NOTIFY on // Postgres). Listen bool + // SingleWriter reports that the engine runs one transaction that writes + // at a time, as SQLite does, rather than locking rows: a transaction + // that writes holds up every other write until it ends. A claim needs no + // SKIP LOCKED to be exclusive, and the transaction that finalizes a job + // settles its batch and promotes its dependents with no other to race + // it, so BatchSettle and WorkflowPromoteReady find nothing a race left. + // With JobFinalizeParams.SkipLocked or JobRecordProgressParams.NoWait, + // a statement waits only briefly for another transaction's writes to + // end, and then skips every job, or returns ErrJobLocked. + SingleWriter bool +} + +// ownerKey is the context key of a context's Owner. +type ownerKey struct{} + +// WithOwner returns ctx marked as running for owner: a job's attempt, whose +// worker's statements, in its own transactions and on the pool, all carry +// it. An engine that runs one writer at a time (Capabilities.SingleWriter) +// uses it to refuse at once a write that would wait for a transaction the +// same owner holds, which would wait until its context ended, holding up +// every other write meanwhile, and to say who holds the lock a write waits +// for. Owners compare by identity, so owner should be a pointer. +func WithOwner(ctx context.Context, owner fmt.Stringer) context.Context { + return context.WithValue(ctx, ownerKey{}, owner) +} + +// Owner returns the owner ctx runs for, or nil. +func Owner(ctx context.Context) fmt.Stringer { + o, _ := ctx.Value(ownerKey{}).(fmt.Stringer) + return o } // Notification channels. Payloads are documented on each. @@ -367,6 +397,19 @@ type MigrationExecutor interface { Close(ctx context.Context) error } +// MigrationScripts is implemented by the MigrationExecutor of an engine +// other than Postgres. hoppermigrate embeds Postgres's scripts; for an +// executor that implements MigrationScripts, it gives Apply the engine's +// own script for each schema version instead. Such an engine has a script, +// up and down, for every schema version hoppermigrate knows, so that a +// version names the same schema on every engine: the tables and columns a +// release's statements need. +type MigrationScripts interface { + // Script returns the engine's script for a schema version: the one that + // applies it (up) or the one that reverts it. + Script(version int, up bool) (string, error) +} + // JobState is a job's position in the state machine. type JobState string @@ -497,7 +540,8 @@ type JobRecordProgressParams struct { // Cursor is a JSON value, or nil for none. Cursor json.RawMessage // NoWait returns ErrJobLocked at once, instead of waiting, if another - // transaction holds the job's row. + // transaction holds the job's row; on a SingleWriter engine, if another + // transaction writes. NoWait bool } @@ -807,7 +851,8 @@ type JobFinalizeParams struct { NoBatches bool // SkipLocked skips the jobs whose rows another transaction holds, such // as a worker's CompleteTx that has not ended, instead of waiting for - // it. A skipped job is not among the applied. + // it. A skipped job is not among the applied. On a SingleWriter engine + // every job is skipped while another transaction writes. SkipLocked bool // KeepLogs is the most attempt logs a job keeps, the latest; a job given // another drops its oldest. Zero keeps them all. diff --git a/driver/hopperpgx/conformance_test.go b/driver/hopperpgx/conformance_test.go index 68d5fd1..2ef5561 100644 --- a/driver/hopperpgx/conformance_test.go +++ b/driver/hopperpgx/conformance_test.go @@ -3,7 +3,6 @@ package hopperpgx_test import ( "context" "testing" - "time" "github.com/jackc/pgx/v5" @@ -13,20 +12,12 @@ import ( "github.com/parallelworks/hopper/internal/testdb" ) -// pgxFixture runs the conformance suite against hopperpgx. -type pgxFixture struct{} +// pgxFixture runs the conformance suite against hopperpgx. Its hooks are +// statements on hopper's schema (drivertest.Postgres), which pgxSQL runs on +// the driver's pool. +type pgxFixture struct{ drivertest.Postgres[pgx.Tx] } -func pool(d driver.Driver[pgx.Tx]) interface { - Exec(ctx context.Context, sql string, args ...any) (interface{ RowsAffected() int64 }, error) -} { - return execAdapter{d.(*hopperpgx.Driver)} -} - -type execAdapter struct{ d *hopperpgx.Driver } - -func (a execAdapter) Exec(ctx context.Context, sql string, args ...any) (interface{ RowsAffected() int64 }, error) { - return a.d.Pool().Exec(ctx, sql, args...) -} +func newFixture() pgxFixture { return pgxFixture{drivertest.Postgres[pgx.Tx]{SQL: pgxSQL{}}} } func (pgxFixture) NewDriver(t *testing.T) driver.Driver[pgx.Tx] { return testdb.Driver(testdb.Pool(t)) @@ -41,50 +32,17 @@ func (pgxFixture) Begin(ctx context.Context, t *testing.T, d driver.Driver[pgx.T return tx, func() error { return tx.Commit(ctx) }, func() error { return tx.Rollback(ctx) } } -func (pgxFixture) ExpireLease(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], clientID int64) { - t.Helper() - if _, err := pool(d).Exec(ctx, "UPDATE hopper_clients SET expires_at = now() - interval '1s' WHERE id = $1", clientID); err != nil { - t.Fatal(err) - } -} +// pgxSQL runs the suite's statements on a driver's pool. +type pgxSQL struct{} -func (pgxFixture) ExpireLeader(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx]) { +func (pgxSQL) Exec(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], query string, args ...any) { t.Helper() - if _, err := pool(d).Exec(ctx, "UPDATE hopper_leader SET expires_at = now() - interval '1s'"); err != nil { - t.Fatal(err) - } -} - -func (pgxFixture) MakeClaimable(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], id driver.JobID) { - t.Helper() - if _, err := pool(d).Exec(ctx, "UPDATE hopper_jobs SET scheduled_at = now() WHERE id = $1", [16]byte(id)); err != nil { - t.Fatal(err) - } -} - -func (pgxFixture) AgeAttempt(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], id driver.JobID, age time.Duration) { - t.Helper() - if _, err := pool(d).Exec(ctx, "UPDATE hopper_jobs SET attempted_at = now() - make_interval(secs => $2) WHERE id = $1", [16]byte(id), age.Seconds()); err != nil { - t.Fatal(err) - } -} - -func (pgxFixture) CountLive(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx]) int { - return count(ctx, t, d, "hopper_jobs") -} - -func (pgxFixture) CountHistory(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx]) int { - return count(ctx, t, d, "hopper_job_history") -} - -func (pgxFixture) Exec(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], query string, args ...any) { - t.Helper() - if _, err := pool(d).Exec(ctx, query, args...); err != nil { + if _, err := d.(*hopperpgx.Driver).Pool().Exec(ctx, query, args...); err != nil { t.Error(err) } } -func (pgxFixture) QueryText(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], query string, args ...any) string { +func (pgxSQL) QueryText(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], query string, args ...any) string { t.Helper() var s string if err := d.(*hopperpgx.Driver).Pool().QueryRow(ctx, query, args...).Scan(&s); err != nil { @@ -93,16 +51,12 @@ func (pgxFixture) QueryText(ctx context.Context, t *testing.T, d driver.Driver[p return s } -func count(ctx context.Context, t *testing.T, d driver.Driver[pgx.Tx], table string) int { - t.Helper() - var n int - if err := d.(*hopperpgx.Driver).Pool().QueryRow(ctx, "SELECT count(*) FROM "+table).Scan(&n); err != nil { - t.Fatal(err) - } - return n +func TestConformance(t *testing.T) { + t.Parallel() + drivertest.Run(t, newFixture()) } -func TestConformance(t *testing.T) { +func TestConformancePostgres(t *testing.T) { t.Parallel() - drivertest.Run(t, pgxFixture{}) + drivertest.RunPostgres(t, newFixture()) } diff --git a/driver/hoppersql/conformance_test.go b/driver/hoppersql/conformance_test.go index 212e987..cd18ba7 100644 --- a/driver/hoppersql/conformance_test.go +++ b/driver/hoppersql/conformance_test.go @@ -22,8 +22,11 @@ import ( ) // sqlFixture runs the conformance suite against hoppersql, through pgx's -// database/sql adapter. -type sqlFixture struct{} +// database/sql adapter. Its hooks are statements on hopper's schema +// (drivertest.Postgres), which sqlSQL runs on the driver's database handle. +type sqlFixture struct{ drivertest.Postgres[*sql.Tx] } + +func newFixture() sqlFixture { return sqlFixture{drivertest.Postgres[*sql.Tx]{SQL: sqlSQL{}}} } // open returns a database/sql handle on the pool's schema. func open(t *testing.T, pool *pgxpool.Pool) *sql.DB { @@ -47,45 +50,17 @@ func (sqlFixture) Begin(ctx context.Context, t *testing.T, d driver.Driver[*sql. return tx, tx.Commit, tx.Rollback } -func exec(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) { - t.Helper() - if _, err := d.(*hoppersql.Driver).DB().ExecContext(ctx, query, args...); err != nil { - t.Fatal(err) - } -} - -func (sqlFixture) ExpireLease(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], clientID int64) { - exec(ctx, t, d, "UPDATE hopper_clients SET expires_at = now() - interval '1s' WHERE id = $1", clientID) -} - -func (sqlFixture) ExpireLeader(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) { - exec(ctx, t, d, "UPDATE hopper_leader SET expires_at = now() - interval '1s'") -} - -func (sqlFixture) MakeClaimable(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { - exec(ctx, t, d, "UPDATE hopper_jobs SET scheduled_at = now() WHERE id = $1", id) -} - -func (sqlFixture) AgeAttempt(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID, age time.Duration) { - exec(ctx, t, d, "UPDATE hopper_jobs SET attempted_at = now() - make_interval(secs => $2) WHERE id = $1", id, age.Seconds()) -} - -func (sqlFixture) CountLive(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) int { - return count(ctx, t, d, "hopper_jobs") -} - -func (sqlFixture) CountHistory(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) int { - return count(ctx, t, d, "hopper_job_history") -} +// sqlSQL runs the suite's statements on a driver's database handle. +type sqlSQL struct{} -func (sqlFixture) Exec(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) { +func (sqlSQL) Exec(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) { t.Helper() if _, err := d.(*hoppersql.Driver).DB().ExecContext(ctx, query, args...); err != nil { t.Error(err) } } -func (sqlFixture) QueryText(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) string { +func (sqlSQL) QueryText(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) string { t.Helper() var s string if err := d.(*hoppersql.Driver).DB().QueryRowContext(ctx, query, args...).Scan(&s); err != nil { @@ -94,18 +69,14 @@ func (sqlFixture) QueryText(ctx context.Context, t *testing.T, d driver.Driver[* return s } -func count(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], table string) int { - t.Helper() - var n int - if err := d.(*hoppersql.Driver).DB().QueryRowContext(ctx, "SELECT count(*) FROM "+table).Scan(&n); err != nil { - t.Fatal(err) - } - return n +func TestConformance(t *testing.T) { + t.Parallel() + drivertest.Run(t, newFixture()) } -func TestConformance(t *testing.T) { +func TestConformancePostgres(t *testing.T) { t.Parallel() - drivertest.Run(t, sqlFixture{}) + drivertest.RunPostgres(t, newFixture()) } // TestMigrate applies and reverts the schema through the database/sql diff --git a/driver/hoppersqlite/chaos_test.go b/driver/hoppersqlite/chaos_test.go new file mode 100644 index 0000000..8eea98e --- /dev/null +++ b/driver/hoppersqlite/chaos_test.go @@ -0,0 +1,192 @@ +package hoppersqlite_test + +import ( + "context" + "fmt" + "log/slog" + "math/rand/v2" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/parallelworks/hopper" + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/hoppersqlite" +) + +type chaosArgs struct { + N int `json:"n"` +} + +func (chaosArgs) Kind() string { return "chaos" } + +// chaosWorker is the "worker" role of TestHelperProcess: a client whose +// worker writes the job's effect and completes the job in one +// transaction, until the test kills it. +func chaosWorker(ctx context.Context, t *testing.T, d *hoppersqlite.Driver) { + workers := hopper.NewWorkers() + hopper.AddWorkFunc(workers, func(ctx context.Context, job *hopper.Job[chaosArgs]) error { + time.Sleep(time.Duration(rand.IntN(20)) * time.Millisecond) + tx, err := d.DB().BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck // committed below + if _, err := tx.ExecContext(ctx, "INSERT INTO effects (n, attempt) VALUES (?, ?)", job.Args.N, job.Attempt); err != nil { + return err + } + if err := hopper.CompleteTx(ctx, tx, job); err != nil { + return err + } + return tx.Commit() + }) + c, err := hopper.NewClient(d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: 8}}, + Workers: workers, + PollInterval: 20 * time.Millisecond, + LeaseTTL: time.Second, + Logger: slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})), + }) + if err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + fmt.Println("started") + time.Sleep(time.Minute) +} + +// hog is the "hog" role of TestHelperProcess: eight goroutines that write +// as fast as they can for three seconds. +func hog(ctx context.Context, t *testing.T, d *hoppersqlite.Driver) { + x := d.Executor() + deadline := time.Now().Add(3 * time.Second) + fmt.Println("hogging") + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + for time.Now().Before(deadline) { + if _, err := x.JobInsertMany(ctx, []driver.JobInsertParams{{Kind: "hog", Queue: "hog", Priority: 2, MaxAttempts: 1}}, driver.JobInsertOpts{}); err != nil { + t.Error(err) + return + } + } + }) + } + wg.Wait() +} + +// TestChaos runs jobs in worker processes that write each job's effect and +// complete it in one transaction, and kills a process at random every few +// hundred milliseconds, often inside a transaction, starting another in +// its place. Every job ends completed with exactly one effect, and the +// database passes its integrity check. +func TestChaos(t *testing.T) { + t.Parallel() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "hopper.db") + d := openAt(t, path) + if _, err := d.DB().ExecContext(ctx, "CREATE TABLE effects (n INTEGER, attempt INTEGER)"); err != nil { + t.Fatal(err) + } + const jobs, procs = 600, 4 + c, err := hopper.NewClient(d, &hopper.Config{Logger: slog.New(slog.DiscardHandler)}) + if err != nil { + t.Fatal(err) + } + params := make([]hopper.InsertParams, jobs) + for i := range params { + params[i] = hopper.InsertParams{Args: chaosArgs{N: i}} + } + if _, err := c.InsertMany(ctx, params); err != nil { + t.Fatal(err) + } + var running []*exec.Cmd + defer func() { + for _, cmd := range running { + _ = cmd.Process.Kill() + _ = cmd.Wait() + } + }() + for range procs { + cmd, _ := helper(t, path, "worker") + running = append(running, cmd) + } + kills := 0 + live := func() int { + var n int + if err := d.DB().QueryRowContext(ctx, "SELECT count(*) FROM hopper_jobs").Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + for start := time.Now(); time.Since(start) < 5*time.Second && live() > 0; { + time.Sleep(time.Duration(100+rand.IntN(200)) * time.Millisecond) + i := rand.IntN(len(running)) + _ = running[i].Process.Signal(syscall.SIGKILL) + _ = running[i].Wait() + kills++ + running = slices.Delete(running, i, i+1) + cmd, _ := helper(t, path, "worker") + running = append(running, cmd) + } + left := live() + for deadline := time.Now().Add(time.Minute); left > 0 && time.Now().Before(deadline); left = live() { + time.Sleep(100 * time.Millisecond) + } + var completed, effects, distinct int + if err := d.DB().QueryRowContext(ctx, `SELECT + (SELECT count(*) FROM hopper_job_history WHERE state = 'completed'), + (SELECT count(*) FROM effects), (SELECT count(DISTINCT n) FROM effects)`).Scan(&completed, &effects, &distinct); err != nil { + t.Fatal(err) + } + t.Logf("killed %d processes; %d jobs left, %d completed, %d effects", kills, left, completed, effects) + if left != 0 || completed != jobs || effects != jobs || distinct != jobs { + t.Errorf("%d jobs: %d left, %d completed, %d effects of %d jobs; want every job completed with one effect", jobs, left, completed, effects, distinct) + } + var check string + if err := d.DB().QueryRowContext(ctx, "PRAGMA integrity_check").Scan(&check); err != nil || check != "ok" { + t.Errorf("integrity_check = %q, %v", check, err) + } +} + +// TestFairness writes in this process while another writes as fast as it +// can, from eight goroutines, for three seconds: writes here are held up, +// since SQLite's busy handler polls for the lock rather than queueing for +// it, but not for as long as the other process writes. It runs alone, not +// in parallel, for its waits to be the lock's. +func TestFairness(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "hopper.db") + d := openAt(t, path) + x := d.Executor() + cmd, out := helper(t, path, "hog") + for !strings.Contains(out.String(), "hogging") { + time.Sleep(5 * time.Millisecond) + } + var waits []time.Duration + for end := time.Now().Add(2 * time.Second); time.Now().Before(end); time.Sleep(20 * time.Millisecond) { + start := time.Now() + if err := x.QueueEnsure(ctx, []string{"q" + strconv.Itoa(len(waits))}); err != nil { + t.Fatal(err) + } + waits = append(waits, time.Since(start)) + } + if err := cmd.Wait(); err != nil { + t.Fatalf("hogging process: %v\n%s", err, out) + } + slices.Sort(waits) + p50, p90, worst := waits[len(waits)/2], waits[len(waits)*9/10], waits[len(waits)-1] + t.Logf("%d writes: p50 %s, p90 %s, max %s", len(waits), p50, p90, worst) + if worst > slowdown*2*time.Second { + t.Errorf("a write waited %s while another process wrote", worst) + } +} diff --git a/driver/hoppersqlite/cluster.go b/driver/hoppersqlite/cluster.go new file mode 100644 index 0000000..db0401f --- /dev/null +++ b/driver/hoppersqlite/cluster.go @@ -0,0 +1,613 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "time" + + "github.com/parallelworks/hopper/driver" +) + +// ClientRegister implements driver.Executor. +func (e *executor) ClientRegister(ctx context.Context, params driver.ClientRegisterParams) (int64, error) { + info, err := jsonValue(params.Info, "{}", "client info") + if err != nil { + return 0, err + } + var id int64 + err = e.write(ctx, func(x *w) error { + return x.QueryRowContext(ctx, `INSERT INTO hopper_clients (hostname, started_at, expires_at, info) VALUES (?, ?, ?, ?) RETURNING id`, + params.Hostname, x.us(), x.at(params.TTL), info).Scan(&id) + }) + if err != nil { + return 0, fmt.Errorf("hopper: register client: %w", err) + } + return id, nil +} + +// strings reads the text values a query selects. +func stringsOf(ctx context.Context, q querier, query string, args ...any) ([]string, error) { + return queryAll(ctx, q, func(r scanner) (string, error) { + var s string + err := r.Scan(&s) + return s, err + }, query, args...) +} + +// ClientRenew implements driver.Executor: it extends an unexpired lease and +// reads the control state the client applies. +func (e *executor) ClientRenew(ctx context.Context, params driver.ClientRenewParams) (driver.ClientRenewResult, error) { + var res driver.ClientRenewResult + err := e.write(ctx, func(x *w) error { + renewed, err := x.ExecContext(ctx, `UPDATE hopper_clients SET expires_at = ? WHERE id = ? AND expires_at > ?`, + x.at(params.TTL), params.ClientID, x.us()) + if err != nil { + return err + } + n, err := renewed.RowsAffected() + if err != nil { + return err + } + res.Renewed = n > 0 + if res.CancelRequested, err = x.ids(ctx, `SELECT id FROM hopper_jobs + WHERE state = 'running' AND attempted_by = ? AND cancel_requested_at IS NOT NULL ORDER BY id`, params.ClientID); err != nil { + return err + } + if res.PausedQueues, err = stringsOf(ctx, x, `SELECT name FROM hopper_queues WHERE paused_at IS NOT NULL ORDER BY name`); err != nil { + return err + } + res.LimitedQueues, err = stringsOf(ctx, x, `SELECT name FROM hopper_queues + WHERE global_limit IS NOT NULL OR rate_per_sec IS NOT NULL OR partition_limit IS NOT NULL ORDER BY name`) + return err + }) + if err != nil { + return driver.ClientRenewResult{}, fmt.Errorf("hopper: renew client lease: %w", err) + } + return res, nil +} + +// ClientDelete implements driver.Executor. +func (e *executor) ClientDelete(ctx context.Context, clientID int64) error { + err := e.write(ctx, func(x *w) error { + _, err := x.ExecContext(ctx, `DELETE FROM hopper_clients WHERE id = ?`, clientID) + return err + }) + if err != nil { + return fmt.Errorf("hopper: delete client: %w", err) + } + return nil +} + +// ClientPruneExpired implements driver.Executor. +func (e *executor) ClientPruneExpired(ctx context.Context) (int64, error) { + var n int64 + err := e.write(ctx, func(x *w) error { + res, err := x.ExecContext(ctx, `DELETE FROM hopper_clients WHERE expires_at < ?`, x.us()) + if err != nil { + return err + } + n, err = res.RowsAffected() + return err + }) + if err != nil { + return 0, fmt.Errorf("hopper: prune clients: %w", err) + } + return n, nil +} + +// ClientList implements driver.Executor. +func (e *executor) ClientList(ctx context.Context) ([]*driver.ClientRow, error) { + clients, err := queryAll(ctx, e.read(), func(r scanner) (*driver.ClientRow, error) { + var ( + c driver.ClientRow + started, expires int64 + info string + ) + if err := r.Scan(&c.ID, &c.Hostname, &started, &expires, &info); err != nil { + return nil, err + } + c.StartedAt, c.ExpiresAt, c.Info = time.UnixMicro(started).UTC(), time.UnixMicro(expires).UTC(), json.RawMessage(info) + return &c, nil + }, `SELECT id, hostname, started_at, expires_at, info FROM hopper_clients ORDER BY id`) + if err != nil { + return nil, fmt.Errorf("hopper: list clients: %w", err) + } + return clients, nil +} + +// LeaderAttempt implements driver.Executor: it takes the lease if it is +// free or expired, or renews it for the holder. +func (e *executor) LeaderAttempt(ctx context.Context, params driver.LeaderParams) (bool, error) { + var leader bool + err := e.write(ctx, func(x *w) error { + var id int64 + err := x.QueryRowContext(ctx, ` + INSERT INTO hopper_leader (name, client_id, elected_at, expires_at) VALUES ('default', ?1, ?2, ?3) + ON CONFLICT (name) DO UPDATE SET + client_id = excluded.client_id, + elected_at = CASE WHEN hopper_leader.client_id = excluded.client_id THEN hopper_leader.elected_at ELSE excluded.elected_at END, + expires_at = excluded.expires_at + WHERE hopper_leader.expires_at < ?2 OR hopper_leader.client_id = excluded.client_id + RETURNING client_id`, params.ClientID, x.us(), x.at(params.TTL)).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + leader = err == nil && id == params.ClientID + return err + }) + if err != nil { + return false, fmt.Errorf("hopper: leader attempt: %w", err) + } + return leader, nil +} + +// LeaderResign implements driver.Executor: it gives the lease up if this +// client holds it and, only then, signals other clients to elect a +// replacement. +func (e *executor) LeaderResign(ctx context.Context, clientID int64) error { + err := e.write(ctx, func(x *w) error { + res, err := x.ExecContext(ctx, `DELETE FROM hopper_leader WHERE name = 'default' AND client_id = ?`, clientID) + if err != nil { + return err + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return err + } + return x.notify(ctx, driver.ChannelLeader, "resigned") + }) + if err != nil { + return fmt.Errorf("hopper: leader resign: %w", err) + } + return nil +} + +// QueueEnsure implements driver.Executor. +func (e *executor) QueueEnsure(ctx context.Context, names []string) error { + if len(names) == 0 { + return nil + } + err := e.write(ctx, func(x *w) error { + for _, name := range names { + if _, err := x.ExecContext(ctx, `INSERT INTO hopper_queues (name, updated_at) VALUES (?, ?) ON CONFLICT (name) DO NOTHING`, name, x.us()); err != nil { + return err + } + } + return nil + }) + if err != nil { + return fmt.Errorf("hopper: ensure queues: %w", err) + } + return nil +} + +// QueuePause implements driver.Executor. +func (e *executor) QueuePause(ctx context.Context, name string) error { + return e.setPaused(ctx, name, true) +} + +// QueueResume implements driver.Executor. +func (e *executor) QueueResume(ctx context.Context, name string) error { + return e.setPaused(ctx, name, false) +} + +func (e *executor) setPaused(ctx context.Context, name string, paused bool) error { + action, stmt := "resume", `INSERT INTO hopper_queues (name, updated_at) VALUES (?1, ?2) + ON CONFLICT (name) DO UPDATE SET paused_at = NULL, updated_at = ?2` + if paused { + action, stmt = "pause", `INSERT INTO hopper_queues (name, paused_at, updated_at) VALUES (?1, ?2, ?2) + ON CONFLICT (name) DO UPDATE SET paused_at = coalesce(hopper_queues.paused_at, ?2), updated_at = ?2` + } + err := e.write(ctx, func(x *w) error { + if _, err := x.ExecContext(ctx, stmt, name, x.us()); err != nil { + return err + } + return x.notify(ctx, driver.ChannelControl, action+":"+name) + }) + if err != nil { + return fmt.Errorf("hopper: %s queue: %w", action, err) + } + return nil +} + +// QueueList implements driver.Executor. +func (e *executor) QueueList(ctx context.Context) ([]*driver.QueueRow, error) { + queues, err := queryAll(ctx, e.read(), func(r scanner) (*driver.QueueRow, error) { + var ( + q driver.QueueRow + paused micros + updated int64 + global, burst, partition, aging sql.NullInt64 + rate sql.NullFloat64 + ) + if err := r.Scan(&q.Name, &paused, &updated, &global, &rate, &burst, &partition, &aging); err != nil { + return nil, err + } + q.PausedAt, q.UpdatedAt = paused.time(), time.UnixMicro(updated).UTC() + q.Limits = driver.QueueLimits{ + Name: q.Name, GlobalLimit: int(global.Int64), RatePerSec: rate.Float64, RateBurst: int(burst.Int64), + PartitionLimit: int(partition.Int64), Aging: time.Duration(aging.Int64) * time.Second, + } + return &q, nil + }, `SELECT name, paused_at, updated_at, global_limit, rate_per_sec, rate_burst, partition_limit, aging_seconds + FROM hopper_queues ORDER BY name`) + if err != nil { + return nil, fmt.Errorf("hopper: list queues: %w", err) + } + return queues, nil +} + +func nullablePositive(v int) any { + if v <= 0 { + return nil + } + return v +} + +// QueueSetLimits implements driver.Executor. A new rate starts with a full +// bucket. +func (e *executor) QueueSetLimits(ctx context.Context, l driver.QueueLimits) error { + var rate any + if l.RatePerSec > 0 { + rate = l.RatePerSec + } + err := e.write(ctx, func(x *w) error { + if _, err := x.ExecContext(ctx, ` + INSERT INTO hopper_queues (name, global_limit, rate_per_sec, rate_burst, partition_limit, aging_seconds, tokens, refilled_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, NULL, NULL, ?7) + ON CONFLICT (name) DO UPDATE SET global_limit = excluded.global_limit, rate_per_sec = excluded.rate_per_sec, + rate_burst = excluded.rate_burst, partition_limit = excluded.partition_limit, aging_seconds = excluded.aging_seconds, + tokens = CASE WHEN hopper_queues.rate_per_sec IS NOT excluded.rate_per_sec THEN NULL ELSE hopper_queues.tokens END, + updated_at = ?7`, + l.Name, nullablePositive(l.GlobalLimit), rate, nullablePositive(l.RateBurst), nullablePositive(l.PartitionLimit), + nullablePositive(int(l.Aging/time.Second)), x.us()); err != nil { + return err + } + return x.notify(ctx, driver.ChannelControl, "limit:"+l.Name) + }) + if err != nil { + return fmt.Errorf("hopper: set queue limits: %w", err) + } + return nil +} + +// PeriodicInsert implements driver.Executor: the job is inserted only by +// the write that advances its slot, so a slot produces at most one job. +func (e *executor) PeriodicInsert(ctx context.Context, params driver.PeriodicInsertParams) (*driver.JobRow, bool, error) { + j, err := encodeInsert(params.Job) + if err != nil { + return nil, false, err + } + var ( + job *driver.JobRow + inserted bool + ) + err = e.write(ctx, func(x *w) error { + var name string + err := x.QueryRowContext(ctx, ` + INSERT INTO hopper_periodic (name, last_slot) VALUES (?, ?) + ON CONFLICT (name) DO UPDATE SET last_slot = excluded.last_slot WHERE hopper_periodic.last_slot < excluded.last_slot + RETURNING name`, params.Name, params.Slot.UnixMicro()).Scan(&name) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return err + } + results, err := x.insertJobs(ctx, []jobInsert{j}, driver.ConflictSkip) + if err != nil { + return err + } + job, inserted = results[0].Job, !results[0].Duplicate + return x.notify(ctx, driver.ChannelInsert, j.queue) + }) + if err != nil { + return nil, false, fmt.Errorf("hopper: insert periodic job %s: %w", params.Name, err) + } + return job, inserted, nil +} + +// PeriodicLastSlots implements driver.Executor. +func (e *executor) PeriodicLastSlots(ctx context.Context) (map[string]time.Time, error) { + type slot struct { + name string + at int64 + } + slots, err := queryAll(ctx, e.read(), func(r scanner) (slot, error) { + var s slot + err := r.Scan(&s.name, &s.at) + return s, err + }, `SELECT name, last_slot FROM hopper_periodic`) + if err != nil { + return nil, fmt.Errorf("hopper: periodic slots: %w", err) + } + out := make(map[string]time.Time, len(slots)) + for _, s := range slots { + out[s.name] = time.UnixMicro(s.at).UTC() + } + return out, nil +} + +// Stats implements driver.Executor, from one snapshot of the database. +func (e *executor) Stats(ctx context.Context) (*driver.Stats, error) { + stats := &driver.Stats{Queues: map[string]*driver.QueueStats{}, RunningByClient: map[int64]int{}} + queue := func(name string) *driver.QueueStats { + q := stats.Queues[name] + if q == nil { + q = &driver.QueueStats{} + stats.Queues[name] = q + } + return q + } + now := e.clock() + err := e.readTx(ctx, func(q querier) error { + rows, err := q.QueryContext(ctx, ` + SELECT queue, state, count(*), + min(CASE WHEN state IN ('available', 'scheduled', 'retryable') AND scheduled_at <= ?1 THEN scheduled_at END) + FROM hopper_jobs GROUP BY queue, state`, now.UnixMicro()) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var ( + name, state string + n int + oldest micros + ) + if err := rows.Scan(&name, &state, &n, &oldest); err != nil { + return err + } + qs := queue(name) + switch driver.JobState(state) { + case driver.JobStateAvailable: + qs.Available = n + case driver.JobStateScheduled: + qs.Scheduled = n + case driver.JobStateRetryable: + qs.Retryable = n + case driver.JobStateRunning: + qs.Running = n + case driver.JobStatePending, driver.JobStateCompleted, driver.JobStateCanceled, driver.JobStateDiscarded: + } + if oldest.Valid { + qs.OldestAvailable = max(qs.OldestAvailable, now.Sub(oldest.time())) + } + } + if err := rows.Err(); err != nil { + return err + } + rows, err = q.QueryContext(ctx, ` + SELECT q.name, q.paused_at IS NOT NULL, + (SELECT count(*) FROM hopper_job_history h WHERE h.queue = q.name AND h.state = 'completed' AND h.finalized_at > ?1) + FROM hopper_queues q`, now.Add(-time.Minute).UnixMicro()) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var ( + name string + paused bool + completed int + ) + if err := rows.Scan(&name, &paused, &completed); err != nil { + return err + } + qs := queue(name) + qs.Paused, qs.CompletedLastMinute = paused, completed + } + if err := rows.Err(); err != nil { + return err + } + if err := q.QueryRowContext(ctx, `SELECT count(*) FROM hopper_clients WHERE expires_at > ?`, now.UnixMicro()).Scan(&stats.LiveClients); err != nil { + return err + } + if err := q.QueryRowContext(ctx, `SELECT coalesce((SELECT client_id FROM hopper_leader WHERE name = 'default' AND expires_at > ?), 0)`, + now.UnixMicro()).Scan(&stats.Leader); err != nil { + return err + } + rows, err = q.QueryContext(ctx, `SELECT attempted_by, count(*) FROM hopper_jobs + WHERE state = 'running' AND attempted_by IS NOT NULL GROUP BY attempted_by`) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var id int64 + var n int + if err := rows.Scan(&id, &n); err != nil { + return err + } + stats.RunningByClient[id] = n + } + return rows.Err() + }) + if err != nil { + return nil, fmt.Errorf("hopper: stats: %w", err) + } + return stats, nil +} + +// retentionBatch is how many rows a retention pass deletes per +// transaction, each of which holds every other write up for as long as it +// runs. +const retentionBatch = 1000 + +// HistoryMaintain implements driver.Executor: retention deletes finished +// jobs and stream events past their retention, a batch per transaction. +// Each batch waits at most params.LockTimeout (lockWaitBrief if zero) for +// the write lock; a group whose batch gives up is reported deferred, and a +// later pass deletes the rest. History has no storage units to create or +// drop. +func (e *executor) HistoryMaintain(ctx context.Context, params driver.HistoryMaintainParams) (driver.HistoryMaintainResult, error) { + var res driver.HistoryMaintainResult + if e.tx != nil { + return res, errors.New("hopper: history maintenance must run on the pool, not in a transaction") + } + wait := params.LockTimeout + if wait <= 0 { + wait = lockWaitBrief + } + groups := []struct { + name, table, where string + keep time.Duration + }{ + {"hopper_job_history_completed", "hopper_job_history", "state = 'completed' AND finalized_at < ?", params.CompletedRetention}, + {"hopper_job_history_failed", "hopper_job_history", "state IN ('canceled', 'discarded') AND finalized_at < ?", params.FailedRetention}, + {"hopper_stream_events", "hopper_stream_events", "created_at < ?", params.StreamRetention}, + } + var errs []error + for _, g := range groups { + if g.keep <= 0 { + continue + } + for { + var n int64 + err := e.writeLocking(ctx, wait, func(x *w) error { + r, err := x.ExecContext(ctx, `DELETE FROM `+g.table+` WHERE rowid IN ( + SELECT rowid FROM `+g.table+` WHERE `+g.where+` LIMIT ?)`, x.at(-g.keep), retentionBatch) + if err != nil { + return err + } + n, err = r.RowsAffected() + return err + }) + if errors.Is(err, errLockBusy) { + res.Deferred = append(res.Deferred, g.name) + break + } + if err != nil { + errs = append(errs, fmt.Errorf("hopper: prune %s: %w", g.name, err)) + break + } + res.Pruned += n + if n < retentionBatch { + break + } + } + } + return res, errors.Join(errs...) +} + +// HistoryIndex implements driver.Executor: history's indexes are built +// with it, so a pass finds nothing to do. +func (e *executor) HistoryIndex(context.Context, driver.HistoryIndexParams) (driver.HistoryIndexResult, error) { + return driver.HistoryIndexResult{}, nil +} + +// maintainedTables are the tables JobsMaintain keeps the planner's +// statistics of: the live table, and the live tags and dependency tables, +// which churn with it. +var maintainedTables = []string{"hopper_jobs", "hopper_job_tags", "hopper_job_deps"} + +// JobsMaintain implements driver.Executor. SQLite reuses the pages of +// deleted rows and needs no vacuum, but its planner, like Postgres's, +// chooses its plans by the row counts its statistics hold: a table +// analyzed while it was empty is planned as if it still were. A table is +// analyzed when the count the statistics hold is off by an order of +// magnitude, or it has none and has grown to a thousand rows. An analyze +// reads a sample of each index (analysis_limit), and waits at most +// lockWaitBrief for the write lock; one that cannot get it waits for the +// next pass. +func (e *executor) JobsMaintain(ctx context.Context) (driver.JobsMaintainResult, error) { + var res driver.JobsMaintainResult + if e.tx != nil { + return res, errors.New("hopper: live table maintenance must run on the pool, not in a transaction") + } + var errs []error + for _, table := range maintainedTables { + believed, err := e.believedRows(ctx, table) + if err != nil { + errs = append(errs, err) + continue + } + floor := max(believed, 100) + // Count up to the point where the counts are an order of magnitude + // apart, which bounds the count's cost by the table's believed size. + var live int64 + if err := e.d.sqlDB.QueryRowContext(ctx, `SELECT count(*) FROM (SELECT 1 FROM `+table+` LIMIT ?)`, 10*floor).Scan(&live); err != nil { + errs = append(errs, fmt.Errorf("hopper: count %s: %w", table, err)) + continue + } + stale := live >= 10*floor || (believed >= 0 && believed >= 10*max(live, 100)) + if believed < 0 { + stale = live >= 1000 + } + if !stale { + continue + } + err = e.writeBrief(ctx, func(x *w) error { + _, err := x.ExecContext(ctx, "PRAGMA analysis_limit = 1000; ANALYZE "+table) + return err + }) + switch { + case errors.Is(err, errLockBusy): + case err != nil: + errs = append(errs, fmt.Errorf("hopper: analyze %s: %w", table, err)) + default: + res.Analyzed = true + } + } + if err := e.truncateLog(ctx); err != nil { + errs = append(errs, err) + } + return res, errors.Join(errs...) +} + +// walTruncateSize is the size of write-ahead log past which JobsMaintain +// checkpoints it and truncates it: twice what SQLite's automatic +// checkpoints, every thousand pages, let it reach. +const walTruncateSize = 8 << 20 + +// truncateLog checkpoints the write-ahead log and truncates it to nothing +// once it is past walTruncateSize. The log grows past that while a reader +// keeps a snapshot from before the checkpoints, which then cannot start it +// over, and SQLite cuts it back only to journalSizeLimit once one can. The +// checkpoint waits for writers and readers at most busyQuantum each, and +// gives up, to be tried at the next pass, if they are still there. +func (e *executor) truncateLog(ctx context.Context) error { + st, err := os.Stat(e.d.db.path + "-wal") + if err != nil || st.Size() <= walTruncateSize { + return nil //nolint:nilerr // no log, or none to speak of + } + var busy, logPages, checkpointed int64 + if err := e.d.sqlDB.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logPages, &checkpointed); err != nil { + return fmt.Errorf("hopper: checkpoint the write-ahead log: %w", err) + } + return nil +} + +// believedRows returns the row count the planner's statistics hold for a +// table, or -1 if it has none. +func (e *executor) believedRows(ctx context.Context, table string) (int64, error) { + var hasStats bool + if err := e.d.sqlDB.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'sqlite_stat1')`).Scan(&hasStats); err != nil { + return 0, err + } + if !hasStats { + return -1, nil + } + // Each index's row starts with its count of rows; a partial index + // counts only its own, and the table's other indexes all of them. + var n sql.NullInt64 + if err := e.d.sqlDB.QueryRowContext(ctx, `SELECT max(CAST(stat AS INTEGER)) FROM sqlite_stat1 WHERE tbl = ?`, table).Scan(&n); err != nil { + return 0, err + } + if !n.Valid { + return -1, nil + } + return n.Int64, nil +} + +// AnalyzeForTest analyzes the live table as it is, for the conformance +// suite to give the planner statistics of a known state. +func (e *executor) AnalyzeForTest(ctx context.Context) error { + return e.write(ctx, func(x *w) error { + _, err := x.ExecContext(ctx, "ANALYZE hopper_jobs") + return err + }) +} diff --git a/driver/hoppersqlite/cmd/hopperbench-sqlite/main.go b/driver/hoppersqlite/cmd/hopperbench-sqlite/main.go new file mode 100644 index 0000000..87a19b0 --- /dev/null +++ b/driver/hoppersqlite/cmd/hopperbench-sqlite/main.go @@ -0,0 +1,432 @@ +// Command hopperbench-sqlite measures hopper's hot paths on SQLite, as +// hopperbench does on Postgres, and prints its results in hopperbench's +// format, one JSON object per scenario, which hopperbench -compare reads. +// +// hopperbench-sqlite [-dir /tmp] [-scenarios throughput,workflow,insert,latency] [-jobs 50000] [-clients 4] +// +// Each scenario runs on a database file of its own in a temporary directory +// under -dir, removed afterwards. -dir must be on a local disk. +// +// Scenarios: +// +// throughput no-op jobs preloaded, then drained by -clients clients +// retry as throughput, with every job failing its first attempt +// workflow as throughput, in workflows of ten steps (one, then eight after it, then one after those), +// each step recording a small output; -jobs is rounded down to a multiple of ten +// insert InsertMany in batches of 100 +// latency commit-to-Work on an idle queue, local wake-up +package main + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "flag" + "fmt" + "log/slog" + "os" + "os/signal" + "path/filepath" + "runtime/pprof" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/parallelworks/hopper" + "github.com/parallelworks/hopper/driver/hoppersqlite" + "github.com/parallelworks/hopper/hoppermigrate" +) + +type benchArgs struct { + N int `json:"n"` +} + +func (benchArgs) Kind() string { return "bench" } + +// result is hopperbench's. +type result struct { + Scenario string `json:"scenario"` + Jobs int `json:"jobs"` + Clients int `json:"clients,omitempty"` + Workers int `json:"workers_per_client,omitempty"` + Seconds float64 `json:"seconds"` + JobsPerSec float64 `json:"jobs_per_sec,omitempty"` + P50Millis float64 `json:"p50_ms,omitempty"` + P99Millis float64 `json:"p99_ms,omitempty"` + MaxMillis float64 `json:"max_ms,omitempty"` +} + +type options struct { + jobs, clients, workers, samples int + synchronous string +} + +func main() { os.Exit(benchMain()) } + +// benchMain runs the command and returns its exit status, after its +// deferred calls, which os.Exit would skip. +func benchMain() int { + var ( + dir = flag.String("dir", os.TempDir(), "directory for the scenarios' databases, on a local disk") + scenarios = flag.String("scenarios", "throughput,workflow,insert,latency", "comma-separated scenarios to run") + jobs = flag.Int("jobs", 50000, "jobs per throughput scenario") + clients = flag.Int("clients", 4, "clients working the throughput scenarios") + workers = flag.Int("workers", 100, "MaxWorkers per client") + samples = flag.Int("samples", 200, "latency samples") + synchronous = flag.String("synchronous", "FULL", "SQLite's synchronous setting: FULL or NORMAL") + cpuProfile = flag.String("cpuprofile", "", "write a CPU profile of the run to this file") + ) + flag.Parse() + if *cpuProfile != "" { + f, err := os.Create(*cpuProfile) + if err != nil { + fmt.Fprintln(os.Stderr, "hopperbench-sqlite:", err) + return 1 + } + if err := pprof.StartCPUProfile(f); err != nil { + fmt.Fprintln(os.Stderr, "hopperbench-sqlite:", err) + return 1 + } + defer pprof.StopCPUProfile() + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + err := run(ctx, *dir, strings.Split(*scenarios, ","), options{jobs: *jobs, clients: *clients, workers: *workers, samples: *samples, synchronous: *synchronous}) + stop() + if err != nil { + fmt.Fprintln(os.Stderr, "hopperbench-sqlite:", err) + return 1 + } + return 0 +} + +func run(ctx context.Context, dir string, scenarios []string, opts options) error { + for _, s := range scenarios { + b, cleanup, err := newBench(ctx, dir, opts) + if err != nil { + return err + } + var r result + switch s = strings.TrimSpace(s); s { + case "throughput": + r, err = b.drain(ctx, "throughput", b.params(opts.jobs), noopWorker) + case "retry": + r, err = b.drain(ctx, "retry", b.params(opts.jobs), failOnceWorker) + case "workflow": + r, err = b.workflows(ctx, opts.jobs) + case "insert": + r, err = b.insert(ctx, opts.jobs, 100) + case "latency": + r, err = b.latency(ctx, opts.samples) + default: + err = fmt.Errorf("unknown scenario %q", s) + } + cleanup() + if err != nil { + return fmt.Errorf("%s: %w", s, err) + } + if err := json.NewEncoder(os.Stdout).Encode(r); err != nil { + return err + } + } + return nil +} + +type bench struct { + d *hoppersqlite.Driver + opts options + logger *slog.Logger +} + +// newBench opens a migrated database in a new directory under dir. +func newBench(ctx context.Context, dir string, opts options) (*bench, func(), error) { + tmp, err := os.MkdirTemp(dir, "hopperbench-sqlite") + if err != nil { + return nil, nil, err + } + d, err := hoppersqlite.Open(filepath.Join(tmp, "hopper.db"), &hoppersqlite.Config{Synchronous: opts.synchronous}) + if err != nil { + _ = os.RemoveAll(tmp) + return nil, nil, err + } + cleanup := func() { + _ = d.Close() + _ = os.RemoveAll(tmp) + } + if _, err := hoppermigrate.Up(ctx, d, &hoppermigrate.Options{Logger: slog.New(slog.DiscardHandler)}); err != nil { + cleanup() + return nil, nil, err + } + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})) + return &bench{d: d, opts: opts, logger: logger}, cleanup, nil +} + +func (b *bench) params(n int) []hopper.InsertParams { + params := make([]hopper.InsertParams, n) + for i := range params { + params[i] = hopper.InsertParams{Args: benchArgs{N: i}} + } + return params +} + +func noopWorker(context.Context, *hopper.Job[benchArgs]) error { return nil } + +var errFirstAttempt = errors.New("first attempt") + +// failOnceWorker fails every first attempt, so each job is finalized twice. +func failOnceWorker(_ context.Context, job *hopper.Job[benchArgs]) error { + if job.Attempt == 1 { + return errFirstAttempt + } + return nil +} + +// immediateRetry is a worker whose failed attempts retry at once. +type immediateRetry struct { + hopper.WorkerDefaults[benchArgs] + fn func(context.Context, *hopper.Job[benchArgs]) error +} + +func (w immediateRetry) Work(ctx context.Context, job *hopper.Job[benchArgs]) error { + return w.fn(ctx, job) +} + +func (immediateRetry) NextRetry(*hopper.Job[benchArgs]) time.Time { return time.Now() } + +const workflowSteps = 10 + +func (b *bench) drain(ctx context.Context, name string, params []hopper.InsertParams, work func(context.Context, *hopper.Job[benchArgs]) error) (result, error) { + return b.drainLoaded(ctx, name, len(params), func(inserter *hopper.Client[*sql.Tx]) error { + for i := 0; i < len(params); i += 10000 { + if _, err := inserter.InsertMany(ctx, params[i:min(i+10000, len(params))]); err != nil { + return err + } + } + return nil + }, work) +} + +func (b *bench) workflows(ctx context.Context, jobs int) (result, error) { + n := jobs / workflowSteps + if n < 1 { + return result{}, fmt.Errorf("-jobs must be at least %d", workflowSteps) + } + load := func(inserter *hopper.Client[*sql.Tx]) error { + return fanOut(ctx, n, 100, func(ctx context.Context, lo, hi int) error { + tx, err := b.d.DB().BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck // committed below + for i := lo; i < hi; i++ { + if _, err := inserter.InsertWorkflowTx(ctx, tx, benchWorkflow(i)); err != nil { + return err + } + } + return tx.Commit() + }) + } + r, err := b.drainLoaded(ctx, "workflow", n*workflowSteps, load, outputWorker) + if err != nil { + return result{}, err + } + var outputs, open int + if err := b.d.DB().QueryRowContext(ctx, `SELECT + (SELECT count(*) FROM hopper_job_history WHERE batch_id IS NOT NULL AND kind = 'bench' AND output IS NOT NULL), + (SELECT count(*) FROM hopper_batches WHERE completed_at IS NULL)`).Scan(&outputs, &open); err != nil { + return result{}, err + } + if outputs != r.Jobs { + return result{}, fmt.Errorf("%d of %d jobs stored an output", outputs, r.Jobs) + } + if open != 0 { + return result{}, fmt.Errorf("%d of %d workflows not completed", open, n) + } + return r, nil +} + +func benchWorkflow(i int) *hopper.Workflow { + wf := hopper.NewWorkflow("bench-"+strconv.Itoa(i), nil) + n := i * workflowSteps + root := wf.Add(benchArgs{N: n}, nil) + middle := make([]hopper.Step, workflowSteps-2) + for j := range middle { + middle[j] = wf.Add(benchArgs{N: n + 1 + j}, hopper.After(root)) + } + wf.Add(benchArgs{N: n + workflowSteps - 1}, hopper.After(middle...)) + return wf +} + +type benchOutput struct { + N int `json:"n"` + Artifact string `json:"artifact"` +} + +func outputWorker(ctx context.Context, job *hopper.Job[benchArgs]) error { + return hopper.SetOutput(ctx, benchOutput{N: job.Args.N, Artifact: "bench/" + strconv.Itoa(job.Args.N)}) +} + +// fanOut runs fn over [0, n) in chunks from eight inserters, as an +// application with several goroutines inserting would; the first error +// cancels the rest. +func fanOut(ctx context.Context, n, chunk int, fn func(ctx context.Context, lo, hi int) error) error { + const inserters = 8 + ctx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + var wg sync.WaitGroup + for w := range inserters { + wg.Go(func() { + for lo := w * chunk; lo < n && ctx.Err() == nil; lo += inserters * chunk { + if err := fn(ctx, lo, min(lo+chunk, n)); err != nil { + cancel(err) + return + } + } + }) + } + wg.Wait() + return context.Cause(ctx) +} + +func (b *bench) drainLoaded(ctx context.Context, name string, jobs int, load func(*hopper.Client[*sql.Tx]) error, work func(context.Context, *hopper.Job[benchArgs]) error) (result, error) { + inserter, err := hopper.NewClient(b.d, &hopper.Config{Logger: b.logger}) + if err != nil { + return result{}, err + } + defer inserter.Stop(ctx) //nolint:errcheck // flushes pending notifications + if err := load(inserter); err != nil { + return result{}, err + } + ws := hopper.NewWorkers() + hopper.AddWorker(ws, immediateRetry{fn: work}) + cs := make([]*hopper.Client[*sql.Tx], b.opts.clients) + for i := range cs { + c, err := hopper.NewClient(b.d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: b.opts.workers}}, + Workers: ws, + Logger: b.logger, + }) + if err != nil { + return result{}, err + } + cs[i] = c + } + defer func() { + sctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + for _, c := range cs { + _ = c.Stop(sctx) + } + }() + start := time.Now() + for _, c := range cs { + if err := c.Start(ctx); err != nil { + return result{}, err + } + } + if err := b.waitDrained(ctx); err != nil { + return result{}, err + } + elapsed := time.Since(start) + for _, c := range cs { + if err := c.Stop(ctx); err != nil { + return result{}, err + } + } + var done int + if err := b.d.DB().QueryRowContext(ctx, "SELECT count(*) FROM hopper_job_history WHERE state = 'completed' AND kind = 'bench'").Scan(&done); err != nil { + return result{}, err + } + if done != jobs { + return result{}, fmt.Errorf("%d of %d jobs completed", done, jobs) + } + return result{ + Scenario: name, Jobs: jobs, Clients: b.opts.clients, Workers: b.opts.workers, + Seconds: elapsed.Seconds(), JobsPerSec: float64(jobs) / elapsed.Seconds(), + }, nil +} + +func (b *bench) waitDrained(ctx context.Context) error { + last, changed := -1, time.Now() + for { + var live int + if err := b.d.DB().QueryRowContext(ctx, "SELECT count(*) FROM hopper_jobs").Scan(&live); err != nil { + return err + } + if live == 0 { + return nil + } + if live != last { + last, changed = live, time.Now() + } else if time.Since(changed) > 2*time.Minute { + return fmt.Errorf("no progress for two minutes with %d jobs left", live) + } + time.Sleep(20 * time.Millisecond) + } +} + +// insert measures InsertMany in batches of the given size. +func (b *bench) insert(ctx context.Context, jobs, batch int) (result, error) { + c, err := hopper.NewClient(b.d, &hopper.Config{Logger: b.logger}) + if err != nil { + return result{}, err + } + defer c.Stop(ctx) //nolint:errcheck // flushes pending notifications + params := b.params(jobs) + start := time.Now() + err = fanOut(ctx, len(params), batch, func(ctx context.Context, lo, hi int) error { + _, err := c.InsertMany(ctx, params[lo:hi]) + return err + }) + elapsed := time.Since(start) + if err != nil { + return result{}, err + } + return result{Scenario: "insert", Jobs: jobs, Seconds: elapsed.Seconds(), JobsPerSec: float64(jobs) / elapsed.Seconds()}, nil +} + +// latency measures commit-to-Work on an idle queue, with the insert coming +// from the working client itself (local wake-up). +func (b *bench) latency(ctx context.Context, samples int) (result, error) { + started := make(chan time.Time, 1) + ws := hopper.NewWorkers() + hopper.AddWorkFunc(ws, func(context.Context, *hopper.Job[benchArgs]) error { + started <- time.Now() + return nil + }) + c, err := hopper.NewClient(b.d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: 10}}, + Workers: ws, + Logger: b.logger, + }) + if err != nil { + return result{}, err + } + if err := c.Start(ctx); err != nil { + return result{}, err + } + defer c.Stop(ctx) //nolint:errcheck // best effort + lat := make([]time.Duration, 0, samples) + begin := time.Now() + for i := range samples { + t0 := time.Now() + if _, err := c.Insert(ctx, benchArgs{N: i}, nil); err != nil { + return result{}, err + } + select { + case t1 := <-started: + lat = append(lat, t1.Sub(t0)) + case <-time.After(10 * time.Second): + return result{}, errors.New("job not picked up within 10s") + } + time.Sleep(5 * time.Millisecond) + } + slices.Sort(lat) + ms := func(d time.Duration) float64 { return float64(d.Microseconds()) / 1000 } + return result{ + Scenario: "latency", Jobs: samples, Seconds: time.Since(begin).Seconds(), + P50Millis: ms(lat[len(lat)/2]), P99Millis: ms(lat[len(lat)*99/100]), MaxMillis: ms(lat[len(lat)-1]), + }, nil +} diff --git a/driver/hoppersqlite/conformance_test.go b/driver/hoppersqlite/conformance_test.go new file mode 100644 index 0000000..6659b42 --- /dev/null +++ b/driver/hoppersqlite/conformance_test.go @@ -0,0 +1,148 @@ +package hoppersqlite_test + +import ( + "context" + "database/sql" + "log/slog" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/hoppersqlite" + "github.com/parallelworks/hopper/drivertest" + "github.com/parallelworks/hopper/hoppermigrate" +) + +// open returns a driver on a new database in the test's temporary +// directory, migrated to the latest schema. +func open(t testing.TB) *hoppersqlite.Driver { + t.Helper() + return openAt(t, filepath.Join(t.TempDir(), "hopper.db")) +} + +// openAt is open on the database at path. +func openAt(t testing.TB, path string) *hoppersqlite.Driver { + t.Helper() + d, err := hoppersqlite.Open(path, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = d.Close() }) + if _, err := hoppermigrate.Up(context.Background(), d, &hoppermigrate.Options{Logger: slog.New(slog.DiscardHandler)}); err != nil { + t.Fatal(err) + } + return d +} + +// fixture runs the conformance suite against hoppersqlite. Its hooks are +// statements on the driver's database handle, with the host's clock as the +// database's. +type fixture struct{} + +func db(d driver.Driver[*sql.Tx]) *sql.DB { return d.(*hoppersqlite.Driver).DB() } + +func now() int64 { return time.Now().UnixMicro() } + +func execSQL(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) { + t.Helper() + if _, err := db(d).ExecContext(ctx, query, args...); err != nil { + t.Error(err) + } +} + +func queryText(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string, args ...any) string { + t.Helper() + var s string + if err := db(d).QueryRowContext(ctx, query, args...).Scan(&s); err != nil { + t.Error(err) + } + return s +} + +func count(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], query string) int { + t.Helper() + n, err := strconv.Atoi(queryText(ctx, t, d, query)) + if err != nil { + t.Error(err) + } + return n +} + +func (fixture) NewDriver(t *testing.T) driver.Driver[*sql.Tx] { return open(t) } + +func (fixture) Begin(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) (*sql.Tx, func() error, func() error) { + t.Helper() + tx, err := db(d).BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + return tx, tx.Commit, tx.Rollback +} + +func (fixture) ExpireLease(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], clientID int64) { + execSQL(ctx, t, d, "UPDATE hopper_clients SET expires_at = ? WHERE id = ?", now()-1e6, clientID) +} + +func (fixture) ExpireLeader(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) { + execSQL(ctx, t, d, "UPDATE hopper_leader SET expires_at = ?", now()-1e6) +} + +func (fixture) MakeClaimable(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { + execSQL(ctx, t, d, "UPDATE hopper_jobs SET scheduled_at = ? WHERE id = ?", now(), id[:]) +} + +func (fixture) AgeAttempt(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID, age time.Duration) { + execSQL(ctx, t, d, "UPDATE hopper_jobs SET attempted_at = ? WHERE id = ? AND attempted_at IS NOT NULL", now()-age.Microseconds(), id[:]) +} + +func (fixture) CountLive(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) int { + return count(ctx, t, d, "SELECT count(*) FROM hopper_jobs") +} + +func (fixture) CountHistory(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) int { + return count(ctx, t, d, "SELECT count(*) FROM hopper_job_history") +} + +func (fixture) ExpireJob(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { + execSQL(ctx, t, d, "UPDATE hopper_jobs SET expires_at = ? WHERE id = ?", now()-1e6, id[:]) +} + +func (fixture) StaleWake(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { + execSQL(ctx, t, d, "UPDATE hopper_jobs SET wake_requested_at = attempted_at - 1000000 WHERE id = ?", id[:]) +} + +func (fixture) MakePending(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) { + execSQL(ctx, t, d, "UPDATE hopper_jobs SET state = 'pending' WHERE id = ?", id[:]) +} + +func (fixture) SnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID) string { + return queryText(ctx, t, d, "SELECT coalesce(snooze_reason, '') FROM hopper_jobs WHERE id = ?", id[:]) +} + +func (fixture) SetSnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID, reason string) { + execSQL(ctx, t, d, "UPDATE hopper_jobs SET snooze_reason = ? WHERE id = ?", reason, id[:]) +} + +func (fixture) SetProgress(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx], id driver.JobID, value string) { + execSQL(ctx, t, d, "UPDATE hopper_jobs SET progress = ? WHERE id = ?", value, id[:]) +} + +func (fixture) CountEdges(ctx context.Context, t *testing.T, d driver.Driver[*sql.Tx]) int { + return count(ctx, t, d, "SELECT count(*) FROM hopper_job_deps") +} + +func (fixture) WaitForLockWait(_ context.Context, t *testing.T, d driver.Driver[*sql.Tx]) { + t.Helper() + for deadline := time.Now().Add(10 * time.Second); hoppersqlite.LockWaiters(d.(*hoppersqlite.Driver)) == 0; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("no statement started waiting for the write lock") + } + } +} + +func TestConformance(t *testing.T) { + t.Parallel() + drivertest.Run(t, fixture{}) +} diff --git a/driver/hoppersqlite/conn.go b/driver/hoppersqlite/conn.go new file mode 100644 index 0000000..a83e748 --- /dev/null +++ b/driver/hoppersqlite/conn.go @@ -0,0 +1,928 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "reflect" + "runtime" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "modernc.org/sqlite" + sqlite3 "modernc.org/sqlite/lib" + + hopperdriver "github.com/parallelworks/hopper/driver" +) + +// SQLite runs one writing transaction at a time, which it holds the +// database's write lock for. A transaction that reads first and writes +// later can find, once it gets the lock, that another wrote in between, and +// must fail: what it read is stale. So every transaction that may write +// takes the lock first, before its first statement reads anything: +// BEGIN IMMEDIATE. +// +// The connections of this package's database/sql handle do that for every +// transaction not begun read-only, but lazily: BeginTx sends nothing, and +// the transaction's first statement, whatever it is, takes the lock. A +// caller that begins a transaction while another holds the lock, and only +// then hands the transaction to a goroutine of its own, therefore never +// blocks in BeginTx. Within this process, transactions queue for the lock +// on a channel, in turn and for as long as their statement's context +// allows; across processes, SQLite's busy handler polls for it, in +// quanta of busyQuantum, between which the context is checked. +// +// After some errors (an interrupted write, a full disk, an I/O error) +// SQLite rolls the whole transaction back itself, and would run the +// transaction's later statements on their own, each committing at once. +// SQLite's rollback hook tells the connection, which then refuses every +// later statement of the transaction (ErrTransactionAborted), as Postgres +// does those of a transaction an error aborted. A COMMIT, END or ROLLBACK +// statement run in the transaction ends it the same way, which the commit +// and rollback hooks tell too: its later statements are refused +// (sql.ErrTxDone) rather than run on their own. + +// busyQuantum is the busy_timeout of every connection: how long SQLite's +// busy handler waits for a lock another process holds before the statement +// fails with SQLITE_BUSY, which this package retries while the statement's +// context allows. +const busyQuantum = 50 * time.Millisecond + +// journalSizeLimit is the size SQLite truncates the write-ahead log to +// when a checkpoint has copied it all back and it starts over, so that a +// burst of writes, or a reader that held up checkpoints for a while, does +// not leave a large log behind for good. +const journalSizeLimit = 32 << 20 + +// ErrTransactionAborted is returned for a statement of a transaction that +// SQLite rolled back itself after an error, such as an interrupted write, a +// full disk or an I/O error: the transaction's earlier statements are +// undone, and its later ones are refused rather than run on their own, as +// Postgres refuses those of a transaction an error aborted. Roll it back; +// Commit rolls it back and returns this error too. +var ErrTransactionAborted = errors.New("hoppersqlite: the transaction was rolled back after an error and runs no more statements; roll it back") + +// ErrLockHeldByOwner is returned for a write that would wait for the +// database's write lock while a transaction of the same owner holds it, +// one its caller began on DB and keeps open, which would wait until its +// context ended and hold every other write up meanwhile. The owner is a +// job's attempt (driver.WithOwner), which the worker's context carries: +// the check applies to a worker's transaction begun with that context, or +// one derived from it, and not to one begun with another. Write through +// that transaction instead, as InsertTx and the other Tx methods do, or end +// it first. Writes of one attempt that are not in a transaction of its own, +// from however many goroutines, wait their turn as any others do. A worker +// that writes from two goroutines at once, one of them in its own +// transaction, gets it, where Postgres would run both; give the other +// goroutine a context without the owner (driver.WithOwner(ctx, nil)) to +// have it wait its turn instead. +var ErrLockHeldByOwner = errors.New("hoppersqlite: the write would wait for the database's write lock, which a transaction the same job attempt began holds open; write through that transaction, or end it first") + +// database is what every connection to one database file in this process +// shares: the write lock, the clock, and the listeners to wake when a +// transaction that notified commits. +type database struct { + path string + // lock holds a token while a connection of this process holds + // SQLite's write lock or waits for another process's to end. + lock chan struct{} + // waiting counts the statements waiting for the write lock, here or + // in another process. + waiting atomic.Int64 + // refs counts the Drivers open on the database (registry.mu). + refs int + // last is the latest time the clock read, in microseconds. + last atomic.Int64 + + holderMu sync.Mutex + holder holder + + listenMu sync.Mutex + listeners map[chan struct{}]struct{} +} + +// holder is who holds the write lock: the owner its transaction runs for, +// or none, since when, and whether the transaction is one a caller began +// and keeps open while its own code runs, rather than the driver's for one +// operation, or a statement on its own. +type holder struct { + owner fmt.Stringer + since time.Time + callers bool +} + +func (h holder) String() string { + if h.since.IsZero() { + return "a transaction that has just ended" + } + who := "a transaction of this process" + if h.owner != nil { + who = "a transaction of " + h.owner.String() + } + return fmt.Sprintf("%s for %s", who, time.Since(h.since).Round(time.Millisecond)) +} + +// registry holds the databases open in this process by path, so that two +// Drivers on one file share its write lock. +var registry = struct { + mu sync.Mutex + dbs map[string]*database +}{dbs: map[string]*database{}} + +func openDatabase(path string) *database { + registry.mu.Lock() + defer registry.mu.Unlock() + db := registry.dbs[path] + if db == nil { + db = &database{path: path, lock: make(chan struct{}, 1), listeners: map[chan struct{}]struct{}{}} + registry.dbs[path] = db + } + db.refs++ + return db +} + +func (db *database) close() { + registry.mu.Lock() + defer registry.mu.Unlock() + if db.refs--; db.refs == 0 { + delete(registry.dbs, db.path) + } +} + +// clock returns the database's time: the host's, in Postgres's precision, +// never earlier than a time it returned before in this process, so that a +// wall clock stepped back (by NTP, say) cannot date a write before the +// writes it follows. +func (db *database) clock() time.Time { + now := time.Now().UTC().Truncate(time.Microsecond).UnixMicro() + for { + last := db.last.Load() + if now <= last { + return time.UnixMicro(last).UTC() + } + if db.last.CompareAndSwap(last, now) { + return time.UnixMicro(now).UTC() + } + } +} + +// acquire takes the write lock for this process for owner, for a +// transaction its caller began (callers) or not, waiting in turn for as +// long as ctx allows. It refuses at once to wait for a lock a transaction +// that a caller of the same owner began holds (ErrLockHeldByOwner), and a +// wait that ctx ends says who held the lock. +func (db *database) acquire(ctx context.Context, owner fmt.Stringer, callers bool) error { + select { + case db.lock <- struct{}{}: + db.held(owner, callers) + return nil + default: + } + db.holderMu.Lock() + h := db.holder + db.holderMu.Unlock() + if h.callers && sameOwner(h.owner, owner) { + return fmt.Errorf("%w (%s)", ErrLockHeldByOwner, owner) + } + db.waiting.Add(1) + defer db.waiting.Add(-1) + start := time.Now() + select { + case db.lock <- struct{}{}: + db.held(owner, callers) + return nil + case <-ctx.Done(): + db.holderMu.Lock() + h := db.holder + db.holderMu.Unlock() + return fmt.Errorf("hoppersqlite: waited %s for the database's write lock, held by %s: %w", + time.Since(start).Round(time.Millisecond), h, ctx.Err()) + } +} + +// sameOwner reports whether a and b are one owner. Owners compare by +// identity; one whose type cannot be compared is never the same as +// another. +func sameOwner(a, b fmt.Stringer) bool { + if a == nil || b == nil { + return false + } + t := reflect.TypeOf(a) + return t == reflect.TypeOf(b) && t.Comparable() && a == b +} + +func (db *database) held(owner fmt.Stringer, callers bool) { + db.holderMu.Lock() + db.holder = holder{owner: owner, since: time.Now(), callers: callers} + db.holderMu.Unlock() +} + +// oneOperationKey marks the context of a transaction the driver begins for +// one operation of its own, which never stays open while a caller's code +// runs. +type oneOperationKey struct{} + +// oneOperation returns ctx marked as oneOperationKey says. +func oneOperation(ctx context.Context) context.Context { + return context.WithValue(ctx, oneOperationKey{}, true) +} + +func (db *database) release() { + db.holderMu.Lock() + db.holder = holder{} + db.holderMu.Unlock() + <-db.lock +} + +// listen registers a channel to be signaled when a transaction that wrote +// notifications commits; the returned function unregisters it. +func (db *database) listen(ch chan struct{}) func() { + db.listenMu.Lock() + defer db.listenMu.Unlock() + db.listeners[ch] = struct{}{} + return func() { + db.listenMu.Lock() + defer db.listenMu.Unlock() + delete(db.listeners, ch) + } +} + +// poke signals every listener of this process. +func (db *database) poke() { + db.listenMu.Lock() + defer db.listenMu.Unlock() + for ch := range db.listeners { + select { + case ch <- struct{}{}: + default: + } + } +} + +// busy reports an error of SQLite's busy family, which a statement that +// waited busyQuantum for another process's lock returns. +func busy(err error) bool { + var e *sqlite.Error + return errors.As(err, &e) && e.Code()&0xff == sqlite3.SQLITE_BUSY +} + +// retryBusy runs fn again while it fails with SQLITE_BUSY and ctx allows, +// counting itself among the database's waiters meanwhile. +func retryBusy[T any](ctx context.Context, db *database, fn func() (T, error)) (T, error) { + v, err := fn() + if !busy(err) { + return v, err + } + db.waiting.Add(1) + defer db.waiting.Add(-1) + start := time.Now() + for busy(err) && ctx.Err() == nil { + v, err = fn() + } + if busy(err) { + return v, fmt.Errorf("hoppersqlite: waited %s for a lock another process holds on the database: %w", + time.Since(start).Round(time.Millisecond), errors.Join(ctx.Err(), err)) + } + return v, err +} + +// innerConn is what a connection of modernc's driver implements. +type innerConn interface { + driver.Conn + driver.ConnPrepareContext + driver.ExecerContext + driver.QueryerContext + driver.Pinger + driver.SessionResetter + driver.Validator + sqlite.HookRegisterer +} + +// connector opens the connections of a Driver's database/sql handle. +type connector struct { + db *database + dsn string + synchronous string +} + +var _ driver.Connector = (*connector)(nil) + +// pragmas sets up a connection: the busy handler's quantum; the +// durability of its commits, on macOS with F_FULLFSYNC, without which fsync +// leaves the commit in the disk's cache; the size the write-ahead log is +// cut back to; and foreign keys off, as SQLite has them by default and +// hopper's schema has none. +func (c *connector) pragmas() []string { + p := []string{ + fmt.Sprintf("PRAGMA busy_timeout = %d", busyQuantum.Milliseconds()), + "PRAGMA synchronous = " + c.synchronous, + fmt.Sprintf("PRAGMA journal_size_limit = %d", journalSizeLimit), + "PRAGMA foreign_keys = OFF", + } + if runtime.GOOS == "darwin" && c.synchronous == "FULL" { + p = append(p, "PRAGMA fullfsync = ON", "PRAGMA checkpoint_fullfsync = ON") + } + return p +} + +func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { + opened, err := sqliteDriver.Open(c.dsn) + if err != nil { + return nil, err + } + inner, ok := opened.(innerConn) + if !ok { + _ = opened.Close() + return nil, fmt.Errorf("hoppersqlite: modernc.org/sqlite's connection is a %T, which lacks a method this driver needs", opened) + } + for _, p := range c.pragmas() { + if _, err := inner.ExecContext(ctx, p, nil); err != nil { + _ = inner.Close() + return nil, fmt.Errorf("hoppersqlite: %s: %w", p, err) + } + } + cn := &conn{inner: inner, db: c.db} + // SQLite calls the hooks on the goroutine running the statement that + // commits or rolls a transaction back, whether by COMMIT or ROLLBACK + // or, for a rollback, after an error. + inner.RegisterRollbackHook(func() { cn.rolledBack = true }) + inner.RegisterCommitHook(func() int32 { cn.committed = true; return 0 }) + return cn, nil +} + +func (c *connector) Driver() driver.Driver { return connectorDriver{c} } + +// connectorDriver is the driver.Driver a connector reports, which opens +// what the connector does whatever the name. +type connectorDriver struct{ c *connector } + +func (d connectorDriver) Open(string) (driver.Conn, error) { return d.c.Connect(context.Background()) } + +// conn is a connection whose transactions take the write lock on their +// first statement. +type conn struct { + inner innerConn + db *database + tx *tx + // rolledBack and committed are set by SQLite's rollback and commit + // hooks when a transaction rolls back or commits; a statement resets + // them before it runs. + rolledBack, committed bool +} + +var ( + _ driver.ConnBeginTx = (*conn)(nil) + _ driver.ConnPrepareContext = (*conn)(nil) + _ driver.ExecerContext = (*conn)(nil) + _ driver.QueryerContext = (*conn)(nil) + _ driver.Pinger = (*conn)(nil) + _ driver.SessionResetter = (*conn)(nil) + _ driver.Validator = (*conn)(nil) +) + +// tx is a transaction on a conn. +type tx struct { + c *conn + readOnly bool + // owner is who the transaction runs for (driver.Owner of the context + // it began with), and oneOperation reports one the driver began for + // one operation of its own. + owner fmt.Stringer + oneOperation bool + // begun reports that BEGIN has run, and locked that the transaction + // holds this process's write lock. + begun, locked bool + // read reports that a read-only transaction has run a statement. + read bool + // notified reports that the transaction wrote notifications, so that + // its commit wakes this process's listeners. + notified bool + // aborted is the error after which SQLite rolled the transaction back. + aborted error + // endedBy names the statement of the caller's that ended the + // transaction (COMMIT, END or ROLLBACK), and committed reports that it + // committed it. + endedBy string + committed bool +} + +// refused returns the error of a statement of a transaction that has +// ended, or nil. +func (t *tx) refused() error { + switch { + case t.aborted != nil: + return abortedError(t.aborted) + case t.endedBy != "": + return fmt.Errorf("hoppersqlite: %s ended the transaction; end it with Commit or Rollback instead: %w", t.endedBy, sql.ErrTxDone) + } + return nil +} + +func (c *conn) exec(ctx context.Context, query string) error { + _, err := c.inner.ExecContext(ctx, query, nil) + return err +} + +func (c *conn) Prepare(query string) (driver.Stmt, error) { + return c.PrepareContext(context.Background(), query) +} + +func (c *conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + s, err := c.inner.PrepareContext(ctx, query) + if err != nil { + return nil, err + } + inner, ok := s.(innerStmt) + if !ok { + _ = s.Close() + return nil, fmt.Errorf("hoppersqlite: modernc.org/sqlite's statement is a %T, which lacks a method this driver needs", s) + } + return &stmt{inner: inner, c: c, query: query}, nil +} + +func (c *conn) Close() error { + if c.tx != nil { + _ = c.tx.Rollback() + } + return c.inner.Close() +} + +func (c *conn) Begin() (driver.Tx, error) { return c.BeginTx(context.Background(), driver.TxOptions{}) } + +// BeginTx begins a transaction. One begun read-only reads a snapshot of +// the database and may not write; any other takes the write lock on its +// first statement. Every isolation level is SQLite's, serializable, which +// is at least as strong as any asked for. +func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if c.tx != nil { + return nil, errors.New("hoppersqlite: a transaction is already open on this connection") + } + t := &tx{c: c, readOnly: opts.ReadOnly, owner: hopperdriver.Owner(ctx), oneOperation: ctx.Value(oneOperationKey{}) != nil} + if opts.ReadOnly { + if err := c.exec(ctx, "PRAGMA query_only = ON"); err != nil { + return nil, err + } + if err := c.exec(ctx, "BEGIN"); err != nil { + _ = c.exec(context.Background(), "PRAGMA query_only = OFF") + return nil, err + } + t.begun = true + } + c.tx = t + return t, nil +} + +// beginWrite takes the write lock for the open transaction and begins it. +func (c *conn) beginWrite(ctx context.Context) error { + if err := c.db.acquire(ctx, c.tx.owner, !c.tx.oneOperation); err != nil { + return err + } + if _, err := retryBusy(ctx, c.db, func() (struct{}, error) { return struct{}{}, c.exec(ctx, "BEGIN IMMEDIATE") }); err != nil { + c.db.release() + return err + } + c.tx.begun, c.tx.locked = true, true + return nil +} + +// abortedError is the error of a statement refused in an aborted +// transaction. It names the cause but does not wrap it: a statement +// refused after an earlier one was interrupted did not see its own context +// end. +func abortedError(cause error) error { + return fmt.Errorf("%w (after: %v)", ErrTransactionAborted, cause) //nolint:errorlint // see above +} + +// enter prepares the open transaction for a statement: it refuses one +// after the transaction ended (refused), begins the transaction on its +// first statement, notes a notification written, and resets the hooks' +// flags. It reports whether the statement may meet SQLITE_BUSY that can be +// retried: the first of a read-only transaction, which has read no +// snapshot yet. +func (c *conn) enter(ctx context.Context, query string) (retry bool, err error) { + t := c.tx + if err := t.refused(); err != nil { + return false, err + } + if !t.begun { + if err := c.beginWrite(ctx); err != nil { + return false, err + } + } + if query == notifySQL { + t.notified = true + } + retry = t.readOnly && !t.read + t.read = true + c.rolledBack, c.committed = false, false + return retry, nil +} + +// leave notes whether a statement of the open transaction, ending with +// err, ended the transaction: SQLite rolled it back after the statement's +// error, or the statement was a ROLLBACK, a COMMIT or an END. +func (c *conn) leave(err error) { + t := c.tx + if t == nil || t.refused() != nil { + return + } + switch { + case c.rolledBack && err != nil: + t.aborted = err + case c.rolledBack: + t.endedBy = "a ROLLBACK statement" + case c.committed && err == nil: + t.endedBy, t.committed = "a COMMIT statement", true + } +} + +// runExec runs a statement that may write: in the open transaction, or on +// its own, holding the write lock while it runs. +func (c *conn) runExec(ctx context.Context, query string, run func() (driver.Result, error)) (driver.Result, error) { + if c.tx == nil { + if err := c.db.acquire(ctx, hopperdriver.Owner(ctx), false); err != nil { + return nil, err + } + defer c.db.release() + return retryBusy(ctx, c.db, run) + } + if _, err := c.enter(ctx, query); err != nil { + return nil, err + } + res, err := run() + c.leave(err) + return res, err +} + +// runQuery runs a query: in the open transaction, or on its own. On its +// own, one that may write (mayWrite) holds the write lock until its rows +// are closed, as a statement that writes in autocommit holds SQLite's +// until it is done. +func (c *conn) runQuery(ctx context.Context, query string, run func() (driver.Rows, error)) (driver.Rows, error) { + if c.tx == nil { + if !mayWrite(query) { + return retryBusy(ctx, c.db, run) + } + if err := c.db.acquire(ctx, hopperdriver.Owner(ctx), false); err != nil { + return nil, err + } + r, err := retryBusy(ctx, c.db, run) + if err != nil { + c.db.release() + return nil, err + } + return &rows{inner: r, release: c.db.release}, nil + } + retry, err := c.enter(ctx, query) + if err != nil { + return nil, err + } + var r driver.Rows + if retry { + r, err = retryBusy(ctx, c.db, run) + } else { + r, err = run() + } + c.leave(err) + if err != nil { + return nil, err + } + return &rows{inner: r, c: c, t: c.tx}, nil +} + +// writeKeywords are the first words of the statements that may write. +var writeKeywords = []string{"INSERT", "UPDATE", "DELETE", "REPLACE", "CREATE", "DROP", "ALTER", "VACUUM", "REINDEX", "ANALYZE"} + +// readPragmas are the pragmas that take an argument in parentheses and +// only read. wal_checkpoint is among them: it waits for writers through +// SQLite's busy handler rather than in turn, as a reader would. +var readPragmas = []string{ + "COLLATION_LIST", "DATABASE_LIST", "FOREIGN_KEY_CHECK", "FOREIGN_KEY_LIST", "FUNCTION_LIST", + "INDEX_INFO", "INDEX_LIST", "INDEX_XINFO", "INTEGRITY_CHECK", "MODULE_LIST", "PRAGMA_LIST", + "QUICK_CHECK", "TABLE_INFO", "TABLE_LIST", "TABLE_XINFO", "WAL_CHECKPOINT", +} + +// mayWrite reports whether a query, run on its own, may write: whether +// any of its statements does, by its first word; a WITH statement by +// whether INSERT, UPDATE, DELETE or REPLACE INTO is among its words; a +// PRAGMA by whether it sets a value, with "=" or in parentheses, or is +// optimize or incremental_vacuum. Words in string literals, quoted +// identifiers and comments do not count. +func mayWrite(query string) bool { + for _, stmt := range sqlTokens(query) { + if statementMayWrite(stmt) { + return true + } + } + return false +} + +func statementMayWrite(t []string) bool { + if len(t) == 0 { + return false + } + switch t[0] { + case "WITH": + for i, w := range t { + switch { + case w == "INSERT" || w == "UPDATE" || w == "DELETE": + return true + case w == "REPLACE" && i+1 < len(t) && t[i+1] == "INTO": + return true + } + } + return false + case "PRAGMA": + name := "" + for i, w := range t[1:] { + switch { + case w == "=": + return true + case w == "(": + return !slices.Contains(readPragmas, name) + case w != "." && i < 3: + name = w // the last of schema.name + } + } + return name == "OPTIMIZE" || name == "INCREMENTAL_VACUUM" + default: + return slices.Contains(writeKeywords, t[0]) + } +} + +// sqlTokens splits SQL into statements, each a list of its words, +// upper-cased, and of its punctuation, one character each. String and blob +// literals, quoted identifiers, numbers, parameters and comments are left +// out. +func sqlTokens(q string) [][]string { + var ( + stmts [][]string + cur []string + ) + isWord := func(c byte) bool { + return c == '_' || c == '$' || c >= '0' && c <= '9' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= 0x80 + } + skipTo := func(i int, end string) int { + if j := strings.Index(q[i:], end); j >= 0 { + return i + j + len(end) + } + return len(q) + } + for i := 0; i < len(q); { + c := q[i] + switch { + case c == '\'' || c == '"' || c == '`': + // A quote doubled inside is the quote itself. + j := i + 1 + for j < len(q) && (q[j] != c || j+1 < len(q) && q[j+1] == c) { + if q[j] == c { + j++ + } + j++ + } + i = j + 1 + case c == '[': + i = skipTo(i, "]") + case strings.HasPrefix(q[i:], "--"): + i = skipTo(i, "\n") + case strings.HasPrefix(q[i:], "/*"): + i = skipTo(i, "*/") + case c == ';': + stmts, cur = append(stmts, cur), nil + i++ + case c == '?' || c == ':' || c == '@' || c >= '0' && c <= '9' || c == '$': + // A parameter or a number, which may run into letters. + j := i + 1 + for j < len(q) && (isWord(q[j]) || q[j] == '.') { + j++ + } + i = j + case isWord(c): + j := i + 1 + for j < len(q) && isWord(q[j]) { + j++ + } + cur = append(cur, strings.ToUpper(q[i:j])) + i = j + case c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r': + // SQLite's whitespace; a vertical tab is not, and fails to prepare. + i++ + default: + cur = append(cur, string(c)) + i++ + } + } + return append(stmts, cur) +} + +func (c *conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + return c.runExec(ctx, query, func() (driver.Result, error) { return c.inner.ExecContext(ctx, query, args) }) +} + +func (c *conn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + return c.runQuery(ctx, query, func() (driver.Rows, error) { return c.inner.QueryContext(ctx, query, args) }) +} + +func (c *conn) Ping(ctx context.Context) error { return c.inner.Ping(ctx) } + +func (c *conn) ResetSession(ctx context.Context) error { + if c.tx != nil { + _ = c.tx.Rollback() + } + return c.inner.ResetSession(ctx) +} + +func (c *conn) IsValid() bool { return c.inner.IsValid() } + +// end ends the transaction's hold on the connection and the lock. +func (t *tx) end() { + if t.locked { + t.c.db.release() + } + if t.readOnly { + _ = t.c.exec(context.Background(), "PRAGMA query_only = OFF") + } + t.c.tx = nil +} + +// Commit commits the transaction. If SQLite rolled it back after an error, +// it rolls back what is left of it and returns ErrTransactionAborted; if a +// statement of the caller's ended it, Commit returns nil for a COMMIT and +// sql.ErrTxDone for a ROLLBACK. +func (t *tx) Commit() error { + defer t.end() + switch { + case !t.begun: + return nil + case t.aborted != nil: + _ = t.rollback() + return abortedError(t.aborted) + case t.endedBy != "" && !t.committed: + return t.refused() + case t.endedBy != "": + if t.notified { + t.c.db.poke() + } + return nil + } + if err := t.c.exec(context.Background(), "COMMIT"); err != nil { + // A COMMIT that failed can leave the transaction open, which + // database/sql does not expect of a connection it reuses. + _ = t.rollback() + return err + } + if t.notified { + t.c.db.poke() + } + return nil +} + +// Rollback rolls the transaction back, or does nothing if it has ended. +func (t *tx) Rollback() error { + defer t.end() + if !t.begun || t.endedBy != "" { + return nil + } + return t.rollback() +} + +// rollback rolls the transaction back. SQLite rolls a transaction back +// itself after some errors (a full disk, an I/O error, an interrupted +// write), which leaves nothing to roll back here. +func (t *tx) rollback() error { + err := t.c.exec(context.Background(), "ROLLBACK") + if err != nil && strings.Contains(err.Error(), "no transaction is active") { + return nil + } + return err +} + +// rows reads a query's rows. In a transaction, it notes a rollback SQLite +// makes while it steps (leave); on its own, a query that may write holds the write +// lock until it is closed. +type rows struct { + inner driver.Rows + c *conn + t *tx + release func() +} + +var ( + _ driver.RowsColumnTypeDatabaseTypeName = (*rows)(nil) + _ driver.RowsColumnTypeLength = (*rows)(nil) + _ driver.RowsColumnTypeNullable = (*rows)(nil) + _ driver.RowsColumnTypePrecisionScale = (*rows)(nil) + _ driver.RowsColumnTypeScanType = (*rows)(nil) +) + +func (r *rows) Columns() []string { return r.inner.Columns() } + +func (r *rows) Close() error { + err := r.inner.Close() + if r.release != nil { + r.release() + r.release = nil + } + return err +} + +func (r *rows) Next(dest []driver.Value) error { + if r.t == nil { + return r.inner.Next(dest) + } + if err := r.t.refused(); err != nil { + return err + } + err := r.inner.Next(dest) + if r.c.tx == r.t { + r.c.leave(err) + } + return err +} + +func (r *rows) ColumnTypeDatabaseTypeName(index int) string { + if t, ok := r.inner.(driver.RowsColumnTypeDatabaseTypeName); ok { + return t.ColumnTypeDatabaseTypeName(index) + } + return "" +} + +func (r *rows) ColumnTypeLength(index int) (int64, bool) { + if t, ok := r.inner.(driver.RowsColumnTypeLength); ok { + return t.ColumnTypeLength(index) + } + return 0, false +} + +func (r *rows) ColumnTypeNullable(index int) (bool, bool) { + if t, ok := r.inner.(driver.RowsColumnTypeNullable); ok { + return t.ColumnTypeNullable(index) + } + return false, false +} + +func (r *rows) ColumnTypePrecisionScale(index int) (int64, int64, bool) { + if t, ok := r.inner.(driver.RowsColumnTypePrecisionScale); ok { + return t.ColumnTypePrecisionScale(index) + } + return 0, 0, false +} + +func (r *rows) ColumnTypeScanType(index int) reflect.Type { + if t, ok := r.inner.(driver.RowsColumnTypeScanType); ok { + return t.ColumnTypeScanType(index) + } + return reflect.TypeFor[any]() +} + +// innerStmt is what a statement of modernc's driver implements. +type innerStmt interface { + driver.Stmt + driver.StmtExecContext + driver.StmtQueryContext +} + +// stmt is a prepared statement whose runs take part in their connection's +// transaction as its statements do. +type stmt struct { + inner innerStmt + c *conn + query string +} + +var ( + _ driver.StmtExecContext = (*stmt)(nil) + _ driver.StmtQueryContext = (*stmt)(nil) +) + +func (s *stmt) Close() error { return s.inner.Close() } +func (s *stmt) NumInput() int { return s.inner.NumInput() } + +func (s *stmt) Exec(args []driver.Value) (driver.Result, error) { + return s.ExecContext(context.Background(), named(args)) +} + +func (s *stmt) Query(args []driver.Value) (driver.Rows, error) { + return s.QueryContext(context.Background(), named(args)) +} + +func (s *stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + return s.c.runExec(ctx, s.query, func() (driver.Result, error) { return s.inner.ExecContext(ctx, args) }) +} + +func (s *stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + return s.c.runQuery(ctx, s.query, func() (driver.Rows, error) { return s.inner.QueryContext(ctx, args) }) +} + +func named(args []driver.Value) []driver.NamedValue { + out := make([]driver.NamedValue, len(args)) + for i, v := range args { + out[i] = driver.NamedValue{Ordinal: i + 1, Value: v} + } + return out +} diff --git a/driver/hoppersqlite/conn_internal_test.go b/driver/hoppersqlite/conn_internal_test.go new file mode 100644 index 0000000..9a34a24 --- /dev/null +++ b/driver/hoppersqlite/conn_internal_test.go @@ -0,0 +1,122 @@ +package hoppersqlite + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +// TestMayWrite checks which statements run on their own take the write +// lock. +func TestMayWrite(t *testing.T) { + for query, want := range map[string]bool{ + "SELECT 1": false, + "select * from t where x = 'INSERT'": false, + "INSERT INTO t VALUES (1) RETURNING x": true, + " -- note\n/* more */ update t SET x = 1": true, + "DELETE FROM t": true, + "REPLACE INTO t VALUES (1)": true, + "WITH c AS (SELECT 1) INSERT INTO t SELECT * FROM c": true, + "WITH c AS (SELECT 1) SELECT * FROM c": false, + "PRAGMA journal_mode": false, + "PRAGMA journal_mode = WAL": true, + "CREATE TABLE t (x)": true, + "ANALYZE t": true, + "-- only a comment": false, + "INSERT\nINTO t VALUES (1)": true, + "delete\tFROM t": true, + "WITH q AS (SELECT name, updated_at FROM hopper_queues) SELECT * FROM q": false, + "WITH j AS (SELECT replace(kind, 'a', 'b') AS k FROM hopper_jobs) SELECT k FROM j": false, + "WITH d AS (SELECT count(*) FROM h WHERE state = 'deleted') SELECT * FROM d": false, + "WITH d AS (SELECT 1 AS \"update\") SELECT * FROM d": false, + "WITH d AS (SELECT 1) SELECT * FROM d -- then DELETE": false, + "WITH d AS (SELECT 'it''s; DELETE') SELECT * FROM d": false, + "WITH d AS (SELECT 1) REPLACE INTO t SELECT * FROM d": true, + "SELECT * FROM t WHERE x = 1; DELETE FROM t": true, + "SELECT ';DELETE FROM t'": false, + "PRAGMA wal_checkpoint(TRUNCATE)": false, + "PRAGMA main.table_info(t)": false, + "PRAGMA user_version(5)": true, + "PRAGMA main.journal_mode = WAL": true, + "PRAGMA optimize": true, + "PRAGMA data_version": false, + "(SELECT 1)": false, + "VALUES (1)": false, + "EXPLAIN DELETE FROM t": false, + "SELECT :update, ?1, $delete": false, + "\fDELETE FROM t": true, + "SELECT 1;\f\fUPDATE t SET x = 1": true, + "WITH d AS (SELECT 1)\fINSERT INTO t SELECT * FROM d": true, + "\f-- note\n\fREPLACE INTO t VALUES (1)": true, + } { + if got := mayWrite(query); got != want { + t.Errorf("mayWrite(%q) = %v, want %v", query, got, want) + } + } +} + +// TestClock checks that the database's clock never goes back, whatever +// the host's does. +func TestClock(t *testing.T) { + db := &database{} + first := db.clock() + if second := db.clock(); second.Before(first) { + t.Errorf("the clock went back from %s to %s", first, second) + } + ahead := time.Now().Add(time.Hour).UnixMicro() + db.last.Store(ahead) // as if the host's clock was stepped back an hour + if got := db.clock(); got.UnixMicro() != ahead { + t.Errorf("clock = %s, want the latest time it read, %s", got, time.UnixMicro(ahead)) + } +} + +type owner struct{ name string } + +func (o *owner) String() string { return o.name } + +type value []int + +func (value) String() string { return "value" } + +// TestOwnerWaits checks that a write fails at once, naming the owner, when +// a transaction of its own owner holds the lock, and that one of another +// owner waits, saying who holds the lock when its context ends. +func TestOwnerWaits(t *testing.T) { + db := &database{lock: make(chan struct{}, 1)} + a, b := &owner{"job a"}, &owner{"job b"} + if err := db.acquire(context.Background(), a, true); err != nil { + t.Fatal(err) + } + err := db.acquire(context.Background(), a, true) + if !errors.Is(err, ErrLockHeldByOwner) || !strings.Contains(err.Error(), "job a") { + t.Errorf("acquire by the holder's owner = %v, want ErrLockHeldByOwner", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + err = db.acquire(ctx, b, true) + if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), "job a") { + t.Errorf("acquire by another owner = %v, want it to wait and name the holder", err) + } + if sameOwner(value{1}, value{1}) || sameOwner(nil, nil) || !sameOwner(a, a) || sameOwner(a, b) { + t.Error("sameOwner compares owners other than by identity, or panics") + } + db.release() + if err := db.acquire(context.Background(), b, true); err != nil { + t.Fatal(err) + } + db.release() + + // A lock held by a transaction the driver began for one operation, or + // by a statement on its own, is waited for by its owner's other writes. + if err := db.acquire(context.Background(), a, false); err != nil { + t.Fatal(err) + } + ctx, cancel = context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + if err := db.acquire(ctx, a, false); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("acquire by the owner of a lock no caller's transaction holds = %v, want it to wait", err) + } + db.release() +} diff --git a/driver/hoppersqlite/executor.go b/driver/hoppersqlite/executor.go new file mode 100644 index 0000000..20c0703 --- /dev/null +++ b/driver/hoppersqlite/executor.go @@ -0,0 +1,487 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "sync" + "time" + + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/internal/jobcodec" +) + +// executor implements driver.Executor on the pool, or inside a caller's +// transaction. +type executor struct { + d *Driver + tx *sql.Tx +} + +var _ driver.Executor = (*executor)(nil) + +// querier is what *sql.DB, *sql.Tx and *sql.Conn share. +type querier interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// errLockBusy reports that a write gave up waiting briefly for the write +// lock (writeBrief). +var errLockBusy = errors.New("hoppersqlite: another transaction holds the database's write lock") + +// clock returns the database's time (database.clock). +func (e *executor) clock() time.Time { return e.d.db.clock() } + +// w is a write in progress: a transaction that holds the write lock, and +// its time, read once the lock was taken, so that every write sees a time +// no earlier than the writes before it. +type w struct { + cached + now time.Time +} + +// maxStatements bounds the statements a Driver keeps prepared. Listings, +// whose text varies with their filters, may otherwise add one for each +// shape of filter. +const maxStatements = 256 + +// statements holds a Driver's prepared statements by text. database/sql +// keeps each prepared on every connection it has run on, and binds it to a +// transaction's connection with Tx.StmtContext, so SQLite parses each text +// once a connection, rather than once a run, which cost most of a +// finalize's time. +type statements struct { + mu sync.Mutex + m map[string]*sql.Stmt +} + +// prepared returns the prepared statement of a query, preparing it on db +// the first time, or nil for a script of several statements, or once +// maxStatements are kept. +func (s *statements) prepared(ctx context.Context, db *sql.DB, query string) (*sql.Stmt, error) { + if strings.Contains(query, ";") { + return nil, nil + } + s.mu.Lock() + defer s.mu.Unlock() + if st, ok := s.m[query]; ok { + return st, nil + } + if len(s.m) >= maxStatements { + return nil, nil + } + st, err := db.PrepareContext(ctx, query) //nolint:sqlclosecheck // kept for the Driver's life; Close closes it + if err != nil { + return nil, err + } + if s.m == nil { + s.m = map[string]*sql.Stmt{} + } + s.m[query] = st + return st, nil +} + +func (s *statements) close() error { + s.mu.Lock() + defer s.mu.Unlock() + var errs []error + for _, st := range s.m { + errs = append(errs, st.Close()) + } + s.m = nil + return errors.Join(errs...) +} + +// cached runs statements on a querier, the pool or a transaction, through +// the Driver's prepared statements. +type cached struct { + q querier + d *Driver +} + +var _ querier = cached{} + +// stmt returns the prepared statement of query bound to the querier, or +// nil to run the text as it is. +func (c cached) stmt(ctx context.Context, query string) (*sql.Stmt, error) { + st, err := c.d.statements.prepared(ctx, c.d.sqlDB, query) + if err != nil || st == nil { + return nil, err + } + if tx, ok := c.q.(*sql.Tx); ok { + return tx.StmtContext(ctx, st), nil + } + return st, nil +} + +func (c cached) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) { + st, err := c.stmt(ctx, query) //nolint:sqlclosecheck // the cache's, or bound to a transaction that closes it + if err != nil { + return nil, err + } + if st == nil { + return c.q.ExecContext(ctx, query, args...) + } + return st.ExecContext(ctx, args...) +} + +func (c cached) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + st, err := c.stmt(ctx, query) //nolint:sqlclosecheck // the cache's, or bound to a transaction that closes it + if err != nil { + return nil, err + } + if st == nil { + return c.q.QueryContext(ctx, query, args...) + } + return st.QueryContext(ctx, args...) +} + +// QueryRowContext runs a query that returns at most one row. A statement +// that fails to prepare reports its error through Scan, as the query's +// would. +func (c cached) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row { + st, err := c.stmt(ctx, query) //nolint:sqlclosecheck // the cache's, or bound to a transaction that closes it + if err != nil || st == nil { + return c.q.QueryRowContext(ctx, query, args...) + } + return st.QueryRowContext(ctx, args...) +} + +// us is the write's time in the unit the schema keeps times in. +func (x *w) us() int64 { return x.now.UnixMicro() } + +// at returns the write's time plus d, in the schema's unit. +func (x *w) at(d time.Duration) int64 { return x.now.Add(d).UnixMicro() } + +// write runs fn in a transaction that holds the write lock: the caller's, +// inside a savepoint so that an error leaves it as it was, or a new one. +func (e *executor) write(ctx context.Context, fn func(*w) error) error { + return e.writeLocking(ctx, 0, fn) +} + +// writeBrief is write that waits at most lockWaitBrief for a write lock +// another transaction holds, and then returns errLockBusy. +func (e *executor) writeBrief(ctx context.Context, fn func(*w) error) error { + return e.writeLocking(ctx, lockWaitBrief, fn) +} + +func (e *executor) writeLocking(ctx context.Context, wait time.Duration, fn func(*w) error) (err error) { + // The first statement takes the lock, under its own deadline when the + // wait is bounded. + lock := func(q cached, statement string) error { + lctx := ctx + if wait > 0 { + var cancel context.CancelFunc + lctx, cancel = context.WithTimeout(ctx, wait) + defer cancel() + } + _, err := q.ExecContext(lctx, statement) + if err != nil && wait > 0 && ctx.Err() == nil && errors.Is(err, context.DeadlineExceeded) { + return errLockBusy + } + return err + } + if e.tx != nil { + q := cached{q: e.tx, d: e.d} + if err := lock(q, "SAVEPOINT hopper"); err != nil { + return err + } + defer func() { + if err != nil { + _, rerr := e.tx.ExecContext(context.WithoutCancel(ctx), "ROLLBACK TO hopper; RELEASE hopper") + err = errors.Join(err, rerr) + } + }() + if err := fn(&w{cached: q, now: e.clock()}); err != nil { + return err + } + _, err := q.ExecContext(ctx, "RELEASE hopper") + return err + } + // A transaction of the driver's own, for this one operation, which may + // wait for a lock its owner's own transaction holds but never holds + // one that another write of the owner should not wait for. + tx, err := e.d.sqlDB.BeginTx(oneOperation(ctx), nil) + if err != nil { + return err + } + defer func() { + if err != nil { + err = errors.Join(err, ignoreDone(tx.Rollback())) + } + }() + q := cached{q: tx, d: e.d} + if err := lock(q, "SELECT 1"); err != nil { + return err + } + if err := fn(&w{cached: q, now: e.clock()}); err != nil { + return err + } + return tx.Commit() +} + +func ignoreDone(err error) error { + if errors.Is(err, sql.ErrTxDone) { + return nil + } + return err +} + +// read returns what a read runs on: the caller's transaction, or the pool, +// on which each statement reads the latest committed state. +func (e *executor) read() querier { + if e.tx != nil { + return cached{q: e.tx, d: e.d} + } + return cached{q: e.d.sqlDB, d: e.d} +} + +// readTx runs fn on one snapshot of the database: the caller's +// transaction, or a read-only one. +func (e *executor) readTx(ctx context.Context, fn func(querier) error) (err error) { + if e.tx != nil { + return fn(cached{q: e.tx, d: e.d}) + } + tx, err := e.d.sqlDB.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + return err + } + defer func() { err = errors.Join(err, ignoreDone(tx.Rollback())) }() + return fn(cached{q: tx, d: e.d}) +} + +// notifySQL writes a notification. conn.go recognizes the statement by its +// text, to wake this process's listeners when the transaction commits. +const notifySQL = "INSERT INTO hopper_notifications (channel, payload, created_at) VALUES (?, ?, ?)" + +// notificationRetention is how long notifications are kept for listeners +// that fall behind; one that falls further behind misses some, and its +// client polls as it would without them. +const notificationRetention = time.Minute + +// notify writes notifications, one per payload, delivered when the write +// commits, and deletes those past notificationRetention. +func (x *w) notify(ctx context.Context, channel string, payloads ...string) error { + if len(payloads) == 0 { + return nil + } + for _, p := range payloads { + if _, err := x.ExecContext(ctx, notifySQL, channel, p, x.us()); err != nil { + return fmt.Errorf("hopper: notify: %w", err) + } + } + if _, err := x.ExecContext(ctx, "DELETE FROM hopper_notifications WHERE created_at < ?", x.at(-notificationRetention)); err != nil { + return fmt.Errorf("hopper: notify: %w", err) + } + return nil +} + +// Notify implements driver.Executor. +func (e *executor) Notify(ctx context.Context, channel string, payloads []string) error { + if len(payloads) == 0 { + return nil + } + return e.write(ctx, func(x *w) error { return x.notify(ctx, channel, payloads...) }) +} + +// Now implements driver.Executor. +func (e *executor) Now(context.Context) (time.Time, error) { return e.clock(), nil } + +// SchemaVersion implements driver.Executor. +func (e *executor) SchemaVersion(ctx context.Context) (int, error) { + versions, err := schemaVersions(ctx, e.read()) + if err != nil { + return 0, fmt.Errorf("hopper: read schema version: %w", err) + } + if len(versions) == 0 { + return 0, nil + } + return slices.Max(versions), nil +} + +// Rows. + +// jobColumns are hopper_jobs' columns in scan order; historyColumns are +// history's, in the same order, with no snooze reason (a finalized job is +// not waiting), followed by finalized_at and output. +const ( + jobColumns = `id, kind, queue, state, priority, attempt, max_attempts, scheduled_at, attempted_at, attempted_by, + args, metadata, errors, unique_key, ordering_key, partition_key, batch_id, expires_at, cancel_requested_at, await, + created_at, snooze_reason, progress, tags, fence` + historyColumns = `id, kind, queue, state, priority, attempt, max_attempts, scheduled_at, attempted_at, attempted_by, + args, metadata, errors, unique_key, ordering_key, partition_key, batch_id, expires_at, cancel_requested_at, await, + created_at, NULL AS snooze_reason, progress, tags, fence, finalized_at, output` + // liveAsHistory reads live rows in historyColumns' shape. + liveAsHistory = jobColumns + `, NULL AS finalized_at, NULL AS output` +) + +// prefixed qualifies a column list with a table alias. +func prefixed(columns, alias string) string { + parts := strings.Split(columns, ",") + for i, p := range parts { + p = strings.TrimSpace(p) + if !strings.HasPrefix(p, "NULL") { + p = alias + "." + p + } + parts[i] = p + } + return strings.Join(parts, ", ") +} + +type scanner interface{ Scan(dest ...any) error } + +// micros scans a nullable time column. +type micros struct{ sql.NullInt64 } + +func (m micros) time() time.Time { + if !m.Valid { + return time.Time{} + } + return time.UnixMicro(m.Int64).UTC() +} + +// scanJob scans a row of jobColumns, plus finalized_at and output when +// history is true, plus trailing columns into extra. +func scanJob(row scanner, history bool, extra ...any) (*driver.JobRow, error) { + var ( + j driver.JobRow + id, batchID []byte + state string + scheduledAt int64 + attemptedAt, expiresAt, cancelAt, createdAt, final micros + attemptedBy sql.NullInt64 + args, metadata, errs string + uniqueKey, orderingKey, partKey, snoozeReason sql.NullString + progress, tags, output sql.NullString + await bool + ) + dest := []any{ + &id, &j.Kind, &j.Queue, &state, &j.Priority, &j.Attempt, &j.MaxAttempts, &scheduledAt, &attemptedAt, &attemptedBy, + &args, &metadata, &errs, &uniqueKey, &orderingKey, &partKey, &batchID, &expiresAt, &cancelAt, &await, + &createdAt, &snoozeReason, &progress, &tags, &j.Fence, + } + if history { + dest = append(dest, &final, &output) + } + if err := row.Scan(append(dest, extra...)...); err != nil { + return nil, err + } + if err := j.ID.Scan(id); err != nil { + return nil, err + } + if len(batchID) > 0 { + if err := j.BatchID.Scan(batchID); err != nil { + return nil, err + } + } + var err error + if j.Errors, err = jobcodec.DecodeErrors(errs); err != nil { + return nil, fmt.Errorf("errors of job %s: %w", j.ID, err) + } + j.State = driver.JobState(state) + j.ScheduledAt = time.UnixMicro(scheduledAt).UTC() + j.AttemptedAt, j.ExpiresAt, j.CancelRequestedAt = attemptedAt.time(), expiresAt.time(), cancelAt.time() + j.CreatedAt, j.FinalizedAt = createdAt.time(), final.time() + j.AttemptedBy = attemptedBy.Int64 + j.Args, j.Metadata = json.RawMessage(args), json.RawMessage(metadata) + if output.Valid { + j.Output = json.RawMessage(output.String) + } + j.UniqueKey, j.OrderingKey, j.PartitionKey = uniqueKey.String, orderingKey.String, partKey.String + j.Await = await + j.Tags = jobcodec.DecodeTags(tags.String) + // Only a snoozed job is waiting for a reason; one left on a job in + // another state by a direct write is not reported. + if j.State == driver.JobStateScheduled { + j.SnoozeReason = snoozeReason.String + } + if progress.Valid { + j.Progress = jobcodec.DecodeProgress(progress.String) + } + return &j, nil +} + +// queryAll runs a query and reads every row it returns through scan. +func queryAll[T any](ctx context.Context, q querier, scan func(scanner) (T, error), query string, args ...any) ([]T, error) { + rows, err := q.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []T + for rows.Next() { + v, err := scan(rows) + if err != nil { + return nil, err + } + out = append(out, v) + } + return out, rows.Err() +} + +// scanHistoryJob scans a row in historyColumns' shape. +func scanHistoryJob(r scanner) (*driver.JobRow, error) { return scanJob(r, true) } + +// Parameters. + +func idParam(id driver.JobID) []byte { return id[:] } + +// nullableID is a BLOB parameter for an ID, NULL for the zero ID. +func nullableID(id driver.JobID) any { + if id.IsZero() { + return nil + } + return id[:] +} + +// nullable is a TEXT parameter, NULL for the empty string. +func nullable(s string) any { + if s == "" { + return nil + } + return s +} + +// smallint checks a value bound for what Postgres keeps in a smallint, so +// that a value means the same on either engine. +func smallint(v int, column string) (int, error) { + if v < -32768 || v > 32767 { + return 0, fmt.Errorf("hopper: %s %d is out of range", column, v) + } + return v, nil +} + +// jsonValue is a JSON parameter: raw, or def when empty. It refuses what +// Postgres's jsonb refuses (checkJSON). +func jsonValue(raw []byte, def, what string) (string, error) { + if len(raw) == 0 { + return def, nil + } + if err := checkJSON(raw); err != nil { + return "", fmt.Errorf("hopper: %s %w: %.40q", what, err, raw) + } + return string(raw), nil +} + +// tagsJSON is a job's tags column: a JSON array, or NULL for none. +func tagsJSON(tags []string) any { + if len(tags) == 0 { + return nil + } + b, _ := json.Marshal(tags) + return string(b) +} + +// placeholders returns n comma-separated parameter markers. +func placeholders(n int) string { + if n <= 0 { + return "" + } + return strings.Repeat("?, ", n-1) + "?" +} diff --git a/driver/hoppersqlite/export_test.go b/driver/hoppersqlite/export_test.go new file mode 100644 index 0000000..54b2e8c --- /dev/null +++ b/driver/hoppersqlite/export_test.go @@ -0,0 +1,5 @@ +package hoppersqlite + +// LockWaiters counts the statements waiting for d's database's write lock, +// in this process or for another process's. +func LockWaiters(d *Driver) int64 { return d.db.waiting.Load() } diff --git a/driver/hoppersqlite/finalize.go b/driver/hoppersqlite/finalize.go new file mode 100644 index 0000000..7f4f6c4 --- /dev/null +++ b/driver/hoppersqlite/finalize.go @@ -0,0 +1,572 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "math" + "time" + + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/internal/jobcodec" + "github.com/parallelworks/hopper/internal/jobtags" +) + +// Every write that finalizes jobs (the batched finalize, canceling or +// deleting a waiting job, expiring) runs in a transaction that holds the +// write lock, so it reads each job, checks it in Go and writes it back with +// no other writer in between: what Postgres's statements do in one +// statement, with row locks and a recheck of every condition, is here a +// sequence of plain statements. The tail does what finalized jobs entail +// in the same transaction: canceling the dependents of a failed job, +// archiving, counting the jobs down in their batches and completing those +// they finish, promoting dependents whose dependencies are all finished, +// and dropping satisfied edges. No other transaction can finalize a +// batch's or a dependent's last jobs at the same time, so nothing is left +// for BatchSettle or WorkflowPromoteReady to find. + +// liveRow is a live job's row with the columns its finalize reads beside +// the JobRow's. +type liveRow struct { + job *driver.JobRow + logs sql.NullString + wakeRequestedAt micros + snoozeReason sql.NullString + // errors is the errors column as stored, which a finalize appends to. + errors string +} + +const liveRowColumns = jobColumns + ", logs, wake_requested_at, snooze_reason, errors" + +func scanLive(row scanner) (*liveRow, error) { + var r liveRow + var err error + r.job, err = scanJob(row, false, &r.logs, &r.wakeRequestedAt, &r.snoozeReason, &r.errors) + return &r, err +} + +// liveRow returns a live job's row, or nil if the job is not live. +func (x *w) liveRow(ctx context.Context, id driver.JobID) (*liveRow, error) { + r, err := scanLive(x.QueryRowContext(ctx, "SELECT "+liveRowColumns+" FROM hopper_jobs WHERE id = ?", idParam(id))) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return r, err +} + +// finished is a job leaving the live table: its final state and what its +// history row keeps. +type finished struct { + row *liveRow + state driver.JobState + errors string + logs sql.NullString + output any + archive bool +} + +func (f finished) failed() bool { return f.state != driver.JobStateCompleted } + +// jobRow returns the job as it is once finished: in its final state, with +// its errors, logs and output, finalized at now, and waiting for nothing. +func (f finished) jobRow(now time.Time) (*driver.JobRow, error) { + j := *f.row.job + j.State, j.FinalizedAt, j.SnoozeReason = f.state, now, "" + var err error + if j.Errors, err = jobcodec.DecodeErrors(f.errors); err != nil { + return nil, err + } + j.Output = nil + if s, ok := f.output.(string); ok { + j.Output = json.RawMessage(s) + } + return &j, nil +} + +// archiveSQL copies a live job to history, with its final state, errors, +// logs, finalization time and output. +const archiveSQL = ` +INSERT INTO hopper_job_history (id, seq, kind, queue, state, priority, attempt, max_attempts, scheduled_at, attempted_at, + attempted_by, args, metadata, errors, unique_key, ordering_key, partition_key, batch_id, expires_at, cancel_requested_at, + await, created_at, progress, tags, fence, logs, finalized_at, output) +SELECT id, seq, kind, queue, ?, priority, attempt, max_attempts, scheduled_at, attempted_at, + attempted_by, args, metadata, ?, unique_key, ordering_key, partition_key, batch_id, expires_at, cancel_requested_at, + await, created_at, progress, tags, fence, ?, ?, ? +FROM hopper_jobs WHERE id = ?` + +// finish moves a job out of the live table, to history if it is archived, +// and announces it to those awaiting it. +func (x *w) finish(ctx context.Context, f finished) error { + id := idParam(f.row.job.ID) + if f.archive { + if _, err := x.ExecContext(ctx, archiveSQL, string(f.state), f.errors, f.logs, x.us(), f.output, id); err != nil { + return fmt.Errorf("archive job %s: %w", f.row.job.ID, err) + } + } + if _, err := x.ExecContext(ctx, "DELETE FROM hopper_jobs WHERE id = ?", id); err != nil { + return fmt.Errorf("delete job %s: %w", f.row.job.ID, err) + } + if f.row.job.Await { + return x.notify(ctx, driver.ChannelDone, f.row.job.ID.String()) + } + return nil +} + +// appendJSON appends an element to a JSON array as stored, keeping the +// elements there as they are. A value that is not an array starts a new one. +func appendJSON(stored string, elem []byte) (string, error) { + var elems []json.RawMessage + _ = json.Unmarshal([]byte(stored), &elems) + b, err := json.Marshal(append(elems, elem)) + return string(b), err +} + +// withErrorAt appends an error record, timestamped at now, to stored. +func withErrorAt(stored string, e *driver.AttemptError, now time.Time) (string, error) { + b, err := jobcodec.MarshalAttemptError(e) + if err != nil { + return "", err + } + var rec map[string]json.RawMessage + if err := json.Unmarshal(b, &rec); err != nil { + return "", err + } + rec["at"], _ = json.Marshal(now) + b, err = json.Marshal(rec) + if err != nil { + return "", err + } + return appendJSON(stored, b) +} + +// hopperError is the errors column of a job that hopper itself fails, such +// as one canceled or expired while waiting: a snoozed job's reason is kept +// in the error, since the row that held it is leaving the live table. +func hopperError(r *liveRow, message string, now time.Time) (string, error) { + if r.job.State == driver.JobStateScheduled && r.snoozeReason.Valid { + message += " while snoozed: " + r.snoozeReason.String + } + return withErrorAt(r.errors, &driver.AttemptError{Attempt: r.job.Attempt, Error: message}, now) +} + +// keepLogs is how many attempt logs a finalize keeps for keep. +func keepLogs(keep int) int { + if keep <= 0 { + return math.MaxInt32 + } + return keep +} + +// appendLog appends an attempt's log, if it has one, to a job's logs as +// stored, keeping the latest keep. A snooze gives its attempt back, so a job +// that snoozes again and again would fill its logs with them: a snooze's +// log, or its lack of one, replaces the log of the job's previous snooze of +// the same attempt number, attempt. Logs that do not change are returned +// as stored; none at all, as NULL. +func appendLog(stored sql.NullString, attempt int, snooze bool, log []byte, keep int) (sql.NullString, error) { + var entries []json.RawMessage + if stored.Valid { + _ = json.Unmarshal([]byte(stored.String), &entries) + } + sameSnooze := func(raw json.RawMessage) bool { + var e map[string]json.RawMessage + if json.Unmarshal(raw, &e) != nil { + return false + } + _, snoozed := e["snoozed"] + var a json.Number + return snoozed && json.Unmarshal(e["attempt"], &a) == nil && a.String() == fmt.Sprint(attempt) + } + replaces := false + if snooze { + for _, e := range entries { + replaces = replaces || sameSnooze(e) + } + } + if log == nil && !replaces { + return stored, nil + } + kept := entries[:0:0] + for _, e := range entries { + if !snooze || !sameSnooze(e) { + kept = append(kept, e) + } + } + if log != nil { + kept = append(kept, log) + } + if len(kept) > keep { + kept = kept[len(kept)-keep:] + } + if len(kept) == 0 { + return sql.NullString{}, nil + } + b, err := json.Marshal(kept) + return sql.NullString{String: string(b), Valid: true}, err +} + +// result is a JobFinalize with its values encoded. +type result struct { + driver.JobFinalize + err *driver.AttemptError + log []byte + output any + reason any +} + +func encodeResults(jobs []driver.JobFinalize) ([]result, error) { + out := make([]result, len(jobs)) + for i, f := range jobs { + switch f.State { + case driver.JobStateCompleted, driver.JobStateCanceled, driver.JobStateDiscarded, + driver.JobStateRetryable, driver.JobStateScheduled: + default: + return nil, fmt.Errorf("hopper: finalize jobs: %s is not a finalize state", f.State) + } + r := result{JobFinalize: f, err: f.Error, reason: nullable(jobcodec.StorableText(f.SnoozeReason))} + log, err := jobcodec.MarshalAttemptLog(f) + if err != nil { + return nil, fmt.Errorf("hopper: finalize jobs: %w", err) + } + r.log = log + if len(f.Output) > 0 { + out, err := jsonValue(f.Output, "", "output") + if err != nil { + return nil, fmt.Errorf("hopper: finalize jobs: %w", err) + } + r.output = out + } + out[i] = r + } + return out, nil +} + +// JobFinalizeMany implements driver.Executor. A result applies only to a +// job running in the claim that produced it, by the same client at the same +// fence, so a stale result never overwrites a newer attempt. With +// SkipLocked, a write lock another transaction holds is waited for only +// briefly, after which no result is applied. +func (e *executor) JobFinalizeMany(ctx context.Context, params driver.JobFinalizeParams) ([]driver.JobID, error) { + if len(params.Jobs) == 0 { + return nil, nil + } + results, err := encodeResults(params.Jobs) + if err != nil { + return nil, err + } + keep := keepLogs(params.KeepLogs) + var applied []driver.JobID + write := e.write + if params.SkipLocked { + write = e.writeBrief + } + err = write(ctx, func(x *w) error { + applied = applied[:0] + var done []finished + for _, r := range results { + row, err := x.liveRow(ctx, r.ID) + if err != nil { + return err + } + if row == nil { + continue + } + j := row.job + if j.State != driver.JobStateRunning || j.AttemptedBy == 0 || j.AttemptedBy != r.AttemptedBy || j.Fence != r.Fence { + continue + } + errs := row.errors + if r.err != nil { + if errs, err = withErrorAt(errs, r.err, x.now); err != nil { + return err + } + } + logs, err := appendLog(row.logs, j.Attempt, r.Snooze, r.log, keep) + if err != nil { + return err + } + if r.State.Terminal() { + f := finished{row: row, state: r.State, errors: errs, logs: logs, output: r.output, archive: r.Archived()} + if err := x.finish(ctx, f); err != nil { + return err + } + done = append(done, f) + applied = append(applied, r.ID) + continue + } + // A wake since the attempt began (JobRetryOrderingKey) makes the + // job runnable at once, and takes the place of a snooze's reason. + woken := row.wakeRequestedAt.Valid && !j.AttemptedAt.IsZero() && !row.wakeRequestedAt.time().Before(j.AttemptedAt) + scheduledAt := x.at(max(r.Delay, 0)) + if woken { + scheduledAt = x.us() + } + attempt := j.Attempt + var reason any + if r.Snooze { + attempt-- + if !woken { + reason = r.reason + } + } + if _, err := x.ExecContext(ctx, ` + UPDATE hopper_jobs SET state = ?, scheduled_at = ?, attempt = ?, errors = ?, logs = ?, attempted_by = NULL, + snooze_reason = ?, wake_requested_at = NULL + WHERE id = ?`, string(r.State), scheduledAt, attempt, errs, logs, reason, idParam(r.ID)); err != nil { + return err + } + applied = append(applied, r.ID) + } + if params.NoBatches { + return nil + } + _, err := x.tail(ctx, done) + return err + }) + if errors.Is(err, errLockBusy) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("hopper: finalize jobs: %w", err) + } + return applied, nil +} + +// tail does what the jobs in done, which have just left the live table, +// entail, and returns done with the dependents it canceled. +func (x *w) tail(ctx context.Context, done []finished) ([]finished, error) { + if len(done) == 0 { + return done, nil + } + // Cancel the dependents of failed jobs, down the graph by cancel edges. + cascaded := map[driver.JobID]bool{} + var queue []driver.JobID + for _, f := range done { + if f.failed() { + queue = append(queue, f.row.job.ID) + } + } + seen := map[driver.JobID]bool{} + for len(queue) > 0 { + id := queue[0] + queue = queue[1:] + deps, err := x.ids(ctx, "SELECT job_id FROM hopper_job_deps WHERE depends_on = ? AND on_failure = 'cancel'", idParam(id)) + if err != nil { + return nil, err + } + for _, dep := range deps { + if seen[dep] { + continue + } + seen[dep] = true + queue = append(queue, dep) + row, err := x.liveRow(ctx, dep) + if err != nil { + return nil, err + } + if row == nil || row.job.State != driver.JobStatePending { + continue + } + errs, err := hopperError(row, "hopper: dependency failed", x.now) + if err != nil { + return nil, err + } + f := finished{row: row, state: driver.JobStateCanceled, errors: errs, logs: row.logs, archive: true} + if err := x.finish(ctx, f); err != nil { + return nil, err + } + done = append(done, f) + cascaded[dep] = true + } + } + + // Count the jobs down in their batches. + type count struct{ n, failed int } + var batches []driver.JobID + counts := map[driver.JobID]*count{} + for _, f := range done { + b := f.row.job.BatchID + if b.IsZero() { + continue + } + c := counts[b] + if c == nil { + c = &count{} + counts[b] = c + batches = append(batches, b) + } + c.n++ + if f.failed() { + c.failed++ + } + } + for _, b := range batches { + if err := x.countBatch(ctx, b, counts[b].n, counts[b].failed); err != nil { + return nil, err + } + } + + // Promote the dependents whose dependencies have all finished, and drop + // the edges of the jobs that finished. + considered := map[driver.JobID]bool{} + for _, f := range done { + deps, err := x.ids(ctx, "SELECT job_id FROM hopper_job_deps WHERE depends_on = ?", idParam(f.row.job.ID)) + if err != nil { + return nil, err + } + for _, dep := range deps { + if considered[dep] || cascaded[dep] { + continue + } + considered[dep] = true + if _, err := x.promoteIfReady(ctx, dep); err != nil { + return nil, err + } + } + } + for _, f := range done { + if _, err := x.ExecContext(ctx, "DELETE FROM hopper_job_deps WHERE job_id = ?", idParam(f.row.job.ID)); err != nil { + return nil, err + } + } + return done, nil +} + +// ids returns the IDs a query selects. +func (x *w) ids(ctx context.Context, query string, args ...any) ([]driver.JobID, error) { + return queryIDs(ctx, x, query, args...) +} + +func queryIDs(ctx context.Context, q querier, query string, args ...any) ([]driver.JobID, error) { + return queryAll(ctx, q, func(r scanner) (driver.JobID, error) { + var b []byte + if err := r.Scan(&b); err != nil { + return driver.JobID{}, err + } + var id driver.JobID + err := id.Scan(b) + return id, err + }, query, args...) +} + +// promoteIfReady makes a pending job claimable if none of its dependencies +// is still live, and reports whether it did. +func (x *w) promoteIfReady(ctx context.Context, id driver.JobID) (bool, error) { + var queue string + err := x.QueryRowContext(ctx, ` + UPDATE hopper_jobs SET state = CASE WHEN scheduled_at > ? THEN 'scheduled' ELSE 'available' END + WHERE id = ? AND state = 'pending' + AND NOT EXISTS (SELECT 1 FROM hopper_job_deps d JOIN hopper_jobs x ON x.id = d.depends_on WHERE d.job_id = ?) + RETURNING queue`, x.us(), idParam(id), idParam(id)).Scan(&queue) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, err + } + return true, x.notify(ctx, driver.ChannelInsert, queue) +} + +// countBatch counts n jobs of a batch, failed of them canceled or +// discarded, as finished, and completes the batch, inserting its +// callbacks, if nothing remains. A batch that has completed already counts +// them too, as a Postgres batch's settle does. +func (x *w) countBatch(ctx context.Context, id driver.JobID, n, failed int) error { + var ( + pending, wasFailed int + completed micros + onSuccess, onFailure, onComplete sql.NullString + ) + err := x.QueryRowContext(ctx, `SELECT pending, failed, completed_at, on_success, on_failure, on_complete FROM hopper_batches WHERE id = ?`, + idParam(id)).Scan(&pending, &wasFailed, &completed, &onSuccess, &onFailure, &onComplete) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("batch %s: %w", id, err) + } + pending -= n + failed += wasFailed + if completed.Valid || pending > 0 { + _, err := x.ExecContext(ctx, `UPDATE hopper_batches SET pending = ?, failed = ? WHERE id = ?`, max(pending, 0), failed, idParam(id)) + return err + } + if _, err := x.ExecContext(ctx, `UPDATE hopper_batches SET pending = 0, failed = ?, completed_at = ? WHERE id = ?`, failed, x.us(), idParam(id)); err != nil { + return err + } + outcome := onSuccess + if failed > 0 { + outcome = onFailure + } + for _, cb := range []sql.NullString{outcome, onComplete} { + if cb.Valid { + if err := x.insertCallback(ctx, cb.String, id, failed); err != nil { + return fmt.Errorf("batch %s: callback: %w", id, err) + } + } + } + return nil +} + +// insertCallback inserts a completed batch's callback. Its values are read +// as Postgres reads them: brought into range, tags as a job's tags column +// reads them, and the batch's ID and failures added to its metadata, so +// that one batch row a direct write made odd cannot fail the finalize that +// completes it. +func (x *w) insertCallback(ctx context.Context, stored string, batch driver.JobID, failed int) error { + var cb struct { + Kind string `json:"kind"` + Queue string `json:"queue"` + Priority json.Number `json:"priority"` + MaxAttempts json.Number `json:"max_attempts"` + Args json.RawMessage `json:"args"` + Metadata json.RawMessage `json:"metadata"` + Tags json.RawMessage `json:"tags"` + } + dec := json.NewDecoder(bytesReader(stored)) + dec.UseNumber() + if err := dec.Decode(&cb); err != nil { + return err + } + clamp := func(n json.Number, def, lo, hi float64) int { + v, err := n.Float64() + if err != nil { + v = def + } + return int(math.Min(math.Max(math.Round(v), lo), hi)) + } + queue := cb.Queue + if queue == "" { + queue = "default" + } + args := "{}" + if len(cb.Args) > 0 && string(cb.Args) != "null" { + args = string(cb.Args) + } + meta := map[string]json.RawMessage{} + if len(cb.Metadata) > 0 { + _ = json.Unmarshal(cb.Metadata, &meta) + } + meta["batch_id"], _ = json.Marshal(batch.String()) + meta["batch_failed"], _ = json.Marshal(failed) + metadata, err := json.Marshal(meta) + if err != nil { + return err + } + var tags []string + for _, t := range jobcodec.DecodeTags(string(cb.Tags)) { + if len(t) <= jobtags.MaxBytes { + tags = append(tags, t) + } + } + _, err = x.insertJob(ctx, jobInsert{ + kind: cb.Kind, queue: queue, priority: clamp(cb.Priority, 2, 1, 4), maxAttempts: clamp(cb.MaxAttempts, 25, 1, 32767), + args: args, metadata: string(metadata), tags: tags, + }) + if err != nil { + return err + } + return x.notify(ctx, driver.ChannelInsert, queue) +} diff --git a/driver/hoppersqlite/flow.go b/driver/hoppersqlite/flow.go new file mode 100644 index 0000000..dac369b --- /dev/null +++ b/driver/hoppersqlite/flow.go @@ -0,0 +1,203 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "math" + "time" + + "github.com/parallelworks/hopper/driver" +) + +// claimLimited claims within a queue's limits: the budget is the smallest +// of the claim's limit, what the global limit leaves, and the whole tokens +// in the rate bucket. The write lock serializes every client's claims on +// the queue, so the limits hold exactly. +func (e *executor) claimLimited(ctx context.Context, params driver.JobClaimParams) (driver.JobClaimResult, error) { + var res driver.JobClaimResult + err := e.write(ctx, func(x *w) error { + var ( + global, burst, partition sql.NullInt64 + rate, tokens sql.NullFloat64 + refilled micros + running int + ) + err := x.QueryRowContext(ctx, `SELECT global_limit, rate_per_sec, rate_burst, tokens, refilled_at, partition_limit + FROM hopper_queues WHERE name = ?`, params.Queue).Scan(&global, &rate, &burst, &tokens, &refilled, &partition) + if errors.Is(err, sql.ErrNoRows) { + // No row, no limits. + res.Jobs, err = x.claim(ctx, params.Queue, params.ClientID, params.Limit, 0) + return err + } + if err != nil { + return fmt.Errorf("hopper: read queue limits: %w", err) + } + if err := x.QueryRowContext(ctx, `SELECT count(*) FROM hopper_jobs WHERE queue = ? AND state = 'running'`, params.Queue).Scan(&running); err != nil { + return fmt.Errorf("hopper: count running: %w", err) + } + budget := params.Limit + if global.Valid { + budget = min(budget, int(global.Int64)-running) + } + var available float64 + if rate.Valid && rate.Float64 > 0 { + capacity := float64(burst.Int64) + if !burst.Valid || capacity <= 0 { + capacity = math.Ceil(rate.Float64) + } + available = capacity + if tokens.Valid { + elapsed := 0.0 + if refilled.Valid { + elapsed = x.now.Sub(refilled.time()).Seconds() + } + available = min(capacity, tokens.Float64+rate.Float64*elapsed) + } + budget = min(budget, int(math.Floor(available))) + if budget <= 0 { + res.Wait = time.Duration((1 - available) / rate.Float64 * float64(time.Second)) + } + } + if budget <= 0 { + return nil + } + if res.Jobs, err = x.claim(ctx, params.Queue, params.ClientID, budget, partition.Int64); err != nil { + return err + } + if rate.Valid && rate.Float64 > 0 { + if _, err := x.ExecContext(ctx, `UPDATE hopper_queues SET tokens = ?, refilled_at = ? WHERE name = ?`, + available-float64(len(res.Jobs)), x.us(), params.Queue); err != nil { + return fmt.Errorf("hopper: update rate bucket: %w", err) + } + } + return nil + }) + if err != nil { + return driver.JobClaimResult{}, err + } + return res, nil +} + +// callbackJSON encodes a batch callback for the batch row, refusing values +// a job could not be inserted with, which would fail the transaction that +// completes the batch. +func callbackJSON(p *driver.JobInsertParams) (any, error) { + if p == nil { + return nil, nil + } + if p.Priority < 1 || p.Priority > 4 { + return nil, fmt.Errorf("hopper: callback priority %d is out of range 1..4", p.Priority) + } + if _, err := smallint(p.MaxAttempts, "callback max_attempts"); err != nil { + return nil, err + } + args, err := jsonValue(p.Args, "{}", "callback args") + if err != nil { + return nil, err + } + metadata, err := jsonValue(p.Metadata, "{}", "callback metadata") + if err != nil { + return nil, err + } + cb := map[string]any{ + "kind": p.Kind, "queue": p.Queue, "priority": p.Priority, "max_attempts": p.MaxAttempts, + "args": json.RawMessage(args), "metadata": json.RawMessage(metadata), + } + if len(p.Tags) > 0 { + cb["tags"] = p.Tags + } + b, err := json.Marshal(cb) + if err != nil { + return nil, err + } + return string(b), nil +} + +// batchRow is a batch's callbacks and metadata, encoded for its row. +type batchRow struct { + onSuccess, onFailure, onComplete any + metadata string +} + +func encodeBatch(onSuccess, onFailure, onComplete *driver.JobInsertParams, metadata []byte) (batchRow, error) { + var b batchRow + var err error + if b.onSuccess, err = callbackJSON(onSuccess); err != nil { + return b, err + } + if b.onFailure, err = callbackJSON(onFailure); err != nil { + return b, err + } + if b.onComplete, err = callbackJSON(onComplete); err != nil { + return b, err + } + b.metadata, err = jsonValue(metadata, "{}", "batch metadata") + return b, err +} + +// insertBatch inserts a batch row expecting total jobs. +func (x *w) insertBatch(ctx context.Context, id driver.JobID, name string, total int, b batchRow, edges any) error { + _, err := x.ExecContext(ctx, `INSERT INTO hopper_batches (id, name, pending, total, on_success, on_failure, on_complete, metadata, edges, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + idParam(id), nullable(name), total, total, b.onSuccess, b.onFailure, b.onComplete, b.metadata, edges, x.us()) + return err +} + +// BatchInsert implements driver.Executor. +func (e *executor) BatchInsert(ctx context.Context, params driver.BatchInsertParams) (driver.JobID, error) { + b, err := encodeBatch(params.OnSuccess, params.OnFailure, params.OnComplete, params.Metadata) + if err != nil { + return driver.JobID{}, err + } + var id driver.JobID + err = e.write(ctx, func(x *w) error { + id = newID(x.now) + return x.insertBatch(ctx, id, "", params.Total, b, nil) + }) + if err != nil { + return driver.JobID{}, fmt.Errorf("hopper: insert batch: %w", err) + } + return id, nil +} + +// BatchSettle implements driver.Executor. The transaction that finalizes a +// batch's jobs counts them down on its row and completes it, so there is no +// progress to fold, and no batch a race left open. +func (e *executor) BatchSettle(context.Context, driver.BatchSettleParams) (driver.BatchSettleResult, error) { + return driver.BatchSettleResult{}, nil +} + +// batchColumns read a batch's progress as BatchRow has it. +const batchColumns = `id, CASE WHEN completed_at IS NULL THEN max(pending, 0) ELSE 0 END, failed, total, created_at, completed_at, metadata` + +func scanBatch(row scanner, extra ...any) (*driver.BatchRow, error) { + var ( + b driver.BatchRow + id []byte + created, completed micros + metadata string + ) + if err := row.Scan(append([]any{&id, &b.Pending, &b.Failed, &b.Total, &created, &completed, &metadata}, extra...)...); err != nil { + return nil, err + } + if err := b.ID.Scan(id); err != nil { + return nil, err + } + b.CreatedAt, b.CompletedAt, b.Metadata = created.time(), completed.time(), json.RawMessage(metadata) + return &b, nil +} + +// BatchGet implements driver.Executor. +func (e *executor) BatchGet(ctx context.Context, id driver.JobID) (*driver.BatchRow, error) { + b, err := scanBatch(e.read().QueryRowContext(ctx, `SELECT `+batchColumns+` FROM hopper_batches WHERE id = ?`, idParam(id))) + if errors.Is(err, sql.ErrNoRows) { + return nil, driver.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("hopper: get batch: %w", err) + } + return b, nil +} diff --git a/driver/hoppersqlite/funcs.go b/driver/hoppersqlite/funcs.go new file mode 100644 index 0000000..af45021 --- /dev/null +++ b/driver/hoppersqlite/funcs.go @@ -0,0 +1,360 @@ +package hoppersqlite + +import ( + "bytes" + sqldriver "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "sync" + "unicode/utf16" + "unicode/utf8" + + "modernc.org/sqlite" +) + +// sqliteDriver is modernc's driver with hopper's SQL functions, which +// only this package's connections have: registered on the package-level +// driver, they would reach every connection the process opens. +var sqliteDriver = func() *sqlite.Driver { + d := &sqlite.Driver{} + d.MustRegisterDeterministicScalarFunction("hopper_topic_matches", 2, topicMatchesSQL) + d.MustRegisterDeterministicScalarFunction("hopper_json_contains", 2, jsonContainsSQL) + return d +}() + +// textArg reads a TEXT (or BLOB) argument of a SQL function, and reports +// false for NULL. +func textArg(v sqldriver.Value) (string, bool) { + switch v := v.(type) { + case string: + return v, true + case []byte: + return string(v), true + } + return "", false +} + +// topicMatchesSQL is hopper_topic_matches(pattern, topic): whether an AMQP +// topic pattern matches a topic, NULL if either is. +func topicMatchesSQL(_ *sqlite.FunctionContext, args []sqldriver.Value) (sqldriver.Value, error) { + pattern, ok := textArg(args[0]) + topic, ok2 := textArg(args[1]) + if !ok || !ok2 { + return nil, nil + } + re, err := topicRegexp(pattern) + if err != nil { + return nil, err + } + return re.MatchString(topic), nil +} + +// topicPatterns caches compiled patterns, which subscriptions, consumers +// and stream reads repeat. It is emptied once it holds maxTopicPatterns, so +// that reads with ever new patterns cannot grow it without bound. +var topicPatterns = struct { + mu sync.Mutex + m map[string]*regexp.Regexp +}{m: map[string]*regexp.Regexp{}} + +const maxTopicPatterns = 1024 + +// topicSpecial are the characters Postgres's hopper_topic_regex escapes +// before it turns the wildcards into regular expressions. +var topicSpecial = regexp.MustCompile(`([.+?^${}()|\[\]\\])`) + +// topicRegexp builds the regular expression an AMQP pattern means, as +// Postgres's hopper_topic_regex does (schema v2): words are separated by +// dots, '*' matches one word and '#' zero or more. The steps are the same, +// in the same order, so that a pattern, however odd, matches the same +// topics on either engine; (?s) gives '.' Postgres's meaning, which takes a +// newline too. +func topicRegexp(pattern string) (*regexp.Regexp, error) { + topicPatterns.mu.Lock() + defer topicPatterns.mu.Unlock() + if re, ok := topicPatterns.m[pattern]; ok { + return re, nil + } + expr := topicSpecial.ReplaceAllString(pattern, `\$1`) + for _, r := range [][2]string{ + {`*`, `[^.]+`}, // one word (before '#', whose rules introduce '.*') + {`\.#\.`, `(\..*)?\.`}, // a.#.b: zero or more words between + {`\.#`, `(\..*)?`}, // a.#: the word and anything under it + {`#\.`, `(.*\.)?`}, // #.b: anything above the word + {`#`, `.*`}, // #: everything + } { + expr = strings.ReplaceAll(expr, r[0], r[1]) + } + re, err := regexp.Compile(`(?s)^` + expr + `$`) + if err != nil { + return nil, fmt.Errorf("hoppersqlite: topic pattern %q: %w", pattern, err) + } + if len(topicPatterns.m) >= maxTopicPatterns { + clear(topicPatterns.m) + } + topicPatterns.m[pattern] = re + return re, nil +} + +// jsonContainsSQL is hopper_json_contains(doc, filter): whether the JSON +// document doc contains filter as Postgres's jsonb @> has it, NULL if +// either is. A document that is not JSON contains nothing. +func jsonContainsSQL(_ *sqlite.FunctionContext, args []sqldriver.Value) (sqldriver.Value, error) { + doc, ok := textArg(args[0]) + filter, ok2 := textArg(args[1]) + if !ok || !ok2 { + return nil, nil + } + want, err := decodeJSON(filter) + if err != nil { + return nil, fmt.Errorf("hoppersqlite: metadata filter: %w", err) + } + if have, err := decodeJSON(doc); err == nil { + return jsonContains(have, want, true), nil + } + return false, nil +} + +// decodeJSON decodes a JSON value, keeping numbers as their text. +func decodeJSON(s string) (any, error) { + dec := json.NewDecoder(strings.NewReader(s)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, err + } + if dec.More() { + return nil, fmt.Errorf("text after the JSON value") + } + return v, nil +} + +// jsonContains reports whether have contains want as jsonb's @> has it: an +// object contains another whose every key it has with a value that +// contains the other's; an array contains another each of whose elements +// some element of it contains, whatever their order and repeats; scalars +// contain the equal scalar, numbers by value. At the top level, an array +// also contains a scalar it has as an element. +func jsonContains(have, want any, top bool) bool { + switch w := want.(type) { + case map[string]any: + h, ok := have.(map[string]any) + if !ok { + return false + } + for k, wv := range w { + hv, ok := h[k] + if !ok || !jsonContains(hv, wv, false) { + return false + } + } + return true + case []any: + h, ok := have.([]any) + if !ok { + return false + } + for _, wv := range w { + found := false + for _, hv := range h { + if jsonContains(hv, wv, false) { + found = true + break + } + } + if !found { + return false + } + } + return true + } + if h, ok := have.([]any); ok && top { + for _, hv := range h { + if jsonScalarEqual(hv, want) { + return true + } + } + return false + } + return jsonScalarEqual(have, want) +} + +func jsonScalarEqual(a, b any) bool { + switch a := a.(type) { + case json.Number: + b, ok := b.(json.Number) + if !ok { + return false + } + ca, err := canonicalNumber(string(a)) + cb, err2 := canonicalNumber(string(b)) + return err == nil && err2 == nil && ca == cb + case string: + b, ok := b.(string) + return ok && a == b + case bool: + b, ok := b.(bool) + return ok && a == b + case nil: + return b == nil + } + return false +} + +// decimal is a JSON number as digits and a power of ten: the number is +// digits × 10^exp, with no leading or trailing zero in digits, which is +// empty for zero. +type decimal struct { + negative bool + digits string + exp int64 +} + +func (d decimal) String() string { + if d.digits == "" { + return "0" + } + sign := "" + if d.negative { + sign = "-" + } + return sign + d.digits + "e" + strconv.FormatInt(d.exp, 10) +} + +// parseDecimal parses a JSON number. Its exponent may be as large as an +// int64 holds, beyond which it is refused. +func parseDecimal(s string) (decimal, error) { + var d decimal + if strings.HasPrefix(s, "-") { + d.negative, s = true, s[1:] + } + mantissa, exponent, hasExp := strings.Cut(strings.ToLower(s), "e") + if hasExp { + e, err := strconv.ParseInt(strings.TrimPrefix(exponent, "+"), 10, 64) + if err != nil { + return decimal{}, fmt.Errorf("number %s: exponent out of range", s) + } + d.exp = e + } + whole, frac, _ := strings.Cut(mantissa, ".") + digits := whole + frac + if d.exp-int64(len(frac)) > d.exp { + return decimal{}, fmt.Errorf("number %s: exponent out of range", s) + } + d.exp -= int64(len(frac)) + digits = strings.TrimLeft(digits, "0") + trimmed := strings.TrimRight(digits, "0") + d.exp += int64(len(digits) - len(trimmed)) + d.digits = trimmed + if d.digits == "" { + d.negative, d.exp = false, 0 + } + return d, nil +} + +// canonicalNumber is a number's text that equal numbers share. +func canonicalNumber(s string) (string, error) { + d, err := parseDecimal(s) + if err != nil { + return "", err + } + return d.String(), nil +} + +// Postgres's numeric, which jsonb keeps numbers in, holds up to 131072 +// digits before the decimal point and 16383 after it. +const ( + maxNumericWhole = 131072 + maxNumericFraction = 16383 +) + +// checkJSON refuses JSON text Postgres's jsonb input refuses, so that a +// value means the same on either engine and a job stored on one could be +// stored on the other: text that is not JSON or not UTF-8, a \u0000 escape +// (text cannot hold NUL), an unpaired surrogate escape, or a number beyond +// the range of Postgres's numeric. It reads the text once, outside its +// strings for numbers and inside them for escapes. +func checkJSON(b []byte) error { + if !utf8.Valid(b) { + return errors.New("is not UTF-8") + } + if !json.Valid(b) { + return errors.New("is not JSON") + } + inString := false + for i := 0; i < len(b); i++ { + c := b[i] + switch { + case inString && c == '"': + inString = false + case inString && c == '\\': + n, err := checkEscape(b[i:]) + if err != nil { + return err + } + i += n - 1 + case inString: + case c == '"': + inString = true + case c == '-' || c >= '0' && c <= '9': + end := i + for end < len(b) && strings.IndexByte("+-.eE0123456789", b[end]) >= 0 { + end++ + } + if err := checkNumber(string(b[i:end])); err != nil { + return err + } + i = end - 1 + } + } + return nil +} + +// checkEscape checks the escape at the start of b, inside a string of valid +// JSON, and returns its length. +func checkEscape(b []byte) (int, error) { + if b[1] != 'u' { + return 2, nil + } + r, _ := strconv.ParseUint(string(b[2:6]), 16, 16) + switch { + case r == 0: + return 0, errors.New("has a \\u0000 escape") + case !utf16.IsSurrogate(rune(r)): + return 6, nil + case r >= 0xdc00 || len(b) < 12 || !bytes.HasPrefix(b[6:], []byte(`\u`)): + return 0, errors.New("has an unpaired surrogate escape") + } + if low, err := strconv.ParseUint(string(b[8:12]), 16, 16); err != nil || low < 0xdc00 || low > 0xdfff { + return 0, errors.New("has an unpaired surrogate escape") + } + return 12, nil +} + +// checkNumber refuses a number beyond the range of Postgres's numeric. +func checkNumber(n string) error { + d, err := parseDecimal(n) + if err != nil { + return err + } + if whole := int64(len(d.digits)) + d.exp; d.digits != "" && whole > maxNumericWhole { + return fmt.Errorf("has a number, %.20s..., beyond the range of Postgres's numeric", n) + } + if d.exp < -maxNumericFraction { + return fmt.Errorf("has a number, %.20s..., with more decimals than Postgres's numeric holds", n) + } + return nil +} + +// checkJSONFilter refuses a metadata filter Postgres's jsonb input would +// (checkJSON), so that a listing's filter means the same on either engine. +func checkJSONFilter(s string) error { + if err := checkJSON([]byte(s)); err != nil { + return fmt.Errorf("the metadata filter %w", err) + } + return nil +} diff --git a/driver/hoppersqlite/funcs_test.go b/driver/hoppersqlite/funcs_test.go new file mode 100644 index 0000000..3253b67 --- /dev/null +++ b/driver/hoppersqlite/funcs_test.go @@ -0,0 +1,145 @@ +package hoppersqlite + +import ( + "encoding/binary" + "strings" + "testing" + "time" +) + +// TestTopicPatterns checks AMQP patterns against topics, with the results +// Postgres's hopper_topic_regex gives, odd patterns included. +func TestTopicPatterns(t *testing.T) { + for _, c := range []struct { + pattern, topic string + want bool + }{ + {"#", "", true}, + {"#", "a.b.c", true}, + {"a.*", "a.b", true}, + {"a.*", "a", false}, + {"a.*", "a.b.c", false}, + {"a.*", "a.", false}, + {"a.#", "a", true}, + {"a.#", "a.b.c", true}, + {"a.#", "ab", false}, + {"#.b", "b", true}, + {"#.b", "a.x.b", true}, + {"#.b", "ab", false}, + {"a.#.b", "a.b", true}, + {"a.#.b", "a.x.y.b", true}, + {"a.#.b", "a.xb", false}, + {"*.created", "order.created", true}, + {"*.created", "order.paid", false}, + {"a#b", "axyzb", true}, // '#' between letters is '.*', as on Postgres + {"a+b", "a+b", true}, + {"a+b", "aab", false}, + {"(x)", "(x)", true}, + {"a.b", "a.b", true}, + {"a.b", "axb", false}, + {"#", "line\nbreak", true}, + } { + re, err := topicRegexp(c.pattern) + if err != nil { + t.Fatalf("%q: %v", c.pattern, err) + } + if got := re.MatchString(c.topic); got != c.want { + t.Errorf("%q matches %q = %v, want %v", c.pattern, c.topic, got, c.want) + } + } +} + +// TestJSONContains checks containment as jsonb's @> has it. +func TestJSONContains(t *testing.T) { + for _, c := range []struct { + have, want string + contains bool + }{ + {`{"a": 1, "b": 2}`, `{"a": 1}`, true}, + {`{"a": 1}`, `{"a": 1, "b": 2}`, false}, + {`{"a": 1.0}`, `{"a": 1}`, true}, + {`{"a": 100}`, `{"a": 1e2}`, true}, + {`{"a": 0.10}`, `{"a": 1e-1}`, true}, + {`{"a": -0}`, `{"a": 0}`, true}, + {`{"a": "1"}`, `{"a": 1}`, false}, + {`{"a": {"b": [1, 2, 3]}}`, `{"a": {"b": [3, 1]}}`, true}, + {`{"a": [1, 2]}`, `{"a": [1, 1, 1]}`, true}, + {`{"a": [1, 2]}`, `{"a": [4]}`, false}, + {`{"a": [[1, 3], 2]}`, `{"a": [[3]]}`, true}, + {`{"a": [[1, 3], 2]}`, `{"a": [3]}`, false}, + {`{"a": null}`, `{"a": null}`, true}, + {`{"a": false}`, `{"a": null}`, false}, + {`{}`, `{}`, true}, + {`["x", "y"]`, `"x"`, true}, + {`{"a": ["x"]}`, `{"a": "x"}`, false}, + {`[1]`, `{}`, false}, + } { + have, err := decodeJSON(c.have) + if err != nil { + t.Fatal(err) + } + want, err := decodeJSON(c.want) + if err != nil { + t.Fatal(err) + } + if got := jsonContains(have, want, true); got != c.contains { + t.Errorf("%s @> %s = %v, want %v", c.have, c.want, got, c.contains) + } + } +} + +// TestJSONFilterChecks checks that a metadata filter is refused where +// Postgres's jsonb input refuses it, and only there. +func TestJSONFilterChecks(t *testing.T) { + for filter, refused := range map[string]bool{ + `{"a": 1}`: false, + `{"a": "é😀"}`: false, + `{"a": 1e131071}`: false, + `{"a": 1e131072}`: true, + `{"a": 1e1000000}`: true, + `{"a": 1e-16383}`: false, + `{"a": 1e-16384}`: true, + `{"a": 12.5e-16382}`: false, + `{"a": 12.5e-16383}`: true, + `{"a": "\u0000"}`: true, + `{"a": "x\\u0000"}`: false, + `{"a": "\ud800x"}`: true, + `{"a": "\udc00\ud800"}`: true, + "{\"a\": \"\xff\"}": true, + `{"a": 1`: true, + `{"a": 99999999999999999999e-20}`: false, + } { + err := checkJSONFilter(filter) + if (err != nil) != refused { + t.Errorf("checkJSONFilter(%s) = %v, want refused %v", filter, err, refused) + } + if err != nil && strings.Contains(err.Error(), "Metadata") { + t.Errorf("checkJSONFilter(%s) names the field: %v", filter, err) + } + } +} + +// TestNewID checks that IDs are UUIDv7s of the time they were minted, in +// order across milliseconds. +func TestNewID(t *testing.T) { + now := time.Date(2026, 10, 22, 10, 0, 0, 500_000, time.UTC) + id := newID(now) + if v := id[6] >> 4; v != 7 { + t.Errorf("version %d", v) + } + if v := id[8] >> 6; v != 2 { + t.Errorf("variant %b", v) + } + if ms := int64(binary.BigEndian.Uint64(id[0:8]) >> 16); ms != now.UnixMilli() { + t.Errorf("time %d, want %d", ms, now.UnixMilli()) + } + if fraction := binary.BigEndian.Uint16(id[6:8]) & 0x0fff; fraction != 2048 { + t.Errorf("fraction %d, want 2048 (half a millisecond)", fraction) + } + if later := newID(now.Add(time.Millisecond)); string(later[:]) <= string(id[:]) { + t.Errorf("%s does not sort after %s", later, id) + } + if again := newID(now); again == id { + t.Error("two IDs minted at one time are equal") + } +} diff --git a/driver/hoppersqlite/go.mod b/driver/hoppersqlite/go.mod new file mode 100644 index 0000000..8a0d0e3 --- /dev/null +++ b/driver/hoppersqlite/go.mod @@ -0,0 +1,28 @@ +module github.com/parallelworks/hopper/driver/hoppersqlite + +go 1.27.0 + +toolchain go1.27.3 + +// hopper does no cryptography of its own, and neither does SQLite. Running +// tests in fips140=only mode proves that nothing non-approved sneaks in. +godebug fips140=only + +require ( + github.com/parallelworks/hopper v0.7.0 // x-release-please-version + golang.org/x/sys v0.48.0 + modernc.org/sqlite v1.60.1 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + modernc.org/libc v1.77.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect +) + +replace github.com/parallelworks/hopper => ../../ diff --git a/driver/hoppersqlite/go.sum b/driver/hoppersqlite/go.sum new file mode 100644 index 0000000..9e6b7d9 --- /dev/null +++ b/driver/hoppersqlite/go.sum @@ -0,0 +1,60 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= +golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= +modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg= +modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE= +modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs= +modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw= +modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/driver/hoppersqlite/hooks_test.go b/driver/hoppersqlite/hooks_test.go new file mode 100644 index 0000000..909ca51 --- /dev/null +++ b/driver/hoppersqlite/hooks_test.go @@ -0,0 +1,176 @@ +package hoppersqlite_test + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "strconv" + "strings" + "testing" + + _ "modernc.org/sqlite" // the "sqlite" driver, unwrapped, to compare with + + "github.com/parallelworks/hopper/driver/hoppersqlite" +) + +// hookCase is a statement run in a transaction after an insert, on a +// table made by table; full limits the database's size first, so that the +// statement fills it. +type hookCase struct { + name, table, stmt string + full bool +} + +// rawEndsTransaction runs c on modernc's own driver and reports whether +// SQLite still had the transaction open after the statement. +func rawEndsTransaction(t *testing.T, c hookCase) bool { + t.Helper() + ctx := context.Background() + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "raw.db")) + if err != nil { + t.Fatal(err) + } + defer db.Close() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + exec := func(q string) { + t.Helper() + if _, err := conn.ExecContext(ctx, q); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + exec(c.table) + var pages int + if err := conn.QueryRowContext(ctx, "PRAGMA page_count").Scan(&pages); err != nil { + t.Fatal(err) + } + exec("BEGIN IMMEDIATE") + exec("INSERT INTO t VALUES (1)") + if c.full { + exec("PRAGMA max_page_count = " + strconv.Itoa(pages+3)) + } + _, _ = conn.ExecContext(ctx, c.stmt) + _, err = conn.ExecContext(ctx, "ROLLBACK") + if err != nil && !strings.Contains(err.Error(), "no transaction is active") { + t.Fatalf("ROLLBACK: %v", err) + } + return err != nil +} + +// TestHooksMatchSQLite runs statements that end a transaction, or leave it +// open, in one of the driver's transactions and on modernc's own +// connection: the driver refuses the transaction's next statement exactly +// when SQLite ended the transaction, with ErrTransactionAborted after an +// error and sql.ErrTxDone after a ROLLBACK, COMMIT or END, and nothing the +// refused statement would have written is left. +func TestHooksMatchSQLite(t *testing.T) { + t.Parallel() + ctx := context.Background() + big := "INSERT INTO t VALUES (randomblob(200000))" + rows := "INSERT INTO t SELECT randomblob(3000) FROM (WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM c WHERE x < 200) SELECT x FROM c)" + trigger := func(action string) string { + return "CREATE TABLE t (x); CREATE TRIGGER tr BEFORE INSERT ON t WHEN NEW.x = 9 BEGIN SELECT RAISE(" + action + ", 'no'); END" + } + for _, c := range []hookCase{ + {"a full disk, one row", "CREATE TABLE t (x)", big, true}, + {"a full disk, many rows", "CREATE TABLE t (x)", rows, true}, + {"a full disk, many rows, a CHECK", "CREATE TABLE t (x CHECK (x IS NOT NULL))", rows, true}, + {"a full disk, an UPDATE", "CREATE TABLE t (x)", "UPDATE t SET x = randomblob(300000)", true}, + {"INSERT OR ROLLBACK", "CREATE TABLE t (x UNIQUE)", "INSERT OR ROLLBACK INTO t VALUES (1)", false}, + {"RAISE(ROLLBACK)", trigger("ROLLBACK"), "INSERT INTO t VALUES (9)", false}, + {"RAISE(ABORT)", trigger("ABORT"), "INSERT INTO t VALUES (9)", false}, + {"RAISE(FAIL)", trigger("FAIL"), "INSERT INTO t VALUES (9)", false}, + {"a unique constraint", "CREATE TABLE t (x UNIQUE)", "INSERT INTO t VALUES (1)", false}, + {"a savepoint rolled back to and released", "CREATE TABLE t (x)", "SAVEPOINT a; INSERT INTO t VALUES (5); ROLLBACK TO a; RELEASE a", false}, + {"ROLLBACK TO no savepoint", "CREATE TABLE t (x)", "ROLLBACK TO nosuch", false}, + {"ROLLBACK", "CREATE TABLE t (x)", "ROLLBACK", false}, + {"COMMIT", "CREATE TABLE t (x)", "COMMIT", false}, + {"END", "CREATE TABLE t (x)", "END", false}, + } { + ended := rawEndsTransaction(t, c) + d := open(t) + db := d.DB() + if _, err := db.ExecContext(ctx, c.table+"; CREATE TABLE side (x)"); err != nil { + t.Fatal(err) + } + var pages int + if err := db.QueryRowContext(ctx, "PRAGMA page_count").Scan(&pages); err != nil { + t.Fatal(err) + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err != nil { + t.Fatal(err) + } + if c.full { + if _, err := tx.ExecContext(ctx, "PRAGMA max_page_count = "+strconv.Itoa(pages+3)); err != nil { + t.Fatal(err) + } + } + stmtErr := func() error { _, err := tx.ExecContext(ctx, c.stmt); return err }() + _, err = tx.ExecContext(ctx, "INSERT INTO side VALUES (1)") + aborted, done := errors.Is(err, hoppersqlite.ErrTransactionAborted), errors.Is(err, sql.ErrTxDone) + switch { + case ended && stmtErr != nil && !aborted: + t.Errorf("%s: SQLite rolled the transaction back after %v; the next statement = %v, want ErrTransactionAborted", c.name, stmtErr, err) + case ended && stmtErr == nil && !done: + t.Errorf("%s: the statement ended the transaction; the next statement = %v, want sql.ErrTxDone", c.name, err) + case !ended && err != nil: + t.Errorf("%s: SQLite kept the transaction open; the next statement = %v", c.name, err) + } + if c.full { + _, _ = tx.ExecContext(ctx, "PRAGMA max_page_count = 1073741823") + } + _ = tx.Rollback() + var side int + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM side").Scan(&side); err != nil || side != 0 { + t.Errorf("%s: %d rows of side left after Rollback, %v", c.name, side, err) + } + } +} + +// TestStatementEndsTransaction ends a transaction with a statement of its +// own: after COMMIT, what it wrote is committed and Commit returns nil; +// after ROLLBACK, nothing is and Commit returns sql.ErrTxDone. Either way +// the write lock is free once the transaction ends. +func TestStatementEndsTransaction(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + db := d.DB() + if _, err := db.ExecContext(ctx, "CREATE TABLE t (x)"); err != nil { + t.Fatal(err) + } + for _, c := range []struct { + stmt string + rows int + err error + }{{"COMMIT", 1, nil}, {"ROLLBACK", 0, sql.ErrTxDone}} { + if _, err := db.ExecContext(ctx, "DELETE FROM t"); err != nil { + t.Fatal(err) + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, c.stmt); err != nil { + t.Fatalf("%s: %v", c.stmt, err) + } + if err := tx.Commit(); !errors.Is(err, c.err) || (c.err == nil && err != nil) { + t.Errorf("Commit after %s = %v, want %v", c.stmt, err, c.err) + } + var n int + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM t").Scan(&n); err != nil || n != c.rows { + t.Errorf("after %s: %d rows, %v; want %d", c.stmt, n, err, c.rows) + } + } +} diff --git a/driver/hoppersqlite/hoppersqlite.go b/driver/hoppersqlite/hoppersqlite.go new file mode 100644 index 0000000..096910e --- /dev/null +++ b/driver/hoppersqlite/hoppersqlite.go @@ -0,0 +1,404 @@ +// Package hoppersqlite is hopper's SQLite driver, for an application that +// runs on one machine and wants hopper without a database server: a +// command-line tool that runs a workflow, a single-node service, local +// development. It is pure Go (modernc.org/sqlite, SQLite transpiled to Go), +// so a program that uses it builds with CGO_ENABLED=0. +// +// d, err := hoppersqlite.Open("/var/lib/app/hopper.db", nil) +// if _, err := hoppermigrate.Up(ctx, d, nil); err != nil { ... } +// client, err := hopper.NewClient(d, cfg) +// +// It passes the drivertest conformance suite, as the Postgres drivers do, +// and several processes may share one database. It uses internal packages +// of hopper's module, so it works with the hopper release of its own +// version: upgrade the two together. What SQLite's design makes different +// from Postgres: +// +// - One transaction writes at a time (driver.Capabilities.SingleWriter). +// A transaction that writes holds the database's write lock until it +// ends, and every other write waits for it: within this process in +// turn, for as long as its context allows, and across processes +// through SQLite's busy handler, which polls rather than queues, so a +// process that writes without pause holds another's writes up for +// longer than its own. A claim is exclusive without SKIP LOCKED, and +// the transaction that finalizes a job settles its batch and promotes +// its dependents itself. A finalize with SkipLocked, or a progress +// write with NoWait, waits 100ms for a write lock held elsewhere, and +// then skips every job, or returns driver.ErrJobLocked. +// - So a transaction held open holds up every client that shares the +// database, their lease renewals included: one held for longer than +// about the lease (hopper.Config.LeaseTTL) costs them their leases, +// and their running jobs are rescued. Keep transactions short. +// - Transactions come from DB, whose connections take the write lock +// when a transaction runs its first statement, not when it begins, so +// that what it reads is never stale by the time it writes, and so that +// beginning one never blocks. A transaction begun read-only takes no +// lock and may not write. Within a transaction, write through it +// (InsertTx and the like), not through the client: a write on the pool +// would wait for the transaction's own lock. In a job's worker, whose +// context names the job's attempt, such a write fails at once with +// ErrLockHeldByOwner instead, if the worker began the transaction with +// that context or one derived from it; begin it so. Writes of the +// attempt with no transaction of its own open wait their turn, from +// however many goroutines. +// - After some errors (an interrupted write, a full disk, an I/O error) +// SQLite rolls the whole transaction back itself. The transaction's +// later statements, and its Commit, then fail with +// ErrTransactionAborted, as those of a transaction an error aborted do +// on Postgres. An error that SQLite undoes only the statement of, such +// as a constraint's, leaves the transaction open, where Postgres would +// abort it. A COMMIT, END or ROLLBACK statement run in a transaction +// ends it: its later statements fail with sql.ErrTxDone. In a read-only +// transaction SQLite tells nothing of a COMMIT, and the statements +// after it, reads all, run on their own. +// - The database must be on a local filesystem: Open refuses one on a +// network filesystem, or a virtual machine's shared folder, with +// ErrNetworkFilesystem, checking where symbolic links in the path +// lead. Its clock is therefore the host's, which every process that +// opens it shares, and the driver reads it as database time, never +// earlier than a time it read before in the same process (a host +// clock stepped back can still date one process's write before +// another's earlier one). A host that +// sleeps and wakes finds its clock moved on, as a database server's +// would, and may rescue the jobs of its own clients, whose leases +// expired while it slept. +// - Commits are durable once they return (Config.Synchronous), on macOS +// with F_FULLFSYNC. The write-ahead log is cut back to 32MiB once a +// checkpoint empties it, and JobsMaintain truncates it once a reader +// that held up checkpoints is done. +// - Notifications are rows of a table, delivered on commit to listeners +// in every process that opens the database: at once in this process, +// and within ListenInterval in others. +// - History is one table, and retention deletes its rows in batches, +// without returning the space to the filesystem; JobsMaintain +// refreshes statistics but never vacuums. A stream's events commit in +// the order they are appended, so a position is an event's sequence +// number (Xid is always 0). +// - Args, metadata and other JSON values are stored as given, not +// normalized as Postgres's jsonb normalizes them, and refused for what +// jsonb refuses. A metadata filter matches as jsonb's @> does. +// - Job IDs are minted in Go, from crypto/rand, rather than by the +// database. There is no COPY: bulk inserts go through the one insert +// path. Nor are there SQL functions to insert jobs from SQL. +package hoppersqlite + +import ( + "context" + "database/sql" + "embed" + "errors" + "fmt" + "io/fs" + "net/url" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/hoppermigrate" +) + +// Config tunes the SQLite driver. A nil *Config means the defaults. +type Config struct { + // Synchronous is SQLite's synchronous setting for the database's + // connections: "FULL" (the default), which makes every commit durable + // before it returns, as Postgres's default does (on macOS with + // F_FULLFSYNC, without which the disk may hold the commit in its + // cache), or "NORMAL", with which a power failure, not a crash of the + // process, can lose the last commits before a checkpoint, and commits + // take less time. + Synchronous string + // ListenInterval is how often a listener looks for notifications + // committed by other processes. Those of this process wake it at once. + // Zero means 50ms. + ListenInterval time.Duration +} + +// DefaultListenInterval is ListenInterval's default. +const DefaultListenInterval = 50 * time.Millisecond + +// lockWaitBrief is how long a finalize with SkipLocked, a progress write +// with NoWait, and the leader's maintenance wait for a write lock another +// transaction holds before they give up: long enough for a statement of +// another client to end, short enough not to hold the caller up behind a +// transaction that stays open. +const lockWaitBrief = 100 * time.Millisecond + +// Driver implements driver.Driver[*sql.Tx] on a SQLite database. +type Driver struct { + sqlDB *sql.DB + db *database + listenInterval time.Duration + statements statements +} + +var _ driver.Driver[*sql.Tx] = (*Driver)(nil) + +// DefaultOpenTimeout is how long Open waits for the database: for another +// process that holds it while it creates it or recovers its log, say. +const DefaultOpenTimeout = 30 * time.Second + +// Open opens the SQLite database at path as OpenContext does, waiting at +// most DefaultOpenTimeout. +func Open(path string, cfg *Config) (*Driver, error) { + ctx, cancel := context.WithTimeout(context.Background(), DefaultOpenTimeout) + defer cancel() + return OpenContext(ctx, path, cfg) +} + +// OpenContext opens the SQLite database at path, creating it if needed, in +// write-ahead log mode, waiting for as long as ctx allows for another +// process that holds it. A symbolic link in the path is resolved, as +// SQLite resolves it, and the database opened where it leads. OpenContext +// returns ErrNetworkFilesystem for a database on a network filesystem. +// Close the Driver to close the database. +func OpenContext(ctx context.Context, path string, cfg *Config) (*Driver, error) { + var c Config + if cfg != nil { + c = *cfg + } + sync := strings.ToUpper(c.Synchronous) + switch sync { + case "": + sync = "FULL" + case "FULL", "NORMAL": + default: + return nil, fmt.Errorf("hoppersqlite: Synchronous is %q, not FULL or NORMAL", c.Synchronous) + } + if c.ListenInterval < 0 { + return nil, errors.New("hoppersqlite: ListenInterval is negative") + } + if c.ListenInterval == 0 { + c.ListenInterval = DefaultListenInterval + } + if path == "" || path == ":memory:" || strings.HasPrefix(path, "file:") { + return nil, fmt.Errorf("hoppersqlite: %q is not a file path; the database must be a file, which every connection and process opens", path) + } + resolved, err := resolve(path) + if err != nil { + return nil, fmt.Errorf("hoppersqlite: %w", err) + } + if err := checkLocal(resolved); err != nil { + return nil, err + } + db := openDatabase(resolved) + sqlDB := sql.OpenDB(&connector{db: db, dsn: dsn(resolved), synchronous: sync}) + sqlDB.SetMaxIdleConns(16) + d := &Driver{sqlDB: sqlDB, db: db, listenInterval: c.ListenInterval} + if err := d.setUp(ctx, resolved); err != nil { + _ = d.Close() + return nil, fmt.Errorf("hoppersqlite: open %s: %w", resolved, err) + } + return d, nil +} + +// setUp puts a database just opened in write-ahead log mode, and checks +// that the file SQLite opened is where Open checked: on a local +// filesystem. +func (d *Driver) setUp(ctx context.Context, resolved string) error { + var mode string + if err := d.sqlDB.QueryRowContext(ctx, "PRAGMA journal_mode = WAL").Scan(&mode); err != nil { + return err + } + if !strings.EqualFold(mode, "wal") { + return fmt.Errorf("the database is in journal mode %s and cannot use a write-ahead log", mode) + } + var file string + if err := d.sqlDB.QueryRowContext(ctx, "SELECT file FROM pragma_database_list WHERE name = 'main'").Scan(&file); err != nil { + return err + } + if file != resolved { + return checkLocal(file) + } + return nil +} + +// dsn is the URI of the database file at an absolute path, for modernc's +// driver, which takes its query parameters out of a plain file name. +func dsn(abs string) string { + p := filepath.ToSlash(abs) + if !strings.HasPrefix(p, "/") { + p = "/" + p // a Windows drive letter + } + return (&url.URL{Scheme: "file", OmitHost: true, Path: p}).String() +} + +// DB returns the database/sql handle the driver uses. Begin the +// transactions you pass to InsertTx and the like on it: its connections +// take the database's write lock as a transaction's first statement runs. +// Leave its limit on open connections alone: each listener, and the +// migrator, holds a connection of its own for as long as it runs, and a +// limit that left a transaction without one would wait for them. +func (d *Driver) DB() *sql.DB { return d.sqlDB } + +// Close closes the database. +func (d *Driver) Close() error { + err := errors.Join(d.statements.close(), d.sqlDB.Close()) + d.db.close() + return err +} + +// Executor implements driver.Driver. +func (d *Driver) Executor() driver.Executor { return &executor{d: d} } + +// UnwrapTx implements driver.Driver. The transaction must come from DB. +func (d *Driver) UnwrapTx(tx *sql.Tx) driver.Executor { return &executor{d: d, tx: tx} } + +// Capabilities implements driver.Driver. +func (d *Driver) Capabilities() driver.Capabilities { + return driver.Capabilities{Listen: true, SingleWriter: true} +} + +// Migrator implements driver.Driver. +func (d *Driver) Migrator() driver.Migrator { return migrator{d} } + +// Listener implements driver.Driver. +func (d *Driver) Listener(ctx context.Context) (driver.Listener, error) { return d.newListener(ctx) } + +//go:embed migrations/*.sql +var migrationFiles embed.FS + +var migrationName = regexp.MustCompile(`^(\d+)_[a-z0-9_]+\.(up|down)\.sql$`) + +// scripts are SQLite's migration scripts by version and direction. +var scripts = func() map[int]map[bool]string { + out := map[int]map[bool]string{} + entries, err := fs.ReadDir(migrationFiles, "migrations") + if err != nil { + panic(err) + } + for _, e := range entries { + m := migrationName.FindStringSubmatch(e.Name()) + if m == nil { + panic("hoppersqlite: malformed migration file name " + e.Name()) + } + version, _ := strconv.Atoi(m[1]) + body, err := fs.ReadFile(migrationFiles, "migrations/"+e.Name()) + if err != nil { + panic(err) + } + if out[version] == nil { + out[version] = map[bool]string{} + } + out[version][m[2] == "up"] = string(body) + } + return out +}() + +// Migrations returns SQLite's migration scripts, one for every schema +// version hoppermigrate knows, in ascending order, for an application that +// applies its schema with a tool of its own rather than hoppermigrate.Up, +// whose Migrations returns Postgres's. Record each version applied in +// hopper_schema, as Up does. +func Migrations() []hoppermigrate.Migration { + ms := hoppermigrate.Migrations() + for i := range ms { + ms[i].Up, ms[i].Down = scripts[ms[i].Version][true], scripts[ms[i].Version][false] + } + return ms +} + +// migrator implements driver.Migrator. Its lock is SQLite's write lock, +// held from Lock to Close on a connection of its own: each version applies +// in a savepoint of that transaction, and Close commits the versions that +// applied. +type migrator struct{ d *Driver } + +func (m migrator) Lock(ctx context.Context) (driver.MigrationExecutor, error) { + conn, err := m.d.sqlDB.Conn(ctx) + if err != nil { + return nil, err + } + tx, err := conn.BeginTx(ctx, nil) + if err == nil { + _, err = tx.ExecContext(ctx, "SELECT 1") // takes the write lock + } + if err != nil { + if tx != nil { + _ = tx.Rollback() + } + _ = conn.Close() + return nil, err + } + return &migration{conn: conn, tx: tx}, nil +} + +// migration implements driver.MigrationExecutor and driver.MigrationScripts. +type migration struct { + conn *sql.Conn + tx *sql.Tx +} + +var _ driver.MigrationScripts = (*migration)(nil) + +func (m *migration) Versions(ctx context.Context) ([]int, error) { return schemaVersions(ctx, m.tx) } + +// Script implements driver.MigrationScripts. +func (m *migration) Script(version int, up bool) (string, error) { + script, ok := scripts[version][up] + if !ok { + return "", fmt.Errorf("hoppersqlite has no script for schema version %d", version) + } + return script, nil +} + +func (m *migration) Apply(ctx context.Context, version int, script string, up bool) (err error) { + if _, err := m.tx.ExecContext(ctx, "SAVEPOINT migration"); err != nil { + return err + } + defer func() { + if err != nil { + _, rerr := m.tx.ExecContext(context.WithoutCancel(ctx), "ROLLBACK TO migration; RELEASE migration") + err = errors.Join(err, rerr) + } + }() + if !up { + // The script of version 1 drops hopper_schema with the rest. + if _, err := m.tx.ExecContext(ctx, "DELETE FROM hopper_schema WHERE version = ?", version); err != nil { + return err + } + } + if _, err := m.tx.ExecContext(ctx, script); err != nil { + return err + } + if up { + if _, err := m.tx.ExecContext(ctx, "INSERT INTO hopper_schema (version, applied_at) VALUES (?, ?)", version, time.Now().UnixMicro()); err != nil { + return err + } + } + _, err = m.tx.ExecContext(ctx, "RELEASE migration") + return err +} + +func (m *migration) Close(context.Context) error { + err := m.tx.Commit() + return errors.Join(err, m.conn.Close()) +} + +// schemaVersions returns the applied schema versions in ascending order, or +// nil if the schema has never been installed. +func schemaVersions(ctx context.Context, q querier) ([]int, error) { + var exists bool + if err := q.QueryRowContext(ctx, "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'hopper_schema')").Scan(&exists); err != nil { + return nil, err + } + if !exists { + return nil, nil + } + rows, err := q.QueryContext(ctx, "SELECT version FROM hopper_schema ORDER BY version") + if err != nil { + return nil, err + } + defer rows.Close() + var versions []int + for rows.Next() { + var v int + if err := rows.Scan(&v); err != nil { + return nil, err + } + versions = append(versions, v) + } + return versions, rows.Err() +} diff --git a/driver/hoppersqlite/id.go b/driver/hoppersqlite/id.go new file mode 100644 index 0000000..7261ce8 --- /dev/null +++ b/driver/hoppersqlite/id.go @@ -0,0 +1,26 @@ +package hoppersqlite + +import ( + "crypto/rand" + "encoding/binary" + "time" + + "github.com/parallelworks/hopper/driver" +) + +// newID returns a UUIDv7 for a job, batch or message: 48 bits of Unix +// milliseconds at now, 12 bits of the millisecond's fraction, as Postgres +// 18's uuidv7() fills them, so that IDs minted in one process in one +// millisecond mostly sort in order, and 62 random bits. SQLite has no +// generator of its own, so the driver mints IDs where Postgres's does in the +// database. crypto/rand reads the operating system's generator through Go's +// FIPS 140-3 DRBG, as fips140=only requires. +func newID(now time.Time) driver.JobID { + var id driver.JobID + ms := uint64(max(now.UnixMilli(), 0)) + fraction := uint64(now.Nanosecond()%int(time.Millisecond)) * 4096 / uint64(time.Millisecond) //nolint:gosec // Nanosecond is in [0, 1e9) + binary.BigEndian.PutUint64(id[0:8], ms<<16|0x7000|fraction) + _, _ = rand.Read(id[8:]) // never fails (crypto/rand.Read) + id[8] = id[8]&0x3f | 0x80 + return id +} diff --git a/driver/hoppersqlite/jobs.go b/driver/hoppersqlite/jobs.go new file mode 100644 index 0000000..fc43bd0 --- /dev/null +++ b/driver/hoppersqlite/jobs.go @@ -0,0 +1,856 @@ +package hoppersqlite + +import ( + "bytes" + "cmp" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "math" + "slices" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/internal/jobcodec" + "github.com/parallelworks/hopper/internal/jobtags" +) + +func bytesReader(s string) *strings.Reader { return strings.NewReader(s) } + +// jobInsert is a job to insert, its values encoded. +type jobInsert struct { + kind, queue string + priority, maxAttempts int + scheduledAt time.Time + args, metadata string + tags []string + uniqueKey string + uniqueFinished []driver.JobState + orderingKey, partitionKey string + batchID driver.JobID + ttl time.Duration + await bool + // pending starts the job waiting on workflow dependencies, and id + // gives it its ID; a zero id has one minted. + pending bool + id driver.JobID +} + +func encodeInsert(p driver.JobInsertParams) (jobInsert, error) { + priority, err := smallint(p.Priority, "priority") + if err != nil { + return jobInsert{}, err + } + maxAttempts, err := smallint(p.MaxAttempts, "max_attempts") + if err != nil { + return jobInsert{}, err + } + args, err := jsonValue(p.Args, "{}", "args") + if err != nil { + return jobInsert{}, err + } + metadata, err := jsonValue(p.Metadata, "{}", "metadata") + if err != nil { + return jobInsert{}, err + } + return jobInsert{ + kind: p.Kind, queue: p.Queue, priority: priority, maxAttempts: maxAttempts, scheduledAt: p.ScheduledAt, + args: args, metadata: metadata, tags: p.Tags, uniqueKey: p.UniqueKey, uniqueFinished: p.UniqueFinished, + orderingKey: p.OrderingKey, partitionKey: p.PartitionKey, batchID: p.BatchID, ttl: p.TTL, await: p.Await, + }, nil +} + +// insertJobSQL inserts a job and returns it as stored. +const insertJobSQL = ` +INSERT INTO hopper_jobs (id, kind, queue, state, priority, max_attempts, scheduled_at, args, metadata, unique_key, + expires_at, await, ordering_key, partition_key, batch_id, tags, created_at) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +RETURNING ` + jobColumns + +// insertJob inserts a job: pending, or scheduled if its time is ahead, or +// available. +func (x *w) insertJob(ctx context.Context, j jobInsert) (*driver.JobRow, error) { + scheduledAt := j.scheduledAt + if scheduledAt.IsZero() { + scheduledAt = x.now + } + state := driver.JobStateAvailable + switch { + case j.pending: + state = driver.JobStatePending + case scheduledAt.After(x.now): + state = driver.JobStateScheduled + } + var expires any + if j.ttl > 0 { + expires = x.at(j.ttl) + } + id := j.id + if id.IsZero() { + id = newID(x.now) + } + return scanJob(x.QueryRowContext(ctx, insertJobSQL, + idParam(id), j.kind, j.queue, string(state), j.priority, j.maxAttempts, scheduledAt.UnixMicro(), j.args, j.metadata, + nullable(j.uniqueKey), expires, j.await, nullable(j.orderingKey), nullable(j.partitionKey), nullableID(j.batchID), + tagsJSON(j.tags), x.us()), false) +} + +// JobInsertMany implements driver.Executor. An input with a unique key is +// first looked for among live jobs, which with ConflictReplace it replaces +// unless running, and then, with UniqueFinished, in history: a job found +// in either is the input's duplicate, and nothing is inserted for it. The +// write lock keeps both looks and the insert free of any other writer. +func (e *executor) JobInsertMany(ctx context.Context, params []driver.JobInsertParams, opts driver.JobInsertOpts) ([]driver.JobInsertResult, error) { + if len(params) == 0 { + return nil, nil + } + jobs := make([]jobInsert, len(params)) + for i, p := range params { + var err error + if jobs[i], err = encodeInsert(p); err != nil { + return nil, err + } + } + var results []driver.JobInsertResult + err := e.write(ctx, func(x *w) error { + var err error + if results, err = x.insertJobs(ctx, jobs, opts.OnConflict); err != nil { + return err + } + if opts.Notify { + return x.notify(ctx, driver.ChannelInsert, distinctQueues(jobs)...) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("hopper: insert jobs: %w", err) + } + return results, nil +} + +func distinctQueues(jobs []jobInsert) []string { + var out []string + for _, j := range jobs { + if !slices.Contains(out, j.queue) { + out = append(out, j.queue) + } + } + return out +} + +// replaceSQL replaces a waiting job's args, metadata, tags, priority, max +// attempts and schedule, as a replacing insert does, and clears its +// progress and snooze reason. +const replaceSQL = ` +UPDATE hopper_jobs SET args = ?, metadata = ?, tags = ?, priority = ?, max_attempts = ?, scheduled_at = ?, state = ?, + progress = NULL, snooze_reason = NULL +WHERE id = ? +RETURNING ` + jobColumns + +func (x *w) insertJobs(ctx context.Context, jobs []jobInsert, onConflict driver.ConflictAction) ([]driver.JobInsertResult, error) { + results := make([]driver.JobInsertResult, len(jobs)) + for i, j := range jobs { + if j.uniqueKey != "" { + dup, err := scanJob(x.QueryRowContext(ctx, "SELECT "+jobColumns+" FROM hopper_jobs WHERE kind = ? AND unique_key = ?", j.kind, j.uniqueKey), false) + switch { + case errors.Is(err, sql.ErrNoRows): + case err != nil: + return nil, err + default: + if onConflict == driver.ConflictReplace && dup.State != driver.JobStateRunning { + scheduledAt := cmp.Or(j.scheduledAt, x.now) + state := driver.JobStateAvailable + if scheduledAt.After(x.now) { + state = driver.JobStateScheduled + } + if dup, err = scanJob(x.QueryRowContext(ctx, replaceSQL, j.args, j.metadata, tagsJSON(j.tags), j.priority, j.maxAttempts, + scheduledAt.UnixMicro(), string(state), idParam(dup.ID)), false); err != nil { + return nil, err + } + } + results[i] = driver.JobInsertResult{Job: dup, Duplicate: true} + continue + } + if len(j.uniqueFinished) > 0 { + found, err := x.finishedDuplicate(ctx, j) + if err != nil { + return nil, err + } + if found != nil { + results[i] = driver.JobInsertResult{Job: found, Duplicate: true} + // The job never joins its batch, which counts it as + // the outcome it was found in, as its finalize would. + if !j.batchID.IsZero() { + failed := 0 + if found.State != driver.JobStateCompleted { + failed = 1 + } + if err := x.countBatch(ctx, j.batchID, 1, failed); err != nil { + return nil, err + } + } + continue + } + } + } + job, err := x.insertJob(ctx, j) + if err != nil { + return nil, err + } + results[i] = driver.JobInsertResult{Job: job} + } + return results, nil +} + +// finishedDuplicate returns the latest job in history with an insert's +// kind and unique key in one of its UniqueFinished states, or nil. +func (x *w) finishedDuplicate(ctx context.Context, j jobInsert) (*driver.JobRow, error) { + args := []any{j.kind, j.uniqueKey} + for _, s := range j.uniqueFinished { + args = append(args, string(s)) + } + found, err := scanJob(x.QueryRowContext(ctx, "SELECT "+historyColumns+` FROM hopper_job_history + WHERE kind = ? AND unique_key = ? AND state IN (`+placeholders(len(j.uniqueFinished))+`) + ORDER BY finalized_at DESC LIMIT 1`, args...), true) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return found, err +} + +// JobInsertCopy implements driver.Executor: SQLite has no COPY. +func (e *executor) JobInsertCopy(context.Context, []driver.JobInsertParams, driver.JobInsertOpts) ([]driver.JobInsertResult, error) { + return nil, driver.ErrNotSupported +} + +// claimMax is the most jobs one claim returns, as on Postgres. +const claimMax = 512 + +// claimEligibleSQL is the WHERE clause of a claim, over an alias cand, with +// the time as ?1 and the queue as ?3. A job with an ordering key is +// claimable only if it is the oldest waiting job of its key and no job of +// that key is running. +const claimEligibleSQL = ` + cand.queue = ?3 AND cand.state IN ('available', 'scheduled', 'retryable') + AND cand.scheduled_at <= ?1 + AND (cand.expires_at IS NULL OR cand.expires_at > ?1) + AND (cand.ordering_key IS NULL OR ( + NOT EXISTS (SELECT 1 FROM hopper_jobs r + WHERE r.queue = cand.queue AND r.ordering_key = cand.ordering_key AND r.state = 'running') + AND cand.seq = (SELECT min(o.seq) FROM hopper_jobs o + WHERE o.queue = cand.queue AND o.ordering_key = cand.ordering_key + AND o.state IN ('available', 'scheduled', 'retryable'))))` + +// claimSetSQL moves claimed jobs to running, owned by the client ?2 at ?1, +// with their fences incremented. +const claimSetSQL = ` +UPDATE hopper_jobs SET state = 'running', attempt = attempt + 1, fence = fence + 1, attempted_at = ?1, attempted_by = ?2, + snooze_reason = NULL +WHERE seq IN (%s) +RETURNING ` + jobColumns + `, seq` + +// claimSQL claims up to ?4 jobs in claim order. +var claimSQL = fmt.Sprintf(claimSetSQL, `SELECT cand.seq FROM hopper_jobs cand WHERE `+claimEligibleSQL+` + ORDER BY cand.priority, cand.scheduled_at, cand.seq LIMIT ?4`) + +// claimPartitionedSQL claims up to ?4 jobs in claim order, admitting only +// those whose rank within their partition key, counting the key's running +// jobs, fits under the partition limit ?5. +var claimPartitionedSQL = fmt.Sprintf(claimSetSQL, `SELECT seq FROM ( + SELECT cand.seq, cand.priority, cand.scheduled_at, cand.partition_key, + row_number() OVER (PARTITION BY cand.partition_key ORDER BY cand.priority, cand.scheduled_at, cand.seq) + + (SELECT count(*) FROM hopper_jobs pr + WHERE pr.queue = cand.queue AND pr.partition_key = cand.partition_key AND pr.state = 'running') AS slot + FROM hopper_jobs cand WHERE `+claimEligibleSQL+` +) WHERE partition_key IS NULL OR slot <= ?5 +ORDER BY priority, scheduled_at, seq LIMIT ?4`) + +// claim claims up to limit jobs of a queue for a client, with a partition +// limit if partition is positive, and returns them in claim order. +func (x *w) claim(ctx context.Context, queue string, clientID int64, limit int, partition int64) ([]*driver.JobRow, error) { + limit = min(limit, claimMax) + if limit <= 0 { + return nil, nil + } + type claimed struct { + job *driver.JobRow + seq int64 + } + query, args := claimSQL, []any{x.us(), clientID, queue, limit} + if partition > 0 { + query, args = claimPartitionedSQL, append(args, partition) + } + rows, err := queryAll(ctx, x, func(r scanner) (claimed, error) { + var c claimed + var err error + c.job, err = scanJob(r, false, &c.seq) + return c, err + }, query, args...) + if err != nil { + return nil, fmt.Errorf("hopper: claim jobs: %w", err) + } + slices.SortFunc(rows, func(a, b claimed) int { + return cmp.Or(cmp.Compare(a.job.Priority, b.job.Priority), a.job.ScheduledAt.Compare(b.job.ScheduledAt), cmp.Compare(a.seq, b.seq)) + }) + jobs := make([]*driver.JobRow, len(rows)) + for i, r := range rows { + jobs[i] = r.job + } + return jobs, nil +} + +// JobClaim implements driver.Executor. The write lock makes the claim +// exclusive: no other transaction claims, or changes, a job between the +// statement's read of it and its update. +func (e *executor) JobClaim(ctx context.Context, params driver.JobClaimParams) (driver.JobClaimResult, error) { + if params.Limit <= 0 { + return driver.JobClaimResult{}, nil + } + if params.Limited { + return e.claimLimited(ctx, params) + } + var jobs []*driver.JobRow + err := e.write(ctx, func(x *w) error { + var err error + jobs, err = x.claim(ctx, params.Queue, params.ClientID, params.Limit, 0) + return err + }) + if err != nil { + return driver.JobClaimResult{}, err + } + return driver.JobClaimResult{Jobs: jobs}, nil +} + +// JobAge implements driver.Executor. Bumped jobs restart their wait, so a +// job climbs one level per period. +func (e *executor) JobAge(ctx context.Context, queue string, after time.Duration) (int64, error) { + if after <= 0 { + return 0, nil + } + var n int64 + err := e.write(ctx, func(x *w) error { + res, err := x.ExecContext(ctx, ` + UPDATE hopper_jobs SET priority = priority - 1, scheduled_at = ? + WHERE queue = ? AND state IN ('available', 'scheduled', 'retryable') AND priority > 1 AND scheduled_at <= ?`, + x.us(), queue, x.at(-after)) + if err != nil { + return err + } + n, err = res.RowsAffected() + return err + }) + if err != nil { + return 0, fmt.Errorf("hopper: age jobs: %w", err) + } + return n, nil +} + +// JobRecordProgress implements driver.Executor. With NoWait, a write lock +// another transaction holds is waited for only briefly, after which it +// returns ErrJobLocked. +func (e *executor) JobRecordProgress(ctx context.Context, params driver.JobRecordProgressParams) (*driver.Progress, error) { + var cursor json.RawMessage + if len(params.Cursor) > 0 { + c, err := jsonValue(params.Cursor, "", "the progress cursor") + if err != nil { + return nil, err + } + cursor = json.RawMessage(c) + } + var progress *driver.Progress + write := e.write + if params.NoWait { + write = e.writeBrief + } + err := write(ctx, func(x *w) error { + var stored sql.NullString + err := x.QueryRowContext(ctx, `SELECT progress FROM hopper_jobs + WHERE id = ? AND state = 'running' AND attempted_by = ? AND fence = ? AND attempt = ?`, + idParam(params.ID), params.AttemptedBy, params.Fence, params.Attempt).Scan(&stored) + if errors.Is(err, sql.ErrNoRows) { + return nil + } + if err != nil { + return err + } + step := params.Step + if params.KeepStep { + step = "" + if p := jobcodec.DecodeProgress(stored.String); p != nil { + step = p.Step + } + } + b, err := json.Marshal(struct { + Step string `json:"step"` + At time.Time `json:"at"` + Cursor json.RawMessage `json:"cursor,omitempty"` + }{step, x.now, cursor}) + if err != nil { + return err + } + if _, err := x.ExecContext(ctx, "UPDATE hopper_jobs SET progress = ? WHERE id = ?", string(b), idParam(params.ID)); err != nil { + return err + } + progress = jobcodec.DecodeProgress(string(b)) + return nil + }) + if errors.Is(err, errLockBusy) { + return nil, driver.ErrJobLocked + } + if err != nil { + return nil, fmt.Errorf("hopper: record job progress: %w", err) + } + return progress, nil +} + +// JobGet implements driver.Executor. +func (e *executor) JobGet(ctx context.Context, id driver.JobID) (*driver.JobRow, error) { + job, err := scanJob(e.read().QueryRowContext(ctx, ` + SELECT `+liveAsHistory+` FROM hopper_jobs WHERE id = ?1 + UNION ALL + SELECT `+historyColumns+` FROM hopper_job_history WHERE id = ?1 + LIMIT 1`, idParam(id)), true) + if errors.Is(err, sql.ErrNoRows) { + return nil, driver.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("hopper: get job: %w", err) + } + return job, nil +} + +// JobLogs implements driver.Executor. +func (e *executor) JobLogs(ctx context.Context, id driver.JobID) ([]driver.AttemptLog, error) { + var logs sql.NullString + err := e.read().QueryRowContext(ctx, ` + SELECT logs FROM hopper_jobs WHERE id = ?1 + UNION ALL + SELECT logs FROM hopper_job_history WHERE id = ?1 + LIMIT 1`, idParam(id)).Scan(&logs) + if errors.Is(err, sql.ErrNoRows) { + return nil, driver.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("hopper: get job logs: %w", err) + } + return jobcodec.DecodeLogs(logs.String), nil +} + +// errOrderingKeyNeedsQueue refuses a key listing across queues: a key is a +// lane only within its queue. +var errOrderingKeyNeedsQueue = fmt.Errorf("%w: an ordering key is listed within a queue; set the queue", driver.ErrInvalidFilter) + +// checkFilterText refuses filter text Postgres would refuse: a NUL, or +// bytes that are not UTF-8. +func checkFilterText(texts ...string) error { + for _, s := range texts { + if !utf8.ValidString(s) || strings.ContainsRune(s, 0) { + return fmt.Errorf("%w: %q is not UTF-8 text without NUL", driver.ErrInvalidFilter, s) + } + } + return nil +} + +// JobList implements driver.Executor. Its filters are refused where +// Postgres's are, so that a listing means the same on either engine. +func (e *executor) JobList(ctx context.Context, params driver.JobListParams) ([]*driver.JobRow, error) { + if params.OrderingKey != "" && params.Queue == "" { + return nil, errOrderingKeyNeedsQueue + } + for _, p := range params.Priorities { + if _, err := smallint(p, "Priorities"); err != nil { + return nil, fmt.Errorf("%w: %w", driver.ErrInvalidFilter, err) + } + } + states := make([]string, len(params.States)) + for i, s := range params.States { + states[i] = string(s) + } + tagsAll, tagsAny := jobtags.Distinct(params.TagsAll), jobtags.Distinct(params.TagsAny) + for _, texts := range [][]string{{params.Queue, params.OrderingKey}, params.Kinds, states, tagsAll, tagsAny} { + if err := checkFilterText(texts...); err != nil { + return nil, err + } + } + if len(params.Metadata) > 0 { + if err := checkJSONFilter(string(params.Metadata)); err != nil { + return nil, fmt.Errorf("%w: %w", driver.ErrInvalidFilter, err) + } + } + live, history := len(params.States) == 0, len(params.States) == 0 + for _, s := range params.States { + if s.Terminal() { + history = true + } else { + live = true + } + } + + var ( + where []string + args []any + ) + in := func(column string, values []any) { + where = append(where, column+" IN ("+placeholders(len(values))+")") + args = append(args, values...) + } + anyOf := func(values []string) []any { + out := make([]any, len(values)) + for i, v := range values { + out[i] = v + } + return out + } + if params.Queue != "" { + where, args = append(where, "queue = ?"), append(args, params.Queue) + } + if len(params.Kinds) > 0 { + in("kind", anyOf(params.Kinds)) + } + if len(states) > 0 { + in("state", anyOf(states)) + } + order := "id" + switch { + case params.NewestFirst: + order = "id DESC" + if !params.After.IsZero() { + where, args = append(where, "id < ?"), append(args, idParam(params.After)) + } + default: + where, args = append(where, "id > ?"), append(args, idParam(params.After)) + } + if params.OrderingKey != "" { + where, args = append(where, "ordering_key = ?"), append(args, params.OrderingKey) + } + if len(params.IDs) > 0 { + ids := make([]any, len(params.IDs)) + for i, id := range params.IDs { + ids[i] = idParam(id) + } + in("id", ids) + } + if len(params.Priorities) > 0 { + ps := make([]any, len(params.Priorities)) + for i, p := range params.Priorities { + ps[i] = p + } + in("priority", ps) + } + if len(params.Metadata) > 0 { + where, args = append(where, "hopper_json_contains(metadata, ?)"), append(args, string(params.Metadata)) + } + // A listing by tags walks the tags table's rows of a tag, and checks + // the job's own tags, which those rows follow, for every tag asked for. + tagged := func(table string) (string, []any) { + var conds []string + var targs []any + walked := tagsAny + if len(tagsAll) > 0 { + walked = tagsAll[:1] + for _, t := range tagsAll { + conds = append(conds, "EXISTS (SELECT 1 FROM json_each(j.tags) WHERE type = 'text' AND value = ?)") + targs = append(targs, t) + } + } + if len(tagsAny) > 0 { + conds = append(conds, "EXISTS (SELECT 1 FROM json_each(j.tags) WHERE type = 'text' AND value IN ("+placeholders(len(tagsAny))+"))") + targs = append(targs, anyOf(tagsAny)...) + } + if len(walked) == 0 { + return "", nil + } + walk := "j.id IN (SELECT t.id FROM " + table + " t WHERE t.tag IN (" + placeholders(len(walked)) + "))" + return " AND " + strings.Join(append([]string{walk}, conds...), " AND "), append(anyOf(walked), targs...) + } + limit := " ORDER BY " + order + " LIMIT " + strconv.Itoa(max(params.Limit, 1)) + filter := " WHERE " + strings.Join(where, " AND ") + if len(where) == 0 { + filter = " WHERE true" + } + var ( + arms []string + armArgs []any + ) + if live { + tags, targs := tagged("hopper_job_tags") + arms = append(arms, "SELECT * FROM (SELECT "+liveAsHistory+" FROM hopper_jobs j"+filter+tags+limit+")") + armArgs = append(append(armArgs, args...), targs...) + } + if history { + tags, targs := tagged("hopper_job_history_tags") + arms = append(arms, "SELECT * FROM (SELECT "+historyColumns+" FROM hopper_job_history j"+filter+tags+limit+")") + armArgs = append(append(armArgs, args...), targs...) + } + query := strings.Join(arms, " UNION ALL ") + limit + jobs, err := queryAll(ctx, e.read(), scanHistoryJob, query, armArgs...) + if err != nil { + return nil, fmt.Errorf("hopper: list jobs: %w", err) + } + return jobs, nil +} + +// lockState returns a live job's row, or nil if it is not live. +func (x *w) lockState(ctx context.Context, id driver.JobID) (*liveRow, error) { + return x.liveRow(ctx, id) +} + +// cancelWaiting finalizes a waiting job as canceled, archived or not, with +// everything that entails for its batch and its dependents, and returns it +// as it is afterwards. +func (x *w) cancelWaiting(ctx context.Context, row *liveRow, archive bool) (*driver.JobRow, error) { + errs, err := hopperError(row, "hopper: canceled", x.now) + if err != nil { + return nil, err + } + f := finished{row: row, state: driver.JobStateCanceled, errors: errs, logs: row.logs, archive: archive} + if err := x.finish(ctx, f); err != nil { + return nil, err + } + if _, err := x.tail(ctx, []finished{f}); err != nil { + return nil, err + } + return f.jobRow(x.now) +} + +// JobCancel implements driver.Executor. +func (e *executor) JobCancel(ctx context.Context, id driver.JobID) (*driver.JobRow, error) { + var job *driver.JobRow + err := e.write(ctx, func(x *w) error { + row, err := x.lockState(ctx, id) + switch { + case err != nil: + return err + case row == nil: + // Not live: finalized already, or unknown. + job, err = (&executor{d: e.d, tx: txOf(x)}).JobGet(ctx, id) + return err + case row.job.State == driver.JobStateRunning: + if job, err = scanJob(x.QueryRowContext(ctx, `UPDATE hopper_jobs SET cancel_requested_at = coalesce(cancel_requested_at, ?) + WHERE id = ? RETURNING `+jobColumns, x.us(), idParam(id)), false); err != nil { + return err + } + return x.notify(ctx, driver.ChannelControl, "cancel:"+id.String()) + } + job, err = x.cancelWaiting(ctx, row, true) + return err + }) + if err != nil { + if errors.Is(err, driver.ErrNotFound) { + return nil, driver.ErrNotFound + } + return nil, fmt.Errorf("hopper: cancel job: %w", err) + } + return job, nil +} + +// txOf returns the transaction a write runs in, for a read within it. +func txOf(x *w) *sql.Tx { + tx, _ := x.q.(*sql.Tx) + return tx +} + +// JobDelete implements driver.Executor. +func (e *executor) JobDelete(ctx context.Context, id driver.JobID) (*driver.JobRow, error) { + var job *driver.JobRow + err := e.write(ctx, func(x *w) error { + row, err := x.lockState(ctx, id) + switch { + case err != nil: + return err + case row == nil: + job, err = scanJob(x.QueryRowContext(ctx, `DELETE FROM hopper_job_history WHERE id = ? RETURNING `+historyColumns, idParam(id)), true) + if errors.Is(err, sql.ErrNoRows) { + return driver.ErrNotFound + } + return err + case row.job.State == driver.JobStateRunning: + return driver.ErrJobRunning + } + job, err = x.cancelWaiting(ctx, row, false) + return err + }) + if err != nil { + if errors.Is(err, driver.ErrNotFound) || errors.Is(err, driver.ErrJobRunning) { + return nil, err + } + return nil, fmt.Errorf("hopper: delete job: %w", err) + } + return job, nil +} + +// runNowSQL is the SET list that makes a waiting job runnable now and drops +// the reason it was waiting. A job that was already due keeps its +// scheduled_at, so it keeps its place in claim order and its aging. +const runNowSQL = `state = 'available', scheduled_at = min(scheduled_at, ?), snooze_reason = NULL, wake_requested_at = NULL` + +// retryHistorySQL moves a finalized job back to the live table with its +// history (attempts, errors, progress unless from the start, logs, fence) +// intact, a fresh seq, and one more attempt if it had run out. Its tags are +// what its tags column reads as. +const retryHistorySQL = ` +INSERT INTO hopper_jobs (id, kind, queue, state, priority, attempt, max_attempts, scheduled_at, args, metadata, + errors, unique_key, ordering_key, partition_key, batch_id, await, created_at, progress, logs, tags, fence) +SELECT id, kind, queue, 'available', priority, attempt, + CASE WHEN attempt >= max_attempts THEN attempt + 1 ELSE max_attempts END, ?, args, metadata, + errors, unique_key, ordering_key, partition_key, batch_id, await, created_at, CASE WHEN ? THEN NULL ELSE progress END, logs, + ?, fence +FROM hopper_job_history WHERE rowid = ? +RETURNING ` + jobColumns + +// JobRetry implements driver.Executor. +func (e *executor) JobRetry(ctx context.Context, params driver.JobRetryParams) (*driver.JobRow, error) { + id := params.ID + var job *driver.JobRow + err := e.write(ctx, func(x *w) error { + row, err := x.lockState(ctx, id) + switch { + case err != nil: + return err + case row == nil: + var ( + rowid int64 + kind, tags sql.NullString + uniqueKey sql.NullString + ) + err := x.QueryRowContext(ctx, `SELECT rowid, kind, unique_key, tags FROM hopper_job_history WHERE id = ? LIMIT 1`, idParam(id)). + Scan(&rowid, &kind, &uniqueKey, &tags) + if errors.Is(err, sql.ErrNoRows) { + return driver.ErrNotFound + } + if err != nil { + return err + } + if uniqueKey.Valid { + var held bool + if err := x.QueryRowContext(ctx, `SELECT EXISTS (SELECT 1 FROM hopper_jobs WHERE kind = ? AND unique_key = ?)`, + kind.String, uniqueKey.String).Scan(&held); err != nil { + return err + } + if held { + return driver.ErrUniqueConflict + } + } + if job, err = scanJob(x.QueryRowContext(ctx, retryHistorySQL, x.us(), params.FromStart, + tagsJSON(jobcodec.DecodeTags(tags.String)), rowid), false); err != nil { + return err + } + _, err = x.ExecContext(ctx, `DELETE FROM hopper_job_history WHERE rowid = ?`, rowid) + return err + case row.job.State == driver.JobStateRunning: + return driver.ErrJobRunning + } + job, err = scanJob(x.QueryRowContext(ctx, `UPDATE hopper_jobs SET `+runNowSQL+`, expires_at = NULL, + progress = CASE WHEN ? THEN NULL ELSE progress END + WHERE id = ? RETURNING `+jobColumns, x.us(), params.FromStart, idParam(id)), false) + return err + }) + if err != nil { + if errors.Is(err, driver.ErrNotFound) || errors.Is(err, driver.ErrJobRunning) || errors.Is(err, driver.ErrUniqueConflict) { + return nil, err + } + return nil, fmt.Errorf("hopper: retry job: %w", err) + } + return job, nil +} + +// JobRetryOrderingKey implements driver.Executor: it marks the key's +// running job, so that its finalize honors the wake, and wakes the key's +// waiting jobs, oldest first, leaving those past their TTL to the expiry +// sweep. +func (e *executor) JobRetryOrderingKey(ctx context.Context, params driver.JobRetryOrderingKeyParams) (int, error) { + var n int64 + err := e.write(ctx, func(x *w) error { + if _, err := x.ExecContext(ctx, `UPDATE hopper_jobs SET wake_requested_at = ? + WHERE queue = ? AND ordering_key = ? AND state = 'running'`, x.us(), params.Queue, params.OrderingKey); err != nil { + return err + } + res, err := x.ExecContext(ctx, `UPDATE hopper_jobs SET `+runNowSQL+` + WHERE seq IN ( + SELECT seq FROM hopper_jobs + WHERE queue = ? AND ordering_key = ? AND state IN ('scheduled', 'retryable') AND (expires_at IS NULL OR expires_at > ?) + ORDER BY seq LIMIT ?)`, x.us(), params.Queue, params.OrderingKey, x.us(), max(params.Limit, 1)) + if err != nil { + return err + } + n, err = res.RowsAffected() + return err + }) + if err != nil { + return 0, fmt.Errorf("hopper: retry ordering key: %w", err) + } + return int(n), nil +} + +// JobDiscardExpired implements driver.Executor. +func (e *executor) JobDiscardExpired(ctx context.Context, limit int) ([]*driver.JobRow, error) { + var jobs []*driver.JobRow + err := e.write(ctx, func(x *w) error { + jobs = jobs[:0] + rows, err := queryAll(ctx, x, scanLive, `SELECT `+liveRowColumns+` FROM hopper_jobs + WHERE expires_at <= ? AND state IN ('available', 'scheduled', 'retryable') + ORDER BY expires_at LIMIT ?`, x.us(), max(limit, 1)) + if err != nil { + return err + } + done := make([]finished, 0, len(rows)) + for _, row := range rows { + errs, err := hopperError(row, "hopper: expired", x.now) + if err != nil { + return err + } + f := finished{row: row, state: driver.JobStateDiscarded, errors: errs, logs: row.logs, archive: true} + if err := x.finish(ctx, f); err != nil { + return err + } + done = append(done, f) + job, err := f.jobRow(x.now) + if err != nil { + return err + } + jobs = append(jobs, job) + } + _, err = x.tail(ctx, done) + return err + }) + if err != nil { + return nil, fmt.Errorf("hopper: discard expired jobs: %w", err) + } + return jobs, nil +} + +// JobRescueCandidates implements driver.Executor: running jobs whose +// client has no live lease, and, with StuckAfter, those claimed longer ago +// than that, oldest first. +func (e *executor) JobRescueCandidates(ctx context.Context, params driver.JobRescueParams) ([]driver.RescueCandidate, error) { + now := e.clock() + stuckBefore := int64(math.MinInt64) + if params.StuckAfter > 0 { + stuckBefore = now.Add(-params.StuckAfter).UnixMicro() + } + candidates, err := queryAll(ctx, e.read(), func(r scanner) (driver.RescueCandidate, error) { + var c driver.RescueCandidate + var err error + c.Job, err = scanJob(r, false, &c.Stuck) + return c, err + }, `SELECT * FROM ( + SELECT `+prefixed(jobColumns, "j")+`, coalesce(j.attempted_at < ?1, false) AS stuck + FROM hopper_jobs j WHERE j.state = 'running' + ) j + WHERE stuck OR NOT EXISTS (SELECT 1 FROM hopper_clients c WHERE c.id = j.attempted_by AND c.expires_at > ?2) + ORDER BY j.attempted_at LIMIT ?3`, stuckBefore, now.UnixMicro(), params.Limit) + if err != nil { + return nil, fmt.Errorf("hopper: rescue candidates: %w", err) + } + return candidates, nil +} + +// sortIDs orders IDs as they sort in the database. +func sortIDs(ids []driver.JobID) { + slices.SortFunc(ids, func(a, b driver.JobID) int { return bytes.Compare(a[:], b[:]) }) +} diff --git a/driver/hoppersqlite/listener.go b/driver/hoppersqlite/listener.go new file mode 100644 index 0000000..04d9a7a --- /dev/null +++ b/driver/hoppersqlite/listener.go @@ -0,0 +1,133 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "errors" + "time" + + "github.com/parallelworks/hopper/driver" +) + +// listener implements driver.Listener on hopper_notifications. It reads +// the rows committed after the last it read, on a connection of its own: +// at once when a transaction of this process that wrote some commits, and +// every listenInterval otherwise, for those of other processes, which it +// reads only when PRAGMA data_version says another connection has +// committed since its last look. +type listener struct { + d *Driver + conn *sql.Conn + channels map[string]bool + poke chan struct{} + unlisten func() + // last is the seq of the last notification read, and version the + // connection's data_version when it was read. + last, version int64 + pending []driver.Notification +} + +// listenPage is how many notifications a listener reads at a time. +const listenPage = 1000 + +func (d *Driver) newListener(ctx context.Context) (*listener, error) { + conn, err := d.sqlDB.Conn(ctx) + if err != nil { + return nil, err + } + l := &listener{d: d, conn: conn, channels: map[string]bool{}, poke: make(chan struct{}, 1), version: -1} + l.unlisten = d.db.listen(l.poke) + // Notifications committed from now on are delivered. + if err := conn.QueryRowContext(ctx, `SELECT coalesce(max(seq), 0) FROM hopper_notifications`).Scan(&l.last); err != nil { + _ = l.Close(ctx) + return nil, err + } + return l, nil +} + +// Listen implements driver.Listener. +func (l *listener) Listen(_ context.Context, channels ...string) error { + for _, c := range channels { + l.channels[c] = true + } + return nil +} + +// Next implements driver.Listener. +func (l *listener) Next(ctx context.Context) (driver.Notification, error) { + poked := true + for { + if len(l.pending) > 0 { + n := l.pending[0] + l.pending = l.pending[1:] + return n, nil + } + if err := l.read(ctx, poked); err != nil { + return driver.Notification{}, err + } + if len(l.pending) > 0 { + continue + } + timer := time.NewTimer(l.d.listenInterval) + select { + case <-l.poke: + poked = true + case <-timer.C: + poked = false + case <-ctx.Done(): + timer.Stop() + return driver.Notification{}, ctx.Err() + } + timer.Stop() + } +} + +// read reads the notifications committed since the last read, if this +// process poked the listener or another connection has committed since. +func (l *listener) read(ctx context.Context, poked bool) error { + var version int64 + if err := l.conn.QueryRowContext(ctx, "PRAGMA data_version").Scan(&version); err != nil { + return err + } + if !poked && version == l.version { + return nil + } + l.version = version + rows, err := l.conn.QueryContext(ctx, `SELECT seq, channel, payload FROM hopper_notifications WHERE seq > ? ORDER BY seq LIMIT ?`, l.last, listenPage) + if err != nil { + return err + } + defer rows.Close() + read := 0 + for rows.Next() { + var n driver.Notification + if err := rows.Scan(&l.last, &n.Channel, &n.Payload); err != nil { + return err + } + read++ + if l.channels[n.Channel] { + l.pending = append(l.pending, n) + } + } + if err := rows.Err(); err != nil { + return err + } + // A full page leaves more to read: look again without waiting. + if read == listenPage { + select { + case l.poke <- struct{}{}: + default: + } + } + return nil +} + +// Close implements driver.Listener. +func (l *listener) Close(context.Context) error { + l.unlisten() + err := l.conn.Close() + if errors.Is(err, sql.ErrConnDone) { + return nil + } + return err +} diff --git a/driver/hoppersqlite/localfs.go b/driver/hoppersqlite/localfs.go new file mode 100644 index 0000000..7b43007 --- /dev/null +++ b/driver/hoppersqlite/localfs.go @@ -0,0 +1,126 @@ +package hoppersqlite + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrNetworkFilesystem is returned by Open for a database on a network or +// cluster filesystem (NFS, SMB, Lustre, GPFS and the like, or FUSE, which +// such filesystems often are), where SQLite's file locks and the shared +// memory its write-ahead log needs do not work reliably: two processes +// could both believe they hold the write lock and corrupt the database. +// Home directories on clusters are often on one, and so are the folders a +// virtual machine or container shares with its host, Docker Desktop's bind +// mounts among them. Choose a path on a local disk instead. +var ErrNetworkFilesystem = errors.New("hoppersqlite: the database is on a network filesystem, where SQLite cannot lock it reliably; choose a path on a local disk") + +// maxLinks bounds the symbolic links resolve follows, as the kernel's +// ELOOP does. +const maxLinks = 40 + +// resolve returns the absolute path with every symbolic link in it +// resolved, a last link to a file that does not exist yet included. SQLite +// opens the file a link names and keeps its write-ahead log and shared +// memory beside it, not beside the link, so the path resolved is the one +// to check and to open. A path that exists is resolved as the kernel +// resolves it (filepath.EvalSymlinks); one that does not yet, component by +// component, as the kernel would to create it. +func resolve(path string) (string, error) { + // Not filepath.Abs, which would clean ".." against what precedes it + // before the links in it are followed. + p := path + if !filepath.IsAbs(p) { + wd, err := os.Getwd() + if err != nil { + return "", err + } + p = wd + string(filepath.Separator) + p + } + r, err := filepath.EvalSymlinks(p) + if err == nil { + return r, nil + } + if !errors.Is(err, fs.ErrNotExist) { + return "", err + } + return resolveMissing(p) +} + +// resolveMissing resolves the links of an absolute path whose last +// component, or the last a link in it leads to, does not exist: each +// component in turn, a link's target in its place, and ".." from the +// directory resolved so far, as the kernel does, not lexically. +func resolveMissing(p string) (string, error) { + vol := filepath.VolumeName(p) + cur := vol + string(filepath.Separator) + rest := split(p[len(vol):]) + links := 0 + for len(rest) > 0 { + name := rest[0] + rest = rest[1:] + switch name { + case ".": + continue + case "..": + cur = filepath.Dir(cur) + continue + } + next := filepath.Join(cur, name) + fi, err := os.Lstat(next) + switch { + case errors.Is(err, fs.ErrNotExist) && len(rest) == 0: + return next, nil // the file SQLite will create + case err != nil: + return "", err + case fi.Mode()&fs.ModeSymlink == 0: + cur = next + continue + } + if links++; links > maxLinks { + return "", fmt.Errorf("%s: more than %d symbolic links", p, maxLinks) + } + target, err := os.Readlink(next) + if err != nil { + return "", err + } + if filepath.IsAbs(target) { + vol = filepath.VolumeName(target) + cur = vol + string(filepath.Separator) + target = target[len(vol):] + } + rest = append(split(target), rest...) + } + return cur, nil +} + +// split returns the components of a path, without empty ones. +func split(p string) []string { + return strings.FieldsFunc(p, func(r rune) bool { return r < 0x80 && os.IsPathSeparator(byte(r)) }) //nolint:gosec // r < 0x80 +} + +// checkLocal returns ErrNetworkFilesystem if the database file at path, or +// the directory it is in, where SQLite also keeps its log and shared +// memory, is on a network filesystem. path has its links resolved +// (resolve). +func checkLocal(path string) error { + dir := filepath.Dir(path) + check := []string{dir} + if _, err := os.Stat(path); err == nil { + check = append(check, path) // a file mounted on its own + } + for _, p := range check { + kind, err := networkFilesystem(p) + if err != nil { + return fmt.Errorf("hoppersqlite: tell whether %s is on a local filesystem: %w", p, err) + } + if kind != "" { + return fmt.Errorf("%w: %s is on %s", ErrNetworkFilesystem, p, kind) + } + } + return nil +} diff --git a/driver/hoppersqlite/localfs_bsd.go b/driver/hoppersqlite/localfs_bsd.go new file mode 100644 index 0000000..9d76bb3 --- /dev/null +++ b/driver/hoppersqlite/localfs_bsd.go @@ -0,0 +1,18 @@ +//go:build darwin || freebsd || openbsd + +package hoppersqlite + +import "golang.org/x/sys/unix" + +// networkFilesystem reports a filesystem the kernel does not mark local +// (MNT_LOCAL), as NFS, SMB, AFP, WebDAV and FUSE mounts are not. +func networkFilesystem(path string) (string, error) { + var st unix.Statfs_t + if err := unix.Statfs(path, &st); err != nil { + return "", err + } + if statfsFlags(&st)&unix.MNT_LOCAL == 0 { + return "a filesystem not marked local (" + statfsType(&st) + ")", nil + } + return "", nil +} diff --git a/driver/hoppersqlite/localfs_darwin.go b/driver/hoppersqlite/localfs_darwin.go new file mode 100644 index 0000000..945ff68 --- /dev/null +++ b/driver/hoppersqlite/localfs_darwin.go @@ -0,0 +1,7 @@ +package hoppersqlite + +import "golang.org/x/sys/unix" + +func statfsFlags(st *unix.Statfs_t) uint64 { return uint64(st.Flags) } + +func statfsType(st *unix.Statfs_t) string { return unix.ByteSliceToString(st.Fstypename[:]) } diff --git a/driver/hoppersqlite/localfs_freebsd.go b/driver/hoppersqlite/localfs_freebsd.go new file mode 100644 index 0000000..9be34e8 --- /dev/null +++ b/driver/hoppersqlite/localfs_freebsd.go @@ -0,0 +1,7 @@ +package hoppersqlite + +import "golang.org/x/sys/unix" + +func statfsFlags(st *unix.Statfs_t) uint64 { return st.Flags } + +func statfsType(st *unix.Statfs_t) string { return unix.ByteSliceToString(st.Fstypename[:]) } diff --git a/driver/hoppersqlite/localfs_linux.go b/driver/hoppersqlite/localfs_linux.go new file mode 100644 index 0000000..eb7ea54 --- /dev/null +++ b/driver/hoppersqlite/localfs_linux.go @@ -0,0 +1,51 @@ +package hoppersqlite + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +// networkMagic names the filesystems, by statfs(2)'s magic number, whose +// locks SQLite cannot rely on: network and cluster filesystems, the shared +// folders of virtual machines, and FUSE, which sshfs, s3fs, GlusterFS and +// many others are, as are the bind mounts of Docker Desktop and of other +// tools that run Linux in a virtual machine. +var networkMagic = map[int64]string{ + 0x6969: "NFS", + 0x517b: "SMB", + 0xfe534d42: "SMB2", + 0xff534d42: "CIFS", + 0x73757245: "Coda", + 0x5346414f: "AFS", + 0x6b414653: "AFS", + 0x01021997: "9P", + 0x564c: "NCP", + 0x00c36400: "CephFS", + 0x01161970: "GFS2", + 0x7461636f: "OCFS2", + 0x0bd00bd0: "Lustre", + 0x47504653: "GPFS", + 0x19830326: "BeeGFS", + 0xaad7aaea: "PanFS", + 0x20030528: "OrangeFS", + 0x6a656a63: "virtiofs", + 0x786f4256: "VirtualBox shared folders", + 0x7c7c6673: "Parallels shared folders", + 0xbacbacbc: "VMware shared folders", + 0x65735546: "FUSE", +} + +// statfs is unix.Statfs, which tests replace. +var statfs = unix.Statfs + +func networkFilesystem(path string) (string, error) { + var st unix.Statfs_t + if err := statfs(path, &st); err != nil { + return "", err + } + if name, ok := networkMagic[int64(st.Type)&0xffffffff]; ok { //nolint:unconvert // st.Type is an int32 or a uint32 on some architectures + return fmt.Sprintf("%s (filesystem type %#x)", name, st.Type), nil + } + return "", nil +} diff --git a/driver/hoppersqlite/localfs_linux_test.go b/driver/hoppersqlite/localfs_linux_test.go new file mode 100644 index 0000000..55050f4 --- /dev/null +++ b/driver/hoppersqlite/localfs_linux_test.go @@ -0,0 +1,125 @@ +package hoppersqlite + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/sys/unix" +) + +// setType sets a statfs type, which is an int32, an int64 or a uint32 by +// architecture, to a magic number. +func setType[T int32 | int64 | uint32](t *T, v int64) { *t = T(v) } + +// TestNetworkFilesystem checks that a directory on NFS, Lustre, GPFS or +// FUSE is refused, naming the filesystem, and one on a local filesystem is +// not. +func TestNetworkFilesystem(t *testing.T) { + if err := checkLocal(t.TempDir() + "/hopper.db"); err != nil { + t.Fatalf("a temporary directory: %v", err) + } + defer func(real func(string, *unix.Statfs_t) error) { statfs = real }(statfs) + for magic, name := range map[int64]string{0x6969: "NFS", 0x0bd00bd0: "Lustre", 0x47504653: "GPFS", 0x65735546: "FUSE", 0xff534d42: "CIFS", 0x7c7c6673: "Parallels", 0xbacbacbc: "VMware"} { + statfs = func(_ string, st *unix.Statfs_t) error { + setType(&st.Type, magic) + return nil + } + err := checkLocal("/home/someone/hopper.db") + if !errors.Is(err, ErrNetworkFilesystem) || !strings.Contains(err.Error(), name) || !strings.Contains(err.Error(), "/home/someone") { + t.Errorf("a directory on %s: %v", name, err) + } + } + statfs = func(string, *unix.Statfs_t) error { return unix.ENOENT } + if err := checkLocal("/nowhere/hopper.db"); err == nil || errors.Is(err, ErrNetworkFilesystem) { + t.Errorf("a directory that cannot be read: %v", err) + } +} + +// TestNetworkFilesystemThroughLink checks that a database reached through +// a symbolic link is checked, and opened, where the link leads: SQLite +// opens the file a link names and keeps its log and shared memory beside +// it, so a link in a local directory to a database on NFS is refused, as +// one whose directory is a link to NFS is, and a dangling link too, which +// SQLite would follow to create the database. +func TestNetworkFilesystemThroughLink(t *testing.T) { + local, remote := t.TempDir(), t.TempDir() // remote stands for a directory on NFS + defer func(real func(string, *unix.Statfs_t) error) { statfs = real }(statfs) + statfs = func(path string, st *unix.Statfs_t) error { + if err := unix.Statfs(path, st); err != nil { + return err + } + if path == remote || strings.HasPrefix(path, remote+"/") { + setType(&st.Type, 0x6969) + } + return nil + } + refused := func(what, path string) { + t.Helper() + d, err := Open(path, nil) + if err == nil { + _ = d.Close() + t.Errorf("%s: opened", what) + return + } + if !errors.Is(err, ErrNetworkFilesystem) || !strings.Contains(err.Error(), remote) { + t.Errorf("%s: %v", what, err) + } + } + refused("a database on NFS", filepath.Join(remote, "direct.db")) + + link := filepath.Join(local, "dangling.db") + if err := os.Symlink(filepath.Join(remote, "dangling.db"), link); err != nil { + t.Fatal(err) + } + refused("a dangling link to NFS", link) + + if err := os.WriteFile(filepath.Join(remote, "existing.db"), nil, 0o600); err != nil { + t.Fatal(err) + } + link = filepath.Join(local, "existing.db") + if err := os.Symlink(filepath.Join("..", filepath.Base(remote), "existing.db"), link); err != nil { + // remote and local share a parent only on some systems + if err := os.Symlink(filepath.Join(remote, "existing.db"), link); err != nil { + t.Fatal(err) + } + } + refused("a link to a database on NFS", link) + + dir := filepath.Join(local, "dir") + if err := os.Symlink(remote, dir); err != nil { + t.Fatal(err) + } + refused("a database in a linked directory on NFS", filepath.Join(dir, "linked.db")) + + if entries, _ := os.ReadDir(remote); len(entries) != 1 { + t.Errorf("the NFS directory holds %d files, want only the one the test made", len(entries)) + } + + // A link to a local database opens it where the link leads, sharing + // the write lock of the Driver that opens it by its own path. + target := filepath.Join(local, "target.db") + link = filepath.Join(local, "link.db") + if err := os.Symlink("target.db", link); err != nil { + t.Fatal(err) + } + statfs = unix.Statfs + d1, err := Open(target, nil) + if err != nil { + t.Fatal(err) + } + defer d1.Close() + d2, err := Open(link, nil) + if err != nil { + t.Fatal(err) + } + defer d2.Close() + if d1.db != d2.db { + t.Error("a link and its target are two databases to the registry") + } + if _, err := os.Stat(target + "-wal"); err != nil { + t.Errorf("the log is not beside the target: %v", err) + } +} diff --git a/driver/hoppersqlite/localfs_netbsd.go b/driver/hoppersqlite/localfs_netbsd.go new file mode 100644 index 0000000..921f838 --- /dev/null +++ b/driver/hoppersqlite/localfs_netbsd.go @@ -0,0 +1,16 @@ +package hoppersqlite + +import "golang.org/x/sys/unix" + +// networkFilesystem reports a filesystem the kernel does not mark local +// (MNT_LOCAL), as NFS, SMB and FUSE mounts are not. +func networkFilesystem(path string) (string, error) { + var st unix.Statvfs_t + if err := unix.Statvfs(path, &st); err != nil { + return "", err + } + if uint64(st.Flag)&unix.MNT_LOCAL == 0 { + return "a filesystem not marked local (" + unix.ByteSliceToString(st.Fstypename[:]) + ")", nil + } + return "", nil +} diff --git a/driver/hoppersqlite/localfs_openbsd.go b/driver/hoppersqlite/localfs_openbsd.go new file mode 100644 index 0000000..e1e7f4e --- /dev/null +++ b/driver/hoppersqlite/localfs_openbsd.go @@ -0,0 +1,7 @@ +package hoppersqlite + +import "golang.org/x/sys/unix" + +func statfsFlags(st *unix.Statfs_t) uint64 { return uint64(st.F_flags) } + +func statfsType(st *unix.Statfs_t) string { return unix.ByteSliceToString(st.F_fstypename[:]) } diff --git a/driver/hoppersqlite/localfs_other.go b/driver/hoppersqlite/localfs_other.go new file mode 100644 index 0000000..c1fdf89 --- /dev/null +++ b/driver/hoppersqlite/localfs_other.go @@ -0,0 +1,11 @@ +//go:build !linux && !darwin && !freebsd && !openbsd && !netbsd && !windows + +package hoppersqlite + +import "errors" + +// networkFilesystem cannot tell on this system, and refuses rather than +// risk a database on a filesystem whose locks do not hold. +func networkFilesystem(string) (string, error) { + return "", errors.New("this system has no way to tell a local filesystem from a network one") +} diff --git a/driver/hoppersqlite/localfs_windows.go b/driver/hoppersqlite/localfs_windows.go new file mode 100644 index 0000000..aad0c0c --- /dev/null +++ b/driver/hoppersqlite/localfs_windows.go @@ -0,0 +1,32 @@ +package hoppersqlite + +import ( + "path/filepath" + "strings" + + "golang.org/x/sys/windows" +) + +// networkFilesystem reports a UNC path or a drive Windows reports as remote, +// such as a mapped network drive. +func networkFilesystem(path string) (string, error) { + abs, err := filepath.Abs(path) + if err != nil { + return "", err + } + if strings.HasPrefix(abs, `\\`) && !strings.HasPrefix(abs, `\\?\`) { + return "a network share", nil + } + p, err := windows.UTF16PtrFromString(abs) + if err != nil { + return "", err + } + volume := make([]uint16, windows.MAX_PATH+1) + if err := windows.GetVolumePathName(p, &volume[0], uint32(len(volume))); err != nil { + return "", err + } + if windows.GetDriveType(&volume[0]) == windows.DRIVE_REMOTE { + return "a network drive", nil + } + return "", nil +} diff --git a/driver/hoppersqlite/messaging.go b/driver/hoppersqlite/messaging.go new file mode 100644 index 0000000..4d56d9e --- /dev/null +++ b/driver/hoppersqlite/messaging.go @@ -0,0 +1,476 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "slices" + "strconv" + "time" + + "github.com/parallelworks/hopper/driver" +) + +// SubscriptionUpsert implements driver.Executor. +func (e *executor) SubscriptionUpsert(ctx context.Context, subs []driver.SubscriptionRow) error { + if len(subs) == 0 { + return nil + } + type row struct { + maxAttempts any + metadata string + } + rows := make([]row, len(subs)) + for i, s := range subs { + if s.MaxAttempts > 0 { + v, err := smallint(s.MaxAttempts, "max_attempts") + if err != nil { + return err + } + rows[i].maxAttempts = v + } + var err error + if rows[i].metadata, err = jsonValue(s.Metadata, "{}", "subscription metadata"); err != nil { + return err + } + } + err := e.write(ctx, func(x *w) error { + for i, s := range subs { + if _, err := x.ExecContext(ctx, ` + INSERT INTO hopper_subscriptions (name, pattern, kind, queue, max_attempts, metadata, created_at) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (name) DO UPDATE SET pattern = excluded.pattern, kind = excluded.kind, queue = excluded.queue, + max_attempts = excluded.max_attempts, metadata = excluded.metadata`, + s.Name, s.Pattern, s.Kind, s.Queue, rows[i].maxAttempts, rows[i].metadata, x.us()); err != nil { + return err + } + } + return nil + }) + if err != nil { + return fmt.Errorf("hopper: upsert subscriptions: %w", err) + } + return nil +} + +// SubscriptionList implements driver.Executor. +func (e *executor) SubscriptionList(ctx context.Context) ([]*driver.SubscriptionRow, error) { + subs, err := queryAll(ctx, e.read(), func(r scanner) (*driver.SubscriptionRow, error) { + var ( + s driver.SubscriptionRow + maxAttempts sql.NullInt64 + metadata string + created int64 + ) + if err := r.Scan(&s.Name, &s.Pattern, &s.Kind, &s.Queue, &maxAttempts, &metadata, &created); err != nil { + return nil, err + } + s.MaxAttempts, s.Metadata, s.CreatedAt = int(maxAttempts.Int64), json.RawMessage(metadata), time.UnixMicro(created).UTC() + return &s, nil + }, `SELECT name, pattern, kind, queue, max_attempts, metadata, created_at FROM hopper_subscriptions ORDER BY name`) + if err != nil { + return nil, fmt.Errorf("hopper: list subscriptions: %w", err) + } + return subs, nil +} + +// mergeObjects returns a JSON object with base's members and over's, over +// winning, as jsonb's || has it for two objects. A value that is not an +// object adds nothing. +func mergeObjects(base, over string) (string, error) { + m := map[string]json.RawMessage{} + _ = json.Unmarshal([]byte(base), &m) + var o map[string]json.RawMessage + _ = json.Unmarshal([]byte(over), &o) + for k, v := range o { + m[k] = v + } + b, err := json.Marshal(m) + return string(b), err +} + +// MessagePublish implements driver.Executor: it inserts one delivery per +// subscription whose pattern matches the topic. Each delivery's metadata +// records the topic, the message ID and the headers, merged with the +// subscription's own; a dedup key becomes the deliveries' unique key, +// scoped by kind, so each subscription deduplicates on its own. +func (e *executor) MessagePublish(ctx context.Context, p driver.MessagePublishParams) ([]driver.JobInsertResult, error) { + if p.Priority == 0 { + p.Priority = 2 + } + priority, err := smallint(p.Priority, "priority") + if err != nil { + return nil, err + } + headers := json.RawMessage("{}") + if p.Headers != nil { + if headers, err = json.Marshal(p.Headers); err != nil { + return nil, fmt.Errorf("hopper: publish: encode headers: %w", err) + } + } + payload, err := jsonValue(p.Payload, "null", "payload") + if err != nil { + return nil, err + } + var results []driver.JobInsertResult + err = e.write(ctx, func(x *w) error { + results = results[:0] + type sub struct { + kind, queue string + maxAttempts sql.NullInt64 + metadata string + } + subs, err := queryAll(ctx, x, func(r scanner) (sub, error) { + var s sub + err := r.Scan(&s.kind, &s.queue, &s.maxAttempts, &s.metadata) + return s, err + }, `SELECT kind, queue, max_attempts, metadata FROM hopper_subscriptions + WHERE hopper_topic_matches(pattern, ?) ORDER BY name`, p.Topic) + if err != nil { + return err + } + messageID := newID(x.now).String() + base, err := json.Marshal(map[string]any{"topic": p.Topic, "message_id": messageID, "headers": headers}) + if err != nil { + return err + } + var queues []string + for _, s := range subs { + metadata, err := mergeObjects(string(base), s.metadata) + if err != nil { + return err + } + attempts := 25 + switch { + case s.maxAttempts.Valid: + attempts = int(s.maxAttempts.Int64) + case p.OrderingKey != "": + attempts = 10 + } + j := jobInsert{ + kind: s.kind, queue: s.queue, priority: priority, maxAttempts: attempts, scheduledAt: x.now.Add(max(p.Delay, 0)), + args: payload, metadata: metadata, orderingKey: p.OrderingKey, ttl: p.TTL, await: p.Await, + } + if p.DedupKey != "" { + j.uniqueKey = "msg:" + p.DedupKey + } + r, err := x.insertJobs(ctx, []jobInsert{j}, driver.ConflictSkip) + if err != nil { + return err + } + results = append(results, r[0]) + if !slices.Contains(queues, s.queue) { + queues = append(queues, s.queue) + } + } + if p.Notify { + return x.notify(ctx, driver.ChannelInsert, queues...) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("hopper: publish: %w", err) + } + return results, nil +} + +// Streams. SQLite runs one writer at a time, so events commit in the order +// they are appended: an event's position is its seq, and a consumer has +// read every event up to the seq it has seen, with nothing committed later +// to sort before it. Positions carry no transaction ID (Xid is 0). + +func position(seq int64) driver.StreamPosition { return driver.StreamPosition{Seq: seq} } + +const streamEventColumns = `seq, topic, key, payload, headers, message_id, created_at` + +func scanEvent(r scanner) (*driver.StreamEvent, error) { + var ( + ev driver.StreamEvent + seq, created int64 + key sql.NullString + payload, headers string + messageID []byte + ) + if err := r.Scan(&seq, &ev.Topic, &key, &payload, &headers, &messageID, &created); err != nil { + return nil, err + } + var id driver.JobID + if err := id.Scan(messageID); err != nil { + return nil, err + } + ev.Position, ev.Key, ev.Payload, ev.MessageID, ev.CreatedAt = position(seq), key.String, json.RawMessage(payload), id.String(), time.UnixMicro(created).UTC() + if headers != "{}" { + if err := json.Unmarshal([]byte(headers), &ev.Headers); err != nil { + return nil, fmt.Errorf("headers: %w", err) + } + } + return &ev, nil +} + +// StreamAppend implements driver.Executor. +func (e *executor) StreamAppend(ctx context.Context, params driver.StreamAppendParams) (*driver.StreamEvent, error) { + headers := []byte("{}") + if params.Headers != nil { + var err error + if headers, err = json.Marshal(params.Headers); err != nil { + return nil, fmt.Errorf("hopper: append: encode headers: %w", err) + } + } + payload, err := jsonValue(params.Payload, "null", "payload") + if err != nil { + return nil, err + } + var ev *driver.StreamEvent + err = e.write(ctx, func(x *w) error { + var err error + if ev, err = scanEvent(x.QueryRowContext(ctx, `INSERT INTO hopper_stream_events (topic, key, payload, headers, message_id, created_at) + VALUES (?, ?, ?, ?, ?, ?) RETURNING `+streamEventColumns, + params.Topic, nullable(params.Key), payload, string(headers), idParam(newID(x.now)), x.us())); err != nil { + return err + } + if params.Notify { + return x.notify(ctx, driver.ChannelStream, params.Topic) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("hopper: append: %w", err) + } + return ev, nil +} + +// StreamRead implements driver.Executor. +func (e *executor) StreamRead(ctx context.Context, params driver.StreamReadParams) ([]*driver.StreamEvent, error) { + events, err := queryAll(ctx, e.read(), scanEvent, `SELECT `+streamEventColumns+` FROM hopper_stream_events + WHERE seq > ? AND (? = '' OR hopper_topic_matches(?, topic)) ORDER BY seq LIMIT ?`, + params.After.Seq, params.Pattern, params.Pattern, max(params.Limit, 1)) + if err != nil { + return nil, fmt.Errorf("hopper: read stream: %w", err) + } + return events, nil +} + +// StreamConsumerUpsert implements driver.Executor. A consumer starting at +// latest has seen every event appended so far. +func (e *executor) StreamConsumerUpsert(ctx context.Context, consumers []driver.StreamConsumerRow) error { + if len(consumers) == 0 { + return nil + } + err := e.write(ctx, func(x *w) error { + var latest int64 + if err := x.QueryRowContext(ctx, `SELECT coalesce(max(seq), 0) FROM hopper_stream_events`).Scan(&latest); err != nil { + return err + } + for _, c := range consumers { + var maxAttempts any + if c.MaxAttempts > 0 { + v, err := smallint(c.MaxAttempts, "max_attempts") + if err != nil { + return err + } + maxAttempts = v + } + metadata, err := jsonValue(c.Metadata, "{}", "consumer metadata") + if err != nil { + return err + } + seen := latest + if c.Start == driver.StreamStartEarliest { + seen = 0 + } + if _, err := x.ExecContext(ctx, ` + INSERT INTO hopper_stream_consumers (name, pattern, kind, queue, max_attempts, metadata, seen, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (name) DO UPDATE SET pattern = excluded.pattern, kind = excluded.kind, queue = excluded.queue, + max_attempts = excluded.max_attempts, metadata = excluded.metadata`, + c.Name, c.Pattern, c.Kind, c.Queue, maxAttempts, metadata, seen, x.us()); err != nil { + return err + } + } + return nil + }) + if err != nil { + return fmt.Errorf("hopper: upsert stream consumers: %w", err) + } + return nil +} + +// StreamConsumerList implements driver.Executor. A consumer's Snapshot is +// the seq it has read up to, and its Position the last event it delivered. +func (e *executor) StreamConsumerList(ctx context.Context) ([]*driver.StreamConsumerRow, error) { + consumers, err := queryAll(ctx, e.read(), func(r scanner) (*driver.StreamConsumerRow, error) { + var ( + c driver.StreamConsumerRow + maxAttempts sql.NullInt64 + metadata string + seen, pos, created int64 + delivered micros + ) + if err := r.Scan(&c.Name, &c.Pattern, &c.Kind, &c.Queue, &maxAttempts, &metadata, &seen, &pos, &delivered, &created); err != nil { + return nil, err + } + c.MaxAttempts, c.Metadata = int(maxAttempts.Int64), json.RawMessage(metadata) + c.Snapshot, c.Position = strconv.FormatInt(seen, 10), position(pos) + c.DeliveredAt, c.CreatedAt = delivered.time(), time.UnixMicro(created).UTC() + return &c, nil + }, `SELECT name, pattern, kind, queue, max_attempts, metadata, seen, position, delivered_at, created_at + FROM hopper_stream_consumers ORDER BY name`) + if err != nil { + return nil, fmt.Errorf("hopper: list stream consumers: %w", err) + } + return consumers, nil +} + +// StreamConsumerSeek implements driver.Executor. +func (e *executor) StreamConsumerSeek(ctx context.Context, name string, params driver.StreamSeekParams) error { + err := e.write(ctx, func(x *w) error { + var ( + seen int64 + latest bool + ) + switch { + case !params.Position.IsZero(): + seen = params.Position.Seq + case !params.Time.IsZero(): + // Just before the first event appended at or after the time. + err := x.QueryRowContext(ctx, `SELECT seq - 1 FROM hopper_stream_events WHERE created_at >= ? ORDER BY seq LIMIT 1`, + params.Time.UnixMicro()).Scan(&seen) + if errors.Is(err, sql.ErrNoRows) { + latest = true + break + } + if err != nil { + return err + } + case params.Earliest: + case params.Latest: + latest = true + default: + return errors.New("nothing to seek to") + } + if latest { + if err := x.QueryRowContext(ctx, `SELECT coalesce(max(seq), 0) FROM hopper_stream_events`).Scan(&seen); err != nil { + return err + } + } + res, err := x.ExecContext(ctx, `UPDATE hopper_stream_consumers SET seen = ?, position = ? WHERE name = ?`, seen, seen, name) + if err != nil { + return err + } + if n, err := res.RowsAffected(); err != nil || n == 0 { + return cmpErr(err, driver.ErrNotFound) + } + return nil + }) + if err != nil { + if errors.Is(err, driver.ErrNotFound) { + return err + } + return fmt.Errorf("hopper: seek stream consumer %s: %w", name, err) + } + return nil +} + +// cmpErr returns err, or def if err is nil. +func cmpErr(err, def error) error { + if err != nil { + return err + } + return def +} + +// StreamPump implements driver.Executor: it turns up to limit events past +// a consumer's seen seq that match its pattern into delivery jobs, and +// advances it past them, or past every event if fewer matched. Deliveries +// carry the topic, message ID, headers, consumer name and position in +// their metadata, over the consumer's own, and the event's key as their +// ordering key. +func (e *executor) StreamPump(ctx context.Context, name string, limit int) (driver.StreamPumpResult, error) { + var res driver.StreamPumpResult + limit = max(limit, 1) + err := e.write(ctx, func(x *w) error { + res = driver.StreamPumpResult{} + var ( + pattern, kind, queue, metadata string + maxAttempts sql.NullInt64 + seen, pos int64 + ) + err := x.QueryRowContext(ctx, `SELECT pattern, kind, queue, max_attempts, metadata, seen, position FROM hopper_stream_consumers WHERE name = ?`, + name).Scan(&pattern, &kind, &queue, &maxAttempts, &metadata, &seen, &pos) + if errors.Is(err, sql.ErrNoRows) { + return driver.ErrNotFound + } + if err != nil { + return err + } + res.Position = position(pos) + events, err := queryAll(ctx, x, scanEvent, `SELECT `+streamEventColumns+` FROM hopper_stream_events + WHERE seq > ? AND hopper_topic_matches(?, topic) ORDER BY seq LIMIT ?`, seen, pattern, limit) + if err != nil { + return err + } + for _, ev := range events { + headers, err := json.Marshal(ev.Headers) + if err != nil { + return err + } + if ev.Headers == nil { + headers = []byte("{}") + } + over, err := json.Marshal(map[string]any{ + "topic": ev.Topic, "message_id": ev.MessageID, "headers": json.RawMessage(headers), + "stream": name, "position": ev.Position.String(), + }) + if err != nil { + return err + } + meta, err := mergeObjects(metadata, string(over)) + if err != nil { + return err + } + attempts := 25 + switch { + case maxAttempts.Valid: + attempts = int(maxAttempts.Int64) + case ev.Key != "": + attempts = 10 + } + if _, err := x.insertJob(ctx, jobInsert{ + kind: kind, queue: queue, priority: 2, maxAttempts: attempts, args: string(ev.Payload), metadata: meta, orderingKey: ev.Key, + }); err != nil { + return err + } + res.Position = ev.Position + seen = ev.Position.Seq + } + res.Delivered = len(events) + if res.Delivered < limit { + // Every event matching the pattern has been delivered: the + // consumer has seen every event appended so far. + if err := x.QueryRowContext(ctx, `SELECT max(?, coalesce(max(seq), 0)) FROM hopper_stream_events`, seen).Scan(&seen); err != nil { + return err + } + } + var delivered any + if res.Delivered > 0 { + delivered = x.us() + res.Queues = []string{queue} + } + if _, err := x.ExecContext(ctx, `UPDATE hopper_stream_consumers SET seen = ?, position = ?, delivered_at = coalesce(?, delivered_at) WHERE name = ?`, + seen, res.Position.Seq, delivered, name); err != nil { + return err + } + if res.Delivered > 0 { + return x.notify(ctx, driver.ChannelInsert, queue) + } + return nil + }) + if err != nil { + if errors.Is(err, driver.ErrNotFound) { + return res, err + } + return res, fmt.Errorf("hopper: pump stream consumer %s: %w", name, err) + } + return res, nil +} diff --git a/driver/hoppersqlite/migrate_test.go b/driver/hoppersqlite/migrate_test.go new file mode 100644 index 0000000..d9b68be --- /dev/null +++ b/driver/hoppersqlite/migrate_test.go @@ -0,0 +1,381 @@ +package hoppersqlite_test + +import ( + "context" + "crypto/rand" + "encoding/json" + "fmt" + "log/slog" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/parallelworks/hopper" + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/hoppersqlite" + "github.com/parallelworks/hopper/hoppermigrate" +) + +var quiet = &hoppermigrate.Options{Logger: slog.New(slog.DiscardHandler)} + +// openEmpty returns a driver on a new database without hopper's schema. +func openEmpty(t *testing.T) *hoppersqlite.Driver { + t.Helper() + d, err := hoppersqlite.Open(filepath.Join(t.TempDir(), "hopper.db"), nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = d.Close() }) + return d +} + +// objects counts hopper's tables, indexes and triggers in the database. +func objects(t *testing.T, d *hoppersqlite.Driver) int { + t.Helper() + var n int + if err := d.DB().QueryRow(`SELECT count(*) FROM sqlite_schema WHERE name LIKE 'hopper%'`).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +// TestMigrate applies every version, reverts them all and applies them +// again: each script, up and down, runs, and down to zero leaves nothing of +// hopper's behind. +func TestMigrate(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := openEmpty(t) + if v, err := hoppermigrate.Version(ctx, d); err != nil || v != 0 { + t.Fatalf("version of an empty database = %d, %v", v, err) + } + if v, err := d.Executor().SchemaVersion(ctx); err != nil || v != 0 { + t.Fatalf("schema version of an empty database = %d, %v", v, err) + } + for range 2 { + res, err := hoppermigrate.Up(ctx, d, quiet) + if err != nil || res.Version != hoppermigrate.Latest() || len(res.Applied) != hoppermigrate.Latest() { + t.Fatalf("Up = %+v, %v", res, err) + } + if res, err := hoppermigrate.Up(ctx, d, quiet); err != nil || len(res.Applied) != 0 { + t.Fatalf("a second Up = %+v, %v", res, err) + } + if v, err := d.Executor().SchemaVersion(ctx); err != nil || v != hoppermigrate.Latest() { + t.Fatalf("schema version = %d, %v", v, err) + } + if _, err := hoppermigrate.Down(ctx, d, quiet); err != nil { + t.Fatalf("Down: %v", err) + } + if n := objects(t, d); n != 0 { + t.Fatalf("%d of hopper's objects left after Down to zero", n) + } + } + // Down and up one version at a time. + if _, err := hoppermigrate.Up(ctx, d, quiet); err != nil { + t.Fatal(err) + } + for v := hoppermigrate.Latest() - 1; v >= 0; v-- { + if _, err := hoppermigrate.Down(ctx, d, &hoppermigrate.Options{Logger: quiet.Logger, Target: v}); err != nil { + t.Fatalf("Down to %d: %v", v, err) + } + } + for v := 1; v <= hoppermigrate.Latest(); v++ { + if _, err := hoppermigrate.Up(ctx, d, &hoppermigrate.Options{Logger: quiet.Logger, Target: v}); err != nil { + t.Fatalf("Up to %d: %v", v, err) + } + } +} + +// TestMigrations checks the scripts Migrations returns: SQLite's, one up +// and one down for every version hoppermigrate knows, under its names, +// which applied in order by hand build the schema Up builds. +func TestMigrations(t *testing.T) { + t.Parallel() + ctx := context.Background() + pg := hoppermigrate.Migrations() + ms := hoppersqlite.Migrations() + if len(ms) != len(pg) { + t.Fatalf("%d migrations, want %d", len(ms), len(pg)) + } + d := openEmpty(t) + for i, m := range ms { + if m.Version != pg[i].Version || m.Name != pg[i].Name || m.Up == "" || m.Down == "" || m.Up == pg[i].Up { + t.Fatalf("migration %d = version %d %q, up %d bytes, down %d bytes; want SQLite's scripts of version %d %q", + i, m.Version, m.Name, len(m.Up), len(m.Down), pg[i].Version, pg[i].Name) + } + if _, err := d.DB().ExecContext(ctx, m.Up); err != nil { + t.Fatalf("version %d: %v", m.Version, err) + } + if _, err := d.DB().ExecContext(ctx, "INSERT INTO hopper_schema (version, applied_at) VALUES (?, 0)", m.Version); err != nil { + t.Fatal(err) + } + } + if v, err := hoppermigrate.Version(ctx, d); err != nil || v != hoppermigrate.Latest() { + t.Errorf("version after applying the scripts by hand = %d, %v", v, err) + } + if res, err := hoppermigrate.Up(ctx, d, quiet); err != nil || len(res.Applied) != 0 { + t.Errorf("Up after applying the scripts by hand = %+v, %v", res, err) + } +} + +// TestUpConcurrent runs Up from several Drivers on one database at once: +// one applies the migrations, and the others find them applied. +func TestUpConcurrent(t *testing.T) { + t.Parallel() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "hopper.db") + var applied atomic.Int64 + var wg sync.WaitGroup + for range 4 { + d, err := hoppersqlite.Open(path, nil) + if err != nil { + t.Fatal(err) + } + defer d.Close() + wg.Go(func() { + res, err := hoppermigrate.Up(ctx, d, quiet) + if err != nil { + t.Error(err) + return + } + applied.Add(int64(len(res.Applied))) + }) + } + wg.Wait() + if n := applied.Load(); n != int64(hoppermigrate.Latest()) { + t.Errorf("applied %d migrations in all, want %d", n, hoppermigrate.Latest()) + } +} + +type upgradeJob struct { + N int `json:"n"` +} + +func (upgradeJob) Kind() string { return "upgrade" } + +// TestUpgradeUnderTraffic migrates to the latest version from every +// earlier one, the upgrades a release can need, while jobs wait in every +// state and inserts, claims and finalizes keep running, and then has a +// client work every job. Those statements are written as a client of an +// earlier release would write them, with the columns of the first version, +// which every later one accepts. The migration holds the write lock while +// it runs, as every transaction that writes does, so they wait for it; none +// of them may fail, and no job may be lost. +func TestUpgradeUnderTraffic(t *testing.T) { + t.Parallel() + for from := 1; from < hoppermigrate.Latest(); from++ { + t.Run(fmt.Sprintf("from_v%d", from), func(t *testing.T) { + t.Parallel() + upgradeUnderTraffic(t, from) + }) + } +} + +func upgradeUnderTraffic(t *testing.T, from int) { + ctx := context.Background() + d := openEmpty(t) + if _, err := hoppermigrate.Up(ctx, d, &hoppermigrate.Options{Logger: quiet.Logger, Target: from}); err != nil { + t.Fatal(err) + } + db := d.DB() + now := time.Now().UnixMicro() + var next atomic.Int64 + insert := func(state string, scheduledAt int64) (driver.JobID, error) { + var id driver.JobID + _, _ = rand.Read(id[:]) + args, _ := json.Marshal(upgradeJob{N: int(next.Add(1))}) + _, err := db.ExecContext(ctx, `INSERT INTO hopper_jobs (id, kind, queue, state, priority, max_attempts, scheduled_at, args, created_at) + VALUES (?, 'upgrade', 'default', ?, 2, 5, ?, ?, ?)`, id[:], state, scheduledAt, string(args), time.Now().UnixMicro()) + return id, err + } + // Jobs in every state, as an earlier release leaves them: available, + // scheduled, retryable, running on a client that has since died, and + // finished in history. + for _, state := range []string{"available", "scheduled", "retryable"} { + at := now + if state == "scheduled" { + at = now + time.Second.Microseconds() + } + for range 5 { + if _, err := insert(state, at); err != nil { + t.Fatal(err) + } + } + } + var dead int64 + if err := db.QueryRowContext(ctx, `INSERT INTO hopper_clients (hostname, started_at, expires_at, info) VALUES ('gone', ?, ?, '{}') RETURNING id`, + now-time.Hour.Microseconds(), now-time.Minute.Microseconds()).Scan(&dead); err != nil { + t.Fatal(err) + } + for range 5 { + id, err := insert("available", now) + if err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, `UPDATE hopper_jobs SET state = 'running', attempt = 1, attempted_at = ?, attempted_by = ? WHERE id = ?`, + now, dead, id[:]); err != nil { + t.Fatal(err) + } + } + finished := func(id driver.JobID) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck // committed below + if _, err := tx.ExecContext(ctx, `INSERT INTO hopper_job_history (id, seq, kind, queue, state, priority, attempt, max_attempts, + scheduled_at, args, metadata, errors, await, created_at, finalized_at) + SELECT id, seq, kind, queue, 'completed', priority, attempt, max_attempts, scheduled_at, args, metadata, errors, await, + created_at, ? FROM hopper_jobs WHERE id = ?`, time.Now().UnixMicro(), id[:]); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `DELETE FROM hopper_jobs WHERE id = ?`, id[:]); err != nil { + return err + } + return tx.Commit() + } + for range 3 { + id, err := insert("available", now) + if err != nil { + t.Fatal(err) + } + if err := finished(id); err != nil { + t.Fatal(err) + } + } + + // Traffic while the migration runs: a producer inserting, and a client + // of the earlier release claiming and finishing jobs with statements + // every version accepts. A claimed job's number is recorded as worked. + var ( + mu sync.Mutex + worked = map[int]int{} + longest time.Duration + stop = make(chan struct{}) + wg sync.WaitGroup + ) + timed := func(fn func() error) error { + start := time.Now() + err := fn() + mu.Lock() + longest = max(longest, time.Since(start)) + mu.Unlock() + return err + } + wg.Go(func() { + for { + select { + case <-stop: + return + default: + } + if err := timed(func() error { _, err := insert("available", time.Now().UnixMicro()); return err }); err != nil { + t.Error(err) + return + } + time.Sleep(time.Millisecond) + } + }) + wg.Go(func() { + for { + select { + case <-stop: + return + default: + } + err := timed(func() error { + var ( + id []byte + args string + ) + err := db.QueryRowContext(ctx, `UPDATE hopper_jobs SET state = 'running', attempt = attempt + 1, attempted_at = ?, attempted_by = ? + WHERE seq = (SELECT seq FROM hopper_jobs WHERE state = 'available' AND scheduled_at <= ? ORDER BY seq LIMIT 1) + RETURNING id, args`, time.Now().UnixMicro(), dead, time.Now().UnixMicro()).Scan(&id, &args) + if err != nil { + return nil //nolint:nilerr // nothing to claim + } + var job upgradeJob + if err := json.Unmarshal([]byte(args), &job); err != nil { + return err + } + mu.Lock() + worked[job.N]++ + mu.Unlock() + return finished(driver.JobID(id)) + }) + if err != nil { + t.Error(err) + return + } + time.Sleep(time.Millisecond) + } + }) + time.Sleep(20 * time.Millisecond) + res, err := hoppermigrate.Up(ctx, d, quiet) + if err != nil { + t.Fatal(err) + } + if res.Version != hoppermigrate.Latest() { + t.Fatalf("Up = %+v", res) + } + time.Sleep(20 * time.Millisecond) + close(stop) + wg.Wait() + if bound := slowdown * 5 * time.Second; longest > bound { + t.Errorf("a statement waited %s for the migration, more than %s", longest, bound) + } + + // The release's client works every job left, the running ones once + // rescued from the dead client. + workers := hopper.NewWorkers() + hopper.AddWorkFunc(workers, func(_ context.Context, job *hopper.Job[upgradeJob]) error { + mu.Lock() + worked[job.Args.N]++ + mu.Unlock() + return nil + }) + c, err := hopper.NewClient(d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: 10}}, + Workers: workers, + PollInterval: 20 * time.Millisecond, + Logger: slog.New(slog.DiscardHandler), + }) + if err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + defer c.Stop(ctx) //nolint:errcheck // stopped below + total := int(next.Load()) + for deadline := time.Now().Add(30 * time.Second); ; time.Sleep(20 * time.Millisecond) { + var live int + if err := db.QueryRowContext(ctx, `SELECT count(*) FROM hopper_jobs`).Scan(&live); err != nil { + t.Fatal(err) + } + if live == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("%d jobs still live", live) + } + } + if err := c.Stop(ctx); err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + // The finished jobs inserted before the migration were never worked. + if len(worked) != total-3 { + t.Errorf("%d jobs worked, want %d", len(worked), total-3) + } + for n, times := range worked { + // A job running on the dead client was worked by it before (not + // recorded) and by the rescue after. + if times != 1 { + t.Errorf("job %d worked %d times", n, times) + } + } +} diff --git a/driver/hoppersqlite/migrations/001_schema.down.sql b/driver/hoppersqlite/migrations/001_schema.down.sql new file mode 100644 index 0000000..5391ab5 --- /dev/null +++ b/driver/hoppersqlite/migrations/001_schema.down.sql @@ -0,0 +1,9 @@ +DROP TABLE hopper_schema; +DROP TABLE hopper_notifications; +DROP TABLE hopper_subscriptions; +DROP TABLE hopper_periodic; +DROP TABLE hopper_queues; +DROP TABLE hopper_leader; +DROP TABLE hopper_clients; +DROP TABLE hopper_job_history; +DROP TABLE hopper_jobs; diff --git a/driver/hoppersqlite/migrations/001_schema.up.sql b/driver/hoppersqlite/migrations/001_schema.up.sql new file mode 100644 index 0000000..f1cdee7 --- /dev/null +++ b/driver/hoppersqlite/migrations/001_schema.up.sql @@ -0,0 +1,145 @@ +-- hopper schema v1 on SQLite. +-- +-- SQLite's scripts follow hoppermigrate's Postgres scripts version for +-- version, so that a version names the same tables and columns on both. +-- What only Postgres has (partitions, enum types, autovacuum settings, SQL +-- functions) is left out, and a version that changes only those is a script +-- of comments here. +-- +-- Times are INTEGER microseconds since the Unix epoch, which is Postgres's +-- timestamptz precision; IDs are 16-byte BLOBs, which sort as UUIDs do; JSON +-- values are TEXT. + +-- Live jobs only. Finalized jobs move to hopper_job_history, so this table and +-- its indexes stay sized to the backlog, not to all-time volume. seq is the +-- rowid, which AUTOINCREMENT never reuses: history keeps it, and it orders a +-- workflow's jobs. +CREATE TABLE hopper_jobs ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + id BLOB NOT NULL UNIQUE CHECK (length(id) = 16), + kind TEXT NOT NULL, + queue TEXT NOT NULL DEFAULT 'default', + state TEXT NOT NULL DEFAULT 'available' + CHECK (state IN ('pending', 'available', 'scheduled', 'running', 'retryable')), + priority INTEGER NOT NULL DEFAULT 2 CHECK (priority BETWEEN 1 AND 4), + attempt INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 25, + scheduled_at INTEGER NOT NULL, + attempted_at INTEGER, + attempted_by INTEGER, -- hopper_clients.id holding the claim + args TEXT NOT NULL DEFAULT '{}', + metadata TEXT NOT NULL DEFAULT '{}', + errors TEXT NOT NULL DEFAULT '[]', + unique_key TEXT, + ordering_key TEXT, + partition_key TEXT, + batch_id BLOB, + expires_at INTEGER, -- TTL: discarded if not started by then + cancel_requested_at INTEGER, + await INTEGER NOT NULL DEFAULT 0, -- notify waiters on finalize + created_at INTEGER NOT NULL +); + +-- Claim path: the only index the hot query touches. +CREATE INDEX hopper_jobs_claim ON hopper_jobs (queue, priority, scheduled_at, seq) + WHERE state IN ('available', 'scheduled', 'retryable'); + +-- Running set: rescue, cancel delivery, global limits. Bounded by total concurrency. +CREATE INDEX hopper_jobs_running ON hopper_jobs (queue, attempted_by) WHERE state = 'running'; + +-- Uniqueness applies only while a job is live. +CREATE UNIQUE INDEX hopper_jobs_unique ON hopper_jobs (kind, unique_key) WHERE unique_key IS NOT NULL; + +-- The expiry sweep's walk. +CREATE INDEX hopper_jobs_expires ON hopper_jobs (expires_at) WHERE expires_at IS NOT NULL; + +-- Finalized jobs, in one table: retention deletes them in batches. +CREATE TABLE hopper_job_history ( + id BLOB NOT NULL, + seq INTEGER NOT NULL, + kind TEXT NOT NULL, + queue TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('completed', 'canceled', 'discarded')), + priority INTEGER NOT NULL, + attempt INTEGER NOT NULL, + max_attempts INTEGER NOT NULL, + scheduled_at INTEGER NOT NULL, + attempted_at INTEGER, + attempted_by INTEGER, + args TEXT NOT NULL DEFAULT '{}', + metadata TEXT NOT NULL DEFAULT '{}', + errors TEXT NOT NULL DEFAULT '[]', + unique_key TEXT, + ordering_key TEXT, + partition_key TEXT, + batch_id BLOB, + expires_at INTEGER, + cancel_requested_at INTEGER, + await INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + finalized_at INTEGER NOT NULL, + output TEXT +); + +CREATE INDEX hopper_job_history_id ON hopper_job_history (id); +CREATE INDEX hopper_job_history_queue ON hopper_job_history (queue, finalized_at); +-- Retention's walk, by outcome. +CREATE INDEX hopper_job_history_retention ON hopper_job_history (state, finalized_at); + +-- Liveness: one lease row per running client process. +CREATE TABLE hopper_clients ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hostname TEXT NOT NULL, + started_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + info TEXT NOT NULL DEFAULT '{}' -- version, queues, pid +); + +CREATE TABLE hopper_leader ( + name TEXT PRIMARY KEY DEFAULT 'default', + client_id INTEGER NOT NULL, + elected_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL +); + +CREATE TABLE hopper_queues ( + name TEXT PRIMARY KEY, + paused_at INTEGER, + global_limit INTEGER, -- NULL = unlimited + rate_per_sec REAL, -- NULL = unlimited + rate_burst INTEGER, + tokens REAL, + refilled_at INTEGER, + metadata TEXT NOT NULL DEFAULT '{}', + updated_at INTEGER NOT NULL +); + +CREATE TABLE hopper_periodic ( -- last inserted slot per periodic job + name TEXT PRIMARY KEY, + last_slot INTEGER NOT NULL +); + +CREATE TABLE hopper_subscriptions ( + name TEXT PRIMARY KEY, + pattern TEXT NOT NULL, -- AMQP topic syntax + kind TEXT NOT NULL, + queue TEXT NOT NULL, + created_at INTEGER NOT NULL +); + +-- Notifications, which SQLite has no LISTEN for: a notifying transaction +-- writes its rows, which commit with it, and listeners in every process that +-- opens the database read the ones after the last they saw. Rows a minute +-- old are deleted as new ones are written. +CREATE TABLE hopper_notifications ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + channel TEXT NOT NULL, + payload TEXT NOT NULL, + created_at INTEGER NOT NULL +); +CREATE INDEX hopper_notifications_created ON hopper_notifications (created_at); + +CREATE TABLE hopper_schema ( + version INTEGER PRIMARY KEY, + applied_at INTEGER NOT NULL +); diff --git a/driver/hoppersqlite/migrations/002_messaging.down.sql b/driver/hoppersqlite/migrations/002_messaging.down.sql new file mode 100644 index 0000000..9f8038a --- /dev/null +++ b/driver/hoppersqlite/migrations/002_messaging.down.sql @@ -0,0 +1,4 @@ +DROP INDEX hopper_jobs_ordering; +DROP INDEX hopper_jobs_ordering_running; +ALTER TABLE hopper_subscriptions DROP COLUMN metadata; +ALTER TABLE hopper_subscriptions DROP COLUMN max_attempts; diff --git a/driver/hoppersqlite/migrations/002_messaging.up.sql b/driver/hoppersqlite/migrations/002_messaging.up.sql new file mode 100644 index 0000000..8a4996d --- /dev/null +++ b/driver/hoppersqlite/migrations/002_messaging.up.sql @@ -0,0 +1,17 @@ +-- hopper schema v2: messaging. +-- +-- Subscriptions carry their own retry budget, and deliveries with an +-- ordering key run one at a time per key. Postgres's SQL insert and publish +-- functions have no SQLite counterpart. + +ALTER TABLE hopper_subscriptions ADD COLUMN max_attempts INTEGER; -- NULL: the client default (10 for ordered deliveries) +ALTER TABLE hopper_subscriptions ADD COLUMN metadata TEXT NOT NULL DEFAULT '{}'; + +-- Backstop for ordering keys: the claim selects at most one delivery per +-- (queue, ordering_key), and this makes running two of one key impossible. +CREATE UNIQUE INDEX hopper_jobs_ordering_running ON hopper_jobs (queue, ordering_key) + WHERE state = 'running' AND ordering_key IS NOT NULL; + +-- Oldest-first lookup per ordering key for the claim. +CREATE INDEX hopper_jobs_ordering ON hopper_jobs (queue, ordering_key, seq) + WHERE ordering_key IS NOT NULL AND state IN ('available', 'scheduled', 'retryable'); diff --git a/driver/hoppersqlite/migrations/003_flow_control.down.sql b/driver/hoppersqlite/migrations/003_flow_control.down.sql new file mode 100644 index 0000000..b6b7fa9 --- /dev/null +++ b/driver/hoppersqlite/migrations/003_flow_control.down.sql @@ -0,0 +1,5 @@ +DROP INDEX hopper_jobs_batch; +DROP TABLE hopper_batches; +DROP INDEX hopper_jobs_partition_running; +ALTER TABLE hopper_queues DROP COLUMN aging_seconds; +ALTER TABLE hopper_queues DROP COLUMN partition_limit; diff --git a/driver/hoppersqlite/migrations/003_flow_control.up.sql b/driver/hoppersqlite/migrations/003_flow_control.up.sql new file mode 100644 index 0000000..85e9991 --- /dev/null +++ b/driver/hoppersqlite/migrations/003_flow_control.up.sql @@ -0,0 +1,26 @@ +-- hopper schema v3: flow control and batches. + +ALTER TABLE hopper_queues ADD COLUMN partition_limit INTEGER; -- max running jobs per partition_key, NULL = unlimited +ALTER TABLE hopper_queues ADD COLUMN aging_seconds INTEGER; -- raise a waiting job's priority after this long, NULL = off + +-- Partitioned limits count running jobs per partition key. +CREATE INDEX hopper_jobs_partition_running ON hopper_jobs (queue, partition_key) + WHERE state = 'running' AND partition_key IS NOT NULL; + +-- A batch counts its pending jobs down in the transaction that finalizes +-- them, which no other writes at the same time, and enqueues its callbacks +-- when the count reaches zero. +CREATE TABLE hopper_batches ( + id BLOB PRIMARY KEY CHECK (length(id) = 16), + pending INTEGER NOT NULL, + failed INTEGER NOT NULL DEFAULT 0, + total INTEGER NOT NULL, + on_success TEXT, -- {kind, queue, args, priority, max_attempts, metadata, tags} + on_failure TEXT, + on_complete TEXT, + metadata TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL, + completed_at INTEGER +); + +CREATE INDEX hopper_jobs_batch ON hopper_jobs (batch_id) WHERE batch_id IS NOT NULL; diff --git a/driver/hoppersqlite/migrations/004_workflows.down.sql b/driver/hoppersqlite/migrations/004_workflows.down.sql new file mode 100644 index 0000000..96874b6 --- /dev/null +++ b/driver/hoppersqlite/migrations/004_workflows.down.sql @@ -0,0 +1,4 @@ +DROP INDEX hopper_job_history_batch; +DROP TABLE hopper_job_deps; +ALTER TABLE hopper_batches DROP COLUMN edges; +ALTER TABLE hopper_batches DROP COLUMN name; diff --git a/driver/hoppersqlite/migrations/004_workflows.up.sql b/driver/hoppersqlite/migrations/004_workflows.up.sql new file mode 100644 index 0000000..2bc2499 --- /dev/null +++ b/driver/hoppersqlite/migrations/004_workflows.up.sql @@ -0,0 +1,23 @@ +-- hopper schema v4: workflows. +-- +-- A workflow is a batch whose jobs depend on each other. Dependents wait in +-- the 'pending' state; the transaction that finalizes a job promotes the +-- dependents whose dependencies are all finalized, and cancels the +-- dependents of a failed job (transitively) unless their edge says to +-- ignore the failure. hopper_job_deps is the working set of edges, deleted +-- as dependents finalize; the batch row keeps the whole graph. + +ALTER TABLE hopper_batches ADD COLUMN name TEXT; +ALTER TABLE hopper_batches ADD COLUMN edges TEXT; -- [[job_id, depends_on], ...] for workflows + +CREATE TABLE hopper_job_deps ( + job_id BLOB NOT NULL, -- the dependent, pending until depends_on is finalized + depends_on BLOB NOT NULL, + on_failure TEXT NOT NULL DEFAULT 'cancel', -- 'cancel' the dependent when depends_on fails, or 'ignore' + PRIMARY KEY (job_id, depends_on) +) WITHOUT ROWID; + +CREATE INDEX hopper_job_deps_depends_on ON hopper_job_deps (depends_on); + +-- Workflow inspection reads a batch's finished jobs from history. +CREATE INDEX hopper_job_history_batch ON hopper_job_history (batch_id) WHERE batch_id IS NOT NULL; diff --git a/driver/hoppersqlite/migrations/005_streams.down.sql b/driver/hoppersqlite/migrations/005_streams.down.sql new file mode 100644 index 0000000..c8d47fa --- /dev/null +++ b/driver/hoppersqlite/migrations/005_streams.down.sql @@ -0,0 +1,2 @@ +DROP TABLE hopper_stream_consumers; +DROP TABLE hopper_stream_events; diff --git a/driver/hoppersqlite/migrations/005_streams.up.sql b/driver/hoppersqlite/migrations/005_streams.up.sql new file mode 100644 index 0000000..7a5cf5b --- /dev/null +++ b/driver/hoppersqlite/migrations/005_streams.up.sql @@ -0,0 +1,31 @@ +-- hopper schema v5: streams. +-- +-- A stream is an append-only log of events. SQLite runs one writer at a +-- time, so events commit in the order they are appended, and an event's +-- position is its seq alone: a consumer reads the log past the last seq it +-- has seen, and nothing committed later can sort before it. + +CREATE TABLE hopper_stream_events ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + topic TEXT NOT NULL, + key TEXT, -- becomes the delivery's ordering key + payload TEXT NOT NULL DEFAULT 'null', + headers TEXT NOT NULL DEFAULT '{}', + message_id BLOB NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE INDEX hopper_stream_events_created ON hopper_stream_events (created_at); + +CREATE TABLE hopper_stream_consumers ( + name TEXT PRIMARY KEY, + pattern TEXT NOT NULL, -- AMQP topic syntax + kind TEXT NOT NULL, -- the delivery job kind + queue TEXT NOT NULL, + max_attempts INTEGER, + metadata TEXT NOT NULL DEFAULT '{}', + seen INTEGER NOT NULL DEFAULT 0, -- every event up to this seq has been read + position INTEGER NOT NULL DEFAULT 0, -- the last event delivered + delivered_at INTEGER, + created_at INTEGER NOT NULL +); diff --git a/driver/hoppersqlite/migrations/006_autovacuum.down.sql b/driver/hoppersqlite/migrations/006_autovacuum.down.sql new file mode 100644 index 0000000..59ffeec --- /dev/null +++ b/driver/hoppersqlite/migrations/006_autovacuum.down.sql @@ -0,0 +1 @@ +-- Nothing to revert: v6 changes nothing on SQLite. diff --git a/driver/hoppersqlite/migrations/006_autovacuum.up.sql b/driver/hoppersqlite/migrations/006_autovacuum.up.sql new file mode 100644 index 0000000..0564685 --- /dev/null +++ b/driver/hoppersqlite/migrations/006_autovacuum.up.sql @@ -0,0 +1,3 @@ +-- hopper schema v6: autovacuum settings for Postgres's live table. SQLite +-- has no autovacuum to tune: the leader refreshes the planner's statistics +-- when the live table's size changes (JobsMaintain). diff --git a/driver/hoppersqlite/migrations/007_vacuum_truncate.down.sql b/driver/hoppersqlite/migrations/007_vacuum_truncate.down.sql new file mode 100644 index 0000000..9e5e853 --- /dev/null +++ b/driver/hoppersqlite/migrations/007_vacuum_truncate.down.sql @@ -0,0 +1 @@ +-- Nothing to revert: v7 changes nothing on SQLite. diff --git a/driver/hoppersqlite/migrations/007_vacuum_truncate.up.sql b/driver/hoppersqlite/migrations/007_vacuum_truncate.up.sql new file mode 100644 index 0000000..b33d791 --- /dev/null +++ b/driver/hoppersqlite/migrations/007_vacuum_truncate.up.sql @@ -0,0 +1,2 @@ +-- hopper schema v7: no heap truncation on Postgres's live table. SQLite +-- reuses the pages of deleted rows, and never runs a VACUUM on its own. diff --git a/driver/hoppersqlite/migrations/008_function_schema.down.sql b/driver/hoppersqlite/migrations/008_function_schema.down.sql new file mode 100644 index 0000000..0bb276e --- /dev/null +++ b/driver/hoppersqlite/migrations/008_function_schema.down.sql @@ -0,0 +1 @@ +-- Nothing to revert: v8 changes nothing on SQLite. diff --git a/driver/hoppersqlite/migrations/008_function_schema.up.sql b/driver/hoppersqlite/migrations/008_function_schema.up.sql new file mode 100644 index 0000000..15e9dc7 --- /dev/null +++ b/driver/hoppersqlite/migrations/008_function_schema.up.sql @@ -0,0 +1,2 @@ +-- hopper schema v8: binds Postgres's SQL producer functions to their +-- schema. SQLite has no such functions. diff --git a/driver/hoppersqlite/migrations/009_snooze_reason.down.sql b/driver/hoppersqlite/migrations/009_snooze_reason.down.sql new file mode 100644 index 0000000..c76ec6b --- /dev/null +++ b/driver/hoppersqlite/migrations/009_snooze_reason.down.sql @@ -0,0 +1 @@ +ALTER TABLE hopper_jobs DROP COLUMN snooze_reason; diff --git a/driver/hoppersqlite/migrations/009_snooze_reason.up.sql b/driver/hoppersqlite/migrations/009_snooze_reason.up.sql new file mode 100644 index 0000000..9fb0965 --- /dev/null +++ b/driver/hoppersqlite/migrations/009_snooze_reason.up.sql @@ -0,0 +1,8 @@ +-- hopper schema v9: why a snoozed job is waiting. +-- +-- A worker can give a reason when it snoozes a job. The finalize stores it +-- with the snooze, and the claim, every other finalize, a retry and a +-- replacing insert clear it, so it only ever describes the current wait. +-- History has no such column: a finalized job is not waiting. + +ALTER TABLE hopper_jobs ADD COLUMN snooze_reason TEXT; diff --git a/driver/hoppersqlite/migrations/010_wake_requested.down.sql b/driver/hoppersqlite/migrations/010_wake_requested.down.sql new file mode 100644 index 0000000..6e0e6cb --- /dev/null +++ b/driver/hoppersqlite/migrations/010_wake_requested.down.sql @@ -0,0 +1 @@ +ALTER TABLE hopper_jobs DROP COLUMN wake_requested_at; diff --git a/driver/hoppersqlite/migrations/010_wake_requested.up.sql b/driver/hoppersqlite/migrations/010_wake_requested.up.sql new file mode 100644 index 0000000..bba92de --- /dev/null +++ b/driver/hoppersqlite/migrations/010_wake_requested.up.sql @@ -0,0 +1,9 @@ +-- hopper schema v10: a wake that finds its key's job running. +-- +-- JobRetryOrderingKey brings a key's waiting jobs forward, and marks a job +-- of the key that is running at the time. If that job then snoozes or +-- fails, the finalize makes it runnable at once instead of after its delay, +-- and clears the mark. Only a mark set since the job's current attempt +-- started counts. History has no such column. + +ALTER TABLE hopper_jobs ADD COLUMN wake_requested_at INTEGER; diff --git a/driver/hoppersqlite/migrations/011_batch_completion.down.sql b/driver/hoppersqlite/migrations/011_batch_completion.down.sql new file mode 100644 index 0000000..ff8892f --- /dev/null +++ b/driver/hoppersqlite/migrations/011_batch_completion.down.sql @@ -0,0 +1 @@ +-- Nothing to revert: v11 changes nothing on SQLite. diff --git a/driver/hoppersqlite/migrations/011_batch_completion.up.sql b/driver/hoppersqlite/migrations/011_batch_completion.up.sql new file mode 100644 index 0000000..88d95dd --- /dev/null +++ b/driver/hoppersqlite/migrations/011_batch_completion.up.sql @@ -0,0 +1,5 @@ +-- hopper schema v11: on Postgres, batch progress in rows of its own, so +-- that the concurrent finalizes of a batch's jobs do not queue on its row. +-- SQLite runs one writer at a time: the transaction that finalizes a +-- batch's jobs counts them down on the batch row, and nothing queues there +-- that would not queue anyway. diff --git a/driver/hoppersqlite/migrations/012_canceled_state.down.sql b/driver/hoppersqlite/migrations/012_canceled_state.down.sql new file mode 100644 index 0000000..b244346 --- /dev/null +++ b/driver/hoppersqlite/migrations/012_canceled_state.down.sql @@ -0,0 +1 @@ +-- Nothing to revert: v12 changes nothing on SQLite. diff --git a/driver/hoppersqlite/migrations/012_canceled_state.up.sql b/driver/hoppersqlite/migrations/012_canceled_state.up.sql new file mode 100644 index 0000000..6f112fe --- /dev/null +++ b/driver/hoppersqlite/migrations/012_canceled_state.up.sql @@ -0,0 +1,2 @@ +-- hopper schema v12: Postgres's canceled state is spelled canceled. SQLite's +-- schema has spelled it so from v1. diff --git a/driver/hoppersqlite/migrations/013_history_ordering.down.sql b/driver/hoppersqlite/migrations/013_history_ordering.down.sql new file mode 100644 index 0000000..4215c5e --- /dev/null +++ b/driver/hoppersqlite/migrations/013_history_ordering.down.sql @@ -0,0 +1 @@ +DROP INDEX hopper_job_history_ordering; diff --git a/driver/hoppersqlite/migrations/013_history_ordering.up.sql b/driver/hoppersqlite/migrations/013_history_ordering.up.sql new file mode 100644 index 0000000..c011e61 --- /dev/null +++ b/driver/hoppersqlite/migrations/013_history_ordering.up.sql @@ -0,0 +1,4 @@ +-- hopper schema v13: list a key's finished jobs without reading all history. + +CREATE INDEX hopper_job_history_ordering ON hopper_job_history (ordering_key, id) + WHERE ordering_key IS NOT NULL; diff --git a/driver/hoppersqlite/migrations/014_history_unique.down.sql b/driver/hoppersqlite/migrations/014_history_unique.down.sql new file mode 100644 index 0000000..8a29894 --- /dev/null +++ b/driver/hoppersqlite/migrations/014_history_unique.down.sql @@ -0,0 +1 @@ +DROP INDEX hopper_job_history_unique; diff --git a/driver/hoppersqlite/migrations/014_history_unique.up.sql b/driver/hoppersqlite/migrations/014_history_unique.up.sql new file mode 100644 index 0000000..2565c55 --- /dev/null +++ b/driver/hoppersqlite/migrations/014_history_unique.up.sql @@ -0,0 +1,5 @@ +-- hopper schema v14: find a unique key's finished jobs without reading all +-- history, for an insert whose UniqueOpts.Finished names finished states. + +CREATE INDEX hopper_job_history_unique ON hopper_job_history (kind, unique_key, finalized_at) + WHERE unique_key IS NOT NULL; diff --git a/driver/hoppersqlite/migrations/015_job_progress.down.sql b/driver/hoppersqlite/migrations/015_job_progress.down.sql new file mode 100644 index 0000000..96c86a1 --- /dev/null +++ b/driver/hoppersqlite/migrations/015_job_progress.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE hopper_jobs DROP COLUMN progress; +ALTER TABLE hopper_job_history DROP COLUMN progress; diff --git a/driver/hoppersqlite/migrations/015_job_progress.up.sql b/driver/hoppersqlite/migrations/015_job_progress.up.sql new file mode 100644 index 0000000..32aa871 --- /dev/null +++ b/driver/hoppersqlite/migrations/015_job_progress.up.sql @@ -0,0 +1,8 @@ +-- hopper schema v15: a job's progress through its steps. +-- +-- A worker records the last step it finished, and optionally a cursor within +-- the next, so that a later attempt resumes there. The progress survives +-- retries, snoozes and a JobRetry from history, so history keeps it too. + +ALTER TABLE hopper_job_history ADD COLUMN progress TEXT; +ALTER TABLE hopper_jobs ADD COLUMN progress TEXT; diff --git a/driver/hoppersqlite/migrations/016_job_logs.down.sql b/driver/hoppersqlite/migrations/016_job_logs.down.sql new file mode 100644 index 0000000..73cd280 --- /dev/null +++ b/driver/hoppersqlite/migrations/016_job_logs.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE hopper_jobs DROP COLUMN logs; +ALTER TABLE hopper_job_history DROP COLUMN logs; diff --git a/driver/hoppersqlite/migrations/016_job_logs.up.sql b/driver/hoppersqlite/migrations/016_job_logs.up.sql new file mode 100644 index 0000000..293b6e7 --- /dev/null +++ b/driver/hoppersqlite/migrations/016_job_logs.up.sql @@ -0,0 +1,6 @@ +-- hopper schema v16: the log lines of a job's latest attempts, written by +-- the transaction that finalizes each attempt and kept through retries and +-- a JobRetry from history, so history keeps them too. + +ALTER TABLE hopper_job_history ADD COLUMN logs TEXT; +ALTER TABLE hopper_jobs ADD COLUMN logs TEXT; diff --git a/driver/hoppersqlite/migrations/017_job_tags.down.sql b/driver/hoppersqlite/migrations/017_job_tags.down.sql new file mode 100644 index 0000000..eb3ef1f --- /dev/null +++ b/driver/hoppersqlite/migrations/017_job_tags.down.sql @@ -0,0 +1,10 @@ +DROP TRIGGER hopper_job_history_tags_update; +DROP TRIGGER hopper_job_history_tags_delete; +DROP TRIGGER hopper_job_history_tags_insert; +DROP TRIGGER hopper_jobs_tags_update; +DROP TRIGGER hopper_jobs_tags_delete; +DROP TRIGGER hopper_jobs_tags_insert; +DROP TABLE hopper_job_history_tags; +DROP TABLE hopper_job_tags; +ALTER TABLE hopper_jobs DROP COLUMN tags; +ALTER TABLE hopper_job_history DROP COLUMN tags; diff --git a/driver/hoppersqlite/migrations/017_job_tags.up.sql b/driver/hoppersqlite/migrations/017_job_tags.up.sql new file mode 100644 index 0000000..4c1db13 --- /dev/null +++ b/driver/hoppersqlite/migrations/017_job_tags.up.sql @@ -0,0 +1,72 @@ +-- hopper schema v17: tags, for finding jobs by label. +-- +-- A job's tags are a JSON array of strings on its row, NULL when it has +-- none, and they are kept in history like the rest of the job. A CHECK keeps +-- a direct SQL write from storing anything but an array; a read skips what +-- in one is not a non-empty string, once each. +-- +-- JobList finds a tag's jobs through tables of (tag, id) rows, one for live +-- jobs and one for history, whose primary keys walk a tag's jobs in ID +-- order. Triggers keep them in step with the tags column on every write, +-- whoever makes it. + +ALTER TABLE hopper_job_history ADD COLUMN tags TEXT CHECK (tags IS NULL OR json_type(tags) = 'array'); +ALTER TABLE hopper_jobs ADD COLUMN tags TEXT CHECK (tags IS NULL OR json_type(tags) = 'array'); + +CREATE TABLE hopper_job_tags ( + tag TEXT NOT NULL, + id BLOB NOT NULL, + PRIMARY KEY (tag, id) +) WITHOUT ROWID; + +CREATE TABLE hopper_job_history_tags ( + tag TEXT NOT NULL, + id BLOB NOT NULL, + PRIMARY KEY (tag, id) +) WITHOUT ROWID; + +CREATE TRIGGER hopper_jobs_tags_insert AFTER INSERT ON hopper_jobs +WHEN NEW.tags IS NOT NULL +BEGIN + INSERT OR IGNORE INTO hopper_job_tags (tag, id) + SELECT value, NEW.id FROM json_each(NEW.tags) WHERE type = 'text' AND value <> ''; +END; + +CREATE TRIGGER hopper_jobs_tags_delete AFTER DELETE ON hopper_jobs +WHEN OLD.tags IS NOT NULL +BEGIN + DELETE FROM hopper_job_tags + WHERE id = OLD.id AND tag IN (SELECT value FROM json_each(OLD.tags) WHERE type = 'text'); +END; + +CREATE TRIGGER hopper_jobs_tags_update AFTER UPDATE OF tags, id ON hopper_jobs +WHEN OLD.tags IS NOT NEW.tags OR OLD.id IS NOT NEW.id +BEGIN + DELETE FROM hopper_job_tags + WHERE OLD.tags IS NOT NULL AND id = OLD.id AND tag IN (SELECT value FROM json_each(OLD.tags) WHERE type = 'text'); + INSERT OR IGNORE INTO hopper_job_tags (tag, id) + SELECT value, NEW.id FROM json_each(coalesce(NEW.tags, '[]')) WHERE type = 'text' AND value <> ''; +END; + +CREATE TRIGGER hopper_job_history_tags_insert AFTER INSERT ON hopper_job_history +WHEN NEW.tags IS NOT NULL +BEGIN + INSERT OR IGNORE INTO hopper_job_history_tags (tag, id) + SELECT value, NEW.id FROM json_each(NEW.tags) WHERE type = 'text' AND value <> ''; +END; + +CREATE TRIGGER hopper_job_history_tags_delete AFTER DELETE ON hopper_job_history +WHEN OLD.tags IS NOT NULL +BEGIN + DELETE FROM hopper_job_history_tags + WHERE id = OLD.id AND tag IN (SELECT value FROM json_each(OLD.tags) WHERE type = 'text'); +END; + +CREATE TRIGGER hopper_job_history_tags_update AFTER UPDATE OF tags, id ON hopper_job_history +WHEN OLD.tags IS NOT NEW.tags OR OLD.id IS NOT NEW.id +BEGIN + DELETE FROM hopper_job_history_tags + WHERE OLD.tags IS NOT NULL AND id = OLD.id AND tag IN (SELECT value FROM json_each(OLD.tags) WHERE type = 'text'); + INSERT OR IGNORE INTO hopper_job_history_tags (tag, id) + SELECT value, NEW.id FROM json_each(coalesce(NEW.tags, '[]')) WHERE type = 'text' AND value <> ''; +END; diff --git a/driver/hoppersqlite/migrations/018_claim_fence.down.sql b/driver/hoppersqlite/migrations/018_claim_fence.down.sql new file mode 100644 index 0000000..7c68de9 --- /dev/null +++ b/driver/hoppersqlite/migrations/018_claim_fence.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE hopper_job_history DROP COLUMN fence; +ALTER TABLE hopper_jobs DROP COLUMN fence; diff --git a/driver/hoppersqlite/migrations/018_claim_fence.up.sql b/driver/hoppersqlite/migrations/018_claim_fence.up.sql new file mode 100644 index 0000000..51afb18 --- /dev/null +++ b/driver/hoppersqlite/migrations/018_claim_fence.up.sql @@ -0,0 +1,11 @@ +-- hopper schema v18: a fencing number per claim. +-- +-- Every claim increments a job's fence, so that a result, a progress write +-- or a rescue applies only to the claim it belongs to: a job rescued and +-- claimed again by the same client, or snoozed and claimed again at the +-- same attempt number, has a new fence. Snoozes, retries and the archive +-- keep it, and a JobRetry from history carries it back, so for a given job +-- it never repeats. History has the column for that reason. + +ALTER TABLE hopper_jobs ADD COLUMN fence INTEGER NOT NULL DEFAULT 0; +ALTER TABLE hopper_job_history ADD COLUMN fence INTEGER NOT NULL DEFAULT 0; diff --git a/driver/hoppersqlite/norace_test.go b/driver/hoppersqlite/norace_test.go new file mode 100644 index 0000000..983b5ce --- /dev/null +++ b/driver/hoppersqlite/norace_test.go @@ -0,0 +1,6 @@ +//go:build !race + +package hoppersqlite_test + +// slowdown is 1 without the race detector (race_test.go). +const slowdown = 1 diff --git a/driver/hoppersqlite/race_test.go b/driver/hoppersqlite/race_test.go new file mode 100644 index 0000000..a600377 --- /dev/null +++ b/driver/hoppersqlite/race_test.go @@ -0,0 +1,8 @@ +//go:build race + +package hoppersqlite_test + +// slowdown is how many times longer the race detector makes the driver's +// work, which is mostly SQLite's transpiled C: bounds on how long a +// statement may wait are multiplied by it. +const slowdown = 6 diff --git a/driver/hoppersqlite/resolve_internal_test.go b/driver/hoppersqlite/resolve_internal_test.go new file mode 100644 index 0000000..cd838ff --- /dev/null +++ b/driver/hoppersqlite/resolve_internal_test.go @@ -0,0 +1,53 @@ +package hoppersqlite + +import ( + "os" + "path/filepath" + "testing" +) + +// TestResolve checks that resolve follows links as the kernel does: ".." +// after a linked directory leads up from the link's target, not from the +// link, for a file that exists and one that does not yet; and a chain of +// links ends where the last leads. +func TestResolve(t *testing.T) { + t.Parallel() + local, remote := t.TempDir(), t.TempDir() + local, _ = filepath.EvalSymlinks(local) + remote, _ = filepath.EvalSymlinks(remote) + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(os.Mkdir(filepath.Join(remote, "inner"), 0o755)) + must(os.Symlink(filepath.Join(remote, "inner"), filepath.Join(local, "sub"))) + must(os.WriteFile(filepath.Join(remote, "other.db"), nil, 0o600)) + must(os.Symlink("sub/../other.db", filepath.Join(local, "existing.db"))) + must(os.Symlink("sub/../missing.db", filepath.Join(local, "dangling.db"))) + must(os.Symlink("dangling.db", filepath.Join(local, "chain.db"))) + for link, want := range map[string]string{ + "existing.db": filepath.Join(remote, "other.db"), + "dangling.db": filepath.Join(remote, "missing.db"), + "chain.db": filepath.Join(remote, "missing.db"), + "sub/../new.db": filepath.Join(remote, "new.db"), + "sub/./x/../../a.db": "", + "plain.db": filepath.Join(local, "plain.db"), + } { + got, err := resolve(local + string(filepath.Separator) + link) // not Join, which cleans ".." + if want == "" { + if err == nil { + t.Errorf("resolve(%s) = %s, want an error for a missing directory", link, got) + } + continue + } + if err != nil || got != want { + t.Errorf("resolve(%s) = %s, %v; want %s", link, got, err, want) + } + } + must(os.Symlink("loop.db", filepath.Join(local, "loop.db"))) + if got, err := resolve(filepath.Join(local, "loop.db")); err == nil { + t.Errorf("resolve of a link to itself = %s, want an error", got) + } +} diff --git a/driver/hoppersqlite/robustness_test.go b/driver/hoppersqlite/robustness_test.go new file mode 100644 index 0000000..dd20aa0 --- /dev/null +++ b/driver/hoppersqlite/robustness_test.go @@ -0,0 +1,555 @@ +package hoppersqlite_test + +import ( + "bytes" + "context" + "database/sql" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/parallelworks/hopper" + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/hoppersqlite" +) + +// longWrite is a write that runs for seconds, for a short context to +// interrupt. +const longWrite = `WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c WHERE x < 100000000) + INSERT INTO t SELECT x + 100 FROM c` + +// interrupt runs longWrite in tx under a context that ends while it runs, +// which makes SQLite roll the whole transaction back. +func interrupt(t *testing.T, tx *sql.Tx) { + t.Helper() + short, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + if _, err := tx.ExecContext(short, longWrite); err == nil { + t.Fatal("the long write was not interrupted") + } +} + +// TestInterruptedTransaction checks a transaction that SQLite rolled back +// itself, after a write its context interrupted: its later statements, the +// executor's included, are refused rather than run on their own; Commit +// returns ErrTransactionAborted and Rollback nothing; nothing it wrote is +// left; and the write lock is free again. +func TestInterruptedTransaction(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + db := d.DB() + if _, err := db.ExecContext(ctx, "CREATE TABLE t (x INTEGER)"); err != nil { + t.Fatal(err) + } + count := func() (rows, jobs int) { + t.Helper() + if err := db.QueryRowContext(ctx, "SELECT (SELECT count(*) FROM t), (SELECT count(*) FROM hopper_jobs)").Scan(&rows, &jobs); err != nil { + t.Fatal(err) + } + return rows, jobs + } + for _, end := range []string{"commit", "rollback"} { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err != nil { + t.Fatal(err) + } + interrupt(t, tx) + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (2)"); !errors.Is(err, hoppersqlite.ErrTransactionAborted) { + t.Errorf("a statement after the interrupted one = %v, want ErrTransactionAborted", err) + } + if err := tx.QueryRowContext(ctx, "SELECT count(*) FROM t").Scan(new(int)); !errors.Is(err, hoppersqlite.ErrTransactionAborted) { + t.Errorf("a query after the interrupted statement = %v, want ErrTransactionAborted", err) + } + _, err = d.UnwrapTx(tx).JobInsertMany(ctx, []driver.JobInsertParams{{Kind: "k", Queue: "default", Priority: 2, MaxAttempts: 1}}, driver.JobInsertOpts{}) + if !errors.Is(err, hoppersqlite.ErrTransactionAborted) { + t.Errorf("an insert through the transaction = %v, want ErrTransactionAborted", err) + } + if end == "commit" { + if err := tx.Commit(); !errors.Is(err, hoppersqlite.ErrTransactionAborted) { + t.Errorf("Commit = %v, want ErrTransactionAborted", err) + } + } else if err := tx.Rollback(); err != nil { + t.Errorf("Rollback = %v", err) + } + if rows, jobs := count(); rows != 0 || jobs != 0 { + t.Errorf("after %s: %d rows and %d jobs left of the transaction", end, rows, jobs) + } + wctx, cancel := context.WithTimeout(ctx, time.Second) + if _, err := db.ExecContext(wctx, "INSERT INTO t VALUES (3); DELETE FROM t"); err != nil { + t.Errorf("a write after %s: %v", end, err) + } + cancel() + } + + // An error that SQLite undoes only the statement of, such as a + // constraint's, leaves the transaction open, as on Postgres it would + // not; and so does a read that is interrupted. + if _, err := db.ExecContext(ctx, "CREATE UNIQUE INDEX t_x ON t (x)"); err != nil { + t.Fatal(err) + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer tx.Rollback() //nolint:errcheck // committed below + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err == nil || errors.Is(err, hoppersqlite.ErrTransactionAborted) { + t.Errorf("a duplicate = %v, want a constraint error", err) + } + short, cancel := context.WithTimeout(ctx, 30*time.Millisecond) + defer cancel() + if err := tx.QueryRowContext(short, `WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c WHERE x < 100000000) SELECT count(*) FROM c`).Scan(new(int)); err == nil { + t.Error("the long read was not interrupted") + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (2)"); err != nil { + t.Errorf("a statement after a constraint error and an interrupted read = %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + if rows, _ := count(); rows != 2 { + t.Errorf("t has %d rows, want 2", rows) + } +} + +// TestInterruptedFinalizeInTx finalizes a job in a worker's transaction +// after a statement of the transaction was interrupted: the finalize is +// refused, and the job stays running once the transaction rolls back, +// rather than completing in a write of its own. +func TestInterruptedFinalizeInTx(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + x := d.Executor() + clientID, err := x.ClientRegister(ctx, driver.ClientRegisterParams{Hostname: "test", TTL: time.Minute}) + if err != nil { + t.Fatal(err) + } + if _, err := x.JobInsertMany(ctx, []driver.JobInsertParams{{Kind: "k", Queue: "default", Priority: 2, MaxAttempts: 3}}, driver.JobInsertOpts{}); err != nil { + t.Fatal(err) + } + claimed, err := x.JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: clientID, Limit: 1}) + if err != nil || len(claimed.Jobs) != 1 { + t.Fatalf("claim = %+v, %v", claimed, err) + } + j := claimed.Jobs[0] + db := d.DB() + if _, err := db.ExecContext(ctx, "CREATE TABLE t (x INTEGER)"); err != nil { + t.Fatal(err) + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + interrupt(t, tx) + applied, err := d.UnwrapTx(tx).JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{{ + ID: j.ID, AttemptedBy: clientID, Fence: j.Fence, State: driver.JobStateCompleted, Archive: true, + }}}) + if !errors.Is(err, hoppersqlite.ErrTransactionAborted) || len(applied) != 0 { + t.Errorf("finalize in the transaction = %d applied, %v; want ErrTransactionAborted", len(applied), err) + } + if err := tx.Rollback(); err != nil { + t.Fatal(err) + } + got, err := x.JobGet(ctx, j.ID) + if err != nil { + t.Fatal(err) + } + if got.State != driver.JobStateRunning { + t.Errorf("the job is %s after its transaction rolled back, want running", got.State) + } +} + +// TestStaleFinalizeAfterRescue checks fencing once a job is rescued: the +// result of the client that lost it applies neither before the job is +// claimed again nor after, nor does a result with the old fence from the +// client that claimed it again; that client's own result does. +func TestStaleFinalizeAfterRescue(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + x := d.Executor() + register := func(host string) int64 { + t.Helper() + id, err := x.ClientRegister(ctx, driver.ClientRegisterParams{Hostname: host, TTL: time.Minute}) + if err != nil { + t.Fatal(err) + } + return id + } + a, b := register("a"), register("b") + if _, err := x.JobInsertMany(ctx, []driver.JobInsertParams{{Kind: "k", Queue: "default", Priority: 2, MaxAttempts: 5}}, driver.JobInsertOpts{}); err != nil { + t.Fatal(err) + } + ca, err := x.JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: a, Limit: 1}) + if err != nil || len(ca.Jobs) != 1 { + t.Fatalf("claim by a = %+v, %v", ca, err) + } + ja := ca.Jobs[0] + if _, err := d.DB().ExecContext(ctx, "UPDATE hopper_clients SET expires_at = 0 WHERE id = ?", a); err != nil { + t.Fatal(err) + } + candidates, err := x.JobRescueCandidates(ctx, driver.JobRescueParams{Limit: 10}) + if err != nil || len(candidates) != 1 { + t.Fatalf("rescue candidates = %+v, %v", candidates, err) + } + c := candidates[0].Job + applied, err := x.JobFinalizeMany(ctx, driver.JobFinalizeParams{SkipLocked: true, Jobs: []driver.JobFinalize{{ + ID: c.ID, AttemptedBy: c.AttemptedBy, Fence: c.Fence, State: driver.JobStateRetryable, Archive: true, + Error: &driver.AttemptError{Attempt: c.Attempt, Fence: c.Fence, Error: "client lost"}, + }}}) + if err != nil || len(applied) != 1 { + t.Fatalf("rescue = %d applied, %v", len(applied), err) + } + finalize := func(f driver.JobFinalize) int { + t.Helper() + applied, err := x.JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{f}}) + if err != nil { + t.Fatal(err) + } + return len(applied) + } + stale := driver.JobFinalize{ID: ja.ID, AttemptedBy: a, Fence: ja.Fence, State: driver.JobStateCompleted, Archive: true} + if n := finalize(stale); n != 0 { + t.Error("the lost client's result applied before the job was claimed again") + } + if _, err := d.DB().ExecContext(ctx, "UPDATE hopper_jobs SET scheduled_at = 0"); err != nil { + t.Fatal(err) + } + cb, err := x.JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: b, Limit: 1}) + if err != nil || len(cb.Jobs) != 1 || cb.Jobs[0].Fence != ja.Fence+1 { + t.Fatalf("claim by b = %+v, %v", cb, err) + } + if n := finalize(stale); n != 0 { + t.Error("the lost client's result applied after the job was claimed again") + } + stale.AttemptedBy = b + if n := finalize(stale); n != 0 { + t.Error("a result of b with the old fence applied") + } + if n := finalize(driver.JobFinalize{ID: ja.ID, AttemptedBy: b, Fence: cb.Jobs[0].Fence, State: driver.JobStateCompleted, Archive: true}); n != 1 { + t.Error("b's result did not apply") + } +} + +type selfArgs struct{} + +func (selfArgs) Kind() string { return "self" } + +// syncBuffer is a bytes.Buffer that a client's logger and the test share. +type syncBuffer struct { + mu sync.Mutex + b bytes.Buffer +} + +func (s *syncBuffer) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.Write(p) +} + +func (s *syncBuffer) String() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.String() +} + +// TestSelfWait runs a worker that writes on the pool while its own +// transaction holds the write lock, which would wait for itself until its +// context ended, and every write of the process with it: the write fails +// at once with ErrLockHeldByOwner, naming the job, and the client keeps +// its lease. +func TestSelfWait(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + var ( + took time.Duration + recErr error + done = make(chan struct{}) + ) + workers := hopper.NewWorkers() + hopper.AddWorkFunc(workers, func(ctx context.Context, job *hopper.Job[selfArgs]) error { + defer close(done) + tx, err := d.DB().BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck // committed below + if _, err := tx.ExecContext(ctx, "UPDATE hopper_queues SET updated_at = updated_at"); err != nil { + return err + } + start := time.Now() + recErr = hopper.RecordStep(ctx, job, "a", nil) + took = time.Since(start) + return tx.Commit() + }) + logs := &syncBuffer{} + c, err := hopper.NewClient(d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: 1}}, + Workers: workers, + JobTimeout: 10 * time.Second, // what a write that waited for itself would wait + LeaseTTL: 3 * time.Second, + Logger: slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelWarn})), + }) + if err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + defer c.Stop(ctx) //nolint:errcheck // cleanup + res, err := c.Insert(ctx, selfArgs{}, nil) + if err != nil { + t.Fatal(err) + } + <-done + if !errors.Is(recErr, hoppersqlite.ErrLockHeldByOwner) || !strings.Contains(recErr.Error(), res.Job.ID.String()) || took > time.Second { + t.Errorf("RecordStep in the worker's transaction = %v after %s, want ErrLockHeldByOwner naming the job at once", recErr, took) + } + for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(20 * time.Millisecond) { + j, err := c.JobGet(ctx, res.Job.ID) + if err != nil { + t.Fatal(err) + } + if j.State == hopper.JobStateCompleted { + break + } + if time.Now().After(deadline) { + t.Fatalf("the job is %s, want completed", j.State) + } + } + if strings.Contains(logs.String(), "lease") { + t.Errorf("the client's lease suffered:\n%s", logs) + } +} + +// TestJSONRefused checks that JSON Postgres's jsonb refuses (a \u0000 +// escape, text that is not UTF-8, a number beyond numeric's range) is +// refused on insert too, as it would be on Postgres. +func TestJSONRefused(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + for _, args := range []string{`{"a":"\u0000"}`, "{\"a\":\"\xff\"}", `{"a":"\ud800"}`, `{"a":1e200000}`} { + _, err := d.Executor().JobInsertMany(ctx, []driver.JobInsertParams{{Kind: "k", Queue: "default", Priority: 2, MaxAttempts: 1, Args: []byte(args)}}, driver.JobInsertOpts{}) + if err == nil { + t.Errorf("args %q inserted", args) + } + } + var n int + if err := d.DB().QueryRowContext(ctx, "SELECT count(*) FROM hopper_jobs").Scan(&n); err != nil || n != 0 { + t.Errorf("jobs = %d, %v", n, err) + } +} + +// TestLogTruncated checks that JobsMaintain cuts back a write-ahead log +// that grew while a reader held up checkpoints, once the reader is done. +func TestLogTruncated(t *testing.T) { + t.Parallel() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "hopper.db") + d := openAt(t, path) + x := d.Executor() + size := func() int64 { + st, err := os.Stat(path + "-wal") + if err != nil { + t.Fatal(err) + } + return st.Size() + } + args := []byte(`{"pad":"` + strings.Repeat("x", 4000) + `"}`) + p := make([]driver.JobInsertParams, 100) + for i := range p { + p[i] = driver.JobInsertParams{Kind: "k", Queue: "default", Priority: 2, MaxAttempts: 1, Args: args} + } + ro, err := d.DB().BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + t.Fatal(err) + } + if err := ro.QueryRowContext(ctx, "SELECT count(*) FROM hopper_jobs").Scan(new(int)); err != nil { + t.Fatal(err) + } + for range 40 { + if _, err := x.JobInsertMany(ctx, p, driver.JobInsertOpts{}); err != nil { + t.Fatal(err) + } + } + if err := ro.Rollback(); err != nil { + t.Fatal(err) + } + grown := size() + if grown < 16<<20 { + t.Fatalf("the log grew to %d bytes only", grown) + } + if _, err := x.JobsMaintain(ctx); err != nil { + t.Fatal(err) + } + if after := size(); after > 1<<20 { + t.Errorf("the log is %d bytes after JobsMaintain, from %d", after, grown) + } +} + +// TestQueryThatWrites checks that a statement that writes, run on its own +// with Query (INSERT ... RETURNING), waits for the write lock in turn, as +// one run with Exec does, saying who holds it when its context ends. +func TestQueryThatWrites(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + db := d.DB() + if _, err := db.ExecContext(ctx, "CREATE TABLE t (x INTEGER)"); err != nil { + t.Fatal(err) + } + // Open the connections the test needs beforehand, for its short + // context to be spent waiting for the lock. + conns := make([]*sql.Conn, 3) + for i := range conns { + c, err := db.Conn(ctx) + if err != nil { + t.Fatal(err) + } + conns[i] = c + } + for _, c := range conns { + _ = c.Close() + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err != nil { + t.Fatal(err) + } + short, cancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancel() + err = db.QueryRowContext(short, "INSERT INTO t VALUES (2) RETURNING x").Scan(new(int)) + if !errors.Is(err, context.DeadlineExceeded) || !strings.Contains(err.Error(), "write lock") { + t.Errorf("INSERT ... RETURNING while a transaction writes = %v, want it to wait for the write lock", err) + } + done := make(chan error, 1) + go func() { + var x int + err := db.QueryRowContext(ctx, "-- a comment\n insert INTO t VALUES (3) RETURNING x").Scan(&x) + done <- err + }() + for hoppersqlite.LockWaiters(d) == 0 { + time.Sleep(time.Millisecond) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + if err := <-done; err != nil { + t.Fatal(err) + } + var n int + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM t").Scan(&n); err != nil || n != 2 { + t.Errorf("rows = %d, %v; want 2", n, err) + } +} + +// TestOpenContext checks that OpenContext gives up when its context ends. +func TestOpenContext(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + d, err := hoppersqlite.OpenContext(ctx, filepath.Join(t.TempDir(), "hopper.db"), nil) + if err == nil { + _ = d.Close() + t.Fatal("OpenContext with an ended context succeeded") + } + if !errors.Is(err, context.Canceled) { + t.Errorf("OpenContext with an ended context = %v", err) + } +} + +type fanArgs struct{} + +func (fanArgs) Kind() string { return "fan" } + +type leafArgs struct{ N int } + +func (leafArgs) Kind() string { return "leaf" } + +// TestConcurrentWritesOfOneAttempt runs a worker that writes on the pool +// from eight goroutines at once, with no transaction of its own open, and +// then records a step while it inserts: every write waits its turn and +// succeeds, as on Postgres, rather than failing with ErrLockHeldByOwner +// because another write of the same attempt holds the lock. +func TestConcurrentWritesOfOneAttempt(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + var ( + mu sync.Mutex + errs []error + worked = make(chan struct{}) + ) + note := func(err error) { + if err != nil { + mu.Lock() + errs = append(errs, err) + mu.Unlock() + } + } + workers := hopper.NewWorkers() + hopper.AddWorkFunc(workers, func(ctx context.Context, job *hopper.Job[fanArgs]) error { + defer close(worked) + c, _ := hopper.ClientFromContext[*sql.Tx](ctx) + var wg sync.WaitGroup + for g := range 8 { + wg.Go(func() { + for i := range 25 { + _, err := c.Insert(ctx, leafArgs{N: g*100 + i}, nil) + note(err) + } + }) + } + wg.Wait() + wg.Go(func() { note(hopper.RecordStep(ctx, job, "a", nil)) }) + wg.Go(func() { _, err := c.Insert(ctx, leafArgs{N: -1}, nil); note(err) }) + wg.Wait() + return nil + }) + hopper.AddWorkFunc(workers, func(context.Context, *hopper.Job[leafArgs]) error { return nil }) + c, err := hopper.NewClient(d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: 1}}, + Workers: workers, + Logger: slog.New(slog.DiscardHandler), + }) + if err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + defer c.Stop(ctx) //nolint:errcheck // cleanup + if _, err := c.Insert(ctx, fanArgs{}, nil); err != nil { + t.Fatal(err) + } + select { + case <-worked: + case <-time.After(time.Minute): + t.Fatal("the worker did not finish") + } + mu.Lock() + defer mu.Unlock() + if len(errs) > 0 { + t.Errorf("%d of 202 writes of one attempt failed, the first: %v", len(errs), errs[0]) + } + var leaves int + if err := d.DB().QueryRowContext(ctx, "SELECT (SELECT count(*) FROM hopper_jobs WHERE kind = 'leaf') + (SELECT count(*) FROM hopper_job_history WHERE kind = 'leaf')").Scan(&leaves); err != nil || leaves != 201 { + t.Errorf("leaf jobs = %d, %v; want 201", leaves, err) + } +} diff --git a/driver/hoppersqlite/sqlite_test.go b/driver/hoppersqlite/sqlite_test.go new file mode 100644 index 0000000..d84c2aa --- /dev/null +++ b/driver/hoppersqlite/sqlite_test.go @@ -0,0 +1,546 @@ +package hoppersqlite_test + +import ( + "bufio" + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/parallelworks/hopper" + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/hoppersqlite" +) + +func TestOpenRefuses(t *testing.T) { + t.Parallel() + dir := t.TempDir() + for _, c := range []struct { + path string + cfg *hoppersqlite.Config + }{ + {"", nil}, + {":memory:", nil}, + {"file:" + filepath.Join(dir, "x.db"), nil}, + {filepath.Join(dir, "x.db"), &hoppersqlite.Config{Synchronous: "OFF"}}, + {filepath.Join(dir, "x.db"), &hoppersqlite.Config{ListenInterval: -time.Second}}, + {filepath.Join(dir, "missing", "x.db"), nil}, + } { + if d, err := hoppersqlite.Open(c.path, c.cfg); err == nil { + _ = d.Close() + t.Errorf("Open(%q, %+v) succeeded", c.path, c.cfg) + } + } +} + +// TestOpen opens one file twice in a process, as two clients of one +// application could: both see each other's writes, share one write lock, +// and keep the database in write-ahead log mode. +func TestOpen(t *testing.T) { + t.Parallel() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "hopper db?.sqlite") + first, err := hoppersqlite.Open(path, &hoppersqlite.Config{Synchronous: "normal"}) + if err != nil { + t.Fatal(err) + } + defer first.Close() + second, err := hoppersqlite.Open(path, nil) + if err != nil { + t.Fatal(err) + } + defer second.Close() + var mode string + if err := second.DB().QueryRowContext(ctx, "PRAGMA journal_mode").Scan(&mode); err != nil || mode != "wal" { + t.Errorf("journal mode = %q, %v", mode, err) + } + if _, err := first.DB().ExecContext(ctx, "CREATE TABLE t (x INTEGER)"); err != nil { + t.Fatal(err) + } + tx, err := first.DB().BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "INSERT INTO t VALUES (1)"); err != nil { + t.Fatal(err) + } + wctx, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + if _, err := second.DB().ExecContext(wctx, "INSERT INTO t VALUES (2)"); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("a write through the other Driver while the first's transaction is open = %v, want it to wait", err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + if _, err := second.DB().ExecContext(ctx, "INSERT INTO t VALUES (2)"); err != nil { + t.Fatal(err) + } + var n int + if err := first.DB().QueryRowContext(ctx, "SELECT count(*) FROM t").Scan(&n); err != nil || n != 2 { + t.Errorf("rows = %d, %v", n, err) + } +} + +// TestTransactions checks how the driver's transactions take the write +// lock: not when they begin but on their first statement, which therefore +// never reads what another transaction changes before it writes; and a +// read-only one not at all. +func TestTransactions(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + db := d.DB() + if _, err := db.ExecContext(ctx, "CREATE TABLE counter (n INTEGER); INSERT INTO counter VALUES (0)"); err != nil { + t.Fatal(err) + } + holder, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := holder.ExecContext(ctx, "UPDATE counter SET n = n + 1"); err != nil { + t.Fatal(err) + } + + // Begin returns at once while another transaction writes. + start := time.Now() + waiter, err := db.BeginTx(ctx, nil) + if err != nil || time.Since(start) > time.Second { + t.Fatalf("BeginTx while another transaction writes = %v after %s", err, time.Since(start)) + } + // A read-only transaction reads the last committed state meanwhile, and + // may not write. + ro, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true}) + if err != nil { + t.Fatal(err) + } + var n int + if err := ro.QueryRowContext(ctx, "SELECT n FROM counter").Scan(&n); err != nil || n != 0 { + t.Errorf("read-only read = %d, %v", n, err) + } + if _, err := ro.ExecContext(ctx, "UPDATE counter SET n = 100"); err == nil { + t.Error("a read-only transaction wrote") + } + if err := ro.Rollback(); err != nil { + t.Fatal(err) + } + // The waiter's first statement, a read, waits for the lock as long as + // its context allows. + short, cancel := context.WithTimeout(ctx, 50*time.Millisecond) + defer cancel() + if err := waiter.QueryRowContext(short, "SELECT n FROM counter").Scan(&n); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("first statement while another transaction writes = %v, want it to wait", err) + } + // Once the holder commits, the read sees its write, and a write after + // it succeeds: what it read cannot have gone stale. + done := make(chan error, 1) + go func() { + var seen int + if err := waiter.QueryRowContext(ctx, "SELECT n FROM counter").Scan(&seen); err != nil { + done <- err + return + } + if seen != 1 { + done <- fmt.Errorf("read %d after the holder committed, want 1", seen) + return + } + _, err := waiter.ExecContext(ctx, "UPDATE counter SET n = ?", seen+1) + done <- err + }() + for hoppersqlite.LockWaiters(d) == 0 { + time.Sleep(time.Millisecond) + } + if err := holder.Commit(); err != nil { + t.Fatal(err) + } + if err := <-done; err != nil { + t.Fatal(err) + } + if err := waiter.Commit(); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, "SELECT n FROM counter").Scan(&n); err != nil || n != 2 { + t.Errorf("counter = %d, %v; want 2", n, err) + } +} + +type countArgs struct { + N int `json:"n"` +} + +func (countArgs) Kind() string { return "count" } + +// TestClient runs hopper's client on the driver: two clients work jobs, +// one with an awaited result and one completed in the worker's own +// transaction, and a workflow, whose steps run after their dependencies. +func TestClient(t *testing.T) { + t.Parallel() + ctx := context.Background() + d := open(t) + var ( + mu sync.Mutex + done = map[int]int{} + ) + workers := hopper.NewWorkers() + hopper.AddWorkFunc(workers, func(ctx context.Context, job *hopper.Job[countArgs]) error { + mu.Lock() + done[job.Args.N]++ + mu.Unlock() + if job.Args.N == -1 { + return hopper.SetOutput(ctx, map[string]int{"answer": 42}) + } + if job.Args.N == -2 { + client, ok := hopper.ClientFromContext[*sql.Tx](ctx) + if !ok { + return errors.New("no client in the worker's context") + } + tx, err := d.DB().BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() //nolint:errcheck // committed below + if _, err := client.InsertTx(ctx, tx, countArgs{N: -3}, nil); err != nil { + return err + } + if err := hopper.CompleteTx(ctx, tx, job); err != nil { + return err + } + return tx.Commit() + } + return nil + }) + clients := make([]*hopper.Client[*sql.Tx], 2) + for i := range clients { + c, err := hopper.NewClient(d, &hopper.Config{ + Queues: map[string]hopper.QueueConfig{hopper.QueueDefault: {MaxWorkers: 10}}, + Workers: workers, + PollInterval: 50 * time.Millisecond, + }) + if err != nil { + t.Fatal(err) + } + if err := c.Start(ctx); err != nil { + t.Fatal(err) + } + defer c.Stop(ctx) //nolint:errcheck // cleanup + clients[i] = c + } + params := make([]hopper.InsertParams, 500) + for i := range params { + params[i] = hopper.InsertParams{Args: countArgs{N: i}} + } + if _, err := clients[0].InsertMany(ctx, params); err != nil { + t.Fatal(err) + } + awaited, err := clients[1].Insert(ctx, countArgs{N: -1}, &hopper.InsertOpts{Await: true}) + if err != nil { + t.Fatal(err) + } + if _, err := clients[1].Insert(ctx, countArgs{N: -2}, nil); err != nil { + t.Fatal(err) + } + wf := hopper.NewWorkflow("chain", nil) + first := wf.Add(countArgs{N: 1000}, nil) + second := wf.Add(countArgs{N: 1001}, hopper.After(first)) + wf.Add(countArgs{N: 1002}, hopper.After(second)) + flow, err := clients[0].InsertWorkflow(ctx, wf) + if err != nil { + t.Fatal(err) + } + + actx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + answer, err := hopper.Await[map[string]int](actx, clients[0], awaited.Job.ID) + if err != nil || answer["answer"] != 42 { + t.Fatalf("awaited job = %+v, %v", answer, err) + } + for deadline := time.Now().Add(30 * time.Second); ; time.Sleep(20 * time.Millisecond) { + mu.Lock() + finished := len(done) + mu.Unlock() + if finished >= 506 { // 500, -1, -2, -3 and the workflow's three + break + } + if time.Now().After(deadline) { + t.Fatalf("%d jobs worked, want %d", finished, 506) + } + } + mu.Lock() + if len(done) != 506 { + t.Errorf("%d jobs worked, want 506", len(done)) + } + for n, times := range done { + if times != 1 { + t.Errorf("job %d worked %d times", n, times) + } + } + mu.Unlock() + for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(20 * time.Millisecond) { + w, err := clients[1].WorkflowGet(ctx, flow.ID) + if err != nil { + t.Fatal(err) + } + if !w.Batch.CompletedAt.IsZero() { + var order []int + for _, j := range w.Jobs { + var args countArgs + if err := json.Unmarshal(j.Args, &args); err != nil { + t.Fatal(err) + } + order = append(order, args.N) + if j.State != hopper.JobStateCompleted { + t.Errorf("workflow job %d is %s", args.N, j.State) + } + } + if fmt.Sprint(order) != "[1000 1001 1002]" { + t.Errorf("workflow jobs = %v", order) + } + break + } + if time.Now().After(deadline) { + t.Fatalf("workflow not completed: %+v", w) + } + } +} + +// helperEnv names the role a re-executed test binary plays: one process of +// several that share a database. +const helperEnv = "HOPPERSQLITE_TEST_HELPER" + +// TestHelperProcess is not a test: TestAcrossProcesses runs the test binary +// again with helperEnv set, and this plays the role it names. +func TestHelperProcess(t *testing.T) { + role := os.Getenv(helperEnv) + if role == "" { + t.Skip("run by TestAcrossProcesses") + } + ctx := context.Background() + d, err := hoppersqlite.Open(os.Getenv("HOPPERSQLITE_TEST_DB"), nil) + if err != nil { + t.Fatal(err) + } + defer d.Close() + x := d.Executor() + switch role { + case "claim": + if err := claimAll(ctx, x, os.Stdout); err != nil { + t.Fatal(err) + } + case "notify": + if _, err := x.JobInsertMany(ctx, []driver.JobInsertParams{{Kind: "k", Queue: "from-another-process", Priority: 2, MaxAttempts: 1}}, + driver.JobInsertOpts{Notify: true}); err != nil { + t.Fatal(err) + } + case "worker": + chaosWorker(ctx, t, d) + case "hog": + hog(ctx, t, d) + case "hold": + tx, err := d.DB().BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tx.ExecContext(ctx, "UPDATE hopper_queues SET updated_at = updated_at"); err != nil { + t.Fatal(err) + } + fmt.Println("holding") + time.Sleep(time.Second) + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + } +} + +// claimAll claims and completes jobs of the default queue until none is +// left, writing the ID and fence of each claim to out. +func claimAll(ctx context.Context, exec driver.Executor, out *os.File) error { + clientID, err := exec.ClientRegister(ctx, driver.ClientRegisterParams{Hostname: "test", TTL: time.Minute}) + if err != nil { + return err + } + w := bufio.NewWriter(out) + defer w.Flush() + for { + res, err := exec.JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: clientID, Limit: 7}) + if err != nil { + return err + } + if len(res.Jobs) == 0 { + return nil + } + fin := make([]driver.JobFinalize, len(res.Jobs)) + for i, j := range res.Jobs { + fmt.Fprintf(w, "claimed %s %d\n", j.ID, j.Fence) + fin[i] = driver.JobFinalize{ID: j.ID, AttemptedBy: clientID, Fence: j.Fence, State: driver.JobStateCompleted, Archive: true} + } + applied, err := exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: fin}) + if err != nil { + return err + } + if len(applied) != len(fin) { + return fmt.Errorf("applied %d of %d results", len(applied), len(fin)) + } + } +} + +// output collects a process's output, which the test reads while the +// process writes it. +type output struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (o *output) Write(p []byte) (int, error) { + o.mu.Lock() + defer o.mu.Unlock() + return o.buf.Write(p) +} + +func (o *output) String() string { + o.mu.Lock() + defer o.mu.Unlock() + return o.buf.String() +} + +// helper starts the test binary as a process playing role on the database +// at path. +func helper(t *testing.T, path, role string) (*exec.Cmd, *output) { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^TestHelperProcess$", "-test.v") + cmd.Env = append(os.Environ(), helperEnv+"="+role, "HOPPERSQLITE_TEST_DB="+path) + out := &output{} + cmd.Stdout, cmd.Stderr = out, out + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + return cmd, out +} + +// TestAcrossProcesses shares one database among processes: claims by four +// processes and by four goroutines of this one take every job exactly +// once; a notification another process commits reaches this one's +// listener; and a write waits for another process's transaction. +func TestAcrossProcesses(t *testing.T) { + t.Parallel() + ctx := context.Background() + path := filepath.Join(t.TempDir(), "hopper.db") + d := openAt(t, path) + x := d.Executor() + + const jobs = 1200 + p := make([]driver.JobInsertParams, jobs) + for i := range p { + p[i] = driver.JobInsertParams{Kind: "k", Queue: "default", Priority: 1 + i%4, MaxAttempts: 1} + } + if _, err := x.JobInsertMany(ctx, p, driver.JobInsertOpts{}); err != nil { + t.Fatal(err) + } + type proc struct { + cmd *exec.Cmd + out *output + } + var procs []proc + for range 4 { + cmd, out := helper(t, path, "claim") + procs = append(procs, proc{cmd, out}) + } + outs := make([]*os.File, 4) + var wg sync.WaitGroup + for i := range outs { + f, err := os.Create(filepath.Join(t.TempDir(), "claims")) + if err != nil { + t.Fatal(err) + } + outs[i] = f + wg.Go(func() { + if err := claimAll(ctx, x, f); err != nil { + t.Error(err) + } + }) + } + wg.Wait() + claims := map[string]int{} + count := func(text string) { + for _, line := range strings.Split(text, "\n") { + if id, ok := strings.CutPrefix(line, "claimed "); ok { + claims[id]++ + } + } + } + for _, p := range procs { + if err := p.cmd.Wait(); err != nil { + t.Fatalf("claiming process: %v\n%s", err, p.out) + } + count(p.out.String()) + } + for _, f := range outs { + b, err := os.ReadFile(f.Name()) + if err != nil { + t.Fatal(err) + } + _ = f.Close() + count(string(b)) + } + twice := 0 + for _, n := range claims { + if n != 1 { + twice++ + } + } + if len(claims) != jobs || twice != 0 { + t.Errorf("%d jobs claimed, %d of them more than once; want %d, once each", len(claims), twice, jobs) + } + var history, fenced int + if err := d.DB().QueryRowContext(ctx, `SELECT count(*), count(*) FILTER (WHERE fence <> 1 OR attempt <> 1 OR state <> 'completed') FROM hopper_job_history`). + Scan(&history, &fenced); err != nil { + t.Fatal(err) + } + if history != jobs || fenced != 0 { + t.Errorf("history has %d jobs, %d not completed in one claim; want %d, 0", history, fenced, jobs) + } + + // A notification committed by another process. + l, err := d.Listener(ctx) + if err != nil { + t.Fatal(err) + } + defer l.Close(ctx) //nolint:errcheck // cleanup + if err := l.Listen(ctx, driver.ChannelInsert); err != nil { + t.Fatal(err) + } + cmd, out := helper(t, path, "notify") + if err := cmd.Wait(); err != nil { + t.Fatalf("notifying process: %v\n%s", err, out) + } + nctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if n, err := l.Next(nctx); err != nil || n.Payload != "from-another-process" { + t.Errorf("notification from another process = %+v, %v", n, err) + } + + // A write waits for another process's transaction, as long as its + // context allows. + cmd, out = helper(t, path, "hold") + for !strings.Contains(out.String(), "holding") { + time.Sleep(5 * time.Millisecond) + } + short, cancelShort := context.WithTimeout(ctx, 100*time.Millisecond) + defer cancelShort() + if err := x.QueueEnsure(short, []string{"q"}); !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("a write while another process's transaction is open = %v, want it to wait", err) + } + if err := x.QueueEnsure(ctx, []string{"q"}); err != nil { + t.Errorf("a write once it ends = %v", err) + } + if err := cmd.Wait(); err != nil { + t.Fatalf("holding process: %v\n%s", err, out) + } +} diff --git a/driver/hoppersqlite/workflow.go b/driver/hoppersqlite/workflow.go new file mode 100644 index 0000000..80811a0 --- /dev/null +++ b/driver/hoppersqlite/workflow.go @@ -0,0 +1,206 @@ +package hoppersqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "slices" + + "github.com/parallelworks/hopper/driver" +) + +// WorkflowInsert implements driver.Executor. +func (e *executor) WorkflowInsert(ctx context.Context, params driver.WorkflowInsertParams) (*driver.WorkflowInsertResult, error) { + n := len(params.Jobs) + if n == 0 { + return nil, errors.New("hopper: insert workflow: no jobs") + } + pending := make([]bool, n) + for _, d := range params.Deps { + if d.Job < 0 || d.Job >= n || d.DependsOn < 0 || d.DependsOn >= n || d.Job == d.DependsOn { + return nil, fmt.Errorf("hopper: insert workflow: dependency %d -> %d is out of range", d.Job, d.DependsOn) + } + pending[d.Job] = true + } + jobs := make([]jobInsert, n) + var notifyQueues []string + for i, p := range params.Jobs { + if p.UniqueKey != "" { + return nil, fmt.Errorf("hopper: insert workflow: job %d has a unique key", i) + } + var err error + if jobs[i], err = encodeInsert(p); err != nil { + return nil, err + } + jobs[i].pending = pending[i] + if !pending[i] && !slices.Contains(notifyQueues, p.Queue) { + notifyQueues = append(notifyQueues, p.Queue) + } + } + batch, err := encodeBatch(params.OnSuccess, params.OnFailure, params.OnComplete, params.Metadata) + if err != nil { + return nil, err + } + res := &driver.WorkflowInsertResult{} + err = e.write(ctx, func(x *w) error { + for i := range jobs { + jobs[i].id = newID(x.now) + } + // The graph, for inspection, and the working set of edges. + edges := make([][2]string, len(params.Deps)) + for i, d := range params.Deps { + edges[i] = [2]string{jobs[d.Job].id.String(), jobs[d.DependsOn].id.String()} + } + edgesJSON, err := json.Marshal(edges) + if err != nil { + return err + } + res.ID = newID(x.now) + for i := range jobs { + jobs[i].batchID = res.ID + } + if err := x.insertBatch(ctx, res.ID, params.Name, n, batch, string(edgesJSON)); err != nil { + return fmt.Errorf("insert batch: %w", err) + } + res.Jobs = make([]driver.JobInsertResult, n) + for i, j := range jobs { + job, err := x.insertJob(ctx, j) + if err != nil { + return fmt.Errorf("insert jobs: %w", err) + } + res.Jobs[i] = driver.JobInsertResult{Job: job} + } + for _, d := range params.Deps { + policy := d.OnFailure + if policy == "" { + policy = driver.DependencyCancel + } + if _, err := x.ExecContext(ctx, `INSERT OR IGNORE INTO hopper_job_deps (job_id, depends_on, on_failure) VALUES (?, ?, ?)`, + idParam(jobs[d.Job].id), idParam(jobs[d.DependsOn].id), string(policy)); err != nil { + return fmt.Errorf("insert dependencies: %w", err) + } + } + if params.Notify { + return x.notify(ctx, driver.ChannelInsert, notifyQueues...) + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("hopper: insert workflow: %w", err) + } + return res, nil +} + +// WorkflowGet implements driver.Executor. +func (e *executor) WorkflowGet(ctx context.Context, id driver.JobID) (*driver.WorkflowRow, error) { + var w driver.WorkflowRow + err := e.readTx(ctx, func(q querier) error { + var ( + name sql.NullString + edges sql.NullString + ) + b, err := scanBatch(q.QueryRowContext(ctx, `SELECT `+batchColumns+`, name, edges FROM hopper_batches WHERE id = ?`, idParam(id)), &name, &edges) + if errors.Is(err, sql.ErrNoRows) { + return driver.ErrNotFound + } + if err != nil { + return err + } + w.Batch = *b + w.Batch.Name = name.String + if edges.Valid { + var pairs [][2]driver.JobID + if err := json.Unmarshal([]byte(edges.String), &pairs); err != nil { + return fmt.Errorf("edges: %w", err) + } + w.Edges = make([]driver.JobEdge, len(pairs)) + for i, p := range pairs { + w.Edges[i] = driver.JobEdge{Job: p[0], DependsOn: p[1]} + } + } + // Jobs in insertion order, live and finished alike. + w.Jobs, err = queryAll(ctx, q, func(r scanner) (*driver.JobRow, error) { + var seq int64 + return scanJob(r, true, &seq) + }, `SELECT * FROM ( + SELECT `+liveAsHistory+`, seq FROM hopper_jobs WHERE batch_id = ?1 + UNION ALL + SELECT `+historyColumns+`, seq FROM hopper_job_history WHERE batch_id = ?1 + ) ORDER BY seq`, idParam(id)) + return err + }) + if err != nil { + if errors.Is(err, driver.ErrNotFound) { + return nil, err + } + return nil, fmt.Errorf("hopper: get workflow: %w", err) + } + return &w, nil +} + +// readySQL selects, of the IDs it is given, the pending jobs none of whose +// dependencies is still live. +const readySQL = `SELECT j.id FROM hopper_jobs j +WHERE j.id IN (%s) AND j.state = 'pending' + AND NOT EXISTS (SELECT 1 FROM hopper_job_deps d JOIN hopper_jobs x ON x.id = d.depends_on WHERE d.job_id = j.id)` + +// WorkflowPromoteReady implements driver.Executor. The transaction that +// finalizes a job promotes its dependents itself, so a pass finds only +// what a direct write left pending: it looks without the write lock, and +// takes the lock only to promote what it found. +func (e *executor) WorkflowPromoteReady(ctx context.Context, params driver.WorkflowPromoteParams) (driver.WorkflowPromoteResult, error) { + if params.DependsOn != nil && len(params.DependsOn) == 0 { + return driver.WorkflowPromoteResult{}, nil + } + var ( + res driver.WorkflowPromoteResult + candidates []driver.JobID + err error + ) + if params.DependsOn == nil { + candidates, err = queryIDs(ctx, e.read(), `SELECT DISTINCT job_id FROM hopper_job_deps WHERE job_id > ? ORDER BY job_id LIMIT ?`, + idParam(params.After), max(params.Limit, 1)) + } else { + ids := make([]any, len(params.DependsOn)) + for i, id := range params.DependsOn { + ids[i] = idParam(id) + } + candidates, err = queryIDs(ctx, e.read(), `SELECT DISTINCT job_id FROM hopper_job_deps WHERE depends_on IN (`+placeholders(len(ids))+`)`, ids...) + } + if err != nil { + return res, fmt.Errorf("hopper: promote ready workflow jobs: %w", err) + } + sortIDs(candidates) + res.Scanned = int64(len(candidates)) + if len(candidates) == 0 { + return res, nil + } + res.Last = candidates[len(candidates)-1] + ids := make([]any, len(candidates)) + for i, id := range candidates { + ids[i] = idParam(id) + } + ready, err := queryIDs(ctx, e.read(), fmt.Sprintf(readySQL, placeholders(len(ids))), ids...) + if err != nil || len(ready) == 0 { + return res, err + } + err = e.write(ctx, func(x *w) error { + res.Promoted = 0 + for _, id := range ready { + promoted, err := x.promoteIfReady(ctx, id) + if err != nil { + return err + } + if promoted { + res.Promoted++ + } + } + return nil + }) + if err != nil { + return driver.WorkflowPromoteResult{}, fmt.Errorf("hopper: promote ready workflow jobs: %w", err) + } + return res, nil +} diff --git a/driver/internal/jobcodec/jobcodec.go b/driver/internal/jobcodec/jobcodec.go new file mode 100644 index 0000000..a94493e --- /dev/null +++ b/driver/internal/jobcodec/jobcodec.go @@ -0,0 +1,120 @@ +// Package jobcodec encodes and decodes the JSON values a job row keeps +// beside its args and metadata: its errors, progress, logs and tags. Every +// engine stores them the same way, and reads them the same way: a value +// that a direct SQL write left in another form reads as what can be made of +// it, rather than failing the read, so that one such row fails neither the +// claim that takes it (whose update has committed by the time it is read) +// nor a listing or the rescuer. +package jobcodec + +import ( + "encoding/json" + "strings" + "time" + + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/internal/jobtags" +) + +// StorableText replaces what a database may refuse in text and JSON +// strings, NUL and invalid UTF-8, so that one worker's text cannot fail the +// statement that finalizes a whole batch. +func StorableText(s string) string { + return strings.ReplaceAll(strings.ToValidUTF8(s, "�"), "\x00", "�") +} + +// MarshalAttemptError encodes an error record without its timestamp, which +// the engine fills in from database time, its text made storable. +func MarshalAttemptError(e *driver.AttemptError) ([]byte, error) { + return json.Marshal(struct { + Attempt int `json:"attempt"` + Fence int64 `json:"fence,omitempty"` + Error string `json:"error"` + Trace string `json:"trace,omitempty"` + }{e.Attempt, e.Fence, StorableText(e.Error), StorableText(e.Trace)}) +} + +// MarshalAttemptLog encodes the log a finalize appends for result f, which +// a snooze marks Snoozed, its text made storable, or returns nil when there +// is none to keep (JobFinalize.KeepsLog). +func MarshalAttemptLog(f driver.JobFinalize) ([]byte, error) { + if f.Log == nil || !f.KeepsLog() { + return nil, nil + } + return json.Marshal(driver.AttemptLog{Attempt: f.Log.Attempt, Snoozed: f.Snooze, Log: StorableText(f.Log.Log)}) +} + +// DecodeErrors reads a job's errors, a JSON array of error records. An +// empty value reads as none. +func DecodeErrors(stored string) ([]driver.AttemptError, error) { + errs := []driver.AttemptError{} + if stored == "" { + return errs, nil + } + if err := json.Unmarshal([]byte(stored), &errs); err != nil { + return nil, err + } + return errs, nil +} + +// DecodeProgress reads a progress value, keeping what it can of one that is +// not in the form an engine writes: a value that is not an object reads as +// none. +func DecodeProgress(stored string) *driver.Progress { + var v struct { + Step any `json:"step"` + Cursor json.RawMessage `json:"cursor"` + At any `json:"at"` + } + if !strings.HasPrefix(strings.TrimSpace(stored), "{") || json.Unmarshal([]byte(stored), &v) != nil { + return nil + } + p := &driver.Progress{Cursor: v.Cursor} + if s, ok := v.Step.(string); ok { + p.Step = s + } + if s, ok := v.At.(string); ok { + p.RecordedAt, _ = time.Parse(time.RFC3339Nano, s) + } + if string(p.Cursor) == "null" { + p.Cursor = nil + } + return p +} + +// DecodeLogs reads a job's logs, skipping entries that are not in the form +// a finalize writes. +func DecodeLogs(stored string) []driver.AttemptLog { + // Anything but an array (or null) fails to decode into entries. + var entries []json.RawMessage + if json.Unmarshal([]byte(stored), &entries) != nil { + return nil + } + var logs []driver.AttemptLog + for _, raw := range entries { + var l driver.AttemptLog + if json.Unmarshal(raw, &l) == nil && l.Log != "" { + logs = append(logs, l) + } + } + return logs +} + +// DecodeTags reads a tags value, a JSON array of strings: a value that is +// not an array reads as no tags, and elements that are not strings are +// skipped, as are empty strings and repeats. +func DecodeTags(stored string) []string { + if stored == "" { + return nil + } + var elems []json.RawMessage + _ = json.Unmarshal([]byte(stored), &elems) // anything but an array reads as no tags + var tags []string + for _, e := range elems { + var s string + if len(e) > 0 && e[0] == '"' && json.Unmarshal(e, &s) == nil && s != "" { + tags = append(tags, s) + } + } + return jobtags.Distinct(tags) +} diff --git a/driver/internal/pgsql/jobs.go b/driver/internal/pgsql/jobs.go index f1e7000..143864f 100644 --- a/driver/internal/pgsql/jobs.go +++ b/driver/internal/pgsql/jobs.go @@ -5,7 +5,6 @@ import ( "cmp" "context" "database/sql" - "encoding/json" "errors" "fmt" "math" @@ -17,6 +16,7 @@ import ( "unicode/utf8" "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/driver/internal/jobcodec" "github.com/parallelworks/hopper/internal/jobtags" ) @@ -90,7 +90,7 @@ func scanJob(row Row, history bool, extra ...any) (*driver.JobRow, error) { j.BatchID = batchID.ID } if progress != "" { - j.Progress = decodeProgress(string(progress)) + j.Progress = jobcodec.DecodeProgress(string(progress)) } return &j, nil } @@ -695,9 +695,9 @@ func (e *Executor) JobFinalizeMany(ctx context.Context, params driver.JobFinaliz return nil, fmt.Errorf("hopper: finalize jobs: %s is not a finalize state", f.State) } ids[i], attemptedBy[i], fences[i], states[i] = f.ID, f.AttemptedBy, f.Fence, string(f.State) - delays[i], snoozes[i], reasons[i] = max(f.Delay, 0).Seconds(), f.Snooze, storableText(f.SnoozeReason) + delays[i], snoozes[i], reasons[i] = max(f.Delay, 0).Seconds(), f.Snooze, jobcodec.StorableText(f.SnoozeReason) if f.Error != nil { - b, err := marshalAttemptError(f.Error) + b, err := jobcodec.MarshalAttemptError(f.Error) if err != nil { return nil, fmt.Errorf("hopper: finalize jobs: %w", err) } @@ -707,13 +707,11 @@ func (e *Executor) JobFinalizeMany(ctx context.Context, params driver.JobFinaliz // Only completed jobs may skip history; the failed partitions are the // dead-letter queue. archives[i] = f.Archived() - if f.Log != nil && f.KeepsLog() { - b, err := json.Marshal(driver.AttemptLog{Attempt: f.Log.Attempt, Snoozed: f.Snooze, Log: storableText(f.Log.Log)}) - if err != nil { - return nil, fmt.Errorf("hopper: finalize jobs: %w", err) - } - logs[i] = b + b, err := jobcodec.MarshalAttemptLog(f) + if err != nil { + return nil, fmt.Errorf("hopper: finalize jobs: %w", err) } + logs[i] = b } args := []any{ uuidArray(ids), int64Array(attemptedBy), textArray(states), floatArray(delays), boolArray(snoozes), @@ -745,17 +743,6 @@ func keepLogs(keep int) int32 { return int32(keep) } -// marshalAttemptError encodes an error record without its timestamp, which -// the finalize statement fills in from database time. -func marshalAttemptError(e *driver.AttemptError) ([]byte, error) { - return json.Marshal(struct { - Attempt int `json:"attempt"` - Fence int64 `json:"fence,omitempty"` - Error string `json:"error"` - Trace string `json:"trace,omitempty"` - }{e.Attempt, e.Fence, storableText(e.Error), storableText(e.Trace)}) -} - var jobGetQuery = fmt.Sprintf(` SELECT %s, NULL::timestamptz AS finalized_at, NULL::jsonb AS output FROM {{schema}}.hopper_jobs WHERE id = $1 UNION ALL @@ -811,33 +798,7 @@ func (e *Executor) JobRecordProgress(ctx context.Context, params driver.JobRecor if err != nil { return nil, fmt.Errorf("hopper: record job progress: %w", err) } - return decodeProgress(stored), nil -} - -// decodeProgress reads a progress value, keeping what it can of one that -// is not in the form JobRecordProgress writes, such as one written by hand: -// a row must still scan, or a claim that took it would fail with the claim -// already applied. A value that is not an object reads as none. -func decodeProgress(stored string) *driver.Progress { - var v struct { - Step any `json:"step"` - Cursor json.RawMessage `json:"cursor"` - At any `json:"at"` - } - if !strings.HasPrefix(strings.TrimSpace(stored), "{") || json.Unmarshal([]byte(stored), &v) != nil { - return nil - } - p := &driver.Progress{Cursor: v.Cursor} - if s, ok := v.Step.(string); ok { - p.Step = s - } - if s, ok := v.At.(string); ok { - p.RecordedAt, _ = time.Parse(time.RFC3339Nano, s) - } - if string(p.Cursor) == "null" { - p.Cursor = nil - } - return p + return jobcodec.DecodeProgress(stored), nil } // JobGet implements driver.Executor. @@ -868,25 +829,7 @@ func (e *Executor) JobLogs(ctx context.Context, id driver.JobID) ([]driver.Attem if err != nil { return nil, fmt.Errorf("hopper: get job logs: %w", err) } - return decodeLogs(string(logs)), nil -} - -// decodeLogs reads a job's logs, skipping entries that are not in the form -// the finalize writes, such as ones written by hand. -func decodeLogs(stored string) []driver.AttemptLog { - // Anything but an array (or null) fails to decode into entries. - var entries []json.RawMessage - if json.Unmarshal([]byte(stored), &entries) != nil { - return nil - } - var logs []driver.AttemptLog - for _, raw := range entries { - var l driver.AttemptLog - if json.Unmarshal(raw, &l) == nil && l.Log != "" { - logs = append(logs, l) - } - } - return logs + return jobcodec.DecodeLogs(string(logs)), nil } var ( diff --git a/driver/internal/pgsql/values.go b/driver/internal/pgsql/values.go index 7a64567..39b4f0d 100644 --- a/driver/internal/pgsql/values.go +++ b/driver/internal/pgsql/values.go @@ -9,7 +9,7 @@ import ( "time" "github.com/parallelworks/hopper/driver" - "github.com/parallelworks/hopper/internal/jobtags" + "github.com/parallelworks/hopper/driver/internal/jobcodec" ) // Array parameters cross the transport as plain slices, which pgx encodes @@ -20,13 +20,6 @@ import ( func textArray(values []string) []string { return orEmpty(values) } -// storableText replaces what PostgreSQL refuses in text and jsonb strings, -// NUL and invalid UTF-8, so that one worker's text cannot fail the -// statement that finalizes a whole batch. -func storableText(s string) string { - return strings.ReplaceAll(strings.ToValidUTF8(s, "�"), "\x00", "�") -} - // nullableTextArray sends empty strings as NULL. func nullableTextArray(values []string) []*string { elems := make([]*string, len(values)) @@ -261,11 +254,12 @@ func (a *attemptErrors) Scan(src any) error { if err := text.Scan(src); err != nil { return err } - *a = attemptErrors{} - if text == "" { - return nil + errs, err := jobcodec.DecodeErrors(string(text)) + if err != nil { + return err } - return json.Unmarshal([]byte(text), (*[]driver.AttemptError)(a)) + *a = errs + return nil } // tagList scans a tags column, a JSON array of strings or NULL. hopper @@ -281,20 +275,7 @@ func (l *tagList) Scan(src any) error { if err := text.Scan(src); err != nil { return err } - *l = nil - if text == "" { - return nil - } - var elems []json.RawMessage - _ = json.Unmarshal([]byte(text), &elems) // anything but an array reads as no tags - var tags []string - for _, e := range elems { - var s string - if len(e) > 0 && e[0] == '"' && json.Unmarshal(e, &s) == nil && s != "" { - tags = append(tags, s) - } - } - *l = jobtags.Distinct(tags) + *l = jobcodec.DecodeTags(string(text)) return nil } diff --git a/driver/owner_test.go b/driver/owner_test.go new file mode 100644 index 0000000..ceb4c05 --- /dev/null +++ b/driver/owner_test.go @@ -0,0 +1,27 @@ +package driver + +import ( + "context" + "testing" +) + +type attempt struct{ name string } + +func (a *attempt) String() string { return a.name } + +// TestOwner checks that a context carries the owner WithOwner gives it, +// and that one without an owner, or given a nil one, has none. +func TestOwner(t *testing.T) { + t.Parallel() + ctx := context.Background() + if Owner(ctx) != nil { + t.Error("a context without an owner has one") + } + a := &attempt{"job 1"} + if got := Owner(WithOwner(ctx, a)); got != a { + t.Errorf("Owner = %v, want %v", got, a) + } + if got := Owner(WithOwner(WithOwner(ctx, a), nil)); got != nil { + t.Errorf("Owner after WithOwner(nil) = %v, want none", got) + } +} diff --git a/drivertest/cluster.go b/drivertest/cluster.go index c11a82a..023cec8 100644 --- a/drivertest/cluster.go +++ b/drivertest/cluster.go @@ -380,7 +380,7 @@ func indexOf(s, sub string) int { // partition, including ones a canceled build left invalid, while leaders // overlap and history is read throughout: no attach may deadlock with a // reader. -func testHistoryIndexes[TTx any](t *testing.T, f Fixture[TTx]) { +func testHistoryIndexes[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -601,7 +601,7 @@ var onceAgain = driver.JobInsertParams{ } // reattachRevalidates runs ReattachRevalidates on the fixture's database. -func reattachRevalidates[TTx any](ctx context.Context, t *testing.T, f Fixture[TTx], d driver.Driver[TTx]) bool { +func reattachRevalidates[TTx any](ctx context.Context, t *testing.T, f PostgresFixture[TTx], d driver.Driver[TTx]) bool { t.Helper() return ReattachRevalidates( func(query string) { f.Exec(ctx, t, d, query) }, @@ -634,7 +634,7 @@ func ReattachRevalidates(exec func(query string), queryText func(query string) s // partitions an outcome table's index lacks, after a schema upgrade // declared it: the index must end valid, which nothing can do afterwards on // releases before May 2026. -func testHistoryRetentionCompletesIndex[TTx any](t *testing.T, f Fixture[TTx]) { +func testHistoryRetentionCompletesIndex[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -1116,7 +1116,7 @@ func testMaintain[TTx any](t *testing.T, f Fixture[TTx]) { // the table has statistics, the planner takes each lookup of a job's edges // for a two-hundredth of the table, and finalizes and the promotion sweep // scan it instead. -func testDepsTableMaintained[TTx any](t *testing.T, f Fixture[TTx]) { +func testDepsTableMaintained[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -1209,7 +1209,7 @@ func waitMaintain(ctx context.Context, t *testing.T, exec driver.Executor, done // past retention needs that transaction to end; maintenance puts the drop // off instead of waiting for it, which would queue every other history // write behind the drop, and drops it once the transaction is over. -func testHistoryMaintainWhileInUse[TTx any](t *testing.T, f Fixture[TTx]) { +func testHistoryMaintainWhileInUse[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() diff --git a/drivertest/drivertest.go b/drivertest/drivertest.go index baa7f43..6bb8c65 100644 --- a/drivertest/drivertest.go +++ b/drivertest/drivertest.go @@ -13,16 +13,23 @@ // (order, ownership, expiry, SKIP LOCKED), every finalize transition and its // fencing to the claim (also inside a caller's transaction), snooze reasons, leases, leader election, rescue candidates, // cancel, retry, TTL expiry, listing, queues, periodic slots, stats, -// notifications and history maintenance. Engine-specific behavior, such as how partitions are managed, -// belongs in the driver's own tests. +// notifications and history maintenance. What depends on whether the engine +// locks rows or runs one writer at a time (driver.Capabilities.SingleWriter) +// is tested either way. Engine-specific behavior, such as how partitions are +// managed, belongs in the driver's own tests; RunPostgres has the tests of +// what hopper's Postgres drivers share. package drivertest import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "io" + "math/big" "slices" + "strconv" "strings" "testing" "time" @@ -30,7 +37,9 @@ import ( "github.com/parallelworks/hopper/driver" ) -// Fixture gives the suite a database per test and the hooks it needs. +// Fixture gives the suite a database per test and the hooks it needs. The +// hooks report a failure with t.Error rather than t.Fatal when the suite may +// call them from another goroutine, as SnoozeReason says. type Fixture[TTx any] interface { // NewDriver returns a driver on a fresh database with hopper's schema // installed, torn down when the test ends. @@ -49,14 +58,31 @@ type Fixture[TTx any] interface { // CountLive and CountHistory count jobs in the live table and in history. CountLive(ctx context.Context, t *testing.T, d driver.Driver[TTx]) int CountHistory(ctx context.Context, t *testing.T, d driver.Driver[TTx]) int - // Exec runs a SQL statement with hopper's tables on the search path, for - // rows the suite writes directly, such as those a client of the previous - // release leaves behind. QueryText returns the single text value a query - // selects, for what the driver's reads do not show. Both report a failure - // with t.Error rather than t.Fatal, so the suite may call them from any - // goroutine. - Exec(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) - QueryText(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) string + // ExpireJob moves a live job's TTL deadline a second into the past. + ExpireJob(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) + // StaleWake marks a running job as woken by JobRetryOrderingKey a second + // before its attempt began, as a wake during an earlier attempt would. + StaleWake(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) + // MakePending puts a live job back in the pending state, as a client + // that stopped before promoting it would leave it. + MakePending(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) + // SnoozeReason reads a live job's stored snooze reason, or "" for none, + // which the driver's reads show only for a scheduled job. The suite + // calls it from any goroutine. SetSnoozeReason stores one, as a client + // of an earlier release, which cleared it neither on claim nor on + // finalize, can leave it on a job that is not snoozed. + SnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) string + SetSnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID, reason string) + // SetProgress stores a JSON value as a live job's progress as it is, as + // a direct SQL write could. + SetProgress(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID, value string) + // CountEdges counts the dependency edges the engine keeps for workflow + // jobs that have not finished. + CountEdges(ctx context.Context, t *testing.T, d driver.Driver[TTx]) int + // WaitForLockWait waits until a statement on the driver's database + // waits for a lock another transaction holds, so that a test knows its + // competing statement is blocked rather than not yet started. + WaitForLockWait(ctx context.Context, t *testing.T, d driver.Driver[TTx]) } // Run runs the suite. @@ -73,17 +99,6 @@ func Run[TTx any](t *testing.T, f Fixture[TTx]) { {"InsertCopy", testInsertCopy[TTx]}, {"JobTags", testJobTags[TTx]}, {"JobListFilters", testJobListFilters[TTx]}, - {"TagsRowsFollowJobs", testTagsRowsFollowJobs[TTx]}, - {"TagsTableMaintained", testTagsTableMaintained[TTx]}, - {"TagsTableMaintainedPastJobsError", testTagsTableMaintainedPastJobsError[TTx]}, - {"TagsFollowHistoryRetention", testTagsFollowHistoryRetention[TTx]}, - {"TagsPartitionWaitsForHistory", testTagsPartitionWaitsForHistory[TTx]}, - {"TagsLeaveWithTheirHour", testTagsLeaveWithTheirHour[TTx]}, - {"TagsHourOutlastsItsDelete", testTagsHourOutlastsItsDelete[TTx]}, - {"TagsHourDroppedMeanwhile", testTagsHourDroppedMeanwhile[TTx]}, - {"TagsDayGoesWithItsLastHour", testTagsDayGoesWithItsLastHour[TTx]}, - {"TagsFollowDirectUpdates", testTagsFollowDirectUpdates[TTx]}, - {"TagsFollowTruncate", testTagsFollowTruncate[TTx]}, {"TagsPagePastFilteredHistory", testTagsPagePastFilteredHistory[TTx]}, {"InsertTx", testInsertTx[TTx]}, {"Claim", testClaim[TTx]}, @@ -128,9 +143,6 @@ func Run[TTx any](t *testing.T, f Fixture[TTx]) { {"Stats", testStats[TTx]}, {"Notify", testNotify[TTx]}, {"HistoryMaintain", testHistoryMaintain[TTx]}, - {"HistoryMaintainWhileInUse", testHistoryMaintainWhileInUse[TTx]}, - {"HistoryIndexes", testHistoryIndexes[TTx]}, - {"HistoryRetentionCompletesIndex", testHistoryRetentionCompletesIndex[TTx]}, {"Now", testNow[TTx]}, {"Subscriptions", testSubscriptions[TTx]}, {"Publish", testPublish[TTx]}, @@ -146,7 +158,63 @@ func Run[TTx any](t *testing.T, f Fixture[TTx]) { {"WorkflowFanInConcurrent", testWorkflowFanInConcurrent[TTx]}, {"WorkflowsConcurrent", testWorkflowsConcurrent[TTx]}, {"Streams", testStreams[TTx]}, + {"StreamsRetention", testStreamsRetention[TTx]}, {"Maintain", testMaintain[TTx]}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.fn(t, f) + }) + } +} + +// PostgresFixture is the Fixture of one of hopper's Postgres drivers, which +// can also run SQL on hopper's schema. RunPostgres runs, with it, the tests +// of what those drivers share beyond the driver interface: history's +// partitions and their indexes, the tags tables and their triggers, the rows +// clients of earlier releases leave, and the planner statistics the leader +// keeps. +type PostgresFixture[TTx any] interface { + Fixture[TTx] + // Exec runs a SQL statement with hopper's tables on the search path, for + // rows the suite writes directly, such as those a client of the previous + // release leaves behind. QueryText returns the single text value a query + // selects, for what the driver's reads do not show. Both report a + // failure with t.Error rather than t.Fatal, so the suite may call them + // from any goroutine. + Exec(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) + QueryText(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) string +} + +// RunPostgres runs the tests of what hopper's Postgres drivers share, which +// a Postgres driver passes besides Run's. +func RunPostgres[TTx any](t *testing.T, f PostgresFixture[TTx]) { + t.Helper() + tests := []struct { + name string + fn func(*testing.T, PostgresFixture[TTx]) + }{ + {"JobTagsStored", testJobTagsStored[TTx]}, + {"TagsRowsFollowJobs", testTagsRowsFollowJobs[TTx]}, + {"TagsTableMaintained", testTagsTableMaintained[TTx]}, + {"TagsTableMaintainedPastJobsError", testTagsTableMaintainedPastJobsError[TTx]}, + {"TagsFollowHistoryRetention", testTagsFollowHistoryRetention[TTx]}, + {"TagsPartitionWaitsForHistory", testTagsPartitionWaitsForHistory[TTx]}, + {"TagsLeaveWithTheirHour", testTagsLeaveWithTheirHour[TTx]}, + {"TagsHourOutlastsItsDelete", testTagsHourOutlastsItsDelete[TTx]}, + {"TagsHourDroppedMeanwhile", testTagsHourDroppedMeanwhile[TTx]}, + {"TagsDayGoesWithItsLastHour", testTagsDayGoesWithItsLastHour[TTx]}, + {"TagsFollowDirectUpdates", testTagsFollowDirectUpdates[TTx]}, + {"TagsFollowTruncate", testTagsFollowTruncate[TTx]}, + {"JobLogsStored", func(t *testing.T, f PostgresFixture[TTx]) { testJobLogsStored(t, f, false) }}, + {"JobLogsStoredNoBatches", func(t *testing.T, f PostgresFixture[TTx]) { testJobLogsStored(t, f, true) }}, + {"RetryOrderingKeyAgingOrder", testRetryOrderingKeyAgingOrder[TTx]}, + {"BatchProgressRaces", testBatchProgressRaces[TTx]}, + {"StreamsPartitioned", testStreamsPartitioned[TTx]}, + {"HistoryMaintainWhileInUse", testHistoryMaintainWhileInUse[TTx]}, + {"HistoryIndexes", testHistoryIndexes[TTx]}, + {"HistoryRetentionCompletesIndex", testHistoryRetentionCompletesIndex[TTx]}, {"DepsTableMaintained", testDepsTableMaintained[TTx]}, } for _, tc := range tests { @@ -157,6 +225,188 @@ func Run[TTx any](t *testing.T, f Fixture[TTx]) { } } +// Postgres implements a PostgresFixture's hooks that are statements on +// hopper's Postgres schema, for the fixtures of hopper's Postgres drivers to +// embed, which then implement only NewDriver, Begin and SQL. +type Postgres[TTx any] struct { + // SQL runs statements on a driver's database, as PostgresFixture's Exec + // and QueryText describe. + SQL interface { + Exec(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) + QueryText(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) string + } +} + +// Exec implements PostgresFixture. +func (p Postgres[TTx]) Exec(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) { + t.Helper() + p.SQL.Exec(ctx, t, d, query, args...) +} + +// QueryText implements PostgresFixture. +func (p Postgres[TTx]) QueryText(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string, args ...any) string { + t.Helper() + return p.SQL.QueryText(ctx, t, d, query, args...) +} + +// ExpireLease implements Fixture. +func (p Postgres[TTx]) ExpireLease(ctx context.Context, t *testing.T, d driver.Driver[TTx], clientID int64) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_clients SET expires_at = now() - interval '1s' WHERE id = $1", clientID) +} + +// ExpireLeader implements Fixture. +func (p Postgres[TTx]) ExpireLeader(ctx context.Context, t *testing.T, d driver.Driver[TTx]) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_leader SET expires_at = now() - interval '1s'") +} + +// MakeClaimable implements Fixture. +func (p Postgres[TTx]) MakeClaimable(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET scheduled_at = now() WHERE id = $1::uuid", id.String()) +} + +// AgeAttempt implements Fixture. +func (p Postgres[TTx]) AgeAttempt(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID, age time.Duration) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET attempted_at = now() - make_interval(secs => $2::float8) WHERE id = $1::uuid", id.String(), age.Seconds()) +} + +// CountLive implements Fixture. +func (p Postgres[TTx]) CountLive(ctx context.Context, t *testing.T, d driver.Driver[TTx]) int { + t.Helper() + return p.count(ctx, t, d, "SELECT count(*)::text FROM hopper_jobs") +} + +// CountHistory implements Fixture. +func (p Postgres[TTx]) CountHistory(ctx context.Context, t *testing.T, d driver.Driver[TTx]) int { + t.Helper() + return p.count(ctx, t, d, "SELECT count(*)::text FROM hopper_job_history") +} + +// ExpireJob implements Fixture. +func (p Postgres[TTx]) ExpireJob(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET expires_at = now() - interval '1s' WHERE id = $1::uuid", id.String()) +} + +// StaleWake implements Fixture. +func (p Postgres[TTx]) StaleWake(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET wake_requested_at = attempted_at - interval '1s' WHERE id = $1::uuid", id.String()) +} + +// MakePending implements Fixture. +func (p Postgres[TTx]) MakePending(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET state = 'pending' WHERE id = $1::uuid", id.String()) +} + +// SnoozeReason implements Fixture. +func (p Postgres[TTx]) SnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID) string { + t.Helper() + return p.SQL.QueryText(ctx, t, d, "SELECT coalesce(snooze_reason, '') FROM hopper_jobs WHERE id = $1::uuid", id.String()) +} + +// SetSnoozeReason implements Fixture. +func (p Postgres[TTx]) SetSnoozeReason(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID, reason string) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET snooze_reason = $2 WHERE id = $1::uuid", id.String(), reason) +} + +// SetProgress implements Fixture. +func (p Postgres[TTx]) SetProgress(ctx context.Context, t *testing.T, d driver.Driver[TTx], id driver.JobID, value string) { + t.Helper() + p.SQL.Exec(ctx, t, d, "UPDATE hopper_jobs SET progress = $2::jsonb WHERE id = $1::uuid", id.String(), value) +} + +// CountEdges implements Fixture. +func (p Postgres[TTx]) CountEdges(ctx context.Context, t *testing.T, d driver.Driver[TTx]) int { + t.Helper() + return p.count(ctx, t, d, "SELECT count(*)::text FROM hopper_job_deps") +} + +// WaitForLockWait implements Fixture: a statement on the driver's schema, +// whose name is in every statement's text, waits for a lock. +func (p Postgres[TTx]) WaitForLockWait(ctx context.Context, t *testing.T, d driver.Driver[TTx]) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for p.SQL.QueryText(ctx, t, d, `SELECT count(*)::text FROM pg_stat_activity + WHERE wait_event_type = 'Lock' AND position(current_schema() IN query) > 0 AND pid <> pg_backend_pid()`) == "0" { + if time.Now().After(deadline) { + t.Fatal("no statement started waiting for a lock") + } + time.Sleep(5 * time.Millisecond) + } +} + +func (p Postgres[TTx]) count(ctx context.Context, t *testing.T, d driver.Driver[TTx], query string) int { + t.Helper() + n, err := strconv.Atoi(p.SQL.QueryText(ctx, t, d, query)) + if err != nil { + t.Error(err) + } + return n +} + +// singleWriter reports whether d's engine runs one writing transaction at a +// time. +func singleWriter[TTx any](d driver.Driver[TTx]) bool { return d.Capabilities().SingleWriter } + +// jsonEqual reports whether two JSON texts hold the same value: an engine +// need not keep the text as given (Postgres's jsonb orders an object's keys +// and spaces them), only the value. Numbers compare by their exact value, +// not as float64s. +func jsonEqual(got []byte, want string) bool { + decode := func(b []byte) (any, bool) { + dec := json.NewDecoder(bytes.NewReader(b)) + dec.UseNumber() + var v any + if dec.Decode(&v) != nil { + return nil, false + } + _, err := dec.Token() // one value, and nothing after it + return v, errors.Is(err, io.EOF) + } + a, ok := decode(got) + b, ok2 := decode([]byte(want)) + return ok && ok2 && fmt.Sprint(normalizeJSON(a)) == fmt.Sprint(normalizeJSON(b)) +} + +// normalizeJSON gives a decoded JSON value a printed form that does not +// depend on its object's key order, nor on how its numbers are written. +func normalizeJSON(v any) any { + switch v := v.(type) { + case map[string]any: + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + slices.Sort(keys) + out := make([]string, len(keys)) + for i, k := range keys { + out[i] = strconv.Quote(k) + ":" + fmt.Sprint(normalizeJSON(v[k])) + } + return "{" + strings.Join(out, ",") + "}" + case []any: + out := make([]string, len(v)) + for i, e := range v { + out[i] = fmt.Sprint(normalizeJSON(e)) + } + return "[" + strings.Join(out, ",") + "]" + case string: + return strconv.Quote(v) + case json.Number: + if r, ok := new(big.Rat).SetString(string(v)); ok { + return r.RatString() + } + return string(v) + default: + return v + } +} + // noBatchesExec finalizes with JobFinalizeParams.NoBatches set, which a // caller may when none of the jobs belongs to a batch: the tests of jobs // outside batches see the same results through it. diff --git a/drivertest/flow.go b/drivertest/flow.go index 5ab1980..f97b030 100644 --- a/drivertest/flow.go +++ b/drivertest/flow.go @@ -243,7 +243,7 @@ func testBatches[TTx any](t *testing.T, f Fixture[TTx]) { if err := json.Unmarshal(kinds["alert"].Metadata, &meta); err != nil || meta.BatchID != id.String() || meta.BatchFailed != 1 { t.Errorf("callback metadata = %s (%v)", kinds["alert"].Metadata, err) } - if j := kinds["alert"]; j.Queue != "callbacks" || j.Priority != 1 || j.MaxAttempts != 3 || string(j.Args) != `{"run": 9}` { + if j := kinds["alert"]; j.Queue != "callbacks" || j.Priority != 1 || j.MaxAttempts != 3 || !jsonEqual(j.Args, `{"run": 9}`) { t.Errorf("callback job = %+v", j) } @@ -273,22 +273,23 @@ func testBatches[TTx any](t *testing.T, f Fixture[TTx]) { } } -// testBatchesConcurrent finalizes a batch's jobs in concurrent statements. -// Each records its progress without waiting on the others, so a -// transaction that finalized a job and is still open holds up none of its -// siblings' finalizes. The last two, each missing the other's uncommitted -// progress, leave the batch open, and BatchSettle, run by each finalizing -// client after its statement commits, completes it once; the sweep finds -// what a client could not settle. -func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { - ctx := context.Background() - d := f.NewDriver(t) - exec := d.Executor() - clientID := register(ctx, t, exec) +// batchHelpers holds what the batch concurrency tests share: inserting a +// batch of n jobs of kind with callbacks on the callbacks queue, settling +// batches, and counting a batch's callbacks by kind. +type batchHelpers struct { + newBatch func(kind string, n int) driver.JobID + settle func(ids ...driver.JobID) driver.BatchSettleResult + sweep func() driver.BatchSettleResult + callbacks func(batch driver.JobID) map[string]int + completed func(id driver.JobID) driver.JobFinalize +} + +func newBatchHelpers(ctx context.Context, t *testing.T, exec driver.Executor, clientID int64) batchHelpers { cb := func(kind string) *driver.JobInsertParams { return &driver.JobInsertParams{Kind: kind, Queue: "callbacks", Priority: 2, MaxAttempts: 3, Args: json.RawMessage(`{}`)} } - callbacks := func(batch driver.JobID) map[string]int { + h := batchHelpers{} + h.callbacks = func(batch driver.JobID) map[string]int { t.Helper() jobs, err := exec.JobList(ctx, driver.JobListParams{Queue: "callbacks", Limit: 1000}) if err != nil { @@ -305,7 +306,7 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } return kinds } - newBatch := func(kind string, n int) driver.JobID { + h.newBatch = func(kind string, n int) driver.JobID { t.Helper() id, err := exec.BatchInsert(ctx, driver.BatchInsertParams{Total: n, OnSuccess: cb("ok"), OnFailure: cb("failed"), OnComplete: cb("always")}) if err != nil { @@ -318,7 +319,7 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { insert(ctx, t, exec, p) return id } - settle := func(ids ...driver.JobID) driver.BatchSettleResult { + h.settle = func(ids ...driver.JobID) driver.BatchSettleResult { t.Helper() p := driver.BatchSettleParams{IDs: ids, Limit: 100} res, err := exec.BatchSettle(ctx, p) @@ -327,62 +328,217 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } return res } - sweep := func() driver.BatchSettleResult { t.Helper(); return settle() } - completed := func(id driver.JobID) driver.JobFinalize { + h.sweep = func() driver.BatchSettleResult { t.Helper(); return h.settle() } + h.completed = func(id driver.JobID) driver.JobFinalize { return driver.JobFinalize{ID: id, AttemptedBy: clientID, Fence: 1, State: driver.JobStateCompleted, Archive: true} } + return h +} - batch := newBatch("pair", 2) +// testBatchesConcurrent finalizes a batch's jobs in concurrent statements. +// On an engine that locks rows, each records its progress without waiting +// on the others, so a transaction that finalized a job and is still open +// holds up none of its siblings' finalizes. The last two, each missing the +// other's uncommitted progress, leave the batch open, and BatchSettle, run +// by each finalizing client after its statement commits, completes it once; +// the sweep finds what a client could not settle. On an engine that runs +// one writer at a time, the sibling's finalize waits for the transaction, +// and completes the batch itself. +func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { + ctx := context.Background() + d := f.NewDriver(t) + exec := d.Executor() + clientID := register(ctx, t, exec) + h := newBatchHelpers(ctx, t, exec, clientID) + + batch := h.newBatch("pair", 2) jobs := claim(ctx, t, exec, clientID, 2) if len(jobs) != 2 { t.Fatalf("claimed %d", len(jobs)) } tx, commit, rollback := f.Begin(ctx, t, d) defer rollback() //nolint:errcheck // already committed on the happy path - finalizeCurrent(ctx, t, d.UnwrapTx(tx), completed(jobs[0].ID)) - sctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - if applied, err := exec.JobFinalizeMany(sctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{completed(jobs[1].ID)}}); err != nil || len(applied) != 1 { - t.Fatalf("a sibling's finalize while the transaction is open = %v, %v; it must not wait for it", applied, err) - } - if res := settle(batch); res.Completed != 0 { - t.Errorf("BatchSettle with a job's finalize uncommitted = %+v", res) - } - if err := commit(); err != nil { - t.Fatal(err) - } - if b, err := exec.BatchGet(ctx, batch); err != nil || b.Pending != 0 || !b.CompletedAt.IsZero() { - t.Fatalf("batch after both finalizes = %+v, %v; the race should leave it open", b, err) - } - if res := settle(batch); res.Completed != 1 { - t.Errorf("BatchSettle = %+v; want 1 completed", res) + finalizeCurrent(ctx, t, d.UnwrapTx(tx), h.completed(jobs[0].ID)) + if singleWriter(d) { + done := make(chan []driver.JobID, 1) + go func() { + applied, err := exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{h.completed(jobs[1].ID)}}) + if err != nil { + t.Error(err) + } + done <- applied + }() + f.WaitForLockWait(ctx, t, d) + if err := commit(); err != nil { + t.Fatal(err) + } + if applied := <-done; len(applied) != 1 { + t.Fatalf("the sibling's finalize after the transaction = %v", applied) + } + if res := h.settle(batch); res.Completed != 0 { + t.Errorf("BatchSettle of a batch its last finalize completed = %+v", res) + } + } else { + sctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if applied, err := exec.JobFinalizeMany(sctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{h.completed(jobs[1].ID)}}); err != nil || len(applied) != 1 { + t.Fatalf("a sibling's finalize while the transaction is open = %v, %v; it must not wait for it", applied, err) + } + if res := h.settle(batch); res.Completed != 0 { + t.Errorf("BatchSettle with a job's finalize uncommitted = %+v", res) + } + if err := commit(); err != nil { + t.Fatal(err) + } + if b, err := exec.BatchGet(ctx, batch); err != nil || b.Pending != 0 || !b.CompletedAt.IsZero() { + t.Fatalf("batch after both finalizes = %+v, %v; the race should leave it open", b, err) + } + if res := h.settle(batch); res.Completed != 1 { + t.Errorf("BatchSettle = %+v; want 1 completed", res) + } } - if res := sweep(); res != (driver.BatchSettleResult{}) { + if res := h.sweep(); res != (driver.BatchSettleResult{}) { t.Errorf("sweep after the batch was settled = %+v", res) } if b, err := exec.BatchGet(ctx, batch); err != nil || b.Pending != 0 || b.Failed != 0 || b.CompletedAt.IsZero() { t.Errorf("settled batch = %+v, %v", b, err) } - if kinds := callbacks(batch); len(kinds) != 2 || kinds["ok"] != 1 || kinds["always"] != 1 { + if kinds := h.callbacks(batch); len(kinds) != 2 || kinds["ok"] != 1 || kinds["always"] != 1 { t.Errorf("callbacks = %v, want ok and always once", kinds) } + // Many batches, their jobs finalized by concurrent clients in mixed + // sets of results, each client settling its batches after each + // statement as the finalizer does: every batch completes exactly once, + // with no sweep, its failures counted and the callbacks its outcome + // calls for. Every third batch has its first two jobs discarded, which + // may land in different statements. + const batches, size, clients, discards = 12, 6, 4, 2 + ids := make([]driver.JobID, batches) + wantFailed := map[string]int{} + for i := range ids { + kind := fmt.Sprintf("many%d", i) + ids[i] = h.newBatch(kind, size) + if i%3 == 0 { + wantFailed[kind] = discards + } + } + var wg sync.WaitGroup + for range clients { + owner := register(ctx, t, exec) + wg.Go(func() { + for { + res, err := exec.JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: owner, Limit: 3}) + if err != nil { + t.Error(err) + return + } + if len(res.Jobs) == 0 { + return + } + results := make([]driver.JobFinalize, len(res.Jobs)) + for i, j := range res.Jobs { + results[i] = driver.JobFinalize{ID: j.ID, AttemptedBy: owner, Fence: j.Fence, State: driver.JobStateCompleted, Archive: true} + if argsN(j) < wantFailed[j.Kind] { + results[i].State, results[i].Error = driver.JobStateDiscarded, &driver.AttemptError{Attempt: 1, Error: "x"} + } + } + if _, err := exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: results}); err != nil { + t.Error(err) + return + } + var batches []driver.JobID + for _, j := range res.Jobs { + if !slices.Contains(batches, j.BatchID) { + batches = append(batches, j.BatchID) + } + } + if _, err := exec.BatchSettle(ctx, driver.BatchSettleParams{IDs: batches}); err != nil { + t.Error(err) + return + } + } + }) + } + wg.Wait() + if res := h.sweep(); res.Completed != 0 { + t.Errorf("the sweep completed %d batches the clients' settles left open", res.Completed) + } + for i, id := range ids { + want := wantFailed[fmt.Sprintf("many%d", i)] + b, err := exec.BatchGet(ctx, id) + if err != nil || b.CompletedAt.IsZero() || b.Pending != 0 || b.Failed != want { + t.Errorf("batch %d = %+v, %v; want %d failed", i, b, err, want) + continue + } + outcome := "ok" + if want > 0 { + outcome = "failed" + } + if kinds := h.callbacks(id); len(kinds) != 2 || kinds["always"] != 1 || kinds[outcome] != 1 { + t.Errorf("batch %d (%d failed): callbacks %v", i, want, kinds) + } + } + + // A failure is counted where retention does not reach: a batch whose + // failed job's history is gone before the batch completes still fails. + pruned := h.newBatch("pruned", 2) + jobs = claim(ctx, t, exec, clientID, 2) + if len(jobs) != 2 { + t.Fatalf("claimed %d", len(jobs)) + } + finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: jobs[0].ID, AttemptedBy: clientID, State: driver.JobStateDiscarded, Error: &driver.AttemptError{Attempt: 1, Error: "x"}}) + h.sweep() + if _, err := exec.JobDelete(ctx, jobs[0].ID); err != nil { + t.Fatal(err) + } + finalizeCurrent(ctx, t, exec, h.completed(jobs[1].ID)) + if b, err := exec.BatchGet(ctx, pruned); err != nil || b.CompletedAt.IsZero() || b.Failed != 1 { + t.Errorf("batch whose failure's history was pruned = %+v, %v", b, err) + } + if kinds := h.callbacks(pruned); kinds["failed"] != 1 || kinds["ok"] != 0 { + t.Errorf("callbacks of the pruned batch = %v", kinds) + } + + // Postgres records a batch's progress in rows of its own, which the + // settles above leave none of once every batch has completed. + if pf, ok := any(f).(PostgresFixture[TTx]); ok { + if left := pf.QueryText(ctx, t, d, "SELECT count(*)::text FROM hopper_batch_progress"); left != "0" { + t.Errorf("%s progress rows left after every batch completed and settled", left) + } + } +} + +// testBatchProgressRaces covers the progress rows Postgres's finalizes +// record a batch's progress in: a settle that cannot get a batch row in +// time leaves the batch to the sweep; of two clients that finalized a +// batch's last jobs at the same time, the one settling second waits for the +// first's settle and completes the batch; the sweep walks batches in ID +// order; a batch row from before callbacks were checked still completes; +// and no progress rows are left once every batch is settled. +func testBatchProgressRaces[TTx any](t *testing.T, f PostgresFixture[TTx]) { + ctx := context.Background() + d := f.NewDriver(t) + exec := d.Executor() + clientID := register(ctx, t, exec) + h := newBatchHelpers(ctx, t, exec, clientID) + // A settle that cannot get a batch row in time leaves the batch to the // sweep, without an error. - held := newBatch("held", 3) - jobs = claim(ctx, t, exec, clientID, 3) + held := h.newBatch("held", 3) + jobs := claim(ctx, t, exec, clientID, 3) if len(jobs) != 3 { t.Fatalf("claimed %d", len(jobs)) } - finalizeCurrent(ctx, t, exec, completed(jobs[0].ID)) - tx, commit, rollback = f.Begin(ctx, t, d) + finalizeCurrent(ctx, t, exec, h.completed(jobs[0].ID)) + tx, commit, rollback := f.Begin(ctx, t, d) defer rollback() //nolint:errcheck // already committed on the happy path if res, err := d.UnwrapTx(tx).BatchSettle(ctx, driver.BatchSettleParams{IDs: []driver.JobID{held}}); err != nil || res.Folded != 1 { t.Fatalf("BatchSettle in a transaction = %+v, %v", res, err) } - finalizeCurrent(ctx, t, exec, completed(jobs[1].ID)) + finalizeCurrent(ctx, t, exec, h.completed(jobs[1].ID)) start := time.Now() - if res := settle(held); res != (driver.BatchSettleResult{}) { + if res := h.settle(held); res != (driver.BatchSettleResult{}) { t.Errorf("BatchSettle of a held batch row = %+v", res) } if took := time.Since(start); took > 3*time.Second { @@ -391,26 +547,26 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { if err := commit(); err != nil { t.Fatal(err) } - finalizeCurrent(ctx, t, exec, completed(jobs[2].ID)) + finalizeCurrent(ctx, t, exec, h.completed(jobs[2].ID)) if b, err := exec.BatchGet(ctx, held); err != nil || b.Pending != 0 || b.CompletedAt.IsZero() { t.Errorf("held batch = %+v, %v", b, err) } - if kinds := callbacks(held); len(kinds) != 2 || kinds["ok"] != 1 || kinds["always"] != 1 { + if kinds := h.callbacks(held); len(kinds) != 2 || kinds["ok"] != 1 || kinds["always"] != 1 { t.Errorf("callbacks of the held batch = %v", kinds) } // Of two clients that finalized a batch's last jobs at the same time, // the one settling second waits for the first's settle, which could not // see its progress, and completes the batch. - raced := newBatch("raced", 2) + raced := h.newBatch("raced", 2) jobs = claim(ctx, t, exec, clientID, 2) if len(jobs) != 2 { t.Fatalf("claimed %d", len(jobs)) } txB, commitB, rollbackB := f.Begin(ctx, t, d) defer rollbackB() //nolint:errcheck // already committed on the happy path - finalizeCurrent(ctx, t, d.UnwrapTx(txB), completed(jobs[1].ID)) - finalizeCurrent(ctx, t, exec, completed(jobs[0].ID)) + finalizeCurrent(ctx, t, d.UnwrapTx(txB), h.completed(jobs[1].ID)) + finalizeCurrent(ctx, t, exec, h.completed(jobs[0].ID)) txA, commitA, rollbackA := f.Begin(ctx, t, d) defer rollbackA() //nolint:errcheck // already committed on the happy path if res, err := d.UnwrapTx(txA).BatchSettle(ctx, driver.BatchSettleParams{IDs: []driver.JobID{raced}}); err != nil || res.Completed != 0 { @@ -446,14 +602,14 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { // The sweep walks batches in ID order from After. var open [2]driver.JobID for i := range open { - open[i] = newBatch(fmt.Sprintf("open%d", i), 2) + open[i] = h.newBatch(fmt.Sprintf("open%d", i), 2) } var rest []driver.JobFinalize for _, j := range claim(ctx, t, exec, clientID, 4) { if argsN(j) == 0 { - finalizeCurrent(ctx, t, exec, completed(j.ID)) + finalizeCurrent(ctx, t, exec, h.completed(j.ID)) } else { - rest = append(rest, completed(j.ID)) + rest = append(rest, h.completed(j.ID)) } } slices.SortFunc(open[:], func(a, b driver.JobID) int { return bytes.Compare(a[:], b[:]) }) @@ -465,7 +621,7 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { if err != nil || second.Folded != 1 || second.Last != open[1] { t.Fatalf("second chunk = %+v, %v; want %s", second, err, open[1]) } - if res := sweep(); res.Folded != 0 { + if res := h.sweep(); res.Folded != 0 { t.Errorf("sweep after both chunks = %+v", res) } for _, id := range open { @@ -485,7 +641,7 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { VALUES (1, 1, '{"kind": "legacy", "queue": "callbacks", "priority": 0, "max_attempts": 99999}') RETURNING id::text`) f.Exec(ctx, t, d, `INSERT INTO hopper_batch_progress (batch_id, done, failed) VALUES ($1::uuid, 1, 0), ('00000000-0000-7000-8000-000000000001', 1, 0)`, legacy) - if res := sweep(); res.Completed != 1 { + if res := h.sweep(); res.Completed != 1 { t.Errorf("sweep with a legacy batch = %+v; want it completed", res) } cbs, err := exec.JobList(ctx, driver.JobListParams{Queue: "callbacks", Kinds: []string{"legacy"}, Limit: 10}) @@ -498,98 +654,6 @@ func testBatchesConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { t.Errorf("batch finished after its sweep = %+v, %v", b, err) } } - - // Many batches, their jobs finalized by concurrent clients in mixed - // sets of results, each client settling its batches after each - // statement as the finalizer does: every batch completes exactly once, - // with no sweep, its failures counted and the callbacks its outcome - // calls for. Every third batch has its first two jobs discarded, which - // may land in different statements. - const batches, size, clients, discards = 12, 6, 4, 2 - ids := make([]driver.JobID, batches) - wantFailed := map[string]int{} - for i := range ids { - kind := fmt.Sprintf("many%d", i) - ids[i] = newBatch(kind, size) - if i%3 == 0 { - wantFailed[kind] = discards - } - } - var wg sync.WaitGroup - for range clients { - owner := register(ctx, t, exec) - wg.Go(func() { - for { - res, err := exec.JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: owner, Limit: 3}) - if err != nil { - t.Error(err) - return - } - if len(res.Jobs) == 0 { - return - } - results := make([]driver.JobFinalize, len(res.Jobs)) - for i, j := range res.Jobs { - results[i] = driver.JobFinalize{ID: j.ID, AttemptedBy: owner, Fence: j.Fence, State: driver.JobStateCompleted, Archive: true} - if argsN(j) < wantFailed[j.Kind] { - results[i].State, results[i].Error = driver.JobStateDiscarded, &driver.AttemptError{Attempt: 1, Error: "x"} - } - } - if _, err := exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: results}); err != nil { - t.Error(err) - return - } - var batches []driver.JobID - for _, j := range res.Jobs { - if !slices.Contains(batches, j.BatchID) { - batches = append(batches, j.BatchID) - } - } - if _, err := exec.BatchSettle(ctx, driver.BatchSettleParams{IDs: batches}); err != nil { - t.Error(err) - return - } - } - }) - } - wg.Wait() - if res := sweep(); res.Completed != 0 { - t.Errorf("the sweep completed %d batches the clients' settles left open", res.Completed) - } - for i, id := range ids { - want := wantFailed[fmt.Sprintf("many%d", i)] - b, err := exec.BatchGet(ctx, id) - if err != nil || b.CompletedAt.IsZero() || b.Pending != 0 || b.Failed != want { - t.Errorf("batch %d = %+v, %v; want %d failed", i, b, err, want) - continue - } - outcome := "ok" - if want > 0 { - outcome = "failed" - } - if kinds := callbacks(id); len(kinds) != 2 || kinds["always"] != 1 || kinds[outcome] != 1 { - t.Errorf("batch %d (%d failed): callbacks %v", i, want, kinds) - } - } - - // A failure is counted where retention does not reach: a batch whose - // failed job's history is gone before the batch completes still fails. - pruned := newBatch("pruned", 2) - jobs = claim(ctx, t, exec, clientID, 2) - if len(jobs) != 2 { - t.Fatalf("claimed %d", len(jobs)) - } - finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: jobs[0].ID, AttemptedBy: clientID, State: driver.JobStateDiscarded, Error: &driver.AttemptError{Attempt: 1, Error: "x"}}) - sweep() - f.Exec(ctx, t, d, "DELETE FROM hopper_job_history WHERE kind = 'pruned'") - finalizeCurrent(ctx, t, exec, completed(jobs[1].ID)) - if b, err := exec.BatchGet(ctx, pruned); err != nil || b.CompletedAt.IsZero() || b.Failed != 1 { - t.Errorf("batch whose failure's history was pruned = %+v, %v", b, err) - } - if kinds := callbacks(pruned); kinds["failed"] != 1 || kinds["ok"] != 0 { - t.Errorf("callbacks of the pruned batch = %v", kinds) - } - if left := f.QueryText(ctx, t, d, "SELECT count(*)::text FROM hopper_batch_progress"); left != "0" { t.Errorf("%s progress rows left after every batch completed and settled", left) } diff --git a/drivertest/jobs.go b/drivertest/jobs.go index 197b67b..688e0e6 100644 --- a/drivertest/jobs.go +++ b/drivertest/jobs.go @@ -129,6 +129,10 @@ func testClaim[TTx any](t *testing.T, f Fixture[TTx]) { } } +// testClaimSkipsLocked claims while another client's claim is held open in +// a transaction. An engine that locks rows skips the held jobs and claims +// the rest at once; one that runs one writer at a time waits for the +// transaction, and then claims the rest. func testClaimSkipsLocked[TTx any](t *testing.T, f Fixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) @@ -136,7 +140,7 @@ func testClaimSkipsLocked[TTx any](t *testing.T, f Fixture[TTx]) { insert(ctx, t, exec, params("k", 4)) a, b := register(ctx, t, exec), register(ctx, t, exec) - tx, _, rollback := f.Begin(ctx, t, d) + tx, commit, rollback := f.Begin(ctx, t, d) defer rollback() //nolint:errcheck // cleanup heldRes, err := d.UnwrapTx(tx).JobClaim(ctx, driver.JobClaimParams{Queue: "default", ClientID: a, Limit: 2}) held := heldRes.Jobs @@ -151,6 +155,17 @@ func testClaimSkipsLocked[TTx any](t *testing.T, f Fixture[TTx]) { } done <- res.Jobs }() + if singleWriter(d) { + f.WaitForLockWait(ctx, t, d) + select { + case jobs := <-done: + t.Fatalf("B claimed %d jobs while A's transaction wrote", len(jobs)) + case <-time.After(100 * time.Millisecond): + } + if err := commit(); err != nil { + t.Fatal(err) + } + } select { case jobs := <-done: if len(jobs) != 2 { @@ -247,8 +262,9 @@ func testFinalize[TTx any](t *testing.T, f Fixture[TTx], noBatches bool) { // testFinalizeSkipLocked finalizes with SkipLocked while another // transaction holds one of the rows, as a worker's CompleteTx that has not -// ended does: the statement returns at once, without that job, and applies -// it once the row is free. +// ended does: the statement returns at once, without that job (on an engine +// with one writer at a time, without any job), and applies it once the row +// is free. func testFinalizeSkipLocked[TTx any](t *testing.T, f Fixture[TTx], noBatches bool) { ctx := context.Background() d := f.NewDriver(t) @@ -269,14 +285,25 @@ func testFinalizeSkipLocked[TTx any](t *testing.T, f Fixture[TTx], noBatches boo applied, err := exec.JobFinalizeMany(sctx, driver.JobFinalizeParams{SkipLocked: true, Jobs: []driver.JobFinalize{ failed, {ID: free, AttemptedBy: clientID, Fence: 1, State: driver.JobStateCompleted, Archive: true}, }}) - if err != nil || !slices.Equal(applied, []driver.JobID{free}) { - t.Fatalf("finalize with a held row = %v, %v; want only %s", applied, err, free) + want := []driver.JobID{free} + if singleWriter(d) { + want = nil + } + if err != nil || !slices.Equal(applied, want) { + t.Fatalf("finalize with a held row = %v, %v; want %v", applied, err, want) } if err := rollback(); err != nil { t.Fatal(err) } - applied, err = exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{SkipLocked: true, Jobs: []driver.JobFinalize{failed}}) - if err != nil || !slices.Equal(applied, []driver.JobID{held}) { + results := []driver.JobFinalize{failed} + want = []driver.JobID{held} + if singleWriter(d) { + results = append(results, driver.JobFinalize{ID: free, AttemptedBy: clientID, Fence: 1, State: driver.JobStateCompleted, Archive: true}) + want = append(want, free) + } + applied, err = exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{SkipLocked: true, Jobs: results}) + slices.SortFunc(applied, func(a, b driver.JobID) int { return slices.Index(want, a) - slices.Index(want, b) }) + if err != nil || !slices.Equal(applied, want) { t.Fatalf("finalize once the row is free = %v, %v", applied, err) } if j, err := exec.JobGet(ctx, held); err != nil || j.State != driver.JobStateRetryable { @@ -365,7 +392,7 @@ func testFinalizeTx[TTx any](t *testing.T, f Fixture[TTx]) { t.Fatal(err) } j, err := exec.JobGet(ctx, id) - if err != nil || j.State != driver.JobStateCompleted || j.FinalizedAt.IsZero() || string(j.Output) != `{"ok": true}` { + if err != nil || j.State != driver.JobStateCompleted || j.FinalizedAt.IsZero() || !jsonEqual(j.Output, `{"ok": true}`) { t.Errorf("job after commit = %+v, %v", j, err) } if live, hist := f.CountLive(ctx, t, d), f.CountHistory(ctx, t, d); live != 0 || hist != 1 { @@ -406,7 +433,7 @@ func testFinalizeTxConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } done <- applied }() - waitForLockWait(ctx, t, f, d) + f.WaitForLockWait(ctx, t, d) end := rollback if commitTx { end = commit @@ -429,21 +456,6 @@ func testFinalizeTxConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } } -// waitForLockWait waits until a statement on the driver's schema is waiting -// for a lock, so a test knows its competing statement is blocked rather -// than not yet started. The schema's name is in every statement's text. -func waitForLockWait[TTx any](ctx context.Context, t *testing.T, f Fixture[TTx], d driver.Driver[TTx]) { - t.Helper() - deadline := time.Now().Add(10 * time.Second) - for f.QueryText(ctx, t, d, `SELECT count(*)::text FROM pg_stat_activity - WHERE wait_event_type = 'Lock' AND position(current_schema() IN query) > 0 AND pid <> pg_backend_pid()`) == "0" { - if time.Now().After(deadline) { - t.Fatal("no statement started waiting for a lock") - } - time.Sleep(5 * time.Millisecond) - } -} - func testCancel[TTx any](t *testing.T, f Fixture[TTx]) { ctx := context.Background() exec := f.NewDriver(t).Executor() @@ -555,7 +567,7 @@ func testRetryOrderingKey[TTx any](t *testing.T, f Fixture[TTx]) { if _, err := exec.JobCancel(ctx, id(4)); err != nil { t.Fatal(err) } - f.Exec(ctx, t, d, "UPDATE hopper_jobs SET expires_at = now() - interval '1s' WHERE id = $1::uuid", id(6).String()) + f.ExpireJob(ctx, t, d, id(6)) wf, err := exec.WorkflowInsert(ctx, driver.WorkflowInsertParams{ Name: "w", Jobs: []driver.JobInsertParams{{Kind: "w", Queue: "wf", Priority: 2, MaxAttempts: 1}, {Kind: "w", Queue: "wf", Priority: 2, MaxAttempts: 1, OrderingKey: "a"}}, Deps: []driver.JobDependency{{Job: 1, DependsOn: 0}}, @@ -620,7 +632,7 @@ func testRetryOrderingKey[TTx any](t *testing.T, f Fixture[TTx]) { finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: id(0), AttemptedBy: clientID, State: driver.JobStateScheduled, Delay: time.Hour, Snooze: true, SnoozeReason: "waiting"}) claimed() wake("default", 10, 1) - if r := storedReason(ctx, t, f, d, id(0)); r != "" { + if r := f.SnoozeReason(ctx, t, d, id(0)); r != "" { t.Errorf("woken job kept reason %q", r) } claimed(id(0)) @@ -628,13 +640,13 @@ func testRetryOrderingKey[TTx any](t *testing.T, f Fixture[TTx]) { // A wake during the attempt makes its snooze run at once, without a reason. wake("default", 10, 0) finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: id(0), AttemptedBy: clientID, State: driver.JobStateScheduled, Delay: time.Hour, Snooze: true, SnoozeReason: "waiting"}) - if r := storedReason(ctx, t, f, d, id(0)); r != "" { + if r := f.SnoozeReason(ctx, t, d, id(0)); r != "" { t.Errorf("snooze after a wake kept reason %q", r) } claimed(id(0)) // A mark from before the attempt started does not count. - f.Exec(ctx, t, d, "UPDATE hopper_jobs SET wake_requested_at = attempted_at - interval '1s' WHERE id = $1::uuid", id(0).String()) + f.StaleWake(ctx, t, d, id(0)) finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: id(0), AttemptedBy: clientID, State: driver.JobStateRetryable, Delay: time.Hour, Error: &driver.AttemptError{Attempt: 1, Error: "x"}}) claimed() if n, err := exec.JobRetryOrderingKey(ctx, driver.JobRetryOrderingKeyParams{Queue: "default", OrderingKey: "none", Limit: 10}); err != nil || n != 0 { @@ -643,8 +655,7 @@ func testRetryOrderingKey[TTx any](t *testing.T, f Fixture[TTx]) { } // testRetryOrderingKeyConcurrent checks that a wake racing the end of an -// attempt is never lost, and that a wake and the leader's aging, which lock -// the same rows, never deadlock. +// attempt is never lost. func testRetryOrderingKeyConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) @@ -673,16 +684,44 @@ func testRetryOrderingKeyConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } }) wg.Wait() - if due := f.QueryText(ctx, t, d, "SELECT (scheduled_at <= now())::text FROM hopper_jobs WHERE id = $1::uuid", job.String()); due != "true" { + if !due(ctx, t, exec, job) { t.Fatalf("round %d: a wake racing the snooze was lost", round) } - f.Exec(ctx, t, d, "DELETE FROM hopper_jobs WHERE id = $1::uuid", job.String()) + if _, err := exec.JobDelete(ctx, job); err != nil { + t.Fatal(err) + } } +} - // Rows laid out in the heap in the reverse of their seq order: the - // aging pass meets them in one order and the wake locks them in the - // other. Each round rewrites them one by one, newest first, which - // appends their new versions in that order. +// due reports whether a live job's scheduled time has come, by the +// database's clock. +func due(ctx context.Context, t *testing.T, exec driver.Executor, id driver.JobID) bool { + t.Helper() + now, err := exec.Now(ctx) + if err != nil { + t.Fatal(err) + } + j, err := exec.JobGet(ctx, id) + if err != nil { + t.Fatal(err) + } + return !j.ScheduledAt.After(now) +} + +// testRetryOrderingKeyAgingOrder checks that a wake and the leader's aging, +// which lock the same rows, never deadlock, with rows laid out in the heap +// in the reverse of their seq order: the aging pass meets them in one order +// and the wake locks them in the other. Each round rewrites them one by +// one, newest first, which appends their new versions in that order. +func testRetryOrderingKeyAgingOrder[TTx any](t *testing.T, f PostgresFixture[TTx]) { + ctx := context.Background() + d := f.NewDriver(t) + exec := d.Executor() + wakeKey := func(key string) { + if _, err := exec.JobRetryOrderingKey(ctx, driver.JobRetryOrderingKeyParams{Queue: "default", OrderingKey: key, Limit: 1000}); err != nil { + t.Error(err) + } + } p := params("k", 300) for i := range p { p[i].OrderingKey, p[i].Priority, p[i].ScheduledAt = "aging", 3, time.Now().Add(time.Hour) @@ -1032,7 +1071,7 @@ func testJobDeleteRacesRetry[TTx any](t *testing.T, f Fixture[TTx]) { deleted, err = exec.JobDelete(ctx, id) done <- err }() - waitForLockWait(ctx, t, f, d) + f.WaitForLockWait(ctx, t, d) if err := commit(); err != nil { t.Fatal(err) } @@ -1071,7 +1110,7 @@ func testRetryOrderingKeyTx[TTx any](t *testing.T, f Fixture[TTx]) { t.Fatal(err) } finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: job, AttemptedBy: clientID, State: driver.JobStateScheduled, Delay: time.Hour, Snooze: true}) - if due := f.QueryText(ctx, t, d, "SELECT (scheduled_at <= now())::text FROM hopper_jobs WHERE id = $1::uuid", job.String()); due != "true" { + if !due(ctx, t, exec, job) { t.Error("a wake in a transaction older than the attempt was lost") } } diff --git a/drivertest/json_internal_test.go b/drivertest/json_internal_test.go new file mode 100644 index 0000000..6b0f199 --- /dev/null +++ b/drivertest/json_internal_test.go @@ -0,0 +1,27 @@ +package drivertest + +import "testing" + +// TestJSONEqual checks that JSON texts compare by value: key order, +// spacing and how a number is written do not count, its exact value does, +// and a text with anything after its value matches nothing. +func TestJSONEqual(t *testing.T) { + t.Parallel() + for _, c := range []struct { + got, want string + equal bool + }{ + {`{"a":1,"b":[true,null]}`, `{ "b": [true, null], "a": 1 }`, true}, + {`{"n":1.50}`, `{"n":1.5}`, true}, + {`{"n":1e2}`, `{"n":100}`, true}, + {`{"n":9007199254740993}`, `{"n":9007199254740992}`, false}, + {`{"a":"x"}`, `{"a":"y"}`, false}, + {`{"a":1} {"b":2}`, `{"a":1}`, false}, + {`{"a":1}`, `{"a":1}x`, false}, + {`{"a":`, `{"a":1}`, false}, + } { + if got := jsonEqual([]byte(c.got), c.want); got != c.equal { + t.Errorf("jsonEqual(%s, %s) = %v, want %v", c.got, c.want, got, c.equal) + } + } +} diff --git a/drivertest/logs.go b/drivertest/logs.go index 26f4547..656d4b4 100644 --- a/drivertest/logs.go +++ b/drivertest/logs.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "slices" - "strconv" "sync" "testing" "time" @@ -111,25 +110,16 @@ func testJobLogs[TTx any](t *testing.T, f Fixture[TTx], noBatches bool) { // A snooze that logs nothing leaves a job without logs as it is, and // takes one whose only log is a snooze's back to none. - noLogs := func(what string) { - t.Helper() - if got := f.QueryText(ctx, t, d, "SELECT coalesce(logs::text, 'NULL') FROM hopper_jobs WHERE id = $1::uuid", other.String()); got != "NULL" { - t.Errorf("logs %s = %s, want NULL", what, got) - } - } run(other, driver.JobFinalize{State: driver.JobStateScheduled, Snooze: true}, 2, false) - noLogs("after a snooze that logged nothing") - attempt, err := strconv.Atoi(f.QueryText(ctx, t, d, "SELECT attempt::text FROM hopper_jobs WHERE id = $1::uuid", other.String())) - if err != nil { - t.Fatal(err) - } + check("after a snooze that logged nothing", logs(other)) + attempt := get(ctx, t, exec, other).Attempt run(other, driver.JobFinalize{State: driver.JobStateScheduled, Snooze: true, Log: &driver.AttemptLog{Attempt: attempt + 1, Log: "x\n"}}, 2, false) run(other, driver.JobFinalize{State: driver.JobStateScheduled, Snooze: true}, 2, false) - noLogs("after a snooze that logged nothing removed the only log") + check("after a snooze that logged nothing removed the only log", logs(other)) // An expired job takes its logs to history too. run(other, driver.JobFinalize{State: driver.JobStateRetryable, Error: &driver.AttemptError{Attempt: 1, Error: "x"}, Log: &a1}, 2, false) - f.Exec(ctx, t, d, "UPDATE hopper_jobs SET expires_at = now() - interval '1s' WHERE id = $1::uuid", other.String()) + f.ExpireJob(ctx, t, d, other) if expired, err := exec.JobDiscardExpired(ctx, 10); err != nil || len(expired) != 1 || expired[0].ID != other { t.Fatalf("expired = %v, %v", expired, err) } @@ -140,6 +130,40 @@ func testJobLogs[TTx any](t *testing.T, f Fixture[TTx], noBatches bool) { } } +// testJobLogsStored checks how the logs column is kept, by both finalize +// statements, keeping two logs as testJobLogs does: a snooze that logs +// nothing leaves a job without logs as it is, stored as NULL rather than +// rewritten, and takes one whose only log is a snooze's back to NULL. +func testJobLogsStored[TTx any](t *testing.T, f PostgresFixture[TTx], noBatches bool) { + ctx := context.Background() + d := f.NewDriver(t) + exec := executor(d, noBatches) + id := insert(ctx, t, exec, params("k", 1))[0].Job.ID + clientID := register(ctx, t, exec) + snooze := func(log *driver.AttemptLog) { + t.Helper() + f.MakeClaimable(ctx, t, d, id) + j := claimOne(ctx, t, exec, clientID, id) + applied, err := exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{KeepLogs: 2, Jobs: []driver.JobFinalize{ + {ID: id, AttemptedBy: clientID, Fence: j.Fence, State: driver.JobStateScheduled, Snooze: true, Log: log}, + }}) + if err != nil || len(applied) != 1 { + t.Fatalf("snooze = %v, %v", applied, err) + } + } + noLogs := func(what string) { + t.Helper() + if got := f.QueryText(ctx, t, d, "SELECT coalesce(logs::text, 'NULL') FROM hopper_jobs WHERE id = $1::uuid", id.String()); got != "NULL" { + t.Errorf("logs %s = %s, want NULL", what, got) + } + } + snooze(nil) + noLogs("after a snooze that logged nothing") + snooze(&driver.AttemptLog{Attempt: 1, Log: "x\n"}) + snooze(nil) + noLogs("after a snooze that logged nothing removed the only log") +} + // testJobLogsFenced finalizes a job from its attempt and, concurrently, // from clients that do not own it, such as one whose lease lapsed: only the // owner's log is appended, once, and only the owner's snooze that logged diff --git a/drivertest/messaging.go b/drivertest/messaging.go index 5b81df3..be093f5 100644 --- a/drivertest/messaging.go +++ b/drivertest/messaging.go @@ -81,7 +81,7 @@ func testPublish[TTx any](t *testing.T, f Fixture[TTx]) { } else if meta.MessageID != messageID { t.Error("deliveries of one message have different IDs") } - if j.OrderingKey != "alloc:42" || j.UniqueKey != "msg:evt-1" || j.Priority != 1 || j.ExpiresAt.IsZero() || !j.Await || string(j.Args) != `{"id": 42}` { + if j.OrderingKey != "alloc:42" || j.UniqueKey != "msg:evt-1" || j.Priority != 1 || j.ExpiresAt.IsZero() || !j.Await || !jsonEqual(j.Args, `{"id": 42}`) { t.Errorf("delivery = %+v", j) } switch j.Kind { diff --git a/drivertest/progress.go b/drivertest/progress.go index 18c11f5..4e5b09d 100644 --- a/drivertest/progress.go +++ b/drivertest/progress.go @@ -48,7 +48,7 @@ func testJobRecordProgress[TTx any](t *testing.T, f Fixture[TTx]) { } // A cursor is stored as given, a null within it included. got := record(exec, driver.JobRecordProgressParams{Fence: 1, Attempt: 1, Step: "download", Cursor: json.RawMessage(`{"page": 2, "etag": null}`)}) - if got == nil || got.Step != "download" || string(got.Cursor) != `{"etag": null, "page": 2}` || got.RecordedAt.IsZero() { + if got == nil || got.Step != "download" || !jsonEqual(got.Cursor, `{"etag": null, "page": 2}`) || got.RecordedAt.IsZero() { t.Fatalf("recorded = %+v", got) } if p := progress(); p == nil || p.Step != "download" || string(p.Cursor) != string(got.Cursor) || !p.RecordedAt.Equal(got.RecordedAt) { @@ -150,7 +150,7 @@ func testJobRecordProgressRacesFinalize[TTx any](t *testing.T, f Fixture[TTx]) { } done <- applied }() - waitForLockWait(ctx, t, f, d) + f.WaitForLockWait(ctx, t, d) if err := commit(); err != nil { t.Fatal(err) } @@ -178,7 +178,7 @@ func testJobRecordProgressRacesFinalize[TTx any](t *testing.T, f Fixture[TTx]) { } recorded <- p }() - waitForLockWait(ctx, t, f, d) + f.WaitForLockWait(ctx, t, d) if err := commit(); err != nil { t.Fatal(err) } @@ -254,7 +254,7 @@ func testJobProgressReset[TTx any](t *testing.T, f Fixture[TTx]) { if j, err := exec.JobRetry(ctx, driver.JobRetryParams{ID: inserted[1].Job.ID, FromStart: true}); err != nil || j.Progress != nil { t.Errorf("JobRetry FromStart from history = %+v, %v", j, err) } - f.Exec(ctx, t, d, `UPDATE hopper_jobs SET progress = '{"step": "e"}' WHERE id = $1`, inserted[0].Job.ID.String()) + f.SetProgress(ctx, t, d, inserted[0].Job.ID, `{"step": "e"}`) if j, err := exec.JobRetry(ctx, driver.JobRetryParams{ID: inserted[0].Job.ID}); err != nil || j.Progress == nil || j.Progress.Step != "e" { t.Errorf("JobRetry of a waiting job = %+v, %v", j, err) } @@ -271,9 +271,10 @@ func testJobProgressReset[TTx any](t *testing.T, f Fixture[TTx]) { `{"step": 3, "cursor": null, "at": "yesterday"}`: {}, `{"step": "s", "cursor": {"p": 1}, "at": "2026-01-02T03:04:05Z"}`: {Step: "s", Cursor: json.RawMessage(`{"p": 1}`), RecordedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)}, } { - f.Exec(ctx, t, d, `UPDATE hopper_jobs SET progress = $1::jsonb WHERE id = $2`, value, inserted[0].Job.ID.String()) + f.SetProgress(ctx, t, d, inserted[0].Job.ID, value) got := get(inserted[0].Job.ID).Progress - if (got == nil) != (want == nil) || (got != nil && (got.Step != want.Step || string(got.Cursor) != string(want.Cursor) || !got.RecordedAt.Equal(want.RecordedAt))) { + if (got == nil) != (want == nil) || (got != nil && (got.Step != want.Step || (got.Cursor != nil) != (want.Cursor != nil) || + (got.Cursor != nil && !jsonEqual(got.Cursor, string(want.Cursor))) || !got.RecordedAt.Equal(want.RecordedAt))) { t.Errorf("progress %s reads as %+v, want %+v", value, got, want) } } diff --git a/drivertest/snooze.go b/drivertest/snooze.go index d7ce0f6..b627f78 100644 --- a/drivertest/snooze.go +++ b/drivertest/snooze.go @@ -12,11 +12,11 @@ import ( "github.com/parallelworks/hopper/driver" ) -// storedReason reads the snooze_reason column itself, which the driver's +// storedReason reads the stored snooze reason itself, which the driver's // reads show only for a scheduled job. func storedReason[TTx any](ctx context.Context, t *testing.T, f Fixture[TTx], d driver.Driver[TTx], id driver.JobID) string { t.Helper() - return f.QueryText(ctx, t, d, "SELECT coalesce(snooze_reason, '') FROM hopper_jobs WHERE id = $1::uuid", id.String()) + return f.SnoozeReason(ctx, t, d, id) } // leaveReason writes a reason on a job directly, as a client of the @@ -24,7 +24,7 @@ func storedReason[TTx any](ctx context.Context, t *testing.T, f Fixture[TTx], d // leaves one on a running or retrying job. func leaveReason[TTx any](ctx context.Context, t *testing.T, f Fixture[TTx], d driver.Driver[TTx], id driver.JobID) { t.Helper() - f.Exec(ctx, t, d, "UPDATE hopper_jobs SET snooze_reason = 'left behind' WHERE id = $1::uuid", id.String()) + f.SetSnoozeReason(ctx, t, d, id, "left behind") } func testSnoozeReason[TTx any](t *testing.T, f Fixture[TTx], noBatches bool) { @@ -155,7 +155,7 @@ func testSnoozeReason[TTx any](t *testing.T, f Fixture[TTx], noBatches bool) { t.Fatal(err) } snooze("fifth") - f.Exec(ctx, t, d, "UPDATE hopper_jobs SET expires_at = now() - interval '1s' WHERE id = $1::uuid", id.String()) + f.ExpireJob(ctx, t, d, id) expired, err := exec.JobDiscardExpired(ctx, 10) if err != nil || len(expired) != 1 || expired[0].Errors[len(expired[0].Errors)-1].Error != "hopper: expired while snoozed: fifth" { t.Errorf("expired = %+v, %v", expired, err) @@ -200,13 +200,14 @@ func testSnoozeReasonConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } // Only this client finalizes the jobs it holds, so what the // claim cleared stays cleared until then. - ids := make([]string, len(res.Jobs)) - for i, j := range res.Jobs { - ids[i] = j.ID.String() + left := 0 + for _, j := range res.Jobs { + if f.SnoozeReason(ctx, t, d, j.ID) != "" { + left++ + } } - if left := f.QueryText(ctx, t, d, "SELECT count(snooze_reason)::text FROM hopper_jobs WHERE id = ANY($1::text[]::uuid[])", - "{"+strings.Join(ids, ",")+"}"); left != "0" { - t.Errorf("claim left snooze reasons on %s of the jobs it took", left) + if left != 0 { + t.Errorf("claim left snooze reasons on %d of the jobs it took", left) } results := make([]driver.JobFinalize, 0, 2*len(res.Jobs)) for i, j := range res.Jobs { diff --git a/drivertest/stream.go b/drivertest/stream.go index 34d2822..4272fd2 100644 --- a/drivertest/stream.go +++ b/drivertest/stream.go @@ -59,7 +59,7 @@ func testStreams[TTx any](t *testing.T, f Fixture[TTx]) { if e1.Position.IsZero() || !e1.Position.Less(e2.Position) || !e2.Position.Less(e3.Position) { t.Errorf("positions not ascending: %v %v %v", e1.Position, e2.Position, e3.Position) } - if e1.Key != "o1" || e1.Headers["n"] != "1" || e1.MessageID == "" || e1.CreatedAt.IsZero() || string(e1.Payload) != `{"i": 1}` { + if e1.Key != "o1" || e1.Headers["n"] != "1" || e1.MessageID == "" || e1.CreatedAt.IsZero() || !jsonEqual(e1.Payload, `{"i": 1}`) { t.Errorf("event = %+v", e1) } if got := read("", driver.StreamPosition{}); len(got) != 3 || got[0].Position != e1.Position || got[2].Position != e3.Position { @@ -89,7 +89,7 @@ func testStreams[TTx any](t *testing.T, f Fixture[TTx]) { if err != nil || len(consumers) != 2 || consumers[0].Name != "audit" || consumers[1].Name != "orders" { t.Fatalf("consumers = %v, %v", consumers, err) } - if c := consumers[1]; !c.Position.IsZero() || c.Queue != "orders" || string(c.Metadata) != `{"c": 1}` || c.CreatedAt.IsZero() || c.Snapshot == "" { + if c := consumers[1]; !c.Position.IsZero() || c.Queue != "orders" || !jsonEqual(c.Metadata, `{"c": 1}`) || c.CreatedAt.IsZero() || c.Snapshot == "" { t.Errorf("orders consumer = %+v", c) } if c := consumers[0]; c.MaxAttempts != 3 || c.Snapshot == consumers[1].Snapshot { @@ -107,7 +107,7 @@ func testStreams[TTx any](t *testing.T, f Fixture[TTx]) { if argsN(j) != 1 { j = jobs[1] } - if j.Kind != "stream:orders" || j.OrderingKey != "o1" || j.MaxAttempts != 10 || string(j.Args) != `{"i": 1}` { + if j.Kind != "stream:orders" || j.OrderingKey != "o1" || j.MaxAttempts != 10 || !jsonEqual(j.Args, `{"i": 1}`) { t.Errorf("delivery = %+v", j) } var meta struct { @@ -194,7 +194,12 @@ func testStreams[TTx any](t *testing.T, f Fixture[TTx]) { } // A transaction that commits late is delivered when it commits, and - // does not hold up the events of younger transactions meanwhile. + // does not hold up the events of younger transactions meanwhile. An + // engine that runs one writer at a time has no younger transaction to + // commit first: events commit in the order they are appended. + if singleWriter(d) { + return + } tx, commit, rollback := f.Begin(ctx, t, d) defer rollback() //nolint:errcheck // already committed on the happy path old, err := d.UnwrapTx(tx).StreamAppend(ctx, driver.StreamAppendParams{Topic: "order.held", Payload: json.RawMessage(`{}`)}) @@ -220,8 +225,30 @@ func testStreams[TTx any](t *testing.T, f Fixture[TTx]) { if res := pump("orders"); res.Delivered != 0 { t.Errorf("late commit delivered twice: %+v", res) } +} - // Retention keeps a partition ahead and drops nothing young. +// testStreamsRetention checks that retention drops no young event. +func testStreamsRetention[TTx any](t *testing.T, f Fixture[TTx]) { + ctx := context.Background() + exec := f.NewDriver(t).Executor() + for i := range 3 { + if _, err := exec.StreamAppend(ctx, driver.StreamAppendParams{Topic: "t", Payload: json.RawMessage(fmt.Sprintf(`{"i":%d}`, i))}); err != nil { + t.Fatal(err) + } + } + if _, err := exec.HistoryMaintain(ctx, driver.HistoryMaintainParams{CompletedRetention: time.Hour, FailedRetention: time.Hour, StreamRetention: time.Hour}); err != nil { + t.Fatal(err) + } + if got, err := exec.StreamRead(ctx, driver.StreamReadParams{Limit: 10}); err != nil || len(got) != 3 { + t.Errorf("events after maintenance = %d, %v; want 3", len(got), err) + } +} + +// testStreamsPartitioned checks that retention keeps a partition of the +// stream log ahead. +func testStreamsPartitioned[TTx any](t *testing.T, f PostgresFixture[TTx]) { + ctx := context.Background() + exec := f.NewDriver(t).Executor() m, err := exec.HistoryMaintain(ctx, driver.HistoryMaintainParams{CompletedRetention: time.Hour, FailedRetention: time.Hour, StreamRetention: time.Hour}) if err != nil { t.Fatal(err) @@ -229,9 +256,6 @@ func testStreams[TTx any](t *testing.T, f Fixture[TTx]) { if !containsPrefix(m.Created, "hopper_stream_events_") { t.Errorf("no stream partition created: %v", m.Created) } - if got := read("", driver.StreamPosition{}); len(got) != 7 { - t.Errorf("events after maintenance = %d, want 7", len(got)) - } } func containsPrefix(names []string, prefix string) bool { diff --git a/drivertest/tags.go b/drivertest/tags.go index 7963168..cccc7ea 100644 --- a/drivertest/tags.go +++ b/drivertest/tags.go @@ -18,7 +18,7 @@ import ( // testJobTags checks that a job's tags are stored with it on every insert // path, returned wherever the job is, kept through history and a retry // from it, and replaced by a replacing insert, and that a job without tags -// stores none. +// has none. func testJobTags[TTx any](t *testing.T, f Fixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) @@ -32,6 +32,102 @@ func testJobTags[TTx any](t *testing.T, f Fixture[TTx]) { t.Errorf("inserted job %d tags = %q, want %q", i, r.Job.Tags, p[i].Tags) } } + if j := get(ctx, t, exec, res[2].Job.ID); j.Tags != nil { + t.Errorf("untagged job's tags = %q, want none", j.Tags) + } + + clientID := register(ctx, t, exec) + running := claim(ctx, t, exec, clientID, 3) + for _, j := range running { + if want := p[argsN(j)].Tags; !slices.Equal(j.Tags, want) { + t.Errorf("claimed job %d tags = %q, want %q", argsN(j), j.Tags, want) + } + finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: j.ID, AttemptedBy: clientID, State: driver.JobStateCompleted, Archive: true}) + } + got, err := exec.JobGet(ctx, res[1].Job.ID) + if err != nil || got.State != driver.JobStateCompleted || !slices.Equal(got.Tags, p[1].Tags) { + t.Fatalf("job in history = %+v, %v", got, err) + } + retried, err := exec.JobRetry(ctx, driver.JobRetryParams{ID: res[0].Job.ID}) + if err != nil || !slices.Equal(retried.Tags, p[0].Tags) { + t.Errorf("job retried from history = %+v, %v", retried, err) + } + + u := params("u", 1)[0] + u.UniqueKey, u.Tags = "one", []string{"old"} + insert(ctx, t, exec, []driver.JobInsertParams{u}) + u.Tags = []string{"new"} + rep, err := exec.JobInsertMany(ctx, []driver.JobInsertParams{u}, driver.JobInsertOpts{OnConflict: driver.ConflictReplace}) + if err != nil || !rep[0].Duplicate || !slices.Equal(rep[0].Job.Tags, []string{"new"}) { + t.Errorf("replaced job = %+v, %v", rep, err) + } + if byTag, err := exec.JobList(ctx, driver.JobListParams{TagsAny: []string{"old", "new"}, Limit: 10}); err != nil || len(byTag) != 1 || !slices.Equal(byTag[0].Tags, []string{"new"}) { + t.Errorf("replaced job listed by its tags = %+v, %v", byTag, err) + } + u.Tags = nil + rep, err = exec.JobInsertMany(ctx, []driver.JobInsertParams{u}, driver.JobInsertOpts{OnConflict: driver.ConflictReplace}) + if err != nil || rep[0].Job.Tags != nil { + t.Errorf("job replaced without tags = %+v, %v", rep, err) + } + + if d.Capabilities().Copy { + c := params("copy", 2) + c[1].Tags = []string{"copied"} + copied, err := exec.JobInsertCopy(ctx, c, driver.JobInsertOpts{}) + if err != nil { + t.Fatal(err) + } + for i, r := range copied { + stored, err := exec.JobGet(ctx, r.Job.ID) + if err != nil || !slices.Equal(stored.Tags, c[i].Tags) || !slices.Equal(r.Job.Tags, c[i].Tags) { + t.Errorf("copied job %d tags = %q (result %q), want %q, %v", i, stored.Tags, r.Job.Tags, c[i].Tags, err) + } + } + } + + w := params("wf", 2) + w[0].Tags = []string{"flow"} + wf, err := exec.WorkflowInsert(ctx, driver.WorkflowInsertParams{Jobs: w, Deps: []driver.JobDependency{{Job: 1, DependsOn: 0}}}) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(wf.Jobs[0].Job.Tags, []string{"flow"}) || wf.Jobs[1].Job.Tags != nil { + t.Errorf("workflow job tags = %q, %q", wf.Jobs[0].Job.Tags, wf.Jobs[1].Job.Tags) + } + + // A batch's callback keeps its tags until the batch completes. + cb := &driver.JobInsertParams{Kind: "done", Queue: "callbacks", Priority: 1, MaxAttempts: 3, Tags: []string{"callback"}} + batch, err := exec.BatchInsert(ctx, driver.BatchInsertParams{Total: 1, OnComplete: cb}) + if err != nil { + t.Fatal(err) + } + b := params("shard", 1) + b[0].Queue, b[0].BatchID = "shards", batch + insert(ctx, t, exec, b) + shards, err := exec.JobClaim(ctx, driver.JobClaimParams{Queue: "shards", ClientID: clientID, Limit: 1}) + if err != nil || len(shards.Jobs) != 1 { + t.Fatalf("claim = %+v, %v", shards, err) + } + finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: shards.Jobs[0].ID, AttemptedBy: clientID, State: driver.JobStateCompleted, Archive: true}) + callbacks, err := exec.JobList(ctx, driver.JobListParams{Queue: "callbacks", Limit: 10}) + if err != nil || len(callbacks) != 1 || !slices.Equal(callbacks[0].Tags, []string{"callback"}) { + t.Errorf("callbacks = %+v, %v", callbacks, err) + } +} + +// testJobTagsStored checks how tags are stored on Postgres: a job without +// tags stores NULL; both tables' CHECKs refuse tags of another shape; tags a +// direct SQL write made odd read as what they hold, in listings, a retry and +// a batch's callback; and a job finished on a new installation keeps its +// tags rows out of the DEFAULT partition. +func testJobTagsStored[TTx any](t *testing.T, f PostgresFixture[TTx]) { + ctx := context.Background() + d := f.NewDriver(t) + exec := d.Executor() + p := params("k", 3) + p[0].Tags = []string{"tenant:acme", "nightly"} + p[1].Tags = []string{"a"} + insert(ctx, t, exec, p) if n := f.QueryText(ctx, t, d, "SELECT count(*)::text FROM hopper_jobs WHERE tags IS NULL"); n != "1" { t.Errorf("%s jobs store NULL tags, want 1 (the untagged one)", n) } @@ -117,82 +213,13 @@ func testJobTags[TTx any](t *testing.T, f Fixture[TTx]) { f.Exec(ctx, t, d, "DELETE FROM hopper_jobs WHERE queue = 'odd-tags'") clientID := register(ctx, t, exec) - running := claim(ctx, t, exec, clientID, 3) - for _, j := range running { - if want := p[argsN(j)].Tags; !slices.Equal(j.Tags, want) { - t.Errorf("claimed job %d tags = %q, want %q", argsN(j), j.Tags, want) - } + for _, j := range claim(ctx, t, exec, clientID, 3) { finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: j.ID, AttemptedBy: clientID, State: driver.JobStateCompleted, Archive: true}) } if n := f.QueryText(ctx, t, d, "SELECT count(*)::text FROM hopper_job_history_tags_completed_default"); n != "0" { t.Errorf("%s tags rows of jobs finished on a new installation in the DEFAULT partition, want 0", n) } - got, err := exec.JobGet(ctx, res[1].Job.ID) - if err != nil || got.State != driver.JobStateCompleted || !slices.Equal(got.Tags, p[1].Tags) { - t.Fatalf("job in history = %+v, %v", got, err) - } - retried, err := exec.JobRetry(ctx, driver.JobRetryParams{ID: res[0].Job.ID}) - if err != nil || !slices.Equal(retried.Tags, p[0].Tags) { - t.Errorf("job retried from history = %+v, %v", retried, err) - } - u := params("u", 1)[0] - u.UniqueKey, u.Tags = "one", []string{"old"} - insert(ctx, t, exec, []driver.JobInsertParams{u}) - u.Tags = []string{"new"} - rep, err := exec.JobInsertMany(ctx, []driver.JobInsertParams{u}, driver.JobInsertOpts{OnConflict: driver.ConflictReplace}) - if err != nil || !rep[0].Duplicate || !slices.Equal(rep[0].Job.Tags, []string{"new"}) { - t.Errorf("replaced job = %+v, %v", rep, err) - } - u.Tags = nil - rep, err = exec.JobInsertMany(ctx, []driver.JobInsertParams{u}, driver.JobInsertOpts{OnConflict: driver.ConflictReplace}) - if err != nil || rep[0].Job.Tags != nil { - t.Errorf("job replaced without tags = %+v, %v", rep, err) - } - - if d.Capabilities().Copy { - c := params("copy", 2) - c[1].Tags = []string{"copied"} - copied, err := exec.JobInsertCopy(ctx, c, driver.JobInsertOpts{}) - if err != nil { - t.Fatal(err) - } - for i, r := range copied { - stored, err := exec.JobGet(ctx, r.Job.ID) - if err != nil || !slices.Equal(stored.Tags, c[i].Tags) || !slices.Equal(r.Job.Tags, c[i].Tags) { - t.Errorf("copied job %d tags = %q (result %q), want %q, %v", i, stored.Tags, r.Job.Tags, c[i].Tags, err) - } - } - } - - w := params("wf", 2) - w[0].Tags = []string{"flow"} - wf, err := exec.WorkflowInsert(ctx, driver.WorkflowInsertParams{Jobs: w, Deps: []driver.JobDependency{{Job: 1, DependsOn: 0}}}) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(wf.Jobs[0].Job.Tags, []string{"flow"}) || wf.Jobs[1].Job.Tags != nil { - t.Errorf("workflow job tags = %q, %q", wf.Jobs[0].Job.Tags, wf.Jobs[1].Job.Tags) - } - - // A batch's callback keeps its tags until the batch completes. - cb := &driver.JobInsertParams{Kind: "done", Queue: "callbacks", Priority: 1, MaxAttempts: 3, Tags: []string{"callback"}} - batch, err := exec.BatchInsert(ctx, driver.BatchInsertParams{Total: 1, OnComplete: cb}) - if err != nil { - t.Fatal(err) - } - b := params("shard", 1) - b[0].Queue, b[0].BatchID = "shards", batch - insert(ctx, t, exec, b) - shards, err := exec.JobClaim(ctx, driver.JobClaimParams{Queue: "shards", ClientID: clientID, Limit: 1}) - if err != nil || len(shards.Jobs) != 1 { - t.Fatalf("claim = %+v, %v", shards, err) - } - finalizeCurrent(ctx, t, exec, driver.JobFinalize{ID: shards.Jobs[0].ID, AttemptedBy: clientID, State: driver.JobStateCompleted, Archive: true}) - callbacks, err := exec.JobList(ctx, driver.JobListParams{Queue: "callbacks", Limit: 10}) - if err != nil || len(callbacks) != 1 || !slices.Equal(callbacks[0].Tags, []string{"callback"}) { - t.Errorf("callbacks = %+v, %v", callbacks, err) - } // A callback whose tags a direct SQL write made odd gets the tags they // read as, rather than failing the finalize that completes its batch. for _, odd := range []struct { @@ -372,7 +399,7 @@ func testJobListFilters[TTx any](t *testing.T, f Fixture[TTx]) { // tags partition beside it is kept, so that a job still in history is still // listed by its tags. A read of the tags partition defers only the tags // partition's drop, which is dropped once the read ends. -func testTagsPartitionWaitsForHistory[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsPartitionWaitsForHistory[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -446,7 +473,7 @@ func testTagsPartitionWaitsForHistory[TTx any](t *testing.T, f Fixture[TTx]) { // hourly history partitions. Its job's tags rows go with it, though the // day's tags partition stays for the second hour: left there, a listing by // tag walked them before the day's jobs still in history. -func testTagsLeaveWithTheirHour[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsLeaveWithTheirHour[TTx any](t *testing.T, f PostgresFixture[TTx]) { dropTaggedHour(t, f, false) } @@ -455,14 +482,14 @@ func testTagsLeaveWithTheirHour[TTx any](t *testing.T, f Fixture[TTx]) { // a busy hour's delete does. The hour goes without deleting them, rather // than staying, with every later hour, for each pass to try again, and its // rows stay in the day's tags partition, which a listing by tag reads past. -func testTagsHourOutlastsItsDelete[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsHourOutlastsItsDelete[TTx any](t *testing.T, f PostgresFixture[TTx]) { dropTaggedHour(t, f, true) } // dropTaggedHour drops the first of a day's two hourly history partitions, // each holding a tagged job, by retention, its tags rows' delete canceled // if canceled, and checks what stays. -func dropTaggedHour[TTx any](t *testing.T, f Fixture[TTx], canceled bool) { +func dropTaggedHour[TTx any](t *testing.T, f PostgresFixture[TTx], canceled bool) { t.Helper() ctx := context.Background() d := f.NewDriver(t) @@ -512,7 +539,7 @@ func dropTaggedHour[TTx any](t *testing.T, f Fixture[TTx], canceled bool) { // The hour's tags rows go with that partition, which the same pass drops // after the hour, rather than be deleted first: a trigger fails any delete // from it. -func testTagsDayGoesWithItsLastHour[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsDayGoesWithItsLastHour[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -542,7 +569,7 @@ func testTagsDayGoesWithItsLastHour[TTx any](t *testing.T, f Fixture[TTx]) { // partition that another session drops first, as an overlapping leader's // pass does, while this pass's delete of its tags rows waits for it. The // pass reports the hour dropped, as it does one gone before its DROP. -func testTagsHourDroppedMeanwhile[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsHourDroppedMeanwhile[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -587,7 +614,7 @@ func testTagsHourDroppedMeanwhile[TTx any](t *testing.T, f Fixture[TTx]) { // jobs, then checks that the tags tables hold exactly the tags of the jobs // beside them, and that a listing by each tag finds exactly the jobs that // have it, in order, across pages. -func testTagsRowsFollowJobs[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsRowsFollowJobs[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -758,7 +785,7 @@ func testTagsRowsFollowJobs[TTx any](t *testing.T, f Fixture[TTx]) { // testTagsTableMaintained checks that the leader's live table maintenance // vacuums the live tags table, whose index a listing by tag walks, once its // dead rows number as many as the live table's would. -func testTagsTableMaintained[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsTableMaintained[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -792,7 +819,7 @@ func testTagsTableMaintained[TTx any](t *testing.T, f Fixture[TTx]) { // testTagsTableMaintainedPastJobsError checks that the live tags table is // maintained even when hopper_jobs' maintenance fails, here by an extended // statistic whose expression its ANALYZE cannot compute. -func testTagsTableMaintainedPastJobsError[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsTableMaintainedPastJobsError[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -829,7 +856,7 @@ func testTagsTableMaintainedPastJobsError[TTx any](t *testing.T, f Fixture[TTx]) // history's DEFAULT partition takes its tags rows along, wherever they are, // and a tags row in the tags' DEFAULT partition stays while its job is in one // of history's time partitions. -func testTagsFollowHistoryRetention[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsFollowHistoryRetention[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -922,7 +949,7 @@ func testTagsFollowHistoryRetention[TTx any](t *testing.T, f Fixture[TTx]) { // and a partition, which fire none of a partitioned table's statement // triggers. Each tags table then holds exactly its jobs' tags, and a listing // by tag finds each job where it now is. -func testTagsFollowDirectUpdates[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsFollowDirectUpdates[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() @@ -1005,7 +1032,7 @@ func testTagsFollowDirectUpdates[TTx any](t *testing.T, f Fixture[TTx]) { // checkTagsRows checks that each tags table holds exactly the tags of the // jobs beside it, comparing the two with except: "EXCEPT", or "EXCEPT ALL" // to count a duplicate tags row as a mismatch too. -func checkTagsRows[TTx any](ctx context.Context, t *testing.T, f Fixture[TTx], d driver.Driver[TTx], except string) { +func checkTagsRows[TTx any](ctx context.Context, t *testing.T, f PostgresFixture[TTx], d driver.Driver[TTx], except string) { t.Helper() const liveExpanded = `SELECT e.tag, j.id FROM hopper_jobs j, LATERAL jsonb_array_elements_text(j.tags) e(tag)` const historyExpanded = `SELECT h.state, h.finalized_at, e.tag, h.id FROM hopper_job_history h, LATERAL jsonb_array_elements_text(h.tags) e(tag)` @@ -1050,7 +1077,7 @@ func mustID(t *testing.T, s string) driver.JobID { // testTagsFollowTruncate truncates hopper_jobs, then history, each alone and // then together with every tags table, as a reset of every table does. The // tags tables follow, though TRUNCATE fires no delete trigger. -func testTagsFollowTruncate[TTx any](t *testing.T, f Fixture[TTx]) { +func testTagsFollowTruncate[TTx any](t *testing.T, f PostgresFixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) exec := d.Executor() diff --git a/drivertest/unique.go b/drivertest/unique.go index 6f1878e..7d3add4 100644 --- a/drivertest/unique.go +++ b/drivertest/unique.go @@ -178,7 +178,7 @@ func testInsertUniqueFinishedRacesFinalize[TTx any](t *testing.T, f Fixture[TTx] } done <- res[0] }() - waitForLockWait(ctx, t, f, d) + f.WaitForLockWait(ctx, t, d) if err := commit(); err != nil { t.Fatal(err) } diff --git a/drivertest/workflow.go b/drivertest/workflow.go index 714cf5b..6cdf69f 100644 --- a/drivertest/workflow.go +++ b/drivertest/workflow.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "strconv" "strings" "sync" "testing" @@ -168,9 +167,12 @@ func testWorkflows[TTx any](t *testing.T, f Fixture[TTx]) { t.Errorf("chain completed = %+v", row.Batch) } cbs := callbacks() + var alert struct { + BatchFailed int `json:"batch_failed"` + } if cbs["wf-alert2"] == nil || cbs["wf-done2"] != nil { t.Errorf("chain callbacks = %v", cbs) - } else if !strings.Contains(string(cbs["wf-alert2"].Metadata), `"batch_failed": 3`) { + } else if err := json.Unmarshal(cbs["wf-alert2"].Metadata, &alert); err != nil || alert.BatchFailed != 3 { t.Errorf("alert metadata = %s", cbs["wf-alert2"].Metadata) } @@ -212,10 +214,12 @@ func testWorkflows[TTx any](t *testing.T, f Fixture[TTx]) { } // testWorkflowFanInConcurrent finalizes a job's two dependencies in -// concurrent transactions. Each statement still sees the other dependency -// live, so neither promotes the dependent; WorkflowPromoteReady, run by -// each finalizing client after its statement commits, does once both have -// committed, and the leader's sweep finds what clients left. +// concurrent transactions. On an engine that locks rows, each statement +// still sees the other dependency live, so neither promotes the dependent; +// WorkflowPromoteReady, run by each finalizing client after its statement +// commits, does once both have committed. On an engine that runs one writer +// at a time, the second finalize waits for the first and promotes the +// dependent itself. Either way, the leader's sweep finds what clients left. func testWorkflowFanInConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { ctx := context.Background() d := f.NewDriver(t) @@ -249,23 +253,44 @@ func testWorkflowFanInConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { if applied := finalizeCurrent(ctx, t, d.UnwrapTx(tx), completed(x)); len(applied) != 1 { t.Fatalf("finalize in tx applied %v", applied) } - // The other dependency's finalize does not wait for the transaction. - fctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - if applied, err := exec.JobFinalizeMany(fctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{completed(y)}}); err != nil || len(applied) != 1 { - t.Fatalf("concurrent finalize = %v, %v", applied, err) - } - if n := promote(y); n != 0 { - t.Errorf("promoted %d while a dependency's finalize was uncommitted", n) - } - if err := commit(); err != nil { - t.Fatal(err) - } - if j, err := exec.JobGet(ctx, dep); err != nil || j.State != driver.JobStatePending { - t.Fatalf("dependent after both finalizes = %+v, %v; the race should leave it pending", j, err) - } - if n := promote(x); n != 1 { - t.Errorf("promoted %d, want 1", n) + if singleWriter(d) { + done := make(chan []driver.JobID, 1) + go func() { + applied, err := exec.JobFinalizeMany(ctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{completed(y)}}) + if err != nil { + t.Error(err) + } + done <- applied + }() + f.WaitForLockWait(ctx, t, d) + if err := commit(); err != nil { + t.Fatal(err) + } + if applied := <-done; len(applied) != 1 { + t.Fatalf("finalize after the transaction applied %v", applied) + } + if j, err := exec.JobGet(ctx, dep); err != nil || j.State != driver.JobStateAvailable { + t.Fatalf("dependent after both finalizes = %+v, %v; the second should promote it", j, err) + } + } else { + // The other dependency's finalize does not wait for the transaction. + fctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if applied, err := exec.JobFinalizeMany(fctx, driver.JobFinalizeParams{Jobs: []driver.JobFinalize{completed(y)}}); err != nil || len(applied) != 1 { + t.Fatalf("concurrent finalize = %v, %v", applied, err) + } + if n := promote(y); n != 0 { + t.Errorf("promoted %d while a dependency's finalize was uncommitted", n) + } + if err := commit(); err != nil { + t.Fatal(err) + } + if j, err := exec.JobGet(ctx, dep); err != nil || j.State != driver.JobStatePending { + t.Fatalf("dependent after both finalizes = %+v, %v; the race should leave it pending", j, err) + } + if n := promote(x); n != 1 { + t.Errorf("promoted %d, want 1", n) + } } if j, err := exec.JobGet(ctx, dep); err != nil || j.State != driver.JobStateAvailable { t.Errorf("dependent after promote = %+v, %v", j, err) @@ -298,7 +323,7 @@ func testWorkflowFanInConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { t.Fatalf("claimed %+v", jobs) } finalizeCurrent(ctx, t, exec, completed(jobs[0].ID)) - f.Exec(ctx, t, d, `UPDATE hopper_jobs SET state = 'pending' WHERE id = $1::uuid`, wf.Jobs[1].Job.ID.String()) + f.MakePending(ctx, t, d, wf.Jobs[1].Job.ID) steps = append(steps, wf.Jobs[1].Job.ID) } sweep := func(after driver.JobID) driver.WorkflowPromoteResult { @@ -340,6 +365,10 @@ func testWorkflowsConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { exec := d.Executor() const workflows, middles = 100, 4 cb := &driver.JobInsertParams{Kind: "done", Queue: "callbacks", Priority: 2, MaxAttempts: 3, Args: json.RawMessage(`{}`)} + var ( + batchIDs []driver.JobID + dependsOn = map[driver.JobID][]driver.JobID{} + ) for w := range workflows { jobs := params("root", 1) mids := params("mid", middles) @@ -351,10 +380,38 @@ func testWorkflowsConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { for m := 1; m <= middles; m++ { deps = append(deps, driver.JobDependency{Job: m, DependsOn: 0}, driver.JobDependency{Job: middles + 1, DependsOn: m}) } - if _, err := exec.WorkflowInsert(ctx, driver.WorkflowInsertParams{Name: "chaos", Jobs: jobs, Deps: deps, OnComplete: cb}); err != nil { + wf, err := exec.WorkflowInsert(ctx, driver.WorkflowInsertParams{Name: "chaos", Jobs: jobs, Deps: deps, OnComplete: cb}) + if err != nil { t.Fatal(err) } + batchIDs = append(batchIDs, wf.ID) + for _, dep := range deps { + job := wf.Jobs[dep.Job].Job.ID + dependsOn[job] = append(dependsOn[job], wf.Jobs[dep.DependsOn].Job.ID) + } } + // live reports whether a job is still in the live table. + live := func(id driver.JobID) (bool, error) { + j, err := exec.JobGet(ctx, id) + if errors.Is(err, driver.ErrNotFound) { + return false, nil + } + if err != nil { + return false, err + } + return j.FinalizedAt.IsZero(), nil + } + list := func(p driver.JobListParams) []*driver.JobRow { + t.Helper() + p.Limit = 10000 + jobs, err := exec.JobList(ctx, p) + if err != nil { + t.Fatal(err) + } + return jobs + } + liveStates := []driver.JobState{driver.JobStatePending, driver.JobStateAvailable, driver.JobStateScheduled, driver.JobStateRunning, driver.JobStateRetryable} + finished := []driver.JobState{driver.JobStateCompleted, driver.JobStateCanceled, driver.JobStateDiscarded} wctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() @@ -418,22 +475,26 @@ func testWorkflowsConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { } fin := make([]driver.JobFinalize, len(res.Jobs)) batches := make([]driver.JobID, len(res.Jobs)) - ids := make([]string, len(res.Jobs)) for i, j := range res.Jobs { fin[i] = driver.JobFinalize{ID: j.ID, AttemptedBy: clientID, Fence: j.Fence, State: driver.JobStateCompleted, Archive: true} if j.Kind == "fail" { fin[i].State = driver.JobStateDiscarded } batches[i] = j.BatchID - ids[i] = j.ID.String() - } - // A dependency leaves the live table when its finalize - // commits, and never returns: one still there ran after - // its dependent was claimed. - if n := f.QueryText(ctx, t, d, `SELECT count(*)::text FROM hopper_job_deps e - JOIN hopper_jobs x ON x.id = e.depends_on WHERE e.job_id = ANY($1::uuid[])`, "{"+strings.Join(ids, ",")+"}"); n != "0" { - fail(fmt.Errorf("claimed %d jobs with %s dependencies still live", len(ids), n)) - return + // A dependency leaves the live table when its finalize + // commits, and never returns: one still there ran after + // its dependent was claimed. + for _, dep := range dependsOn[j.ID] { + isLive, err := live(dep) + if err != nil { + fail(err) + return + } + if isLive { + fail(fmt.Errorf("claimed job %s with dependency %s still live", j.ID, dep)) + return + } + } } finished := finalizeRetrying(fin) record(finished) @@ -482,8 +543,22 @@ func testWorkflowsConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { time.Sleep(5 * time.Millisecond) } }) - for wctx.Err() == nil && f.QueryText(ctx, t, d, `SELECT ((SELECT count(*) FROM hopper_jobs WHERE queue = 'default') = 0 - AND (SELECT count(*) FROM hopper_batches WHERE completed_at IS NULL) = 0)::text`) != "true" { + done := func() bool { + if len(list(driver.JobListParams{Queue: "default", States: liveStates})) > 0 { + return false + } + for _, id := range batchIDs { + b, err := exec.BatchGet(ctx, id) + if err != nil { + t.Fatal(err) + } + if b.CompletedAt.IsZero() { + return false + } + } + return true + } + for wctx.Err() == nil && !done() { time.Sleep(20 * time.Millisecond) } timedOut := wctx.Err() != nil @@ -494,8 +569,11 @@ func testWorkflowsConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { t.Fatal(err) } if timedOut { - t.Fatalf("not finished in a minute (last finalize error: %v): %s", lastErr, f.QueryText(ctx, t, d, - "SELECT coalesce(string_agg(state || ' ' || n, ', '), 'none') FROM (SELECT state::text, count(*) AS n FROM hopper_jobs GROUP BY state) s")) + states := map[driver.JobState]int{} + for _, j := range list(driver.JobListParams{Queue: "default", States: liveStates}) { + states[j.State]++ + } + t.Fatalf("not finished in a minute (last finalize error: %v): %v", lastErr, states) } failing := workflows / 5 @@ -509,26 +587,50 @@ func testWorkflowsConcurrent[TTx any](t *testing.T, f Fixture[TTx]) { if len(applied) != want || twice != 0 { t.Errorf("results applied for %d jobs, %d of them more than once; want %d jobs, once each", len(applied), twice, want) } - checks := []struct{ what, query, want string }{ - { - "history rows, distinct jobs", `SELECT count(*) || ' ' || count(DISTINCT id) FROM hopper_job_history WHERE queue = 'default'`, - fmt.Sprintf("%d %d", workflows*(middles+2), workflows*(middles+2)), - }, - { - "last steps by state", `SELECT string_agg(state || ' ' || n, ', ' ORDER BY state) FROM ( - SELECT state::text, count(*) AS n FROM hopper_job_history WHERE kind = 'last' GROUP BY state) s`, - fmt.Sprintf("canceled %d, completed %d", failing, workflows-failing), - }, - {"canceled last steps that ran", `SELECT count(*) FROM hopper_job_history WHERE kind = 'last' AND state = 'canceled' AND attempted_at IS NOT NULL`, "0"}, - {"workflows whose failures miscount", `SELECT count(*) FROM hopper_batches b - WHERE b.failed <> (SELECT count(*) FROM hopper_job_history h WHERE h.batch_id = b.id AND h.state <> 'completed')`, "0"}, - {"failures", `SELECT sum(failed) FROM hopper_batches`, strconv.Itoa(2 * failing)}, - {"completion callbacks", `SELECT count(*) FROM hopper_jobs WHERE queue = 'callbacks'`, strconv.Itoa(workflows)}, - {"edges left", `SELECT count(*) FROM hopper_job_deps`, "0"}, + history := list(driver.JobListParams{Queue: "default", States: finished}) + distinct := map[driver.JobID]bool{} + lastByState := map[driver.JobState]int{} + ranCanceled := 0 + failedByBatch := map[driver.JobID]int{} + for _, j := range history { + distinct[j.ID] = true + if j.Kind == "last" { + lastByState[j.State]++ + if j.State == driver.JobStateCanceled && !j.AttemptedAt.IsZero() { + ranCanceled++ + } + } + if j.State != driver.JobStateCompleted { + failedByBatch[j.BatchID]++ + } + } + if len(history) != workflows*(middles+2) || len(distinct) != len(history) { + t.Errorf("history has %d rows of %d distinct jobs, want %d of as many", len(history), len(distinct), workflows*(middles+2)) + } + if lastByState[driver.JobStateCanceled] != failing || lastByState[driver.JobStateCompleted] != workflows-failing || len(lastByState) != 2 { + t.Errorf("last steps by state = %v, want %d canceled and %d completed", lastByState, failing, workflows-failing) + } + if ranCanceled != 0 { + t.Errorf("%d canceled last steps ran", ranCanceled) } - for _, c := range checks { - if got := f.QueryText(ctx, t, d, "SELECT ("+c.query+")::text"); got != c.want { - t.Errorf("%s = %s, want %s", c.what, got, c.want) + failures := 0 + for _, id := range batchIDs { + b, err := exec.BatchGet(ctx, id) + if err != nil { + t.Fatal(err) + } + failures += b.Failed + if b.Failed != failedByBatch[id] { + t.Errorf("workflow %s counts %d failures, its history %d", id, b.Failed, failedByBatch[id]) } } + if failures != 2*failing { + t.Errorf("failures = %d, want %d", failures, 2*failing) + } + if n := len(list(driver.JobListParams{Queue: "callbacks"})); n != workflows { + t.Errorf("completion callbacks = %d, want %d", n, workflows) + } + if n := f.CountEdges(ctx, t, d); n != 0 { + t.Errorf("edges left = %d, want 0", n) + } } diff --git a/execute.go b/execute.go index 04eae96..603c3d7 100644 --- a/execute.go +++ b/execute.go @@ -42,6 +42,7 @@ func (c *Client[TTx]) execute(ctx context.Context, row *driver.JobRow, qcfg Queu defer func() { a.closeLog(c.logCloseTimeout(ctx)) }() jobCtx = context.WithValue(jobCtx, attemptKey{}, a) jobCtx = context.WithValue(jobCtx, clientKey{}, c) + jobCtx = driver.WithOwner(jobCtx, a) claim := c.trackRunning(row.ID, cancel) defer c.untrackRunning(row.ID, claim) diff --git a/hoppermigrate/hoppermigrate.go b/hoppermigrate/hoppermigrate.go index de60242..3c8f533 100644 --- a/hoppermigrate/hoppermigrate.go +++ b/hoppermigrate/hoppermigrate.go @@ -5,6 +5,12 @@ // so any number of replicas can call Up at startup. The raw files are also // available through Migrations for teams that run migrations with their own // tool; hopper_schema records the applied versions either way. +// +// The embedded scripts are Postgres's. A driver for another engine supplies +// its own script for each version (driver.MigrationScripts), and Up and Down +// apply those, at the same versions; such a driver also returns its scripts +// for other tools, as hoppersqlite.Migrations does. A new schema version +// therefore needs a script for each engine. package hoppermigrate import ( @@ -33,7 +39,9 @@ type Migration struct { var migrations = mustLoad() -// Migrations returns every known migration in ascending version order. +// Migrations returns every known migration in ascending version order, +// with Postgres's scripts. A driver for another engine has its own +// (hoppersqlite.Migrations, for SQLite). func Migrations() []Migration { return slices.Clone(migrations) } // Latest returns the newest schema version this package knows. @@ -122,7 +130,11 @@ func run[TTx any](ctx context.Context, d driver.Driver[TTx], opts *Options, up b if m.Version > target || slices.Contains(applied, m.Version) { continue } - if err := exec.Apply(ctx, m.Version, m.Up, true); err != nil { + script, err := scriptFor(exec, m, true) + if err != nil { + return nil, err + } + if err := exec.Apply(ctx, m.Version, script, true); err != nil { return nil, fmt.Errorf("hoppermigrate: apply %03d_%s: %w", m.Version, m.Name, err) } logger.InfoContext(ctx, "hopper: applied migration", "version", m.Version, "name", m.Name) @@ -136,7 +148,11 @@ func run[TTx any](ctx context.Context, d driver.Driver[TTx], opts *Options, up b if m.Version <= target || !slices.Contains(applied, m.Version) { continue } - if err := exec.Apply(ctx, m.Version, m.Down, false); err != nil { + script, err := scriptFor(exec, m, false) + if err != nil { + return nil, err + } + if err := exec.Apply(ctx, m.Version, script, false); err != nil { return nil, fmt.Errorf("hoppermigrate: revert %03d_%s: %w", m.Version, m.Name, err) } logger.InfoContext(ctx, "hopper: reverted migration", "version", m.Version, "name", m.Name) @@ -146,6 +162,23 @@ func run[TTx any](ctx context.Context, d driver.Driver[TTx], opts *Options, up b return res, nil } +// scriptFor returns m's up or down script for the engine exec migrates: its +// own, if it has scripts of its own, or the embedded Postgres one. +func scriptFor(exec driver.MigrationExecutor, m Migration, up bool) (string, error) { + scripts, ok := exec.(driver.MigrationScripts) + if !ok { + if up { + return m.Up, nil + } + return m.Down, nil + } + script, err := scripts.Script(m.Version, up) + if err != nil { + return "", fmt.Errorf("hoppermigrate: the engine's script for %03d_%s: %w", m.Version, m.Name, err) + } + return script, nil +} + func current(applied []int) int { if len(applied) == 0 { return 0 diff --git a/hoppermigrate/scripts_test.go b/hoppermigrate/scripts_test.go new file mode 100644 index 0000000..635e39f --- /dev/null +++ b/hoppermigrate/scripts_test.go @@ -0,0 +1,96 @@ +package hoppermigrate_test + +import ( + "context" + "errors" + "fmt" + "log/slog" + "slices" + "strings" + "testing" + + "github.com/parallelworks/hopper/driver" + "github.com/parallelworks/hopper/hoppermigrate" +) + +// engineDriver is a driver whose engine has migration scripts of its own, +// which its executor records as they are applied. +type engineDriver struct{ exec *engineExec } + +func (engineDriver) Executor() driver.Executor { return nil } +func (engineDriver) UnwrapTx(struct{}) driver.Executor { return nil } +func (engineDriver) Capabilities() driver.Capabilities { return driver.Capabilities{} } +func (d engineDriver) Migrator() driver.Migrator { return d } +func (engineDriver) Listener(context.Context) (driver.Listener, error) { + return nil, driver.ErrNotSupported +} + +func (d engineDriver) Lock(context.Context) (driver.MigrationExecutor, error) { return d.exec, nil } + +type engineExec struct { + versions []int + applied []string + // missing is a version the engine has no script for. + missing int +} + +func (e *engineExec) Versions(context.Context) ([]int, error) { return slices.Clone(e.versions), nil } + +func (e *engineExec) Script(version int, up bool) (string, error) { + if version == e.missing { + return "", errors.New("no script") + } + if up { + return fmt.Sprintf("engine up %d", version), nil + } + return fmt.Sprintf("engine down %d", version), nil +} + +func (e *engineExec) Apply(_ context.Context, version int, script string, up bool) error { + e.applied = append(e.applied, script) + if up { + e.versions = append(e.versions, version) + } else { + e.versions = slices.DeleteFunc(e.versions, func(v int) bool { return v == version }) + } + return nil +} + +func (*engineExec) Close(context.Context) error { return nil } + +// TestEngineScripts migrates an engine that has scripts of its own: Up and +// Down apply its scripts, not the embedded Postgres ones, at hoppermigrate's +// versions, and an engine without a version's script fails the run, naming +// the migration. +func TestEngineScripts(t *testing.T) { + t.Parallel() + ctx := context.Background() + opts := &hoppermigrate.Options{Logger: slog.New(slog.DiscardHandler)} + exec := &engineExec{} + d := engineDriver{exec} + res, err := hoppermigrate.Up(ctx, d, opts) + if err != nil || res.Version != hoppermigrate.Latest() { + t.Fatalf("Up = %+v, %v", res, err) + } + for i, script := range exec.applied { + if want := fmt.Sprintf("engine up %d", i+1); script != want { + t.Fatalf("script %d = %q, want %q", i, script, want) + } + } + exec.applied = nil + if _, err := hoppermigrate.Down(ctx, d, &hoppermigrate.Options{Logger: opts.Logger, Target: hoppermigrate.Latest() - 2}); err != nil { + t.Fatal(err) + } + if want := []string{fmt.Sprintf("engine down %d", hoppermigrate.Latest()), fmt.Sprintf("engine down %d", hoppermigrate.Latest()-1)}; !slices.Equal(exec.applied, want) { + t.Errorf("Down applied %q, want %q", exec.applied, want) + } + + exec = &engineExec{missing: 3} + _, err = hoppermigrate.Up(ctx, engineDriver{exec}, opts) + if err == nil || !strings.Contains(err.Error(), "003_"+hoppermigrate.Migrations()[2].Name) { + t.Errorf("Up without a script for version 3 = %v", err) + } + if len(exec.applied) != 2 { + t.Errorf("applied %q before the missing script, want versions 1 and 2", exec.applied) + } +} diff --git a/lease_internal_test.go b/lease_internal_test.go new file mode 100644 index 0000000..98876ee --- /dev/null +++ b/lease_internal_test.go @@ -0,0 +1,114 @@ +package hopper + +import ( + "context" + "log/slog" + "slices" + "sync" + "testing" + "testing/synctest" + "time" + + "github.com/parallelworks/hopper/driver" +) + +// leaseExec registers clients and answers renewals: it renews at once +// while renew is set, and otherwise holds each renewal until its context +// ends, as a renewal behind a write lock another transaction holds waits, +// recording how long each was given. +type leaseExec struct { + driver.Executor + mu sync.Mutex + renew bool + waits []time.Duration +} + +func (e *leaseExec) ClientRegister(context.Context, driver.ClientRegisterParams) (int64, error) { + return 1, nil +} + +func (e *leaseExec) ClientRenew(ctx context.Context, _ driver.ClientRenewParams) (driver.ClientRenewResult, error) { + deadline, _ := ctx.Deadline() + e.mu.Lock() + e.waits = append(e.waits, time.Until(deadline)) + renew := e.renew + e.mu.Unlock() + if renew { + return driver.ClientRenewResult{Renewed: true}, nil + } + <-ctx.Done() + return driver.ClientRenewResult{}, ctx.Err() +} + +func (e *leaseExec) setRenew(renew bool) { + e.mu.Lock() + e.renew = renew + e.mu.Unlock() +} + +// TestRenewalWaitsUntilTheLeaseExpires checks the lease renewal's bound: a +// renewal held up waits until the lease it renews would expire, counted +// from when it was last taken or renewed (by registering again too), and +// at least one renewal interval once it has. +func TestRenewalWaitsUntilTheLeaseExpires(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + exec := &leaseExec{renew: true} + tun := defaultTuning.withLease(15 * time.Second) // renewals every 5s + c := &Client[struct{}]{exec: exec, logger: slog.New(slog.DiscardHandler), tuning: tun} + if _, err := c.register(ctx); err != nil { // the lease from 0s + t.Fatal(err) + } + done := make(chan struct{}) + go func() { defer close(done); c.leaseLoop(ctx) }() + + time.Sleep(6 * time.Second) // renewed at 5s + exec.setRenew(false) + time.Sleep(2 * time.Second) + if _, err := c.register(ctx); err != nil { // registered again at 8s + t.Fatal(err) + } + // Renewals at 10s, held until 23s, the new lease's expiry; then + // at 23s, the expired lease's, held one interval, to 28s. + time.Sleep(25 * time.Second) + cancel() + <-done + exec.mu.Lock() + defer exec.mu.Unlock() + if want := []time.Duration{10 * time.Second, 13 * time.Second, 5 * time.Second}; !slices.Equal(exec.waits[:min(len(exec.waits), 3)], want) { + t.Errorf("renewals were given %v, want %v first", exec.waits, want) + } + }) +} + +// TestRenewalBoundMovesWithRenewals checks the common case: each renewal +// that succeeds moves the bound, so a renewal held up after two that +// succeeded is given until the lease the last of them took expires. +func TestRenewalBoundMovesWithRenewals(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + exec := &leaseExec{renew: true} + tun := defaultTuning.withLease(15 * time.Second) // renewals every 5s + c := &Client[struct{}]{exec: exec, logger: slog.New(slog.DiscardHandler), tuning: tun} + if _, err := c.register(ctx); err != nil { // the lease from 0s + t.Fatal(err) + } + done := make(chan struct{}) + go func() { defer close(done); c.leaseLoop(ctx) }() + + time.Sleep(12 * time.Second) // renewed at 5s and 10s + exec.setRenew(false) + // The renewal at 15s is held, and given until 25s, when the lease + // renewed at 10s expires. + time.Sleep(5 * time.Second) + cancel() + <-done + exec.mu.Lock() + defer exec.mu.Unlock() + if want := []time.Duration{10 * time.Second, 10 * time.Second, 10 * time.Second}; !slices.Equal(exec.waits, want) { + t.Errorf("renewals were given %v, want %v", exec.waits, want) + } + }) +} diff --git a/release-please-config.json b/release-please-config.json index b0ac00b..2acdae5 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -66,6 +66,10 @@ { "type": "generic", "path": "hopperui/go.mod" + }, + { + "type": "generic", + "path": "driver/hoppersqlite/go.mod" } ] } diff --git a/steps.go b/steps.go index a850450..d6f1a90 100644 --- a/steps.go +++ b/steps.go @@ -52,6 +52,12 @@ func RecordStep[T JobArgs](ctx context.Context, job *Job[T], step string, cursor // before the step returns, since RunSteps records the next finished step on // the pool, and a record on the pool while tx is open returns ErrJobLocked. // A job's finalize skips a row tx still holds, and is retried once tx ends. +// On an engine that runs one writer at a time +// (driver.Capabilities.SingleWriter, SQLite), tx holds the whole database +// for writes from its first statement until it ends: every other write +// waits for it, lease renewals included, and a write of the worker's own +// on the pool fails rather than wait for tx, provided tx was begun with the +// worker's ctx. func RecordStepTx[TTx any, T JobArgs](ctx context.Context, tx TTx, job *Job[T], step string, cursor any) error { return record(ctx, job, func() (driver.Executor, error) { return txExecutor(ctx, tx, job.ID) }, step, false, cursor) }