diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f9632f42a..c0419fceb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: # Categories are declared by the property in each test project. A category can # span several projects that share infrastructure, so that the container is provisioned once # and the union of their build closures is compiled once. - test-category: [ DefaultCore, DefaultMonitoring, Raven, SqlServer, PostgreSql, RabbitMQ, AzureServiceBus, AzureStorageQueues, MSMQ, SQS, IBMMQ ] + test-category: [ DefaultCore, DefaultMonitoring, Migration, Raven, SqlServer, PostgreSql, RabbitMQ, AzureServiceBus, AzureStorageQueues, MSMQ, SQS, IBMMQ ] include: - os: windows-latest os-name: Windows @@ -87,7 +87,7 @@ jobs: id: select run: ./tools/select-test-projects.ps1 -Category ${{ matrix.test-category }} - name: Download RavenDB Server - if: matrix.test-category == 'DefaultCore' + if: contains(fromJSON('["DefaultCore", "Migration"]'), matrix.test-category) run: ./tools/download-ravendb-server.ps1 - name: Build id: build diff --git a/README.md b/README.md index dbdd477cb5..56dddcd55b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ It's also possible to [locally test containers built from PRs in GitHub Containe If the instance is executed for the first time, it must set up the required infrastructure. To do so, once the instance is configured to use the selected transport and persister, run it in setup mode. This can be done by using the `Setup {instance name}` launch profile that is defined in the `launchSettings.json` file of each instance. When started in setup mode, the instance will start as usual, execute the setup process, and exit. At this point the instance can be run normally by using the non-setup launch profile. +## Migrating from RavenDB to SQL Server or PostgreSQL + +See [Migrating from RavenDB to SQL Server or PostgreSQL](docs/migration/ravendb-to-sql-migration-instructions.md). + ## Secrets Testing using the [CI workflow](/.github/workflows/ci.yml) depends on the following secrets. The Particular values for these secrets are stored in the secure note named **ServiceControl Repo Secrets**. diff --git a/docs/migration/migration-system-design-diagram.png b/docs/migration/migration-system-design-diagram.png new file mode 100644 index 0000000000..15845976ae Binary files /dev/null and b/docs/migration/migration-system-design-diagram.png differ diff --git a/docs/migration/ravendb-to-sql-migration-instructions.md b/docs/migration/ravendb-to-sql-migration-instructions.md new file mode 100644 index 0000000000..7b3fc8c7ae --- /dev/null +++ b/docs/migration/ravendb-to-sql-migration-instructions.md @@ -0,0 +1,54 @@ +# Migrating from RavenDB to SQL Server or PostgreSQL + +This page covers what you can run today. How the migration works, and what is planned, is in the [migration overview](ravendb-to-sql-migration-overview.md) and the [system design diagram](migration-system-design-diagram.png). + +> [!NOTE] +> Copying data is not built yet. The one migration command available is the source report. It sends RavenDB only reads, but loading a database lets RavenDB's own expiration, its automatic deletion of documents past their retention date, run against it. If you are keeping the RavenDB database as a fallback, back it up before you run the report, as [Goals](ravendb-to-sql-migration-overview.md#goals) explains. + +## Before you start + +The source is a ServiceControl error instance on RavenDB. Keep its RavenDB settings in its configuration: the migration reads RavenDB through them, including after `PersistenceType` is switched to SQL Server or PostgreSQL. + +| Setting | Environment variable | What it is | +| --- | --- | --- | +| `ServiceControl/RavenDB/ConnectionString` | `SERVICECONTROL_RAVENDB_CONNECTIONSTRING` | An external RavenDB server. Leave unset for an embedded database | +| `ServiceControl/DbPath` | `SERVICECONTROL_DBPATH` | The embedded database's data directory | +| `ServiceControl/RavenDB/DatabaseName` | `SERVICECONTROL_RAVENDB_DATABASENAME` | The primary database, `primary` by default | +| `LicensingComponent/RavenDB/ThroughputDatabaseName` | `LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME` | The throughput database, `throughput` by default | +| `ServiceControl/RavenDB/ClientCertificatePath` or `ServiceControl/RavenDB/ClientCertificateBase64`, with `ServiceControl/RavenDB/ClientCertificatePassword` | `SERVICECONTROL_RAVENDB_CLIENTCERTIFICATEPATH` and so on | A secured external server's client certificate | +| `ServiceControl/ErrorRetentionPeriod` | `SERVICECONTROL_ERRORRETENTIONPERIOD` | Required. Don't change it during the move | + +`ServiceControl/Migration/SourcePersistenceType` defaults to `RavenDB` and needs no setting. + +## Report on the source + +Run the instance's executable with `--migration-source-report`: + +```powershell +# Installed on Windows, from the instance's installation folder +.\ServiceControl.exe --migration-source-report +``` + +```shell +# Container, against an external RavenDB server +docker run --rm --env-file servicecontrol.env ghcr.io/particular/servicecontrol: --migration-source-report +``` + +From source, build `src/ServiceControl` and run the same command from its output folder, as in [How to run/debug locally](../../README.md#how-to-rundebug-locally). + +The report prints the RavenDB server version, whether the source is embedded or external and where it is, both database names with the setting each came from, and a row count for every collection in both databases. + +- **External server:** run it while ServiceControl is running. It sends only reads, and the note above about expiration applies to a server you are keeping as a fallback. +- **Embedded database:** stop the ServiceControl service, run the report, then start the service again. The report starts its own RavenDB process against the data directory, which cannot happen while the instance holds it. +- **Container with an embedded database:** not supported, because the container image does not ship the RavenDB server. Point the instance at an external RavenDB server instead. + +## If the report fails + +The error names the setting to fix: + +- **"has no database named ..."**: the database name setting it quotes is wrong. +- **"refused its client certificate access ..."**: grant that certificate Read access to the database, or supply a certificate that has it. + +## Not available yet + +Copying the data (`MigrationMode`), the dry run, and the status and verify commands are planned but not built. The planned steps are in [Migration workflow](ravendb-to-sql-migration-overview.md#migration-workflow). diff --git a/docs/migration/ravendb-to-sql-migration-overview.md b/docs/migration/ravendb-to-sql-migration-overview.md new file mode 100644 index 0000000000..de2c322b14 --- /dev/null +++ b/docs/migration/ravendb-to-sql-migration-overview.md @@ -0,0 +1,373 @@ +# Moving data from RavenDB to SQL + +## Problem + +A customer can already point ServiceControl at SQL Server or PostgreSQL. They cannot bring their existing data with them. + +This covers the error instance only. The audit instance has no SQL persister, so a customer who finishes this migration still runs RavenDB for audit. + +## Strategy + +- Switch over first, and copy only what has to be copied. Retention does most of the work: error retention is between 5 and 45 days and event retention is shorter, so most of the source ages out on its own within weeks. That is why archived and resolved messages are optional rather than required. Retention would have deleted them anyway. +- The required set is small because unresolved failures are the only category a customer can act on, and the strategy assumes customers keep that number low by resolving and archiving. **A neglected instance breaks that assumption**: unresolved failures can legitimately be months old, and a large backlog of them makes the closed window long rather than short. The dry run is what tells a customer which case they are in. +- Anything not selected simply ages out of RavenDB, and the customer deletes the old database when they are ready. + +## Goals + +- **Minimal downtime**. Only the required data copies with ServiceControl closed. Optional data copies in the background while it serves traffic. +- **All three RavenDB sources are supported**. Embedded, a container, or RavenDB Cloud, on one code path rather than three. +- **No writes through the client**. The copier never changes the source, but RavenDB's own expiration does: the primary database already has it configured, and the sweep keeps deleting failed messages and event log items throughout the migration and for as long afterwards as the instance is left running. The old database is a fallback that degrades from the moment you start. +- **Abandonable up to a known point, and only up to that point**. While ServiceControl is closed the copy can be thrown away at no cost, because nothing but the copier has written to SQL and the migration has written nothing to RavenDB: see [the one point you can go back](#the-one-point-you-can-go-back). Once the host opens there is no way back at all. +- **No duplicates and no gaps**. Rows and the resume cursor commit in one transaction, so a crash needs no reconciliation. +- **Every identifier anything depends on is carried across**. The event log and historic retry operations are renumbered, because nothing references their keys. +- **Refuse rather than half-migrate**. Every check runs before the first row moves, and a failure is a host that will not start. +- **No silent loss**. A migration cannot end with a selected category still in progress or halted, only with each one finished or explicitly abandoned. Abandoning is a deliberate choice, and an abandoned category lets the host open. Skipped rows are counted and reported. +- **Bounded impact on a live instance**. Throttled behind normal ingestion and streamed, so memory does not track the size of the database. +- **Known before it starts, visible while it runs**. A dry run reports what will move and how long ServiceControl is closed, and every category transition is reported as it happens. +- **Use existing functionality where possible**. Progress goes through custom checks and the activity feed, so no new client or screen is needed. + +## Deliberately not built, and not currently planned + +- **Zero downtime.** The required data is copied with ServiceControl closed, so there is a real, if short, outage. +- **Reversible once ServiceControl opens.** Nothing copies SQL rows back to RavenDB, so once the host has served traffic there is no rollback of any kind. +- **Steerable while running.** No pause, resume, or abort. Changing anything means editing configuration and restarting. +- **A general-purpose migration tool.** The source is always RavenDB and the target is always a ServiceControl EF Core persister, both at versions this build can read. +- **Custom migration UI via ServicePulse.** Custom checks and the event log will be used for progress reporting, but migration configuration and migration engine control will not be available via the UI. + +## Supported migration scenarios + +The copier runs inside the ServiceControl host, so every row and every message body travels from the source, through ServiceControl, to the target. There is no database-to-database transfer, no backup and restore, and no replication. Whether a combination works therefore comes down to whether the ServiceControl host can reach both ends at once, and how long it takes comes down to how far the data has to travel. + +**Supported locations:** + +- **The RavenDB source**: embedded on the ServiceControl host (Windows installations only, see below), self-hosted on the same network in a container, VM or bare metal, or RavenDB Cloud +- **The SQL target**: SQL Server or PostgreSQL on the ServiceControl host, elsewhere on the same network, in a container, or as a managed cloud service such as Azure SQL, Amazon RDS or Google Cloud SQL +- **Any combination of the two**, subject to the requirements below + +**By where the data has to travel:** + +- **On-prem to on-prem.** The common case and the fastest. Embedded or self-hosted RavenDB to SQL on the same host or the same network. +- **On-prem to cloud.** RavenDB on the network, managed SQL in a cloud. Works, but each batch is one round trip, so write latency multiplies by the number of batches rather than being amortised away. +- **Cloud to on-prem.** RavenDB Cloud down to local SQL. Works, and the customer pays egress on everything copied, most of which is archived messages and their bodies. +- **Cloud to cloud.** Works, and is only sensible when ServiceControl runs alongside one of them. A host sitting on-prem between two clouds pulls every byte down and pushes it straight back up. +- **The message body store is a third location.** Bodies come out of RavenDB attachments and go wherever the target is configured to put them: a filesystem, Azure Blob or S3, with small text bodies kept inline in the database. That decision is made at the same time as the database move. + +**Infrastructure requirements:** + +- The ServiceControl host needs network access to the RavenDB source, the SQL target and the body store simultaneously +- Both RavenDB databases, primary and throughput, on one server or cluster +- A SQL Server target must have Full-Text Search installed. `--setup` checks `SERVERPROPERTY('IsFullTextInstalled')` and fails if it is absent, because message search is not optional. A stock SQL Server container image does not include it. PostgreSQL needs nothing extra, since its index is a GIN over `to_tsvector` +- A managed target's transient failures are already survivable: retry on failure is on by default and there is no setting to turn it off + +**Not supported:** + +- Any server-to-server copy: no backup and restore, no RavenDB ETL or replication into SQL, no external data pipeline +- A host that can reach only one of the two databases at a time, so no staged move by way of an offline copy +- **An embedded RavenDB source when ServiceControl runs in a container.** Reading an embedded database means starting a RavenDB server process, and the container image does not carry one: `ServiceControl.Persistence.RavenDB.csproj:36` excludes the `RavenDBServer` directory from the artifact, and the copy that would restore it at `:44` is conditional on `CI` not being set, which the Dockerfile sets. A containerised instance migrating away from embedded RavenDB has to point at an external RavenDB server rather than at a data directory. Windows installations are unaffected: the installer unzips the server unconditionally +- Primary and throughput RavenDB databases in different locations +- Anything but RavenDB as the source, or anything but a ServiceControl EF Core persister as the target + +## Migration workflow + +1. Upgrade ServiceControl as normal, still on RavenDB. +2. Set four things in configuration: the new `PersistenceType`, its connection string, `MigrationMode=true`, and which [optional data](#data-to-be-migrated-categories) they want copied. +3. Run `--setup` to create the SQL schema. It fails against a SQL Server instance without Full-Text Search installed. +4. Run the [dry run](#dry-run). It reports what it resolved as a source, what each category holds, and an estimate of how long ServiceControl will be closed. Read [what the dry run reports](#dry-run) before booking an outage around its estimate. +5. Start ServiceControl (`MigrationMode=true`). +6. Every check runs before a single row moves. If one fails the host does not start and names which, having copied nothing, so a wrong database name or unconfigured body storage costs a restart rather than a half-finished migration. +7. The copying of [required data](#required) starts, with ServiceControl still closed. This is assumed to be a small amount of data. +8. ServiceControl opens, and whatever [optional data](#optional) they asked for is copied in the background while the instance runs normally. They can watch it from ServicePulse custom checks and events, but not steer it. +9. They run the verification pass once the background job has completed, which reports row counts on both sides category by category, accounting for deliberate skips so a difference is explained rather than reported as a fault, then set `MigrationMode=false` and restart. It tolerates more rows in SQL than in RavenDB, because RavenDB keeps expiring rows the copier already took. +10. RavenDB data can be removed. + +- If `MigrationMode=false` is set while a selected category is still incomplete, the host refuses to start and names exactly what is outstanding. +- A category that ended *complete with errors* counts as complete and does not block, though its skipped count is printed so the loss is stated rather than silent. +- An explicit override exists for a customer who has changed their mind and accepts leaving data behind. It marks the outstanding categories as abandoned, which is a deliberate end state rather than a failure, so the progress check settles and the guard stays armed for any later migration. +- **Steps 5 to 7 are the abort window**, which is not the override above: see [the one point you can go back](#the-one-point-you-can-go-back). +- A category that stops because too many rows failed is *halted*, and it stays that way until someone acts: fix the cause and restart to carry on from where it stopped, or abandon it deliberately if you accept the loss. See [a halt stops one category, and clearing it is a restart](#a-halt-stops-one-category-and-clearing-it-is-a-restart). + +## Architecture + +```mermaid +flowchart TB + cfg["Configuration + restart
the only way to change anything"] + checks["Custom checks + activity feed
progress, with no new client needed"] + + subgraph host["One ServiceControl host process, started with MigrationMode = true"] + direction LR + raven["RavenDB persister
own AssemblyLoadContext
new read-only lifecycle"] + engine["MigrationEngine
categories, throttle,
dry run, verification"] + target["EF Core persister
SQL Server or PostgreSQL
own AssemblyLoadContext"] + raven -->|"IMigrationSource"| engine + engine -->|"IMigrationTarget"| target + end + + old[("Old RavenDB
read only, never written to")] + sql[("SQL Server or PostgreSQL
plus a new checkpoint table")] + bodies[("Message body store
filesystem, Azure Blob or S3")] + + cfg --> host + host --> checks + old --> raven + target --> sql + target --> bodies +``` + +- **The engine and the host know no store.** They deal in categories, cursors and counts. The source maps a category to what it reads and describes itself as labelled facts, the target maps a category to where it writes and how to count it, and each contributes its own startup checks. RavenDB to SQL is the only pair built, and another pair, such as SQL to RavenDB, would add a source or a target without changing the engine. +- **The source reads the instance's own RavenDB settings**, so an existing customer sets nothing new. Leave them in place when switching `PersistenceType`. +- **Both persisters load into the same process**, each into its own `AssemblyLoadContext`. +- **The engine references neither assembly.** It knows only `IMigrationSource` and `IMigrationTarget`, and treats the resume cursor as an opaque value it passes from one to the other, so it can be tested against fakes on either side. + +## Startup sequence + +```mermaid +flowchart TB + A["Restart with MigrationMode = true"] --> B["Open the SQL target, exactly as today"] + B --> C["Open the old RavenDB, read only"] + C --> D{"All checks pass?"} + D -->|"No"| E["Host does not start.
Says which check failed.
Nothing has been copied."] + D -->|"Yes"| F["Copy what cannot be recreated.
Minutes. ServiceControl still closed."] + F --> G["ServiceControl opens.
New failed messages go straight to SQL."] + G --> H["Copy the selected history in the background,
throttled behind normal ingestion"] + H --> I["Verify row counts on both sides,
category by category"] + I --> J{"MigrationMode = false,
everything complete?"} + J -->|"No"| K["Host does not start.
Names what is outstanding.
An override exists."] + J -->|"Yes"| L["RavenDB is never opened again"] +``` + +**Checked before a single row moves:** + +- The SQL schema is current +- Message body storage is writable +- Both RavenDB databases are reachable +- The client certificate is valid, where the source is an external server +- The source is at a version this build can read +- The selected categories are valid +- `RetryHistoryDepth` is greater than zero. At zero or less, the first completed retry after the migration deletes the entire copied retry history, and no row count would ever show it + +## Data to be migrated (Categories) + +### Required + +- Unresolved **and retry-issued** failed messages, with their bodies. Attempt history collapses to the newest attempt, because the SQL model has no attempts table. Retry-issued messages are required for the same reason unresolved ones are: issuing a retry deletes the expiry, so they never age out. Leaving one behind means the retry confirmation arrives with no row to mark resolved, and the message stays missing from the customer's list while the retry actually succeeded +- Message redirects +- Endpoint settings +- Known endpoints, including the monitored flag. One category, because the flag is a property of the endpoint row and cannot be copied without it +- Notification settings +- The licence trial end date +- Throughput history +- Retry operations, unacknowledged and historic. One category, because RavenDB holds both lists in a single document +- Licensing report masks +- The uploaded licensed endpoint details file, which nothing recomputes: skipping it means the customer re-downloads it from the licence portal and uploads it again +- Subscriptions + +### Optional + +- Archived and resolved failed messages: the biggest category by far, and most of the copying time +- The event log, without which the ServicePulse activity feed starts empty +- Custom checks, which cost almost nothing to skip because every check re-reports on its next interval +- Failed error imports, the record of errors that could not be ingested +- Group comments, **copied last of everything**, after archived and resolved messages. A comment survives only once the failed messages its group is built from have arrived, so on a large archive the comments are the last thing to appear. An empty comment field partway through a migration is the copy still running, not data loss +- Failed message edits + +### Not migrated + +- The RavenDB index definitions +- The transient in-flight collections, which are empty when nothing is running: `RetryBatches`, `RetryBatchNowForwardings`, `FailedMessageRetries`, `ArchiveOperations` and `UnarchiveOperations` +- `ArchiveBatches` and `UnarchiveBatches`, which exist only because of how RavenDB works +- `ConnectedApplications`, which only versions 6.0 and 6.1 wrote and nothing has read since +- Integration events still waiting to be sent when you switch over are never sent +- Broker and audit service version details, which refill on the throughput collector's next run + +## What does not come across + +**Whole categories are never copied.** Which ones, and why nothing needs them, is the [not migrated](#not-migrated) list above. Anything in an optional category you did not select is also never copied, and nothing later goes back for it. + +**Rows skipped one at a time, and counted.** Each of these shows up in the skipped count for its category, broken out by reason, so you can see how much went and why: + +- A failed message whose `UniqueMessageId` is not a GUID. The target column is a `uniqueidentifier` and the value is never regenerated, because it is simultaneously the primary key, the ServicePulse URL, the retry correlation key and the body lookup key. +- A failed message with no processing attempts recorded against it. The SQL model keeps the newest attempt and derives the failure time, the failing endpoint and the exception from it, all of which are required columns, so a message with nothing to derive them from cannot be written at all rather than being written blank. +- A failed message whose body cannot be read after three attempts. **The whole message is skipped, not just its body**, because a message with no body is worse than no message. +- A subscription whose message type or transport address exceeds 200 characters. The target key columns are capped at 200 characters, so it cannot be stored at all. +- An archived or resolved failed message, or an event log item, already past its retention period. SQL's retention clean-up would delete it on its first pass, so it is counted rather than copied only to be deleted. +- A group comment whose failure group has no failed messages in SQL once the messages are copied. SQL's clean-up removes such a comment, where RavenDB never expired one. +- Endpoint settings for an endpoint ServiceControl does not know. ServiceControl removes those settings shortly after it starts. +- A row missing a value SQL requires, such as a known endpoint with no name or host, or a failed message with no failing endpoint address. An empty group comment is left behind the same way, because ServiceControl never stores one. + +**Things that change shape, and are not counted as skips at all.** The dry run counts these before anything moves, so they are a number you see in advance rather than a discovery afterwards. They are also the ones to read twice: + +- **Processing attempt history collapses to the newest attempt.** The SQL model has no attempts table. This affects every failed message that failed more than once, in the one category every customer copies. A message that failed five times arrives showing one attempt, and the other four are gone. +- **Subscriptions that differ only in message-type version merge onto one row**, because the target key carries the type name without the version. +- **Endpoint settings for two endpoint names that differ only in case merge onto one row on SQL Server**, because SQL Server's default collation compares names without case, so one of the two settings is kept. PostgreSQL keeps both, and so does a SQL Server database created with a case-sensitive collation. The dry run counts this one too, by asking SQL Server how the name column compares, though for unusual characters its count can differ from what the copy does. +- **Event log items and historic retry operations are renumbered.** Their keys are database identities and nothing references them, so this is safe, but the old numbers do not survive. + +**Rows RavenDB deletes while the copy is running are an absence, not a skip.** Expiration only deletes a document carrying `@expires`, and only two kinds ever get one: a resolved or archived failed message, and an event log item (`ExpirationManager.cs:34,41`). Everything in the [required](#required) set is therefore safe, since unresolved and retry-issued messages have their expiry removed when the retry is issued, so only the archived and resolved messages category and the event log category can shrink underneath the copier, and both copy in the background where the window is longest. A document the sweep removes before the stream reaches it is never read, so it is counted nowhere: the counts are of rows the source actually handed over, and there is no expected total to fall short of. It is the same population as the retention skip above, and which of the two it becomes is a race with the sweep. The consequence to know is that the dry run's count is a snapshot rather than a promise, and for those two categories the difference between it and the final copied count is not attributed to anything. + +**A category can finish with a small amount of loss and still count as complete.** A few skipped rows in a large table leave the category in a *complete with errors* state, which blocks nothing. Its skipped count is printed and the ids of the skipped rows are written to the log, so while the RavenDB database still exists you can go and look at exactly what did not make it. + +## The one point you can go back + +While ServiceControl is closed and the required copy is running, nothing except the copier has written to SQL, and the migration has written nothing to RavenDB, which is still authoritative. RavenDB's own expiration still runs, though: unless you disabled it, it keeps deleting expired failed messages and event log items, as [Goals](#goals) describes. If you need your instance back, set `MigrationMode=false`, point `PersistenceType` back at RavenDB, and start. You lose the copy, not your data, and you can start again later. + +That window closes the moment ServiceControl opens. From then on new failed messages are ingesting into SQL, RavenDB is no longer current, and there is no rollback: nothing copies SQL rows back. The choice at that point is to finish the migration or to accept losing whatever has not been copied. + +## Reading from RavenDB + +- A third RavenDB lifecycle opens the source: connect, check the version, stop. It never calls `DatabaseSetup.Execute`. +- Both source databases must be on the same server or cluster (`LicensingDataStore.cs:35`). +- The source has to be at a ServiceControl version this build can read, and nothing in RavenDB records one today. The only version check that exists compares the RavenDB server version to the RavenDB client version, and runs only for an external source. So a marker is stamped into the database on upgrade, and a source without one, or one from a newer major version, is refused by name rather than misread. +- Duration scales with distance to the source. The copier already holds the document from the stream, so each body costs **one** round trip rather than two, but it is one per message and they are not batched. Egress out of RavenDB Cloud is billed to the customer. See [batching and throttling](#batching-and-throttling). + +## Writing to SQL + +- A whole `FailedMessage` is written with its stored status intact. No existing caller does that, though the dialect upsert already accepts a status, so the gap is smaller than it looks. +- `UniqueMessageId` keeps its value, but converts type: the source holds a string and the target column is a `uniqueidentifier`. It is the primary key, the ServicePulse URL, the retry correlation key and the body lookup key at once. +- `StatusChangedAt` is reconstructed from `@expires` for resolved and archived messages, which is the only place RavenDB sets it. Unresolved and retry-issued messages have no `@expires`, so the copier uses the newest processing attempt's timestamp. The column is `NOT NULL`, so it cannot be left empty, but the value is harmless for those two: the retention sweep only considers resolved and archived rows, so an unresolved message never ages out whatever is written here. +- Message bodies go through `IBodyStoragePersistence`, which owns the compression threshold and the choice of filesystem, Azure Blob or S3. The separate 102,400-byte inline threshold is not there: it lives on the ingestion path, so the copier has to apply it rather than inherit it. +- Throughput rows are written directly rather than through the collector, and the write sets each day's count rather than adding to it. Throughput is a required category, so it copies while ServiceControl is closed, before any collector has written to SQL. Setting is what makes the category safe to resume after a crash, where adding would double-count. Copying the rows is also what stops the audit and broker collectors re-gathering the same days when the host opens, because `LastCollectedDate` is derived from the newest throughput row rather than stored (`LicensingDataStore.cs:45`). The checkpoint is what stops a second pass overwriting days the collectors have written since. +- Identifiers narrow on the way across, and the dry run counts every kind. What narrows, merges or cannot be stored at all is in [what does not come across](#what-does-not-come-across). + +## Batching and throttling + +- Batch size comes from the provider: SQL Server divides its own parameter budget by the column count, PostgreSQL uses a flat 50 rows. +- The throttle is a configurable pause between batches, defaulting to 100 ms. Lowering it, or turning `MigrationMode` off, is the only remedy for a copy competing with production. + +## Checkpointing and resume + +A copy that runs for hours will be interrupted at some point: a restart, a dropped connection, a machine reboot. The checkpoint is what makes an interruption cost only the batch that was in flight. It is one row per category, kept on the target and created by `--setup` along with the rest of the schema, and it is written in the same database transaction as the rows it describes. Only the copier writes to it; the status and verify commands read it. + +**What one row holds:** the category it tracks, its state, the resume cursor, how many rows were copied, skipped and already present, a count per skip reason, how many rows the source held when the category started, when it started, when it last made progress, when it settled, the last error, and a version number used to spot a second writer. + +**The states, and which ones a restart re-enters.** `Complete`, `CompleteWithErrors` and `Abandoned` are terminal, so a restart passes straight over the category. `Halted` and `Blocked` are not: a halt is resumed from its cursor once the cause is fixed, and a block clears itself once the category it waits on settles, which is how group comments end up behind archived messages. `NotStarted` and `InProgress` both mean there is work to do. + +### One batch, and why nothing provisional is ever saved + +```mermaid +sequenceDiagram + participant E as Migration engine + participant S as RavenDB source + participant T as SQL target + participant C as Checkpoint row + + E->>C: Read this category's row + C-->>E: State, cursor, totals so far + loop One batch at a time + E->>S: Read the next batch after the cursor + S-->>E: Rows, and the cursor they end at + opt The category carries message bodies + E->>S: Read each body, up to three attempts + S-->>E: The bodies, and which ones could not be read + end + E->>T: Write the rows, with the totals so far,
the unreadable bodies and the new cursor + Note over T,C: One transaction. The rows, the target's own skips,
the new totals and the cursor all commit, or none of them do + T-->>E: The checkpoint exactly as it committed + E->>E: Halt if too much of this run was skipped + end + E->>C: Settle as complete, complete with errors, or halted +``` + +The thing to read twice is that the counts never travel back through the engine to be saved on some later write. The engine hands the target the totals so far, the target adds its own outcome to them and saves the result beside the rows, and the engine then keeps whatever committed. So there is no window in which the stored row claims rows that are not there, and a crash at any instant leaves counts and cursor that both describe exactly the rows in SQL. + +**What that buys, and why each part is needed:** + +- **Progress never gets ahead of the data.** The rows and the cursor commit together, so a restart cannot skip past rows that were never written. +- **A crash costs the batch in flight and nothing else.** The next run reads from the committed cursor. +- **Re-reading a batch cannot double-count it.** Unreadable bodies stay off the checkpoint until the write commits, so a batch that is read twice is counted once, and rows the earlier attempt did write come back as *already present* rather than as fresh copies. +- **Every skipped row has a reason, or the save is refused.** The checkpoint rejects a batch reporting more skips than it explains, because verification has to account for each one rather than report a healthy migration as broken. +- **Each category resumes independently**, so a half-copied category picks up where it stopped while its neighbours are untouched. +- **The halt counters are per run and deliberately not stored.** If the skips that tripped a halt stayed on the row, a restart with the cause fixed would re-trip it on its first batch. +- **A second writer is caught rather than merged.** Each save carries the version it read, and a save against a row that has moved on is refused, so two hosts pointed at one target cannot quietly interleave their progress. +- **If a message is already in SQL the SQL row wins and the copier skips it**, which is what makes every category safe to run twice. + +## Error handling + +- Which rows are skipped, and why, is in [what does not come across](#what-does-not-come-across). What follows is the mechanics around those rules. +- A body is read up to three times before the message is skipped whole, and the exhausted attempts count toward the halt threshold. +- Deciding whether a row is past the target's retention cutoff needs two retention periods: the source's reverses `@expires` back into the status-change instant, and the target's current one decides whether that instant is past the cutoff. +- A bad row does not stop the copy. Its category finishes in a separate complete-with-errors state. +- The halt threshold is proportional with an absolute floor, and a category halts only when both are exceeded. Proportional alone halts a three-row category on one bad row; absolute alone halts a five-million-row table on its 101st failure at the default floor of 100. Together, a large category keeps going through losses under the percentage and finishes complete with errors, so ten thousand skipped rows out of five million do not halt it. +- Rows left behind because SQL would remove them anyway (past retention, orphaned group comments, settings for unknown endpoints) are counted and reported, but never halt a category. The target reports them apart from its real failures, so they land in the skipped count and the log without moving the category toward a halt. +- The percentage is measured against what the run has processed so far rather than against the category's total, so a run that starts badly looks worse than it is. The floor is what keeps that harmless, since fewer than 101 skipped rows never consults the percentage at all. More than that, bunched at the start, does halt a category whose overall rate would have been fine, and the cost is one restart: the skipped rows commit with the cursor, so the next run resumes past them with its counters back at zero. +- A source therefore must not read a category in an order that puts the rows most likely to be skipped at the front of it. +- Verification therefore cannot treat any count difference as a fault. It accounts for every skip rule, or it reports every successful migration as broken. + +### A halt stops one category, and clearing it is a restart + +A halt is the copy refusing to keep going on one category because something is wrong beyond the odd bad row. It is not a crash and not data loss: everything already copied is committed, the cursor points at the row after the last one that committed, and the reason is written on the category. Nothing is retried in the background and nothing waits for a timer. The category sits halted until a person does something about it. + +```mermaid +stateDiagram-v2 + [*] --> NotStarted: nothing has run yet + NotStarted --> InProgress: the host starts with MigrationMode = true + NotStarted --> Blocked: the category it must follow has not settled + Blocked --> InProgress: that category settles, then the next restart + InProgress --> InProgress: the host was stopped mid-copy,
so the next start resumes from the cursor + InProgress --> Complete: every row reached, none skipped + InProgress --> CompleteWithErrors: every row reached, some skipped + InProgress --> Halted: too many rows skipped in this run,
or the copy hit an error it did not expect + Halted --> InProgress: fix the cause, restart,
carry on from the cursor + Halted --> Abandoned: accept the loss, deliberately + InProgress --> Abandoned: accept the loss, deliberately + Complete --> [*] + CompleteWithErrors --> [*] + Abandoned --> [*] +``` + +**Two things halt a category.** Either the skipped rows in this run pass both the percentage and the floor, which says the failures are systematic rather than incidental, or the copy hits an error it did not expect, in which case the error type and the cursor it stopped at are recorded. A host being shut down is neither: it leaves the category in progress, to be picked up from the cursor next time. Nor is a second host writing to the same checkpoint, which is refused so that the other host's progress stands. + +**A halt stops that category and nothing else.** The remaining categories still run, with one exception: a category that must follow the halted one goes to blocked rather than running early, which is how group comments stay behind the archived messages they belong to. A blocked category is not a failure and needs no separate action, since clearing the halt clears the block on the next restart. + +**What it costs depends on which category halted.** A halted optional category means the instance keeps serving traffic and that one slice of history is missing until it is resumed. A halted required category means the host stays closed, so the outage carries on until the halt is cleared or the category is abandoned. That is deliberate: opening the host is the point of no return, and it should not happen with required data left behind by accident. + +**Clearing it:** + +1. Read the reason on the category, in the custom check or the status command. It names the count that tripped the threshold, or the error, and the cursor either way. +2. Fix the cause. It is usually outside the migration: the body store unreachable, a certificate expired, the source or the target down, or the disk full. +3. Restart the host with `MigrationMode=true`. The category picks up at its cursor, its run counters start again at zero, and the skips already recorded stay on the row so the totals still add up at the end. +4. Repeat only if it halts again. A restart that halts at the same point is telling you the cause is still there, and a restart that gets further has made real progress, because the rows it skipped are committed and will not be read again. + +**Or abandon it, on purpose.** Abandoning marks the category as deliberately given up rather than failed, which lets the host open and lets the migration end. It is the right answer when the data is not worth the outage, and the wrong one if it was picked by accident, because nothing goes back for an abandoned category afterwards. What it leaves behind is stated in the counts rather than guessed at. + +## Dry run + +Runnable before anything starts, and again later against whatever is still outstanding. It never writes to RavenDB. + +What it resolves and reports: + +- Whether the source is embedded or external, and which server +- Which two RavenDB databases, and the setting each name came from +- What it found in each of them +- Rows per category, and message-body volume per category +- A duration for the window while ServiceControl is closed, as a range + +It runs the same startup checks that gate startup, so a missing setting surfaces before a customer books an outage. + +It counts, before anything moves, the rows that cannot cross as they stand: + +- Documents whose `UniqueMessageId` will not parse as a GUID +- Subscriptions that differ only in message-type version, and so merge onto one row +- Subscriptions whose message type or transport address exceeds the 200-character key limit +- Integration event dispatches still queued, which are not copied and will never be sent + +It reports no duration for the optional categories, and nothing about load on the source. + +### When you can run the read-only commands + +`--migration-source-report`, `--migration-verify` and `--migration-dry-run` all open the RavenDB source. **On an embedded source that means stopping the ServiceControl service first**, because a second RavenDB process cannot attach to a data directory the first one holds. Plan the dry run as part of the outage rather than as something you run the day before while the instance keeps serving traffic. On an external source, a container or RavenDB Cloud, all three run against a live instance with no interruption. + +A containerised instance runs all three as a one-off `docker run` of the same image with the command's flag, against an external RavenDB server, as the [instructions](ravendb-to-sql-migration-instructions.md#report-on-the-source) show for the source report. It cannot use an embedded source, because the image does not ship the RavenDB server. + +`--migration-status` is the exception and is deliberately so: it reads only the checkpoint table in SQL and never opens the source, so it works on every source shape at any time, including during the background copy. It is the command to use for watching progress. + +## Configuration and control + +- A customer sets `MigrationMode` and the list of categories next to it. A status command and a custom check report back. +- Categories are read fresh at every startup. Adding one copies it on the next restart, removing one deletes nothing. +- There is no HTTP API, no pause, no resume, no abort, and no way to add a category to a running instance. All of those mean editing configuration and restarting. +- The checkpoint table is a record of what happened, not a control channel. +- Stopping a copy takes a restart, so it cannot be stopped in ten seconds. + +## Out of scope + +- The audit instance, which has no EF Core persister at all, so a customer who finishes this migration is still running RavenDB for audit. This is stated up front under [Problem](#problem), because it changes whether the migration is worth doing at all +- The monitoring instance, which keeps its data in memory, so there is nothing to move diff --git a/src/ServiceControl.Migration.Tests/.editorconfig b/src/ServiceControl.Migration.Tests/.editorconfig new file mode 100644 index 0000000000..ca5ad8bd2e --- /dev/null +++ b/src/ServiceControl.Migration.Tests/.editorconfig @@ -0,0 +1,5 @@ +[*.cs] + +# Justification: Test project +dotnet_diagnostic.CA2007.severity = none +dotnet_diagnostic.PS0018.severity = none diff --git a/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs b/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs new file mode 100644 index 0000000000..737a26bd4f --- /dev/null +++ b/src/ServiceControl.Migration.Tests/MigrationSourceReportCommandTests.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.IO; +using System.Threading.Tasks; +using NUnit.Framework; +using Particular.ServiceControl.Hosting; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Hosting.Commands; + +[TestFixture] +[NonParallelizable] +class MigrationSourceReportCommandTests +{ + [SetUp] + public void SetUp() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", MigrationSourceServer.ServerUrl); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_DATABASENAME", MigrationSourceServer.PrimaryDatabase); + Environment.SetEnvironmentVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", MigrationSourceServer.ThroughputDatabase); + } + + [TearDown] + public void TearDown() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_RAVENDB_DATABASENAME", null); + Environment.SetEnvironmentVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", null); + } + + [Test] + public async Task The_report_names_the_databases_the_settings_and_the_collections() + { + var report = await RunReport(); + + Assert.Multiple(() => + { + Assert.That(report, Does.Match(@"Mode\s+: external")); + Assert.That(report, Does.Contain(MigrationSourceServer.ServerUrl)); + Assert.That(report, Does.Contain("from ServiceControl/RavenDB/DatabaseName")); + Assert.That(report, Does.Contain("from LicensingComponent/RavenDB/ThroughputDatabaseName")); + Assert.That(report, Does.Match(@"EndpointSettings\s+1"), "The report has to render a count, not just name the collection."); + }); + } + + static async Task RunReport() + { + var settings = new Settings(persisterType: "RavenDB", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)); + + var writer = new StringWriter(); + var original = Console.Out; + Console.SetOut(writer); + + try + { + await new MigrationSourceReportCommand().Execute(new HostArguments([]), settings); + } + finally + { + Console.SetOut(original); + } + + return writer.ToString(); + } +} diff --git a/src/ServiceControl.Migration.Tests/MigrationSourceServer.cs b/src/ServiceControl.Migration.Tests/MigrationSourceServer.cs new file mode 100644 index 0000000000..9b6e933c03 --- /dev/null +++ b/src/ServiceControl.Migration.Tests/MigrationSourceServer.cs @@ -0,0 +1,94 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; +using Raven.Client.Documents; +using Raven.Client.Documents.Conventions; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations; +using ServiceControl.Persistence; +using ServiceControl.RavenDB; +using TestHelper; + +/// +/// One embedded RavenDB for the whole assembly, holding a seeded database that stands in for a +/// customer's old instance. EmbeddedServer.Instance is process wide, so there can only be one. +/// +static class MigrationSourceServer +{ + public static string ServerUrl { get; private set; } + + public static string PrimaryDatabase { get; private set; } + + public static string ThroughputDatabase { get; private set; } + + public static async Task Start(CancellationToken cancellationToken = default) + { + var dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "MigrationSource", "Data"); + var logPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "MigrationSource", "Logs"); + var port = PortUtility.GetAssignedOrAvailablePort(33377); + + ServerUrl = $"http://localhost:{port}"; + PrimaryDatabase = "primary"; + ThroughputDatabase = "throughput"; + + var configuration = new EmbeddedDatabaseConfiguration(ServerUrl, PrimaryDatabase, dbPath, logPath, "Operations") { RunInMemory = true }; + database = EmbeddedDatabase.Start(configuration, lifetime); + + using var store = await database.Connect(cancellationToken); + + await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(PrimaryDatabase)), cancellationToken); + await store.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(ThroughputDatabase)), cancellationToken); + + using var seeder = new DocumentStore + { + Urls = [ServerUrl], + Database = PrimaryDatabase, + Conventions = new DocumentConventions { SaveEnumsAsIntegers = true } + }.Initialize(); + + using var session = seeder.OpenAsyncSession(); + await session.StoreAsync(new EndpointSettings { Name = "Sales", TrackInstances = true }, "EndpointSettings/1", cancellationToken); + await session.SaveChangesAsync(cancellationToken); + } + + public static async Task Stop() + { + if (database is null) + { + return; + } + + await database.Stop(CancellationToken.None); + database.Dispose(); + database = null; + lifetime.StopApplication(); + } + + static EmbeddedDatabase database; + static readonly TestLifetime lifetime = new(); + + sealed class TestLifetime : IHostApplicationLifetime + { + public CancellationToken ApplicationStarted => CancellationToken.None; + public CancellationToken ApplicationStopping => stopping.Token; + public CancellationToken ApplicationStopped => CancellationToken.None; + public void StopApplication() => stopping.Cancel(); + + readonly CancellationTokenSource stopping = new(); + } +} + +[SetUpFixture] +public class MigrationSourceServerFixture +{ + [OneTimeSetUp] + public Task StartSource() => MigrationSourceServer.Start(); + + [OneTimeTearDown] + public Task StopSource() => MigrationSourceServer.Stop(); +} diff --git a/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj b/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj new file mode 100644 index 0000000000..da4cda3938 --- /dev/null +++ b/src/ServiceControl.Migration.Tests/ServiceControl.Migration.Tests.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + Migration + + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs new file mode 100644 index 0000000000..ba677f8b42 --- /dev/null +++ b/src/ServiceControl.Migration.Tests/TwoPersistersInOneProcessTests.cs @@ -0,0 +1,120 @@ +namespace ServiceControl.Migration.Tests; + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.Loader; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using NUnit.Framework; +using ServiceBus.Management.Infrastructure.Settings; +using ServiceControl.Infrastructure; +using ServiceControl.Persistence; + +[TestFixture] +[NonParallelizable] +class TwoPersistersInOneProcessTests +{ + Settings settings; + string bodyStoragePath; + + [SetUp] + public void SetUp() + { + bodyStoragePath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Bodies", Guid.NewGuid().ToString("n")); + + // Both persistence configurations read ErrorRetentionPeriod through the settings reader rather + // than off the Settings object, so the constructor argument below does not satisfy either of them. + SetVariable("SERVICECONTROL_ERRORRETENTIONPERIOD", "10.00:00:00"); + // Registering the SQL Server persistence never connects, so any connection string satisfies it. + SetVariable("SERVICECONTROL_DATABASE_CONNECTIONSTRING", + Environment.GetEnvironmentVariable("ServiceControl_Persistence_SqlServer_ConnectionString") + ?? "Server=localhost;Database=ServiceControl;Trusted_Connection=True;TrustServerCertificate=True"); + SetVariable("SERVICECONTROL_MESSAGEBODY_STORAGETYPE", "FileSystem"); + SetVariable("SERVICECONTROL_MESSAGEBODY_FILESYSTEM_STORAGEPATH", bodyStoragePath); + SetVariable("SERVICECONTROL_RAVENDB_CONNECTIONSTRING", MigrationSourceServer.ServerUrl); + SetVariable("SERVICECONTROL_RAVENDB_DATABASENAME", MigrationSourceServer.PrimaryDatabase); + SetVariable("LICENSINGCOMPONENT_RAVENDB_THROUGHPUTDATABASENAME", MigrationSourceServer.ThroughputDatabase); + + settings = new Settings(persisterType: "SQLServer", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)); + } + + [TearDown] + public void TearDown() + { + foreach (var name in variables) + { + Environment.SetEnvironmentVariable(name, null); + } + + variables.Clear(); + } + + [Test] + public async Task A_source_and_a_target_load_side_by_side_and_share_one_type_identity() + { + Assert.That(settings.AssemblyLoadContextResolver(typeof(Settings).Assembly.Location), Is.InstanceOf(), + "This test is meaningless if the resolver is pinned to AssemblyLoadContext.Default the way every acceptance test pins it."); + + var services = new ServiceCollection(); + services.AddPersistence(settings); + var targetSettings = settings.PersisterSpecificSettings; + + await using var source = await PersistenceFactory.OpenMigrationSource(settings); + + var sourceContext = AssemblyLoadContext.GetLoadContext(source.GetType().Assembly); + var targetContext = AssemblyLoadContext.GetLoadContext(settings.PersisterSpecificSettings.GetType().Assembly); + + Assert.Multiple(() => + { + Assert.That(sourceContext, Is.Not.SameAs(AssemblyLoadContext.Default), + "The RavenDB persister must load into its own plugin context, or this proves nothing."); + Assert.That(targetContext, Is.Not.SameAs(AssemblyLoadContext.Default), + "The target persister must be isolated too, otherwise only one half of the pairing is being tested."); + Assert.That(sourceContext, Is.Not.SameAs(targetContext), + "Source and target must land in separate contexts. That separation is the whole feature, and nothing else here asserts it."); + Assert.That(settings.PersisterSpecificSettings, Is.SameAs(targetSettings), + "Opening a source must leave the target's settings object exactly where it was."); + }); + + var description = await source.Describe(); + + Assert.That(description.Facts.Single(fact => fact.Label == "Primary database").Value, Is.EqualTo(MigrationSourceServer.PrimaryDatabase), + "The source read its own RavenDB settings rather than the target's."); + } + + [Test] + public void Asking_a_SQL_persister_for_a_source_names_the_setting_to_change() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_SOURCEPERSISTENCETYPE", "SQLServer"); + + try + { + var refusal = Assert.ThrowsAsync(async () => await PersistenceFactory.OpenMigrationSource(new Settings(persisterType: "SQLServer", forwardErrorMessages: false, errorRetentionPeriod: TimeSpan.FromDays(10)))); + + Assert.That(refusal.Message, Does.Contain("cannot be read as a migration source").And.Contain("ServiceControl/Migration/SourcePersistenceType")); + } + finally + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_SOURCEPERSISTENCETYPE", null); + } + } + + [Test] + public void Asking_a_SQL_persister_for_maintenance_mode_is_still_refused() + { + var refusal = Assert.Throws(() => PersistenceFactory.Create(settings, maintenanceMode: true)); + + Assert.That(refusal.Message, Does.Contain("Maintenance mode is not supported").And.Contain("SQLServer"), + "PersistenceFactory's SupportsMaintenanceMode guard is what turns a persister mismatch into a sentence instead of a cast exception. A restructure that drops it compiles and passes every other test."); + } + + void SetVariable(string name, string value) + { + Environment.SetEnvironmentVariable(name, value); + variables.Add(name); + } + + readonly List variables = []; +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs new file mode 100644 index 0000000000..cc3c59a998 --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenMigrationSource.cs @@ -0,0 +1,64 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents.Operations; +using Raven.Client.ServerWide.Operations; +using ServiceControl.Persistence.DataMigration; + +sealed class RavenMigrationSource(RavenReadOnlySourceLifecycle lifecycle) : IMigrationSource +{ + public Task Open(CancellationToken cancellationToken = default) => lifecycle.Open(cancellationToken); + + public async Task Describe(CancellationToken cancellationToken = default) + { + var settings = lifecycle.Settings; + var build = await lifecycle.DocumentStore.Maintenance.Server.SendAsync(new GetBuildNumberOperation(), cancellationToken); + + var server = settings.UseEmbeddedServer + ? new MigrationSourceFact("Server", settings.ServerUrl, $"{lifecycle.SettingsRoot}/{RavenBootstrapper.DatabaseMaintenancePortKey}") + : new MigrationSourceFact("Server", settings.ConnectionString, $"{lifecycle.SettingsRoot}/{RavenBootstrapper.ConnectionStringKey}"); + + return new MigrationSourceDescription(build.ProductVersion, + [ + new MigrationSourceFact("Mode", settings.UseEmbeddedServer ? "embedded" : "external"), + server, + new MigrationSourceFact("Primary database", settings.DatabaseName, $"{lifecycle.SettingsRoot}/{RavenBootstrapper.DatabaseNameKey}"), + new MigrationSourceFact("Throughput database", settings.ThroughputDatabaseName, $"{ThroughputSettings.SettingsNamespace}/{ThroughputSettings.DatabaseNameKey}") + ]); + } + + public async Task> Inventory(CancellationToken cancellationToken = default) + { + var entries = new List(); + + foreach (var databaseName in new[] { lifecycle.Settings.DatabaseName, lifecycle.Settings.ThroughputDatabaseName }) + { + var statistics = await lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetCollectionStatisticsOperation(), cancellationToken); + entries.AddRange(statistics.Collections.Select(collection => new MigrationSourceInventoryEntry(databaseName, collection.Key, collection.Value))); + } + + return entries; + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => + throw new NotSupportedException($"The RavenDB migration source cannot count category {category.Id} yet"); + + public IAsyncEnumerable Read( + MigrationCategory category, + string? resumeAfter, + int batchSize, + CancellationToken cancellationToken = default) => + throw new NotSupportedException($"The RavenDB migration source cannot read category {category.Id} yet"); + + public Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default) => + throw new NotSupportedException($"The RavenDB migration source cannot read bodies for category {category.Id} yet"); + + public ValueTask DisposeAsync() => lifecycle.DisposeAsync(); +} diff --git a/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs new file mode 100644 index 0000000000..354587cb5e --- /dev/null +++ b/src/ServiceControl.Persistence.RavenDB/DataMigration/RavenReadOnlySourceLifecycle.cs @@ -0,0 +1,189 @@ +#nullable enable + +namespace ServiceControl.Persistence.RavenDB.DataMigration; + +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Particular.LicensingComponent.Contracts; +using Raven.Client.Documents; +using Raven.Client.Documents.Conventions; +using Raven.Client.Documents.Operations; +using Raven.Client.Documents.Session; +using Raven.Client.Exceptions.Database; +using Raven.Client.Exceptions.Security; +using ServiceControl.Configuration; +using ServiceControl.RavenDB; + +sealed class RavenReadOnlySourceLifecycle(RavenPersisterSettings settings, SettingsRootNamespace settingsRoot) : IAsyncDisposable +{ + public RavenPersisterSettings Settings => settings; + + public SettingsRootNamespace SettingsRoot => settingsRoot; + + public IDocumentStore DocumentStore => documentStore ?? throw new InvalidOperationException($"The migration source is not open. Call {nameof(Open)} first."); + + public async Task Open(CancellationToken cancellationToken = default) + { + if (documentStore is not null) + { + throw new InvalidOperationException("The migration source is already open. Opening it twice would abandon the first server without stopping it."); + } + + try + { + var serverUrl = settings.UseEmbeddedServer ? StartEmbedded() : settings.ConnectionString; + documentStore = Connect(serverUrl); + + if (!settings.UseEmbeddedServer) + { + // Only an external server can be older than the client; an embedded one ships beside it. + await StartupChecks.EnsureServerVersion(documentStore, cancellationToken); + } + + await EnsureReadable(settings.DatabaseName, $"{settingsRoot}/{RavenBootstrapper.DatabaseNameKey}", cancellationToken); + await EnsureReadable(settings.ThroughputDatabaseName, $"{ThroughputSettings.SettingsNamespace}/{ThroughputSettings.DatabaseNameKey}", cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + await DisposeAsync(); + throw; + } + catch (Exception) + { + await DisposeAsync(); + throw; + } + } + + public IAsyncDocumentSession OpenSession(string databaseName) => + DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName, NoTracking = true }); + + async Task EnsureReadable(string databaseName, string settingKey, CancellationToken cancellationToken) + { + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + await DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation(), cancellationToken); + return; + } + catch (DatabaseLoadTimeoutException) when (settings.UseEmbeddedServer) + { + // A large embedded database routinely exceeds the load timeout on first open, which + // RavenEmbeddedPersistenceLifecycle already allows for the same way. + await Task.Delay(EmbeddedLoadRetryDelay, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (DatabaseDoesNotExistException e) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} has no database named '{databaseName}'. That name comes from the '{settingKey}' setting. Correct it before migrating: a wrong name reads a database that is not there rather than the one that is.", e); + } + catch (AuthorizationException e) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} refused its client certificate access to the database '{databaseName}'. Grant that certificate Read access to '{databaseName}', or supply one that has it in '{settingsRoot}/{RavenBootstrapper.ClientCertificateBase64Key}' or '{settingsRoot}/{RavenBootstrapper.ClientCertificatePathKey}'. If '{databaseName}' is the wrong name, correct the '{settingKey}' setting instead: RavenDB refuses a certificate that has no access to a database whether or not that database exists.", e); + } + catch (Exception e) when (e is not DatabaseLoadTimeoutException) + { + throw new InvalidOperationException($"The RavenDB migration source at {Located()} has a database named '{databaseName}', from the '{settingKey}' setting, but could not load it.", e); + } + } + } + + string Located() => Located(settings, settingsRoot); + + internal static string Located(RavenPersisterSettings sourceSettings, SettingsRootNamespace root) => sourceSettings.UseEmbeddedServer + ? $"{sourceSettings.ServerUrl} (embedded, data directory '{sourceSettings.DatabasePath}', from '{root}/{RavenBootstrapper.DatabasePathKey}')" + : sourceSettings.ConnectionString; + + string StartEmbedded() + { + var configuration = new EmbeddedDatabaseConfiguration(settings.ServerUrl, settings.DatabaseName, settings.DatabasePath, settings.LogPath, settings.LogsMode); + + embedded = EmbeddedDatabase.Start(configuration, lifetime); + + return embedded.ServerUrl; + } + + IDocumentStore Connect(string serverUrl) + { + var store = new DocumentStore + { + Database = settings.DatabaseName, + Urls = [serverUrl], + Conventions = new DocumentConventions { SaveEnumsAsIntegers = true } + }; + + if (!settings.UseEmbeddedServer) + { + store.Certificate = RavenClientCertificate.FindClientCertificate(settings); + } + + store.OnBeforeRequest += RefuseWrite; + + return store.Initialize(); + } + + static void RefuseWrite(object? sender, BeforeRequestEventArgs e) + { + if (IsRead(e.Request.Method, new Uri(e.Url).AbsolutePath)) + { + return; + } + + throw new InvalidOperationException($"The RavenDB migration source is open read-only and refused a {e.Request.Method} to '{e.Url}'. Nothing in a migration may write to the database it is reading."); + } + + static bool IsRead(HttpMethod method, string path) + { + if (method == HttpMethod.Get || method == HttpMethod.Head) + { + // HiLo persists the id range it hands out, so it writes despite being a GET. + return !path.Contains(HiLoPathSegment, StringComparison.OrdinalIgnoreCase); + } + + return method == HttpMethod.Post && Array.Exists(ReadOnlyPostPaths, suffix => path.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)); + } + + public async ValueTask DisposeAsync() + { + documentStore?.Dispose(); + documentStore = null; + + if (embedded is not null) + { + // Stop force-kills only once this token cancels; EmbeddedDatabase sets the graceful wait to an hour. + using var shutdown = new CancellationTokenSource(EmbeddedShutdownTimeout); + await embedded.Stop(shutdown.Token); + embedded.Dispose(); + embedded = null; + } + } + + const string HiLoPathSegment = "/hilo/"; + static readonly string[] ReadOnlyPostPaths = ["/queries", "/multi_get", "/streams/queries"]; + static readonly TimeSpan EmbeddedShutdownTimeout = TimeSpan.FromSeconds(30); + static readonly TimeSpan EmbeddedLoadRetryDelay = TimeSpan.FromMilliseconds(500); + + IDocumentStore? documentStore; + EmbeddedDatabase? embedded; + readonly SourceLifetime lifetime = new(); + + sealed class SourceLifetime : IHostApplicationLifetime + { + public CancellationToken ApplicationStarted => CancellationToken.None; + public CancellationToken ApplicationStopping => CancellationToken.None; + public CancellationToken ApplicationStopped => CancellationToken.None; + + public void StopApplication() + { + } + } +} diff --git a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs index 44b20a23a2..c0a2dd3cea 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceConfiguration.cs @@ -5,11 +5,13 @@ namespace ServiceControl.Persistence.RavenDB using System.Reflection; using Configuration; using CustomChecks; + using DataMigration; using Microsoft.Extensions.Logging; using Particular.LicensingComponent.Contracts; using ServiceControl.Infrastructure; + using ServiceControl.Persistence.DataMigration; - class RavenPersistenceConfiguration : PersistenceConfiguration, IPersistenceConfiguration + class RavenPersistenceConfiguration : PersistenceConfiguration, IPersistenceConfiguration, IMigrationSourceFactory { public const string DataSpaceRemainingThresholdKey = "DataSpaceRemainingThreshold"; const string AuditRetentionPeriodKey = "AuditRetentionPeriod"; @@ -90,5 +92,8 @@ public IPersistence Create(PersistenceSettings settings) var specificSettings = (RavenPersisterSettings)settings; return new RavenPersistence(specificSettings); } + + public IMigrationSource CreateSource(SettingsRootNamespace settingsRoot) => + new RavenMigrationSource(new RavenReadOnlySourceLifecycle((RavenPersisterSettings)CreateSettings(settingsRoot), settingsRoot)); } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs new file mode 100644 index 0000000000..859e02c505 --- /dev/null +++ b/src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/ReadOnlySourceLifecycleTests.cs @@ -0,0 +1,366 @@ +namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration; + +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; +using Raven.Client.Documents; +using Raven.Client.Documents.Conventions; +using Raven.Client.Documents.Operations; +using Raven.Client.Documents.Operations.Expiration; +using Raven.Client.Documents.Session; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations; +using ServiceControl.Configuration; +using ServiceControl.MessageFailures; +using ServiceControl.Operations.BodyStorage.RavenAttachments; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.Persistence.RavenDB; +using ServiceControl.Persistence.RavenDB.DataMigration; +using ServiceControl.Persistence.Tests; + +[TestFixture] +class ReadOnlySourceLifecycleTests +{ + static readonly SettingsRootNamespace SettingsRoot = new("ServiceControl"); + + string databaseName; + IDocumentStore bootstrapStore; + RavenPersisterSettings sourceSettings; + + [SetUp] + public async Task SetUp() + { + var embeddedServer = await SharedEmbeddedServer.GetInstance(); + databaseName = Guid.NewGuid().ToString("n"); + + bootstrapStore = new DocumentStore + { + Urls = [embeddedServer.ServerUrl], + Database = databaseName, + Conventions = new DocumentConventions { SaveEnumsAsIntegers = true } + }.Initialize(); + + await bootstrapStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord(databaseName))); + await bootstrapStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(new DatabaseRecord($"{databaseName}-throughput"))); + + using (var session = bootstrapStore.OpenAsyncSession()) + { + await session.StoreAsync(new FailedMessage { UniqueMessageId = "abc", Status = FailedMessageStatus.Archived }, "FailedMessages/abc"); + await session.SaveChangesAsync(); + } + + sourceSettings = new RavenPersisterSettings + { + DatabaseName = databaseName, + ThroughputDatabaseName = $"{databaseName}-throughput", + ConnectionString = embeddedServer.ServerUrl, + ErrorRetentionPeriod = TimeSpan.FromDays(10) + }; + } + + [TearDown] + public void TearDown() => bootstrapStore?.Dispose(); + + [Test] + public async Task Opening_the_source_creates_no_index() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + var statistics = await bootstrapStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation()); + + Assert.That(statistics.CountOfIndexes, Is.Zero, "Opening a migration source must not create the fifteen ServiceControl indexes on it."); + } + [Test] + public async Task Opening_the_source_creates_no_database() + { + var absentThroughput = $"{databaseName}-absent"; + sourceSettings.ThroughputDatabaseName = absentThroughput; + + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + + var exception = Assert.ThrowsAsync(async () => await lifecycle.Open()); + + Assert.That(exception.Message, Does.Contain(absentThroughput).And.Contain("LicensingComponent/RavenDB/ThroughputDatabaseName")); + Assert.That(await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(absentThroughput)), Is.Null, "Opening a migration source must not create a database that was missing."); + } + + [Test] + public async Task Opening_the_source_writes_no_database_settings() + { + var before = (await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName))).Settings; + + await using (var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)) + { + await lifecycle.Open(); + } + + var after = (await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName))).Settings; + + Assert.That(after, Is.EquivalentTo(before), "Opening a migration source must not rewrite the settings of the database it reads."); + } + + [Test] + public async Task Opening_the_source_configures_no_expiration() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + var record = await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName)); + + Assert.That(record.Expiration, Is.Null, "Opening a migration source must not enable RavenDB document expiry against it, or the customer's fallback data is deleted while they are migrating."); + } + + [Test] + public async Task Writing_through_the_source_store_throws() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); + await session.StoreAsync(new FailedMessage { UniqueMessageId = "def", Status = FailedMessageStatus.Unresolved }, "FailedMessages/def"); + + var exception = Assert.ThrowsAsync(async () => await session.SaveChangesAsync()); + + Assert.That(exception.Message, Does.Contain("open read-only")); + await AssertAbsent("FailedMessages/def"); + } + + [Test] + public async Task A_source_session_cannot_even_stage_a_write() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.OpenSession(databaseName); + + var exception = Assert.ThrowsAsync(async () => + await session.StoreAsync(new FailedMessage { UniqueMessageId = "def", Status = FailedMessageStatus.Unresolved }, "FailedMessages/def")); + + Assert.That(exception.Message, Does.Contain("tracking is disabled"), "OpenSession is NoTracking, so a write through it fails at Store rather than reaching the RefuseWrite request hook. Both guards have to hold: this one is the only one a copier's own sessions ever meet."); + } + + [Test] + public async Task Failed_message_status_reads_back_as_stored() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.OpenSession(databaseName); + var loaded = await session.LoadAsync("FailedMessages/abc"); + + Assert.That(loaded.Status, Is.EqualTo(FailedMessageStatus.Archived), "Without SaveEnumsAsIntegers every message status is misread, and it looks like data corruption rather than a missing convention."); + } + + [Test] + public async Task A_failed_open_leaves_no_store_behind() + { + sourceSettings.ThroughputDatabaseName = $"{databaseName}-absent"; + + var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + + Assert.ThrowsAsync(async () => await lifecycle.Open()); + + var afterFailure = Assert.Throws(() => _ = lifecycle.DocumentStore); + + Assert.That(afterFailure.Message, Does.Contain("is not open"), "A failed Open must dispose what it created. The caller never received a source, so nothing else can, and on the embedded path what leaks is a live RavenDB server process holding the customer's data directory."); + + await lifecycle.DisposeAsync(); + } + [Test] + public async Task A_source_describes_itself_as_facts_naming_the_setting_each_came_from() + { + await using var source = new RavenMigrationSource(new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)); + await source.Open(); + var description = await source.Describe(); + var facts = description.Facts.ToDictionary(fact => fact.Label); + + Assert.Multiple(() => + { + Assert.That(description.Version, Does.StartWith("6.")); + Assert.That(facts["Mode"].Value, Is.EqualTo("external")); + Assert.That(facts["Primary database"].Value, Is.EqualTo(databaseName)); + Assert.That(facts["Primary database"].SettingKey, Is.EqualTo("ServiceControl/RavenDB/DatabaseName")); + Assert.That(facts["Throughput database"].Value, Is.EqualTo($"{databaseName}-throughput")); + }); + } + + [Test] + public async Task An_unopened_source_refuses_to_describe_itself() + { + await using var source = new RavenMigrationSource(new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)); + + var exception = Assert.ThrowsAsync(async () => await source.Describe()); + + Assert.That(exception.Message, Does.Contain("is not open"), "Separating construction from Open is what lets a host register a source in its container, and it buys a state where nothing is connected yet. That state has to fail loudly rather than return an empty description."); + } + + [Test] + public async Task The_source_inventories_every_collection_by_database() + { + await using var source = new RavenMigrationSource(new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot)); + await source.Open(); + var inventory = await source.Inventory(); + + Assert.That(inventory.Single(entry => entry.Scope == databaseName && entry.Name == "FailedMessages").Count, Is.EqualTo(1)); + } + + [Test] + public void An_embedded_source_names_the_data_directory_setting() + { + sourceSettings.ConnectionString = null; + + var location = RavenReadOnlySourceLifecycle.Located(sourceSettings, SettingsRoot); + + Assert.That(location, Does.Contain("'ServiceControl/DbPath'"), "An operator told the embedded source cannot be read needs the setting that points at its data directory."); + } + + [Test] + public async Task Opening_the_source_twice_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + var exception = Assert.ThrowsAsync(async () => await lifecycle.Open()); + + Assert.That(exception.Message, Does.Contain("already open"), "A second Open would abandon the first store, and on the embedded path a running server process with it."); + } + + [Test] + public async Task Generating_an_id_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); + + Assert.CatchAsync(async () => + await session.StoreAsync(new FailedMessage { UniqueMessageId = "hilo", Status = FailedMessageStatus.Unresolved })); + + Assert.That(await CountDocuments(), Is.EqualTo(1), "HiLo reserves an id range with a request of its own before SaveChanges, so only the request hook can refuse it."); + } + + [Test] + public async Task A_patch_against_the_source_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + Assert.CatchAsync(async () => + await lifecycle.DocumentStore.Operations.ForDatabase(databaseName).SendAsync( + new PatchOperation("FailedMessages/abc", null, new PatchRequest { Script = "this.Status = 1;" }))); + + Assert.That(await LoadStatus("FailedMessages/abc"), Is.EqualTo(FailedMessageStatus.Archived), "A patch is a request of its own with no session, so only the request hook can refuse it; the RavenDB persister writes this way."); + } + + [Test] + public async Task A_bulk_insert_into_the_source_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + Assert.CatchAsync(async () => + { + await using var bulk = lifecycle.DocumentStore.BulkInsert(databaseName); + await bulk.StoreAsync(new FailedMessage { UniqueMessageId = "bulk", Status = FailedMessageStatus.Unresolved }, "FailedMessages/bulk"); + }); + + await AssertAbsent("FailedMessages/bulk"); + } + + [Test] + public async Task Reconfiguring_expiry_on_the_source_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + Assert.CatchAsync(async () => + await lifecycle.DocumentStore.Maintenance.ForDatabase(databaseName).SendAsync( + new ConfigureExpirationOperation(new ExpirationConfiguration { Disabled = false, DeleteFrequencyInSec = 60 }))); + + var record = await bootstrapStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName)); + + Assert.That(record.Expiration, Is.Null, "Enabling expiry on the source would start deleting the customer fallback data."); + } + + [Test] + public async Task Deleting_an_untracked_document_is_refused() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.DocumentStore.OpenAsyncSession(new SessionOptions { Database = databaseName }); + session.Delete("FailedMessages/abc"); + + Assert.CatchAsync(async () => await session.SaveChangesAsync()); + + Assert.That(await CountDocuments(), Is.EqualTo(1), "A delete by id of an untracked document only goes out at SaveChanges, as a batch request the hook must refuse."); + } + + [Test] + public async Task Reading_the_source_still_works() + { + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var session = lifecycle.OpenSession(databaseName); + + var loaded = await session.LoadAsync("FailedMessages/abc"); + var queried = await session.Query().ToListAsync(); + + Assert.Multiple(() => + { + Assert.That(loaded, Is.Not.Null, "A guard that fails closed still has to let every read the migration needs through."); + Assert.That(queried, Has.Count.EqualTo(1)); + }); + } + + [Test] + public async Task A_source_session_reads_a_message_body_attachment() + { + using (var session = bootstrapStore.OpenAsyncSession()) + { + using var stored = new MemoryStream(Encoding.UTF8.GetBytes("1")); + session.Advanced.Attachments.Store("FailedMessages/abc", RavenAttachmentsBodyStorage.AttachmentName, stored, "text/xml"); + await session.SaveChangesAsync(); + } + + await using var lifecycle = new RavenReadOnlySourceLifecycle(sourceSettings, SettingsRoot); + await lifecycle.Open(); + + using var sourceSession = lifecycle.OpenSession(databaseName); + using var attachment = await sourceSession.Advanced.Attachments.GetAsync("FailedMessages/abc", RavenAttachmentsBodyStorage.AttachmentName); + using var read = new MemoryStream(); + await attachment.Stream.CopyToAsync(read); + + Assert.Multiple(() => + { + Assert.That(Encoding.UTF8.GetString(read.ToArray()), Is.EqualTo("1"), "The body copy reads every attachment through this no-tracking session, so a read that needed tracking would need a second kind of source session."); + Assert.That(attachment.Details.ContentType, Is.EqualTo("text/xml")); + }); + } + + async Task AssertAbsent(string documentId) + { + using var session = bootstrapStore.OpenAsyncSession(); + var found = await session.LoadAsync(documentId); + + Assert.That(found, Is.Null, $"The source refused the write, so '{documentId}' must not be in the database. Asserting the exception alone tests the guard, not the guarantee."); + } + + async Task CountDocuments() + { + var statistics = await bootstrapStore.Maintenance.ForDatabase(databaseName).SendAsync(new GetStatisticsOperation()); + return statistics.CountOfDocuments; + } + + async Task LoadStatus(string documentId) + { + using var session = bootstrapStore.OpenAsyncSession(); + var found = await session.LoadAsync(documentId); + return found.Status; + } +} diff --git a/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs new file mode 100644 index 0000000000..92d6a73ce7 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/HaltThreshold.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Persistence.DataMigration; + +public static class HaltThreshold +{ + // Both must be exceeded: the floor ignores a few bad rows in a small category, the percentage a small share of a large one. + public static bool Exceeded(long skippedCount, long totalCount, int percentThreshold, int minimumFloor) + { + if (skippedCount <= minimumFloor || totalCount == 0) + { + return false; + } + + var percent = skippedCount * 100m / totalCount; + return percent > percentThreshold; + } +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs new file mode 100644 index 0000000000..bc71283d01 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationCheckpointStore.cs @@ -0,0 +1,80 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +public enum MigrationCategoryState +{ + NotStarted, + InProgress, + Complete, + CompleteWithErrors, + Halted, + Abandoned, + Blocked +} + +/// A category's saved progress: where a restart carries on from, and what the status and verify commands report. +public sealed record MigrationCheckpoint( + string CategoryId, + MigrationCategoryState State, + string? Cursor, + long CopiedCount, + long SkippedCount, + long? SourceTotal, + IReadOnlyDictionary? SkipReasons, + DateTime? StartedAt, + DateTime? LastProgressAt, + // The moment the category stopped running, whatever state it stopped in. Read it beside State: a halt settles too. + DateTime? SettledAt, + string? LastError, + long AlreadyPresentCount = 0, + // The optimistic concurrency token. A store sets it on save and refuses one carrying a value the stored row no longer holds. + long Version = 0) +{ + /// Adds one batch's outcome to this checkpoint. A target calls it inside the transaction that writes the rows, so the saved counts are the real ones. + public MigrationCheckpoint Extend(int copied, int skipped, int alreadyPresent, IReadOnlyDictionary? skipReasons) + { + var explained = skipReasons?.Values.Sum() ?? 0; + if (explained != skipped) + { + throw new InvalidOperationException($"The target reported {skipped} skipped rows in category {CategoryId} but gave reasons for {explained}. Every skipped row needs a reason, or --migration-verify cannot account for it."); + } + + return this with + { + CopiedCount = CopiedCount + copied, + SkippedCount = SkippedCount + skipped, + AlreadyPresentCount = AlreadyPresentCount + alreadyPresent, + SkipReasons = AddSkipReasons(SkipReasons, skipReasons) + }; + } + + internal static IReadOnlyDictionary? AddSkipReasons(IReadOnlyDictionary? totals, IReadOnlyDictionary? additions) + { + if (additions is not { Count: > 0 }) + { + return totals; + } + + Dictionary sum = totals is null ? [] : new(totals); + foreach (var (reason, count) in additions) + { + sum[reason] = sum.GetValueOrDefault(reason) + count; + } + + return sum; + } +} + +public interface IMigrationCheckpointStore +{ + Task> ReadAll(CancellationToken cancellationToken = default); + Task Read(string categoryId, CancellationToken cancellationToken = default); + + /// Saves the checkpoint and returns it as stored, carrying the version the save landed on. Throws when the stored row has moved on. + Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs new file mode 100644 index 0000000000..bd19e623aa --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationSource.cs @@ -0,0 +1,32 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +public interface IMigrationSource : IAsyncDisposable +{ + /// Connects to the source read-only. Every other member throws until this has run. + Task Open(CancellationToken cancellationToken = default); + + /// What the source report and dry run print about the source. + Task Describe(CancellationToken cancellationToken = default); + + /// A count of everything the source holds, including data no category copies. + Task> Inventory(CancellationToken cancellationToken = default); + + /// How many rows the source holds for one category, for progress and verify. + Task Count(MigrationCategory category, CancellationToken cancellationToken = default); + + /// Reads a category in batches, after the checkpoint cursor if provided, or from the start when it is null. Throws on a cursor it never issued. + /// A ceiling, not a target: returning fewer costs nothing, returning more fails the target's write. + IAsyncEnumerable Read( + MigrationCategory category, + string? resumeAfter, + int batchSize, + CancellationToken cancellationToken = default); + + /// Reads one row's message body when Read did not attach it. Returns null when the row has no body. + Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs new file mode 100644 index 0000000000..1f4de16640 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationSourceFactory.cs @@ -0,0 +1,10 @@ +namespace ServiceControl.Persistence.DataMigration; + +using ServiceControl.Configuration; + +/// Implemented by a persister that can be read as the old database a migration copies from. +public interface IMigrationSourceFactory +{ + /// Builds the source from the instance's own settings, without connecting to it. + IMigrationSource CreateSource(SettingsRootNamespace settingsRoot); +} diff --git a/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs new file mode 100644 index 0000000000..78026fc571 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/IMigrationTarget.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +/// Implemented by a persister that can be the new database a migration copies into. +public interface IMigrationTarget +{ + /// The most rows a source may return in one batch for this category. The target picks it because its own database sets the limit, and a batch over it fails the write. + int BatchSizeFor(MigrationCategory category); + + /// Saves the batch's rows and the checkpoint in one transaction, so progress never gets ahead of the data. Extend checkpointToExtend with this batch's own outcome through and save the result, so what lands is the real split rather than a guess the next save has to correct. + /// Prior totals, the cursor this batch reached, and any rows the engine itself skipped. Not yet counting anything the target does. + Task Write( + MigrationCategory category, + MigrationBatch batch, + MigrationCheckpoint checkpointToExtend, + CancellationToken cancellationToken = default); + + /// How many rows the target holds for one category, for progress and verify. Counts only that category, even where two categories share a table. + Task Count(MigrationCategory category, CancellationToken cancellationToken = default); +} + +/// What the target did with one batch, and the checkpoint it committed alongside the rows. Every skipped row must have a reason in SkipReasons. +/// How many of Skipped the target would have deleted anyway, such as a row already past retention. Counted and reported like any skip, but never counted toward the halt threshold. +public sealed record MigrationWriteResult(MigrationCheckpoint Saved, int Copied, int Skipped, IReadOnlyList SkippedIds, int AlreadyPresent = 0, IReadOnlyDictionary? SkipReasons = null, int BenignSkipped = 0); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs b/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs new file mode 100644 index 0000000000..431730ba82 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationBatch.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; + +public enum MigrationCategoryKind { Required, Optional } + +/// One kind of data, such as endpoint settings, copied as a unit and resumed from its own cursor. +public sealed record MigrationCategory( + string Id, + MigrationCategoryKind Kind, + bool CarriesBodies, + int Order, + string? MustFollow = null); + +/// A message body read from the source, as bytes plus its content type. +public sealed record MigrationBody(ReadOnlyMemory Content, string ContentType); + +/// One item read from the source. Body is null unless the source attached it or the engine fetched it. +public sealed record MigrationRow( + string SourceId, + object Document, + IReadOnlyDictionary Metadata, + MigrationBody? Body = null); + +/// Rows read from the source together, plus the cursor to resume after them. +public sealed record MigrationBatch(IReadOnlyList Rows, string Cursor); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs new file mode 100644 index 0000000000..6f15bcdf24 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryIds.cs @@ -0,0 +1,24 @@ +namespace ServiceControl.Persistence.DataMigration; + +public static class MigrationCategoryIds +{ + public const string KnownEndpoints = nameof(KnownEndpoints); + public const string EndpointSettings = nameof(EndpointSettings); + public const string MessageRedirects = nameof(MessageRedirects); + public const string Subscriptions = nameof(Subscriptions); + public const string NotificationSettings = nameof(NotificationSettings); + public const string TrialEndDate = nameof(TrialEndDate); + public const string RetryOperations = nameof(RetryOperations); + public const string LicensingEndpoints = nameof(LicensingEndpoints); + public const string LicensingThroughput = nameof(LicensingThroughput); + public const string LicensingReportMasks = nameof(LicensingReportMasks); + public const string LicensedEndpointDetails = nameof(LicensedEndpointDetails); + public const string UnresolvedAndRetryIssuedFailedMessages = nameof(UnresolvedAndRetryIssuedFailedMessages); + + public const string EventLog = nameof(EventLog); + public const string CustomChecks = nameof(CustomChecks); + public const string FailedErrorImports = nameof(FailedErrorImports); + public const string FailedMessageEdits = nameof(FailedMessageEdits); + public const string ArchivedAndResolvedFailedMessages = nameof(ArchivedAndResolvedFailedMessages); + public const string GroupComments = nameof(GroupComments); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs new file mode 100644 index 0000000000..69665e34e1 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCategoryRegistry.cs @@ -0,0 +1,38 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; +using System.Linq; +using ServiceControl.MessageFailures; + +public static class MigrationCategoryRegistry +{ + public static readonly IReadOnlyList UnresolvedAndRetryIssuedStatuses = [FailedMessageStatus.Unresolved, FailedMessageStatus.RetryIssued]; + public static readonly IReadOnlyList ArchivedAndResolvedStatuses = [FailedMessageStatus.Archived, FailedMessageStatus.Resolved]; + + public static readonly IReadOnlyList All = + [ + // Required, copied with ServiceControl closed. + new(MigrationCategoryIds.KnownEndpoints, MigrationCategoryKind.Required, CarriesBodies: false, Order: 1), + new(MigrationCategoryIds.EndpointSettings, MigrationCategoryKind.Required, CarriesBodies: false, Order: 2, MustFollow: MigrationCategoryIds.KnownEndpoints), + new(MigrationCategoryIds.MessageRedirects, MigrationCategoryKind.Required, CarriesBodies: false, Order: 3), + new(MigrationCategoryIds.Subscriptions, MigrationCategoryKind.Required, CarriesBodies: false, Order: 4), + new(MigrationCategoryIds.NotificationSettings, MigrationCategoryKind.Required, CarriesBodies: false, Order: 5), + new(MigrationCategoryIds.TrialEndDate, MigrationCategoryKind.Required, CarriesBodies: false, Order: 6), + new(MigrationCategoryIds.RetryOperations, MigrationCategoryKind.Required, CarriesBodies: false, Order: 7), + new(MigrationCategoryIds.LicensingEndpoints, MigrationCategoryKind.Required, CarriesBodies: false, Order: 8), + new(MigrationCategoryIds.LicensingThroughput, MigrationCategoryKind.Required, CarriesBodies: false, Order: 9, MustFollow: MigrationCategoryIds.LicensingEndpoints), + new(MigrationCategoryIds.LicensingReportMasks, MigrationCategoryKind.Required, CarriesBodies: false, Order: 10), + new(MigrationCategoryIds.LicensedEndpointDetails, MigrationCategoryKind.Required, CarriesBodies: false, Order: 11), + new(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages, MigrationCategoryKind.Required, CarriesBodies: true, Order: 12), + + // Optional, copied in the background once the host is open. + new(MigrationCategoryIds.EventLog, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 1), + new(MigrationCategoryIds.CustomChecks, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 2), + new(MigrationCategoryIds.FailedErrorImports, MigrationCategoryKind.Optional, CarriesBodies: true, Order: 3), + new(MigrationCategoryIds.FailedMessageEdits, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 4), + new(MigrationCategoryIds.ArchivedAndResolvedFailedMessages, MigrationCategoryKind.Optional, CarriesBodies: true, Order: 5), + new(MigrationCategoryIds.GroupComments, MigrationCategoryKind.Optional, CarriesBodies: false, Order: 6, MustFollow: MigrationCategoryIds.ArchivedAndResolvedFailedMessages), + ]; + + public static MigrationCategory? Find(string id) => All.FirstOrDefault(c => c.Id == id); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationCheckpointConflictException.cs b/src/ServiceControl.Persistence/DataMigration/MigrationCheckpointConflictException.cs new file mode 100644 index 0000000000..cc2822822f --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationCheckpointConflictException.cs @@ -0,0 +1,6 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; + +/// Thrown when a checkpoint save carries a version the stored row no longer holds, because another writer moved it on. +public sealed class MigrationCheckpointConflictException(string message, Exception? innerException = null) : Exception(message, innerException); diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs new file mode 100644 index 0000000000..3503321796 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngine.cs @@ -0,0 +1,240 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +public sealed class MigrationEngine( + IMigrationSource source, + IMigrationTarget target, + IMigrationCheckpointStore checkpointStore, + TimeProvider timeProvider, + MigrationEngineOptions options, + ILogger logger) +{ + public const int MaxBodyReadAttempts = 3; + + public IReadOnlyList SelectCategories(MigrationCategoryKind kind) => + MigrationCategoryRegistry.All + .Where(c => c.Kind == kind) + .Where(c => kind == MigrationCategoryKind.Required || options.SelectedOptionalCategoryIds.Contains(c.Id)) + .OrderBy(c => c.Order) + .ToArray(); + + // Runs in the order given without re-sorting: required and optional orders both start at 1, so + // sorting a mixed list would put an optional category in front of a required one. + public async Task> RunCategories( + IReadOnlyList categories, + CancellationToken cancellationToken = default) + { + var results = new List(categories.Count); + + foreach (var category in categories) + { + results.Add(await RunCategoryAsync(category, cancellationToken)); + } + + return results; + } + + public async Task RunCategoryAsync(MigrationCategory category, CancellationToken cancellationToken = default) + { + var checkpoint = await checkpointStore.Read(category.Id, cancellationToken) ?? NotStarted(category); + + // Halted is deliberately not one of them: a halt says "stopped, and here is why", and a restart after the cause is fixed has to be able to pick it up again. + if (checkpoint.State is MigrationCategoryState.Complete or MigrationCategoryState.CompleteWithErrors or MigrationCategoryState.Abandoned) + { + return checkpoint; + } + + if (category.MustFollow is { } mustFollowId) + { + var predecessor = await checkpointStore.Read(mustFollowId, cancellationToken); + if (predecessor is not { State: MigrationCategoryState.Complete or MigrationCategoryState.CompleteWithErrors or MigrationCategoryState.Abandoned }) + { + var predecessorState = predecessor?.State.ToString() ?? "not started"; + var blocked = checkpoint with + { + State = MigrationCategoryState.Blocked, + LastError = $"Blocked: {category.Id} must follow {mustFollowId}, which is {predecessorState}" + }; + logger.LogWarning("Category {CategoryId} did not run: it must follow {PredecessorId}, which is {PredecessorState}", + category.Id, mustFollowId, predecessorState); + return await checkpointStore.Upsert(blocked, cancellationToken); + } + } + + if (checkpoint.State is MigrationCategoryState.NotStarted or MigrationCategoryState.Halted or MigrationCategoryState.Blocked) + { + checkpoint = await checkpointStore.Upsert(checkpoint with + { + State = MigrationCategoryState.InProgress, + StartedAt = checkpoint.StartedAt ?? timeProvider.GetUtcNow().UtcDateTime, + SettledAt = null, + LastError = null + }, cancellationToken); + } + + var isFirstBatch = true; + // Per run, not the persisted totals: the skips that tripped a halt stay on the row, so + // counting them again would re-halt a restart whose cause has been fixed. + var skippedThisRun = 0L; + var processedThisRun = 0L; + + try + { + var batchSize = target.BatchSizeFor(category); + await foreach (var batch in source.Read(category, checkpoint.Cursor, batchSize, cancellationToken).WithCancellation(cancellationToken)) + { + if (!isFirstBatch && category.Kind == MigrationCategoryKind.Optional) + { + await Pause(options.ThrottlePause, cancellationToken); + } + isFirstBatch = false; + + var batchToWrite = batch; + // Stays off checkpoint until the write commits: the catch persists checkpoint, and a restart + // re-reads an uncommitted batch and would count these skips again. + var bodySkips = 0; + if (category.CarriesBodies) + { + var (withBodies, failed) = await FetchBodiesWithRetry(category, batch, cancellationToken); + batchToWrite = withBodies; + bodySkips = failed.Count; + + foreach (var (id, lastAttemptError) in failed) + { + logger.LogWarning(lastAttemptError, "Skipped {SourceId} in category {CategoryId}: body unreadable after {MaxAttempts} attempts", id, category.Id, MaxBodyReadAttempts); + } + } + + // Prior totals, the new cursor, and the rows this engine already skipped. The target adds its own + // outcome inside the transaction that writes the rows, so nothing provisional is ever stored. + var checkpointToExtend = checkpoint with + { + Cursor = batch.Cursor, + SkippedCount = checkpoint.SkippedCount + bodySkips, + SkipReasons = MigrationCheckpoint.AddSkipReasons(checkpoint.SkipReasons, bodySkips == 0 ? null : new Dictionary { [MigrationSkipReason.BodyUnreadable] = bodySkips }) + }; + + var result = await target.Write(category, batchToWrite, checkpointToExtend, cancellationToken); + checkpoint = result.Saved; + + foreach (var id in result.SkippedIds) + { + logger.LogWarning("Skipped {SourceId} in category {CategoryId}", id, category.Id); + } + + // A negative fault count would silently disarm the halt threshold for the rest of the run. + if (result.BenignSkipped > result.Skipped) + { + throw new InvalidOperationException($"The target reported {result.BenignSkipped} benign skips in category {category.Id} out of {result.Skipped} skipped rows. Benign skips are a subset of the skipped rows."); + } + + processedThisRun += bodySkips + result.Copied + result.Skipped + result.AlreadyPresent; + // Rows the target would have deleted anyway are not faults, so they never halt a category. + skippedThisRun += bodySkips + result.Skipped - result.BenignSkipped; + + if (HaltThreshold.Exceeded(skippedThisRun, processedThisRun, options.HaltThresholdPercent, options.HaltThresholdMinimum)) + { + var reason = $"Halted: {skippedThisRun} of {processedThisRun} rows skipped in this run exceeds the configured threshold of {options.HaltThresholdPercent}% and {options.HaltThresholdMinimum} rows. Fix the cause and restart to resume from the cursor, or abandon the category to accept the loss."; + logger.LogError("Category {CategoryId} halted at cursor {Cursor}: {LastError}", category.Id, checkpoint.Cursor, reason); + return await Settle(checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }, cancellationToken); + } + } + } + // A shutdown is not a halt, and there is nothing to reconcile: the last committed batch stored its + // real split with its own rows, so the row on disk is already correct and resumable. + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + // Another writer holds this row, which no amount of halting resolves. Leave the state alone so their row stands. + catch (MigrationCheckpointConflictException) + { + throw; + } + catch (Exception ex) + { + var position = checkpoint.Cursor is null ? "at the start" : $"at cursor {checkpoint.Cursor}"; + var reason = $"{ex.GetType().Name} {position}: {ex.Message}"; + logger.LogError(ex, "Category {CategoryId} halted at cursor {Cursor}", category.Id, checkpoint.Cursor); + return await Settle(checkpoint with { State = MigrationCategoryState.Halted, LastError = reason }, cancellationToken); + } + + return await Settle(checkpoint with { State = checkpoint.SkippedCount > 0 ? MigrationCategoryState.CompleteWithErrors : MigrationCategoryState.Complete }, cancellationToken); + } + + // Halts log before settling: the store shares the target's database, so a failed save would hide the cause. + Task Settle(MigrationCheckpoint settled, CancellationToken cancellationToken) => + checkpointStore.Upsert(settled with { SettledAt = timeProvider.GetUtcNow().UtcDateTime }, cancellationToken); + + async Task<(MigrationBatch Batch, IReadOnlyList<(string SourceId, Exception LastAttemptError)> Failed)> FetchBodiesWithRetry(MigrationCategory category, MigrationBatch batch, CancellationToken cancellationToken) + { + var survivors = new List(batch.Rows.Count); + var failed = new List<(string SourceId, Exception LastAttemptError)>(); + + foreach (var row in batch.Rows) + { + if (row.Body is not null) + { + survivors.Add(row); + continue; + } + + MigrationBody? body = null; + Exception? lastAttemptError = null; + var succeeded = false; + + for (var attempt = 1; attempt <= MaxBodyReadAttempts && !succeeded; attempt++) + { + try + { + body = await source.ReadBody(category, row.SourceId, cancellationToken); + succeeded = true; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // A shutdown is not a transient body failure. Retrying it to the attempt limit and then + // recording the message as permanently unreadable would lose a row to a restart. + throw; + } + catch (Exception ex) when (!IsDefect(ex)) + { + lastAttemptError = ex; + logger.LogWarning(ex, "Attempt {Attempt} to read the body for {SourceId} failed", attempt, row.SourceId); + if (attempt < MaxBodyReadAttempts) + { + await Pause(options.BodyRetryBackoff, cancellationToken); + } + } + } + + if (succeeded) + { + survivors.Add(row with { Body = body }); + } + else + { + failed.Add((row.SourceId, lastAttemptError!)); + } + } + + return (batch with { Rows = survivors }, failed); + } + + // These fail the same way on every attempt, so retrying would only turn a code defect into skipped messages. + static bool IsDefect(Exception exception) => + exception is NotSupportedException or NotImplementedException or InvalidOperationException or ArgumentException or NullReferenceException or InvalidCastException; + + // A configured pause of zero means "do not throttle", and a timer that is never going to be + // waited on is worse than no timer: against a fake clock nobody advances, it never completes. + Task Pause(TimeSpan duration, CancellationToken cancellationToken) => + duration <= TimeSpan.Zero ? Task.CompletedTask : Task.Delay(duration, timeProvider, cancellationToken); + + static MigrationCheckpoint NotStarted(MigrationCategory category) => + new(category.Id, MigrationCategoryState.NotStarted, null, 0, 0, null, null, null, null, null, null); +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs new file mode 100644 index 0000000000..8433836893 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationEngineOptions.cs @@ -0,0 +1,44 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System; +using System.Collections.Generic; +using System.Linq; +using ServiceControl.Configuration; + +/// The engine's tuning: the pause between optional batches, when a category halts, and which optional categories to copy. +public sealed record MigrationEngineOptions( + TimeSpan ThrottlePause, + int HaltThresholdPercent, + int HaltThresholdMinimum, + IReadOnlyCollection SelectedOptionalCategoryIds) +{ + public static readonly TimeSpan DefaultBodyRetryBackoff = TimeSpan.FromMilliseconds(200); + + /// How long to wait before trying again to read a message body that failed. Not read from settings. + public TimeSpan BodyRetryBackoff { get; init; } = DefaultBodyRetryBackoff; + + /// Reads the options from settings. Throws if the optional categories setting names one that doesn't exist or isn't optional. + public static MigrationEngineOptions FromSettings(SettingsRootNamespace settingsRootNamespace) + { + var throttleMilliseconds = SettingsReader.Read(settingsRootNamespace, MigrationSettings.ThrottlePauseMillisecondsKey, MigrationSettings.DefaultThrottlePauseMilliseconds); + var haltPercent = SettingsReader.Read(settingsRootNamespace, MigrationSettings.HaltThresholdPercentKey, MigrationSettings.DefaultHaltThresholdPercent); + var haltMinimum = SettingsReader.Read(settingsRootNamespace, MigrationSettings.HaltThresholdMinimumKey, MigrationSettings.DefaultHaltThresholdMinimum); + var optionalCategories = SettingsReader.Read(settingsRootNamespace, MigrationSettings.OptionalCategoriesKey, string.Empty); + + var selectedIds = optionalCategories + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToArray(); + + var unknown = selectedIds + .Where(id => MigrationCategoryRegistry.Find(id) is not { Kind: MigrationCategoryKind.Optional }) + .ToArray(); + + if (unknown.Length > 0) + { + throw new InvalidOperationException( + $"{MigrationSettings.OptionalCategoriesKey} names categories that do not exist or are not optional: {string.Join(", ", unknown)}"); + } + + return new MigrationEngineOptions(TimeSpan.FromMilliseconds(throttleMilliseconds), haltPercent, haltMinimum, selectedIds); + } +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs new file mode 100644 index 0000000000..2b4faf5c91 --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSettings.cs @@ -0,0 +1,19 @@ +namespace ServiceControl.Persistence.DataMigration; + +/// The migration's setting names, relative to the instance's settings root, and their defaults. explains the tuning ones. +public static class MigrationSettings +{ + public const string ThrottlePauseMillisecondsKey = "Migration/ThrottlePauseMilliseconds"; + public const string HaltThresholdPercentKey = "Migration/HaltThresholdPercent"; + public const string HaltThresholdMinimumKey = "Migration/HaltThresholdMinimum"; + /// A comma-separated list of the optional categories to copy, such as "EventLog, CustomChecks". + public const string OptionalCategoriesKey = "Migration/OptionalCategories"; + /// Which persister holds the old data being copied from. + public const string SourcePersistenceTypeKey = "Migration/SourcePersistenceType"; + + public const int DefaultThrottlePauseMilliseconds = 100; + public const int DefaultHaltThresholdPercent = 5; + public const int DefaultHaltThresholdMinimum = 100; + /// RavenDB is the only source supported today. + public const string DefaultSourcePersistenceType = "RavenDB"; +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs new file mode 100644 index 0000000000..4129eb939d --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSkipReason.cs @@ -0,0 +1,9 @@ +namespace ServiceControl.Persistence.DataMigration; + +public enum MigrationSkipReason +{ + BodyUnreadable, + PastRetention, + // Never written by a copier. A database a newer build wrote still reads rather than throwing where the host decides whether to start. + Unknown +} diff --git a/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs b/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs new file mode 100644 index 0000000000..3d5692a56d --- /dev/null +++ b/src/ServiceControl.Persistence/DataMigration/MigrationSourceDescription.cs @@ -0,0 +1,12 @@ +namespace ServiceControl.Persistence.DataMigration; + +using System.Collections.Generic; + +/// What the source report and dry run print about the source: its version and a list of facts. +public sealed record MigrationSourceDescription(string Version, IReadOnlyList Facts); + +/// One line the source report prints, such as which server or database the source read, with the setting to change if it is wrong. +public sealed record MigrationSourceFact(string Label, string Value, string? SettingKey = null); + +/// A count the source report prints, such as the rows in one collection of one database, so an operator sees how much there is to copy. +public sealed record MigrationSourceInventoryEntry(string Scope, string Name, long Count); diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt index abd4c98313..f73200052b 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.PlatformSampleSettings.approved.txt @@ -63,6 +63,7 @@ "VirtualDirectory": "", "HeartbeatGracePeriod": "00:00:40", "TransportType": "ServiceControl.Transports.Learning.LearningTransportCustomization, ServiceControl.Transports.Learning", + "MigrationSourcePersistenceType": "RavenDB", "ErrorLogQueue": "error.log", "ErrorQueue": "error", "ForwardErrorMessages": false, diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt new file mode 100644 index 0000000000..b77ea84b2b --- /dev/null +++ b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Every_category_runs_in_a_fixed_order.approved.txt @@ -0,0 +1,18 @@ +Required 1: KnownEndpoints +Required 2: EndpointSettings, after KnownEndpoints +Required 3: MessageRedirects +Required 4: Subscriptions +Required 5: NotificationSettings +Required 6: TrialEndDate +Required 7: RetryOperations +Required 8: LicensingEndpoints +Required 9: LicensingThroughput, after LicensingEndpoints +Required 10: LicensingReportMasks +Required 11: LicensedEndpointDetails +Required 12: UnresolvedAndRetryIssuedFailedMessages, with bodies +Optional 1: EventLog +Optional 2: CustomChecks +Optional 3: FailedErrorImports, with bodies +Optional 4: FailedMessageEdits +Optional 5: ArchivedAndResolvedFailedMessages, with bodies +Optional 6: GroupComments, after ArchivedAndResolvedFailedMessages \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt new file mode 100644 index 0000000000..6c96bee11f --- /dev/null +++ b/src/ServiceControl.UnitTests/ApprovalFiles/MigrationEngineCategorySelectionTests.Only_configured_optional_categories_are_selected.approved.txt @@ -0,0 +1,6 @@ +Configured: GroupComments, EventLog, ArchivedAndResolvedFailedMessages +Runs as: +Optional 1: EventLog +Optional 5: ArchivedAndResolvedFailedMessages, with bodies +Optional 6: GroupComments, after ArchivedAndResolvedFailedMessages +Never copied: CustomChecks, FailedErrorImports, FailedMessageEdits \ No newline at end of file diff --git a/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs b/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs new file mode 100644 index 0000000000..d3c8851ed0 --- /dev/null +++ b/src/ServiceControl.UnitTests/Hosting/MigrationSourceReportArgumentTests.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.UnitTests.Hosting; + +using NUnit.Framework; +using Particular.ServiceControl.Hosting; +using ServiceControl.Hosting.Commands; + +[TestFixture] +class MigrationSourceReportArgumentTests +{ + [Test] + public void The_flag_selects_the_report_command() => + Assert.That(new HostArguments(["--migration-source-report"]).Command, Is.EqualTo(typeof(MigrationSourceReportCommand))); + + [Test] + public void No_flag_still_selects_the_run_command() => + Assert.That(new HostArguments([]).Command, Is.EqualTo(typeof(RunCommand))); + + [Test] + public void The_setup_flag_is_undisturbed() => + Assert.That(new HostArguments(["--setup"]).Command, Is.EqualTo(typeof(SetupCommand))); +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/CapturingLogger.cs b/src/ServiceControl.UnitTests/Migration/Fakes/CapturingLogger.cs new file mode 100644 index 0000000000..24aee91e82 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/CapturingLogger.cs @@ -0,0 +1,19 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using ServiceControl.Persistence.DataMigration; + +public sealed class CapturingLogger : ILogger +{ + public List<(LogLevel Level, string Message, Exception? Exception)> Entries { get; } = []; + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => + Entries.Add((logLevel, formatter(state, exception), exception)); +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs new file mode 100644 index 0000000000..78edd726b6 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationCheckpointStore.cs @@ -0,0 +1,33 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +public sealed class InMemoryMigrationCheckpointStore : IMigrationCheckpointStore +{ + readonly ConcurrentDictionary checkpoints = new(); + + public Task> ReadAll(CancellationToken cancellationToken = default) => + Task.FromResult>(checkpoints.Values.ToArray()); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => + Task.FromResult(checkpoints.TryGetValue(categoryId, out var checkpoint) ? checkpoint : null); + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) + { + var storedVersion = checkpoints.TryGetValue(checkpoint.CategoryId, out var stored) ? stored.Version : 0; + if (storedVersion != checkpoint.Version) + { + throw new MigrationCheckpointConflictException($"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, but the stored row is at version {storedVersion}."); + } + + var saved = checkpoint with { Version = checkpoint.Version + 1 }; + checkpoints[checkpoint.CategoryId] = saved; + return Task.FromResult(saved); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs new file mode 100644 index 0000000000..66341cbf20 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSource.cs @@ -0,0 +1,92 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +public sealed class InMemoryMigrationSource : IMigrationSource +{ + readonly Dictionary> rowsByCategory = []; + readonly Dictionary bodyFailures = []; + readonly Dictionary bodyReadAttempts = []; + readonly Dictionary bodies = []; + + public MigrationSourceDescription Description { get; set; } = new("in-memory", [new MigrationSourceFact("Store", "in memory")]); + + public void Seed(string categoryId, params MigrationRow[] rows) => rowsByCategory[categoryId] = [.. rows]; + + public void SetBody(string sourceId, MigrationBody? body) => bodies[sourceId] = body; + + public void FailBodyReads(string sourceId, int times, Exception failure) => bodyFailures[sourceId] = (times, failure); + + /// Makes the body read for this row behave like a host shutting down: the token is cancelled and the read throws. + public (string SourceId, CancellationTokenSource Source)? StopOnBodyRead { get; set; } + + public int BodyReadAttempts(string sourceId) => bodyReadAttempts.GetValueOrDefault(sourceId); + + public Task Open(CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task Describe(CancellationToken cancellationToken = default) => Task.FromResult(Description); + + public Task> Inventory(CancellationToken cancellationToken = default) => + Task.FromResult>( + [.. rowsByCategory.Select(pair => new MigrationSourceInventoryEntry("memory", pair.Key, pair.Value.Count))]); + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => + Task.FromResult((long)(rowsByCategory.TryGetValue(category.Id, out var rows) ? rows.Count : 0)); + + public async IAsyncEnumerable Read( + MigrationCategory category, + string? resumeAfter, + int batchSize, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var rows = rowsByCategory.GetValueOrDefault(category.Id) ?? []; + var startIndex = 0; + + if (resumeAfter is not null) + { + var cursorIndex = rows.FindIndex(r => r.SourceId == resumeAfter); + if (cursorIndex < 0) + { + throw new InvalidOperationException($"Category {category.Id} was asked to resume after {resumeAfter}, a cursor this source never issued"); + } + startIndex = cursorIndex + 1; + } + + for (var i = startIndex; i < rows.Count; i += batchSize) + { + cancellationToken.ThrowIfCancellationRequested(); + var slice = rows.Skip(i).Take(batchSize).ToArray(); + yield return new MigrationBatch(slice, slice[^1].SourceId); + await Task.Yield(); + } + } + + public async Task ReadBody(MigrationCategory category, string sourceId, CancellationToken cancellationToken = default) + { + await Task.Yield(); + + var attempt = bodyReadAttempts[sourceId] = BodyReadAttempts(sourceId) + 1; + + if (StopOnBodyRead is { } stop && stop.SourceId == sourceId) + { + await stop.Source.CancelAsync(); + throw new OperationCanceledException(stop.Source.Token); + } + + if (bodyFailures.TryGetValue(sourceId, out var failures) && attempt <= failures.Times) + { + throw failures.Failure; + } + + return bodies.GetValueOrDefault(sourceId); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs new file mode 100644 index 0000000000..23492909d4 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationSourceTests.cs @@ -0,0 +1,82 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class InMemoryMigrationSourceTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task Streams_seeded_rows_in_batches_of_the_requested_size() + { + var source = new InMemoryMigrationSource(); + source.Seed("EndpointSettings", Row("a"), Row("b"), Row("c"), Row("d"), Row("e")); + + var batches = new List(); + await foreach (var batch in source.Read(MigrationCategoryRegistry.Find("EndpointSettings")!, resumeAfter: null, batchSize: 2)) + { + batches.Add(batch); + } + + using (Assert.EnterMultipleScope()) + { + Assert.That(batches, Has.Count.EqualTo(3)); + Assert.That(batches[0].Rows.Select(r => r.SourceId), Is.EqualTo(new[] { "a", "b" })); + Assert.That(batches[2].Rows.Select(r => r.SourceId), Is.EqualTo(new[] { "e" })); + } + } + + [Test] + public async Task ResumeAfter_skips_everything_up_to_and_including_that_id() + { + var source = new InMemoryMigrationSource(); + source.Seed("EndpointSettings", Row("a"), Row("b"), Row("c")); + + var batches = new List(); + await foreach (var batch in source.Read(MigrationCategoryRegistry.Find("EndpointSettings")!, resumeAfter: "a", batchSize: 10)) + { + batches.Add(batch); + } + + Assert.That(batches.Single().Rows.Select(r => r.SourceId), Is.EqualTo(new[] { "b", "c" })); + } + + [Test] + public void A_cursor_the_source_never_issued_throws_rather_than_starting_again() + { + var source = new InMemoryMigrationSource(); + source.Seed("EndpointSettings", Row("a"), Row("b")); + + Assert.ThrowsAsync(async () => + { + await foreach (var _ in source.Read(MigrationCategoryRegistry.Find("EndpointSettings")!, resumeAfter: "no-such-row", batchSize: 10)) + { + } + }); + } + + [Test] + public async Task ReadBody_fails_the_configured_number_of_times_then_returns_the_seeded_body() + { + var source = new InMemoryMigrationSource(); + var finalBody = new MigrationBody(new byte[] { 9 }, "text/plain"); + source.SetBody("msg-1", finalBody); + source.FailBodyReads("msg-1", times: 1, new TimeoutException("transient")); + + Assert.ThrowsAsync(() => source.ReadBody(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!, "msg-1")); + var secondAttempt = await source.ReadBody(MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!, "msg-1"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(secondAttempt, Is.EqualTo(finalBody)); + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(2)); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs new file mode 100644 index 0000000000..cdfaff9777 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTarget.cs @@ -0,0 +1,105 @@ +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using ServiceControl.Persistence.DataMigration; + +public sealed class InMemoryMigrationTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget +{ + readonly Dictionary> writtenKeysByCategory = []; + readonly Dictionary> writtenRowsByCategory = []; + readonly HashSet preExistingKeys = []; + readonly Dictionary rejectedKeys = []; + + public int DefaultBatchSize { get; set; } = 3; + public string NoBatchSizeFor { get; set; } + public int? FailOnCallNumber { get; set; } + + /// What FailOnCallNumber throws, when the default simulated failure is the wrong shape for the test. + public Exception FailWith { get; set; } + + /// Cancels the token on this call and then writes normally, so the stop surfaces from the source's next batch. + public (int CallNumber, CancellationTokenSource Source)? CancelOnCall { get; set; } + public (int CallNumber, CancellationTokenSource Source)? StopOnCall { get; set; } + int callCount; + + public void SeedExistingKey(string sourceId) => preExistingKeys.Add(sourceId); + + public void RejectKey(string sourceId, MigrationSkipReason reason, bool benign = false) => rejectedKeys[sourceId] = (reason, benign); + + public IReadOnlyList WrittenRows(string categoryId) => + writtenRowsByCategory.TryGetValue(categoryId, out var rows) ? rows : []; + + public int BatchSizeFor(MigrationCategory category) => + category.Id == NoBatchSizeFor ? throw new InvalidOperationException($"No batch size is mapped for category {category.Id}") : DefaultBatchSize; + + public async Task Write( + MigrationCategory category, + MigrationBatch batch, + MigrationCheckpoint checkpointToExtend, + CancellationToken cancellationToken = default) + { + callCount++; + + if (StopOnCall is { } stop && stop.CallNumber == callCount) + { + await stop.Source.CancelAsync(); + throw new OperationCanceledException(stop.Source.Token); + } + + if (FailOnCallNumber == callCount) + { + throw FailWith ?? new InvalidOperationException($"Simulated failure on write {callCount}"); + } + + if (CancelOnCall is { } cancel && cancel.CallNumber == callCount) + { + await cancel.Source.CancelAsync(); + } + + var keys = writtenKeysByCategory.TryGetValue(category.Id, out var existingKeys) ? existingKeys : writtenKeysByCategory[category.Id] = []; + var rows = writtenRowsByCategory.TryGetValue(category.Id, out var existingRows) ? existingRows : writtenRowsByCategory[category.Id] = []; + + var copied = 0; + var alreadyPresent = 0; + var benignSkipped = 0; + var skippedIds = new List(); + var skipReasons = new Dictionary(); + + foreach (var row in batch.Rows) + { + if (rejectedKeys.TryGetValue(row.SourceId, out var rejection)) + { + skippedIds.Add(row.SourceId); + skipReasons[rejection.Reason] = skipReasons.GetValueOrDefault(rejection.Reason) + 1; + if (rejection.Benign) + { + benignSkipped++; + } + continue; + } + + if (preExistingKeys.Contains(row.SourceId) || !keys.Add(row.SourceId)) + { + alreadyPresent++; + continue; + } + + rows.Add(row); + copied++; + } + + // Extended and saved in the same operation as the rows, as the real targets do, so what lands + // is this batch's real split rather than a provisional one the next save has to correct. + var saved = await checkpointStore.Upsert( + checkpointToExtend.Extend(copied, skippedIds.Count, alreadyPresent, skipReasons), + cancellationToken); + + return new MigrationWriteResult(saved, copied, skippedIds.Count, skippedIds, alreadyPresent, skipReasons, benignSkipped); + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => + Task.FromResult((long)(writtenRowsByCategory.TryGetValue(category.Id, out var rows) ? rows.Count : 0)); +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs new file mode 100644 index 0000000000..ae52e815dc --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/InMemoryMigrationTargetTests.cs @@ -0,0 +1,81 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System.Collections.Generic; +using System.Threading.Tasks; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class InMemoryMigrationTargetTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + static MigrationCheckpoint EmptyCheckpoint(string categoryId) => + new(categoryId, MigrationCategoryState.InProgress, Cursor: null, 0, 0, null, null, null, null, null, null); + + [Test] + public async Task Writes_new_rows_and_commits_the_extended_checkpoint_with_them() + { + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var category = MigrationCategoryRegistry.Find("EndpointSettings")!; + var batch = new MigrationBatch([Row("a"), Row("b")], Cursor: "b"); + // Prior totals and the new cursor. The target adds this batch's own outcome before it saves. + var checkpointToExtend = EmptyCheckpoint(category.Id) with { Cursor = "b" }; + + var result = await target.Write(category, batch, checkpointToExtend); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(2)); + Assert.That(result.Skipped, Is.Zero); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(2)); + Assert.That(await checkpointStore.Read(category.Id), Is.EqualTo(checkpointToExtend with { CopiedCount = 2, Version = 1 })); + Assert.That(result.Saved, Is.EqualTo(checkpointToExtend with { CopiedCount = 2, Version = 1 }), "the result carries the row as stored"); + } + } + + [Test] + public async Task The_committed_checkpoint_carries_the_real_split_rather_than_the_rows_handed_over() + { + // Three rows in, one copied: a checkpoint saying three would survive a crash as three. + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + target.SeedExistingKey("already-present"); + target.RejectKey("rejected", MigrationSkipReason.BodyUnreadable); + var batch = new MigrationBatch([Row("already-present"), Row("rejected"), Row("new-row")], Cursor: "new-row"); + var checkpointToExtend = EmptyCheckpoint(category.Id) with { Cursor = "new-row" }; + + var result = await target.Write(category, batch, checkpointToExtend); + + using (Assert.EnterMultipleScope()) + { + Assert.That(result.Copied, Is.EqualTo(1)); + Assert.That(result.Skipped, Is.EqualTo(1)); + Assert.That(result.SkippedIds, Is.EqualTo(new[] { "rejected" })); + Assert.That(result.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 })); + Assert.That(result.AlreadyPresent, Is.EqualTo(1)); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(1)); + var stored = (await checkpointStore.Read(category.Id))!; + Assert.That((stored.CopiedCount, stored.SkippedCount, stored.AlreadyPresentCount), Is.EqualTo((1L, 1L, 1L)), "copied, skipped, already present as committed"); + } + } + + [Test] + public async Task Count_answers_for_one_category_rather_than_one_table() + { + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var required = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var archive = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + await target.Write(required, new MigrationBatch([Row("a"), Row("b")], "b"), EmptyCheckpoint(required.Id)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await target.Count(required), Is.EqualTo(2)); + Assert.That(await target.Count(archive), Is.Zero); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/Fakes/TimerRecordingTimeProvider.cs b/src/ServiceControl.UnitTests/Migration/Fakes/TimerRecordingTimeProvider.cs new file mode 100644 index 0000000000..6178cbde97 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/Fakes/TimerRecordingTimeProvider.cs @@ -0,0 +1,35 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration.Fakes; + +using System; +using System.Collections.Generic; +using System.Threading; +using Microsoft.Extensions.Time.Testing; + +// Signals each timer the engine creates, so a test only advances the clock once a pause is waiting on it. +public sealed class TimerRecordingTimeProvider : TimeProvider +{ + readonly FakeTimeProvider clock = new(); + + public SemaphoreSlim TimerCreated { get; } = new(0); + + public List DueTimes { get; } = []; + + public void Advance(TimeSpan delta) => clock.Advance(delta); + + public override DateTimeOffset GetUtcNow() => clock.GetUtcNow(); + + public override long GetTimestamp() => clock.GetTimestamp(); + + public override long TimestampFrequency => clock.TimestampFrequency; + + public override TimeZoneInfo LocalTimeZone => clock.LocalTimeZone; + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = clock.CreateTimer(callback, state, dueTime, period); + DueTimes.Add(dueTime); + TimerCreated.Release(); + return timer; + } +} diff --git a/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs b/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs new file mode 100644 index 0000000000..fbe2bebe97 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/HaltThresholdTests.cs @@ -0,0 +1,78 @@ +namespace ServiceControl.UnitTests.Migration; + +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class HaltThresholdTests +{ + [Test] + public void Does_not_halt_a_three_row_category_with_one_bad_row() + { + // 1 of 3 is 33%, well past the 5% proportion, but 1 skip never passes the 100-row floor. + var exceeded = HaltThreshold.Exceeded(skippedCount: 1, totalCount: 3, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void Halts_a_large_category_with_a_systemic_failure() + { + // 600 of 10,000 is 6%, past both the proportion and the floor. + var exceeded = HaltThreshold.Exceeded(skippedCount: 600, totalCount: 10_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.True); + } + + [Test] + public void Does_not_halt_a_large_category_with_only_a_small_proportion_skipped() + { + // 10,000 of 5,000,000 is 0.2%: past the floor, nowhere near the proportion. + var exceeded = HaltThreshold.Exceeded(skippedCount: 10_000, totalCount: 5_000_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void Exactly_at_both_boundaries_does_not_halt_because_both_must_be_exceeded() + { + // Exactly the floor, which settles it before the proportion is ever worked out. "Exceed" means strictly past, not "at". + var exceeded = HaltThreshold.Exceeded(skippedCount: 100, totalCount: 2_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void Exactly_on_the_proportion_does_not_halt_once_the_floor_is_behind_it() + { + // 101 of 2,020 is exactly 5% with the floor already passed, so this is the only shape that + // reaches the proportion comparison and depends on it being strictly greater. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 2_020, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void A_hair_past_the_proportion_halts_once_the_floor_is_behind_it() + { + // 101 of 2,000 is 5.05%: the same skip count as above, one row's worth over the line. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 2_000, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.True); + } + + [Test] + public void A_skip_count_with_nothing_processed_never_halts_and_never_divides_by_zero() + { + // Past the floor with a zero total, which is the only input that reaches the division guard. + var exceeded = HaltThreshold.Exceeded(skippedCount: 101, totalCount: 0, percentThreshold: 5, minimumFloor: 100); + + Assert.That(exceeded, Is.False); + } + + [Test] + public void No_rows_processed_yet_never_halts() + { + Assert.That(HaltThreshold.Exceeded(skippedCount: 0, totalCount: 0, percentThreshold: 5, minimumFloor: 100), Is.False); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs new file mode 100644 index 0000000000..9b5e637ec3 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationCategoryRegistryTests.cs @@ -0,0 +1,110 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using ServiceControl.MessageFailures; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationCategoryRegistryTests +{ + [Test] + public void Contains_all_eighteen_categories_with_unique_ids() + { + using (Assert.EnterMultipleScope()) + { + Assert.That(MigrationCategoryRegistry.All, Has.Count.EqualTo(18)); + Assert.That(MigrationCategoryRegistry.All.Select(c => c.Id).Distinct().Count(), Is.EqualTo(18)); + } + } + + [Test] + public void Required_categories_are_ordered_exactly_as_the_contract_lists_them() + { + var requiredIds = MigrationCategoryRegistry.All + .Where(c => c.Kind == MigrationCategoryKind.Required) + .OrderBy(c => c.Order) + .Select(c => c.Id) + .ToArray(); + + Assert.That(requiredIds, Is.EqualTo(new[] + { + "KnownEndpoints", "EndpointSettings", "MessageRedirects", "Subscriptions", + "NotificationSettings", "TrialEndDate", "RetryOperations", + "LicensingEndpoints", "LicensingThroughput", "LicensingReportMasks", + "LicensedEndpointDetails", "UnresolvedAndRetryIssuedFailedMessages" + })); + } + + [Test] + public void Optional_categories_are_ordered_exactly_as_the_contract_lists_them() + { + var optionalIds = MigrationCategoryRegistry.All + .Where(c => c.Kind == MigrationCategoryKind.Optional) + .OrderBy(c => c.Order) + .Select(c => c.Id) + .ToArray(); + + Assert.That(optionalIds, Is.EqualTo(new[] + { + "EventLog", "CustomChecks", "FailedErrorImports", + "FailedMessageEdits", "ArchivedAndResolvedFailedMessages", "GroupComments" + })); + } + + [Test] + public void Three_categories_declare_a_MustFollow() + { + var declared = MigrationCategoryRegistry.All + .Where(c => c.MustFollow is not null) + .ToDictionary(c => c.Id, c => c.MustFollow); + + Assert.That(declared, Is.EqualTo(new Dictionary + { + // A cascading foreign key: throughput rows cannot exist before their endpoint row. + ["LicensingThroughput"] = "LicensingEndpoints", + // Keeps settings back while known endpoints are unfinished. The heartbeat sync is kept off + // the copied settings because both categories finish before the host opens. + ["EndpointSettings"] = "KnownEndpoints", + // A group comment whose failed messages have not arrived reads as an orphan to the + // retention sweeper, which deletes it. + ["GroupComments"] = "ArchivedAndResolvedFailedMessages" + })); + } + + [Test] + public void The_two_failed_message_categories_and_failed_imports_carry_bodies() + { + var carriesBodies = MigrationCategoryRegistry.All.Where(c => c.CarriesBodies).Select(c => c.Id).ToArray(); + + Assert.That(carriesBodies, Is.EquivalentTo(new[] { "UnresolvedAndRetryIssuedFailedMessages", "ArchivedAndResolvedFailedMessages", "FailedErrorImports" })); + } + + [Test] + public void The_two_failed_message_categories_split_every_status_between_them() + { + var split = MigrationCategoryRegistry.UnresolvedAndRetryIssuedStatuses.Concat(MigrationCategoryRegistry.ArchivedAndResolvedStatuses).ToArray(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(split, Is.Unique); + Assert.That(split, Is.EquivalentTo(Enum.GetValues()), + "a status neither category claims is a failed message no migration copies"); + } + } + + [Test] + public void Find_returns_null_for_an_unknown_id() + { + Assert.That(MigrationCategoryRegistry.Find("NoSuchCategory"), Is.Null); + } + + [Test] + public void Find_returns_the_category_for_a_known_id() + { + Assert.That(MigrationCategoryRegistry.Find("EventLog")?.Kind, Is.EqualTo(MigrationCategoryKind.Optional)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs new file mode 100644 index 0000000000..3df3d4e123 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationContractShapeTests.cs @@ -0,0 +1,64 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +class MigrationContractShapeTests +{ + [Test] + public void MigrationCategory_carries_no_fact_about_where_a_store_keeps_its_rows() + { + var properties = typeof(MigrationCategory).GetProperties().Select(p => p.Name); + + Assert.That(properties, Is.EquivalentTo(new[] { "Id", "Kind", "CarriesBodies", "Order", "MustFollow" }), + "a source database or target table belongs in that store's own adapter, where a different store pair can map it differently"); + } + + [Test] + public void Category_states_keep_the_integers_stored_checkpoints_already_hold() + { + var stored = System.Enum.GetValues().ToDictionary(state => state.ToString(), state => (int)state); + + Assert.That(stored, Is.EqualTo(new Dictionary + { + ["NotStarted"] = 0, + ["InProgress"] = 1, + ["Complete"] = 2, + ["CompleteWithErrors"] = 3, + ["Halted"] = 4, + ["Abandoned"] = 5, + ["Blocked"] = 6 + }), "checkpoints store State as an integer, so a reordered or inserted member silently changes what every saved checkpoint means"); + } + + [Test] + public void MigrationRow_body_defaults_to_null() + { + var row = new MigrationRow("id-1", new object(), new Dictionary()); + + Assert.That(row.Body, Is.Null); + } + + [Test] + public void MigrationCheckpoint_AlreadyPresentCount_defaults_to_zero() + { + var checkpoint = new MigrationCheckpoint( + CategoryId: "EndpointSettings", + State: MigrationCategoryState.NotStarted, + Cursor: null, + CopiedCount: 0, + SkippedCount: 0, + SourceTotal: null, + SkipReasons: null, + StartedAt: null, + LastProgressAt: null, + SettledAt: null, + LastError: null); + + Assert.That(checkpoint.AlreadyPresentCount, Is.Zero); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs new file mode 100644 index 0000000000..dc938d6543 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineBodyRetryTests.cs @@ -0,0 +1,234 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineBodyRetryTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task A_body_that_fails_until_the_last_attempt_then_succeeds_is_written_with_its_body_attached() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + var body = new MigrationBody(new byte[] { 1 }, "text/plain"); + source.SetBody("msg-1", body); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts - 1, new TimeoutException("blip")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + // Zeroed because FakeTimeProvider is never advanced here: a 200 ms Task.Delay against a clock + // nobody moves never completes, and the test hangs until the runner kills it. + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(target.WrittenRows(category.Id).Single().Body, Is.EqualTo(body)); + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(MigrationEngine.MaxBodyReadAttempts)); + } + } + + // Every type the engine treats as a defect. Each fails the same way on every attempt, so retrying one + // would only turn a code fault into skipped messages. + static readonly Exception[] Defects = + [ + new NotSupportedException("this source cannot read bodies"), + new NotImplementedException("not written yet"), + new InvalidOperationException("the session is closed"), + new ArgumentException("the id is not a document id"), + new NullReferenceException("no attachment"), + new InvalidCastException("not an attachment") + ]; + + [TestCaseSource(nameof(Defects))] + public async Task A_body_read_that_fails_as_a_defect_halts_the_category_on_the_first_attempt_without_skipping_the_message(Exception defect) + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, defect); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain(defect.GetType().Name)); + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(1)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.SkipReasons, Is.Null); + } + } + + [Test] + public async Task A_shutdown_during_a_body_read_stops_the_run_instead_of_skipping_the_message() + { + // Retrying a shutdown to the attempt limit and then recording the message as permanently + // unreadable is the one path here that silently loses a customer's failed message. + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + using var stopping = new CancellationTokenSource(); + source.StopOnBodyRead = ("msg-1", stopping); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category, stopping.Token)); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(source.BodyReadAttempts("msg-1"), Is.EqualTo(1), "a shutdown is not a transient body failure, so it is not retried"); + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(persisted.SkippedCount, Is.Zero, "the message is still there to copy on the next run"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + + [Test] + public async Task The_configured_backoff_is_waited_between_body_read_attempts() + { + // Without the wait, three attempts against a body store that is briefly down all fail inside a + // millisecond and the message is skipped for an outage it would have survived. + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + var body = new MigrationBody(new byte[] { 1 }, "text/plain"); + source.SetBody("msg-1", body); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts - 1, new TimeoutException("body store unreachable")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var clock = new TimerRecordingTimeProvider(); + var backoff = TimeSpan.FromMilliseconds(200); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = backoff }; + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); + + var runTask = engine.RunCategoryAsync(category); + + // Two failures, so a wait after each before the attempt that succeeds. + for (var waitNumber = 1; waitNumber <= MigrationEngine.MaxBodyReadAttempts - 1; waitNumber++) + { + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(5)), Is.True, $"backoff {waitNumber} never started"); + Assert.That(runTask.IsCompleted, Is.False, $"backoff {waitNumber} should still be pending"); + clock.Advance(backoff); + } + + var checkpoint = await runTask.WaitAsync(TimeSpan.FromSeconds(5)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(clock.DueTimes, Is.EqualTo(new[] { backoff, backoff }), "no wait after the final attempt, which has nothing left to retry"); + Assert.That(target.WrittenRows(category.Id).Single().Body, Is.EqualTo(body)); + } + } + + [Test] + public async Task The_skip_warning_for_an_unreadable_body_carries_the_last_attempts_exception() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1")); + var failure = new TimeoutException("body store unreachable"); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, failure); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var logger = new CapturingLogger(); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, logger); + + await engine.RunCategoryAsync(category); + + Assert.That(logger.Entries.Single(e => e.Message.StartsWith("Skipped msg-1")).Exception, Is.SameAs(failure)); + } + + [Test] + public async Task A_body_the_source_already_attached_is_written_without_reading_it_again() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + var attached = new MigrationBody(new byte[] { 7 }, "text/plain"); + source.Seed(category.Id, Row("msg-1") with { Body = attached }); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("a body read nobody needed")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(target.WrittenRows(category.Id).Single().Body, Is.EqualTo(attached)); + } + } + + [Test] + public async Task Exhausted_attempts_skip_the_whole_message_and_count_toward_the_halt_threshold() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("down")); + source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + // msg-1 is excluded entirely: not written with a missing body, not written at all. + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "msg-2" })); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(1)); + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + } + } + + [Test] + public async Task A_body_store_outage_across_many_messages_halts_the_category() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + var ids = Enumerable.Range(1, 1_000).Select(i => $"msg-{i}").ToArray(); + source.Seed(category.Id, [.. ids.Select(Row)]); + // Every body read fails every attempt: a down body store, not a per-message fluke. + foreach (var id in ids) + { + source.FailBodyReads(id, MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + } + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs new file mode 100644 index 0000000000..9c74a7b605 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCategorySelectionTests.cs @@ -0,0 +1,80 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using Particular.Approvals; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineCategorySelectionTests +{ + static MigrationEngine BuildEngine(IReadOnlyCollection selectedOptionalIds, out InMemoryMigrationCheckpointStore checkpointStore) + { + checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.FromSeconds(1), 5, 100, selectedOptionalIds); + return new MigrationEngine(new InMemoryMigrationSource(), target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + } + + static string Describe(MigrationCategory category) => + $"{category.Kind} {category.Order}: {category.Id}" + + (category.CarriesBodies ? ", with bodies" : string.Empty) + + (category.MustFollow is null ? string.Empty : $", after {category.MustFollow}"); + + [Test] + public void Every_category_runs_in_a_fixed_order() + { + var everyOptionalId = MigrationCategoryRegistry.All + .Where(category => category.Kind == MigrationCategoryKind.Optional) + .Select(category => category.Id) + .ToArray(); + var engine = BuildEngine(everyOptionalId, out _); + + var runOrder = engine.SelectCategories(MigrationCategoryKind.Required) + .Concat(engine.SelectCategories(MigrationCategoryKind.Optional)) + .Select(Describe); + + Approver.Verify(string.Join(Environment.NewLine, runOrder)); + } + + [Test] + public void Only_configured_optional_categories_are_selected() + { + string[] configured = [MigrationCategoryIds.GroupComments, MigrationCategoryIds.EventLog, MigrationCategoryIds.ArchivedAndResolvedFailedMessages]; + var engine = BuildEngine(configured, out _); + + var selected = engine.SelectCategories(MigrationCategoryKind.Optional); + var left = MigrationCategoryRegistry.All + .Where(category => category.Kind == MigrationCategoryKind.Optional && !selected.Contains(category)) + .Select(category => category.Id); + + Approver.Verify(string.Join(Environment.NewLine, + [ + $"Configured: {string.Join(", ", configured)}", + "Runs as:", + .. selected.Select(Describe), + $"Never copied: {string.Join(", ", left)}" + ])); + } + + [Test] + public void A_category_removed_from_configuration_leaves_its_checkpoint_row_untouched() + { + var engine = BuildEngine([], out var checkpointStore); + var previousRun = new MigrationCheckpoint("EventLog", MigrationCategoryState.CompleteWithErrors, "cursor-99", 40, 2, 42, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); + checkpointStore.Upsert(previousRun).GetAwaiter().GetResult(); + + var selected = engine.SelectCategories(MigrationCategoryKind.Optional); + + using (Assert.EnterMultipleScope()) + { + Assert.That(selected.Select(c => c.Id), Does.Not.Contain("EventLog")); + Assert.That(checkpointStore.Read("EventLog").GetAwaiter().GetResult(), Is.EqualTo(previousRun with { Version = 1 })); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs new file mode 100644 index 0000000000..59b53c6a07 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCollisionTests.cs @@ -0,0 +1,45 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineCollisionTests +{ + static MigrationRow Row(string id, string name) => new(id, new { Name = name }, new Dictionary()); + + [Test] + public async Task Rows_already_present_in_the_target_are_counted_as_already_present_across_batches_not_skipped_or_overwritten() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1", "from-source"), Row("msg-2", "from-source"), Row("msg-3", "from-source"), Row("msg-4", "from-source")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; + // msg-1 and msg-4 already exist in the target: they failed again after cutover, and real ingestion wrote them. + target.SeedExistingKey("msg-1"); + target.SeedExistingKey("msg-4"); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(2)); + // Asserted through the fake's recorded rows, never by the engine inspecting a document. + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "msg-2", "msg-3" })); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs new file mode 100644 index 0000000000..aafb9d4935 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineCopyTests.cs @@ -0,0 +1,130 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineCopyTests +{ + static MigrationRow Row(string id) => new(id, new { Name = id }, new Dictionary()); + + [Test] + public async Task Copies_every_row_and_finishes_Complete_when_nothing_was_skipped() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + Assert.That(checkpoint.SkippedCount, Is.Zero); + Assert.That(checkpoint.Cursor, Is.EqualTo("c")); + Assert.That(checkpoint.StartedAt, Is.Not.Null); + Assert.That(checkpoint.SettledAt, Is.Not.Null); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(3)); + } + } + + [Test] + public async Task A_category_with_no_rows_finishes_Complete_without_a_cursor() + { + // The ordinary state of several required categories on a small instance: nothing to copy is a + // finished category, not a category that never ran. + var category = MigrationCategoryRegistry.Find("MessageRedirects")!; + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((0L, 0L))); + Assert.That(checkpoint.Cursor, Is.Null); + Assert.That(checkpoint.SettledAt, Is.Not.Null); + } + } + + [Test] + public async Task The_moment_a_category_settles_comes_from_the_injected_clock() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var settledAt = new DateTimeOffset(2026, 3, 4, 5, 6, 7, TimeSpan.Zero); + var clock = new FakeTimeProvider(settledAt); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.SettledAt, Is.EqualTo(settledAt.UtcDateTime), "a wall-clock read here would drift from every other time the migration reports"); + } + + [Test] + public async Task A_category_already_CompleteWithErrors_is_left_alone_on_a_second_run() + { + // Finished with a few skips is finished. Re-reading it would copy the whole category again and + // count its skips a second time. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var finishedWithSkips = new MigrationCheckpoint(category.Id, MigrationCategoryState.CompleteWithErrors, "b", 1, 1, 2, + new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 }, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); + await checkpointStore.Upsert(finishedWithSkips); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(finishedWithSkips with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + + [Test] + public async Task A_category_already_Complete_is_left_alone_on_a_second_run() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var alreadyDone = new MigrationCheckpoint(category.Id, MigrationCategoryState.Complete, "a", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null); + await checkpointStore.Upsert(alreadyDone); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(alreadyDone with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs new file mode 100644 index 0000000000..fb26628421 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineFailurePathTests.cs @@ -0,0 +1,376 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineFailurePathTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + static MigrationEngine BuildEngine(InMemoryMigrationSource source, InMemoryMigrationCheckpointStore checkpointStore, InMemoryMigrationTarget target) => + new(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + [Test] + public async Task A_failing_write_halts_the_category_and_records_the_error_instead_of_throwing() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("Simulated failure")); + // The first batch committed, so the cursor is real and a later run resumes from it. + Assert.That(checkpoint.Cursor, Is.EqualTo("b")); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(2)); + } + } + + [Test] + public async Task The_halt_is_durable_so_the_health_check_and_the_guard_can_read_it() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 1 }; + var engine = BuildEngine(source, checkpointStore, target); + + await engine.RunCategoryAsync(category); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(persisted.LastError, Is.Not.Null); + } + } + + [Test] + public void A_cancelled_run_is_a_shutdown_rather_than_a_failure_and_is_not_swallowed() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, StopOnCall = (1, stopping) }; + var engine = BuildEngine(source, checkpointStore, target); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category, stopping.Token)); + } + + [Test] + public async Task A_halted_category_is_re_attempted_on_the_next_run_and_resumes_from_its_cursor() + { + // A halt that no restart can clear would leave abandoning the category as the only way out of + // a fault the customer has already repaired. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var firstRun = BuildEngine(source, checkpointStore, target); + var halted = await firstRun.RunCategoryAsync(category); + Assert.That(halted.State, Is.EqualTo(MigrationCategoryState.Halted)); + + // Stopped on its first write, so the saved row is the restarted one rather than the completed one. + target.FailOnCallNumber = null; + using var stopping = new CancellationTokenSource(); + target.StopOnCall = (3, stopping); + Assert.ThrowsAsync(() => BuildEngine(source, checkpointStore, target).RunCategoryAsync(category, stopping.Token)); + var restarted = await checkpointStore.Read(category.Id); + + target.StopOnCall = null; + var lastRun = BuildEngine(source, checkpointStore, target); + var finished = await lastRun.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(restarted!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(restarted.SettledAt, Is.Null, "a copy running again does not keep the time its halt settled at"); + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finished.LastError, Is.Null, "a cleared halt does not leave a stale error on the row"); + Assert.That(finished.CopiedCount, Is.EqualTo(4)); + var writtenIds = target.WrittenRows(category.Id).Select(r => r.SourceId).ToArray(); + Assert.That(writtenIds, Is.EquivalentTo(new[] { "a", "b", "c", "d" })); + Assert.That(writtenIds.Distinct().Count(), Is.EqualTo(writtenIds.Length), "no duplicates across the halt"); + } + } + + [Test] + public async Task Body_skips_in_a_batch_whose_write_fails_are_counted_once_across_the_restart() + { + var category = MigrationCategoryRegistry.Find("UnresolvedAndRetryIssuedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + // msg-1's body is unreadable on both runs: every attempt fails on each. + source.FailBodyReads("msg-1", 2 * MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + source.SetBody("msg-2", new MigrationBody(new byte[] { 2 }, "text/plain")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 1 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var firstRun = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var halted = await firstRun.RunCategoryAsync(category); + + target.FailOnCallNumber = null; + var secondRun = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var finished = await secondRun.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(halted.State, Is.EqualTo(MigrationCategoryState.Halted)); + // The failed batch never committed, so its skip is recorded only when the batch is read again. + Assert.That(halted.SkippedCount, Is.Zero); + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(finished.SkippedCount, Is.EqualTo(1)); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "msg-2" })); + } + } + + [Test] + public async Task An_abandoned_category_is_left_exactly_as_it_is() + { + // Abandoned is the one end a person chooses: this category will never be copied, and the + // guard and the health check both treat it as settled. A later run must not quietly restart it. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var abandoned = new MigrationCheckpoint(category.Id, MigrationCategoryState.Abandoned, "a", 1, 3, 4, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, "Halted: the body store was unreachable"); + await checkpointStore.Upsert(abandoned); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint, Is.EqualTo(abandoned with { Version = 1 }), "the row is read back untouched, at the version the seeding save left it"); + Assert.That(target.WrittenRows(category.Id), Is.Empty); + } + } + + [Test] + public async Task A_target_with_no_batch_size_for_a_category_halts_it_and_the_next_category_still_runs() + { + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var unmapped = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var next = MigrationCategoryRegistry.Find("MessageRedirects")!; + source.Seed(unmapped.Id, Row("k-1")); + source.Seed(next.Id, Row("r-1")); + var target = new InMemoryMigrationTarget(checkpointStore) { NoBatchSizeFor = unmapped.Id }; + var engine = BuildEngine(source, checkpointStore, target); + + var results = await engine.RunCategories([unmapped, next]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(results[0].LastError, Does.Contain("No batch size")); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete)); + } + } + + [Test] + public void A_halt_whose_save_fails_still_logs_the_exception_that_caused_it() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new HaltSaveFailsCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { FailOnCallNumber = 1 }; + var logger = new CapturingLogger(); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), logger); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category)); + + Assert.That(logger.Entries.Where(e => e.Level == LogLevel.Error).Select(e => e.Exception?.Message), Does.Contain("Simulated failure on write 1")); + } + + [Test] + public void A_threshold_halt_whose_save_fails_still_logs_why_it_halted() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new HaltSaveFailsCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + target.RejectKey("a", MigrationSkipReason.BodyUnreadable); + var logger = new CapturingLogger(); + // A floor of zero lets the one rejected row halt the category. + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 0, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, logger); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category)); + + Assert.That(logger.Entries.Where(e => e.Level == LogLevel.Error).Select(e => e.Message), Has.Some.Contains("Halted: 1 of 1 rows skipped")); + } + + [Test] + public async Task A_checkpoint_conflict_leaves_the_other_writer_alone_instead_of_halting_over_it() + { + // Two hosts pointed at one target is what the version token exists for. Settling this as halted + // would write over the progress of whichever host is still copying. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + // Save 1 moves the row to in progress; save 2 is the first batch, by which point the other host has moved it on. + var checkpointStore = new ConflictOnNthSaveCheckpointStore { ConflictOnSave = 2 }; + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2 }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category)); + + var persisted = await checkpointStore.Read(category.Id); + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress), "no halted row was written over the conflict"); + } + + [Test] + public async Task A_cancellation_that_is_not_a_shutdown_halts_the_category_like_any_other_failure() + { + // An inner timeout surfaces as the same exception type as a host stopping, and only the token + // says which. Treating a timeout as a shutdown would end the run with no reason on the row. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) + { + FailOnCallNumber = 1, + FailWith = new OperationCanceledException("the query timed out") + }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("OperationCanceledException").And.Contain("the query timed out")); + } + } + + [Test] + public async Task The_halt_reason_names_the_cursor_the_copy_had_reached() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.LastError, Does.Contain("at cursor b"), "the cursor is the only pointer an operator has to where it stopped"); + } + + [Test] + public async Task The_halt_reason_says_at_the_start_when_the_first_batch_never_committed() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { FailOnCallNumber = 1 }; + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.LastError, Does.Contain("at the start")); + } + + [Test] + public async Task Every_row_the_target_skips_is_named_in_the_log() + { + // The counts say how much was left behind. Only the log says which rows, and it is the way back + // to them while the RavenDB database still exists. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + target.RejectKey("b", MigrationSkipReason.PastRetention); + var logger = new CapturingLogger(); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), logger); + + await engine.RunCategoryAsync(category); + + Assert.That(logger.Entries.Select(entry => entry.Message), Has.Some.EqualTo("Skipped b in category KnownEndpoints")); + } + + [Test] + public async Task A_stop_between_batches_leaves_the_row_in_progress_at_the_batch_that_committed() + { + // The stop lands in the source rather than in a write, so nothing is mid-transaction: the row + // still has to describe the batches that did commit, and stay resumable. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, CancelOnCall = (1, stopping) }; + var engine = BuildEngine(source, checkpointStore, target); + + Assert.ThrowsAsync(() => engine.RunCategoryAsync(category, stopping.Token)); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress), "a shutdown is not a halt"); + Assert.That(persisted.Cursor, Is.EqualTo("b")); + Assert.That(persisted.CopiedCount, Is.EqualTo(2)); + Assert.That(persisted.SettledAt, Is.Null); + } + } + + // Another host moved the row on between this host reading it and saving it. + sealed class ConflictOnNthSaveCheckpointStore : IMigrationCheckpointStore + { + readonly InMemoryMigrationCheckpointStore saved = new(); + int saves; + + public int ConflictOnSave { get; init; } + + public Task> ReadAll(CancellationToken cancellationToken = default) => saved.ReadAll(cancellationToken); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => saved.Read(categoryId, cancellationToken); + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + ++saves == ConflictOnSave + ? throw new MigrationCheckpointConflictException($"Checkpoint {checkpoint.CategoryId} was saved from version {checkpoint.Version}, but the stored row has moved on.") + : saved.Upsert(checkpoint, cancellationToken); + } + + // The store shares the target's database, which has become unreachable by the time the halt is saved. + sealed class HaltSaveFailsCheckpointStore : IMigrationCheckpointStore + { + readonly InMemoryMigrationCheckpointStore saved = new(); + + public Task> ReadAll(CancellationToken cancellationToken = default) => saved.ReadAll(cancellationToken); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => saved.Read(categoryId, cancellationToken); + + public Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + checkpoint.State == MigrationCategoryState.Halted ? throw new TimeoutException("checkpoint store unreachable") : saved.Upsert(checkpoint, cancellationToken); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs new file mode 100644 index 0000000000..47a77f85d7 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineHaltTests.cs @@ -0,0 +1,189 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineHaltTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task A_systemic_failure_halts_the_category_partway_through() + { + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + // Every 5th of 1,000 rows is rejected: a steady 20% spread evenly rather than clustered at the + // start, past both the 5% proportion and the 100-row floor. + var rows = Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}")).ToArray(); + source.Seed(category.Id, rows); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("Halted")); + Assert.That(checkpoint.SettledAt, Is.Not.Null); + // Stopped partway: the 1,000th row was never reached. + Assert.That(target.WrittenRows(category.Id).Count, Is.LessThan(800)); + } + } + + [Test] + public async Task Rows_the_target_would_have_deleted_anyway_never_count_toward_the_halt_threshold() + { + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + // The same 20% that halts the category above, except these rows are past the target's retention + // cutoff, so leaving them behind is the copy working rather than failing. + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention, benign: true); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That((checkpoint.CopiedCount, checkpoint.SkippedCount), Is.EqualTo((800L, 200L))); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(800), "the last row was reached, so nothing halted partway"); + } + } + + [TestCase(25, MigrationCategoryState.CompleteWithErrors, TestName = "A_mix_of_benign_and_fault_skips_runs_on_while_the_faults_stay_under_the_threshold")] + [TestCase(10, MigrationCategoryState.Halted, TestName = "A_mix_of_benign_and_fault_skips_halts_once_the_faults_alone_pass_the_threshold")] + public async Task A_batch_mixing_benign_and_fault_skips_is_judged_on_the_faults_alone(int everyNthIsAFault, MigrationCategoryState expected) + { + // A real archive copy loses rows both ways at once: retention takes some, unreadable bodies take + // others. This is the only shape where the subtraction has to do arithmetic rather than pick a side. + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 5_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + // A fifth of the category is past retention either way, which on its own is four times the threshold. + foreach (var i in Enumerable.Range(1, 5_000).Where(i => i % 5 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.PastRetention, benign: true); + } + // The offset keeps the faults clear of the benign rows: 4% of the category in one case, 10% in the other. + foreach (var i in Enumerable.Range(1, 5_000).Where(i => i % everyNthIsAFault == 3)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.State, Is.EqualTo(expected)); + } + + [Test] + public async Task Rows_already_present_keep_a_category_under_the_halt_threshold() + { + // Already-present rows are in the denominator because the run did handle them. Drop them from it + // and this category's 4% fault rate reads as 12%, halting a copy that is merely being re-run. + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 3_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 2_000)) + { + target.SeedExistingKey($"row-{i}"); + } + // 120 faults spread through the whole category: past the 100-row floor, and 4% of 3,000. + foreach (var i in Enumerable.Range(1, 3_000).Where(i => i % 25 == 0)) + { + target.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(120)); + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(1_920), "the 80 already-present rows that are also faults are refused before the collision check"); + } + } + + [Test] + public async Task Rows_already_present_in_the_target_never_count_toward_the_halt_threshold() + { + var category = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + target.SeedExistingKey($"row-{i}"); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(checkpoint.CopiedCount, Is.EqualTo(800)); + Assert.That(checkpoint.AlreadyPresentCount, Is.EqualTo(200)); + } + } + + [Test] + public async Task A_restart_after_a_threshold_halt_counts_only_its_own_skips_and_keeps_the_earlier_ones() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 1_000).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var failingTarget = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + foreach (var i in Enumerable.Range(1, 1_000).Where(i => i % 5 == 0)) + { + failingTarget.RejectKey($"row-{i}", MigrationSkipReason.BodyUnreadable); + } + var options = new MigrationEngineOptions(TimeSpan.Zero, HaltThresholdPercent: 5, HaltThresholdMinimum: 100, []); + var halted = await new MigrationEngine(source, failingTarget, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + // The cause is fixed: the remaining rows now write cleanly. + var fixedTarget = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 100 }; + var finished = await new MigrationEngine(source, fixedTarget, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance).RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + // 120 skips in 600 rows is the first point past both the floor and 5%. + Assert.That((halted.State, halted.CopiedCount, halted.SkippedCount), Is.EqualTo((MigrationCategoryState.Halted, 480L, 120L))); + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors), "the skips still on the row must not halt a run that skips nothing"); + Assert.That((finished.CopiedCount, finished.SkippedCount), Is.EqualTo((880L, 120L)), "copied, skipped at the end"); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs new file mode 100644 index 0000000000..0ede2f09b3 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOptionsTests.cs @@ -0,0 +1,78 @@ +namespace ServiceControl.UnitTests.Migration; + +using System; +using NUnit.Framework; +using ServiceControl.Configuration; +using ServiceControl.Persistence.DataMigration; + +[TestFixture] +[NonParallelizable] +class MigrationEngineOptionsTests +{ + static readonly SettingsRootNamespace Namespace = new("ServiceControl"); + + [TearDown] + public void ClearEnvironmentVariables() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_THROTTLEPAUSEMILLISECONDS", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDPERCENT", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDMINIMUM", null); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_OPTIONALCATEGORIES", null); + } + + [Test] + public void Defaults_match_the_contract_when_nothing_is_configured() + { + var options = MigrationEngineOptions.FromSettings(Namespace); + + using (Assert.EnterMultipleScope()) + { + Assert.That(options.ThrottlePause, Is.EqualTo(TimeSpan.FromMilliseconds(100))); + Assert.That(options.HaltThresholdPercent, Is.EqualTo(5)); + Assert.That(options.HaltThresholdMinimum, Is.EqualTo(100)); + Assert.That(options.SelectedOptionalCategoryIds, Is.Empty); + Assert.That(options.BodyRetryBackoff, Is.EqualTo(TimeSpan.FromMilliseconds(200))); + } + } + + [Test] + public void Reads_configured_values_from_environment_variables() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_THROTTLEPAUSEMILLISECONDS", "2500"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDPERCENT", "10"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_HALTTHRESHOLDMINIMUM", "50"); + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_OPTIONALCATEGORIES", "EventLog, CustomChecks"); + + var options = MigrationEngineOptions.FromSettings(Namespace); + + using (Assert.EnterMultipleScope()) + { + Assert.That(options.ThrottlePause, Is.EqualTo(TimeSpan.FromMilliseconds(2500))); + Assert.That(options.HaltThresholdPercent, Is.EqualTo(10)); + Assert.That(options.HaltThresholdMinimum, Is.EqualTo(50)); + Assert.That(options.SelectedOptionalCategoryIds, Is.EquivalentTo(new[] { "EventLog", "CustomChecks" })); + } + } + + [Test] + public void Refuses_an_unknown_optional_category_id() + { + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_OPTIONALCATEGORIES", "NoSuchCategory"); + + var ex = Assert.Throws(() => MigrationEngineOptions.FromSettings(Namespace)); + + Assert.That(ex.Message, Does.Contain("NoSuchCategory")); + } + + [Test] + public void Refuses_a_required_category_id_named_as_optional() + { + // EndpointSettings is required, not optional: naming it here is a customer mistake, not + // a valid way to force it. Required categories are never a matter of configuration. + Environment.SetEnvironmentVariable("SERVICECONTROL_MIGRATION_OPTIONALCATEGORIES", "EndpointSettings"); + + var ex = Assert.Throws(() => MigrationEngineOptions.FromSettings(Namespace)); + + Assert.That(ex.Message, Does.Contain("EndpointSettings")); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs new file mode 100644 index 0000000000..a7f2c99e25 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineOrderingTests.cs @@ -0,0 +1,173 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineOrderingTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + static MigrationEngine BuildEngine(InMemoryMigrationSource source, InMemoryMigrationCheckpointStore checkpointStore, InMemoryMigrationTarget target) => + new(source, target, checkpointStore, new FakeTimeProvider(), new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + [Test] + public async Task LicensingThroughput_does_not_start_before_LicensingEndpoints_completes_and_says_so_on_its_checkpoint_row() + { + var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; + var source = new InMemoryMigrationSource(); + source.Seed(throughputCategory.Id, Row("t-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + // LicensingEndpoints has never run: no checkpoint row for it at all. + var checkpoint = await engine.RunCategoryAsync(throughputCategory); + + var persisted = await checkpointStore.Read(throughputCategory.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(target.WrittenRows(throughputCategory.Id), Is.Empty); + // A row exists, so status can print it. Without one, an operator cannot tell a category + // waiting on another from a category nobody asked for. + Assert.That(persisted, Is.Not.Null); + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(persisted.LastError, Does.Contain("LicensingEndpoints")); + } + } + + [Test] + public async Task EndpointSettings_does_not_start_before_KnownEndpoints_completes() + { + // Not a foreign key: settings wait for known endpoints, and a halted required category keeps + // the host, and with it the heartbeat sync, closed. + var settingsCategory = MigrationCategoryRegistry.Find("EndpointSettings")!; + var source = new InMemoryMigrationSource(); + source.Seed(settingsCategory.Id, Row("EndpointSettings/1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(settingsCategory); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(target.WrittenRows(settingsCategory.Id), Is.Empty); + } + } + + [TestCase(MigrationCategoryState.InProgress)] + [TestCase(MigrationCategoryState.Halted)] + public async Task GroupComments_does_not_start_before_the_archive_completes(MigrationCategoryState archiveState) + { + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("ArchivedAndResolvedFailedMessages", archiveState, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(comments); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(checkpoint.LastError, Is.EqualTo($"Blocked: GroupComments must follow ArchivedAndResolvedFailedMessages, which is {archiveState}")); + Assert.That(target.WrittenRows(comments.Id), Is.Empty); + } + } + + [Test] + public async Task A_blocked_category_runs_once_the_category_it_follows_settles() + { + // Every real migration starts group comments blocked, so a block nothing can clear would strand + // the last category and leave the migration unable to end. + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var archive = MigrationCategoryRegistry.Find("ArchivedAndResolvedFailedMessages")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var archiveRunning = new MigrationCheckpoint(archive.Id, MigrationCategoryState.InProgress, "m-500", 500, 0, null, null, DateTime.UtcNow, DateTime.UtcNow, null, null); + var saved = await checkpointStore.Upsert(archiveRunning); + var target = new InMemoryMigrationTarget(checkpointStore); + + var blocked = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + await checkpointStore.Upsert(saved with { State = MigrationCategoryState.Complete, SettledAt = DateTime.UtcNow }); + var unblocked = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + using (Assert.EnterMultipleScope()) + { + Assert.That(blocked.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(unblocked.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(unblocked.LastError, Is.Null, "the block reason does not outlive the block"); + Assert.That(target.WrittenRows(comments.Id), Has.Count.EqualTo(1)); + } + } + + [Test] + public async Task A_blocked_category_has_not_settled_because_it_has_not_finished() + { + // Anything reading SettledAt beside State would otherwise take a blocked category for a finished one. + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + + await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + var persisted = await checkpointStore.Read(comments.Id); + Assert.That(persisted!.SettledAt, Is.Null); + } + + [TestCase(MigrationCategoryState.NotStarted)] + [TestCase(MigrationCategoryState.Blocked)] + public async Task A_predecessor_that_has_a_row_but_has_not_run_is_named_by_the_state_on_that_row(MigrationCategoryState archiveState) + { + // A missing row reads as "not started"; a row that exists says what it actually holds, which is + // how an operator tells a category waiting its turn from one waiting on a chain. + var comments = MigrationCategoryRegistry.Find("GroupComments")!; + var source = new InMemoryMigrationSource(); + source.Seed(comments.Id, Row("GroupComment/g-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("ArchivedAndResolvedFailedMessages", archiveState, null, 0, 0, null, null, null, null, null, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + + var checkpoint = await BuildEngine(source, checkpointStore, target).RunCategoryAsync(comments); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Blocked)); + Assert.That(checkpoint.LastError, Is.EqualTo($"Blocked: GroupComments must follow ArchivedAndResolvedFailedMessages, which is {archiveState}")); + } + } + + [TestCase(MigrationCategoryState.Complete)] + [TestCase(MigrationCategoryState.CompleteWithErrors)] + [TestCase(MigrationCategoryState.Abandoned)] + public async Task LicensingThroughput_proceeds_once_LicensingEndpoints_is_finished_or_abandoned(MigrationCategoryState endpointsState) + { + var throughputCategory = MigrationCategoryRegistry.Find("LicensingThroughput")!; + var source = new InMemoryMigrationSource(); + source.Seed(throughputCategory.Id, Row("t-1")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + await checkpointStore.Upsert(new MigrationCheckpoint("LicensingEndpoints", endpointsState, "e-1", 1, 0, 1, null, DateTime.UtcNow, DateTime.UtcNow, DateTime.UtcNow, null)); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = BuildEngine(source, checkpointStore, target); + + var checkpoint = await engine.RunCategoryAsync(throughputCategory); + + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs new file mode 100644 index 0000000000..d21760171f --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineResumeTests.cs @@ -0,0 +1,161 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineResumeTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task A_restart_keeps_the_moment_the_category_first_started() + { + // StartedAt is what an operator reads to see how long a background copy has been going. Stamping + // it again on every restart would report minutes for a copy that has been running for days. + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, FailOnCallNumber = 2 }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var firstStart = new DateTimeOffset(2026, 3, 1, 9, 0, 0, TimeSpan.Zero); + var halted = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(firstStart), options, NullLogger.Instance) + .RunCategoryAsync(category); + + // A day later, the cause is fixed and the host is started again. + target.FailOnCallNumber = null; + var restart = firstStart.AddDays(1); + var finished = await new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(restart), options, NullLogger.Instance) + .RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(halted.StartedAt, Is.EqualTo(firstStart.UtcDateTime)); + Assert.That(finished.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finished.StartedAt, Is.EqualTo(firstStart.UtcDateTime), "the restart carries on a copy that started a day ago"); + Assert.That(finished.SettledAt, Is.EqualTo(restart.UtcDateTime)); + } + } + + [Test] + public async Task Restarting_after_a_mid_category_stop_produces_no_duplicates_and_no_gaps() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + var allIds = Enumerable.Range(1, 6).Select(i => $"row-{i}").ToArray(); + source.Seed(category.Id, [.. allIds.Select(Row)]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, StopOnCall = (3, stopping) }; + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var firstEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + // Batches are 2 rows each; the host stops on the 3rd call to Write, so exactly 2 batches (4 rows) land. + Assert.ThrowsAsync(() => firstEngine.RunCategoryAsync(category, stopping.Token)); + + var afterStop = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + // Still InProgress, not Halted: a cancelled run is a shutdown, not a failure. + Assert.That(afterStop!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That(afterStop.CopiedCount, Is.EqualTo(4)); + Assert.That(afterStop.Cursor, Is.EqualTo("row-4")); + Assert.That(target.WrittenRows(category.Id), Has.Count.EqualTo(4)); + } + + // Restart: same source, same target, same checkpoint store, nothing stopping it this time. + target.StopOnCall = null; + var secondEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var finalCheckpoint = await secondEngine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.Complete)); + Assert.That(finalCheckpoint.CopiedCount, Is.EqualTo(6)); + var writtenIds = target.WrittenRows(category.Id).Select(r => r.SourceId).ToArray(); + Assert.That(writtenIds, Is.EquivalentTo(allIds), "no gaps"); + Assert.That(writtenIds.Distinct().Count(), Is.EqualTo(writtenIds.Length), "no duplicates"); + } + } + + [Test] + public async Task A_graceful_stop_saves_the_real_split_of_the_last_committed_batch() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 6).Select(i => Row($"row-{i}"))]); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + using var stopping = new CancellationTokenSource(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 2, StopOnCall = (3, stopping) }; + // Both in the second batch, the last one to commit before the stop. + target.SeedExistingKey("row-3"); + target.RejectKey("row-4", MigrationSkipReason.BodyUnreadable); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var firstEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + Assert.ThrowsAsync(() => firstEngine.RunCategoryAsync(category, stopping.Token)); + var afterStop = await checkpointStore.Read(category.Id); + + target.StopOnCall = null; + var secondEngine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + var finalCheckpoint = await secondEngine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(afterStop!.State, Is.EqualTo(MigrationCategoryState.InProgress)); + Assert.That((afterStop.CopiedCount, afterStop.SkippedCount, afterStop.AlreadyPresentCount), Is.EqualTo((2L, 1L, 1L)), "copied, skipped, already present after the stop"); + Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That((finalCheckpoint.CopiedCount, finalCheckpoint.SkippedCount, finalCheckpoint.AlreadyPresentCount), Is.EqualTo((4L, 1L, 1L)), "copied, skipped, already present at the end"); + Assert.That(finalCheckpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 })); + } + } + + [Test] + public async Task A_hard_crash_after_a_committed_write_loses_nothing_because_the_target_saved_the_real_split() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, [.. Enumerable.Range(1, 4).Select(i => Row($"row-{i}"))]); + var committed = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(committed) { DefaultBatchSize = 2 }; + target.SeedExistingKey("row-3"); + target.RejectKey("row-4", MigrationSkipReason.BodyUnreadable); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var crashingEngine = new MigrationEngine(source, target, new CrashAfterCommitCheckpointStore(committed, crashAfterCursor: "row-4"), new FakeTimeProvider(), options, NullLogger.Instance); + await crashingEngine.RunCategoryAsync(category); + + var restartedEngine = new MigrationEngine(source, target, committed, new FakeTimeProvider(), options, NullLogger.Instance); + var finalCheckpoint = await restartedEngine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + // The engine's own settle was lost, but every count came from the target's own transaction. + Assert.That(finalCheckpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That((finalCheckpoint.CopiedCount, finalCheckpoint.SkippedCount, finalCheckpoint.AlreadyPresentCount), Is.EqualTo((2L, 1L, 1L)), "copied, skipped, already present at the end"); + Assert.That(target.WrittenRows(category.Id).Select(r => r.SourceId), Is.EqualTo(new[] { "row-1", "row-2" })); + } + } + + // Once the target has committed crashAfterCursor, the engine's own saves are lost, as if the process died there. + sealed class CrashAfterCommitCheckpointStore(IMigrationCheckpointStore committed, string crashAfterCursor) : IMigrationCheckpointStore + { + public Task> ReadAll(CancellationToken cancellationToken = default) => committed.ReadAll(cancellationToken); + + public Task Read(string categoryId, CancellationToken cancellationToken = default) => committed.Read(categoryId, cancellationToken); + + public async Task Upsert(MigrationCheckpoint checkpoint, CancellationToken cancellationToken = default) => + (await committed.Read(checkpoint.CategoryId, cancellationToken))?.Cursor == crashAfterCursor + ? checkpoint + : await committed.Upsert(checkpoint, cancellationToken); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs new file mode 100644 index 0000000000..e4e8393de0 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineRunCategoriesTests.cs @@ -0,0 +1,84 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineRunCategoriesTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task Runs_every_category_it_is_given_in_the_order_it_is_given_them() + { + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + // The reverse of registry order, so an engine that re-sorted by Order would run KnownEndpoints first. + MigrationCategory[] given = [MigrationCategoryRegistry.Find("MessageRedirects")!, MigrationCategoryRegistry.Find("KnownEndpoints")!]; + foreach (var category in given) + { + source.Seed(category.Id, Row($"{category.Id}-1")); + } + + var results = await engine.RunCategories(given); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results.Select(c => c.CategoryId), Is.EqualTo(new[] { "MessageRedirects", "KnownEndpoints" })); + Assert.That(results.Select(c => c.State), Is.All.EqualTo(MigrationCategoryState.Complete)); + } + } + + [Test] + public async Task Running_no_categories_copies_nothing_and_reports_nothing() + { + // What an instance that selected no optional categories asks for on every background pass. + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var engine = new MigrationEngine(new InMemoryMigrationSource(), target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + + var results = await engine.RunCategories([]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results, Is.Empty); + Assert.That(await checkpointStore.ReadAll(), Is.Empty, "a category nobody ran gets no row"); + } + } + + [Test] + public async Task One_category_halting_does_not_stop_the_ones_after_it() + { + // Everything that can still be copied is copied, and the guard decides what an incomplete migration + // may do. Stopping at the first halt would leave later categories untouched with no reason recorded. + var source = new InMemoryMigrationSource(); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1, FailOnCallNumber = 1 }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), + new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []), NullLogger.Instance); + var first = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var second = MigrationCategoryRegistry.Find("MessageRedirects")!; + source.Seed(first.Id, Row("k-1")); + source.Seed(second.Id, Row("r-1")); + + var results = await engine.RunCategories([first, second]); + + using (Assert.EnterMultipleScope()) + { + Assert.That(results[0].State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(results[1].State, Is.EqualTo(MigrationCategoryState.Complete)); + } + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs new file mode 100644 index 0000000000..2e07501603 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineSkipReasonTests.cs @@ -0,0 +1,160 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineSkipReasonTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task Reasons_the_target_reports_add_up_across_batches_on_the_checkpoint() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c"), Row("d")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; + // One reason exists, so this pins the total rather than the split between reasons. + target.RejectKey("a", MigrationSkipReason.BodyUnreadable); + target.RejectKey("b", MigrationSkipReason.BodyUnreadable); + target.RejectKey("d", MigrationSkipReason.BodyUnreadable); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.SkippedCount, Is.EqualTo(3)); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 3 })); + Assert.That((await checkpointStore.Read(category.Id))!.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 3 })); + } + } + + [Test] + public async Task A_message_whose_body_is_never_read_is_counted_under_BodyUnreadable() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2")); + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("down")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary { [MigrationSkipReason.BodyUnreadable] = 1 })); + } + + [Test] + public async Task A_batch_that_loses_rows_two_different_ways_records_both_reasons() + { + // The breakdown is what tells a customer what they lost and why. A merge that overwrote instead of + // summing would leave the total right and the reasons wrong, and verification would still balance. + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.UnresolvedAndRetryIssuedFailedMessages)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("msg-1"), Row("msg-2"), Row("msg-3")); + // One the engine skips itself because the body will not read, one the target refuses. + source.FailBodyReads("msg-1", MigrationEngine.MaxBodyReadAttempts, new TimeoutException("body store unreachable")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 3 }; + target.RejectKey("msg-2", MigrationSkipReason.PastRetention, benign: true); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []) { BodyRetryBackoff = TimeSpan.Zero }; + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.CompleteWithErrors)); + Assert.That(checkpoint.SkippedCount, Is.EqualTo(2)); + Assert.That(checkpoint.SkipReasons, Is.EquivalentTo(new Dictionary + { + [MigrationSkipReason.BodyUnreadable] = 1, + [MigrationSkipReason.PastRetention] = 1 + })); + Assert.That(target.WrittenRows(category.Id).Select(row => row.SourceId), Is.EqualTo(new[] { "msg-3" })); + } + } + + [Test] + public async Task A_target_whose_skip_reasons_do_not_add_up_to_its_skips_halts_the_category() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new UnexplainedSkipTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("reported 1 skipped").And.Contain("reasons for 0")); + Assert.That(checkpoint.Cursor, Is.Null, "the target refused ahead of its commit, so nothing was written and a restart reads the batch again"); + } + } + + [Test] + public async Task A_target_counting_more_benign_skips_than_skipped_rows_halts_the_category() + { + var category = MigrationCategoryRegistry.Find(MigrationCategoryIds.KnownEndpoints)!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new OverCountedBenignTarget(checkpointStore); + var options = new MigrationEngineOptions(TimeSpan.Zero, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.State, Is.EqualTo(MigrationCategoryState.Halted)); + Assert.That(checkpoint.LastError, Does.Contain("2 benign skips").And.Contain("out of 1 skipped")); + } + } + + sealed class OverCountedBenignTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget + { + public int BatchSizeFor(MigrationCategory category) => 10; + + public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + var reasons = new Dictionary { [MigrationSkipReason.PastRetention] = batch.Rows.Count }; + var saved = await checkpointStore.Upsert(checkpointToExtend.Extend(0, batch.Rows.Count, 0, reasons), cancellationToken); + return new MigrationWriteResult(saved, 0, batch.Rows.Count, [], 0, reasons, BenignSkipped: batch.Rows.Count + 1); + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + } + + sealed class UnexplainedSkipTarget(IMigrationCheckpointStore checkpointStore) : IMigrationTarget + { + public int BatchSizeFor(MigrationCategory category) => 10; + + public async Task Write(MigrationCategory category, MigrationBatch batch, MigrationCheckpoint checkpointToExtend, CancellationToken cancellationToken = default) + { + var saved = await checkpointStore.Upsert(checkpointToExtend.Extend(0, batch.Rows.Count, 0, null), cancellationToken); + return new MigrationWriteResult(saved, 0, batch.Rows.Count, []); + } + + public Task Count(MigrationCategory category, CancellationToken cancellationToken = default) => Task.FromResult(0L); + } +} diff --git a/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs b/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs new file mode 100644 index 0000000000..e568edb6a1 --- /dev/null +++ b/src/ServiceControl.UnitTests/Migration/MigrationEngineThrottleTests.cs @@ -0,0 +1,97 @@ +#nullable enable +namespace ServiceControl.UnitTests.Migration; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using NUnit.Framework; +using ServiceControl.Persistence.DataMigration; +using ServiceControl.UnitTests.Migration.Fakes; + +[TestFixture] +class MigrationEngineThrottleTests +{ + static MigrationRow Row(string id) => new(id, new object(), new Dictionary()); + + [Test] + public async Task Optional_categories_pause_between_batches_for_the_configured_duration() + { + var category = MigrationCategoryRegistry.Find("EventLog")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1 }; + var clock = new TimerRecordingTimeProvider(); + var pause = TimeSpan.FromSeconds(1); + var options = new MigrationEngineOptions(pause, 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); + + var runTask = engine.RunCategoryAsync(category); + + // Three batches, so a pause before the second and another before the third. + for (var pauseNumber = 1; pauseNumber <= 2; pauseNumber++) + { + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(5)), Is.True, $"pause {pauseNumber} never started"); + Assert.That(runTask.IsCompleted, Is.False, $"pause {pauseNumber} should still be pending"); + clock.Advance(pause); + } + + var checkpoint = await runTask.WaitAsync(TimeSpan.FromSeconds(5)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + Assert.That(clock.DueTimes, Is.EqualTo(new[] { pause, pause })); + } + } + + [Test] + public async Task A_stop_during_a_pause_ends_the_run_as_a_shutdown() + { + // Most of a throttled background copy's life is spent in this pause, so it is where a host being + // stopped most often lands. + var category = MigrationCategoryRegistry.Find("EventLog")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1 }; + var clock = new TimerRecordingTimeProvider(); + using var stopping = new CancellationTokenSource(); + var options = new MigrationEngineOptions(TimeSpan.FromSeconds(1), 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, clock, options, NullLogger.Instance); + + var runTask = engine.RunCategoryAsync(category, stopping.Token); + Assert.That(await clock.TimerCreated.WaitAsync(TimeSpan.FromSeconds(5)), Is.True, "the first pause never started"); + await stopping.CancelAsync(); + + Assert.ThrowsAsync(() => runTask); + + var persisted = await checkpointStore.Read(category.Id); + using (Assert.EnterMultipleScope()) + { + Assert.That(persisted!.State, Is.EqualTo(MigrationCategoryState.InProgress), "a shutdown mid-pause is not a halt"); + Assert.That(persisted.Cursor, Is.EqualTo("a"), "the batch before the pause committed"); + } + } + + [Test] + public async Task Required_categories_never_pause() + { + var category = MigrationCategoryRegistry.Find("KnownEndpoints")!; + var source = new InMemoryMigrationSource(); + source.Seed(category.Id, Row("a"), Row("b"), Row("c")); + var checkpointStore = new InMemoryMigrationCheckpointStore(); + var target = new InMemoryMigrationTarget(checkpointStore) { DefaultBatchSize = 1 }; + // A FakeTimeProvider that is never advanced: if the engine tried to pause, this would hang + // until the test runner's own timeout, which WaitAsync turns into a clear failure instead. + var options = new MigrationEngineOptions(TimeSpan.FromSeconds(1), 5, 100, []); + var engine = new MigrationEngine(source, target, checkpointStore, new FakeTimeProvider(), options, NullLogger.Instance); + + var checkpoint = await engine.RunCategoryAsync(category).WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.That(checkpoint.CopiedCount, Is.EqualTo(3)); + } +} diff --git a/src/ServiceControl.slnx b/src/ServiceControl.slnx index 622050f094..38f69745a8 100644 --- a/src/ServiceControl.slnx +++ b/src/ServiceControl.slnx @@ -51,6 +51,7 @@ + diff --git a/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs new file mode 100644 index 0000000000..ce0ac0501b --- /dev/null +++ b/src/ServiceControl/Hosting/Commands/MigrationSourceReportCommand.cs @@ -0,0 +1,42 @@ +namespace ServiceControl.Hosting.Commands +{ + using System; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Particular.ServiceControl.Hosting; + using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Persistence; + + class MigrationSourceReportCommand : AbstractCommand + { + public override async Task Execute(HostArguments args, Settings settings, CancellationToken cancellationToken = default) + { + await using var source = await PersistenceFactory.OpenMigrationSource(settings, cancellationToken); + + var description = await source.Describe(cancellationToken); + + Console.Out.WriteLine("ServiceControl migration source report"); + Console.Out.WriteLine(); + Console.Out.WriteLine($"{"Source persistence",-20}: {settings.MigrationSourcePersistenceType}"); + Console.Out.WriteLine($"{"Version",-20}: {description.Version}"); + + foreach (var fact in description.Facts) + { + var origin = fact.SettingKey is null ? string.Empty : $" (from {fact.SettingKey})"; + Console.Out.WriteLine($"{fact.Label,-20}: {fact.Value}{origin}"); + } + + foreach (var scope in (await source.Inventory(cancellationToken)).GroupBy(entry => entry.Scope)) + { + Console.Out.WriteLine(); + Console.Out.WriteLine($"{scope.Key}:"); + + foreach (var entry in scope.OrderBy(entry => entry.Name, StringComparer.Ordinal)) + { + Console.Out.WriteLine($" {entry.Name,-42}{entry.Count,12:N0}"); + } + } + } + } +} diff --git a/src/ServiceControl/Hosting/Help.txt b/src/ServiceControl/Hosting/Help.txt index 4925b8c494..4289cde5c8 100644 --- a/src/ServiceControl/Hosting/Help.txt +++ b/src/ServiceControl/Hosting/Help.txt @@ -22,6 +22,21 @@ it owns the retry pipeline, the retention sweep, integration event dispatch and Message bodies must be stored somewhere every host can read, so this mode should not be combined with file system body storage unless the path is a shared mount. +MIGRATION SOURCE REPORT + + ServiceControl.exe --migration-source-report + +Reports what a migration would read from the source persistence named by +ServiceControl/Migration/SourcePersistenceType: the facts the source reports about itself, with the +setting each came from, and a row count for everything it holds. The source reads the instance's own +RavenDB settings, so keep them in place when switching PersistenceType. + +For a RavenDB source: an EXTERNAL server can be reported on while ServiceControl is running. An EMBEDDED source +cannot: ServiceControl starts its own RavenDB process against that data directory, and a second one +cannot attach to it. Stop the ServiceControl service first, run the report, and start it again. An +instance that does not ship the RavenDB server, such as the container image, cannot report on an +embedded source at all. + SERVICE INSTALL AND UNINSTALL AND CONFIGURATION OPTIONS As of Service Control 1.7 the command line uninstall and install switches have been removed. diff --git a/src/ServiceControl/Hosting/HostArguments.cs b/src/ServiceControl/Hosting/HostArguments.cs index b260543662..0fa750392c 100644 --- a/src/ServiceControl/Hosting/HostArguments.cs +++ b/src/ServiceControl/Hosting/HostArguments.cs @@ -62,6 +62,15 @@ public HostArguments(string[] args) } }; + var migrationSourceReportOptions = new OptionSet + { + { + "migration-source-report", + "Report what a migration would read from the source persistence", + s => Command = typeof(MigrationSourceReportCommand) + } + }; + try { externalInstallerOptions.Parse(args); @@ -92,6 +101,13 @@ public HostArguments(string[] args) return; } + migrationSourceReportOptions.Parse(args); + + if (Command == typeof(MigrationSourceReportCommand)) + { + return; + } + defaultOptions.Parse(args); } catch (Exception e) diff --git a/src/ServiceControl/Infrastructure/Settings/Settings.cs b/src/ServiceControl/Infrastructure/Settings/Settings.cs index af52f4bf88..9b39e03426 100644 --- a/src/ServiceControl/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl/Infrastructure/Settings/Settings.cs @@ -16,6 +16,7 @@ using ServiceControl.Infrastructure.Settings; using ServiceControl.Infrastructure.WebApi; using ServiceControl.Persistence; + using ServiceControl.Persistence.DataMigration; using ServiceControl.Transports; using ServicePulse; using JsonSerializer = System.Text.Json.JsonSerializer; @@ -185,6 +186,7 @@ public string InstanceId public string TransportType { get; set; } public string PersistenceType { get; private set; } + public string MigrationSourcePersistenceType => SettingsReader.Read(SettingsRootNamespace, MigrationSettings.SourcePersistenceTypeKey, MigrationSettings.DefaultSourcePersistenceType); public string ErrorLogQueue { get; set; } public string ErrorQueue { get; set; } diff --git a/src/ServiceControl/Persistence/PersistenceFactory.cs b/src/ServiceControl/Persistence/PersistenceFactory.cs index 892920cf18..5aac4cdda8 100644 --- a/src/ServiceControl/Persistence/PersistenceFactory.cs +++ b/src/ServiceControl/Persistence/PersistenceFactory.cs @@ -2,13 +2,17 @@ namespace ServiceControl.Persistence { using System; using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; using ServiceBus.Management.Infrastructure.Settings; + using ServiceControl.Persistence.DataMigration; static class PersistenceFactory { public static IPersistence Create(Settings settings, bool maintenanceMode = false) { - var persistenceConfiguration = CreatePersistenceConfiguration(settings); + var persistenceConfiguration = CreatePersistenceConfiguration(settings.PersistenceType, settings); if (maintenanceMode && !persistenceConfiguration.SupportsMaintenanceMode) { @@ -24,11 +28,39 @@ public static IPersistence Create(Settings settings, bool maintenanceMode = fals return persistence; } - static IPersistenceConfiguration CreatePersistenceConfiguration(Settings settings) + public static IMigrationSource CreateMigrationSource(Settings settings) { + var persistenceType = settings.MigrationSourcePersistenceType; + var persistenceConfiguration = CreatePersistenceConfiguration(persistenceType, settings); + + if (persistenceConfiguration is not IMigrationSourceFactory sourceFactory) + { + throw new Exception($"The '{persistenceType}' persistence cannot be read as a migration source. Set {Settings.SettingsRootNamespace}/{MigrationSettings.SourcePersistenceTypeKey} to the persistence that holds the data being migrated away from."); + } + + return sourceFactory.CreateSource(Settings.SettingsRootNamespace); + } + + public static async Task OpenMigrationSource(Settings settings, CancellationToken cancellationToken = default) + { + var source = CreateMigrationSource(settings); + await source.Open(cancellationToken); + + return source; + } + + static IPersistenceConfiguration CreatePersistenceConfiguration(string persistenceType, Settings settings) + { + var persistenceManifest = PersistenceManifestLibrary.Find(persistenceType) + ?? throw new Exception($"There is no persistence named '{persistenceType}'. Available: {string.Join(", ", PersistenceManifestLibrary.PersistenceManifests.Where(m => m.IsSupported).Select(m => m.Name))}."); + + if (persistenceManifest.TypeName is null) + { + throw new Exception($"The '{persistenceManifest.DisplayName}' persistence no longer ships an assembly and cannot be loaded."); + } + try { - var persistenceManifest = PersistenceManifestLibrary.Find(settings.PersistenceType); var assemblyPath = Path.Combine(persistenceManifest.Location, $"{persistenceManifest.AssemblyName}.dll"); var loadContext = settings.AssemblyLoadContextResolver(assemblyPath); var customizationType = Type.GetType(persistenceManifest.TypeName, loadContext.LoadFromAssemblyName, null, true); @@ -37,7 +69,7 @@ static IPersistenceConfiguration CreatePersistenceConfiguration(Settings setting } catch (Exception e) { - throw new Exception($"Could not load persistence customization type {settings.PersistenceType}.", e); + throw new Exception($"Could not load persistence customization type {persistenceType}.", e); } } } diff --git a/src/ServiceControl/ServiceControl.csproj b/src/ServiceControl/ServiceControl.csproj index 7afd8f88cf..d9b9958c43 100644 --- a/src/ServiceControl/ServiceControl.csproj +++ b/src/ServiceControl/ServiceControl.csproj @@ -68,6 +68,7 @@ +