From f2cc13ac98e18ef888a540b5c3a41a5e9e9eba4c Mon Sep 17 00:00:00 2001 From: Kevin Dethlefs Date: Sun, 18 Oct 2026 13:07:13 -0500 Subject: [PATCH 1/2] EBS: check the Twitch PubSub credentials at startup; app: warn when Twitch refuses them A wrong Twitch__ExtensionSecret was completely silent. The EBS started, stored every card, answered /healthz ok, and never delivered a live update (every PubSub call 403 'JWT could not be verified'); viewers who loaded the channel still saw the card, so nothing looked broken. EBS - TwitchCredentialProbe sends the real PubSub endpoint a request signed like a broadcast but aimed at a target Twitch cannot deliver to, so nothing ever reaches a viewer. 401/403 = rejected (logged at error, naming the three settings to check); 400 or success = not refused; anything else is inconclusive and changes nothing. It never throws. - TwitchCredentialCheckBackgroundService runs it ~10 s after start and every 6 h. It decides at run time whether it is under the test host (the test host applies its environment late). - TwitchCredentialStatus is shared with every real broadcast: a refusal sets it, a success clears it, so a real publish is the authority over the startup check. - /healthz stays status ok (a wrong secret must not restart the container in a loop) and adds pubsub / pubsubStatus / pubsubCheckedAt. - A publish that fails because Twitch refuses the EBS's credentials is a 503 with X-EDNexus-Pubsub-Rejected: credentials, instead of a 502 a proxy can rewrite. Other PubSub failures stay 502. App - StreamStatePublishStatus.BackendRejectedByTwitch, mapped from that header. It is retried like any failure (the operator can fix the backend while the app runs) and is not a re-login. - A warning bar on the main window (the Settings status line is only seen with Settings open), raised by StreamCardWarning and cleared by the next success. Tests: EBS 212 (+26), full solution 1809 passed. The key app tests were shown to fail without the change. Limits, stated in the README: that Twitch verifies the token before the body is reasoned from its docs, not observed, so the startup check's accepted means not refused; only a real broadcast proves the credentials. Co-Authored-By: Claude Sonnet 5.5 --- extension/README.md | 2 +- .../ViewModels/MainWindowViewModel.cs | 19 + src/EDNexus.App/Views/MainWindow.axaml | 8 + src/EDNexus.App/Views/SettingsWindow.axaml.cs | 2 + src/EDNexus.Core/Twitch/StreamCardWarning.cs | 28 ++ .../Twitch/StreamStateApiClient.cs | 15 + .../Twitch/TwitchStreamCardService.cs | 3 +- src/EDNexus.Ebs/Program.cs | 37 +- src/EDNexus.Ebs/README.md | 46 ++- .../TwitchCredentialCheckBackgroundService.cs | 73 ++++ .../Services/TwitchCredentialProbe.cs | 143 ++++++++ .../Services/TwitchCredentialStatus.cs | 80 ++++ .../Services/TwitchPubSubClient.cs | 9 + tests/EDNexus.Ebs.Tests/EbsUnitTests.cs | 1 + .../TwitchCredentialCheckTests.cs | 346 ++++++++++++++++++ .../Twitch/StreamCardWarningTests.cs | 40 ++ .../Twitch/StreamStateApiClientTests.cs | 34 ++ .../Twitch/TwitchStreamCardServiceTests.cs | 40 ++ 18 files changed, 922 insertions(+), 4 deletions(-) create mode 100644 src/EDNexus.Core/Twitch/StreamCardWarning.cs create mode 100644 src/EDNexus.Ebs/Services/TwitchCredentialCheckBackgroundService.cs create mode 100644 src/EDNexus.Ebs/Services/TwitchCredentialProbe.cs create mode 100644 src/EDNexus.Ebs/Services/TwitchCredentialStatus.cs create mode 100644 tests/EDNexus.Ebs.Tests/TwitchCredentialCheckTests.cs create mode 100644 tests/EDNexus.Tests/Twitch/StreamCardWarningTests.cs diff --git a/extension/README.md b/extension/README.md index c756f2b..d3d647d 100644 --- a/extension/README.md +++ b/extension/README.md @@ -203,7 +203,7 @@ and is not repeated here. ### 0. Before touching the console - [ ] The EBS is deployed and reachable at `https://ednexus.signal-and-thread.com` over valid TLS - (`GET /healthz` returns `{"status":"ok"}`), and the same origin is what the desktop app's + (`GET /healthz` returns `status: "ok"`, plus `pubsub: "unknown"`, `"accepted"` or `"rejected"`: a `rejected` means Twitch is refusing the EBS's credentials and viewers get no live updates, see the EBS README's credential self-check), and the same origin is what the desktop app's `AppSettings.Twitch.EbsBaseUrl` defaults to. - [ ] The desktop app build that publishes a **heartbeat** has shipped (see [Publishing contract](#publishing-contract-with-the-desktop-app)). Without it, an idle docked diff --git a/src/EDNexus.App/ViewModels/MainWindowViewModel.cs b/src/EDNexus.App/ViewModels/MainWindowViewModel.cs index 981518f..f89a4f3 100644 --- a/src/EDNexus.App/ViewModels/MainWindowViewModel.cs +++ b/src/EDNexus.App/ViewModels/MainWindowViewModel.cs @@ -265,6 +265,7 @@ private EngineHost BuildHost() { if (result.IsSuccess) Trace.TraceInformation("Twitch: stream card published."); else Trace.TraceWarning($"Twitch: stream card publish failed ({result.Status}) — {result.Error}"); + UpdateTwitchWarning(result); TwitchPublishCompleted?.Invoke(result); }; twitchCard.ReauthRequired += () => @@ -374,6 +375,24 @@ private void UpdateJournalStatus() private void OnDiscordSettingsChanged(DiscordSettings settings) => _host.ApplyDiscordSettings(settings); [ObservableProperty] private string _journalStatus = ""; + + /// + /// A warning, shown on the main window, that the stream card is failing for a reason the commander + /// cannot fix (Twitch refusing the backend's credentials). Empty when there is nothing to say. The + /// Settings window's status line is only seen with Settings open; this is seen while playing. + /// + [ObservableProperty, NotifyPropertyChangedFor(nameof(HasTwitchWarning))] private string _twitchWarning = ""; + + public bool HasTwitchWarning => TwitchWarning.Length > 0; + + /// Raises or clears from a publish result; runs on the UI thread, since results arrive on the publisher's. + private void UpdateTwitchWarning(EDNexus.Core.Twitch.StreamStatePublishResult result) + { + if (EDNexus.Core.Twitch.StreamCardWarning.For(result) is { } warning) + Dispatcher.UIThread.Post(() => TwitchWarning = warning); + else if (EDNexus.Core.Twitch.StreamCardWarning.Clears(result)) + Dispatcher.UIThread.Post(() => TwitchWarning = ""); + } [ObservableProperty] private string _privacyStatus = ""; [ObservableProperty] private string _commanderName = "—"; [ObservableProperty] private string _balance = "0 cr"; diff --git a/src/EDNexus.App/Views/MainWindow.axaml b/src/EDNexus.App/Views/MainWindow.axaml index e08347d..da31a0d 100644 --- a/src/EDNexus.App/Views/MainWindow.axaml +++ b/src/EDNexus.App/Views/MainWindow.axaml @@ -217,6 +217,14 @@ + + + + + diff --git a/src/EDNexus.App/Views/SettingsWindow.axaml.cs b/src/EDNexus.App/Views/SettingsWindow.axaml.cs index a3713f8..d5bb4fd 100644 --- a/src/EDNexus.App/Views/SettingsWindow.axaml.cs +++ b/src/EDNexus.App/Views/SettingsWindow.axaml.cs @@ -623,6 +623,8 @@ private void OnTwitchPublishCompleted(StreamStatePublishResult result) => StreamStatePublishStatus.Unauthorized => "The backend rejected this machine's credential — sign in again.", StreamStatePublishStatus.TooLarge => "The last card was too large for Twitch — try hiding a section.", StreamStatePublishStatus.RateLimited => "Publishing is being rate-limited; updates are slowing down.", + StreamStatePublishStatus.BackendRejectedByTwitch => + "Twitch is rejecting the backend's credentials, so viewers are not getting live updates. It is a backend problem, not your sign-in.", StreamStatePublishStatus.Cleared => "Card taken off the air.", StreamStatePublishStatus.ClearedNotDelivered => "Card taken off the air; viewers already watching will be told on the next retry.", diff --git a/src/EDNexus.Core/Twitch/StreamCardWarning.cs b/src/EDNexus.Core/Twitch/StreamCardWarning.cs new file mode 100644 index 0000000..12ae24e --- /dev/null +++ b/src/EDNexus.Core/Twitch/StreamCardWarning.cs @@ -0,0 +1,28 @@ +namespace EDNexus.Core.Twitch; + +/// +/// The warning the app shows when publishing the stream card is failing for a reason the commander +/// cannot fix from their side. Kept apart from the UI so the wording and the rule for when it appears +/// and clears are tested. +/// +public static class StreamCardWarning +{ + /// + /// What to show after a publish, or null for nothing. Only a + /// raises one: a plain failure (the network dropped, a rate limit) already has its own status line + /// and clears itself on the next success, whereas a credential the backend operator must fix would + /// otherwise go unnoticed, since the card keeps looking fine on the commander's own machine. + /// + public static string? For(StreamStatePublishResult result) => + result.Status == StreamStatePublishStatus.BackendRejectedByTwitch + ? "Twitch is rejecting the EDNexus backend's credentials, so viewers are not getting live updates. " + + "This is a problem with the backend's configuration, not with your sign-in; it keeps retrying and " + + "the card will start updating again as soon as it is fixed." + : null; + + /// + /// True when this result shows the problem is over: the card reached viewers (or was taken down + /// cleanly). A failure of some other kind proves nothing either way, so it leaves a shown warning alone. + /// + public static bool Clears(StreamStatePublishResult result) => result.IsSuccess; +} diff --git a/src/EDNexus.Core/Twitch/StreamStateApiClient.cs b/src/EDNexus.Core/Twitch/StreamStateApiClient.cs index 0ecafae..ca44ffa 100644 --- a/src/EDNexus.Core/Twitch/StreamStateApiClient.cs +++ b/src/EDNexus.Core/Twitch/StreamStateApiClient.cs @@ -34,6 +34,14 @@ public enum StreamStatePublishStatus /// already watching. Worth retrying for their sake; nothing is left public to anyone new. /// ClearedNotDelivered, + + /// + /// Twitch is refusing the backend's own PubSub credentials (503 with the + /// header). The card is stored, so new + /// viewers still get it, but nothing reaches viewers already watching. It is a server + /// misconfiguration: not the commander's sign-in (so no re-login) and not worth retrying hard. + /// + BackendRejectedByTwitch, } /// Whether the snapshot reached viewers, and if not, why not. @@ -103,6 +111,12 @@ public StreamStateApiClient(HttpClient? http = null) /// public const string SnapshotRemovedHeader = "X-EDNexus-Snapshot-Removed"; + /// + /// Header the EBS sets when Twitch is refusing the EBS's own PubSub credentials. Mirrors + /// EDNexus.Ebs.Services.TwitchCredentialStatus.RejectedHeader. + /// + public const string PubSubRejectedHeader = "X-EDNexus-Pubsub-Rejected"; + public Task PublishAsync( string updateStateEndpoint, string token, StreamCardSnapshot snapshot, CancellationToken ct = default) { @@ -156,6 +170,7 @@ private async Task SendAsync( var status = response.StatusCode switch { _ when response.Headers.Contains(SnapshotRemovedHeader) => StreamStatePublishStatus.ClearedNotDelivered, + _ when response.Headers.Contains(PubSubRejectedHeader) => StreamStatePublishStatus.BackendRejectedByTwitch, HttpStatusCode.Unauthorized => StreamStatePublishStatus.Unauthorized, HttpStatusCode.RequestEntityTooLarge => StreamStatePublishStatus.TooLarge, HttpStatusCode.TooManyRequests => StreamStatePublishStatus.RateLimited, diff --git a/src/EDNexus.Core/Twitch/TwitchStreamCardService.cs b/src/EDNexus.Core/Twitch/TwitchStreamCardService.cs index c41b1dc..fa422c1 100644 --- a/src/EDNexus.Core/Twitch/TwitchStreamCardService.cs +++ b/src/EDNexus.Core/Twitch/TwitchStreamCardService.cs @@ -290,7 +290,8 @@ private async Task PumpAsync(CancellationToken ct) try { var published = await PublishIfChangedAsync(ct).ConfigureAwait(false); - if (published is { Status: StreamStatePublishStatus.RateLimited or StreamStatePublishStatus.Failed }) + if (published is { Status: StreamStatePublishStatus.RateLimited or StreamStatePublishStatus.Failed + or StreamStatePublishStatus.BackendRejectedByTwitch }) { // Give a struggling EBS (or a tighter-than-expected rate limit) room to recover // rather than retrying at the floor interval. diff --git a/src/EDNexus.Ebs/Program.cs b/src/EDNexus.Ebs/Program.cs index 37fe155..fbc476c 100644 --- a/src/EDNexus.Ebs/Program.cs +++ b/src/EDNexus.Ebs/Program.cs @@ -122,6 +122,14 @@ // long a single misbehaving/slow call can hold resources. client.Timeout = TimeSpan.FromSeconds(10); }); + // Whether Twitch accepts the credentials PubSub calls are signed with: written by every real + // broadcast and by the startup check, read by /healthz and by the update-state response. + builder.Services.AddSingleton(); + builder.Services.AddHttpClient(client => + { + client.Timeout = TimeSpan.FromSeconds(10); + }); + builder.Services.AddHostedService(); builder.Services.AddHttpClient(client => { // Same reasoning as above, and a login/refresh waits on this: fail in seconds, not the 100s default. @@ -306,7 +314,20 @@ await Results.Problem( // project site rather than a 404. app.MapGet("/", () => Results.Redirect("https://signal-thread.github.io/EDNexus/")); - app.MapGet("/healthz", () => Results.Ok(new { status = "ok" })); + // "ok" means the process is up and answering, which is all a container health check should + // gate on, so a credential problem is reported alongside it, never instead of it: a wrong secret + // should not get the container restarted in a loop. `pubsub` is "unknown", "accepted" or "rejected". + app.MapGet("/healthz", (TwitchCredentialStatus credentials) => + { + var (state, httpStatus, at) = credentials.Current; + return Results.Ok(new + { + status = "ok", + pubsub = state.ToString().ToLowerInvariant(), + pubsubStatus = httpStatus, + pubsubCheckedAt = at, + }); + }); app.MapOAuthEndpoints(); @@ -315,6 +336,7 @@ await Results.Problem( UpdateStateRequest body, IChannelStateStore stateStore, ITwitchPubSubClient pubSubClient, + TwitchCredentialStatus credentials, IBroadcasterTokenStore tokenStore, ChannelLocks channelLocks, IOptions ebsOptions, @@ -368,6 +390,19 @@ await Results.Problem( if (!published) { logger.LogWarning("Failed to publish state update for channel {ChannelId} to Twitch PubSub.", channelId); + + // Twitch refusing the EBS's own credentials is a server misconfiguration, not an outage, + // and no amount of retrying fixes it. Say so with a header (a proxy in front can replace + // a 502's body) and a 503 (which one does not rewrite), so the desktop app can tell the + // commander it is not their sign-in, instead of a bare "502". + if (credentials.State == TwitchCredentialState.Rejected) + { + httpRequest.HttpContext.Response.Headers[TwitchCredentialStatus.RejectedHeader] = "credentials"; + return Results.Problem( + "Twitch is rejecting this service's PubSub credentials, so live updates cannot be delivered. The card is stored and new viewers still see it. This is a server configuration problem.", + statusCode: StatusCodes.Status503ServiceUnavailable); + } + return Results.Problem("Failed to publish the update to Twitch PubSub.", statusCode: StatusCodes.Status502BadGateway); } diff --git a/src/EDNexus.Ebs/README.md b/src/EDNexus.Ebs/README.md index 70144e2..51936cb 100644 --- a/src/EDNexus.Ebs/README.md +++ b/src/EDNexus.Ebs/README.md @@ -111,6 +111,47 @@ end to end instead of finding out when a viewer reports a blank card: 5. Switch the card off (`DELETE /api/update-state`, or the app's toggle) and confirm the card disappears and `initial-state` answers `404`. +### Credential self-check + +A wrong `Twitch__ExtensionSecret` used to be silent: the service started, stored every card, answered +`/healthz` with `ok`, and never delivered a live update. Viewers who load the channel still saw the +card (it is served from the store), so nothing looked broken until someone noticed a card that never +changed. Three things now make it loud: + +1. **A startup check.** About 10 seconds after start, then every 6 hours, the EBS sends the real + Helix PubSub endpoint a request signed exactly like a broadcast but aimed at a target Twitch cannot + deliver to, so **nothing is ever sent to a viewer**. Twitch refusing the signed token + (`401`/`403`) logs at **error**: `Twitch REJECTED the EBS's PubSub credentials (403: ...)`, naming + the three settings to check. A `400` (the credentials got through and only the bogus target was + refused) logs `Twitch did not refuse the EBS's PubSub credentials`. Anything else (a network error, + a 5xx, a 429) is "inconclusive" and changes nothing. It never throws or stops the service. +2. **Real broadcasts keep the record.** Every actual PubSub call updates the same status: a refusal + sets it, a success clears it. This is the authority, so a stale startup verdict cannot outlive the + first real publish. +3. **`/healthz` reports it, and publishes carry it.** `/healthz` still answers `status: "ok"` (a wrong + secret must not get the container restarted in a loop, which would not fix it) with + `pubsub: "unknown" | "accepted" | "rejected"`, plus `pubsubStatus` (the HTTP status behind a + rejection) and `pubsubCheckedAt`. A `POST /api/update-state` that fails *because* Twitch is + refusing these credentials returns **`503`** with the header `X-EDNexus-Pubsub-Rejected: credentials` + instead of a bare `502` (a proxy in front can replace a 502's body, so the header and the status + code carry the meaning). The desktop app turns that into a warning on its main window. + +```bash +curl -s https:///healthz +# {"status":"ok","pubsub":"rejected","pubsubStatus":403,"pubsubCheckedAt":"2026-10-18T17:50:43+00:00"} +``` + +What `rejected` means and how to fix it: `Twitch__ExtensionSecret` must be this extension's own base64 +secret from the Developer Console (not another extension's, a rotated one, or the application's Client +Secret); `Twitch__ExtensionId` must be this extension's Client ID; and set `Twitch__OwnerUserId` to the +numeric Twitch user id of the extension's owner. Restart the service and watch `/healthz` (it updates +on the next real publish; the check also re-runs on its 6 hour cycle). + +> **What the check does not prove.** Twitch verifying the token before it looks at the request body is +> reasoned from its documentation, not observed: the refusal side was seen in production (`403 JWT could +> not be verified`), the acceptance side was not. So `accepted` from the startup check means "not +> refused"; only a real broadcast proves the credentials work, and that is why it overrides the check. + ## Behind a reverse proxy The container speaks plain HTTP and expects a TLS-terminating proxy (Caddy in the compose file) in @@ -203,7 +244,10 @@ Called by the desktop client on behalf of the broadcaster. - **Responses**: `200 OK` on success, `401 Unauthorized` for a missing/invalid/revoked token, `400 Bad Request` for a missing/`null`/non-object `state`, `413 Payload Too Large` if the state or the request body exceeds its limit, `429 Too Many Requests` if - the per-channel rate limit is exceeded, `502 Bad Gateway` if Twitch PubSub rejects the message. + the per-channel rate limit is exceeded, `502 Bad Gateway` if Twitch PubSub fails the message, and + `503 Service Unavailable` with `X-EDNexus-Pubsub-Rejected: credentials` when the failure is Twitch + refusing **this service's own credentials** (see [Credential self-check](#credential-self-check)). + The state is stored in every one of these cases, so new viewers still get the card. ### `DELETE /api/update-state` diff --git a/src/EDNexus.Ebs/Services/TwitchCredentialCheckBackgroundService.cs b/src/EDNexus.Ebs/Services/TwitchCredentialCheckBackgroundService.cs new file mode 100644 index 0000000..aa5d3d0 --- /dev/null +++ b/src/EDNexus.Ebs/Services/TwitchCredentialCheckBackgroundService.cs @@ -0,0 +1,73 @@ +namespace EDNexus.Ebs.Services; + +/// +/// Checks, shortly after startup and then every few hours, that Twitch accepts the credentials the +/// EBS signs PubSub calls with, and says so loudly in the log when it does not. +/// +/// +/// Without this a wrong Twitch__ExtensionSecret is silent: the service starts, stores every +/// card, answers /healthz with ok, and simply never delivers a live update, which nobody +/// notices until a viewer's card stops changing. It never throws or stops the host, and it only +/// reports: the real broadcasts keep the authoritative record (see ). +/// +public sealed class TwitchCredentialCheckBackgroundService : BackgroundService +{ + /// Delay before the first check, so the host is listening and a restart loop does not spam Twitch. + public static readonly TimeSpan InitialDelay = TimeSpan.FromSeconds(10); + + /// Between checks once running: often enough to notice a rotated secret, rare enough to be no load. + public static readonly TimeSpan Interval = TimeSpan.FromHours(6); + + private readonly ITwitchCredentialProbe _probe; + private readonly IHostEnvironment _environment; + private readonly TimeProvider _time; + private readonly ILogger _logger; + + public TwitchCredentialCheckBackgroundService( + ITwitchCredentialProbe probe, + IHostEnvironment environment, + TimeProvider time, + ILogger logger) + { + _probe = probe; + _environment = environment; + _time = time; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + // Decided here, not when the service is registered: the test host applies its environment late, + // and a check that reached out to Twitch from every long-running test would be a flaky one. + if (_environment.IsEnvironment("Testing")) return; + + try + { + await Task.Delay(InitialDelay, _time, stoppingToken).ConfigureAwait(false); + + while (!stoppingToken.IsCancellationRequested) + { + await RunOnceAsync(stoppingToken).ConfigureAwait(false); + await Task.Delay(Interval, _time, stoppingToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + // Shutting down. + } + } + + /// One check, exposed so tests can drive it without waiting on the delays. + internal async Task RunOnceAsync(CancellationToken ct) + { + try + { + await _probe.CheckAsync(ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + // The probe already contains its own failures; this is the last line of defence for the host. + _logger.LogError(ex, "Unhandled error in the Twitch credential check."); + } + } +} diff --git a/src/EDNexus.Ebs/Services/TwitchCredentialProbe.cs b/src/EDNexus.Ebs/Services/TwitchCredentialProbe.cs new file mode 100644 index 0000000..5e66742 --- /dev/null +++ b/src/EDNexus.Ebs/Services/TwitchCredentialProbe.cs @@ -0,0 +1,143 @@ +using System.Net.Http.Headers; +using System.Net.Http.Json; +using EDNexus.Ebs.Options; +using EDNexus.Ebs.Security; +using Microsoft.Extensions.Options; + +namespace EDNexus.Ebs.Services; + +/// What a credential check found. +public enum TwitchCredentialProbeResult +{ + /// The check could not tell (a network error, or a response that says nothing about the credentials). + Inconclusive = 0, + + /// Twitch did not refuse the credentials. + Accepted, + + /// Twitch refused the credentials. + Rejected, +} + +/// Asks Twitch, without broadcasting anything, whether it accepts the credentials PubSub calls are signed with. +public interface ITwitchCredentialProbe +{ + /// Runs one check and records a definite answer in . + Task CheckAsync(CancellationToken cancellationToken); +} + +/// +/// Checks the PubSub credentials by sending the real endpoint a request that is signed exactly like a +/// real broadcast but carries a deliberately invalid target, so it can never reach a viewer. +/// +/// +/// Twitch verifies the signed token before it looks at the body, so a refused credential comes back +/// 401/403 and an accepted one comes back 400 (the bogus target is what it then +/// rejects). Going through the same endpoint, Client-Id and token claims as a real publish +/// is the point: a different endpoint would be checking a different set of rules. +/// +/// Two limits, stated rather than hidden. The refusal side is the one seen in production +/// (403 JWT could not be verified); that Twitch really does check the token before the body is +/// reasoned from its documentation, not observed, so an +/// here means "not refused", and only a real broadcast proves the credentials work. And this never +/// guesses: anything it does not recognise is , +/// which changes nothing. +/// +/// +public sealed class TwitchCredentialProbe : ITwitchCredentialProbe +{ + private const string PubSubUrl = "https://api.twitch.tv/helix/extensions/pubsub"; + + /// Not a target Twitch knows (broadcast, global, whisper-…), so it is refused whatever else is true. + internal const string InvalidTarget = "__ednexus_credential_check__"; + + private readonly HttpClient _httpClient; + private readonly ITwitchExtensionJwtService _jwtService; + private readonly TwitchEbsOptions _options; + private readonly TwitchCredentialStatus _status; + private readonly ILogger _logger; + + public TwitchCredentialProbe( + HttpClient httpClient, + ITwitchExtensionJwtService jwtService, + IOptions options, + TwitchCredentialStatus status, + ILogger logger) + { + _httpClient = httpClient; + _jwtService = jwtService; + _options = options.Value; + _status = status; + _logger = logger; + } + + /// + public async Task CheckAsync(CancellationToken cancellationToken) + { + // Any channel id will do for a request that is going to be refused on its target; the owner's + // is the most natural, and "0" cannot be a real one. + var channelId = !string.IsNullOrWhiteSpace(_options.OwnerUserId) ? _options.OwnerUserId.Trim() : "0"; + + try + { + using var request = new HttpRequestMessage(HttpMethod.Post, PubSubUrl) + { + Content = JsonContent.Create(new + { + broadcaster_id = channelId, + target = new[] { InvalidTarget }, + message = "{}", + }), + }; + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _jwtService.CreateExternalServiceToken(channelId)); + request.Headers.Add("Client-Id", TwitchPubSubClient.ClientIdHeader(_options)); + + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + return await ClassifyAsync(response, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + // A check that cannot reach Twitch (or cannot even sign: a malformed secret) says nothing + // about whether Twitch accepts the credentials, and must never take the service down. + _logger.LogWarning(ex, "The Twitch credential check could not be completed; it proves nothing either way."); + return TwitchCredentialProbeResult.Inconclusive; + } + } + + private async Task ClassifyAsync(HttpResponseMessage response, CancellationToken ct) + { + var status = (int)response.StatusCode; + + if (TwitchCredentialStatus.IsCredentialRefusal(status)) + { + var body = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); + _status.MarkRejected(status); + _logger.LogError( + "Twitch REJECTED the EBS's PubSub credentials ({StatusCode}: {Body}). Live updates will NOT reach viewers " + + "until this is fixed. Check Twitch__ExtensionSecret (this extension's own base64 secret), Twitch__ExtensionId " + + "(this extension's Client ID) and Twitch__OwnerUserId (the numeric Twitch user id of the extension's owner).", + status, body); + return TwitchCredentialProbeResult.Rejected; + } + + // 400: the credentials got past verification and only the deliberately invalid target was + // refused, which is the expected answer. 2xx would mean Twitch accepted a message it should not + // have; the credentials are plainly not refused, but it is worth saying. + if (status == 400 || response.IsSuccessStatusCode) + { + _status.MarkAccepted(); + if (response.IsSuccessStatusCode) + _logger.LogWarning("The Twitch credential check was accepted outright ({StatusCode}); it should have been refused for its invalid target.", status); + else + _logger.LogInformation("Twitch did not refuse the EBS's PubSub credentials (the check's invalid target was refused, as intended)."); + return TwitchCredentialProbeResult.Accepted; + } + + _logger.LogWarning("The Twitch credential check got {StatusCode}, which says nothing about the credentials; leaving the status unchanged.", status); + return TwitchCredentialProbeResult.Inconclusive; + } +} diff --git a/src/EDNexus.Ebs/Services/TwitchCredentialStatus.cs b/src/EDNexus.Ebs/Services/TwitchCredentialStatus.cs new file mode 100644 index 0000000..99c9055 --- /dev/null +++ b/src/EDNexus.Ebs/Services/TwitchCredentialStatus.cs @@ -0,0 +1,80 @@ +namespace EDNexus.Ebs.Services; + +/// What the EBS last learned about whether Twitch accepts the credentials it signs PubSub calls with. +public enum TwitchCredentialState +{ + /// Nothing learned yet: the startup check has not run or could not tell. + Unknown = 0, + + /// Twitch accepted the credentials (a real broadcast succeeded, or the startup check was not refused). + Accepted, + + /// + /// Twitch refused the credentials (401/403, typically "JWT could not be verified"): + /// a wrong Twitch__ExtensionSecret, a Twitch__ExtensionId that is not the extension's, + /// or a wrong Twitch__OwnerUserId. Every broadcast fails until it is fixed, so viewers + /// already watching never receive a live update, though new viewers still get the stored card. + /// + Rejected, +} + +/// +/// The EBS's running answer to "does Twitch accept our PubSub credentials?", shared by the startup +/// check, every real broadcast, /healthz and the /api/update-state response. +/// +/// +/// A real broadcast is the authority: it uses exactly the credentials that matter, so a success +/// clears a rejection recorded by the startup check and a refusal sets one. Thread-safe: it is +/// written from request threads and the background check and read from both. +/// +public sealed class TwitchCredentialStatus +{ + /// + /// Response header on a /api/update-state failure caused by Twitch refusing the EBS's + /// credentials. A header, because a proxy in front of the EBS can replace the body of a 502; the + /// desktop app reads it to tell a server misconfiguration from a transient outage. Mirrored by + /// EDNexus.Core.Twitch.StreamStateApiClient.PubSubRejectedHeader. + /// + public const string RejectedHeader = "X-EDNexus-Pubsub-Rejected"; + + private readonly Lock _gate = new(); + private readonly TimeProvider _time; + private TwitchCredentialState _state; + private int? _httpStatus; + private DateTimeOffset? _at; + + public TwitchCredentialStatus(TimeProvider? time = null) => _time = time ?? TimeProvider.System; + + /// A consistent snapshot of the state, the HTTP status behind it, and when it was recorded. + public (TwitchCredentialState State, int? HttpStatus, DateTimeOffset? At) Current + { + get { lock (_gate) return (_state, _httpStatus, _at); } + } + + public TwitchCredentialState State => Current.State; + + /// Records that Twitch accepted the credentials. + public void MarkAccepted() + { + lock (_gate) + { + _state = TwitchCredentialState.Accepted; + _httpStatus = null; + _at = _time.GetUtcNow(); + } + } + + /// Records that Twitch refused the credentials with (401 or 403). + public void MarkRejected(int httpStatus) + { + lock (_gate) + { + _state = TwitchCredentialState.Rejected; + _httpStatus = httpStatus; + _at = _time.GetUtcNow(); + } + } + + /// True for the statuses Twitch uses to refuse a signed token: the credentials, not the payload or Twitch itself. + public static bool IsCredentialRefusal(int httpStatus) => httpStatus is 401 or 403; +} diff --git a/src/EDNexus.Ebs/Services/TwitchPubSubClient.cs b/src/EDNexus.Ebs/Services/TwitchPubSubClient.cs index 07d0e8b..491664b 100644 --- a/src/EDNexus.Ebs/Services/TwitchPubSubClient.cs +++ b/src/EDNexus.Ebs/Services/TwitchPubSubClient.cs @@ -20,6 +20,7 @@ public sealed class TwitchPubSubClient : ITwitchPubSubClient private readonly ITwitchExtensionJwtService _jwtService; private readonly TwitchEbsOptions _twitchOptions; private readonly EbsOptions _ebsOptions; + private readonly TwitchCredentialStatus _credentials; private readonly ILogger _logger; public TwitchPubSubClient( @@ -27,12 +28,14 @@ public TwitchPubSubClient( ITwitchExtensionJwtService jwtService, IOptions twitchOptions, IOptions ebsOptions, + TwitchCredentialStatus credentials, ILogger logger) { _httpClient = httpClient; _jwtService = jwtService; _twitchOptions = twitchOptions.Value; _ebsOptions = ebsOptions.Value; + _credentials = credentials; _logger = logger; } @@ -65,9 +68,15 @@ public async Task BroadcastAsync(string broadcasterId, JsonElement state, broadcasterId, (int)response.StatusCode, body); + // Only a refusal of the signed token says anything about the credentials; a 400 is the + // payload and a 5xx is Twitch, and neither should flip the state. + if (TwitchCredentialStatus.IsCredentialRefusal((int)response.StatusCode)) + _credentials.MarkRejected((int)response.StatusCode); return false; } + // A real broadcast is the authority: it clears a rejection the startup check recorded. + _credentials.MarkAccepted(); return true; } } diff --git a/tests/EDNexus.Ebs.Tests/EbsUnitTests.cs b/tests/EDNexus.Ebs.Tests/EbsUnitTests.cs index aa8d6e0..cb72e49 100644 --- a/tests/EDNexus.Ebs.Tests/EbsUnitTests.cs +++ b/tests/EDNexus.Ebs.Tests/EbsUnitTests.cs @@ -126,6 +126,7 @@ public class TwitchPubSubClientHeaderTests new TwitchExtensionJwtService(Microsoft.Extensions.Options.Options.Create(twitch)), Microsoft.Extensions.Options.Options.Create(twitch), Microsoft.Extensions.Options.Options.Create(new EbsOptions()), + new TwitchCredentialStatus(), NullLogger.Instance); private const string Secret = "c3VwZXItc2VjcmV0LWV4dGVuc2lvbi1rZXktMTIzNA=="; diff --git a/tests/EDNexus.Ebs.Tests/TwitchCredentialCheckTests.cs b/tests/EDNexus.Ebs.Tests/TwitchCredentialCheckTests.cs new file mode 100644 index 0000000..36f0328 --- /dev/null +++ b/tests/EDNexus.Ebs.Tests/TwitchCredentialCheckTests.cs @@ -0,0 +1,346 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using EDNexus.Ebs.Options; +using EDNexus.Ebs.Security; +using EDNexus.Ebs.Services; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; + +namespace EDNexus.Ebs.Tests; + +/// +/// The EBS's answer to "does Twitch accept our PubSub credentials?": the startup check, the status +/// every real broadcast keeps, /healthz, and what a commander's publish gets back. A wrong +/// Twitch__ExtensionSecret used to be completely silent (the card was stored, /healthz said ok, +/// and nothing was ever delivered), which is exactly what these make loud. +/// +public class TwitchCredentialCheckTests +{ + private static TwitchEbsOptions Options(string? secret = EbsHostFactory.ExtensionSecret, string? owner = null) => new() + { + ClientId = "test-client-id", + ExtensionId = "test-extension-id", + ExtensionSecret = secret ?? "", + OwnerUserId = owner, + }; + + private static ITwitchExtensionJwtService Jwt(TwitchEbsOptions options) => new TwitchExtensionJwtService(Microsoft.Extensions.Options.Options.Create(options)); + + private static (TwitchCredentialProbe Probe, TwitchCredentialStatus Status, RecordingHandler Handler) Probe( + HttpStatusCode code, string body = "{}", string? secret = EbsHostFactory.ExtensionSecret, string? owner = null) + { + var options = Options(secret, owner); + var status = new TwitchCredentialStatus(); + var handler = new RecordingHandler(code, body); + var probe = new TwitchCredentialProbe(new HttpClient(handler), Jwt(options), Microsoft.Extensions.Options.Options.Create(options), status, NullLogger.Instance); + return (probe, status, handler); + } + + private static (TwitchPubSubClient Client, TwitchCredentialStatus Status) PubSub(HttpStatusCode code, TwitchCredentialStatus? status = null) + { + var options = Options(); + status ??= new TwitchCredentialStatus(); + var client = new TwitchPubSubClient( + new HttpClient(new RecordingHandler(code, "{}")), Jwt(options), + Microsoft.Extensions.Options.Options.Create(options), Microsoft.Extensions.Options.Options.Create(new EbsOptions()), + status, NullLogger.Instance); + return (client, status); + } + + private static JsonElement State() => JsonDocument.Parse("""{"v":1,"headline":"Docked"}""").RootElement.Clone(); + + // --- the startup check's verdict ----------------------------------------------------------------- + + [Theory] + [InlineData(HttpStatusCode.Forbidden)] // the one seen in production: "JWT could not be verified" + [InlineData(HttpStatusCode.Unauthorized)] + public async Task A_refused_token_is_a_rejection_and_is_recorded(HttpStatusCode code) + { + var (probe, status, _) = Probe(code, """{"error":"Forbidden","status":403,"message":"Error (403): JWT could not be verified"}"""); + + var result = await probe.CheckAsync(CancellationToken.None); + + Assert.Equal(TwitchCredentialProbeResult.Rejected, result); + Assert.Equal(TwitchCredentialState.Rejected, status.State); + Assert.Equal((int)code, status.Current.HttpStatus); + } + + [Fact] + public async Task A_400_means_the_credentials_got_through_and_only_the_bogus_target_was_refused() + { + var (probe, status, _) = Probe(HttpStatusCode.BadRequest, """{"error":"Bad Request","message":"Invalid target"}"""); + + Assert.Equal(TwitchCredentialProbeResult.Accepted, await probe.CheckAsync(CancellationToken.None)); + Assert.Equal(TwitchCredentialState.Accepted, status.State); + } + + [Fact] + public async Task An_outright_success_is_still_not_a_refusal() + { + var (probe, status, _) = Probe(HttpStatusCode.NoContent); + + Assert.Equal(TwitchCredentialProbeResult.Accepted, await probe.CheckAsync(CancellationToken.None)); + Assert.Equal(TwitchCredentialState.Accepted, status.State); + } + + [Theory] + [InlineData(HttpStatusCode.InternalServerError)] + [InlineData(HttpStatusCode.BadGateway)] + [InlineData(HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.NotFound)] + public async Task A_response_that_says_nothing_about_the_credentials_changes_nothing(HttpStatusCode code) + { + var (probe, status, _) = Probe(code); + + Assert.Equal(TwitchCredentialProbeResult.Inconclusive, await probe.CheckAsync(CancellationToken.None)); + Assert.Equal(TwitchCredentialState.Unknown, status.State); + } + + [Fact] + public async Task An_unreachable_Twitch_is_inconclusive_and_never_throws() + { + var options = Options(); + var status = new TwitchCredentialStatus(); + var probe = new TwitchCredentialProbe(new HttpClient(new ThrowingHandler()), Jwt(options), Microsoft.Extensions.Options.Options.Create(options), status, NullLogger.Instance); + + Assert.Equal(TwitchCredentialProbeResult.Inconclusive, await probe.CheckAsync(CancellationToken.None)); + Assert.Equal(TwitchCredentialState.Unknown, status.State); + } + + [Fact] + public async Task A_secret_that_is_not_even_base64_is_inconclusive_and_never_throws() + { + var (probe, status, handler) = Probe(HttpStatusCode.Forbidden, secret: "this is not base64!"); + + Assert.Equal(TwitchCredentialProbeResult.Inconclusive, await probe.CheckAsync(CancellationToken.None)); + Assert.Equal(TwitchCredentialState.Unknown, status.State); + Assert.Equal(0, handler.CallCount); // it could not sign, so it sent nothing + } + + [Fact] + public async Task The_check_cannot_reach_a_viewer_and_is_signed_like_a_real_broadcast() + { + var (probe, _, handler) = Probe(HttpStatusCode.BadRequest, owner: "424242"); + + await probe.CheckAsync(CancellationToken.None); + + var sent = JsonDocument.Parse(Assert.Single(handler.Bodies)).RootElement; + // The point of the design: a target Twitch cannot deliver to, so nothing is ever broadcast. + Assert.Equal(TwitchCredentialProbe.InvalidTarget, Assert.Single(sent.GetProperty("target").EnumerateArray()).GetString()); + Assert.DoesNotContain("\"broadcast\"", sent.GetProperty("target").GetRawText()); + Assert.Equal("424242", sent.GetProperty("broadcaster_id").GetString()); + Assert.Equal("test-extension-id", Assert.Single(handler.ClientIds)); // the same Client-Id a real publish sends + Assert.Equal("https://api.twitch.tv/helix/extensions/pubsub", Assert.Single(handler.Uris)!.ToString()); + } + + // --- real broadcasts keep the status honest ------------------------------------------------------ + + [Theory] + [InlineData(HttpStatusCode.Forbidden)] + [InlineData(HttpStatusCode.Unauthorized)] + public async Task A_real_broadcast_refused_for_its_token_marks_the_credentials_rejected(HttpStatusCode code) + { + var (client, status) = PubSub(code); + + Assert.False(await client.BroadcastAsync("chan", State(), CancellationToken.None)); + + Assert.Equal(TwitchCredentialState.Rejected, status.State); + } + + [Theory] + [InlineData(HttpStatusCode.BadRequest)] // the payload, not the credentials + [InlineData(HttpStatusCode.InternalServerError)] // Twitch, not the credentials + [InlineData(HttpStatusCode.TooManyRequests)] + public async Task Other_failures_say_nothing_about_the_credentials(HttpStatusCode code) + { + var (client, status) = PubSub(code); + + Assert.False(await client.BroadcastAsync("chan", State(), CancellationToken.None)); + + Assert.Equal(TwitchCredentialState.Unknown, status.State); + } + + [Fact] + public async Task A_real_broadcast_that_succeeds_clears_a_rejection_the_startup_check_recorded() + { + var status = new TwitchCredentialStatus(); + status.MarkRejected(403); + var (client, _) = PubSub(HttpStatusCode.NoContent, status); + + Assert.True(await client.BroadcastAsync("chan", State(), CancellationToken.None)); + + Assert.Equal(TwitchCredentialState.Accepted, status.State); + Assert.Null(status.Current.HttpStatus); + } + + [Fact] + public async Task A_rejection_after_a_success_is_noticed_straight_away() + { + var status = new TwitchCredentialStatus(); + var (ok, _) = PubSub(HttpStatusCode.NoContent, status); + await ok.BroadcastAsync("chan", State(), CancellationToken.None); + var (refused, _) = PubSub(HttpStatusCode.Forbidden, status); + + await refused.BroadcastAsync("chan", State(), CancellationToken.None); + + Assert.Equal(TwitchCredentialState.Rejected, status.State); + } + + // --- what the commander's app and the operator see ----------------------------------------------- + + [Fact] + public async Task A_publish_Twitch_refuses_for_the_EBS_credentials_is_a_503_with_the_marker_header() + { + using var factory = new EbsHostFactory(); + var (client, _) = factory.CreateBroadcasterClient("chan-rejected"); + factory.PubSub.RejectFor.Add("chan-rejected"); + factory.Services.GetRequiredService().MarkRejected(403); + + var response = await client.PostAsJsonAsync("/api/update-state", new { state = new { v = 1, headline = "Docked at Home" } }); + + Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode); + Assert.Equal("credentials", Assert.Single(response.Headers.GetValues(TwitchCredentialStatus.RejectedHeader))); + Assert.Contains("credentials", await response.Content.ReadAsStringAsync()); + // The card is still stored, so a viewer who opens the channel gets it: only live pushes are lost. + Assert.Equal(HttpStatusCode.OK, (await factory.CreateClient().GetAsync("/api/initial-state/chan-rejected")).StatusCode); + } + + [Fact] + public async Task An_ordinary_PubSub_failure_is_still_a_plain_502_without_the_marker() + { + using var factory = new EbsHostFactory(); + var (client, _) = factory.CreateBroadcasterClient("chan-flaky"); + factory.PubSub.RejectFor.Add("chan-flaky"); // fails, but Twitch never said the credentials were refused + + var response = await client.PostAsJsonAsync("/api/update-state", new { state = new { v = 1, headline = "Docked at Home" } }); + + Assert.Equal(HttpStatusCode.BadGateway, response.StatusCode); + Assert.False(response.Headers.Contains(TwitchCredentialStatus.RejectedHeader)); + } + + [Fact] + public async Task A_successful_publish_does_not_carry_the_marker() + { + using var factory = new EbsHostFactory(); + var (client, _) = factory.CreateBroadcasterClient("chan-fine"); + + var response = await client.PostAsJsonAsync("/api/update-state", new { state = new { v = 1, headline = "Docked at Home" } }); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.False(response.Headers.Contains(TwitchCredentialStatus.RejectedHeader)); + } + + [Fact] + public async Task Healthz_stays_ok_but_reports_what_is_known_about_the_credentials() + { + using var factory = new EbsHostFactory(); + var http = factory.CreateClient(); + var status = factory.Services.GetRequiredService(); + + var unknown = await ReadHealth(http); + Assert.Equal("ok", unknown.GetProperty("status").GetString()); + Assert.Equal("unknown", unknown.GetProperty("pubsub").GetString()); + + status.MarkRejected(403); + var rejected = await ReadHealth(http); + // Still "ok": a wrong secret must not get the container restarted in a loop, which is what a + // failing health check does and which would not fix it. + Assert.Equal("ok", rejected.GetProperty("status").GetString()); + Assert.Equal("rejected", rejected.GetProperty("pubsub").GetString()); + Assert.Equal(403, rejected.GetProperty("pubsubStatus").GetInt32()); + Assert.NotEqual(JsonValueKind.Null, rejected.GetProperty("pubsubCheckedAt").ValueKind); + + status.MarkAccepted(); + var accepted = await ReadHealth(http); + Assert.Equal("accepted", accepted.GetProperty("pubsub").GetString()); + Assert.Equal(JsonValueKind.Null, accepted.GetProperty("pubsubStatus").ValueKind); + } + + private static async Task ReadHealth(HttpClient http) + { + var response = await http.GetAsync("/healthz"); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + return JsonDocument.Parse(await response.Content.ReadAsStringAsync()).RootElement.Clone(); + } + + // --- the background service ---------------------------------------------------------------------- + + [Fact] + public async Task One_run_asks_the_probe_once() + { + var probe = new CountingProbe(); + var service = new TwitchCredentialCheckBackgroundService(probe, new TestEnvironment("Production"), TimeProvider.System, NullLogger.Instance); + + await service.RunOnceAsync(CancellationToken.None); + + Assert.Equal(1, probe.Calls); + } + + [Fact] + public async Task A_probe_that_throws_cannot_take_the_host_down() + { + var probe = new CountingProbe { Throw = true }; + var service = new TwitchCredentialCheckBackgroundService(probe, new TestEnvironment("Production"), TimeProvider.System, NullLogger.Instance); + + await service.RunOnceAsync(CancellationToken.None); // does not throw + + Assert.Equal(1, probe.Calls); + } + + [Fact] + public async Task Under_the_test_host_it_never_reaches_out_at_all() + { + var probe = new CountingProbe(); + var service = new TwitchCredentialCheckBackgroundService(probe, new TestEnvironment("Testing"), TimeProvider.System, NullLogger.Instance); + + await service.StartAsync(CancellationToken.None); + await service.ExecuteTask!; + + Assert.Equal(0, probe.Calls); + } + + [Fact] + public async Task Shutting_down_before_the_first_check_stops_it_cleanly_without_asking_Twitch() + { + var probe = new CountingProbe(); + var service = new TwitchCredentialCheckBackgroundService(probe, new TestEnvironment("Production"), TimeProvider.System, NullLogger.Instance); + + await service.StartAsync(CancellationToken.None); + await service.StopAsync(CancellationToken.None); + + Assert.Equal(0, probe.Calls); // the 10 s initial delay had not passed + } + + // --- helpers ------------------------------------------------------------------------------------- + + private sealed class ThrowingHandler : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + => throw new HttpRequestException("network is down"); + } + + private sealed class CountingProbe : ITwitchCredentialProbe + { + public int Calls; + public bool Throw; + + public Task CheckAsync(CancellationToken cancellationToken) + { + Calls++; + if (Throw) throw new InvalidOperationException("boom"); + return Task.FromResult(TwitchCredentialProbeResult.Inconclusive); + } + } + + private sealed class TestEnvironment(string name) : IHostEnvironment + { + public string EnvironmentName { get; set; } = name; + public string ApplicationName { get; set; } = "test"; + public string ContentRootPath { get; set; } = ""; + public IFileProvider ContentRootFileProvider { get; set; } = new NullFileProvider(); + } +} diff --git a/tests/EDNexus.Tests/Twitch/StreamCardWarningTests.cs b/tests/EDNexus.Tests/Twitch/StreamCardWarningTests.cs new file mode 100644 index 0000000..974f736 --- /dev/null +++ b/tests/EDNexus.Tests/Twitch/StreamCardWarningTests.cs @@ -0,0 +1,40 @@ +using EDNexus.Core.Twitch; +using Xunit; + +namespace EDNexus.Tests.Twitch; + +public class StreamCardWarningTests +{ + private static StreamStatePublishResult Result(StreamStatePublishStatus status) => new(status, "x"); + + [Fact] + public void Twitch_refusing_the_backends_credentials_raises_a_warning_that_says_it_is_not_the_commanders_fault() + { + var warning = StreamCardWarning.For(Result(StreamStatePublishStatus.BackendRejectedByTwitch)); + + Assert.NotNull(warning); + Assert.Contains("credentials", warning); + Assert.Contains("live updates", warning); + Assert.Contains("not with your sign-in", warning); + } + + [Theory] + [InlineData(StreamStatePublishStatus.Published)] + [InlineData(StreamStatePublishStatus.Cleared)] + [InlineData(StreamStatePublishStatus.Unauthorized)] // has its own, different, prompt + [InlineData(StreamStatePublishStatus.TooLarge)] + [InlineData(StreamStatePublishStatus.RateLimited)] + [InlineData(StreamStatePublishStatus.Failed)] // a dropped connection is not a banner + [InlineData(StreamStatePublishStatus.ClearedNotDelivered)] + public void Nothing_else_raises_one(StreamStatePublishStatus status) + => Assert.Null(StreamCardWarning.For(Result(status))); + + [Theory] + [InlineData(StreamStatePublishStatus.Published, true)] + [InlineData(StreamStatePublishStatus.Cleared, true)] + [InlineData(StreamStatePublishStatus.Failed, false)] // proves nothing either way + [InlineData(StreamStatePublishStatus.RateLimited, false)] + [InlineData(StreamStatePublishStatus.BackendRejectedByTwitch, false)] + public void Only_a_success_clears_a_shown_warning(StreamStatePublishStatus status, bool clears) + => Assert.Equal(clears, StreamCardWarning.Clears(Result(status))); +} diff --git a/tests/EDNexus.Tests/Twitch/StreamStateApiClientTests.cs b/tests/EDNexus.Tests/Twitch/StreamStateApiClientTests.cs index 1d1ff8b..f8bcd61 100644 --- a/tests/EDNexus.Tests/Twitch/StreamStateApiClientTests.cs +++ b/tests/EDNexus.Tests/Twitch/StreamStateApiClientTests.cs @@ -70,6 +70,40 @@ public async Task An_unreachable_ebs_is_reported_not_thrown() Assert.Contains("no route to host", result.Error); } + [Fact] + public async Task Twitch_refusing_the_backends_credentials_is_told_apart_from_an_ordinary_failure() + { + // The EBS marks it with a header (a proxy can rewrite a 502's body) on a 503. + var refused = new StubHandler(_ => + { + var response = new HttpResponseMessage(HttpStatusCode.ServiceUnavailable); + response.Headers.Add(StreamStateApiClient.PubSubRejectedHeader, "credentials"); + return Task.FromResult(response); + }); + using (var client = new StreamStateApiClient(new HttpClient(refused))) + { + var result = await client.PublishAsync(Endpoint, "ebs-token", Snapshot()); + + Assert.Equal(StreamStatePublishStatus.BackendRejectedByTwitch, result.Status); + Assert.False(result.IsSuccess); + // Not the commander's sign-in: re-login is the wrong advice and must not stop publishing. + Assert.False(result.RequiresReauth); + } + + // The same status with no marker is just a failure, as before. + var plain = new StubHandler(_ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable))); + using var plainClient = new StreamStateApiClient(new HttpClient(plain)); + Assert.Equal(StreamStatePublishStatus.Failed, (await plainClient.PublishAsync(Endpoint, "ebs-token", Snapshot())).Status); + } + + [Fact] + public void The_header_names_match_what_the_EBS_sends() + { + // Mirrored constants: the two projects cannot reference each other, so a typo would silently + // turn every credential problem back into an anonymous failure. + Assert.Equal("X-EDNexus-Pubsub-Rejected", StreamStateApiClient.PubSubRejectedHeader); + } + [Fact] public async Task A_clear_the_EBS_confirms_removed_is_told_apart_from_a_proxy_502() { diff --git a/tests/EDNexus.Tests/Twitch/TwitchStreamCardServiceTests.cs b/tests/EDNexus.Tests/Twitch/TwitchStreamCardServiceTests.cs index 2a8982c..f865312 100644 --- a/tests/EDNexus.Tests/Twitch/TwitchStreamCardServiceTests.cs +++ b/tests/EDNexus.Tests/Twitch/TwitchStreamCardServiceTests.cs @@ -468,6 +468,46 @@ public async Task A_failed_publish_is_retried_without_waiting_for_a_change() Assert.True(attempts >= 3, $"Expected the pump to retry unprompted; it stopped after {attempts} attempt(s)."); } + [Fact] + public async Task A_backend_whose_credentials_Twitch_refuses_is_retried_and_recovers_once_fixed() + { + var state = new CommanderState { StarSystem = "Nervi" }; + var attempts = 0; + var client = new FakeStreamStateApiClient(); + // The operator fixes the secret while the app is running: no commander action is needed, so + // the pump must keep coming back by itself and the card must start updating again. + client.Respond = _ => ++attempts < 3 + ? new StreamStatePublishResult(StreamStatePublishStatus.BackendRejectedByTwitch, "HTTP 503") + : StreamStatePublishResult.Ok; + + using var service = Create(state, client); + service.RequestPublish(); + + var deadline = DateTime.UtcNow.AddSeconds(10); + while (DateTime.UtcNow < deadline && attempts < 3) + await Task.Delay(FastInterval * 2); + + Assert.True(attempts >= 3, $"Expected the pump to keep retrying a backend fault; it stopped after {attempts} attempt(s)."); + } + + [Fact] + public async Task A_backend_fault_does_not_stop_publishing_the_way_a_revoked_token_does() + { + var state = new CommanderState { StarSystem = "Nervi" }; + var reauth = 0; + var client = new FakeStreamStateApiClient + { + Respond = _ => new StreamStatePublishResult(StreamStatePublishStatus.BackendRejectedByTwitch, "HTTP 503"), + }; + + using var service = Create(state, client); + service.ReauthRequired += () => reauth++; + service.RequestPublish(); + await Task.Delay(FastInterval * 6); + + Assert.Equal(0, reauth); // the commander's sign-in is not the problem + } + [Fact] public async Task Retries_are_bounded_so_a_dead_ebs_is_not_polled_forever() { From 7693081d81ebb651d6e28a902003a3242e1110e0 Mon Sep 17 00:00:00 2001 From: Kevin Dethlefs Date: Sun, 18 Oct 2026 13:26:15 -0500 Subject: [PATCH 2/2] Address review: classify each broadcast by its own result; keep retrying a backend fault - BroadcastAsync now returns PubSubBroadcastResult (Published, CredentialsRejected, Failed) and the update-state endpoint branches on THIS call's result. It used to read the shared credential status, which stays 'rejected' after an earlier 403, so a later 400 or Twitch outage was reported as a credentials problem (503 + marker) instead of a plain 502. The shared status is unchanged and still feeds /healthz. - A BackendRejectedByTwitch publish no longer counts against the bounded retry budget: it retries indefinitely, but slowly (24x the publish floor, capped at MaxRetryAfter: two minutes at the default floor). With no earlier successful publish even the heartbeat wake is skipped, so giving up after five tries left an idle commander's card stale until something changed, contradicting the on-screen warning's 'it keeps retrying'. Ordinary failures stay bounded. Both new regression tests were shown to fail on the previous behaviour. EDNexus.Ebs.Tests 214 (+2), EDNexus.Tests 1126 (+2); full solution twice, no failures. Co-Authored-By: Claude Sonnet 5.5 --- .../Twitch/TwitchStreamCardService.cs | 25 ++++++++++- src/EDNexus.Ebs/Program.cs | 13 +++--- .../Services/ChannelStateClearing.cs | 2 +- .../Services/ITwitchPubSubClient.cs | 21 ++++++++- .../Services/TwitchPubSubClient.cs | 10 +++-- tests/EDNexus.Ebs.Tests/EbsHardeningTests.cs | 4 +- tests/EDNexus.Ebs.Tests/EbsUnitTests.cs | 2 +- .../EDNexus.Ebs.Tests/OAuthEndpointsTests.cs | 2 +- .../TwitchCredentialCheckTests.cs | 40 ++++++++++++++--- .../UpdateStateEndpointTests.cs | 10 +++-- .../Twitch/TwitchStreamCardServiceTests.cs | 43 +++++++++++++++++++ 11 files changed, 147 insertions(+), 25 deletions(-) diff --git a/src/EDNexus.Core/Twitch/TwitchStreamCardService.cs b/src/EDNexus.Core/Twitch/TwitchStreamCardService.cs index fa422c1..9c48c0d 100644 --- a/src/EDNexus.Core/Twitch/TwitchStreamCardService.cs +++ b/src/EDNexus.Core/Twitch/TwitchStreamCardService.cs @@ -193,6 +193,15 @@ internal TwitchStreamCardService( // the EBS's initial state. The host calls RequestPublish() once the replay is done. } + /// + /// How long to wait between retries while the backend's Twitch credentials are being refused: + /// 24 times the publish floor, capped at (so two minutes at the default + /// five-second floor). Scaled from the floor rather than fixed so a test with a tiny floor does not + /// have to wait minutes. + /// + internal TimeSpan BackendFaultRetryInterval => + TimeSpan.FromTicks(Math.Min(MaxRetryAfter.Ticks, _minInterval.Ticks * 24)); + /// The floor between two publishes this service was built with. public TimeSpan MinInterval => _minInterval; @@ -300,7 +309,21 @@ private async Task PumpAsync(CancellationToken ct) // Capped, so a misconfigured proxy's header cannot silence the card for the session. if (published.RetryAfter is { } wait && wait > backoff) backoff = wait < MaxRetryAfter ? wait : MaxRetryAfter; - retry = ++_consecutiveFailures <= MaxPublishRetries; + if (published.Status == StreamStatePublishStatus.BackendRejectedByTwitch) + { + // A backend fault lasts as long as it takes an operator to fix a secret: minutes or + // hours, not the seconds a bounded retry covers. The warning on screen promises it keeps + // retrying, and with no earlier success even the heartbeat wake is skipped, so giving up + // would leave an idle commander's card stale until something changed. So retry for as long + // as it takes, slowly (one small request every BackendFaultRetryInterval), and outside + // the bounded budget that stops a dead EBS from being polled forever. + if (backoff < BackendFaultRetryInterval) backoff = BackendFaultRetryInterval; + retry = true; + } + else + { + retry = ++_consecutiveFailures <= MaxPublishRetries; + } } else if (published is not null) { diff --git a/src/EDNexus.Ebs/Program.cs b/src/EDNexus.Ebs/Program.cs index fbc476c..279784e 100644 --- a/src/EDNexus.Ebs/Program.cs +++ b/src/EDNexus.Ebs/Program.cs @@ -336,7 +336,6 @@ await Results.Problem( UpdateStateRequest body, IChannelStateStore stateStore, ITwitchPubSubClient pubSubClient, - TwitchCredentialStatus credentials, IBroadcasterTokenStore tokenStore, ChannelLocks channelLocks, IOptions ebsOptions, @@ -371,7 +370,7 @@ await Results.Problem( // not the request's original whitespace, which the limit never saw. using var compactState = JsonDocument.Parse(pubSubRequest.Message); - bool published; + PubSubBroadcastResult broadcast; using (await channelLocks.AcquireAsync(channelId, cancellationToken).ConfigureAwait(false)) { // Authenticated again now that this channel is ours alone: a sign-out (POST /oauth/revoke) that @@ -384,18 +383,20 @@ await Results.Problem( } stateStore.Set(channelId, compactState.RootElement); - published = await pubSubClient.BroadcastAsync(channelId, compactState.RootElement, cancellationToken).ConfigureAwait(false); + broadcast = await pubSubClient.BroadcastAsync(channelId, compactState.RootElement, cancellationToken).ConfigureAwait(false); } - if (!published) + if (broadcast != PubSubBroadcastResult.Published) { logger.LogWarning("Failed to publish state update for channel {ChannelId} to Twitch PubSub.", channelId); // Twitch refusing the EBS's own credentials is a server misconfiguration, not an outage, // and no amount of retrying fixes it. Say so with a header (a proxy in front can replace // a 502's body) and a 503 (which one does not rewrite), so the desktop app can tell the - // commander it is not their sign-in, instead of a bare "502". - if (credentials.State == TwitchCredentialState.Rejected) + // commander it is not their sign-in, instead of a bare "502". Decided from THIS call's + // outcome, not the shared status: that stays "rejected" after an earlier refusal, and a + // later 400 or Twitch outage must not be reported as a credentials problem. + if (broadcast == PubSubBroadcastResult.CredentialsRejected) { httpRequest.HttpContext.Response.Headers[TwitchCredentialStatus.RejectedHeader] = "credentials"; return Results.Problem( diff --git a/src/EDNexus.Ebs/Services/ChannelStateClearing.cs b/src/EDNexus.Ebs/Services/ChannelStateClearing.cs index a0a556e..ba4cc50 100644 --- a/src/EDNexus.Ebs/Services/ChannelStateClearing.cs +++ b/src/EDNexus.Ebs/Services/ChannelStateClearing.cs @@ -33,7 +33,7 @@ public static class ChannelStateClearing public static async Task ClearAsync(string channelId, IChannelStateStore stateStore, ITwitchPubSubClient pubSubClient, CancellationToken ct) { stateStore.Remove(channelId); - try { return await pubSubClient.BroadcastAsync(channelId, OfflineMessage, ct).ConfigureAwait(false); } + try { return await pubSubClient.BroadcastAsync(channelId, OfflineMessage, ct).ConfigureAwait(false) == PubSubBroadcastResult.Published; } catch (Exception ex) when (ex is not OperationCanceledException) { return false; } } } diff --git a/src/EDNexus.Ebs/Services/ITwitchPubSubClient.cs b/src/EDNexus.Ebs/Services/ITwitchPubSubClient.cs index a9adfa7..3874b0a 100644 --- a/src/EDNexus.Ebs/Services/ITwitchPubSubClient.cs +++ b/src/EDNexus.Ebs/Services/ITwitchPubSubClient.cs @@ -2,6 +2,23 @@ namespace EDNexus.Ebs.Services; +/// How a single PubSub broadcast ended. +public enum PubSubBroadcastResult +{ + /// Twitch accepted the message (HTTP 204). + Published, + + /// + /// Twitch refused the credentials the call was signed with (401/403): a server + /// misconfiguration that no retry fixes. Reported per call so a caller never has to infer it from + /// shared state, which can be left over from an earlier call. + /// + CredentialsRejected, + + /// Anything else: a bad payload, a Twitch outage, a rate limit. + Failed, +} + /// Broadcasts state payloads to viewers of a channel via Twitch's Extensions PubSub API. public interface ITwitchPubSubClient { @@ -9,6 +26,6 @@ public interface ITwitchPubSubClient /// Sends to every viewer of currently /// listening on the extension's "broadcast" PubSub target. /// - /// True when Twitch accepted the message (HTTP 204). - Task BroadcastAsync(string broadcasterId, JsonElement state, CancellationToken cancellationToken); + /// How this call ended; when Twitch accepted the message (HTTP 204). + Task BroadcastAsync(string broadcasterId, JsonElement state, CancellationToken cancellationToken); } diff --git a/src/EDNexus.Ebs/Services/TwitchPubSubClient.cs b/src/EDNexus.Ebs/Services/TwitchPubSubClient.cs index 491664b..00e3882 100644 --- a/src/EDNexus.Ebs/Services/TwitchPubSubClient.cs +++ b/src/EDNexus.Ebs/Services/TwitchPubSubClient.cs @@ -44,7 +44,7 @@ internal static string ClientIdHeader(TwitchEbsOptions options) => string.IsNullOrWhiteSpace(options.ExtensionId) ? options.ClientId : options.ExtensionId; /// - public async Task BroadcastAsync(string broadcasterId, JsonElement state, CancellationToken cancellationToken) + public async Task BroadcastAsync(string broadcasterId, JsonElement state, CancellationToken cancellationToken) { var request = PubSubBroadcastRequest.Create(broadcasterId, state, _ebsOptions.MaxStatePayloadBytes); var token = _jwtService.CreateExternalServiceToken(broadcasterId); @@ -71,12 +71,16 @@ public async Task BroadcastAsync(string broadcasterId, JsonElement state, // Only a refusal of the signed token says anything about the credentials; a 400 is the // payload and a 5xx is Twitch, and neither should flip the state. if (TwitchCredentialStatus.IsCredentialRefusal((int)response.StatusCode)) + { _credentials.MarkRejected((int)response.StatusCode); - return false; + return PubSubBroadcastResult.CredentialsRejected; + } + + return PubSubBroadcastResult.Failed; } // A real broadcast is the authority: it clears a rejection the startup check recorded. _credentials.MarkAccepted(); - return true; + return PubSubBroadcastResult.Published; } } diff --git a/tests/EDNexus.Ebs.Tests/EbsHardeningTests.cs b/tests/EDNexus.Ebs.Tests/EbsHardeningTests.cs index 5404e3b..7a694c9 100644 --- a/tests/EDNexus.Ebs.Tests/EbsHardeningTests.cs +++ b/tests/EDNexus.Ebs.Tests/EbsHardeningTests.cs @@ -474,7 +474,7 @@ private sealed class GatedOfflinePubSubClient(string gatedChannel) : ITwitchPubS public ManualResetEventSlim OfflineEntered { get; } = new(); public TaskCompletionSource ReleaseOffline { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); - public async Task BroadcastAsync(string broadcasterId, JsonElement state, CancellationToken cancellationToken) + public async Task BroadcastAsync(string broadcasterId, JsonElement state, CancellationToken cancellationToken) { if (broadcasterId == gatedChannel && state.TryGetProperty("offline", out _)) { @@ -482,7 +482,7 @@ public async Task BroadcastAsync(string broadcasterId, JsonElement state, await ReleaseOffline.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); } - return true; + return PubSubBroadcastResult.Published; } } } diff --git a/tests/EDNexus.Ebs.Tests/EbsUnitTests.cs b/tests/EDNexus.Ebs.Tests/EbsUnitTests.cs index cb72e49..7b80095 100644 --- a/tests/EDNexus.Ebs.Tests/EbsUnitTests.cs +++ b/tests/EDNexus.Ebs.Tests/EbsUnitTests.cs @@ -137,7 +137,7 @@ public async Task The_Client_Id_header_is_the_extensions_id_not_the_oauth_applic var handler = new RecordingHandler(HttpStatusCode.NoContent); var client = Create(handler, new TwitchEbsOptions { ClientId = "oauth-app", ExtensionId = "the-extension", ExtensionSecret = Secret }); - Assert.True(await client.BroadcastAsync("123", JsonSerializer.SerializeToElement(new { v = 1 }), CancellationToken.None)); + Assert.Equal(PubSubBroadcastResult.Published, await client.BroadcastAsync("123", JsonSerializer.SerializeToElement(new { v = 1 }), CancellationToken.None)); Assert.Equal("the-extension", Assert.Single(handler.ClientIds)); } diff --git a/tests/EDNexus.Ebs.Tests/OAuthEndpointsTests.cs b/tests/EDNexus.Ebs.Tests/OAuthEndpointsTests.cs index e425841..17bf10d 100644 --- a/tests/EDNexus.Ebs.Tests/OAuthEndpointsTests.cs +++ b/tests/EDNexus.Ebs.Tests/OAuthEndpointsTests.cs @@ -65,7 +65,7 @@ protected override void ConfigureWebHost(IWebHostBuilder builder) /// Fake PubSub relay — the OAuth flow tests only care that authentication succeeded, not that Twitch PubSub itself is reachable. public sealed class FakePubSubClient : ITwitchPubSubClient { - public Task BroadcastAsync(string broadcasterId, JsonElement state, CancellationToken cancellationToken) => Task.FromResult(true); + public Task BroadcastAsync(string broadcasterId, JsonElement state, CancellationToken cancellationToken) => Task.FromResult(PubSubBroadcastResult.Published); } public sealed class FakeTwitchOAuthClient : ITwitchOAuthClient diff --git a/tests/EDNexus.Ebs.Tests/TwitchCredentialCheckTests.cs b/tests/EDNexus.Ebs.Tests/TwitchCredentialCheckTests.cs index 36f0328..559b48e 100644 --- a/tests/EDNexus.Ebs.Tests/TwitchCredentialCheckTests.cs +++ b/tests/EDNexus.Ebs.Tests/TwitchCredentialCheckTests.cs @@ -146,7 +146,7 @@ public async Task A_real_broadcast_refused_for_its_token_marks_the_credentials_r { var (client, status) = PubSub(code); - Assert.False(await client.BroadcastAsync("chan", State(), CancellationToken.None)); + Assert.Equal(PubSubBroadcastResult.CredentialsRejected, await client.BroadcastAsync("chan", State(), CancellationToken.None)); Assert.Equal(TwitchCredentialState.Rejected, status.State); } @@ -159,7 +159,8 @@ public async Task Other_failures_say_nothing_about_the_credentials(HttpStatusCod { var (client, status) = PubSub(code); - Assert.False(await client.BroadcastAsync("chan", State(), CancellationToken.None)); + // An ordinary failure, reported as one: never as a credentials problem. + Assert.Equal(PubSubBroadcastResult.Failed, await client.BroadcastAsync("chan", State(), CancellationToken.None)); Assert.Equal(TwitchCredentialState.Unknown, status.State); } @@ -171,12 +172,26 @@ public async Task A_real_broadcast_that_succeeds_clears_a_rejection_the_startup_ status.MarkRejected(403); var (client, _) = PubSub(HttpStatusCode.NoContent, status); - Assert.True(await client.BroadcastAsync("chan", State(), CancellationToken.None)); + Assert.Equal(PubSubBroadcastResult.Published, await client.BroadcastAsync("chan", State(), CancellationToken.None)); Assert.Equal(TwitchCredentialState.Accepted, status.State); Assert.Null(status.Current.HttpStatus); } + [Fact] + public async Task An_ordinary_failure_after_a_refusal_is_reported_as_one_and_does_not_change_the_status() + { + var status = new TwitchCredentialStatus(); + status.MarkRejected(403); + var (client, _) = PubSub(HttpStatusCode.InternalServerError, status); + + Assert.Equal(PubSubBroadcastResult.Failed, await client.BroadcastAsync("chan", State(), CancellationToken.None)); + + // Still what the last real refusal said: a 500 proves nothing about the credentials either way. + Assert.Equal(TwitchCredentialState.Rejected, status.State); + Assert.Equal(403, status.Current.HttpStatus); + } + [Fact] public async Task A_rejection_after_a_success_is_noticed_straight_away() { @@ -197,8 +212,7 @@ public async Task A_publish_Twitch_refuses_for_the_EBS_credentials_is_a_503_with { using var factory = new EbsHostFactory(); var (client, _) = factory.CreateBroadcasterClient("chan-rejected"); - factory.PubSub.RejectFor.Add("chan-rejected"); - factory.Services.GetRequiredService().MarkRejected(403); + factory.PubSub.RejectCredentialsFor.Add("chan-rejected"); // Twitch refuses the EBS's credentials for this call var response = await client.PostAsJsonAsync("/api/update-state", new { state = new { v = 1, headline = "Docked at Home" } }); @@ -222,6 +236,22 @@ public async Task An_ordinary_PubSub_failure_is_still_a_plain_502_without_the_ma Assert.False(response.Headers.Contains(TwitchCredentialStatus.RejectedHeader)); } + [Fact] + public async Task A_stale_rejection_does_not_make_a_later_ordinary_failure_look_like_a_credentials_problem() + { + using var factory = new EbsHostFactory(); + var (client, _) = factory.CreateBroadcasterClient("chan-stale"); + // Some earlier call was refused, so the shared status says "rejected"... + factory.Services.GetRequiredService().MarkRejected(403); + // ...but THIS broadcast fails for an ordinary reason (a Twitch outage, a 400). + factory.PubSub.RejectFor.Add("chan-stale"); + + var response = await client.PostAsJsonAsync("/api/update-state", new { state = new { v = 1, headline = "Docked at Home" } }); + + Assert.Equal(HttpStatusCode.BadGateway, response.StatusCode); + Assert.False(response.Headers.Contains(TwitchCredentialStatus.RejectedHeader)); + } + [Fact] public async Task A_successful_publish_does_not_carry_the_marker() { diff --git a/tests/EDNexus.Ebs.Tests/UpdateStateEndpointTests.cs b/tests/EDNexus.Ebs.Tests/UpdateStateEndpointTests.cs index 83c519f..72926f8 100644 --- a/tests/EDNexus.Ebs.Tests/UpdateStateEndpointTests.cs +++ b/tests/EDNexus.Ebs.Tests/UpdateStateEndpointTests.cs @@ -335,13 +335,17 @@ public sealed class FakeTwitchPubSubClient : ITwitchPubSubClient public Dictionary LastMessage { get; } = new(); - /// Channels whose broadcasts Twitch rejects. + /// Channels whose broadcasts fail for an ordinary reason (a bad payload, a Twitch outage). public HashSet RejectFor { get; } = new(); - public Task BroadcastAsync(string broadcasterId, JsonElement state, CancellationToken cancellationToken) + /// Channels whose broadcasts fail because Twitch refuses the EBS's own credentials. + public HashSet RejectCredentialsFor { get; } = new(); + + public Task BroadcastAsync(string broadcasterId, JsonElement state, CancellationToken cancellationToken) { BroadcastCalls[broadcasterId] = BroadcastCalls.GetValueOrDefault(broadcasterId) + 1; LastMessage[broadcasterId] = state.Clone(); - return Task.FromResult(!RejectFor.Contains(broadcasterId)); + if (RejectCredentialsFor.Contains(broadcasterId)) return Task.FromResult(PubSubBroadcastResult.CredentialsRejected); + return Task.FromResult(RejectFor.Contains(broadcasterId) ? PubSubBroadcastResult.Failed : PubSubBroadcastResult.Published); } } diff --git a/tests/EDNexus.Tests/Twitch/TwitchStreamCardServiceTests.cs b/tests/EDNexus.Tests/Twitch/TwitchStreamCardServiceTests.cs index f865312..2a06399 100644 --- a/tests/EDNexus.Tests/Twitch/TwitchStreamCardServiceTests.cs +++ b/tests/EDNexus.Tests/Twitch/TwitchStreamCardServiceTests.cs @@ -490,6 +490,49 @@ public async Task A_backend_whose_credentials_Twitch_refuses_is_retried_and_reco Assert.True(attempts >= 3, $"Expected the pump to keep retrying a backend fault; it stopped after {attempts} attempt(s)."); } + [Fact] + public async Task A_backend_fault_keeps_being_retried_past_the_cap_so_an_idle_card_recovers_when_it_is_fixed() + { + var state = new CommanderState { StarSystem = "Nervi" }; + var attempts = 0; + var fixedAt = TwitchStreamCardService.MaxPublishRetries + 4; // well past the bounded budget + var client = new FakeStreamStateApiClient(); + // No successful publish ever happened this session, so the heartbeat is skipped too: the retry + // loop is the card's only way back, and an operator fixing the secret takes far longer than + // the handful of retries an ordinary failure gets. + client.Respond = _ => ++attempts < fixedAt + ? new StreamStatePublishResult(StreamStatePublishStatus.BackendRejectedByTwitch, "HTTP 503") + : StreamStatePublishResult.Ok; + + using var service = new TwitchStreamCardService( + state, StreamCardSources.Empty, client, static () => Endpoint, static () => "ebs-token", + visibility: null, isSuppressed: null, minInterval: TimeSpan.FromMilliseconds(5), clock: null); + service.RequestPublish(); + + var deadline = DateTime.UtcNow.AddSeconds(15); + while (DateTime.UtcNow < deadline && attempts < fixedAt) + await Task.Delay(50); + + Assert.True(attempts >= fixedAt, + $"Expected retries to continue past the cap until the backend recovered; stopped after {attempts} attempt(s)."); + } + + [Fact] + public void The_backend_fault_retry_is_slow_in_real_use_and_never_longer_than_the_retry_ceiling() + { + var state = new CommanderState(); + using var realistic = new TwitchStreamCardService( + state, StreamCardSources.Empty, new FakeStreamStateApiClient(), static () => Endpoint, static () => "t"); + using var tiny = new TwitchStreamCardService( + state, StreamCardSources.Empty, new FakeStreamStateApiClient(), static () => Endpoint, static () => "t", + visibility: null, isSuppressed: null, minInterval: TimeSpan.FromMilliseconds(5), clock: null); + + // At the default five-second floor: two minutes, i.e. one small request per two minutes. + Assert.Equal(TwitchStreamCardService.MaxRetryAfter, realistic.BackendFaultRetryInterval); + Assert.Equal(TimeSpan.FromMilliseconds(120), tiny.BackendFaultRetryInterval); + Assert.True(realistic.BackendFaultRetryInterval > realistic.MinInterval * 2, "must be slower than an ordinary retry"); + } + [Fact] public async Task A_backend_fault_does_not_stop_publishing_the_way_a_revoked_token_does() {