From 5aa84c45e58113674786f2066e7e3ff118744278 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 13:08:57 +0000 Subject: [PATCH 1/8] feat: sync .NET client with Apify OpenAPI spec v2-2026-10-20T121847Z Add RunChargeOptions.RequestId, document rental pricing removal, and fix webhook integration tests to use an owned Actor as the condition. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01UUMWnU4gH573HweHieMhHi --- CHANGELOG.md | 11 ++++++++ docs/misc.md | 2 +- docs/runs.md | 1 + src/Apify.Client/Apify.Client.csproj | 2 +- src/Apify.Client/ApifyClientVersion.cs | 4 +-- src/Apify.Client/Options/RunChargeOptions.cs | 7 +++++ src/Apify.Client/Options/StoreListOptions.cs | 4 +-- src/Apify.Client/Resources/RunClient.cs | 5 ++++ .../Integration/IntegrationTestBase.cs | 15 ++++++++--- .../Integration/IterationIntegrationTests.cs | 4 ++- .../Integration/WebhookIntegrationTests.cs | 26 ++++++++++++++++--- .../Unit/RequestShapeTests.cs | 13 ++++++++++ 12 files changed, 80 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8406474..d715a07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 0.7.0 + +- Bumped `ApifyClientVersion.ApiSpecVersion` to the Apify OpenAPI spec `v2-2026-10-20T121847Z` and + the project version to `0.7.0`. +- Added `RequestId` to `RunChargeOptions`, sent as `requestId` in the charge request body; it identifies + the Server Actor request being charged (required for multi-tenant Server Actor runs). +- Documented that the rental pricing model (`FLAT_PRICE_PER_MONTH`) is no longer supported and is not an + accepted `StoreListOptions.PricingModel` filter value. +- Fixed the webhook integration tests: the API now requires a webhook condition to reference an existing + resource, so they use a throwaway Actor owned by the test account instead of a made-up run ID. + ## 0.6.0 - Bumped `ApifyClientVersion.ApiSpecVersion` to the Apify OpenAPI spec `v2-2026-10-19T133704Z` and diff --git a/docs/misc.md b/docs/misc.md index 3081f9d..eaaa2dd 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -21,7 +21,7 @@ Browse public Actors in the [Apify Store](https://apify.com/store). | `SortBy` | `string?` | Field to sort by (e.g. `popularity`, `newest`). | | `Category` | `string?` | Restrict results to a Store category. | | `Username` | `string?` | Restrict results to a given owner's Actors. | -| `PricingModel` | `string?` | Filter by pricing model (`FREE`, `FLAT_PRICE_PER_MONTH`, `PRICE_PER_DATASET_ITEM`, …). | +| `PricingModel` | `string?` | Filter by pricing model (`FREE`, `PRICE_PER_DATASET_ITEM`, `PAY_PER_EVENT`; the rental model `FLAT_PRICE_PER_MONTH` is no longer supported). | | `IncludeUnrunnableActors` | `bool?` | Include Actors that cannot currently be run. | | `AllowsAgenticUsers` | `bool?` | Only Actors that permit agentic (automated) users. | | `ResponseFormat` | `string?` | Requested response format. | diff --git a/docs/runs.md b/docs/runs.md index 75be828..6b6901a 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -53,6 +53,7 @@ and `RestartOnError` (`bool?`). See [Actors](actors.md) for each field's meaning | `EventName` | `string` | Name of the pay-per-event event to charge for (required, non-empty). | | `Count` | `int?` | Number of event occurrences to charge (defaults to 1 server-side). | | `IdempotencyKey` | `string?` | Key that deduplicates retried charges; auto-generated when omitted. | +| `RequestId` | `string?` | ID of the Server Actor request being charged (the `X-Actor-Request-ID` header of the incoming request); required for multi-tenant Server Actor runs. Set via object initializer. | ```csharp using Apify.Client; diff --git a/src/Apify.Client/Apify.Client.csproj b/src/Apify.Client/Apify.Client.csproj index 2c789f3..ee43fae 100644 --- a/src/Apify.Client/Apify.Client.csproj +++ b/src/Apify.Client/Apify.Client.csproj @@ -6,7 +6,7 @@ Apify.Client - 0.6.0 + 0.7.0 Apify Apify Apify API client for .NET diff --git a/src/Apify.Client/ApifyClientVersion.cs b/src/Apify.Client/ApifyClientVersion.cs index 4fea34c..3e1945e 100644 --- a/src/Apify.Client/ApifyClientVersion.cs +++ b/src/Apify.Client/ApifyClientVersion.cs @@ -14,11 +14,11 @@ public static class ApifyClientVersion /// The semantic version of this client library (see https://semver.org/). Changes to the public /// interface other than additive ones are considered breaking changes. /// - public const string ClientVersion = "0.6.0"; + public const string ClientVersion = "0.7.0"; /// /// The version of the Apify OpenAPI specification this client was generated and verified against. /// Corresponds to the info.version field of the Apify OpenAPI document. /// - public const string ApiSpecVersion = "v2-2026-10-19T133704Z"; + public const string ApiSpecVersion = "v2-2026-10-20T121847Z"; } diff --git a/src/Apify.Client/Options/RunChargeOptions.cs b/src/Apify.Client/Options/RunChargeOptions.cs index 3c6d829..3e82fc0 100644 --- a/src/Apify.Client/Options/RunChargeOptions.cs +++ b/src/Apify.Client/Options/RunChargeOptions.cs @@ -26,6 +26,13 @@ public RunChargeOptions(string eventName, int? count = null, string? idempotency /// A key that deduplicates the charge across retries. public string? IdempotencyKey { get; } + /// + /// The ID of the Server Actor request being charged, taken from the X-Actor-Request-ID header of + /// the incoming request. Required for multi-tenant Server Actor runs, where it identifies the request + /// the charge belongs to. + /// + public string? RequestId { get; init; } + /// The count to send, defaulting to 1. internal int CountValue() => Count ?? 1; } diff --git a/src/Apify.Client/Options/StoreListOptions.cs b/src/Apify.Client/Options/StoreListOptions.cs index 5b8070b..4d2da97 100644 --- a/src/Apify.Client/Options/StoreListOptions.cs +++ b/src/Apify.Client/Options/StoreListOptions.cs @@ -24,8 +24,8 @@ public sealed class StoreListOptions public string? Username { get; init; } /// - /// Filter Actors by pricing model (FREE, FLAT_PRICE_PER_MONTH, - /// PRICE_PER_DATASET_ITEM, PAY_PER_EVENT). + /// Filter Actors by pricing model (FREE, PRICE_PER_DATASET_ITEM, + /// PAY_PER_EVENT). The rental model (FLAT_PRICE_PER_MONTH) is no longer supported. /// public string? PricingModel { get; init; } diff --git a/src/Apify.Client/Resources/RunClient.cs b/src/Apify.Client/Resources/RunClient.cs index 8babaa6..e42f5ff 100644 --- a/src/Apify.Client/Resources/RunClient.cs +++ b/src/Apify.Client/Resources/RunClient.cs @@ -166,6 +166,11 @@ public async Task ChargeAsync(RunChargeOptions options, CancellationToken cancel ["eventName"] = options.EventName, ["count"] = options.CountValue(), }; + if (!string.IsNullOrEmpty(options.RequestId)) + { + body["requestId"] = options.RequestId; + } + using var response = await _http.CallAsync( HttpMethod.Post, _ctx.SubUrl("charge"), diff --git a/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs b/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs index c1bb2e6..3f38387 100644 --- a/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs +++ b/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs @@ -114,15 +114,22 @@ protected static string UniqueName(string prefix) actions = Array.Empty(), }; - /// A minimal ad-hoc webhook definition targeting a condition that never actually fires. - protected static object MinimalWebhook(string requestUrl) => new + /// + /// A minimal webhook definition that fires on runs of the given Actor. The API requires the condition's + /// resource to exist and be accessible, so the Actor must be one the test account owns (see + /// ). + /// + protected static object MinimalWebhook(string requestUrl, string actorId) => new { - isAdHoc = true, eventTypes = new[] { "ACTOR.RUN.SUCCEEDED" }, - condition = new { actorRunId = "ZZZZZZZZZZZZZZZZZ" }, + condition = new { actorId }, requestUrl, }; + /// Creates a throwaway Actor owned by the test account, to use as a webhook condition. Delete it when done. + protected static async Task CreateWebhookActorAsync(ApifyClient client) + => (await client.Actors().CreateAsync(MinimalActor(UniqueName("wh-act")))).Id!; + /// /// Retries up to times, sleeping /// between attempts, until it returns true. Used to tolerate diff --git a/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs index d44015f..6558fe8 100644 --- a/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs @@ -178,11 +178,12 @@ public async Task IterateWebhooks() { var client = RequireClient(); var ids = new HashSet(); + var actorId = await CreateWebhookActorAsync(client); try { for (var i = 0; i < 2; i++) { - ids.Add((await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/it-wh-" + i))).Id!); + ids.Add((await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/it-wh-" + i, actorId))).Id!); } Assert.True( @@ -198,6 +199,7 @@ await FindsAllEventuallyAsync( { await client.Webhook(id).DeleteAsync(); } + await client.Actor(actorId).DeleteAsync(); } } diff --git a/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs index 61ba64f..24b3cf7 100644 --- a/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs @@ -31,7 +31,8 @@ public async Task ListWebhookDispatches() public async Task GetWebhook() { var client = RequireClient(); - var wh = await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook")); + var actorId = await CreateWebhookActorAsync(client); + var wh = await CreateWebhookOrCleanUpAsync(client, actorId); try { var got = await client.Webhook(wh.Id!).GetAsync(); @@ -41,6 +42,7 @@ public async Task GetWebhook() finally { await client.Webhook(wh.Id!).DeleteAsync(); + await client.Actor(actorId).DeleteAsync(); } } @@ -48,7 +50,8 @@ public async Task GetWebhook() public async Task GetWebhookDispatch() { var client = RequireClient(); - var wh = await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook")); + var actorId = await CreateWebhookActorAsync(client); + var wh = await CreateWebhookOrCleanUpAsync(client, actorId); try { var dispatch = await client.Webhook(wh.Id!).TestAsync(); @@ -59,6 +62,7 @@ public async Task GetWebhookDispatch() finally { await client.Webhook(wh.Id!).DeleteAsync(); + await client.Actor(actorId).DeleteAsync(); } } @@ -66,7 +70,8 @@ public async Task GetWebhookDispatch() public async Task WebhookCrudFlow() { var client = RequireClient(); - var wh = await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook")); + var actorId = await CreateWebhookActorAsync(client); + var wh = await CreateWebhookOrCleanUpAsync(client, actorId); try { var webhook = client.Webhook(wh.Id!); @@ -79,6 +84,21 @@ public async Task WebhookCrudFlow() finally { await client.Webhook(wh.Id!).DeleteAsync(); + await client.Actor(actorId).DeleteAsync(); + } + } + + /// Creates the webhook, deleting the Actor if that fails so a failed test leaks nothing. + private static async Task CreateWebhookOrCleanUpAsync(ApifyClient client, string actorId) + { + try + { + return await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook", actorId)); + } + catch + { + await client.Actor(actorId).DeleteAsync(); + throw; } } } diff --git a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs index c1eb7c2..0c2e8fb 100644 --- a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs +++ b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs @@ -117,6 +117,19 @@ public async Task RunChargeSendsBodyAndIdempotencyKey() Assert.Equal(3, body["count"]!.GetValue()); } + [Fact] + public async Task RunChargeSendsRequestIdOnlyWhenSet() + { + var transport = new MockTransport().QueueResponse(200, string.Empty).QueueResponse(200, string.Empty); + var run = Client(transport).Run("run1"); + + await run.ChargeAsync(new RunChargeOptions("e") { RequestId = "req-1" }); + Assert.Equal("req-1", JsonNode.Parse(transport.LastRequest.Body)!["requestId"]!.GetValue()); + + await run.ChargeAsync(new RunChargeOptions("e")); + Assert.Null(JsonNode.Parse(transport.LastRequest.Body)!.AsObject()["requestId"]); + } + [Fact] public async Task RunChargeUsesProvidedIdempotencyKey() { From 3ce796724e82d63f88bdfcff969aed38489e626c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 13:25:07 +0000 Subject: [PATCH 2/8] fix: address review items for .NET client sync (invalid body error, run dataset iteration, tests, docs) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01UUMWnU4gH573HweHieMhHi --- .editorconfig | 8 +- CHANGELOG.md | 11 +- docs/README.md | 22 ++- docs/runs.md | 24 +++ docs/webhooks.md | 4 +- src/Apify.Client/Apify.Client.csproj | 2 +- src/Apify.Client/ApifyClient.cs | 2 +- .../Exceptions/ApifyClientException.cs | 3 +- .../InvalidResponseBodyException.cs | 20 +++ src/Apify.Client/Internal/Json.cs | 17 ++- .../Options/RunIterateDatasetItemsOptions.cs | 53 +++++++ src/Apify.Client/Resources/ActorClient.cs | 2 +- src/Apify.Client/Resources/DatasetClient.cs | 8 + src/Apify.Client/Resources/RunClient.cs | 133 +++++++++++++++- src/Apify.Client/Resources/TaskClient.cs | 2 +- .../Integration/ActorRunIntegrationTests.cs | 24 +++ .../Integration/IntegrationTestBase.cs | 16 +- .../Integration/IterationIntegrationTests.cs | 78 +++++++--- .../RequestQueueIntegrationTests.cs | 8 + .../Integration/WebhookIntegrationTests.cs | 46 ++++-- .../Unit/ApifyClientExceptionTests.cs | 19 +++ .../Unit/RunIterateDatasetItemsTests.cs | 142 ++++++++++++++++++ 22 files changed, 578 insertions(+), 66 deletions(-) create mode 100644 src/Apify.Client/Exceptions/InvalidResponseBodyException.cs create mode 100644 src/Apify.Client/Options/RunIterateDatasetItemsOptions.cs create mode 100644 tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs diff --git a/.editorconfig b/.editorconfig index 42ce52a..a36b2b0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -12,10 +12,10 @@ indent_size = 4 # Use file-scoped namespaces throughout. csharp_style_namespace_declarations = file_scoped:warning -# CA1711: the model type `RequestQueue` deliberately mirrors the Apify API resource name (and the -# JS/PHP/Java/Go/Rust sibling clients). Renaming it away from the reserved "Queue" suffix would break -# cross-client naming consistency, which the client requirements mandate. Justified suppression. -dotnet_diagnostic.CA1711.severity = none +# CA1711: the `RequestQueue*` types deliberately mirror the Apify API resource name and the reference +# client. Renaming them away from the "Queue" suffix would break cross-client naming consistency, which +# the client requirements mandate. The rule stays on for every other suffix. +dotnet_code_quality.CA1711.allowed_suffixes = Queue [*.{csproj,props,targets}] indent_size = 2 diff --git a/CHANGELOG.md b/CHANGELOG.md index d715a07..779ae8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,17 @@ the Server Actor request being charged (required for multi-tenant Server Actor runs). - Documented that the rental pricing model (`FLAT_PRICE_PER_MONTH`) is no longer supported and is not an accepted `StoreListOptions.PricingModel` filter value. +- Added `InvalidResponseBodyException` (an `ApifyClientException`), thrown when a successful response's + body is not valid JSON, instead of a raw `System.Text.Json.JsonException`. It is not retried. An error + status with an unparsable body keeps throwing the `ApifyApiException` for its status code. +- Added `RunClient.IterateDatasetItemsAsync` (with `RunIterateDatasetItemsOptions`), which iterates a run's + default dataset while the run is still producing items, matching the reference client's + `iterateDatasetItems()`. - Fixed the webhook integration tests: the API now requires a webhook condition to reference an existing - resource, so they use a throwaway Actor owned by the test account instead of a made-up run ID. + Actor ID, so they use the ID of the public `apify/hello-world` Actor instead of a made-up run ID. +- Added integration tests for Actor and task webhook listing, request-queue request update/list and + dataset item iteration, and narrowed the CA1711 analyzer suppression to the `Queue` suffix. +- Removed the experimental/AI notice from the NuGet package description (it stays in the top-level README). ## 0.6.0 diff --git a/docs/README.md b/docs/README.md index 81a0b60..59f1006 100644 --- a/docs/README.md +++ b/docs/README.md @@ -74,7 +74,7 @@ the `using` directives for whichever ones a file references: | `Apify.Client.Resources` | Every resource client the entry point returns — `ActorClient`, `RunClient`, `BuildClient`, `DatasetClient`, `KeyValueStoreClient`, `RequestQueueClient`, `TaskClient`, `ScheduleClient`, `LogClient`, `UserClient`, `ActorVersionClient`, `ActorEnvVarClient`, the `…CollectionClient` types (including `ActorVersionCollectionClient`, `ActorEnvVarCollectionClient`, `NestedWebhookCollectionClient`, and `WebhookDispatchCollectionClient`), etc. | | `Apify.Client.Models` | Data models returned by the clients — `Actor`, `ActorRun`, `Build`, `Dataset`, `RequestQueueRequest`, `ActorEnvVar`, `PaginationList`, and so on. | | `Apify.Client.Options` | The option/request objects passed into methods — `ActorStartOptions`, `ActorBuildOptions`, `ActorListOptions`, `RunListOptions`, `RunResurrectOptions`, `RunChargeOptions`, `MetamorphOptions`, `LastRunOptions`, `DatasetListItemsOptions`, `DatasetDownloadOptions`, `DownloadItemsFormat`, `GetRecordOptions`, `SetRecordOptions`, `ListKeysOptions`, `IterateKeysOptions`, `ListRequestsOptions`, `PaginateRequestsOptions`, `BatchAddRequestsOptions`, `RequestQueueClientOptions`, `TaskStartOptions`, `ValidateInputOptions`, `LogOptions`, `ListOptions`, `StorageListOptions`, `StoreListOptions`. | -| `Apify.Client.Exceptions` | `ApifyClientException` (common base), `ApifyApiException` (and its per-status subclasses) and `ApifyTransportException`. | +| `Apify.Client.Exceptions` | `ApifyClientException` (common base), `ApifyApiException` (and its per-status subclasses), `ApifyTransportException` and `InvalidResponseBodyException`. | | `Apify.Client.Http` | The replaceable transport: `IHttpTransport` and the default `HttpClientTransport`. | | `System.Text.Json.Nodes` (BCL) | Not an Apify namespace, but required whenever you name the JSON escape-hatch types the client returns/accepts — `JsonObject`/`JsonNode` (e.g. dataset items, `GetInputAsync`/`UpdateInputAsync`, `GetStatisticsAsync`, `GetOpenApiDefinitionAsync`, `MonthlyUsageAsync`/`LimitsAsync`, `RequestQueueRequest.UserData`). Add `using System.Text.Json.Nodes;`. | @@ -205,11 +205,25 @@ catch (ApifyApiException e) `ApifyTransportException` is thrown instead when the request never reaches the API (network failure, timeout, or DNS error) after all retries are exhausted. -Both `ApifyApiException` and `ApifyTransportException` extend the common `ApifyClientException`, so -`catch (ApifyClientException)` catches every error this client raises for a failed call without naming -either concrete type: +`InvalidResponseBodyException` is thrown when a successful response's body is not valid JSON. The response +arrived in full, so it is not retried. An error status with an unparsable body throws the +`ApifyApiException` for its status code, with the raw body in its message. + +`ApifyApiException`, `ApifyTransportException` and `InvalidResponseBodyException` extend the common +`ApifyClientException`, so `catch (ApifyClientException)` catches every error this client raises for a +failed call without naming the concrete types: ```csharp +using System; +using Apify.Client; +using Apify.Client.Exceptions; + +var client = new ApifyClient("my-api-token"); +try +{ + var user = await client.User().GetAsync(); + Console.WriteLine(user?.Username); +} catch (ApifyClientException e) { Console.WriteLine("client call failed: " + e.Message); diff --git a/docs/runs.md b/docs/runs.md index 6b6901a..2dcaad8 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -27,6 +27,10 @@ collections). - `WaitForFinishAsync(int? waitSecs = null)` → `ActorRun` — client-side polling until terminal (`null` waits indefinitely). A run id that keeps answering 404 for 3 seconds straight (replica lag resolves faster than that) throws `NotFoundException` instead of polling for the rest of `waitSecs`. +- `IterateDatasetItemsAsync(RunIterateDatasetItemsOptions? options = null)` → `IAsyncEnumerable` — + lazily iterates the run's default dataset while the run is still producing items: it reads the rows + below the dataset's `itemCount`, waits for the run to finish between polls, and after the run finishes + reads any remaining rows. On a `LastRun()` client it sticks to the run its first request resolved to. - `Dataset()`, `KeyValueStore()`, `RequestQueue()` — the run's default storages. Their own `GetAsync()`/`DeleteAsync()` throw on a 404 instead of resolving `null`/no-op, since the run itself (not just the storage) could be what's missing; a record/request looked up by its own key or id @@ -45,6 +49,26 @@ collections). `MemoryMbytes` (`int?`), `TimeoutSecs` (`int?`), `MaxItems` (`int?`), `MaxTotalChargeUsd` (`double?`), and `RestartOnError` (`bool?`). See [Actors](actors.md) for each field's meaning. +`RunIterateDatasetItemsOptions` fields (all optional): `Offset` (rows to skip), `Limit` (maximum rows to +scan; `null` or `0` for no limit), `ChunkSize` (rows per API call, default 1000), `PollIntervalSecs` +(seconds to wait for the run to finish between polls, default 5), and the item filters `Clean`, `Fields`, +`Omit`, `Unwind`, `SkipEmpty` and `SkipHidden` (same meaning as in `DatasetListItemsOptions`). + +```csharp +using System; +using Apify.Client; +using Apify.Client.Options; + +var client = new ApifyClient("my-api-token"); +var run = await client.Actor("apify/hello-world").StartAsync(); + +// Items are yielded as the run produces them, until the run has finished and its dataset is drained. +await foreach (var item in client.Run(run.Id!).IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { ChunkSize = 100 })) +{ + Console.WriteLine(item); +} +``` + `RunChargeOptions` describes a pay-per-event charge and is built via its constructor `RunChargeOptions(string eventName, int? count = null, string? idempotencyKey = null)`: diff --git a/docs/webhooks.md b/docs/webhooks.md index 11a2aa6..6243eb8 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -26,10 +26,12 @@ using Apify.Client; using Apify.Client.Options; var client = new ApifyClient("my-api-token"); +// The webhook condition needs the Actor's ID, not its "username/name" slug. +var actor = await client.Actor("apify/hello-world").GetAsync(); var webhook = await client.Webhooks().CreateAsync(new { eventTypes = new[] { "ACTOR.RUN.SUCCEEDED" }, - condition = new { actorId = "apify/hello-world" }, + condition = new { actorId = actor!.Id }, requestUrl = "https://example.com/webhook", }); diff --git a/src/Apify.Client/Apify.Client.csproj b/src/Apify.Client/Apify.Client.csproj index ee43fae..d07613f 100644 --- a/src/Apify.Client/Apify.Client.csproj +++ b/src/Apify.Client/Apify.Client.csproj @@ -10,7 +10,7 @@ Apify Apify Apify API client for .NET - Official, experimental (AI-generated and AI-maintained) Apify API client for .NET: run Actors, manage storages (datasets, key-value stores, request queues), schedules, webhooks and more. + Apify API client for .NET: run Actors, manage storages (datasets, key-value stores, request queues), schedules, webhooks and more. apify;api;client;actor;scraping;crawler;automation Apache-2.0 https://github.com/apify/apify-client-dotnet diff --git a/src/Apify.Client/ApifyClient.cs b/src/Apify.Client/ApifyClient.cs index 5450648..a3ff367 100644 --- a/src/Apify.Client/ApifyClient.cs +++ b/src/Apify.Client/ApifyClient.cs @@ -141,7 +141,7 @@ private static string ToApiBaseUrl(string url) /// A client for a specific Actor run. Must not be empty. /// The run ID. - public RunClient Run(string id) => new(_http, _baseUrl, "actor-runs", RequireNonEmptyId(id, nameof(id))); + public RunClient Run(string id) => new(this, _http, _baseUrl, "actor-runs", RequireNonEmptyId(id, nameof(id))); // ----- Dataset accessors --------------------------------------------------- diff --git a/src/Apify.Client/Exceptions/ApifyClientException.cs b/src/Apify.Client/Exceptions/ApifyClientException.cs index 8a12787..919739c 100644 --- a/src/Apify.Client/Exceptions/ApifyClientException.cs +++ b/src/Apify.Client/Exceptions/ApifyClientException.cs @@ -4,7 +4,8 @@ namespace Apify.Client.Exceptions; /// /// Base class of every error this client throws for a failed API call: -/// (and its per-status subclasses) and . A single +/// (and its per-status subclasses), and +/// . A single /// catch (ApifyClientException) catches all of them, matching the reference client's /// ApifyClientError. /// diff --git a/src/Apify.Client/Exceptions/InvalidResponseBodyException.cs b/src/Apify.Client/Exceptions/InvalidResponseBodyException.cs new file mode 100644 index 0000000..ea1b0dd --- /dev/null +++ b/src/Apify.Client/Exceptions/InvalidResponseBodyException.cs @@ -0,0 +1,20 @@ +using System; + +namespace Apify.Client.Exceptions; + +/// +/// Thrown when the body of a successful response does not parse as JSON, for example a proxy's HTML page +/// served with a success status. The response arrived in full, so the request is not retried. An error +/// status with an unparsable body throws the for its status code instead. +/// Matches the reference client's InvalidResponseBodyError. +/// +public sealed class InvalidResponseBodyException : ApifyClientException +{ + /// Creates an invalid-response-body exception. + /// A description of the failure. + /// The JSON parse error, if any. + public InvalidResponseBodyException(string message, Exception? innerException = null) + : base(message, innerException) + { + } +} diff --git a/src/Apify.Client/Internal/Json.cs b/src/Apify.Client/Internal/Json.cs index ddd8f7f..c460c0a 100644 --- a/src/Apify.Client/Internal/Json.cs +++ b/src/Apify.Client/Internal/Json.cs @@ -1,5 +1,6 @@ using System.Text.Encodings.Web; using System.Text.Json; +using Apify.Client.Exceptions; using System.Text.Json.Nodes; namespace Apify.Client.Internal; @@ -25,7 +26,10 @@ internal static class Json public static string Encode(JsonNode? value) => value?.ToJsonString(SerializerOptions) ?? "null"; - /// Decodes a JSON string into a (null for an empty body). + /// + /// Decodes a JSON response body into a (null for an empty body). + /// Throws when the body is not valid JSON. + /// public static JsonNode? Decode(string body) { if (string.IsNullOrEmpty(body)) @@ -33,7 +37,14 @@ public static string Encode(JsonNode? value) => return null; } - return JsonNode.Parse(body); + try + { + return JsonNode.Parse(body); + } + catch (JsonException ex) + { + throw new InvalidResponseBodyException("response body is not valid JSON: " + ex.Message, ex); + } } /// @@ -57,7 +68,7 @@ public static string Encode(JsonNode? value) => { return Decode(body); } - catch (JsonException) + catch (InvalidResponseBodyException) { return null; } diff --git a/src/Apify.Client/Options/RunIterateDatasetItemsOptions.cs b/src/Apify.Client/Options/RunIterateDatasetItemsOptions.cs new file mode 100644 index 0000000..c3c33c7 --- /dev/null +++ b/src/Apify.Client/Options/RunIterateDatasetItemsOptions.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; + +namespace Apify.Client.Options; + +/// +/// Configures , which iterates over +/// the items of a run's default dataset while the run is still producing them. Mirrors the reference +/// client's RunIterateDatasetItemsOptions. All fields are optional. +/// +public sealed class RunIterateDatasetItemsOptions +{ + /// Default maximum number of dataset rows requested per API call. + public const int DefaultChunkSize = 1000; + + /// Default time, in seconds, to wait for the run to finish between polls. + public const int DefaultPollIntervalSecs = 5; + + /// Number of dataset rows to skip from the beginning (defaults to 0). + public int? Offset { get; init; } + + /// + /// Maximum number of dataset rows to scan (null or 0 for no limit). Fewer items are + /// yielded when filters drop some, more when splits a row into several. + /// + public int? Limit { get; init; } + + /// Maximum number of dataset rows requested per API call (defaults to ). + public int? ChunkSize { get; init; } + + /// How long to wait for the run to finish between polls, in seconds (defaults to ). + public int? PollIntervalSecs { get; init; } + + /// Return only non-empty items and skip hidden fields. + public bool? Clean { get; init; } + + /// Restrict the output to these fields. + public IReadOnlyList? Fields { get; init; } + + /// Exclude these fields from the output. + public IReadOnlyList? Omit { get; init; } + + /// Expand these fields (each array element becomes a separate item). + public IReadOnlyList? Unwind { get; init; } + + /// Skip empty items. + public bool? SkipEmpty { get; init; } + + /// Skip hidden fields (those starting with #). + public bool? SkipHidden { get; init; } + + /// Whether any filter that can drop or split rows (, , ) is set. + internal bool HasRowFilters => Clean == true || SkipEmpty == true || Unwind is { Count: > 0 }; +} diff --git a/src/Apify.Client/Resources/ActorClient.cs b/src/Apify.Client/Resources/ActorClient.cs index 5a8e61a..8027543 100644 --- a/src/Apify.Client/Resources/ActorClient.cs +++ b/src/Apify.Client/Resources/ActorClient.cs @@ -156,7 +156,7 @@ public async Task DefaultBuildAsync(int? waitForFinish = null, Canc /// Optional last-run filters. public RunClient LastRun(LastRunOptions? options = null) { - var client = new RunClient(_http, _ctx.SubUrl(""), "runs", "last"); + var client = new RunClient(_root, _http, _ctx.SubUrl(""), "runs", "last"); client.SetLastRunParams(options ?? new LastRunOptions()); return client; } diff --git a/src/Apify.Client/Resources/DatasetClient.cs b/src/Apify.Client/Resources/DatasetClient.cs index 48c6b19..58312f7 100644 --- a/src/Apify.Client/Resources/DatasetClient.cs +++ b/src/Apify.Client/Resources/DatasetClient.cs @@ -131,6 +131,14 @@ public async Task UpdateAsync(object newFields, CancellationToken cance } } + /// Fetches a single page of items for the given options (see the query overload). + internal Task<(PaginationList Page, long ScannedCount)> FetchItemsPageAsync(DatasetListItemsOptions options, CancellationToken cancellationToken) + { + var q = new QueryParams(); + options.AppendTo(q); + return FetchItemsPageAsync(q, options.Desc ?? false, cancellationToken); + } + /// /// Fetches a single page of dataset items. The endpoint returns a bare JSON array with pagination in /// X-Apify-Pagination-* response headers. Returns the page alongside the number of rows the API diff --git a/src/Apify.Client/Resources/RunClient.cs b/src/Apify.Client/Resources/RunClient.cs index e42f5ff..80298c8 100644 --- a/src/Apify.Client/Resources/RunClient.cs +++ b/src/Apify.Client/Resources/RunClient.cs @@ -3,6 +3,7 @@ using System.Globalization; using System.IO; using System.Net.Http; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using System.Text.Json.Nodes; @@ -25,12 +26,14 @@ public sealed class RunClient /// Header the API uses to deduplicate charge requests. private const string ChargeIdempotencyHeader = "idempotency-key"; + private readonly ApifyClient _root; private readonly HttpClientCore _http; private readonly ResourceContext _ctx; private readonly string _id; - internal RunClient(HttpClientCore http, string baseUrl, string resourcePath, string id) + internal RunClient(ApifyClient root, HttpClientCore http, string baseUrl, string resourcePath, string id) { + _root = root; _http = http; _id = id; _ctx = ResourceContext.Single(http, baseUrl, resourcePath, id); @@ -232,6 +235,134 @@ internal async Task WaitForFinishWithLogAsync(int? waitSecs, Action + /// Lazily iterates over the items of the run's default dataset while the run is still producing them. + /// Mirrors the reference client's iterateDatasetItems(). + /// + /// + /// While the run has not finished, each poll yields the rows below the dataset's itemCount and + /// then waits up to for the run to finish, + /// so the last rows are read as soon as it does. Each page is requested with a limit that ends at + /// itemCount, so it covers exactly the rows it asks for, whatever the filters or + /// do to the items. itemCount lags a few + /// seconds behind the pushed items, so once the run reaches a terminal status the rows past it are read + /// a page at a time until none are left. On a last-run client, the iteration sticks to the run its + /// first request resolves to, so a newer run cannot swap the dataset mid-iteration. + /// + /// Optional pagination, filtering and polling options. + /// A token to cancel the iteration. + public async IAsyncEnumerable IterateDatasetItemsAsync( + RunIterateDatasetItemsOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + options ??= new RunIterateDatasetItemsOptions(); + var chunkSize = options.ChunkSize is > 0 ? options.ChunkSize.Value : RunIterateDatasetItemsOptions.DefaultChunkSize; + var pollIntervalSecs = options.PollIntervalSecs ?? RunIterateDatasetItemsOptions.DefaultPollIntervalSecs; + ArgumentOutOfRangeException.ThrowIfNegative(pollIntervalSecs, nameof(options)); + ArgumentOutOfRangeException.ThrowIfNegative(options.Offset ?? 0, nameof(options)); + ArgumentOutOfRangeException.ThrowIfNegative(options.Limit ?? 0, nameof(options)); + + var position = options.Offset ?? 0; + int? end = options.Limit is > 0 ? position + options.Limit.Value : null; + + var run = await GetAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + // A last-run client resolves `runs/last` on each request, so a newer run would swap the dataset + // mid-iteration: pin to the run the first request resolved to. + var runClient = this; + if (run?.Id is { } resolvedId && resolvedId != _id) + { + runClient = _root.Run(resolvedId); + if (_ctx.RequestTimeout is { } timeout) + { + runClient.WithTimeout(timeout); + } + } + + var datasetClient = runClient.Dataset(); + + while (true) + { + var isFinished = run is null || run.IsTerminal; + var dataset = await datasetClient.GetAsync(cancellationToken).ConfigureAwait(false); + var itemCount = dataset?.ItemCount ?? 0; + if (end is not null) + { + itemCount = Math.Min(itemCount, end.Value); + } + + while (position < itemCount) + { + var pageLimit = (int)Math.Min(chunkSize, itemCount - position); + var (page, _) = await datasetClient.FetchItemsPageAsync(ItemsPageOptions(options, position, pageLimit), cancellationToken).ConfigureAwait(false); + foreach (var item in page.Items) + { + yield return item; + } + + position += pageLimit; + } + + if (end is not null && position >= end) + { + yield break; + } + + if (isFinished) + { + break; + } + + run = await runClient.WaitForFinishAsync(pollIntervalSecs, cancellationToken).ConfigureAwait(false); + } + + // The run is finished, but `itemCount` lags the pushed items: read the remaining rows page by page. + while (true) + { + var pageLimit = end is not null ? Math.Min(chunkSize, end.Value - position) : chunkSize; + var (page, scannedCount) = await datasetClient.FetchItemsPageAsync(ItemsPageOptions(options, position, pageLimit), cancellationToken).ConfigureAwait(false); + foreach (var item in page.Items) + { + yield return item; + } + + // Only a page that returned nothing and scanned no rows marks the end: filters can shorten a + // full page, or empty one that still scanned rows. When a filter could have hidden every row + // past a lagging `itemCount`, a plain one-row read confirms nothing is left. + if (page.Items.Count == 0 && scannedCount == 0 + && (!options.HasRowFilters || await IsDatasetExhaustedAsync(datasetClient, position, cancellationToken).ConfigureAwait(false))) + { + yield break; + } + + position += pageLimit; + if (end is not null && position >= end) + { + yield break; + } + } + } + + /// Builds the dataset-items options for one page of . + private static DatasetListItemsOptions ItemsPageOptions(RunIterateDatasetItemsOptions options, int offset, int limit) => new() + { + Offset = offset, + Limit = limit, + Clean = options.Clean, + Fields = options.Fields, + Omit = options.Omit, + Unwind = options.Unwind, + SkipEmpty = options.SkipEmpty, + SkipHidden = options.SkipHidden, + }; + + /// Checks, without any filter, whether the dataset has no row at or after . + private static async Task IsDatasetExhaustedAsync(DatasetClient datasetClient, int position, CancellationToken cancellationToken) + { + var plain = new DatasetListItemsOptions { Offset = position, Limit = 1 }; + var (page, _) = await datasetClient.FetchItemsPageAsync(plain, cancellationToken).ConfigureAwait(false); + return page.Items.Count == 0; + } + // Nested accessors inherit this client's params so last-run status/origin filters (see // SetLastRunParams) resolve the intended run's storage/log rather than the latest run's. diff --git a/src/Apify.Client/Resources/TaskClient.cs b/src/Apify.Client/Resources/TaskClient.cs index 220d349..7f4261b 100644 --- a/src/Apify.Client/Resources/TaskClient.cs +++ b/src/Apify.Client/Resources/TaskClient.cs @@ -165,7 +165,7 @@ public async Task CallAsync( /// Optional last-run filters. public RunClient LastRun(LastRunOptions? options = null) { - var client = new RunClient(_http, _ctx.SubUrl(""), "runs", "last"); + var client = new RunClient(_root, _http, _ctx.SubUrl(""), "runs", "last"); client.SetLastRunParams(options ?? new LastRunOptions()); return client; } diff --git a/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs index 24a889b..ce68f4f 100644 --- a/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs @@ -38,6 +38,30 @@ public async Task RunActorAndReadOutputs() await client.Run(run.Id!).KeyValueStore().GetRecordAsync("OUTPUT"); } + [SkippableFact] + public async Task IterateRunDatasetItems() + { + var client = RequireClient(); + var run = await client.Actor("apify/hello-world").CallAsync(null, null, 120); + + var iterated = 0; + await foreach (var item in client.Run(run.Id!).IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { ChunkSize = 1 })) + { + Assert.NotNull(item); + iterated++; + } + + // The run is finished, so its dataset is complete. The dataset's reported item count lags the pushed + // items by a few seconds (which is why the iterator drains past it), so it must catch up to what the + // iterator already read. + Assert.True( + await PollUntilAsync( + EventualConsistencyAttempts, + EventualConsistencyBackoff, + async () => (await client.Run(run.Id!).Dataset().ListItemsAsync(new DatasetListItemsOptions())).Total == iterated), + "expected the dataset's item count to catch up with the iterated items"); + } + [SkippableFact] public async Task UpdateAndDeleteRun() { diff --git a/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs b/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs index 3f38387..46efb42 100644 --- a/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs +++ b/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs @@ -26,14 +26,14 @@ public abstract class IntegrationTestBase /// Retry budget for : how many times a freshly created /// resource's collection listing is re-scanned before giving up on it having propagated. /// - private const int EventualConsistencyAttempts = 16; + protected const int EventualConsistencyAttempts = 16; /// /// Delay between retries in . Combined with /// , the total wait budget is /// (EventualConsistencyAttempts - 1) * EventualConsistencyBackoff = ~15s. /// - private static readonly TimeSpan EventualConsistencyBackoff = TimeSpan.FromSeconds(1); + protected static readonly TimeSpan EventualConsistencyBackoff = TimeSpan.FromSeconds(1); /// /// Derives the client base URL from an optional APIFY_API_URL. The variable includes the @@ -115,9 +115,9 @@ protected static string UniqueName(string prefix) }; /// - /// A minimal webhook definition that fires on runs of the given Actor. The API requires the condition's - /// resource to exist and be accessible, so the Actor must be one the test account owns (see - /// ). + /// A minimal webhook definition that fires on runs of the given Actor. The API requires the condition to + /// reference an existing Actor by its ID (not its username/name slug), see + /// . /// protected static object MinimalWebhook(string requestUrl, string actorId) => new { @@ -126,9 +126,9 @@ protected static string UniqueName(string prefix) requestUrl, }; - /// Creates a throwaway Actor owned by the test account, to use as a webhook condition. Delete it when done. - protected static async Task CreateWebhookActorAsync(ApifyClient client) - => (await client.Actors().CreateAsync(MinimalActor(UniqueName("wh-act")))).Id!; + /// Resolves the ID of the public apify/hello-world Actor, for use as a webhook condition. + protected static async Task ResolveHelloWorldActorIdAsync(ApifyClient client) + => (await client.Actor("apify/hello-world").GetAsync())!.Id!; /// /// Retries up to times, sleeping diff --git a/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs index 6558fe8..dcec197 100644 --- a/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Apify.Client.Models; using Apify.Client.Options; @@ -7,29 +8,15 @@ namespace Apify.Client.Tests.Integration; /// -/// Iteration coverage for the 12 collection-of-resources IterateAsync clients that had no -/// dedicated test before this suite was added (Actors, Actor versions, Actor environment variables, -/// datasets, key-value stores, request queues, tasks, schedules, webhooks, builds, runs, and webhook -/// dispatches — exercised by the 11 test methods below, since Actor versions and Actor environment -/// variables share one). already covers the thirteenth -/// (StoreCollectionClient), so it is not duplicated here. +/// Live iteration tests for the collection-of-resources IterateAsync clients, plus dataset item +/// iteration. Where creation is cheap, a test creates a couple of uniquely-named resources and asserts that +/// iterating the collection (newest-first) eventually surfaces every one of them. Builds, runs and webhook +/// dispatches are slow to create on demand, so those tests drain a Limit-bounded slice of what +/// already exists on the account and check the iterator terminates, respects the cap and yields +/// well-formed items. covers the store collection; key +/// iteration and request iteration have their own tests below and in +/// RequestQueueIntegrationTests.RequestQueuePaginateMultiplePages. /// -/// -/// This suite is about iterating a collection of resources (e.g. "every dataset"), not a resource's own -/// contents: dataset item iteration (DatasetClient.IterateItemsAsync) and key-value-store key -/// iteration (KeyValueStoreClient.IterateKeysAsync) are covered by unit tests (AutoPagingTests) -/// against a mocked transport, since they need to exercise multi-page paging logic deterministically; -/// below covers the same iterator against the real API. -/// Request-queue request iteration has its own integration test -/// (RequestQueueIntegrationTests.RequestQueuePaginateMultiplePages). -/// -/// Where creation is cheap, each test below creates a couple of uniquely-named resources and asserts that -/// iterating the collection (newest-first) eventually surfaces every one of them — exercising the real -/// async generator, not just a single ListAsync call. Builds, runs, and webhook dispatches are -/// expensive or slow to create on demand, so those tests instead drain a Limit-bounded slice of -/// whatever already exists on the test account and assert the iterator behaves (terminates, respects the -/// cap, yields well-formed items). -/// [Trait("Category", "Integration")] public sealed class IterationIntegrationTests : IntegrationTestBase { @@ -178,7 +165,7 @@ public async Task IterateWebhooks() { var client = RequireClient(); var ids = new HashSet(); - var actorId = await CreateWebhookActorAsync(client); + var actorId = await ResolveHelloWorldActorIdAsync(client); try { for (var i = 0; i < 2; i++) @@ -199,7 +186,6 @@ await FindsAllEventuallyAsync( { await client.Webhook(id).DeleteAsync(); } - await client.Actor(actorId).DeleteAsync(); } } @@ -351,4 +337,48 @@ public async Task IterateWebhookDispatchesBounded() Assert.True(count <= 5, "the total-cap limit must bound iteration; got " + count); } + + [SkippableFact] + public async Task IterateDatasetItemsMultiplePages() + { + var client = RequireClient(); + var ds = await client.Datasets().GetOrCreateAsync(UniqueName("ds-iter")); + try + { + const int total = 5; + var dataset = client.Dataset(ds.Id!); + await dataset.PushItemsAsync(Enumerable.Range(0, total).Select(static n => (object)new { n }).ToArray()); + + // Dataset item counts propagate with a delay, so retry until every pushed item is readable. + var seen = new List(); + Assert.True( + await PollUntilAsync( + EventualConsistencyAttempts, + EventualConsistencyBackoff, + async () => + { + seen.Clear(); + await foreach (var item in dataset.IterateItemsAsync()) + { + seen.Add(item!["n"]!.GetValue()); + } + + return seen.Count == total; + }), + "expected iteration to eventually return every pushed item"); + Assert.Equal(Enumerable.Range(0, total), seen); + + var limited = new List(); + await foreach (var item in dataset.IterateItemsAsync(new DatasetListItemsOptions { Offset = 1, Limit = 2 })) + { + limited.Add(item!["n"]!.GetValue()); + } + + Assert.Equal(new[] { 1, 2 }, limited); + } + finally + { + await client.Dataset(ds.Id!).DeleteAsync(); + } + } } diff --git a/tests/Apify.Client.Tests/Integration/RequestQueueIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/RequestQueueIntegrationTests.cs index 9f53fcc..946448f 100644 --- a/tests/Apify.Client.Tests/Integration/RequestQueueIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/RequestQueueIntegrationTests.cs @@ -55,6 +55,14 @@ public async Task RequestQueueCrudFlow() Assert.Equal("https://example.com", got!.Url); Assert.NotEmpty((await queue.ListHeadAsync(10)).Items); + + got.Method = "POST"; + await queue.UpdateRequestAsync(got); + Assert.Equal("POST", (await queue.GetRequestAsync(info.RequestId!))!.Method); + Assert.Contains( + (await queue.ListRequestsAsync(new ListRequestsOptions { Limit = 10 })).Items, + r => r.Id == info.RequestId); + await queue.UpdateAsync(new { name = UniqueName("rq-renamed") }); await queue.DeleteRequestAsync(info.RequestId!); } diff --git a/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs index 24b3cf7..84b8ff3 100644 --- a/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs @@ -31,8 +31,7 @@ public async Task ListWebhookDispatches() public async Task GetWebhook() { var client = RequireClient(); - var actorId = await CreateWebhookActorAsync(client); - var wh = await CreateWebhookOrCleanUpAsync(client, actorId); + var wh = await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook", await ResolveHelloWorldActorIdAsync(client))); try { var got = await client.Webhook(wh.Id!).GetAsync(); @@ -42,7 +41,6 @@ public async Task GetWebhook() finally { await client.Webhook(wh.Id!).DeleteAsync(); - await client.Actor(actorId).DeleteAsync(); } } @@ -50,8 +48,7 @@ public async Task GetWebhook() public async Task GetWebhookDispatch() { var client = RequireClient(); - var actorId = await CreateWebhookActorAsync(client); - var wh = await CreateWebhookOrCleanUpAsync(client, actorId); + var wh = await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook", await ResolveHelloWorldActorIdAsync(client))); try { var dispatch = await client.Webhook(wh.Id!).TestAsync(); @@ -62,7 +59,6 @@ public async Task GetWebhookDispatch() finally { await client.Webhook(wh.Id!).DeleteAsync(); - await client.Actor(actorId).DeleteAsync(); } } @@ -70,8 +66,7 @@ public async Task GetWebhookDispatch() public async Task WebhookCrudFlow() { var client = RequireClient(); - var actorId = await CreateWebhookActorAsync(client); - var wh = await CreateWebhookOrCleanUpAsync(client, actorId); + var wh = await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook", await ResolveHelloWorldActorIdAsync(client))); try { var webhook = client.Webhook(wh.Id!); @@ -84,21 +79,42 @@ public async Task WebhookCrudFlow() finally { await client.Webhook(wh.Id!).DeleteAsync(); - await client.Actor(actorId).DeleteAsync(); } } - /// Creates the webhook, deleting the Actor if that fails so a failed test leaks nothing. - private static async Task CreateWebhookOrCleanUpAsync(ApifyClient client, string actorId) + [SkippableFact] + public async Task ListActorWebhooks() + { + var client = RequireClient(); + var actor = await client.Actors().CreateAsync(MinimalActor(UniqueName("wh-act"))); + try + { + var page = await client.Actor(actor.Id!).Webhooks().ListAsync(new ListOptions { Limit = 5 }); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); + } + finally + { + await client.Actor(actor.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task ListTaskWebhooks() { + var client = RequireClient(); + var task = await client.Tasks().CreateAsync(MinimalTask(UniqueName("wh-task"))); try { - return await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook", actorId)); + var page = await client.Task(task.Id!).Webhooks().ListAsync(new ListOptions { Limit = 5 }); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); } - catch + finally { - await client.Actor(actorId).DeleteAsync(); - throw; + await client.Task(task.Id!).DeleteAsync(); } } } diff --git a/tests/Apify.Client.Tests/Unit/ApifyClientExceptionTests.cs b/tests/Apify.Client.Tests/Unit/ApifyClientExceptionTests.cs index 9b7413f..533cddf 100644 --- a/tests/Apify.Client.Tests/Unit/ApifyClientExceptionTests.cs +++ b/tests/Apify.Client.Tests/Unit/ApifyClientExceptionTests.cs @@ -47,6 +47,25 @@ public async Task ApifyTransportExceptionIsAnApifyClientException() Assert.IsAssignableFrom(ex); } + [Fact] + public async Task SuccessWithUnparsableBodyThrowsInvalidResponseBodyWithoutRetry() + { + var transport = new MockTransport().QueueResponse(200, "not json").QueueResponse(200, "{\"data\":{\"id\":\"a\"}}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actor("x").GetAsync()); + Assert.IsAssignableFrom(ex); + Assert.Equal(1, transport.CallCount); + } + + [Fact] + public async Task ErrorStatusWithUnparsableBodyThrowsApiExceptionForItsStatus() + { + var transport = new MockTransport().QueueResponse(400, "Bad Request"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actors().CreateAsync(new { name = "x" })); + Assert.Contains("Bad Request", ex.Message, StringComparison.Ordinal); + } + [Fact] public async Task CatchingApifyClientExceptionCatchesBothFamilies() { diff --git a/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs b/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs new file mode 100644 index 0000000..85bd727 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs @@ -0,0 +1,142 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// Verifies RunClient.IterateDatasetItemsAsync: it reads a run's dataset while the run is still +/// producing items, pages by chunkSize up to itemCount, and drains rows past a lagging +/// itemCount once the run is finished. +/// +[Trait("Category", "Unit")] +public sealed class RunIterateDatasetItemsTests +{ + private const string FinishedRun = "{\"data\":{\"id\":\"run1\",\"status\":\"SUCCEEDED\"}}"; + private const string RunningRun = "{\"data\":{\"id\":\"run1\",\"status\":\"RUNNING\"}}"; + + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + private static string DatasetWithItemCount(int itemCount) => "{\"data\":{\"id\":\"ds1\",\"itemCount\":" + itemCount + "}}"; + + private static string Items(params int[] ids) => "[" + string.Join(",", ids.Select(static i => "{\"n\":" + i + "}")) + "]"; + + private static async Task> CollectAsync(IAsyncEnumerable items) + { + var result = new List(); + await foreach (var item in items) + { + result.Add(item!["n"]!.GetValue()); + } + + return result; + } + + [Fact] + public async Task FinishedRunPagesByChunkSizeAndDrainsRowsPastLaggingItemCount() + { + var transport = new MockTransport() + .QueueResponse(200, FinishedRun) + .QueueResponse(200, DatasetWithItemCount(3)) + .QueueResponse(200, Items(1, 2)) + .QueueResponse(200, Items(3)) + .QueueResponse(200, Items(4)) // past itemCount: the count lags the pushed items + .QueueResponse(200, Items()); + + var items = await CollectAsync(Client(transport).Run("run1").IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { ChunkSize = 2 })); + + Assert.Equal(new[] { 1, 2, 3, 4 }, items); + var uris = transport.Received.Select(static r => r.Uri).ToList(); + Assert.Contains("offset=0", uris[2], StringComparison.Ordinal); + Assert.Contains("limit=2", uris[2], StringComparison.Ordinal); + Assert.Contains("offset=2", uris[3], StringComparison.Ordinal); + Assert.Contains("limit=1", uris[3], StringComparison.Ordinal); + Assert.Contains("offset=3", uris[4], StringComparison.Ordinal); + } + + [Fact] + public async Task RunningRunPollsForFinishAndReadsRowsAsTheyAppear() + { + var transport = new MockTransport() + .QueueResponse(200, RunningRun) + .QueueResponse(200, DatasetWithItemCount(1)) + .QueueResponse(200, Items(1)) + .QueueResponse(200, FinishedRun) // waitForFinish poll + .QueueResponse(200, DatasetWithItemCount(2)) + .QueueResponse(200, Items(2)) + .QueueResponse(200, Items()); + + var items = await CollectAsync(Client(transport).Run("run1").IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { PollIntervalSecs = 1 })); + + Assert.Equal(new[] { 1, 2 }, items); + Assert.Contains("waitForFinish=", transport.Received[3].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task LimitBoundsScannedRowsAndStopsWithoutFurtherReads() + { + var transport = new MockTransport() + .QueueResponse(200, FinishedRun) + .QueueResponse(200, DatasetWithItemCount(10)) + .QueueResponse(200, Items(2, 3)); + + var items = await CollectAsync(Client(transport).Run("run1").IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { Offset = 1, Limit = 2 })); + + Assert.Equal(new[] { 2, 3 }, items); + Assert.Equal(3, transport.CallCount); + Assert.Contains("offset=1", transport.LastRequest.Uri, StringComparison.Ordinal); + Assert.Contains("limit=2", transport.LastRequest.Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task LastRunClientPinsToTheRunItFirstResolved() + { + var transport = new MockTransport() + .QueueResponse(200, "{\"data\":{\"id\":\"resolvedRun\",\"status\":\"SUCCEEDED\"}}") + .QueueResponse(200, DatasetWithItemCount(1)) + .QueueResponse(200, Items(1)) + .QueueResponse(200, Items()); + + var items = await CollectAsync(Client(transport).Actor("me~act").LastRun().IterateDatasetItemsAsync()); + + Assert.Equal(new[] { 1 }, items); + Assert.Contains("/actors/me~act/runs/last", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.Contains("/actor-runs/resolvedRun/dataset", transport.Received[1].Uri, StringComparison.Ordinal); + Assert.Contains("/actor-runs/resolvedRun/dataset/items", transport.Received[2].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task FilteredEmptyTailIsConfirmedWithAPlainRead() + { + var transport = new MockTransport() + .QueueResponse(200, FinishedRun) + .QueueResponse(200, DatasetWithItemCount(0)) + .QueueResponse(200, Items()) // filtered page: empty, nothing scanned + .QueueResponse(200, Items(7)) // plain read finds a row the filter page missed + .QueueResponse(200, Items(7)) + .QueueResponse(200, Items()) + .QueueResponse(200, Items()); + + var items = await CollectAsync(Client(transport).Run("run1").IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { SkipEmpty = true })); + + Assert.Equal(new[] { 7 }, items); + } + + [Fact] + public async Task NegativeArgumentsAreRejected() + { + var run = Client(new MockTransport()).Run("run1"); + await Assert.ThrowsAsync(() => CollectAsync(run.IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { Offset = -1 }))); + await Assert.ThrowsAsync(() => CollectAsync(run.IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { PollIntervalSecs = -1 }))); + } +} From a65c79de866ae34dbb5527386a11c1364446b370 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 13:41:59 +0000 Subject: [PATCH 3/8] fix: address second review round (chunkSize, response validation, run iteration fixes, doc snippet compile check) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01UUMWnU4gH573HweHieMhHi --- .../workflows/dotnet-integration-tests.yml | 4 +- CHANGELOG.md | 17 ++- docs/README.md | 15 +- docs/actors.md | 1 + docs/builds.md | 2 +- docs/misc.md | 1 + docs/runs.md | 2 +- docs/storages.md | 12 +- .../Exceptions/ApifyClientException.cs | 4 +- .../InvalidResponseBodyException.cs | 22 ++- .../Exceptions/ResponseValidationException.cs | 39 ++++++ src/Apify.Client/Internal/HttpClientCore.cs | 3 + src/Apify.Client/Internal/Json.cs | 31 +---- src/Apify.Client/Internal/PageSizing.cs | 41 ++++++ src/Apify.Client/Internal/ResourceContext.cs | 67 ++++++--- src/Apify.Client/Internal/ResponseReader.cs | 91 +++++++++++++ src/Apify.Client/Options/ActorListOptions.cs | 7 + .../Options/DatasetListItemsOptions.cs | 7 + .../Options/IterateKeysOptions.cs | 11 +- src/Apify.Client/Options/ListOptions.cs | 7 + .../Options/PaginateRequestsOptions.cs | 15 +- .../Options/RunIterateDatasetItemsOptions.cs | 5 +- .../Options/StorageListOptions.cs | 7 + src/Apify.Client/Options/StoreListOptions.cs | 11 +- .../AbstractWebhookCollectionClient.cs | 2 +- .../Resources/ActorCollectionClient.cs | 2 +- src/Apify.Client/Resources/BuildClient.cs | 5 +- .../Resources/BuildCollectionClient.cs | 2 +- src/Apify.Client/Resources/DatasetClient.cs | 26 ++-- .../Resources/DatasetCollectionClient.cs | 2 +- .../Resources/KeyValueStoreClient.cs | 18 ++- .../KeyValueStoreCollectionClient.cs | 2 +- src/Apify.Client/Resources/LogClient.cs | 4 +- .../Resources/RequestQueueClient.cs | 14 +- .../Resources/RequestQueueCollectionClient.cs | 2 +- src/Apify.Client/Resources/RunClient.cs | 26 ++-- .../Resources/RunCollectionClient.cs | 2 +- .../Resources/ScheduleCollectionClient.cs | 2 +- .../Resources/StoreCollectionClient.cs | 11 +- src/Apify.Client/Resources/TaskClient.cs | 5 +- .../Resources/TaskCollectionClient.cs | 2 +- .../WebhookDispatchCollectionClient.cs | 2 +- .../Apify.Client.Tests.csproj | 2 + .../Examples/DocumentationSnippetTests.cs | 94 +++++++++++++ .../Integration/ActorRunIntegrationTests.cs | 26 +++- .../Integration/IterationIntegrationTests.cs | 6 +- .../Unit/AutoPagingTests.cs | 2 +- .../Apify.Client.Tests/Unit/ChunkSizeTests.cs | 128 ++++++++++++++++++ .../Unit/ResponseValidationTests.cs | 87 ++++++++++++ .../Unit/RunIterateDatasetItemsTests.cs | 66 +++++++-- 50 files changed, 808 insertions(+), 154 deletions(-) create mode 100644 src/Apify.Client/Exceptions/ResponseValidationException.cs create mode 100644 src/Apify.Client/Internal/PageSizing.cs create mode 100644 src/Apify.Client/Internal/ResponseReader.cs create mode 100644 tests/Apify.Client.Tests/Examples/DocumentationSnippetTests.cs create mode 100644 tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs create mode 100644 tests/Apify.Client.Tests/Unit/ResponseValidationTests.cs diff --git a/.github/workflows/dotnet-integration-tests.yml b/.github/workflows/dotnet-integration-tests.yml index 2e80494..960447c 100644 --- a/.github/workflows/dotnet-integration-tests.yml +++ b/.github/workflows/dotnet-integration-tests.yml @@ -69,8 +69,8 @@ jobs: APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} run: dotnet test --configuration Release --no-build --filter Category=Integration - # Standalone CI step that verifies the documentation examples work end-to-end against the live - # API (the Examples suite runs each example). + # Standalone CI step that verifies the documentation examples: the Examples suite runs each example + # end-to-end against the live API and compiles every C# snippet in the README and docs/. - name: Test examples env: APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 779ae8b..0454156 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,21 @@ - Documented that the rental pricing model (`FLAT_PRICE_PER_MONTH`) is no longer supported and is not an accepted `StoreListOptions.PricingModel` filter value. - Added `InvalidResponseBodyException` (an `ApifyClientException`), thrown when a successful response's - body is not valid JSON, instead of a raw `System.Text.Json.JsonException`. It is not retried. An error - status with an unparsable body keeps throwing the `ApifyApiException` for its status code. + body is not valid JSON, instead of a raw `System.Text.Json.JsonException`. It is not retried and exposes + the status code, HTTP method, path and raw body. An error status with an unparsable body keeps throwing + the `ApifyApiException` for its status code. +- Added `ResponseValidationException` (an `ApifyClientException`), thrown when a successful response is + missing the `data` envelope or has a `data` of the wrong type (for example an array where an object is + expected), instead of silently returning `null` or an empty object. Matches the reference client's + `ResponseValidationError`. +- Added `ChunkSize` to `ListOptions`, `StorageListOptions`, `ActorListOptions`, `StoreListOptions`, + `DatasetListItemsOptions`, `IterateKeysOptions` and `PaginateRequestsOptions`: the page size iterators + request (the smaller of it and the remaining `Limit`), matching the reference client's `chunkSize`. + `MaxPageLimit` stays as the older name of the same option. A non-positive `ChunkSize` throws + `ArgumentOutOfRangeException`. +- Nested accessors of a run (`Dataset()`, `KeyValueStore()`, `RequestQueue()`, `Log()`) and of a build + (`Log()`) now inherit the run's or build's `WithTimeout` override. +- Added a documentation test that compiles every C# snippet in the README and `docs/`. - Added `RunClient.IterateDatasetItemsAsync` (with `RunIterateDatasetItemsOptions`), which iterates a run's default dataset while the run is still producing items, matching the reference client's `iterateDatasetItems()`. diff --git a/docs/README.md b/docs/README.md index 59f1006..6e2e487 100644 --- a/docs/README.md +++ b/docs/README.md @@ -74,7 +74,7 @@ the `using` directives for whichever ones a file references: | `Apify.Client.Resources` | Every resource client the entry point returns — `ActorClient`, `RunClient`, `BuildClient`, `DatasetClient`, `KeyValueStoreClient`, `RequestQueueClient`, `TaskClient`, `ScheduleClient`, `LogClient`, `UserClient`, `ActorVersionClient`, `ActorEnvVarClient`, the `…CollectionClient` types (including `ActorVersionCollectionClient`, `ActorEnvVarCollectionClient`, `NestedWebhookCollectionClient`, and `WebhookDispatchCollectionClient`), etc. | | `Apify.Client.Models` | Data models returned by the clients — `Actor`, `ActorRun`, `Build`, `Dataset`, `RequestQueueRequest`, `ActorEnvVar`, `PaginationList`, and so on. | | `Apify.Client.Options` | The option/request objects passed into methods — `ActorStartOptions`, `ActorBuildOptions`, `ActorListOptions`, `RunListOptions`, `RunResurrectOptions`, `RunChargeOptions`, `MetamorphOptions`, `LastRunOptions`, `DatasetListItemsOptions`, `DatasetDownloadOptions`, `DownloadItemsFormat`, `GetRecordOptions`, `SetRecordOptions`, `ListKeysOptions`, `IterateKeysOptions`, `ListRequestsOptions`, `PaginateRequestsOptions`, `BatchAddRequestsOptions`, `RequestQueueClientOptions`, `TaskStartOptions`, `ValidateInputOptions`, `LogOptions`, `ListOptions`, `StorageListOptions`, `StoreListOptions`. | -| `Apify.Client.Exceptions` | `ApifyClientException` (common base), `ApifyApiException` (and its per-status subclasses), `ApifyTransportException` and `InvalidResponseBodyException`. | +| `Apify.Client.Exceptions` | `ApifyClientException` (common base), `ApifyApiException` (and its per-status subclasses), `ApifyTransportException`, `InvalidResponseBodyException` and `ResponseValidationException`. | | `Apify.Client.Http` | The replaceable transport: `IHttpTransport` and the default `HttpClientTransport`. | | `System.Text.Json.Nodes` (BCL) | Not an Apify namespace, but required whenever you name the JSON escape-hatch types the client returns/accepts — `JsonObject`/`JsonNode` (e.g. dataset items, `GetInputAsync`/`UpdateInputAsync`, `GetStatisticsAsync`, `GetOpenApiDefinitionAsync`, `MonthlyUsageAsync`/`LimitsAsync`, `RequestQueueRequest.UserData`). Add `using System.Text.Json.Nodes;`. | @@ -205,11 +205,14 @@ catch (ApifyApiException e) `ApifyTransportException` is thrown instead when the request never reaches the API (network failure, timeout, or DNS error) after all retries are exhausted. -`InvalidResponseBodyException` is thrown when a successful response's body is not valid JSON. The response -arrived in full, so it is not retried. An error status with an unparsable body throws the -`ApifyApiException` for its status code, with the raw body in its message. +`InvalidResponseBodyException` is thrown when a successful response's body is not valid JSON, and +`ResponseValidationException` when it is valid JSON but does not have the shape the API specification +promises (for example the `data` envelope is missing). The response arrived in full, so neither is +retried. Both expose `StatusCode`, `HttpMethod`, `Path` and the raw `Body`. An error status with an +unparsable body throws the `ApifyApiException` for its status code, with the raw body in its message. -`ApifyApiException`, `ApifyTransportException` and `InvalidResponseBodyException` extend the common +`ApifyApiException`, `ApifyTransportException`, `InvalidResponseBodyException` and +`ResponseValidationException` extend the common `ApifyClientException`, so `catch (ApifyClientException)` catches every error this client raises for a failed call without naming the concrete types: @@ -221,7 +224,7 @@ using Apify.Client.Exceptions; var client = new ApifyClient("my-api-token"); try { - var user = await client.User().GetAsync(); + var user = await client.Me().GetAsync(); Console.WriteLine(user?.Username); } catch (ApifyClientException e) diff --git a/docs/actors.md b/docs/actors.md index 5d5f695..c55ee7c 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -20,6 +20,7 @@ Access the Actor collection with `client.Actors()` and a specific Actor with `cl | `Desc` | `bool?` | Sort newest-first when `true`. | | `My` | `bool?` | Return only Actors owned by the current account when `true`. | | `SortBy` | `string?` | Field to sort by (e.g. `createdAt`, `lastRunStartedAt`). | +| `ChunkSize` | `int?` | Page size used by `IterateAsync` (positive; the smaller of this and the remaining `Limit` is requested per call). Never sent to the API. | ```csharp using System; diff --git a/docs/builds.md b/docs/builds.md index cfb39c7..33fb81f 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -5,7 +5,7 @@ Access the account-wide build collection with `client.Builds()`, an Actor's buil ## Collection -- `ListAsync(ListOptions? options = null)` → `PaginationList` (`Offset`, `Limit`, `Desc`). +- `ListAsync(ListOptions? options = null)` → `PaginationList` (`Offset`, `Limit`, `Desc`; `ChunkSize` sizes the pages of `IterateAsync`). - `IterateAsync(ListOptions? options = null)` → `IAsyncEnumerable` (lazy, all pages). ## Single build — `client.Build(buildId)` diff --git a/docs/misc.md b/docs/misc.md index eaaa2dd..0a19e58 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -17,6 +17,7 @@ Browse public Actors in the [Apify Store](https://apify.com/store). |---|---|---| | `Offset` | `int?` | Number of Actors to skip from the start. | | `Limit` | `int?` | Maximum number of Actors to return in the page. | +| `ChunkSize` | `int?` | Page size used by `IterateAsync` (positive); takes precedence over `Limit` as the page size. Never sent to the API. | | `Search` | `string?` | Full-text search string to filter Actors by. | | `SortBy` | `string?` | Field to sort by (e.g. `popularity`, `newest`). | | `Category` | `string?` | Restrict results to a Store category. | diff --git a/docs/runs.md b/docs/runs.md index 2dcaad8..5ba78ad 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -9,7 +9,7 @@ Access the account-wide run collection with `client.Runs()`, an Actor's or task' - `IterateAsync(ListOptions? options = null, RunListOptions? filter = null)` → `IAsyncEnumerable` (lazy, all pages). -`ListOptions` fields: `Offset`, `Limit`, `Desc` (standard pagination). `RunListOptions` fields: +`ListOptions` fields: `Offset`, `Limit`, `Desc` (standard pagination) and `ChunkSize` (the page size `IterateAsync` requests; the smaller of it and the remaining `Limit` is used per call). `RunListOptions` fields: `Status` (`IReadOnlyList?`, filter by one or more run statuses such as `SUCCEEDED`/`RUNNING`), `StartedAfter` and `StartedBefore` (ISO 8601 bounds, honoured only by the Actor- and task-scoped run collections). diff --git a/docs/storages.md b/docs/storages.md index 890fe30..c574b55 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -19,6 +19,7 @@ reached from a run (`client.Run(id).Dataset()`, `.KeyValueStore()`, `.RequestQue | `Desc` | `bool?` | Sort newest-first when `true`. | | `Unnamed` | `bool?` | Include unnamed storages when `true` (they are excluded by default). | | `Ownership` | `string?` | Filter by ownership (e.g. only storages owned by the current account). | +| `ChunkSize` | `int?` | Page size used by `IterateAsync` (positive; the smaller of this and the remaining `Limit` is requested per call). Never sent to the API. | ## Datasets @@ -39,7 +40,8 @@ reached from a run (`client.Run(id).Dataset()`, `.KeyValueStore()`, `.RequestQue - `GetStatisticsAsync()` → `JsonObject` (throws if the dataset does not exist). - `CreateItemsPublicUrlAsync(DatasetListItemsOptions? options = null, int? expiresInSecs = null, DownloadItemsFormat? format = null)` → `string` (a signed public URL; `format` defaults to `Json`). -`DatasetListItemsOptions` selects and reshapes items: `Offset`/`Limit` (pagination), `Desc` (reverse +`DatasetListItemsOptions` selects and reshapes items: `Offset`/`Limit` (pagination), `ChunkSize` (the page +size `IterateItemsAsync` requests, never sent to the API), `Desc` (reverse order), `Fields`/`OutputFields`/`Omit` (choose columns), `Unwind`/`Flatten` (restructure nested fields), `Clean`/`SkipEmpty`/`SkipHidden`/`SkipFailedPages` (drop unwanted rows), `Simplified`, `View`, and `Signature` (for signed public URLs). @@ -99,7 +101,7 @@ await csvStream.CopyToAsync(file); - `GetAsync()` → `KeyValueStore?`; `UpdateAsync(object newFields)` → `KeyValueStore`; `DeleteAsync()`. - `ListKeysAsync(ListKeysOptions? options = null)` → `KeyValueStoreKeysPage` (one page). - `IterateKeysAsync(IterateKeysOptions? options = null)` → `IAsyncEnumerable` — lazily - iterate every key across pages, fetching each page (sized by `MaxPageLimit`) on demand. + iterate every key across pages, fetching each page (sized by `ChunkSize`) on demand. - `RecordExistsAsync(string key)` → `bool`. - `GetRecordAsync(string key, GetRecordOptions? options = null)` → `KeyValueStoreRecord?`. `KeyValueStoreRecord.Value` is a `byte[]` of the record's raw bytes (so binary records survive intact); @@ -114,8 +116,8 @@ await csvStream.CopyToAsync(file); `ListKeysOptions` fields: `Limit` (page size), `ExclusiveStartKey` (start after this key), `Prefix` (only keys with this prefix), `Collection` (a named record collection), and `Signature` (for signed public URLs). `IterateKeysOptions` fields: `Limit` (total across all pages), -`MaxPageLimit` (page size; client-side only, never sent to the API — defaults to 1000), -`ExclusiveStartKey`, `Collection`, and `Prefix`. `GetRecordOptions` fields: `Attachment` (request a +`ChunkSize` (page size; client-side only, never sent to the API — defaults to 1000; `MaxPageLimit` is its +older name, and `ChunkSize` wins if both are set), `ExclusiveStartKey`, `Collection`, and `Prefix`. `GetRecordOptions` fields: `Attachment` (request a download disposition) and `Signature`. `SetRecordOptions` fields: `TimeoutSecs`, `DoNotRetryTimeouts`, and `ContentEncoding` (upload a value that is already compressed — see below). @@ -210,7 +212,7 @@ leaves unprocessed), `MaxParallel` (how many chunks are sent concurrently), and `MinDelayBetweenUnprocessedRequestsRetriesMillis` (backoff before retrying unprocessed requests). `ListRequestsOptions`/`PaginateRequestsOptions` fields: `Limit`, `ExclusiveStartId`, `Cursor`, and `Filter` (an `IReadOnlyList?` — one or more of `"pending"`/`"locked"`, so several states can be -requested at once); `PaginateRequestsOptions` also has `MaxPageLimit`. +requested at once); `PaginateRequestsOptions` also has `ChunkSize` (page size, defaults to 1000; `MaxPageLimit` is its older name). ```csharp using System; diff --git a/src/Apify.Client/Exceptions/ApifyClientException.cs b/src/Apify.Client/Exceptions/ApifyClientException.cs index 919739c..9a65b6a 100644 --- a/src/Apify.Client/Exceptions/ApifyClientException.cs +++ b/src/Apify.Client/Exceptions/ApifyClientException.cs @@ -4,8 +4,8 @@ namespace Apify.Client.Exceptions; /// /// Base class of every error this client throws for a failed API call: -/// (and its per-status subclasses), and -/// . A single +/// (and its per-status subclasses), , +/// and . A single /// catch (ApifyClientException) catches all of them, matching the reference client's /// ApifyClientError. /// diff --git a/src/Apify.Client/Exceptions/InvalidResponseBodyException.cs b/src/Apify.Client/Exceptions/InvalidResponseBodyException.cs index ea1b0dd..712455d 100644 --- a/src/Apify.Client/Exceptions/InvalidResponseBodyException.cs +++ b/src/Apify.Client/Exceptions/InvalidResponseBodyException.cs @@ -12,9 +12,29 @@ public sealed class InvalidResponseBodyException : ApifyClientException { /// Creates an invalid-response-body exception. /// A description of the failure. + /// The HTTP status code of the response. + /// The HTTP method of the request that produced the response. + /// The API endpoint path of the request (URL excluding origin). + /// The raw response body that failed to parse. /// The JSON parse error, if any. - public InvalidResponseBodyException(string message, Exception? innerException = null) + public InvalidResponseBodyException(string message, int statusCode, string httpMethod, string path, string body, Exception? innerException = null) : base(message, innerException) { + StatusCode = statusCode; + HttpMethod = httpMethod; + Path = path; + Body = body; } + + /// The HTTP status code of the response. + public int StatusCode { get; } + + /// The HTTP method of the request that produced the response (e.g. GET). + public string HttpMethod { get; } + + /// The API endpoint path of the request (URL excluding origin). + public string Path { get; } + + /// The raw response body that failed to parse. + public string Body { get; } } diff --git a/src/Apify.Client/Exceptions/ResponseValidationException.cs b/src/Apify.Client/Exceptions/ResponseValidationException.cs new file mode 100644 index 0000000..2962c6c --- /dev/null +++ b/src/Apify.Client/Exceptions/ResponseValidationException.cs @@ -0,0 +1,39 @@ +using System; + +namespace Apify.Client.Exceptions; + +/// +/// Thrown when a successful response is valid JSON but does not have the shape the Apify API +/// specification promises, for example it lacks the data envelope or carries an array where the +/// endpoint returns an object. The response arrived in full, so the request is not retried. Matches the +/// reference client's ResponseValidationError. +/// +public sealed class ResponseValidationException : ApifyClientException +{ + /// Creates a response-validation exception. + /// A description of how the response deviates from the specification. + /// The HTTP status code of the response. + /// The HTTP method of the request that produced the response. + /// The API endpoint path of the request (URL excluding origin). + /// The raw response body that failed validation. + public ResponseValidationException(string message, int statusCode, string httpMethod, string path, string body) + : base(message) + { + StatusCode = statusCode; + HttpMethod = httpMethod; + Path = path; + Body = body; + } + + /// The HTTP status code of the response. + public int StatusCode { get; } + + /// The HTTP method of the request that produced the response (e.g. GET). + public string HttpMethod { get; } + + /// The API endpoint path of the request (URL excluding origin). + public string Path { get; } + + /// The raw response body that failed validation. + public string Body { get; } +} diff --git a/src/Apify.Client/Internal/HttpClientCore.cs b/src/Apify.Client/Internal/HttpClientCore.cs index ce7c851..9bb4858 100644 --- a/src/Apify.Client/Internal/HttpClientCore.cs +++ b/src/Apify.Client/Internal/HttpClientCore.cs @@ -169,6 +169,9 @@ public async Task CallAsync( var status = (int)response.StatusCode; if (status < MaxSuccessStatus) { + // The sent request is disposed by now; keep a body-less copy of it so response readers can + // report the method and path in the errors they raise. + response.RequestMessage = new HttpRequestMessage(method, url); return response; } diff --git a/src/Apify.Client/Internal/Json.cs b/src/Apify.Client/Internal/Json.cs index c460c0a..89e0764 100644 --- a/src/Apify.Client/Internal/Json.cs +++ b/src/Apify.Client/Internal/Json.cs @@ -1,6 +1,5 @@ using System.Text.Encodings.Web; using System.Text.Json; -using Apify.Client.Exceptions; using System.Text.Json.Nodes; namespace Apify.Client.Internal; @@ -27,8 +26,9 @@ public static string Encode(JsonNode? value) => value?.ToJsonString(SerializerOptions) ?? "null"; /// - /// Decodes a JSON response body into a (null for an empty body). - /// Throws when the body is not valid JSON. + /// Decodes a JSON string into a (null for an empty body). Throws + /// when it is not valid JSON; response bodies go through + /// , which turns that into an . /// public static JsonNode? Decode(string body) { @@ -37,28 +37,7 @@ public static string Encode(JsonNode? value) => return null; } - try - { - return JsonNode.Parse(body); - } - catch (JsonException ex) - { - throw new InvalidResponseBodyException("response body is not valid JSON: " + ex.Message, ex); - } - } - - /// - /// Decodes a JSON response body wrapped in a {"data": ...} envelope, returning the unwrapped - /// data value (or null if it is absent/null). - /// - public static JsonNode? DecodeData(string body) - { - if (Decode(body) is JsonObject obj && obj.TryGetPropertyValue("data", out var data)) - { - return data; - } - - return null; + return JsonNode.Parse(body); } /// Attempts to decode a body, returning null on any parse error. @@ -68,7 +47,7 @@ public static string Encode(JsonNode? value) => { return Decode(body); } - catch (InvalidResponseBodyException) + catch (JsonException) { return null; } diff --git a/src/Apify.Client/Internal/PageSizing.cs b/src/Apify.Client/Internal/PageSizing.cs new file mode 100644 index 0000000..d5c58fe --- /dev/null +++ b/src/Apify.Client/Internal/PageSizing.cs @@ -0,0 +1,41 @@ +using System; + +namespace Apify.Client.Internal; + +/// +/// Shared handling of the client-side ChunkSize iteration option: the maximum number of items +/// requested per API call while iterating across pages. It never reaches the query string itself; it only +/// sizes each page's limit, matching the reference client's chunkSize. +/// +internal static class PageSizing +{ + /// Rejects a zero or negative chunk size, which the reference client's options schema also refuses. + /// The chunk size from the caller's options, or null when unset. + /// The option's public name, used in the error message. + public static void ValidateChunkSize(int? chunkSize, string optionName) + { + if (chunkSize is <= 0) + { + throw new ArgumentOutOfRangeException(optionName, chunkSize, optionName + " must be positive."); + } + } + + /// + /// The limit to request for the next page: the smaller of the chunk size and the number of items + /// still wanted. A value of null or 0 means unset (the API itself reads limit=0 as + /// unbounded), so it loses to any other value; null is returned when both are unset. + /// + /// The validated chunk size, or null when unset. + /// The number of items still wanted under the caller's total limit, or null when unbounded. + public static int? PageLimit(int? chunkSize, int? remaining) + { + var chunk = chunkSize is 0 ? null : chunkSize; + var wanted = remaining is 0 ? null : remaining; + if (chunk is null) + { + return wanted; + } + + return wanted is null ? chunk : Math.Min(chunk.Value, wanted.Value); + } +} diff --git a/src/Apify.Client/Internal/ResourceContext.cs b/src/Apify.Client/Internal/ResourceContext.cs index d5398fe..c5b5b46 100644 --- a/src/Apify.Client/Internal/ResourceContext.cs +++ b/src/Apify.Client/Internal/ResourceContext.cs @@ -170,8 +170,7 @@ public string PublicUrl(string subPath) { var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); using var response = await Http.CallAsync(HttpMethod.Get, url, timeout: _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); - var body = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); - return Json.DecodeData(body); + return await response.ReadDataAsync(ct).ConfigureAwait(false); } /// PUT to update a resource with a JSON-serializable body, returning the decoded data. @@ -179,7 +178,7 @@ public async Task UpdateResourceAsync(string subPath, object? body, { var url = MergedParams(new QueryParams()).ApplyToUrl(SubUrl(subPath)); using var response = await Http.CallAsync(HttpMethod.Put, url, Json.Encode(body), ContentTypeJson, _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); - return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); } /// @@ -233,18 +232,25 @@ public async IAsyncEnumerable IterateListAsync( QueryParams baseQuery, int startOffset, int? limit, + int? chunkSize, Func hydrate, RequestTier tier, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct) { + PageSizing.ValidateChunkSize(chunkSize, "ChunkSize"); var offset = Math.Max(startOffset, 0); var yielded = 0; while (true) { var q = baseQuery.Copy().Set("offset", offset); - if (limit is not null) + var pageLimit = PageSizing.PageLimit(chunkSize, limit is null ? null : Math.Max(limit.Value - yielded, 0)); + if (pageLimit is not null) { - q.Set("limit", Math.Max(limit.Value - yielded, 0)); + q.Set("limit", pageLimit.Value); + } + else if (limit is not null) + { + q.Set("limit", 0); } var page = await ListResourceAsync(subPath, q, hydrate, tier, ct).ConfigureAwait(false); @@ -267,7 +273,7 @@ public async Task CreateResourceAsync(QueryParams p, object? body, R { var url = MergedParams(p).ApplyToUrl(SubUrl(string.Empty)); using var response = await Http.CallAsync(HttpMethod.Post, url, Json.Encode(body), ContentTypeJson, _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); - return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); } /// @@ -286,7 +292,7 @@ public async Task GetOrCreateNamedAsync(string? name, JsonNode? sche using var response = schema is not null ? await Http.CallAsync(HttpMethod.Post, url, Json.Encode(new JsonObject { ["schema"] = schema.DeepClone() }), ContentTypeJson, _requestTimeout, tier: RequestTier.Short, cancellationToken: ct).ConfigureAwait(false) : await Http.CallAsync(HttpMethod.Post, url, timeout: _requestTimeout, tier: RequestTier.Short, cancellationToken: ct).ConfigureAwait(false); - return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); } /// @@ -298,7 +304,7 @@ public async Task PostWithBodyAsync(string subPath, QueryParams p, s { var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); using var response = await Http.CallAsync(HttpMethod.Post, url, body, contentType, _requestTimeout, bodyBytes: bodyBytes, tier: tier, cancellationToken: ct).ConfigureAwait(false); - return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); } /// @@ -311,7 +317,7 @@ public async Task PostWithBodyAsync(string subPath, QueryParams p, s { var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); using var response = await Http.CallAsync(HttpMethod.Post, url, body, contentType, _requestTimeout, bodyBytes: bodyBytes, tier: tier, cancellationToken: ct).ConfigureAwait(false); - return Json.Decode(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false)); + return await response.ReadJsonAsync(ct).ConfigureAwait(false); } /// @@ -340,7 +346,7 @@ public async Task DeleteWithBodyAsync(string subPath, QueryParams p, { var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); using var response = await Http.CallAsync(HttpMethod.Delete, url, Json.Encode(body), ContentTypeJson, _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); - return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); } /// @@ -373,6 +379,40 @@ public async Task GetRawRequiredAsync(string subPath, QueryParams p, Req return await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); } + /// + /// GET returning a JSON body that is not wrapped in a data envelope; propagates a 404 (see + /// for why). Returns a JSON null node for an empty body. + /// + public async Task GetJsonRequiredAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); + using var response = await Http.CallAsync(HttpMethod.Get, url, timeout: _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); + return await response.ReadJsonAsync(ct).ConfigureAwait(false); + } + + /// + /// GET returning a JSON body that is not wrapped in a data envelope. Returns null on not-found. + /// + public async Task GetJsonAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) + { + try + { + return await GetJsonRequiredAsync(subPath, p, tier, ct).ConfigureAwait(false); + } + catch (ApifyApiException e) when (HttpClientCore.IsNotFound(e)) + { + return null; + } + } + + /// GET returning a data-enveloped object; propagates a 404 (see for why). + public async Task GetDataObjectRequiredAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); + using var response = await Http.CallAsync(HttpMethod.Get, url, timeout: _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); + } + /// /// GET returning the raw response body as bytes (no data envelope, no UTF-8 decoding — useful when the /// body is not necessarily text). Returns null on not-found. @@ -538,13 +578,6 @@ public async Task WaitForFinishAsync(int? waitSecs, string resourceN $"waiting for {resourceName} to finish failed: cannot fetch {resourceName} details from the server"); } - /// - /// Coerces a decoded value to a JSON object. Endpoints that return a resource object always decode to - /// an object; a non-object (e.g. an unexpected null data field) becomes an empty object so model - /// construction stays type-safe. - /// - private static JsonObject AsObject(JsonNode? value) => value as JsonObject ?? new JsonObject(); - // ---- URL / id helpers ----------------------------------------------------- /// diff --git a/src/Apify.Client/Internal/ResponseReader.cs b/src/Apify.Client/Internal/ResponseReader.cs new file mode 100644 index 0000000..34ca2c1 --- /dev/null +++ b/src/Apify.Client/Internal/ResponseReader.cs @@ -0,0 +1,91 @@ +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Exceptions; + +namespace Apify.Client.Internal; + +/// +/// Reads a successful JSON response into a and reports a body that is unparsable +/// () or lacks the shape the API specification promises +/// (), with the request's method and path as context. Every +/// successful JSON response is decoded through here, matching the reference client's parseResponse. +/// +internal static class ResponseReader +{ + private const string DataEnvelopeKey = "data"; + + /// Reads a JSON body that is not wrapped in a data envelope (null for an empty body). + public static async Task ReadJsonAsync(this HttpResponseMessage response, CancellationToken cancellationToken) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + return Parse(response, body); + } + + /// + /// Reads a body wrapped in the {"data": ...} envelope and returns the unwrapped value. The + /// envelope and a non-null data are required. + /// + public static async Task ReadDataAsync(this HttpResponseMessage response, CancellationToken cancellationToken) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + if (Parse(response, body) is not JsonObject envelope || !envelope.TryGetPropertyValue(DataEnvelopeKey, out var data)) + { + throw Invalid(response, body, "the response is missing the `data` envelope"); + } + + return data ?? throw Invalid(response, body, "the response's `data` is null"); + } + + /// Reads a data-enveloped body whose data must be an object. + public static async Task ReadDataObjectAsync(this HttpResponseMessage response, CancellationToken cancellationToken) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + if (Parse(response, body) is not JsonObject envelope || !envelope.TryGetPropertyValue(DataEnvelopeKey, out var data)) + { + throw Invalid(response, body, "the response is missing the `data` envelope"); + } + + return data as JsonObject ?? throw Invalid(response, body, "the response's `data` is not an object"); + } + + /// Reads a bare (not enveloped) JSON array body, such as a dataset's items. + public static async Task ReadJsonArrayAsync(this HttpResponseMessage response, CancellationToken cancellationToken) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + return Parse(response, body) as JsonArray ?? throw Invalid(response, body, "the response is not a JSON array"); + } + + private static JsonNode? Parse(HttpResponseMessage response, string body) + { + try + { + return Json.Decode(body); + } + catch (JsonException ex) + { + throw new InvalidResponseBodyException( + "response body is not valid JSON: " + ex.Message, + (int)response.StatusCode, + MethodOf(response), + PathOf(response), + body, + ex); + } + } + + private static ResponseValidationException Invalid(HttpResponseMessage response, string body, string problem) + => new( + "response does not match the API specification: " + problem, + (int)response.StatusCode, + MethodOf(response), + PathOf(response), + body); + + private static string MethodOf(HttpResponseMessage response) => response.RequestMessage?.Method.Method ?? string.Empty; + + private static string PathOf(HttpResponseMessage response) + => response.RequestMessage?.RequestUri is { } uri ? HttpClientCore.ExtractPath(uri.ToString()) : string.Empty; +} diff --git a/src/Apify.Client/Options/ActorListOptions.cs b/src/Apify.Client/Options/ActorListOptions.cs index 3acf7a8..442b30c 100644 --- a/src/Apify.Client/Options/ActorListOptions.cs +++ b/src/Apify.Client/Options/ActorListOptions.cs @@ -11,6 +11,13 @@ public sealed class ActorListOptions /// Maximum number of Actors to return. public int? Limit { get; init; } + /// + /// Maximum number of items requested per API call when iterating across pages (must be positive). Only + /// iteration uses it, as the page size; it is never sent to the API. The effective page size is the + /// smaller of this and the remaining Limit. Defaults to the API's own page size. + /// + public int? ChunkSize { get; init; } + /// If true, return Actors newest-first. public bool? Desc { get; init; } diff --git a/src/Apify.Client/Options/DatasetListItemsOptions.cs b/src/Apify.Client/Options/DatasetListItemsOptions.cs index c8f4edf..5b037d8 100644 --- a/src/Apify.Client/Options/DatasetListItemsOptions.cs +++ b/src/Apify.Client/Options/DatasetListItemsOptions.cs @@ -15,6 +15,13 @@ public sealed class DatasetListItemsOptions /// Maximum number of items to return. public int? Limit { get; init; } + /// + /// Maximum number of items requested per API call when iterating across pages (must be positive). Only + /// iteration uses it, as the page size; it is never sent to the API. The effective page size is the + /// smaller of this and the remaining Limit. Defaults to the API's own page size. + /// + public int? ChunkSize { get; init; } + /// Return items newest-first. public bool? Desc { get; init; } diff --git a/src/Apify.Client/Options/IterateKeysOptions.cs b/src/Apify.Client/Options/IterateKeysOptions.cs index e560b5f..8385dc6 100644 --- a/src/Apify.Client/Options/IterateKeysOptions.cs +++ b/src/Apify.Client/Options/IterateKeysOptions.cs @@ -14,8 +14,15 @@ public sealed class IterateKeysOptions public int? Limit { get; init; } /// - /// Maximum number of keys fetched per page (defaults to ). Never sent - /// to the API as-is; it only sizes each page request, matching the reference client's chunkSize. + /// Maximum number of keys requested per API call (must be positive; defaults to + /// ). Never sent to the API as-is; it only sizes each page request. + /// Matches the reference client's chunkSize. + /// + public int? ChunkSize { get; init; } + + /// + /// The earlier name of , kept so existing callers keep working. + /// takes precedence when both are set. /// public int? MaxPageLimit { get; init; } diff --git a/src/Apify.Client/Options/ListOptions.cs b/src/Apify.Client/Options/ListOptions.cs index e028c4d..521899c 100644 --- a/src/Apify.Client/Options/ListOptions.cs +++ b/src/Apify.Client/Options/ListOptions.cs @@ -15,6 +15,13 @@ public sealed class ListOptions /// Maximum number of items to return. public int? Limit { get; init; } + /// + /// Maximum number of items requested per API call when iterating across pages (must be positive). Only + /// iteration uses it, as the page size; it is never sent to the API. The effective page size is the + /// smaller of this and the remaining Limit. Defaults to the API's own page size. + /// + public int? ChunkSize { get; init; } + /// If true, return items newest-first. public bool? Desc { get; init; } diff --git a/src/Apify.Client/Options/PaginateRequestsOptions.cs b/src/Apify.Client/Options/PaginateRequestsOptions.cs index 6db4d7f..3a3f008 100644 --- a/src/Apify.Client/Options/PaginateRequestsOptions.cs +++ b/src/Apify.Client/Options/PaginateRequestsOptions.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using Apify.Client.Internal; namespace Apify.Client.Options; @@ -23,7 +24,16 @@ public sealed class PaginateRequestsOptions /// Maximum total number of requests to iterate across all pages (null for no bound). public int? Limit { get; init; } - /// Maximum number of requests fetched per page (defaults to ). + /// + /// Maximum number of requests fetched per API call (must be positive; defaults to + /// ). Matches the reference client's chunkSize. + /// + public int? ChunkSize { get; init; } + + /// + /// The earlier name of , kept so existing callers keep working. + /// takes precedence when both are set. + /// public int? MaxPageLimit { get; init; } /// Start iterating after this request ID (first page only; mutually exclusive with cursor). @@ -41,6 +51,9 @@ public sealed class PaginateRequestsOptions /// Validates the options for API-level constraints. internal void Validate() { + PageSizing.ValidateChunkSize(ChunkSize, nameof(ChunkSize)); + PageSizing.ValidateChunkSize(MaxPageLimit, nameof(MaxPageLimit)); + if (ExclusiveStartId is not null && Cursor is not null) { throw new ArgumentException("PaginateRequestsOptions: ExclusiveStartId and Cursor are mutually exclusive"); diff --git a/src/Apify.Client/Options/RunIterateDatasetItemsOptions.cs b/src/Apify.Client/Options/RunIterateDatasetItemsOptions.cs index c3c33c7..abd5bd7 100644 --- a/src/Apify.Client/Options/RunIterateDatasetItemsOptions.cs +++ b/src/Apify.Client/Options/RunIterateDatasetItemsOptions.cs @@ -24,7 +24,10 @@ public sealed class RunIterateDatasetItemsOptions /// public int? Limit { get; init; } - /// Maximum number of dataset rows requested per API call (defaults to ). + /// + /// Maximum number of dataset rows requested per API call (must be positive; defaults to + /// ). + /// public int? ChunkSize { get; init; } /// How long to wait for the run to finish between polls, in seconds (defaults to ). diff --git a/src/Apify.Client/Options/StorageListOptions.cs b/src/Apify.Client/Options/StorageListOptions.cs index 1428a11..259162e 100644 --- a/src/Apify.Client/Options/StorageListOptions.cs +++ b/src/Apify.Client/Options/StorageListOptions.cs @@ -15,6 +15,13 @@ public sealed class StorageListOptions /// Maximum number of items to return. public int? Limit { get; init; } + /// + /// Maximum number of items requested per API call when iterating across pages (must be positive). Only + /// iteration uses it, as the page size; it is never sent to the API. The effective page size is the + /// smaller of this and the remaining Limit. Defaults to the API's own page size. + /// + public int? ChunkSize { get; init; } + /// If true, return items newest-first. public bool? Desc { get; init; } diff --git a/src/Apify.Client/Options/StoreListOptions.cs b/src/Apify.Client/Options/StoreListOptions.cs index 4d2da97..bffadf8 100644 --- a/src/Apify.Client/Options/StoreListOptions.cs +++ b/src/Apify.Client/Options/StoreListOptions.cs @@ -11,6 +11,13 @@ public sealed class StoreListOptions /// Maximum number of Actors to return (also the per-page size when iterating). public int? Limit { get; init; } + /// + /// Maximum number of items requested per API call when iterating across pages (must be positive). Only + /// iteration uses it, as the page size; it is never sent to the API. The effective page size is the + /// smaller of this and the remaining Limit. Defaults to the API's own page size. + /// + public int? ChunkSize { get; init; } + /// Full-text search query. public string? Search { get; init; } @@ -39,10 +46,10 @@ public sealed class StoreListOptions public string? ResponseFormat { get; init; } /// Returns a copy of these options with a new (used by lazy iteration). - internal StoreListOptions WithOffset(int? offset) => new() + internal StoreListOptions WithPage(int? offset, int? limit) => new() { Offset = offset, - Limit = Limit, + Limit = limit, Search = Search, SortBy = SortBy, Category = Category, diff --git a/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs b/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs index 93a3b6f..91470ca 100644 --- a/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs +++ b/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs @@ -41,7 +41,7 @@ public IAsyncEnumerable IterateAsync(ListOptions? options = null, Cance options ??= new ListOptions(); var q = new QueryParams(); options.AppendTo(q); - return Ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Webhook(d), RequestTier.Medium, cancellationToken); + return Ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new Webhook(d), RequestTier.Medium, cancellationToken); } } diff --git a/src/Apify.Client/Resources/ActorCollectionClient.cs b/src/Apify.Client/Resources/ActorCollectionClient.cs index 1c88039..cb3b01a 100644 --- a/src/Apify.Client/Resources/ActorCollectionClient.cs +++ b/src/Apify.Client/Resources/ActorCollectionClient.cs @@ -39,7 +39,7 @@ public IAsyncEnumerable IterateAsync(ActorListOptions? options = null, Ca options ??= new ActorListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Actor(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new Actor(d), RequestTier.Medium, cancellationToken); } /// Creates a new Actor. diff --git a/src/Apify.Client/Resources/BuildClient.cs b/src/Apify.Client/Resources/BuildClient.cs index 3b58c6e..6c35e95 100644 --- a/src/Apify.Client/Resources/BuildClient.cs +++ b/src/Apify.Client/Resources/BuildClient.cs @@ -63,12 +63,11 @@ public async Task WaitForFinishAsync(int? waitSecs = null, CancellationTo /// A token to cancel the request. public async Task GetOpenApiDefinitionAsync(CancellationToken cancellationToken = default) { - var body = await _ctx.GetRawAsync("openapi.json", new QueryParams(), RequestTier.Medium, cancellationToken).ConfigureAwait(false); - return body is null ? null : Json.Decode(body) as JsonObject; + return await _ctx.GetJsonAsync("openapi.json", new QueryParams(), RequestTier.Medium, cancellationToken).ConfigureAwait(false) as JsonObject; } /// A client for accessing this build's log. - public LogClient Log() => LogClient.Nested(_http, _ctx.SubUrl("")); + public LogClient Log() => LogClient.Nested(_http, _ctx.SubUrl(""), timeout: _ctx.RequestTimeout); /// /// Returns this client with every subsequent call's timeout set to , diff --git a/src/Apify.Client/Resources/BuildCollectionClient.cs b/src/Apify.Client/Resources/BuildCollectionClient.cs index 55c3b9d..ed355cd 100644 --- a/src/Apify.Client/Resources/BuildCollectionClient.cs +++ b/src/Apify.Client/Resources/BuildCollectionClient.cs @@ -40,7 +40,7 @@ public IAsyncEnumerable IterateAsync(ListOptions? options = null, Cancell options ??= new ListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Build(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new Build(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/DatasetClient.cs b/src/Apify.Client/Resources/DatasetClient.cs index 58312f7..011cb0b 100644 --- a/src/Apify.Client/Resources/DatasetClient.cs +++ b/src/Apify.Client/Resources/DatasetClient.cs @@ -28,8 +28,8 @@ private DatasetClient(HttpClientCore http, ResourceContext ctx) internal static DatasetClient ForId(HttpClientCore http, string baseUrl, string id) => new(http, ResourceContext.Single(http, baseUrl, "datasets", id)); - internal static DatasetClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null) - => new(http, ResourceContext.NestedSingleton(http, baseUrl, subPath, inheritedParams)); + internal static DatasetClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null, TimeSpan? timeout = null) + => new(http, ResourceContext.NestedSingleton(http, baseUrl, subPath, inheritedParams).WithTimeout(timeout)); internal DatasetClient WithPublicBase(string publicBaseUrl) { @@ -102,6 +102,7 @@ public async Task UpdateAsync(object newFields, CancellationToken cance [EnumeratorCancellation] CancellationToken cancellationToken = default) { options ??= new DatasetListItemsOptions(); + PageSizing.ValidateChunkSize(options.ChunkSize, nameof(DatasetListItemsOptions.ChunkSize)); var desc = options.Desc ?? false; var baseQuery = new QueryParams(); options.AppendTo(baseQuery); @@ -111,9 +112,14 @@ public async Task UpdateAsync(object newFields, CancellationToken cance while (true) { var q = baseQuery.Copy().Set("offset", offset); - if (limit is not null) + var pageLimit = PageSizing.PageLimit(options.ChunkSize, limit is null ? null : Math.Max(limit.Value - yielded, 0)); + if (pageLimit is not null) { - q.Set("limit", Math.Max(limit.Value - yielded, 0)); + q.Set("limit", pageLimit.Value); + } + else if (limit is not null) + { + q.Set("limit", 0); } var (page, scannedCount) = await FetchItemsPageAsync(q, desc, cancellationToken).ConfigureAwait(false); @@ -149,15 +155,10 @@ public async Task UpdateAsync(object newFields, CancellationToken cance { var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("items")); using var response = await _http.CallAsync(HttpMethod.Get, url, timeout: _ctx.RequestTimeout, tier: RequestTier.Long, cancellationToken: cancellationToken).ConfigureAwait(false); - var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - var items = new List(); - if (Json.Decode(body) is JsonArray array) + foreach (var item in await response.ReadJsonArrayAsync(cancellationToken).ConfigureAwait(false)) { - foreach (var item in array) - { - items.Add(item); - } + items.Add(item); } var count = items.Count; @@ -249,8 +250,7 @@ public async Task PushItemsAsync(object items, CancellationToken cancellationTok /// A token to cancel the request. public async Task GetStatisticsAsync(CancellationToken cancellationToken = default) { - var body = await _ctx.GetRawRequiredAsync("statistics", new QueryParams(), RequestTier.Short, cancellationToken).ConfigureAwait(false); - return Json.DecodeData(body) as JsonObject ?? new JsonObject(); + return await _ctx.GetDataObjectRequiredAsync("statistics", new QueryParams(), RequestTier.Short, cancellationToken).ConfigureAwait(false); } /// diff --git a/src/Apify.Client/Resources/DatasetCollectionClient.cs b/src/Apify.Client/Resources/DatasetCollectionClient.cs index 577807c..de136ac 100644 --- a/src/Apify.Client/Resources/DatasetCollectionClient.cs +++ b/src/Apify.Client/Resources/DatasetCollectionClient.cs @@ -51,7 +51,7 @@ public IAsyncEnumerable IterateAsync(StorageListOptions? options = null options ??= new StorageListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Dataset(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new Dataset(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/KeyValueStoreClient.cs b/src/Apify.Client/Resources/KeyValueStoreClient.cs index 03afe09..8b3c3a2 100644 --- a/src/Apify.Client/Resources/KeyValueStoreClient.cs +++ b/src/Apify.Client/Resources/KeyValueStoreClient.cs @@ -26,8 +26,8 @@ private KeyValueStoreClient(HttpClientCore http, ResourceContext ctx) internal static KeyValueStoreClient ForId(HttpClientCore http, string baseUrl, string id) => new(http, ResourceContext.Single(http, baseUrl, "key-value-stores", id)); - internal static KeyValueStoreClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null) - => new(http, ResourceContext.NestedSingleton(http, baseUrl, subPath, inheritedParams)); + internal static KeyValueStoreClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null, TimeSpan? timeout = null) + => new(http, ResourceContext.NestedSingleton(http, baseUrl, subPath, inheritedParams).WithTimeout(timeout)); internal KeyValueStoreClient WithPublicBase(string publicBaseUrl) { @@ -74,13 +74,13 @@ public async Task ListKeysAsync(ListKeysOptions? options /// until the store is exhausted. Limit caps the total number of keys yielded across all pages — /// 0 yields none, without issuing any request (the API itself reads a literal limit=0 as /// unbounded, not zero, so this is checked before building the request rather than left to the API), and - /// must not be negative. MaxPageLimit caps the page size (never sent to the API itself) and must - /// be positive, and ExclusiveStartKey chooses the starting point (first page only). + /// must not be negative. ChunkSize caps the page size (never sent to the API itself) and must + /// be positive (MaxPageLimit is its older name), and ExclusiveStartKey chooses the starting point (first page only). /// /// Optional iteration options. /// A token to cancel the iteration. /// - /// is zero or negative, or + /// or is zero or negative, or /// is negative. /// public async IAsyncEnumerable IterateKeysAsync( @@ -88,17 +88,15 @@ public async IAsyncEnumerable IterateKeysAsync( [EnumeratorCancellation] CancellationToken cancellationToken = default) { options ??= new IterateKeysOptions(); - if (options.MaxPageLimit is <= 0) - { - throw new ArgumentException("IterateKeysOptions.MaxPageLimit must be positive.", nameof(options)); - } + PageSizing.ValidateChunkSize(options.ChunkSize, nameof(IterateKeysOptions.ChunkSize)); + PageSizing.ValidateChunkSize(options.MaxPageLimit, nameof(IterateKeysOptions.MaxPageLimit)); if (options.Limit is < 0) { throw new ArgumentException("IterateKeysOptions.Limit must not be negative.", nameof(options)); } - var maxPageLimit = options.MaxPageLimit ?? IterateKeysOptions.DefaultMaxPageLimit; + var maxPageLimit = options.ChunkSize ?? options.MaxPageLimit ?? IterateKeysOptions.DefaultMaxPageLimit; var limit = options.Limit; // total across all pages; null = unbounded var nextExclusiveStartKey = options.ExclusiveStartKey; var iterated = 0; diff --git a/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs b/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs index d2bf650..def2ad5 100644 --- a/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs +++ b/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs @@ -50,7 +50,7 @@ public IAsyncEnumerable IterateAsync(StorageListOptions? options options ??= new StorageListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new KeyValueStore(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new KeyValueStore(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/LogClient.cs b/src/Apify.Client/Resources/LogClient.cs index 35ffceb..dec89a2 100644 --- a/src/Apify.Client/Resources/LogClient.cs +++ b/src/Apify.Client/Resources/LogClient.cs @@ -35,8 +35,8 @@ internal static LogClient ForId(HttpClientCore http, string baseUrl, string id) /// missing — so and throw instead of resolving /// null, matching the reference client's catchNotFoundForResourceOrThrow(). /// - internal static LogClient Nested(HttpClientCore http, string baseUrl, QueryParams? inheritedParams = null) - => new(http, ResourceContext.Collection(http, baseUrl, "log", inheritedParams), ambiguousNotFound: true); + internal static LogClient Nested(HttpClientCore http, string baseUrl, QueryParams? inheritedParams = null, TimeSpan? timeout = null) + => new(http, ResourceContext.Collection(http, baseUrl, "log", inheritedParams).WithTimeout(timeout), ambiguousNotFound: true); /// /// Fetches the log as text. For a log addressed by its own id (), a 404 diff --git a/src/Apify.Client/Resources/RequestQueueClient.cs b/src/Apify.Client/Resources/RequestQueueClient.cs index 412ef26..9036740 100644 --- a/src/Apify.Client/Resources/RequestQueueClient.cs +++ b/src/Apify.Client/Resources/RequestQueueClient.cs @@ -49,8 +49,8 @@ internal static RequestQueueClient ForId(HttpClientCore http, string baseUrl, st return new RequestQueueClient(http, ctx, options?.ClientKey, timeout); } - internal static RequestQueueClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null) - => new(http, ResourceContext.NestedSingleton(http, baseUrl, subPath, inheritedParams), null, null); + internal static RequestQueueClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null, TimeSpan? timeout = null) + => new(http, ResourceContext.NestedSingleton(http, baseUrl, subPath, inheritedParams).WithTimeout(timeout), null, timeout); /// /// Returns a copy of the client that identifies its requests with . A @@ -130,8 +130,7 @@ public async Task UpdateRequestAsync(RequestQueueRequ ApplyClientKey(q); var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("requests/" + ResourceContext.EncodePathSegment(request.Id ?? string.Empty))); using var response = await _http.CallAsync(HttpMethod.Put, url, Json.Encode(request.ToJsonObject()), ResourceContext.ContentTypeJson, _timeout, tier: RequestTier.Medium, cancellationToken: cancellationToken).ConfigureAwait(false); - var data = Json.DecodeData(await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)); - return new RequestQueueOperationInfo(data as JsonObject ?? new JsonObject()); + return new RequestQueueOperationInfo(await response.ReadDataObjectAsync(cancellationToken).ConfigureAwait(false)); } /// Deletes a request by ID. @@ -483,8 +482,7 @@ public async Task ProlongRequestLockAsync(string id, int lockSe ApplyClientKey(q); var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("requests/" + ResourceContext.EncodePathSegment(id) + "/lock")); using var response = await _http.CallAsync(HttpMethod.Put, url, null, "", _timeout, tier: RequestTier.Medium, cancellationToken: cancellationToken).ConfigureAwait(false); - var data = Json.DecodeData(await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)); - return RequestLockInfo.FromData(data); + return RequestLockInfo.FromData(await response.ReadDataObjectAsync(cancellationToken).ConfigureAwait(false)); } /// @@ -526,7 +524,7 @@ public async Task UnlockRequestsAsync(CancellationToken ca /// /// With no options it fetches pages of up to /// requests until the queue is exhausted. The options mirror the reference client: Limit caps the - /// total number of requests yielded across all pages, MaxPageLimit caps the page size, + /// total number of requests yielded across all pages, ChunkSize (or its older name MaxPageLimit) caps the page size, /// ExclusiveStartId/Cursor choose the starting point (first page only), and Filter /// restricts to locked/pending requests. /// @@ -539,7 +537,7 @@ public async IAsyncEnumerable PaginateRequestsAsync( options ??= new PaginateRequestsOptions(); options.Validate(); - var maxPageLimit = options.MaxPageLimit ?? PaginateRequestsOptions.DefaultMaxPageLimit; + var maxPageLimit = options.ChunkSize ?? options.MaxPageLimit ?? PaginateRequestsOptions.DefaultMaxPageLimit; var limit = options.Limit; // total across all pages; null = unbounded var nextCursor = options.Cursor; var nextExclusiveStartId = options.ExclusiveStartId; // used for the first page only diff --git a/src/Apify.Client/Resources/RequestQueueCollectionClient.cs b/src/Apify.Client/Resources/RequestQueueCollectionClient.cs index 298a649..bb865cd 100644 --- a/src/Apify.Client/Resources/RequestQueueCollectionClient.cs +++ b/src/Apify.Client/Resources/RequestQueueCollectionClient.cs @@ -48,7 +48,7 @@ public IAsyncEnumerable IterateAsync(StorageListOptions? options = options ??= new StorageListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new RequestQueue(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new RequestQueue(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/RunClient.cs b/src/Apify.Client/Resources/RunClient.cs index 80298c8..d42ca04 100644 --- a/src/Apify.Client/Resources/RunClient.cs +++ b/src/Apify.Client/Resources/RunClient.cs @@ -256,14 +256,16 @@ internal async Task WaitForFinishWithLogAsync(int? waitSecs, Action 0 ? options.ChunkSize.Value : RunIterateDatasetItemsOptions.DefaultChunkSize; + PageSizing.ValidateChunkSize(options.ChunkSize, nameof(RunIterateDatasetItemsOptions.ChunkSize)); + var chunkSize = options.ChunkSize ?? RunIterateDatasetItemsOptions.DefaultChunkSize; var pollIntervalSecs = options.PollIntervalSecs ?? RunIterateDatasetItemsOptions.DefaultPollIntervalSecs; - ArgumentOutOfRangeException.ThrowIfNegative(pollIntervalSecs, nameof(options)); - ArgumentOutOfRangeException.ThrowIfNegative(options.Offset ?? 0, nameof(options)); - ArgumentOutOfRangeException.ThrowIfNegative(options.Limit ?? 0, nameof(options)); + ArgumentOutOfRangeException.ThrowIfNegative(pollIntervalSecs, nameof(RunIterateDatasetItemsOptions.PollIntervalSecs)); + ArgumentOutOfRangeException.ThrowIfNegative(options.Offset ?? 0, nameof(RunIterateDatasetItemsOptions.Offset)); + ArgumentOutOfRangeException.ThrowIfNegative(options.Limit ?? 0, nameof(RunIterateDatasetItemsOptions.Limit)); var position = options.Offset ?? 0; - int? end = options.Limit is > 0 ? position + options.Limit.Value : null; + // Wider than int: Offset + Limit can exceed int.MaxValue (e.g. Limit = int.MaxValue as "everything"). + long? end = options.Limit is > 0 ? (long)position + options.Limit.Value : null; var run = await GetAsync(cancellationToken: cancellationToken).ConfigureAwait(false); // A last-run client resolves `runs/last` on each request, so a newer run would swap the dataset @@ -318,7 +320,7 @@ internal async Task WaitForFinishWithLogAsync(int? waitSecs, Action IsDatasetExhaustedAsync(DatasetClient datasetCli return page.Items.Count == 0; } - // Nested accessors inherit this client's params so last-run status/origin filters (see - // SetLastRunParams) resolve the intended run's storage/log rather than the latest run's. + // Nested accessors inherit this client's params, so last-run status/origin filters (see SetLastRunParams) + // resolve the intended run's storage/log rather than the latest run's, and its WithTimeout override. /// A client for this run's default dataset. - public DatasetClient Dataset() => DatasetClient.Nested(_http, _ctx.SubUrl(""), "dataset", _ctx.BaseParams); + public DatasetClient Dataset() => DatasetClient.Nested(_http, _ctx.SubUrl(""), "dataset", _ctx.BaseParams, _ctx.RequestTimeout); /// A client for this run's default key-value store. - public KeyValueStoreClient KeyValueStore() => KeyValueStoreClient.Nested(_http, _ctx.SubUrl(""), "key-value-store", _ctx.BaseParams); + public KeyValueStoreClient KeyValueStore() => KeyValueStoreClient.Nested(_http, _ctx.SubUrl(""), "key-value-store", _ctx.BaseParams, _ctx.RequestTimeout); /// A client for this run's default request queue. - public RequestQueueClient RequestQueue() => RequestQueueClient.Nested(_http, _ctx.SubUrl(""), "request-queue", _ctx.BaseParams); + public RequestQueueClient RequestQueue() => RequestQueueClient.Nested(_http, _ctx.SubUrl(""), "request-queue", _ctx.BaseParams, _ctx.RequestTimeout); /// A client for accessing this run's log. - public LogClient Log() => LogClient.Nested(_http, _ctx.SubUrl(""), _ctx.BaseParams); + public LogClient Log() => LogClient.Nested(_http, _ctx.SubUrl(""), _ctx.BaseParams, _ctx.RequestTimeout); /// /// Opens a live stream of this run's raw log bytes. The caller reads (and disposes) the returned diff --git a/src/Apify.Client/Resources/RunCollectionClient.cs b/src/Apify.Client/Resources/RunCollectionClient.cs index c1f63de..23f3309 100644 --- a/src/Apify.Client/Resources/RunCollectionClient.cs +++ b/src/Apify.Client/Resources/RunCollectionClient.cs @@ -50,7 +50,7 @@ public IAsyncEnumerable IterateAsync( var q = new QueryParams(); options.AppendTo(q); (filter ?? new RunListOptions()).AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new ActorRun(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new ActorRun(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/ScheduleCollectionClient.cs b/src/Apify.Client/Resources/ScheduleCollectionClient.cs index ff26c93..e7985e9 100644 --- a/src/Apify.Client/Resources/ScheduleCollectionClient.cs +++ b/src/Apify.Client/Resources/ScheduleCollectionClient.cs @@ -45,7 +45,7 @@ public IAsyncEnumerable IterateAsync(ListOptions? options = null, Canc options ??= new ListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Schedule(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new Schedule(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/StoreCollectionClient.cs b/src/Apify.Client/Resources/StoreCollectionClient.cs index 4b32a8c..c1ffbf2 100644 --- a/src/Apify.Client/Resources/StoreCollectionClient.cs +++ b/src/Apify.Client/Resources/StoreCollectionClient.cs @@ -30,20 +30,23 @@ public Task> ListAsync(StoreListOptions? opti } /// - /// Lazily iterates over all Store Actors matching the options, fetching pages on demand. The options' - /// Limit (if set) is used as the per-page size. + /// Lazily iterates over all Store Actors matching the options, fetching pages on demand. The page size + /// is the options' ChunkSize, falling back to Limit (which here is the per-page size, not + /// a total cap), falling back to the API default. /// - /// Optional listing filters; Limit is used as the page size. + /// Optional listing filters; ChunkSize (or else Limit) is the page size. /// A token to cancel the iteration. public async IAsyncEnumerable IterateAsync( StoreListOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { options ??= new StoreListOptions(); + PageSizing.ValidateChunkSize(options.ChunkSize, nameof(StoreListOptions.ChunkSize)); + var pageSize = options.ChunkSize ?? options.Limit; var offset = options.Offset ?? 0; while (true) { - var page = await ListAsync(options.WithOffset(offset), cancellationToken).ConfigureAwait(false); + var page = await ListAsync(options.WithPage(offset, pageSize), cancellationToken).ConfigureAwait(false); var items = page.Items; foreach (var item in items) { diff --git a/src/Apify.Client/Resources/TaskClient.cs b/src/Apify.Client/Resources/TaskClient.cs index 7f4261b..27e1a05 100644 --- a/src/Apify.Client/Resources/TaskClient.cs +++ b/src/Apify.Client/Resources/TaskClient.cs @@ -141,8 +141,7 @@ public async Task CallAsync( /// A token to cancel the request. public async Task GetInputAsync(CancellationToken cancellationToken = default) { - var body = await _ctx.GetRawRequiredAsync("input", new QueryParams(), RequestTier.Short, cancellationToken).ConfigureAwait(false); - return Json.Decode(body); + return await _ctx.GetJsonRequiredAsync("input", new QueryParams(), RequestTier.Short, cancellationToken).ConfigureAwait(false); } /// Replaces the task's stored input and returns the updated input. @@ -158,7 +157,7 @@ public async Task CallAsync( timeout: _ctx.RequestTimeout, tier: RequestTier.Short, cancellationToken: cancellationToken).ConfigureAwait(false); - return Json.Decode(await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)); + return await response.ReadJsonAsync(cancellationToken).ConfigureAwait(false); } /// Returns a client for the last run of this task, optionally filtered by status and/or origin. diff --git a/src/Apify.Client/Resources/TaskCollectionClient.cs b/src/Apify.Client/Resources/TaskCollectionClient.cs index 288a8e5..76410f9 100644 --- a/src/Apify.Client/Resources/TaskCollectionClient.cs +++ b/src/Apify.Client/Resources/TaskCollectionClient.cs @@ -45,7 +45,7 @@ public IAsyncEnumerable IterateAsync(ListOptions? options = null, Can options ??= new ListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new ActorTask(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new ActorTask(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs b/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs index c5a5e7b..7134177 100644 --- a/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs +++ b/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs @@ -40,7 +40,7 @@ public IAsyncEnumerable IterateAsync(ListOptions? options = nul options ??= new ListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new WebhookDispatch(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new WebhookDispatch(d), RequestTier.Medium, cancellationToken); } diff --git a/tests/Apify.Client.Tests/Apify.Client.Tests.csproj b/tests/Apify.Client.Tests/Apify.Client.Tests.csproj index 4c62faf..9f2b502 100644 --- a/tests/Apify.Client.Tests/Apify.Client.Tests.csproj +++ b/tests/Apify.Client.Tests/Apify.Client.Tests.csproj @@ -14,6 +14,8 @@ + + diff --git a/tests/Apify.Client.Tests/Examples/DocumentationSnippetTests.cs b/tests/Apify.Client.Tests/Examples/DocumentationSnippetTests.cs new file mode 100644 index 0000000..b73e846 --- /dev/null +++ b/tests/Apify.Client.Tests/Examples/DocumentationSnippetTests.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Xunit; + +namespace Apify.Client.Tests.Examples; + +/// +/// Compiles every csharp code block of the top-level README and of docs/*.md against the +/// library, so a snippet that no longer compiles (a renamed method, a missing argument, a bare fragment) +/// fails CI. The runnable scenarios in docs/examples.md are additionally executed by +/// . +/// +/// +/// A block that starts with using directives is a complete program (top-level statements). Any +/// other block is a fragment that continues the most recent complete block of the same file, so it is +/// compiled after that block's directives and declarations. +/// +[Trait("Category", "Examples")] +public sealed class DocumentationSnippetTests +{ + private static readonly Regex CodeBlock = new(@"```csharp\r?\n(?.*?)```", RegexOptions.Singleline | RegexOptions.Compiled); + + public static IEnumerable Snippets() + { + var root = FindRepositoryRoot(); + var files = new[] { Path.Combine(root, "README.md") } + .Concat(Directory.GetFiles(Path.Combine(root, "docs"), "*.md").OrderBy(static f => f, StringComparer.Ordinal)); + foreach (var file in files) + { + var relative = Path.GetRelativePath(root, file).Replace('\\', '/'); + var lastComplete = string.Empty; + var index = 0; + foreach (Match match in CodeBlock.Matches(File.ReadAllText(file))) + { + index++; + var code = match.Groups["code"].Value; + var isComplete = code.TrimStart().StartsWith("using ", StringComparison.Ordinal); + var program = isComplete ? code : lastComplete + "\n" + code; + if (isComplete) + { + lastComplete = code; + } + + yield return new object[] { relative + " snippet #" + index, program }; + } + } + } + + [Theory] + [MemberData(nameof(Snippets))] + public void SnippetCompiles(string name, string program) + { + var tree = CSharpSyntaxTree.ParseText(program, new CSharpParseOptions(LanguageVersion.Latest)); + var compilation = CSharpCompilation.Create( + "DocSnippet", + new[] { tree }, + References(), + new CSharpCompilationOptions(OutputKind.ConsoleApplication, nullableContextOptions: NullableContextOptions.Enable)); + + var errors = compilation.GetDiagnostics() + .Where(static d => d.Severity == DiagnosticSeverity.Error) + .Select(static d => d.ToString()) + .ToList(); + + Assert.True(errors.Count == 0, name + " does not compile:\n" + string.Join("\n", errors)); + } + + private static IEnumerable References() + { + var trusted = (string)AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES")!; + foreach (var path in trusted.Split(Path.PathSeparator)) + { + yield return MetadataReference.CreateFromFile(path); + } + + yield return MetadataReference.CreateFromFile(typeof(ApifyClient).Assembly.Location); + } + + private static string FindRepositoryRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "Apify.Client.sln"))) + { + directory = directory.Parent; + } + + return directory?.FullName ?? throw new InvalidOperationException("Could not locate the repository root (Apify.Client.sln)."); + } +} diff --git a/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs index ce68f4f..8c08d68 100644 --- a/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs @@ -1,3 +1,5 @@ +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Apify.Client.Options; using Xunit; @@ -44,22 +46,32 @@ public async Task IterateRunDatasetItems() var client = RequireClient(); var run = await client.Actor("apify/hello-world").CallAsync(null, null, 120); - var iterated = 0; + var iterated = new List(); await foreach (var item in client.Run(run.Id!).IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { ChunkSize = 1 })) { Assert.NotNull(item); - iterated++; + iterated.Add(item!.ToJsonString()); } - // The run is finished, so its dataset is complete. The dataset's reported item count lags the pushed - // items by a few seconds (which is why the iterator drains past it), so it must catch up to what the - // iterator already read. + // hello-world pushes an item, so an iterator that yields nothing is a failure. + Assert.NotEmpty(iterated); + + // The run is finished, so its dataset is complete. The dataset listing lags the pushed items by a few + // seconds (which is why the iterator drains past the reported item count), so it must catch up to + // exactly the items the iterator already read. + var listed = new List(); Assert.True( await PollUntilAsync( EventualConsistencyAttempts, EventualConsistencyBackoff, - async () => (await client.Run(run.Id!).Dataset().ListItemsAsync(new DatasetListItemsOptions())).Total == iterated), - "expected the dataset's item count to catch up with the iterated items"); + async () => + { + var page = await client.Run(run.Id!).Dataset().ListItemsAsync(new DatasetListItemsOptions()); + listed = page.Items.Select(static i => i!.ToJsonString()).ToList(); + return listed.Count == iterated.Count; + }), + "expected the dataset listing to catch up with the iterated items"); + Assert.Equal(listed, iterated); } [SkippableFact] diff --git a/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs index dcec197..082ab4a 100644 --- a/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs @@ -358,7 +358,7 @@ await PollUntilAsync( async () => { seen.Clear(); - await foreach (var item in dataset.IterateItemsAsync()) + await foreach (var item in dataset.IterateItemsAsync(new DatasetListItemsOptions { ChunkSize = 2 })) { seen.Add(item!["n"]!.GetValue()); } @@ -369,12 +369,12 @@ await PollUntilAsync( Assert.Equal(Enumerable.Range(0, total), seen); var limited = new List(); - await foreach (var item in dataset.IterateItemsAsync(new DatasetListItemsOptions { Offset = 1, Limit = 2 })) + await foreach (var item in dataset.IterateItemsAsync(new DatasetListItemsOptions { Offset = 1, Limit = 3, ChunkSize = 2 })) { limited.Add(item!["n"]!.GetValue()); } - Assert.Equal(new[] { 1, 2 }, limited); + Assert.Equal(new[] { 1, 2, 3 }, limited); } finally { diff --git a/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs index a7d0187..747b970 100644 --- a/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs +++ b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs @@ -268,7 +268,7 @@ public async Task KeyValueStoreIterateKeysRejectsNonPositiveMaxPageLimit(int max { var transport = new MockTransport(); - await Assert.ThrowsAsync(async () => + await Assert.ThrowsAsync(async () => { await foreach (var _ in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { MaxPageLimit = maxPageLimit })) { diff --git a/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs b/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs new file mode 100644 index 0000000..5ab2a69 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs @@ -0,0 +1,128 @@ +using System; +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// Verifies the ChunkSize iteration option, which sizes each page's limit (the smaller of the +/// chunk size and the remaining total Limit) and is never sent as a query parameter itself. +/// +[Trait("Category", "Unit")] +public sealed class ChunkSizeTests +{ + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + private static string Page(params string[] ids) + => "{\"data\":{\"total\":99,\"items\":[" + string.Join(",", Array.ConvertAll(ids, static id => "{\"id\":\"" + id + "\"}")) + "]}}"; + + private static async Task DrainAsync(System.Collections.Generic.IAsyncEnumerable items) + { + await foreach (var unused in items) + { + _ = unused; + } + } + + [Fact] + public async Task CollectionIterationRequestsTheSmallerOfChunkSizeAndRemainingLimit() + { + var transport = new MockTransport() + .QueueResponse(200, Page("a", "b")) + .QueueResponse(200, Page("c", "d")) + .QueueResponse(200, Page("e")); + + await DrainAsync(Client(transport).Tasks().IterateAsync(new ListOptions { Limit = 5, ChunkSize = 2 })); + + Assert.Equal(3, transport.CallCount); + Assert.Contains("limit=2", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.Contains("limit=2", transport.Received[1].Uri, StringComparison.Ordinal); + Assert.Contains("limit=1", transport.Received[2].Uri, StringComparison.Ordinal); + Assert.DoesNotContain("chunkSize", transport.Received[0].Uri, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task CollectionIterationWithoutLimitUsesChunkSizeAsPageLimit() + { + var transport = new MockTransport().QueueResponse(200, Page("a")).QueueResponse(200, Page()); + + await DrainAsync(Client(transport).Actors().IterateAsync(new ActorListOptions { ChunkSize = 7 })); + + Assert.All(transport.Received, static r => Assert.Contains("limit=7", r.Uri, StringComparison.Ordinal)); + } + + [Fact] + public async Task ZeroLimitDoesNotBeatChunkSize() + { + // The API reads limit=0 as unset, so it must lose to the chunk size. + var transport = new MockTransport().QueueResponse(200, Page("a")); + + await DrainAsync(Client(transport).Schedules().IterateAsync(new ListOptions { Limit = 0, ChunkSize = 3 })); + + Assert.Contains("limit=3", transport.Received[0].Uri, StringComparison.Ordinal); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public async Task NonPositiveChunkSizeIsRejectedBeforeAnyRequest(int chunkSize) + { + var transport = new MockTransport(); + + var ex = await Assert.ThrowsAsync(() => DrainAsync(Client(transport).Runs().IterateAsync(new ListOptions { ChunkSize = chunkSize }))); + + Assert.Equal("ChunkSize", ex.ParamName); + Assert.Equal(0, transport.CallCount); + } + + [Fact] + public async Task DatasetItemIterationSizesPagesByChunkSize() + { + var headers = new System.Collections.Generic.Dictionary { ["X-Apify-Pagination-Total"] = "3" }; + var transport = new MockTransport() + .QueueResponse(200, "[{\"n\":1},{\"n\":2}]", headers) + .QueueResponse(200, "[{\"n\":3}]", headers); + + await DrainAsync(Client(transport).Dataset("ds1").IterateItemsAsync(new DatasetListItemsOptions { Limit = 3, ChunkSize = 2 })); + + Assert.Contains("limit=2", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.Contains("limit=1", transport.Received[1].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task KeyIterationHonoursChunkSizeOverItsOlderName() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[],\"isTruncated\":false}}"); + + await DrainAsync(Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { ChunkSize = 4, MaxPageLimit = 9 })); + + Assert.Contains("limit=4", transport.Received[0].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task RequestIterationHonoursChunkSize() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[],\"limit\":5}}"); + + await DrainAsync(Client(transport).RequestQueue("q1").PaginateRequestsAsync(new PaginateRequestsOptions { ChunkSize = 5 })); + + Assert.Contains("limit=5", transport.Received[0].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task StoreIterationUsesChunkSizeAsPageSize() + { + var transport = new MockTransport().QueueResponse(200, Page()); + + await DrainAsync(Client(transport).Store().IterateAsync(new StoreListOptions { ChunkSize = 6, Limit = 100 })); + + Assert.Contains("limit=6", transport.Received[0].Uri, StringComparison.Ordinal); + } +} diff --git a/tests/Apify.Client.Tests/Unit/ResponseValidationTests.cs b/tests/Apify.Client.Tests/Unit/ResponseValidationTests.cs new file mode 100644 index 0000000..fb2752f --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/ResponseValidationTests.cs @@ -0,0 +1,87 @@ +using System; +using System.Threading.Tasks; +using Apify.Client.Exceptions; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// Verifies how a successful (2xx) response that is not what the API specification promises is reported: +/// unparsable JSON as , a missing data envelope or a +/// wrongly-typed data as , both with the request context. +/// +[Trait("Category", "Unit")] +public sealed class ResponseValidationTests +{ + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + [Fact] + public async Task UnparsableBodyExceptionCarriesStatusMethodPathAndBody() + { + var transport = new MockTransport().QueueResponse(200, "oops"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actor("me~act").GetAsync()); + + Assert.Equal(200, ex.StatusCode); + Assert.Equal("GET", ex.HttpMethod); + Assert.EndsWith("/actors/me~act", ex.Path, StringComparison.Ordinal); + Assert.Equal("oops", ex.Body); + } + + [Fact] + public async Task MissingDataEnvelopeOnGetThrowsInsteadOfLookingLikeNotFound() + { + var transport = new MockTransport().QueueResponse(200, "{}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actor("me~act").GetAsync()); + + Assert.IsAssignableFrom(ex); + Assert.Equal(200, ex.StatusCode); + Assert.Equal("GET", ex.HttpMethod); + Assert.Equal("{}", ex.Body); + Assert.Equal(1, transport.CallCount); + } + + [Fact] + public async Task NullDataOnGetThrows() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":null}"); + + await Assert.ThrowsAsync(() => Client(transport).Actor("me~act").GetAsync()); + } + + [Fact] + public async Task ArrayDataWhereAnObjectIsExpectedThrowsOnUpdate() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":[]}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actor("me~act").UpdateAsync(new { title = "x" })); + + Assert.Equal("PUT", ex.HttpMethod); + } + + [Fact] + public async Task MissingDataEnvelopeOnCreateThrows() + { + var transport = new MockTransport().QueueResponse(201, "{\"id\":\"a\"}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actors().CreateAsync(new { name = "x" })); + + Assert.Equal("POST", ex.HttpMethod); + Assert.Equal(201, ex.StatusCode); + } + + [Fact] + public async Task NonArrayDatasetItemsBodyThrows() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":[]}"); + + await Assert.ThrowsAsync(() => Client(transport).Dataset("ds1").ListItemsAsync()); + } +} diff --git a/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs b/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs index 85bd727..465f9b5 100644 --- a/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs +++ b/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs @@ -122,21 +122,69 @@ public async Task FilteredEmptyTailIsConfirmedWithAPlainRead() .QueueResponse(200, FinishedRun) .QueueResponse(200, DatasetWithItemCount(0)) .QueueResponse(200, Items()) // filtered page: empty, nothing scanned - .QueueResponse(200, Items(7)) // plain read finds a row the filter page missed - .QueueResponse(200, Items(7)) - .QueueResponse(200, Items()) - .QueueResponse(200, Items()); + .QueueResponse(200, "[{}]") // plain read: a row the filter dropped, so iteration must go on + .QueueResponse(200, Items()) // next filtered page + .QueueResponse(200, Items()); // plain read: nothing left var items = await CollectAsync(Client(transport).Run("run1").IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { SkipEmpty = true })); - Assert.Equal(new[] { 7 }, items); + Assert.Empty(items); + Assert.Equal(6, transport.CallCount); + Assert.Contains("skipEmpty=1", transport.Received[2].Uri, StringComparison.Ordinal); + Assert.Contains("offset=0", transport.Received[3].Uri, StringComparison.Ordinal); + Assert.DoesNotContain("skipEmpty", transport.Received[3].Uri, StringComparison.Ordinal); + Assert.Contains("offset=1000", transport.Received[4].Uri, StringComparison.Ordinal); + Assert.Contains("offset=1000", transport.Received[5].Uri, StringComparison.Ordinal); } [Fact] - public async Task NegativeArgumentsAreRejected() + public async Task OffsetPlusLimitBeyondIntRangeStillIteratesAllRows() { - var run = Client(new MockTransport()).Run("run1"); - await Assert.ThrowsAsync(() => CollectAsync(run.IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { Offset = -1 }))); - await Assert.ThrowsAsync(() => CollectAsync(run.IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { PollIntervalSecs = -1 }))); + var transport = new MockTransport() + .QueueResponse(200, FinishedRun) + .QueueResponse(200, DatasetWithItemCount(3)) + .QueueResponse(200, Items(2, 3)) + .QueueResponse(200, Items()); + + var items = await CollectAsync(Client(transport).Run("run1").IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { Offset = 1, Limit = int.MaxValue })); + + Assert.Equal(new[] { 2, 3 }, items); + } + + [Fact] + public async Task RunTimeoutAppliesToTheDatasetRequests() + { + var transport = new MockTransport() + .QueueResponse(200, FinishedRun) + .QueueResponse(200, DatasetWithItemCount(1)) + .QueueResponse(200, Items(1)) + .QueueResponse(200, Items()); + + await CollectAsync(Client(transport).Run("run1").WithTimeout(TimeSpan.FromSeconds(3)).IterateDatasetItemsAsync()); + + Assert.All(transport.Timeouts, static t => Assert.Equal(3, t)); + } + + [Theory] + [InlineData("Offset", -1)] + [InlineData("Limit", -1)] + [InlineData("PollIntervalSecs", -1)] + [InlineData("ChunkSize", 0)] + [InlineData("ChunkSize", -1)] + public async Task InvalidArgumentsAreRejectedNamingTheOption(string option, int value) + { + var options = option switch + { + "Offset" => new RunIterateDatasetItemsOptions { Offset = value }, + "Limit" => new RunIterateDatasetItemsOptions { Limit = value }, + "PollIntervalSecs" => new RunIterateDatasetItemsOptions { PollIntervalSecs = value }, + _ => new RunIterateDatasetItemsOptions { ChunkSize = value }, + }; + var transport = new MockTransport(); + + var ex = await Assert.ThrowsAsync(() => CollectAsync(Client(transport).Run("run1").IterateDatasetItemsAsync(options))); + + Assert.Equal(option, ex.ParamName); + Assert.Equal(0, transport.CallCount); } } From a45650e55749cceb3682af07f93c36f0c9b14d98 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 14:02:33 +0000 Subject: [PATCH 4/8] fix: validate responses against the spec schemas, align store/dataset iteration with the reference, add timeout tests Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01UUMWnU4gH573HweHieMhHi --- CHANGELOG.md | 12 +- docs/README.md | 6 +- docs/misc.md | 6 +- src/Apify.Client/Apify.Client.csproj | 5 + src/Apify.Client/Internal/PageSizing.cs | 23 +- src/Apify.Client/Internal/ResourceContext.cs | 20 +- src/Apify.Client/Internal/ResponseReader.cs | 52 ++-- src/Apify.Client/Internal/ResponseSchemas.cs | 289 ++++++++++++++++++ .../Internal/ResponseSchemas.json | 1 + src/Apify.Client/Options/StoreListOptions.cs | 14 - src/Apify.Client/Resources/ActorClient.cs | 2 +- src/Apify.Client/Resources/DatasetClient.cs | 15 +- .../Resources/StoreCollectionClient.cs | 33 +- src/Apify.Client/Resources/UserClient.cs | 6 +- .../Integration/StoreIntegrationTests.cs | 16 +- .../Unit/AutoPagingTests.cs | 58 ++-- .../Apify.Client.Tests/Unit/ChunkSizeTests.cs | 71 ++++- .../Unit/HttpClientTests.cs | 25 +- .../Unit/RequestShapeTests.cs | 38 +-- .../Unit/ResponseDecompressionTests.cs | 2 +- .../Unit/ResponseSchemaTests.cs | 136 +++++++++ .../Unit/RunIterateDatasetItemsTests.cs | 8 +- .../Unit/RunNestedClientTimeoutTests.cs | 92 ++++++ tests/Apify.Client.Tests/Unit/SpecFixtures.cs | 95 ++++++ .../Unit/WaitForFinishTests.cs | 2 +- tools/generate_response_schemas.py | 109 +++++++ 26 files changed, 958 insertions(+), 178 deletions(-) create mode 100644 src/Apify.Client/Internal/ResponseSchemas.cs create mode 100644 src/Apify.Client/Internal/ResponseSchemas.json create mode 100644 tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs create mode 100644 tests/Apify.Client.Tests/Unit/RunNestedClientTimeoutTests.cs create mode 100644 tests/Apify.Client.Tests/Unit/SpecFixtures.cs create mode 100644 tools/generate_response_schemas.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0454156..7fcb72b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,15 +12,21 @@ body is not valid JSON, instead of a raw `System.Text.Json.JsonException`. It is not retried and exposes the status code, HTTP method, path and raw body. An error status with an unparsable body keeps throwing the `ApifyApiException` for its status code. -- Added `ResponseValidationException` (an `ApifyClientException`), thrown when a successful response is - missing the `data` envelope or has a `data` of the wrong type (for example an array where an object is - expected), instead of silently returning `null` or an empty object. Matches the reference client's +- Added `ResponseValidationException` (an `ApifyClientException`), thrown when a successful response does + not match the schema the OpenAPI specification declares for its operation (missing `data` envelope, + missing required properties, wrong JSON types, `null` where it is not allowed), instead of silently + returning `null` or an empty object. Unknown properties and enum values are accepted. The schemas are + embedded and generated by `tools/generate_response_schemas.py`. Matches the reference client's `ResponseValidationError`. - Added `ChunkSize` to `ListOptions`, `StorageListOptions`, `ActorListOptions`, `StoreListOptions`, `DatasetListItemsOptions`, `IterateKeysOptions` and `PaginateRequestsOptions`: the page size iterators request (the smaller of it and the remaining `Limit`), matching the reference client's `chunkSize`. `MaxPageLimit` stays as the older name of the same option. A non-positive `ChunkSize` throws `ArgumentOutOfRangeException`. +- Fixed `StoreCollectionClient.IterateAsync` to treat `Limit` as a cap on the total number of Actors + yielded (as every other iterator does) instead of as the page size, and made it, like + `DatasetClient.IterateItemsAsync`, keep paging until a page is empty instead of stopping at the total + an earlier page reported, so items added while iterating are no longer missed. - Nested accessors of a run (`Dataset()`, `KeyValueStore()`, `RequestQueue()`, `Log()`) and of a build (`Log()`) now inherit the run's or build's `WithTimeout` override. - Added a documentation test that compiles every C# snippet in the README and `docs/`. diff --git a/docs/README.md b/docs/README.md index 6e2e487..26a22b9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -206,8 +206,10 @@ catch (ApifyApiException e) timeout, or DNS error) after all retries are exhausted. `InvalidResponseBodyException` is thrown when a successful response's body is not valid JSON, and -`ResponseValidationException` when it is valid JSON but does not have the shape the API specification -promises (for example the `data` envelope is missing). The response arrived in full, so neither is +`ResponseValidationException` when it is valid JSON but does not match the schema the API specification +declares for that operation (for example the `data` envelope or a required property is missing, or a +property has the wrong JSON type). Unknown properties and enum values are accepted, so new API fields do +not break the client. The response arrived in full, so neither is retried. Both expose `StatusCode`, `HttpMethod`, `Path` and the raw `Body`. An error status with an unparsable body throws the `ApifyApiException` for its status code, with the raw body in its message. diff --git a/docs/misc.md b/docs/misc.md index 0a19e58..8871884 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -9,15 +9,15 @@ Browse public Actors in the [Apify Store](https://apify.com/store). - `ListAsync(StoreListOptions? options = null)` → `PaginationList` (one page). - `IterateAsync(StoreListOptions? options = null)` → `IAsyncEnumerable` (lazy, all - pages; `Limit` is the page size). + pages; `Limit` caps the total number of Actors yielded and `ChunkSize` sets the page size). `StoreListOptions` fields: | Field | Type | Description | |---|---|---| | `Offset` | `int?` | Number of Actors to skip from the start. | -| `Limit` | `int?` | Maximum number of Actors to return in the page. | -| `ChunkSize` | `int?` | Page size used by `IterateAsync` (positive); takes precedence over `Limit` as the page size. Never sent to the API. | +| `Limit` | `int?` | Maximum number of Actors to return (for `IterateAsync`, in total across all pages). | +| `ChunkSize` | `int?` | Page size used by `IterateAsync` (positive; the smaller of this and the remaining `Limit` is requested per call). Never sent to the API. | | `Search` | `string?` | Full-text search string to filter Actors by. | | `SortBy` | `string?` | Field to sort by (e.g. `popularity`, `newest`). | | `Category` | `string?` | Restrict results to a Store category. | diff --git a/src/Apify.Client/Apify.Client.csproj b/src/Apify.Client/Apify.Client.csproj index d07613f..2776f9c 100644 --- a/src/Apify.Client/Apify.Client.csproj +++ b/src/Apify.Client/Apify.Client.csproj @@ -25,6 +25,11 @@ + + + + + diff --git a/src/Apify.Client/Internal/PageSizing.cs b/src/Apify.Client/Internal/PageSizing.cs index d5c58fe..9b0bb64 100644 --- a/src/Apify.Client/Internal/PageSizing.cs +++ b/src/Apify.Client/Internal/PageSizing.cs @@ -21,21 +21,22 @@ public static void ValidateChunkSize(int? chunkSize, string optionName) } /// - /// The limit to request for the next page: the smaller of the chunk size and the number of items - /// still wanted. A value of null or 0 means unset (the API itself reads limit=0 as - /// unbounded), so it loses to any other value; null is returned when both are unset. + /// Sets the limit query parameter of the next page: the smaller of the chunk size and the number + /// of items still wanted under the caller's total limit. A total limit of 0 means unset (the API + /// itself reads limit=0 as unbounded), so it loses to the chunk size, and when neither is set the + /// query is left as the caller built it. /// + /// The page query, which already carries the caller's own limit if any. /// The validated chunk size, or null when unset. - /// The number of items still wanted under the caller's total limit, or null when unbounded. - public static int? PageLimit(int? chunkSize, int? remaining) + /// The caller's total limit across all pages, or null when unbounded. + /// The number of items already yielded. + public static void SetPageLimit(QueryParams query, int? chunkSize, int? totalLimit, int yielded) { - var chunk = chunkSize is 0 ? null : chunkSize; - var wanted = remaining is 0 ? null : remaining; - if (chunk is null) + var remaining = totalLimit is null or 0 ? null : (int?)Math.Max(totalLimit.Value - yielded, 0); + var pageLimit = chunkSize is null ? remaining : (remaining is null ? chunkSize : Math.Min(chunkSize.Value, remaining.Value)); + if (pageLimit is not null) { - return wanted; + query.Set("limit", pageLimit.Value); } - - return wanted is null ? chunk : Math.Min(chunk.Value, wanted.Value); } } diff --git a/src/Apify.Client/Internal/ResourceContext.cs b/src/Apify.Client/Internal/ResourceContext.cs index c5b5b46..5a32911 100644 --- a/src/Apify.Client/Internal/ResourceContext.cs +++ b/src/Apify.Client/Internal/ResourceContext.cs @@ -148,7 +148,7 @@ public string PublicUrl(string subPath) /// resource or the nested one is missing) and propagates instead. A non-empty /// (e.g. a request looked up by its own id) is never ambiguous regardless of how this context was built. /// - public async Task GetResourceAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) + public async Task GetResourceAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) { if (_ambiguousNotFound && subPath.Length == 0) { @@ -165,12 +165,12 @@ public string PublicUrl(string subPath) } } - /// GET a single resource, returning its decoded data (propagates errors). - public async Task GetResourceRequiredAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) + /// GET a single resource, returning its decoded data object (propagates errors). + public async Task GetResourceRequiredAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) { var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); using var response = await Http.CallAsync(HttpMethod.Get, url, timeout: _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); - return await response.ReadDataAsync(ct).ConfigureAwait(false); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); } /// PUT to update a resource with a JSON-serializable body, returning the decoded data. @@ -243,15 +243,7 @@ public async IAsyncEnumerable IterateListAsync( while (true) { var q = baseQuery.Copy().Set("offset", offset); - var pageLimit = PageSizing.PageLimit(chunkSize, limit is null ? null : Math.Max(limit.Value - yielded, 0)); - if (pageLimit is not null) - { - q.Set("limit", pageLimit.Value); - } - else if (limit is not null) - { - q.Set("limit", 0); - } + PageSizing.SetPageLimit(q, chunkSize, limit, yielded); var page = await ListResourceAsync(subPath, q, hydrate, tier, ct).ConfigureAwait(false); foreach (var item in page.Items) @@ -381,7 +373,7 @@ public async Task GetRawRequiredAsync(string subPath, QueryParams p, Req /// /// GET returning a JSON body that is not wrapped in a data envelope; propagates a 404 (see - /// for why). Returns a JSON null node for an empty body. + /// for why). Returns null for an empty body. /// public async Task GetJsonRequiredAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) { diff --git a/src/Apify.Client/Internal/ResponseReader.cs b/src/Apify.Client/Internal/ResponseReader.cs index 34ca2c1..de4d4ee 100644 --- a/src/Apify.Client/Internal/ResponseReader.cs +++ b/src/Apify.Client/Internal/ResponseReader.cs @@ -9,9 +9,10 @@ namespace Apify.Client.Internal; /// /// Reads a successful JSON response into a and reports a body that is unparsable -/// () or lacks the shape the API specification promises -/// (), with the request's method and path as context. Every -/// successful JSON response is decoded through here, matching the reference client's parseResponse. +/// () or does not match the schema the API specification +/// declares for the operation (, see ), +/// with the request's method and path as context. Every successful JSON response is decoded through here, +/// matching the reference client's parseResponse. /// internal static class ResponseReader { @@ -21,41 +22,50 @@ internal static class ResponseReader public static async Task ReadJsonAsync(this HttpResponseMessage response, CancellationToken cancellationToken) { var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - return Parse(response, body); + var node = Parse(response, body); + ValidateAgainstSpec(response, body, node); + return node; } - /// - /// Reads a body wrapped in the {"data": ...} envelope and returns the unwrapped value. The - /// envelope and a non-null data are required. - /// - public static async Task ReadDataAsync(this HttpResponseMessage response, CancellationToken cancellationToken) + /// Reads a bare (not enveloped) JSON array body, such as a dataset's items. + public static async Task ReadJsonArrayAsync(this HttpResponseMessage response, CancellationToken cancellationToken) { var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - if (Parse(response, body) is not JsonObject envelope || !envelope.TryGetPropertyValue(DataEnvelopeKey, out var data)) - { - throw Invalid(response, body, "the response is missing the `data` envelope"); - } - - return data ?? throw Invalid(response, body, "the response's `data` is null"); + var node = Parse(response, body); + ValidateAgainstSpec(response, body, node); + return node as JsonArray ?? throw Invalid(response, body, "the response is not a JSON array"); } /// Reads a data-enveloped body whose data must be an object. public static async Task ReadDataObjectAsync(this HttpResponseMessage response, CancellationToken cancellationToken) { var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - if (Parse(response, body) is not JsonObject envelope || !envelope.TryGetPropertyValue(DataEnvelopeKey, out var data)) + return UnwrapData(response, body) as JsonObject ?? throw Invalid(response, body, "the response's `data` is not an object"); + } + + /// + /// Parses an enveloped body, validates it against the specification's schema for the operation, and + /// returns the unwrapped non-null data. + /// + private static JsonNode UnwrapData(HttpResponseMessage response, string body) + { + var node = Parse(response, body); + ValidateAgainstSpec(response, body, node); + if (node is not JsonObject envelope || !envelope.TryGetPropertyValue(DataEnvelopeKey, out var data)) { throw Invalid(response, body, "the response is missing the `data` envelope"); } - return data as JsonObject ?? throw Invalid(response, body, "the response's `data` is not an object"); + return data ?? throw Invalid(response, body, "the response's `data` is null"); } - /// Reads a bare (not enveloped) JSON array body, such as a dataset's items. - public static async Task ReadJsonArrayAsync(this HttpResponseMessage response, CancellationToken cancellationToken) + private static void ValidateAgainstSpec(HttpResponseMessage response, string body, JsonNode? node) { - var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - return Parse(response, body) as JsonArray ?? throw Invalid(response, body, "the response is not a JSON array"); + var issues = ResponseSchemas.Validate(MethodOf(response), PathOf(response), node); + if (issues.Count > 0) + { + throw Invalid(response, body, string.Join("; ", issues)); + } } private static JsonNode? Parse(HttpResponseMessage response, string body) diff --git a/src/Apify.Client/Internal/ResponseSchemas.cs b/src/Apify.Client/Internal/ResponseSchemas.cs new file mode 100644 index 0000000..56717d3 --- /dev/null +++ b/src/Apify.Client/Internal/ResponseSchemas.cs @@ -0,0 +1,289 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; + +namespace Apify.Client.Internal; + +/// +/// Validates a successful JSON response against the schema the Apify OpenAPI specification declares for +/// its operation, the way the reference client's parseResponse does with its generated schemas. +/// +/// +/// The schemas come from the embedded ResponseSchemas.json, generated from the specification by +/// tools/generate_response_schemas.py. Validation covers JSON types, required properties (including +/// nullability), nested properties and array items, and the allOf/anyOf/oneOf +/// combinators. Like the reference client's schemas, it lets unknown properties and unknown enum values +/// through, so an API that grows new fields or values does not break existing clients. A request that +/// matches no documented operation is not validated. +/// +internal static class ResponseSchemas +{ + private const string ResourceName = "Apify.Client.ResponseSchemas.json"; + private const string ApiVersionSegment = "/v2/"; + private const string RefPrefix = "#/components/schemas/"; + + /// The most issues reported for one response, so a badly wrong body stays readable. + private const int MaxReportedIssues = 10; + + private static readonly Lazy Loaded = new(Load); + + /// The OpenAPI specification version the embedded schemas were generated from. + public static string SpecVersion => Loaded.Value.SpecVersion; + + /// + /// Validates against the schema documented for the operation identified by + /// and (the request path, optionally with a query). + /// Returns the problems found, empty when the body is valid or the operation is not documented. + /// + public static IReadOnlyList Validate(string method, string path, JsonNode? body) + { + var set = Loaded.Value; + var operation = set.Find(method, path); + if (operation is null) + { + return Array.Empty(); + } + + var issues = new List(); + set.Check(body, operation, "$", issues); + return issues; + } + + private static SchemaSet Load() + { + using var stream = typeof(ResponseSchemas).Assembly.GetManifestResourceStream(ResourceName) + ?? throw new InvalidOperationException("The embedded resource " + ResourceName + " is missing."); + var document = (JsonObject)JsonNode.Parse(stream)!; + return new SchemaSet( + (string)document["specVersion"]!, + document["operations"]!.AsObject(), + document["schemas"]!.AsObject()); + } + + private sealed class SchemaSet + { + private readonly JsonObject _schemas; + private readonly Dictionary> _operationsByMethod = new(StringComparer.Ordinal); + + public SchemaSet(string specVersion, JsonObject operations, JsonObject schemas) + { + SpecVersion = specVersion; + _schemas = schemas; + foreach (var (key, schema) in operations) + { + var space = key.IndexOf(' ', StringComparison.Ordinal); + var method = key[..space]; + var template = key[(space + 1)..]; + if (!_operationsByMethod.TryGetValue(method, out var list)) + { + list = new List(); + _operationsByMethod[method] = list; + } + + list.Add(new OperationSchema(template, schema!)); + } + + // The most specific template wins (e.g. `/runs/last` over `/runs/{runId}`): fewest parameters first. + foreach (var list in _operationsByMethod.Values) + { + list.Sort(static (a, b) => a.ParameterCount.CompareTo(b.ParameterCount)); + } + } + + public string SpecVersion { get; } + + public JsonNode? Find(string method, string path) + { + if (!_operationsByMethod.TryGetValue(method.ToUpperInvariant(), out var candidates)) + { + return null; + } + + var normalized = NormalizePath(path); + foreach (var candidate in candidates) + { + if (candidate.Pattern.IsMatch(normalized)) + { + return candidate.Schema; + } + } + + return null; + } + + /// Appends a message to for each way violates . + public void Check(JsonNode? value, JsonNode? schema, string location, List issues) + { + if (schema is not JsonObject rules || issues.Count >= MaxReportedIssues) + { + return; + } + + if (rules["$ref"] is JsonNode reference) + { + Check(value, Resolve((string)reference!), location, issues); + } + + if (rules["allOf"] is JsonArray all) + { + foreach (var part in all) + { + Check(value, part, location, issues); + } + } + + if (rules["anyOf"] is JsonArray any) + { + CheckAny(value, any, location, issues); + } + + if (rules["oneOf"] is JsonArray one) + { + // A value satisfying several alternatives is accepted too, like the reference client's unions. + CheckAny(value, one, location, issues); + } + + if (rules["type"] is JsonNode typeNode && !MatchesType(value, typeNode)) + { + issues.Add(location + ": expected " + DescribeType(typeNode) + " but got " + DescribeKind(value)); + return; + } + + if (value is JsonObject obj) + { + CheckObject(obj, rules, location, issues); + } + else if (value is JsonArray array && rules["items"] is JsonNode items) + { + for (var i = 0; i < array.Count; i++) + { + Check(array[i], items, location + "[" + i.ToString(CultureInfo.InvariantCulture) + "]", issues); + } + } + } + + private void CheckObject(JsonObject obj, JsonObject rules, string location, List issues) + { + if (rules["required"] is JsonArray required) + { + foreach (var name in required) + { + if (!obj.ContainsKey((string)name!)) + { + issues.Add(location + "." + (string)name! + ": required property is missing"); + } + } + } + + if (rules["properties"] is JsonObject properties) + { + foreach (var (name, propertySchema) in properties) + { + // An absent optional property is fine; an absent required one was reported above. + if (obj.TryGetPropertyValue(name, out var propertyValue)) + { + Check(propertyValue, propertySchema, location + "." + name, issues); + } + } + } + } + + private void CheckAny(JsonNode? value, JsonArray alternatives, string location, List issues) + { + foreach (var alternative in alternatives) + { + var attempt = new List(); + Check(value, alternative, location, attempt); + if (attempt.Count == 0) + { + return; + } + } + + issues.Add(location + ": matches none of the allowed variants (got " + DescribeKind(value) + ")"); + } + + private JsonNode? Resolve(string reference) + { + if (!reference.StartsWith(RefPrefix, StringComparison.Ordinal)) + { + throw new InvalidOperationException("Unsupported schema reference: " + reference); + } + + return _schemas[reference[RefPrefix.Length..]]; + } + + private static bool MatchesType(JsonNode? value, JsonNode typeNode) + => typeNode is JsonArray types + ? types.Any(t => MatchesSingleType(value, (string)t!)) + : MatchesSingleType(value, (string)typeNode!); + + private static bool MatchesSingleType(JsonNode? value, string type) + { + if (value is null) + { + return type == "null"; + } + + return type switch + { + "object" => value is JsonObject, + "array" => value is JsonArray, + "string" => value.GetValueKind() == JsonValueKind.String, + "boolean" => value.GetValueKind() is JsonValueKind.True or JsonValueKind.False, + "number" => value.GetValueKind() == JsonValueKind.Number, + "integer" => IsInteger(value), + _ => false, + }; + } + + private static bool IsInteger(JsonNode value) + { + if (value.GetValueKind() != JsonValueKind.Number) + { + return false; + } + + var number = value.GetValue(); + return number.TryGetInt64(out _) || (number.TryGetDouble(out var d) && d == Math.Floor(d) && !double.IsInfinity(d)); + } + + private static string DescribeType(JsonNode typeNode) + => typeNode is JsonArray types ? string.Join(" or ", types.Select(static t => (string)t!)) : (string)typeNode!; + + private static string DescribeKind(JsonNode? value) => value is null ? "null" : value.GetValueKind().ToString().ToLowerInvariant(); + + private static string NormalizePath(string path) + { + var query = path.IndexOf('?', StringComparison.Ordinal); + var withoutQuery = query >= 0 ? path[..query] : path; + var version = withoutQuery.IndexOf(ApiVersionSegment, StringComparison.Ordinal); + return (version >= 0 ? withoutQuery[version..] : withoutQuery).TrimEnd('/'); + } + } + + private sealed class OperationSchema + { + /// Matches a path parameter ({name}) in a template after escaped its opening brace. + private static readonly Regex Parameter = new(@"\\\{[^}/]+\}", RegexOptions.Compiled); + + public OperationSchema(string template, JsonNode schema) + { + Schema = schema; + ParameterCount = template.Count(static c => c == '{'); + Pattern = new Regex("^" + Parameter.Replace(Regex.Escape(template), "[^/]+") + "$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + } + + public JsonNode Schema { get; } + + public int ParameterCount { get; } + + public Regex Pattern { get; } + } +} diff --git a/src/Apify.Client/Internal/ResponseSchemas.json b/src/Apify.Client/Internal/ResponseSchemas.json new file mode 100644 index 0000000..577a835 --- /dev/null +++ b/src/Apify.Client/Internal/ResponseSchemas.json @@ -0,0 +1 @@ +{"operations":{"DELETE /v2/actor-runs/{runId}/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/actor-tasks/{actorTaskId}":{"type":"object"},"DELETE /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/actors/{actorId}":{"type":"object"},"DELETE /v2/actors/{actorId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/actors/{actorId}/versions/{versionNumber}":{"type":"object"},"DELETE /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"type":"object"},"DELETE /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"DELETE /v2/request-queues/{queueId}/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/schedules/{scheduleId}":{"type":"object"},"DELETE /v2/webhooks/{webhookId}":{"type":"object"},"GET /v2/actor-builds":{"$ref":"#/components/schemas/ListOfBuildsResponse"},"GET /v2/actor-builds/{buildId}":{"allOf":[{"$ref":"#/components/schemas/BuildResponse"}]},"GET /v2/actor-builds/{buildId}/openapi.json":{"type":"object"},"GET /v2/actor-runs":{"allOf":[{"$ref":"#/components/schemas/ListOfRunsResponse"}]},"GET /v2/actor-runs/{runId}":{"allOf":[{"$ref":"#/components/schemas/RunResponse"}]},"GET /v2/actor-runs/{runId}/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/actor-runs/{runId}/dataset/items":{"items":{"type":"object"},"type":"array"},"GET /v2/actor-runs/{runId}/dataset/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/actor-runs/{runId}/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/actor-runs/{runId}/key-value-store/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/actor-runs/{runId}/key-value-store/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/actor-runs/{runId}/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/actor-runs/{runId}/request-queue/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/actor-runs/{runId}/request-queue/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/actor-runs/{runId}/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/actor-tasks":{"$ref":"#/components/schemas/ListOfTasksResponse"},"GET /v2/actor-tasks/{actorTaskId}":{"properties":{"data":{"$ref":"#/components/schemas/Task"}},"required":["data"],"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/input":{"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/run-sync":{"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/run-sync-get-dataset-items":{"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/runs":{"properties":{"data":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/RunListItem"},"type":"array"}},"required":["items"],"type":"object"}]}},"required":["data"],"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/runs/last":{"properties":{"data":{"$ref":"#/components/schemas/Run"}},"required":["data"],"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/dataset/items":{"items":{"type":"object"},"type":"array"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/dataset/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/actor-tasks/{actorTaskId}/webhooks":{"properties":{"data":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/WebhookResource"},"type":"array"}},"required":["items"],"type":"object"}]}},"required":["data"],"type":"object"},"GET /v2/actors":{"$ref":"#/components/schemas/ListOfActorsResponse"},"GET /v2/actors/{actorId}":{"$ref":"#/components/schemas/ActorResponse"},"GET /v2/actors/{actorId}/builds":{"$ref":"#/components/schemas/ListOfBuildsResponse"},"GET /v2/actors/{actorId}/builds/default":{"$ref":"#/components/schemas/BuildResponse"},"GET /v2/actors/{actorId}/builds/{buildId}":{"$ref":"#/components/schemas/BuildResponse"},"GET /v2/actors/{actorId}/builds/{buildId}/openapi.json":{"type":"object"},"GET /v2/actors/{actorId}/run-sync":{"type":"object"},"GET /v2/actors/{actorId}/run-sync-get-dataset-items":{"items":{"type":"object"},"type":"array"},"GET /v2/actors/{actorId}/runs":{"$ref":"#/components/schemas/ListOfRunsResponse"},"GET /v2/actors/{actorId}/runs/last":{"$ref":"#/components/schemas/RunResponse"},"GET /v2/actors/{actorId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/actors/{actorId}/runs/last/dataset/items":{"items":{"type":"object"},"type":"array"},"GET /v2/actors/{actorId}/runs/last/dataset/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/actors/{actorId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/actors/{actorId}/runs/last/key-value-store/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/actors/{actorId}/runs/last/key-value-store/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/actors/{actorId}/runs/{runId}":{"$ref":"#/components/schemas/RunResponse"},"GET /v2/actors/{actorId}/versions":{"$ref":"#/components/schemas/ListOfVersionsResponse"},"GET /v2/actors/{actorId}/versions/{versionNumber}":{"$ref":"#/components/schemas/VersionResponse"},"GET /v2/actors/{actorId}/versions/{versionNumber}/env-vars":{"$ref":"#/components/schemas/ListOfEnvVarsResponse"},"GET /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"$ref":"#/components/schemas/EnvVarResponse"},"GET /v2/actors/{actorId}/webhooks":{"$ref":"#/components/schemas/ListOfWebhooksResponse"},"GET /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"GET /v2/datasets":{"$ref":"#/components/schemas/ListOfDatasetsResponse"},"GET /v2/datasets/{datasetId}":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/datasets/{datasetId}/items":{"items":{"type":"object"},"type":"array"},"GET /v2/datasets/{datasetId}/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/key-value-stores":{"$ref":"#/components/schemas/ListOfKeyValueStoresResponse"},"GET /v2/key-value-stores/{storeId}":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/key-value-stores/{storeId}/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/key-value-stores/{storeId}/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/request-queues":{"$ref":"#/components/schemas/ListOfRequestQueuesResponse"},"GET /v2/request-queues/{queueId}":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/request-queues/{queueId}/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/request-queues/{queueId}/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/request-queues/{queueId}/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/schedules":{"$ref":"#/components/schemas/ListOfSchedulesResponse"},"GET /v2/schedules/{scheduleId}":{"$ref":"#/components/schemas/ScheduleResponse"},"GET /v2/schedules/{scheduleId}/log":{"$ref":"#/components/schemas/ScheduleLogResponse"},"GET /v2/store":{"$ref":"#/components/schemas/ListOfStoreActorsResponse"},"GET /v2/users/me":{"$ref":"#/components/schemas/UserPrivateInfoResponse"},"GET /v2/users/me/limits":{"$ref":"#/components/schemas/AccountLimitsResponse"},"GET /v2/users/me/usage/monthly":{"$ref":"#/components/schemas/MonthlyUsageResponse"},"GET /v2/users/{userId}":{"$ref":"#/components/schemas/UserPublicInfoResponse"},"GET /v2/webhook-dispatches":{"$ref":"#/components/schemas/ListOfWebhookDispatchesResponse"},"GET /v2/webhook-dispatches/{dispatchId}":{"$ref":"#/components/schemas/WebhookDispatchResponse"},"GET /v2/webhooks":{"$ref":"#/components/schemas/ListOfWebhooksResponse"},"GET /v2/webhooks/{webhookId}":{"$ref":"#/components/schemas/WebhookResponse"},"GET /v2/webhooks/{webhookId}/dispatches":{"$ref":"#/components/schemas/ListOfWebhookDispatchesResponse"},"POST /v2/actor-builds/{buildId}/abort":{"allOf":[{"$ref":"#/components/schemas/BuildResponse"}]},"POST /v2/actor-runs/{runId}/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-runs/{runId}/charge":{"type":"object"},"POST /v2/actor-runs/{runId}/dataset/items":{"type":"object"},"POST /v2/actor-runs/{runId}/key-value-store/records/{recordKey}":{"type":"object"},"POST /v2/actor-runs/{runId}/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-runs/{runId}/reboot":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-runs/{runId}/request-queue/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/actor-runs/{runId}/request-queue/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/actor-runs/{runId}/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/actor-runs/{runId}/request-queue/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/actor-runs/{runId}/resurrect":{"allOf":[{"$ref":"#/components/schemas/RunResponse"}]},"POST /v2/actor-tasks":{"$ref":"#/components/schemas/TaskResponse"},"POST /v2/actor-tasks/{actorTaskId}/run-sync":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/run-sync-get-dataset-items":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs":{"properties":{"data":{"$ref":"#/components/schemas/Run"}},"required":["data"],"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/dataset/items":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/reboot":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/actors":{"$ref":"#/components/schemas/ActorResponse"},"POST /v2/actors/{actorId}/builds":{"$ref":"#/components/schemas/BuildResponse"},"POST /v2/actors/{actorId}/builds/{buildId}/abort":{"$ref":"#/components/schemas/BuildResponse"},"POST /v2/actors/{actorId}/run-sync":{"type":"object"},"POST /v2/actors/{actorId}/run-sync-get-dataset-items":{"items":{"type":"object"},"type":"array"},"POST /v2/actors/{actorId}/runs":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/dataset/items":{"type":"object"},"POST /v2/actors/{actorId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"POST /v2/actors/{actorId}/runs/last/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/reboot":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/actors/{actorId}/runs/{runId}/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/{runId}/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/{runId}/resurrect":{"allOf":[{"$ref":"#/components/schemas/RunResponse"}]},"POST /v2/actors/{actorId}/validate-input":{"properties":{"valid":{"type":"boolean"}},"required":["valid"],"type":"object"},"POST /v2/actors/{actorId}/versions":{"$ref":"#/components/schemas/VersionResponse"},"POST /v2/actors/{actorId}/versions/{versionNumber}":{"$ref":"#/components/schemas/VersionResponse"},"POST /v2/actors/{actorId}/versions/{versionNumber}/env-vars":{"$ref":"#/components/schemas/EnvVarResponse"},"POST /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"$ref":"#/components/schemas/EnvVarResponse"},"POST /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"POST /v2/datasets":{"$ref":"#/components/schemas/DatasetResponse"},"POST /v2/datasets/{datasetId}/items":{"type":"object"},"POST /v2/key-value-stores":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"POST /v2/key-value-stores/{storeId}/records/{recordKey}":{"type":"object"},"POST /v2/request-queues":{"$ref":"#/components/schemas/RequestQueueResponse"},"POST /v2/request-queues/{queueId}/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/request-queues/{queueId}/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/request-queues/{queueId}/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/request-queues/{queueId}/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/schedules":{"$ref":"#/components/schemas/ScheduleResponse"},"POST /v2/tools/decode-and-verify":{"$ref":"#/components/schemas/DecodeAndVerifyResponse"},"POST /v2/tools/encode-and-sign":{"$ref":"#/components/schemas/EncodeAndSignResponse"},"POST /v2/webhooks":{"$ref":"#/components/schemas/WebhookResponse"},"POST /v2/webhooks/{webhookId}/test":{"$ref":"#/components/schemas/WebhookDispatchResponse"},"PUT /v2/actor-runs/{runId}":{"$ref":"#/components/schemas/RunResponse"},"PUT /v2/actor-runs/{runId}/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/actor-runs/{runId}/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/actor-runs/{runId}/key-value-store/records/{recordKey}":{"type":"object"},"PUT /v2/actor-runs/{runId}/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/actor-runs/{runId}/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/actor-runs/{runId}/request-queue/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/actor-tasks/{actorTaskId}":{"properties":{"data":{"$ref":"#/components/schemas/Task"}},"required":["data"],"type":"object"},"PUT /v2/actor-tasks/{actorTaskId}/input":{"type":"object"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/actors/{actorId}":{"$ref":"#/components/schemas/ActorResponse"},"PUT /v2/actors/{actorId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/actors/{actorId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/actors/{actorId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"PUT /v2/actors/{actorId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/actors/{actorId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/actors/{actorId}/runs/last/request-queue/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/actors/{actorId}/versions/{versionNumber}":{"$ref":"#/components/schemas/VersionResponse"},"PUT /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"$ref":"#/components/schemas/EnvVarResponse"},"PUT /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"PUT /v2/datasets/{datasetId}":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/key-value-stores/{storeId}":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/key-value-stores/{storeId}/records/{recordKey}":{"type":"object"},"PUT /v2/request-queues/{queueId}":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/request-queues/{queueId}/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/request-queues/{queueId}/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/schedules/{scheduleId}":{"$ref":"#/components/schemas/ScheduleResponse"},"PUT /v2/users/me/limits":{"type":"object"},"PUT /v2/webhooks/{webhookId}":{"$ref":"#/components/schemas/WebhookResponse"}},"schemas":{"AccountLimits":{"properties":{"current":{"$ref":"#/components/schemas/CurrentUsage"},"limits":{"$ref":"#/components/schemas/Limits"},"monthlyUsageCycle":{"$ref":"#/components/schemas/UsageCycle"}},"required":["monthlyUsageCycle","limits","current"],"type":"object"},"AccountLimitsResponse":{"properties":{"data":{"$ref":"#/components/schemas/AccountLimits"}},"required":["data"],"type":"object"},"ActorDefinition":{"properties":{"actorSpecification":{"type":"integer"},"buildTag":{"type":"string"},"changelog":{"type":["string","null"]},"defaultMemoryMbytes":{"oneOf":[{"type":"string"},{"type":"integer"}]},"dockerContextDir":{"type":"string"},"dockerfile":{"type":"string"},"environmentVariables":{"type":"object"},"input":{"type":["object","null"]},"maxMemoryMbytes":{"type":"integer"},"minMemoryMbytes":{"type":"integer"},"name":{"type":"string"},"readme":{"type":["string","null"]},"storages":{"properties":{"dataset":{"type":"object"}},"type":"object"},"usesStandbyMode":{"type":"boolean"},"version":{"type":"string"}},"type":"object"},"ActorJobStatus":{"type":"string"},"ActorListItem":{"properties":{"createdAt":{"type":"string"},"id":{"type":"string"},"modifiedAt":{"type":"string"},"name":{"type":"string"},"stats":{"anyOf":[{"$ref":"#/components/schemas/ActorStats"},{"type":"null"}]},"title":{"type":"string"},"username":{"type":"string"}},"required":["id","createdAt","modifiedAt","name","username"],"type":"object"},"ActorNotice":{"type":["string","null"]},"ActorPermissionLevel":{"type":"string"},"ActorResource":{"properties":{"actorPermissionLevel":{"$ref":"#/components/schemas/ActorPermissionLevel"},"actorStandby":{"anyOf":[{"$ref":"#/components/schemas/ActorStandby"},{"type":"null"}]},"categories":{"items":{"type":"string"},"type":"array"},"createdAt":{"type":"string"},"defaultRunOptions":{"$ref":"#/components/schemas/DefaultRunOptions"},"deploymentKey":{"type":"string"},"description":{"type":["string","null"]},"exampleRunInput":{"anyOf":[{"$ref":"#/components/schemas/ExampleRunInput"},{"type":"null"}]},"hasNoDataset":{"type":"boolean"},"id":{"type":"string"},"isCritical":{"type":"boolean"},"isDeprecated":{"type":["boolean","null"]},"isGeneric":{"type":"boolean"},"isPublic":{"type":"boolean"},"isSourceCodeHidden":{"type":"boolean"},"modifiedAt":{"type":"string"},"name":{"type":"string"},"notice":{"$ref":"#/components/schemas/ActorNotice"},"pictureUrl":{"type":["string","null"]},"pricingInfos":{"items":{"$ref":"#/components/schemas/ActorRunPricingInfo"},"type":"array"},"readmeSummary":{"type":"string"},"restartOnError":{"type":["boolean","null"]},"seoDescription":{"type":["string","null"]},"seoTitle":{"type":["string","null"]},"standbyUrl":{"type":["string","null"]},"stats":{"$ref":"#/components/schemas/ActorStats"},"taggedBuilds":{"anyOf":[{"$ref":"#/components/schemas/TaggedBuilds"},{"type":"null"}]},"title":{"type":["string","null"]},"userId":{"type":"string"},"username":{"type":"string"},"versions":{"items":{"$ref":"#/components/schemas/Version"},"type":"array"}},"required":["id","userId","name","username","isPublic","createdAt","modifiedAt","stats","versions","defaultRunOptions"],"type":"object"},"ActorResponse":{"properties":{"data":{"$ref":"#/components/schemas/ActorResource"}},"required":["data"],"type":"object"},"ActorRunPricingInfo":{"oneOf":[{"$ref":"#/components/schemas/PayPerEventActorPricingInfo"},{"$ref":"#/components/schemas/PricePerDatasetItemActorPricingInfo"},{"$ref":"#/components/schemas/FlatPricePerMonthActorPricingInfo"},{"$ref":"#/components/schemas/FreeActorPricingInfo"}]},"ActorStandby":{"properties":{"build":{"type":["string","null"]},"desiredRequestsPerActorRun":{"type":["integer","null"]},"disableStandbyFieldsOverride":{"type":["boolean","null"]},"idleTimeoutSecs":{"type":["integer","null"]},"isEnabled":{"type":["boolean","null"]},"maxRequestsPerActorRun":{"type":["integer","null"]},"memoryMbytes":{"type":["integer","null"]},"shouldPassActorInput":{"type":["boolean","null"]}},"type":"object"},"ActorStats":{"properties":{"actorReviewCount":{"type":"integer"},"actorReviewRating":{"type":"number"},"bookmarkCount":{"type":"integer"},"lastRunStartedAt":{"type":"string"},"publicActorRunStats30Days":{"properties":{"ABORTED":{"type":"integer"},"FAILED":{"type":"integer"},"SUCCEEDED":{"type":"integer"},"TIMED-OUT":{"type":"integer"},"TOTAL":{"type":"integer"}},"type":"object"},"totalBuilds":{"type":"integer"},"totalMetamorphs":{"type":"integer"},"totalRuns":{"type":"integer"},"totalUsers":{"type":"integer"},"totalUsers30Days":{"type":"integer"},"totalUsers7Days":{"type":"integer"},"totalUsers90Days":{"type":"integer"}},"type":"object"},"AddRequestResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestRegistration"}},"required":["data"],"type":"object"},"AddedRequest":{"properties":{"requestId":{"$ref":"#/components/schemas/RequestId"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"wasAlreadyHandled":{"$ref":"#/components/schemas/WasAlreadyHandled"},"wasAlreadyPresent":{"$ref":"#/components/schemas/WasAlreadyPresent"}},"required":["requestId","uniqueKey","wasAlreadyHandled","wasAlreadyPresent"],"type":"object"},"AvailableProxyGroups":{"type":"object"},"BatchAddResponse":{"properties":{"data":{"$ref":"#/components/schemas/BatchAddResult"}},"required":["data"],"type":"object"},"BatchAddResult":{"properties":{"processedRequests":{"items":{"$ref":"#/components/schemas/AddedRequest"},"type":"array"},"unprocessedRequests":{"items":{"$ref":"#/components/schemas/UnprocessedRequest"},"type":"array"}},"required":["processedRequests","unprocessedRequests"],"type":"object"},"BatchDeleteResponse":{"properties":{"data":{"$ref":"#/components/schemas/BatchDeleteResult"}},"required":["data"],"type":"object"},"BatchDeleteResult":{"properties":{"processedRequests":{"items":{"$ref":"#/components/schemas/DeletedRequest"},"type":"array"},"unprocessedRequests":{"items":{"$ref":"#/components/schemas/UnprocessedRequest"},"type":"array"}},"required":["processedRequests","unprocessedRequests"],"type":"object"},"BrowserInfoResponse":{"properties":{"bodyLength":{"type":"integer"},"clientIp":{"type":["string","null"]},"countryCode":{"type":["string","null"]},"headers":{"type":"object"},"method":{"type":"string"},"rawHeaders":{"items":{"type":"string"},"type":"array"}},"required":["method","clientIp","countryCode","bodyLength"],"type":"object"},"Build":{"properties":{"actId":{"type":"string"},"actVersion":{"properties":{"buildTag":{"type":["string","null"]},"gitRepoUrl":{"type":"string"},"sourceFiles":{"items":{"$ref":"#/components/schemas/SourceCodeFile"},"type":"array"},"sourceType":{"$ref":"#/components/schemas/VersionSourceType"},"versionNumber":{"type":"string"}},"type":"object"},"actorDefinition":{"anyOf":[{"$ref":"#/components/schemas/ActorDefinition"},{"type":"null"}]},"buildNumber":{"type":"string"},"finishedAt":{"type":["string","null"]},"id":{"type":"string"},"imageDigest":{"type":["string","null"]},"inputSchema":{"type":["string","null"]},"meta":{"$ref":"#/components/schemas/BuildMeta"},"options":{"anyOf":[{"$ref":"#/components/schemas/BuildOptions"},{"type":"null"}]},"readme":{"type":["string","null"]},"startedAt":{"type":"string"},"stats":{"anyOf":[{"$ref":"#/components/schemas/BuildStats"},{"type":"null"}]},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"usage":{"anyOf":[{"$ref":"#/components/schemas/BuildUsage"},{"type":"null"}]},"usageTotalUsd":{"type":["number","null"]},"usageUsd":{"anyOf":[{"$ref":"#/components/schemas/BuildUsage"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","actId","userId","startedAt","status","meta","buildNumber"],"type":"object"},"BuildListItem":{"properties":{"actId":{"type":"string"},"buildNumber":{"type":"string"},"buildNumberInt":{"type":"integer"},"finishedAt":{"type":["string","null"]},"id":{"type":"string"},"meta":{"$ref":"#/components/schemas/BuildMeta"},"startedAt":{"type":"string"},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"usageTotalUsd":{"type":"number"},"userId":{"type":"string"}},"required":["id","status","startedAt","usageTotalUsd","buildNumber"],"type":"object"},"BuildMeta":{"properties":{"clientIp":{"type":["string"]},"origin":{"$ref":"#/components/schemas/RunOrigin"},"userAgent":{"type":["string"]}},"required":["origin"],"type":"object"},"BuildOptions":{"properties":{"betaPackages":{"type":["boolean","null"]},"diskMbytes":{"type":["integer","null"]},"memoryMbytes":{"type":["integer","null"]},"useCache":{"type":["boolean","null"]}},"type":"object"},"BuildResponse":{"properties":{"data":{"$ref":"#/components/schemas/Build"}},"required":["data"],"type":"object"},"BuildStats":{"properties":{"computeUnits":{"type":"number"},"durationMillis":{"type":"integer"},"imageSizeBytes":{"type":["integer","null"]},"runTimeSecs":{"type":"number"}},"type":"object"},"BuildUsage":{"properties":{"ACTOR_COMPUTE_UNITS":{"type":["number","null"]}},"type":"object"},"CommonActorPricingInfo":{"properties":{"apifyMarginPercentage":{"type":"number"},"createdAt":{"type":"string"},"forceContainsSignificantPriceChange":{"type":"boolean"},"isPriceChangeNotificationSuppressed":{"type":"boolean"},"notifiedAboutChangeAt":{"type":["string","null"]},"notifiedAboutFutureChangeAt":{"type":["string","null"]},"reasonForChange":{"type":["string","null"]},"startedAt":{"type":"string"}},"required":["apifyMarginPercentage","createdAt","startedAt"],"type":"object"},"CurrentPricingInfo":{"properties":{"apifyMarginPercentage":{"type":"number"},"createdAt":{"type":"string"},"forceContainsSignificantPriceChange":{"type":"boolean"},"isPPEPlatformUsagePaidByUser":{"type":"boolean"},"isPriceChangeNotificationSuppressed":{"type":"boolean"},"minimalMaxTotalChargeUsd":{"type":["number","null"]},"notifiedAboutChangeAt":{"type":["string","null"]},"notifiedAboutFutureChangeAt":{"type":["string","null"]},"pricingModel":{"type":"string"},"pricingPerEvent":{"type":["object","null"]},"reasonForChange":{"type":["string","null"]},"startedAt":{"type":"string"}},"required":["pricingModel"],"type":"object"},"CurrentUsage":{"properties":{"activeActorJobCount":{"type":"integer"},"actorCount":{"type":"integer"},"actorMemoryGbytes":{"type":"number"},"actorTaskCount":{"type":"integer"},"monthlyActorComputeUnits":{"type":"number"},"monthlyExternalDataTransferGbytes":{"type":"number"},"monthlyProxySerps":{"type":"integer"},"monthlyResidentialProxyGbytes":{"type":"number"},"monthlyUsageUsd":{"type":"number"},"scheduleCount":{"type":"integer"},"teamAccountSeatCount":{"type":"integer"}},"required":["monthlyUsageUsd","monthlyActorComputeUnits","monthlyExternalDataTransferGbytes","monthlyProxySerps","monthlyResidentialProxyGbytes","actorMemoryGbytes","actorCount","actorTaskCount","activeActorJobCount","teamAccountSeatCount"],"type":"object"},"DailyServiceUsage":{"properties":{"date":{"type":"string"},"serviceUsage":{"$ref":"#/components/schemas/ServiceUsage"},"totalUsageCreditsUsd":{"type":"number"}},"required":["date","serviceUsage","totalUsageCreditsUsd"],"type":"object"},"DatasetListItem":{"properties":{"accessedAt":{"type":"string"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"cleanItemCount":{"type":"integer"},"createdAt":{"type":"string"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"id":{"type":"string"},"itemCount":{"type":"integer"},"modifiedAt":{"type":"string"},"name":{"type":"string"},"stats":{"$ref":"#/components/schemas/DatasetStats"},"title":{"type":["string","null"]},"userId":{"type":"string"},"username":{"type":"string"}},"required":["id","name","userId","createdAt","modifiedAt","accessedAt","itemCount","cleanItemCount"],"type":"object"},"DatasetResource":{"properties":{"accessedAt":{"type":"string"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"cleanItemCount":{"type":"integer"},"consoleUrl":{"type":"string"},"createdAt":{"type":"string"},"fields":{"items":{"type":"string"},"type":["array","null"]},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"id":{"type":"string"},"itemCount":{"type":"integer"},"itemsPublicUrl":{"type":"string"},"modifiedAt":{"type":"string"},"name":{"type":["string","null"]},"schema":{"type":["object","null"]},"stats":{"$ref":"#/components/schemas/DatasetStats"},"urlSigningSecretKey":{"type":["string","null"]},"userId":{"type":"string"}},"required":["id","userId","createdAt","modifiedAt","accessedAt","itemCount","cleanItemCount","consoleUrl"],"type":"object"},"DatasetResponse":{"properties":{"data":{"$ref":"#/components/schemas/DatasetResource"}},"required":["data"],"type":"object"},"DatasetStatistics":{"properties":{"fieldStatistics":{"type":["object","null"]}},"type":"object"},"DatasetStatisticsResponse":{"properties":{"data":{"$ref":"#/components/schemas/DatasetStatistics"}},"required":["data"],"type":"object"},"DatasetStats":{"properties":{"inflatedBytes":{"type":"integer"},"readCount":{"type":"integer"},"storageBytes":{"type":"integer"},"writeCount":{"type":"integer"}},"type":"object"},"DecodeAndVerifyResponse":{"properties":{"data":{"$ref":"#/components/schemas/DecodeAndVerifyResult"}},"required":["data"],"type":"object"},"DecodeAndVerifyResult":{"properties":{"decoded":{},"encodedByUserId":{"type":["string","null"]},"isVerifiedUser":{"type":"boolean"}},"required":["decoded","encodedByUserId","isVerifiedUser"],"type":"object"},"DefaultRunOptions":{"properties":{"build":{"type":"string"},"forcePermissionLevel":{"anyOf":[{"$ref":"#/components/schemas/ActorPermissionLevel"},{"type":"null"}]},"maxItems":{"type":["integer","null"]},"memoryMbytes":{"type":"integer"},"restartOnError":{"type":["boolean","null"]},"timeoutSecs":{"type":"integer"}},"type":"object"},"DeletedRequest":{"anyOf":[{"$ref":"#/components/schemas/DeletedRequestById"},{"$ref":"#/components/schemas/DeletedRequestByUniqueKey"}]},"DeletedRequestById":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"}},"required":["id"],"type":"object"},"DeletedRequestByUniqueKey":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"}},"required":["uniqueKey"],"type":"object"},"EffectivePlatformFeature":{"properties":{"disabledReason":{"type":["string","null"]},"disabledReasonType":{"type":["string","null"]},"isEnabled":{"type":"boolean"},"isTrial":{"type":"boolean"},"trialExpirationAt":{"type":["string","null"]}},"required":["isEnabled","disabledReason","disabledReasonType","isTrial","trialExpirationAt"],"type":"object"},"EffectivePlatformFeatures":{"properties":{"ACTORS":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"ACTORS_PUBLIC_ALL":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"ACTORS_PUBLIC_DEVELOPER":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY_EXTERNAL_ACCESS":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY_RESIDENTIAL":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY_SERPS":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"SCHEDULER":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"STORAGE":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"WEBHOOKS":{"$ref":"#/components/schemas/EffectivePlatformFeature"}},"required":["ACTORS","STORAGE","SCHEDULER","PROXY","PROXY_EXTERNAL_ACCESS","PROXY_RESIDENTIAL","PROXY_SERPS","WEBHOOKS","ACTORS_PUBLIC_ALL","ACTORS_PUBLIC_DEVELOPER"],"type":"object"},"EncodeAndSignResponse":{"properties":{"data":{"$ref":"#/components/schemas/EncodeAndSignResult"}},"required":["data"],"type":"object"},"EncodeAndSignResult":{"properties":{"encoded":{"type":"string"}},"required":["encoded"],"type":"object"},"EnvVar":{"properties":{"isSecret":{"type":["boolean","null"]},"name":{"type":"string"},"value":{"type":"string"}},"required":["name"],"type":"object"},"EnvVarResponse":{"properties":{"data":{"$ref":"#/components/schemas/EnvVar"}},"required":["data"],"type":"object"},"ExampleRunInput":{"properties":{"body":{"type":"string"},"contentType":{"type":"string"}},"type":"object"},"FlatPricePerMonthActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"pricePerUnitUsd":{"type":"number"},"pricingModel":{"type":"string"},"trialMinutes":{"type":"integer"}},"required":["pricingModel","pricePerUnitUsd","trialMinutes"],"type":"object"}]},"FreeActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"pricingModel":{"type":"string"}},"required":["pricingModel"],"type":"object"}]},"GeneralAccess":{"type":"string"},"HadMultipleClients":{"type":"boolean"},"HandledRequestCount":{"type":"integer"},"HeadLimit":{"type":"integer"},"HttpMethod":{"type":"string"},"KeyValueStoreKey":{"properties":{"key":{"type":"string"},"recordPublicUrl":{"type":"string"},"size":{"type":"integer"}},"required":["key","size","recordPublicUrl"],"type":"object"},"KeyValueStoreResource":{"properties":{"accessedAt":{"type":"string"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"consoleUrl":{"type":"string"},"createdAt":{"type":"string"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"id":{"type":"string"},"keysPublicUrl":{"type":"string"},"modifiedAt":{"type":"string"},"name":{"type":["string","null"]},"recordsPublicUrl":{"type":"string"},"schema":{"type":["object","null"]},"stats":{"$ref":"#/components/schemas/KeyValueStoreStats"},"urlSigningSecretKey":{"type":["string","null"]},"userId":{"type":["string","null"]},"username":{"type":["string","null"]}},"required":["id","createdAt","modifiedAt","accessedAt"],"type":"object"},"KeyValueStoreResponse":{"properties":{"data":{"$ref":"#/components/schemas/KeyValueStoreResource"}},"required":["data"],"type":"object"},"KeyValueStoreStats":{"properties":{"deleteCount":{"type":"integer"},"listCount":{"type":"integer"},"readCount":{"type":"integer"},"s3StorageBytes":{"type":"integer"},"storageBytes":{"type":"integer"},"writeCount":{"type":"integer"}},"type":"object"},"Limits":{"properties":{"dataRetentionDays":{"type":"integer"},"maxActorCount":{"type":"integer"},"maxActorMemoryGbytes":{"type":"number"},"maxActorTaskCount":{"type":"integer"},"maxConcurrentActorJobs":{"type":"integer"},"maxMonthlyActorComputeUnits":{"type":"number"},"maxMonthlyExternalDataTransferGbytes":{"type":"number"},"maxMonthlyProxySerps":{"type":"integer"},"maxMonthlyResidentialProxyGbytes":{"type":"number"},"maxMonthlyUsageUsd":{"type":"number"},"maxScheduleCount":{"type":"integer"},"maxTeamAccountSeatCount":{"type":"integer"}},"required":["maxMonthlyUsageUsd","maxMonthlyActorComputeUnits","maxMonthlyExternalDataTransferGbytes","maxMonthlyProxySerps","maxMonthlyResidentialProxyGbytes","maxActorMemoryGbytes","maxActorCount","maxActorTaskCount","maxConcurrentActorJobs","maxTeamAccountSeatCount","dataRetentionDays"],"type":"object"},"ListOfActors":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/ActorListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfActorsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfActors"}},"required":["data"],"type":"object"},"ListOfBuilds":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/BuildListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfBuildsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfBuilds"}},"required":["data"],"type":"object"},"ListOfDatasets":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/DatasetListItem"},"type":"array"},"unnamed":{"type":"boolean"}},"required":["items"],"type":"object"}]},"ListOfDatasetsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfDatasets"}},"required":["data"],"type":"object"},"ListOfEnvVars":{"properties":{"items":{"items":{"$ref":"#/components/schemas/EnvVar"},"type":"array"},"total":{"type":"integer"}},"required":["total","items"],"type":"object"},"ListOfEnvVarsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfEnvVars"}},"required":["data"],"type":"object"},"ListOfKeyValueStores":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/KeyValueStoreResource"},"type":"array"},"unnamed":{"type":"boolean"}},"required":["items"],"type":"object"}]},"ListOfKeyValueStoresResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfKeyValueStores"}},"required":["data"],"type":"object"},"ListOfKeys":{"properties":{"count":{"type":"integer"},"exclusiveStartKey":{"type":["string","null"]},"isTruncated":{"type":"boolean"},"items":{"items":{"$ref":"#/components/schemas/KeyValueStoreKey"},"type":"array"},"limit":{"type":"integer"},"nextExclusiveStartKey":{"type":["string","null"]}},"required":["items","count","limit","isTruncated"],"type":"object"},"ListOfKeysResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfKeys"}},"required":["data"],"type":"object"},"ListOfRequestQueues":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/RequestQueueListItem"},"type":"array"},"unnamed":{"type":"boolean"}},"required":["items"],"type":"object"}]},"ListOfRequestQueuesResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfRequestQueues"}},"required":["data"],"type":"object"},"ListOfRequests":{"properties":{"cursor":{"type":"string"},"exclusiveStartId":{"type":"string"},"items":{"items":{"$ref":"#/components/schemas/RequestResource"},"type":"array"},"limit":{"type":"integer"},"nextCursor":{"type":"string"}},"required":["items","limit"],"type":"object"},"ListOfRequestsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfRequests"}},"required":["data"],"type":"object"},"ListOfRuns":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/RunListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfRunsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfRuns"}},"required":["data"],"type":"object"},"ListOfSchedules":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/ScheduleListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfSchedulesResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfSchedules"}},"required":["data"],"type":"object"},"ListOfStoreActors":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/StoreActor"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfStoreActorsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfStoreActors"}},"required":["data"],"type":"object"},"ListOfTasks":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/TaskListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfTasksResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfTasks"}},"required":["data"],"type":"object"},"ListOfVersions":{"properties":{"items":{"items":{"$ref":"#/components/schemas/Version"},"type":"array"},"total":{"type":"integer"}},"required":["total","items"],"type":"object"},"ListOfVersionsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfVersions"}},"required":["data"],"type":"object"},"ListOfWebhookDispatches":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/WebhookDispatch"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfWebhookDispatchesResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfWebhookDispatches"}},"required":["data"],"type":"object"},"ListOfWebhooks":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/WebhookListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfWebhooksResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfWebhooks"}},"required":["data"],"type":"object"},"LockExpiresAt":{"type":"string"},"LockedRequestQueueHead":{"properties":{"clientKey":{"type":"string"},"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"items":{"items":{"$ref":"#/components/schemas/LockedRequestQueueHeadItem"},"type":"array"},"limit":{"$ref":"#/components/schemas/HeadLimit"},"lockSecs":{"type":"integer"},"queueHasLockedRequests":{"type":"boolean"},"queueModifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"}},"required":["limit","queueModifiedAt","hadMultipleClients","lockSecs","items"],"type":"object"},"LockedRequestQueueHeadItem":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"lockExpiresAt":{"$ref":"#/components/schemas/LockExpiresAt"},"method":{"$ref":"#/components/schemas/RequestMethod"},"retryCount":{"$ref":"#/components/schemas/RetryCount"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"}},"required":["id","uniqueKey","url","lockExpiresAt"],"type":"object"},"LockedRequestQueueHeadResponse":{"properties":{"data":{"$ref":"#/components/schemas/LockedRequestQueueHead"}},"required":["data"],"type":"object"},"MonthlyServiceUsage":{"type":"object"},"MonthlyUsage":{"properties":{"dailyServiceUsages":{"items":{"$ref":"#/components/schemas/DailyServiceUsage"},"type":"array"},"monthlyServiceUsage":{"$ref":"#/components/schemas/MonthlyServiceUsage"},"totalUsageCreditsUsdAfterVolumeDiscount":{"type":"number"},"totalUsageCreditsUsdBeforeVolumeDiscount":{"type":"number"},"usageCycle":{"$ref":"#/components/schemas/UsageCycle"}},"required":["usageCycle","monthlyServiceUsage","dailyServiceUsages","totalUsageCreditsUsdBeforeVolumeDiscount","totalUsageCreditsUsdAfterVolumeDiscount"],"type":"object"},"MonthlyUsageResponse":{"properties":{"data":{"$ref":"#/components/schemas/MonthlyUsage"}},"required":["data"],"type":"object"},"PaginationResponse":{"properties":{"count":{"type":"integer"},"desc":{"type":"boolean"},"limit":{"type":"integer"},"offset":{"type":"integer"},"total":{"type":"integer"}},"required":["total","offset","limit","desc","count"],"type":"object"},"PayPerEventActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"minimalMaxTotalChargeUsd":{"type":["number","null"]},"pricingModel":{"type":"string"},"pricingPerEvent":{"properties":{"actorChargeEvents":{"type":"object"}},"type":"object"}},"required":["pricingModel","pricingPerEvent"],"type":"object"}]},"PendingRequestCount":{"type":"integer"},"PricePerDatasetItemActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"pricePerUnitUsd":{"type":"number"},"pricingModel":{"type":"string"},"tieredPricing":{"$ref":"#/components/schemas/TieredPricingPerDatasetItem"},"unitName":{"type":"string"}},"required":["pricingModel","unitName"],"type":"object"}]},"ProlongRequestLockResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestLockInfo"}},"required":["data"],"type":"object"},"ProxyGroup":{"properties":{"availableCount":{"type":"integer"},"description":{"type":["string","null"]},"name":{"type":"string"}},"required":["name","description","availableCount"],"type":"object"},"ProxyResource":{"properties":{"groups":{"items":{"$ref":"#/components/schemas/ProxyGroup"},"type":"array"},"password":{"type":"string"}},"required":["password","groups"],"type":"object"},"QueueAccessedAt":{"type":"string"},"QueueCreatedAt":{"type":"string"},"QueueId":{"type":"string"},"QueueModifiedAt":{"type":"string"},"QueueUserId":{"type":"string"},"RecordResponse":{"type":"object"},"RequestBase":{"properties":{"errorMessages":{"items":{"type":"string"},"type":["array","null"]},"handledAt":{"type":["string","null"]},"headers":{"type":["object","null"]},"loadedUrl":{"type":["string","null"]},"method":{"$ref":"#/components/schemas/RequestMethod"},"noRetry":{"type":["boolean","null"]},"payload":{"type":["string","object","null"]},"retryCount":{"$ref":"#/components/schemas/RetryCount"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"},"userData":{"$ref":"#/components/schemas/RequestUserData"}},"required":["uniqueKey","url"],"type":"object"},"RequestId":{"type":"string"},"RequestLockInfo":{"properties":{"lockExpiresAt":{"$ref":"#/components/schemas/LockExpiresAt"}},"required":["lockExpiresAt"],"type":"object"},"RequestMethod":{"$ref":"#/components/schemas/HttpMethod"},"RequestQueueHead":{"properties":{"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"items":{"items":{"$ref":"#/components/schemas/RequestQueueHeadItem"},"type":"array"},"limit":{"$ref":"#/components/schemas/HeadLimit"},"queueModifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"}},"required":["limit","queueModifiedAt","hadMultipleClients","items"],"type":"object"},"RequestQueueHeadItem":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"method":{"$ref":"#/components/schemas/RequestMethod"},"retryCount":{"$ref":"#/components/schemas/RetryCount"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"}},"required":["id","uniqueKey","url"],"type":"object"},"RequestQueueHeadResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestQueueHead"}},"required":["data"],"type":"object"},"RequestQueueListItem":{"properties":{"accessedAt":{"$ref":"#/components/schemas/QueueAccessedAt"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"createdAt":{"$ref":"#/components/schemas/QueueCreatedAt"},"expireAt":{"type":"string"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"handledRequestCount":{"$ref":"#/components/schemas/HandledRequestCount"},"id":{"$ref":"#/components/schemas/QueueId"},"modifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"},"name":{"type":"string"},"pendingRequestCount":{"$ref":"#/components/schemas/PendingRequestCount"},"stats":{"$ref":"#/components/schemas/RequestQueueStats"},"totalRequestCount":{"$ref":"#/components/schemas/TotalRequestCount"},"userId":{"$ref":"#/components/schemas/QueueUserId"},"username":{"type":"string"}},"required":["id","name","userId","username","createdAt","modifiedAt","accessedAt","totalRequestCount","handledRequestCount","pendingRequestCount","hadMultipleClients"],"type":"object"},"RequestQueueResource":{"properties":{"accessedAt":{"$ref":"#/components/schemas/QueueAccessedAt"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"consoleUrl":{"type":"string"},"createdAt":{"$ref":"#/components/schemas/QueueCreatedAt"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"handledRequestCount":{"$ref":"#/components/schemas/HandledRequestCount"},"id":{"$ref":"#/components/schemas/QueueId"},"modifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"},"name":{"type":["string","null"]},"pendingRequestCount":{"$ref":"#/components/schemas/PendingRequestCount"},"stats":{"$ref":"#/components/schemas/RequestQueueStats"},"totalRequestCount":{"$ref":"#/components/schemas/TotalRequestCount"},"userId":{"$ref":"#/components/schemas/QueueUserId"}},"required":["id","userId","createdAt","modifiedAt","accessedAt","totalRequestCount","handledRequestCount","pendingRequestCount","hadMultipleClients","consoleUrl"],"type":"object"},"RequestQueueResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestQueueResource"}},"required":["data"],"type":"object"},"RequestQueueStats":{"properties":{"deleteCount":{"type":"integer"},"headItemReadCount":{"type":"integer"},"readCount":{"type":"integer"},"storageBytes":{"type":"integer"},"writeCount":{"type":"integer"}},"type":"object"},"RequestRegistration":{"properties":{"requestId":{"$ref":"#/components/schemas/RequestId"},"wasAlreadyHandled":{"$ref":"#/components/schemas/WasAlreadyHandled"},"wasAlreadyPresent":{"$ref":"#/components/schemas/WasAlreadyPresent"}},"required":["requestId","wasAlreadyPresent","wasAlreadyHandled"],"type":"object"},"RequestResource":{"allOf":[{"$ref":"#/components/schemas/RequestBase","required":["id"]},{"properties":{"id":{"$ref":"#/components/schemas/RequestId"}}}]},"RequestResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestResource"}},"required":["data"],"type":"object"},"RequestUrl":{"type":"string"},"RequestUserData":{"type":"object"},"RetryCount":{"type":"integer"},"Run":{"properties":{"actId":{"type":"string"},"actorTaskId":{"type":["string","null"]},"buildId":{"type":"string"},"buildNumber":{"type":["string","null"]},"chargedEventCounts":{"type":"object"},"containerUrl":{"type":"string"},"defaultDatasetId":{"type":"string"},"defaultKeyValueStoreId":{"type":"string"},"defaultRequestQueueId":{"type":"string"},"exitCode":{"type":["integer","null"]},"finishedAt":{"type":["string","null"]},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"gitBranchName":{"type":["string","null"]},"id":{"type":"string"},"isContainerServerReady":{"type":["boolean","null"]},"isStatusMessageTerminal":{"type":["boolean","null"]},"meta":{"$ref":"#/components/schemas/RunMeta"},"metamorphs":{"anyOf":[{"items":{"$ref":"#/components/schemas/RunMetamorphEvent"},"type":"array"},{"type":"null"}]},"options":{"$ref":"#/components/schemas/RunOptions"},"platformUsageBillingModel":{"type":"string"},"pricingInfo":{"$ref":"#/components/schemas/ActorRunPricingInfo"},"startedAt":{"type":"string"},"stats":{"$ref":"#/components/schemas/RunStats"},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"statusMessage":{"type":["string","null"]},"storageIds":{"properties":{"datasets":{"properties":{"default":{"type":"string"}},"type":"object"},"keyValueStores":{"properties":{"default":{"type":"string"}},"type":"object"},"requestQueues":{"properties":{"default":{"type":"string"}},"type":"object"}},"type":"object"},"usage":{"anyOf":[{"$ref":"#/components/schemas/RunUsage"},{"type":"null"}]},"usageTotalUsd":{"type":["number","null"]},"usageUsd":{"anyOf":[{"$ref":"#/components/schemas/RunUsageUsd"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","actId","userId","startedAt","status","meta","stats","options","buildId","defaultKeyValueStoreId","defaultDatasetId","defaultRequestQueueId"],"type":"object"},"RunListItem":{"properties":{"actId":{"type":"string"},"actorTaskId":{"type":["string","null"]},"buildId":{"type":"string"},"buildNumber":{"type":"string"},"buildNumberInt":{"type":"integer"},"defaultDatasetId":{"type":"string"},"defaultKeyValueStoreId":{"type":"string"},"defaultRequestQueueId":{"type":"string"},"finishedAt":{"type":["string","null"]},"id":{"type":"string"},"meta":{"$ref":"#/components/schemas/RunMeta"},"startedAt":{"type":"string"},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"usageTotalUsd":{"type":"number"},"userId":{"type":"string"}},"required":["id","actId","status","startedAt","buildId","meta","usageTotalUsd","defaultKeyValueStoreId","defaultDatasetId","defaultRequestQueueId"],"type":"object"},"RunMeta":{"properties":{"clientIp":{"type":["string","null"]},"origin":{"$ref":"#/components/schemas/RunOrigin"},"scheduleId":{"type":["string","null"]},"scheduledAt":{"type":["string","null"]},"userAgent":{"type":["string","null"]}},"required":["origin"],"type":"object"},"RunMetamorphEvent":{"properties":{"actorId":{"type":"string"},"buildId":{"type":"string"},"createdAt":{"type":"string"},"inputKey":{"type":["string","null"]}},"required":["createdAt","actorId","buildId"],"type":"object"},"RunOptions":{"properties":{"build":{"type":"string"},"diskMbytes":{"type":"integer"},"maxItems":{"type":["integer","null"]},"maxTotalChargeUsd":{"type":["number","null"]},"memoryMbytes":{"type":"integer"},"timeoutSecs":{"type":"integer"}},"required":["build","timeoutSecs","memoryMbytes","diskMbytes"],"type":"object"},"RunOrigin":{"type":"string"},"RunResponse":{"properties":{"data":{"$ref":"#/components/schemas/Run"}},"required":["data"],"type":"object"},"RunStats":{"properties":{"computeUnits":{"type":"number"},"cpuAvgUsage":{"type":"number"},"cpuCurrentUsage":{"type":"number"},"cpuMaxUsage":{"type":"number"},"durationMillis":{"type":"integer"},"inputBodyLen":{"type":["integer","null"]},"memAvgBytes":{"type":"number"},"memCurrentBytes":{"type":"integer"},"memMaxBytes":{"type":"integer"},"metamorph":{"type":"integer"},"migrationCount":{"type":"integer"},"netRxBytes":{"type":"integer"},"netTxBytes":{"type":"integer"},"rebootCount":{"type":"integer"},"restartCount":{"type":"integer"},"resurrectCount":{"type":"integer"},"runTimeSecs":{"type":"number"}},"type":"object"},"RunUsage":{"properties":{"ACTOR_COMPUTE_UNITS":{"type":["number","null"]},"DATASET_READS":{"type":["integer","null"]},"DATASET_WRITES":{"type":["integer","null"]},"DATA_TRANSFER_EXTERNAL_GBYTES":{"type":["number","null"]},"DATA_TRANSFER_INTERNAL_GBYTES":{"type":["number","null"]},"KEY_VALUE_STORE_LISTS":{"type":["integer","null"]},"KEY_VALUE_STORE_READS":{"type":["integer","null"]},"KEY_VALUE_STORE_WRITES":{"type":["integer","null"]},"PROXY_RESIDENTIAL_TRANSFER_GBYTES":{"type":["number","null"]},"PROXY_SERPS":{"type":["integer","null"]},"REQUEST_QUEUE_READS":{"type":["integer","null"]},"REQUEST_QUEUE_WRITES":{"type":["integer","null"]}},"type":"object"},"RunUsageUsd":{"properties":{"ACTOR_COMPUTE_UNITS":{"type":["number","null"]},"DATASET_READS":{"type":["number","null"]},"DATASET_WRITES":{"type":["number","null"]},"DATA_TRANSFER_EXTERNAL_GBYTES":{"type":["number","null"]},"DATA_TRANSFER_INTERNAL_GBYTES":{"type":["number","null"]},"KEY_VALUE_STORE_LISTS":{"type":["number","null"]},"KEY_VALUE_STORE_READS":{"type":["number","null"]},"KEY_VALUE_STORE_WRITES":{"type":["number","null"]},"PROXY_RESIDENTIAL_TRANSFER_GBYTES":{"type":["number","null"]},"PROXY_SERPS":{"type":["number","null"]},"REQUEST_QUEUE_READS":{"type":["number","null"]},"REQUEST_QUEUE_WRITES":{"type":["number","null"]}},"type":"object"},"Schedule":{"allOf":[{"$ref":"#/components/schemas/ScheduleBase"},{"properties":{"actions":{"items":{"$ref":"#/components/schemas/ScheduleAction"},"type":"array"},"description":{"type":["string","null"]},"notifications":{"$ref":"#/components/schemas/ScheduleNotifications"},"title":{"type":["string","null"]}},"required":["actions"],"type":"object"}]},"ScheduleAction":{"oneOf":[{"$ref":"#/components/schemas/ScheduleActionRunActor"},{"$ref":"#/components/schemas/ScheduleActionRunActorTask"}]},"ScheduleActionRunActor":{"properties":{"actorId":{"type":"string"},"id":{"type":"string"},"runInput":{"anyOf":[{"$ref":"#/components/schemas/ScheduleActionRunInput"},{"type":"null"}]},"runOptions":{"anyOf":[{"$ref":"#/components/schemas/TaskOptions"},{"type":"null"}]},"type":{}},"required":["id","type","actorId"],"type":"object"},"ScheduleActionRunActorTask":{"properties":{"actorTaskId":{"type":"string"},"id":{"type":"string"},"input":{"anyOf":[{"type":"object"},{"type":"null"}]},"type":{}},"required":["id","type","actorTaskId"],"type":"object"},"ScheduleActionRunInput":{"properties":{"body":{"type":["string","null"]},"contentType":{"type":["string","null"]}},"type":"object"},"ScheduleBase":{"properties":{"createdAt":{"type":"string"},"cronExpression":{"type":"string"},"id":{"type":"string"},"isEnabled":{"type":"boolean"},"isExclusive":{"type":"boolean"},"lastRunAt":{"type":["string","null"]},"modifiedAt":{"type":"string"},"name":{"type":"string"},"nextRunAt":{"type":["string","null"]},"timezone":{"type":"string"},"userId":{"type":"string"}},"required":["id","userId","name","cronExpression","timezone","isEnabled","isExclusive","createdAt","modifiedAt"],"type":"object"},"ScheduleListItem":{"allOf":[{"$ref":"#/components/schemas/ScheduleBase"},{"properties":{"actions":{"items":{"$ref":"#/components/schemas/ScheduleListItemAction"},"type":"array"}},"required":["actions"],"type":"object"}]},"ScheduleListItemAction":{"oneOf":[{"$ref":"#/components/schemas/ScheduleListItemActionRunActor"},{"$ref":"#/components/schemas/ScheduleListItemActionRunActorTask"}]},"ScheduleListItemActionRunActor":{"properties":{"actorId":{"type":"string"},"id":{"type":"string"},"type":{}},"required":["id","type","actorId"],"type":"object"},"ScheduleListItemActionRunActorTask":{"properties":{"actorTaskId":{"type":"string"},"id":{"type":"string"},"type":{}},"required":["id","type","actorTaskId"],"type":"object"},"ScheduleLogEntry":{"properties":{"createdAt":{"type":"string"},"level":{"type":"string"},"message":{"type":"string"}},"required":["message","level","createdAt"],"type":"object"},"ScheduleLogResponse":{"properties":{"data":{"items":{"$ref":"#/components/schemas/ScheduleLogEntry"},"type":"array"}},"required":["data"],"type":"object"},"ScheduleNotifications":{"properties":{"email":{"type":"boolean"}},"type":"object"},"ScheduleResponse":{"properties":{"data":{"$ref":"#/components/schemas/Schedule"}},"required":["data"],"type":"object"},"ServiceUsage":{"type":"object"},"SourceCodeFile":{"properties":{"content":{"type":"string"},"format":{"$ref":"#/components/schemas/SourceCodeFileFormat"},"name":{"type":"string"}},"required":["name"],"type":"object"},"SourceCodeFileFormat":{"type":"string"},"SourceCodeFolder":{"properties":{"folder":{"type":"boolean"},"name":{"type":"string"}},"required":["name","folder"],"type":"object"},"StoreActor":{"properties":{"actorReviewCount":{"type":"integer"},"actorReviewRating":{"type":"number"},"badge":{"type":["string","null"]},"bookmarkCount":{"type":"integer"},"categories":{"items":{"type":"string"},"type":"array"},"currentPricingInfo":{"$ref":"#/components/schemas/CurrentPricingInfo"},"description":{"type":["string","null"]},"id":{"type":"string"},"isWhiteListedForAgenticPayments":{"type":["boolean","null"]},"name":{"type":"string"},"notice":{"$ref":"#/components/schemas/ActorNotice"},"pictureUrl":{"type":["string","null"]},"readmeSummary":{"type":"string"},"stats":{"$ref":"#/components/schemas/ActorStats"},"title":{"type":"string"},"url":{"type":["string","null"]},"userFullName":{"type":["string","null"]},"userPictureUrl":{"type":["string","null"]},"username":{"type":"string"}},"required":["id","title","name","username","stats"],"type":"object"},"TaggedBuilds":{"type":"object"},"Task":{"properties":{"actId":{"type":"string"},"actorStandby":{"anyOf":[{"$ref":"#/components/schemas/ActorStandby"},{"type":"null"}]},"createdAt":{"type":"string"},"description":{"type":["string","null"]},"id":{"type":"string"},"input":{"anyOf":[{"$ref":"#/components/schemas/TaskInput"},{"items":{"$ref":"#/components/schemas/TaskInput"},"type":"array"},{"type":"null"}]},"isPublic":{"type":"boolean"},"modifiedAt":{"type":"string"},"name":{"type":"string"},"options":{"anyOf":[{"$ref":"#/components/schemas/TaskOptions"},{"type":"null"}]},"publicConfig":{"anyOf":[{"$ref":"#/components/schemas/TaskPublicConfig"},{"type":"null"}]},"removedAt":{"type":["string","null"]},"standbyUrl":{"type":["string","null"]},"stats":{"anyOf":[{"$ref":"#/components/schemas/TaskStats"},{"type":"null"}]},"title":{"type":["string","null"]},"userId":{"type":"string"},"username":{"type":["string","null"]}},"required":["id","userId","actId","name","createdAt","modifiedAt"],"type":"object"},"TaskInput":{"type":"object"},"TaskListItem":{"properties":{"actId":{"type":"string"},"actName":{"type":["string","null"]},"actUsername":{"type":["string","null"]},"createdAt":{"type":"string"},"id":{"type":"string"},"modifiedAt":{"type":"string"},"name":{"type":"string"},"stats":{"anyOf":[{"$ref":"#/components/schemas/TaskStats"},{"type":"null"}]},"userId":{"type":"string"},"username":{"type":["string","null"]}},"required":["id","userId","actId","name","createdAt","modifiedAt"],"type":"object"},"TaskOptions":{"properties":{"build":{"type":["string","null"]},"maxItems":{"type":["integer","null"]},"maxTotalChargeUsd":{"type":["number","null"]},"memoryMbytes":{"type":["integer","null"]},"restartOnError":{"type":["boolean","null"]},"timeoutSecs":{"type":["integer","null"]}},"type":"object"},"TaskPublicConfig":{"properties":{"datasetName":{"type":["string","null"]},"datasetView":{"type":["string","null"]},"inputSchemaFields":{"items":{"type":"string"},"type":["array","null"]},"publishedAt":{"type":["string","null"]},"seoDescription":{"type":["string","null"]},"seoTitle":{"type":["string","null"]}},"type":"object"},"TaskResponse":{"properties":{"data":{"$ref":"#/components/schemas/Task"}},"required":["data"],"type":"object"},"TaskStats":{"properties":{"totalRuns":{"type":"integer"}},"type":"object"},"TieredPricingPerDatasetItem":{"type":"object"},"TotalRequestCount":{"type":"integer"},"UniqueKey":{"type":"string"},"UnlockRequestsResponse":{"properties":{"data":{"$ref":"#/components/schemas/UnlockRequestsResult"}},"required":["data"],"type":"object"},"UnlockRequestsResult":{"properties":{"unlockedCount":{"type":"integer"}},"required":["unlockedCount"],"type":"object"},"UnprocessedRequest":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"method":{"$ref":"#/components/schemas/RequestMethod"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"}},"required":["uniqueKey","url"],"type":"object"},"UpdateRequestResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestRegistration"}},"required":["data"],"type":"object"},"UsageCycle":{"properties":{"endAt":{"type":"string"},"startAt":{"type":"string"}},"required":["startAt","endAt"],"type":"object"},"UserPlan":{"properties":{"apiRateLimitBoosts":{"type":"integer"},"availableAddOns":{"items":{"type":"string"},"type":"array"},"availableProxyGroups":{"$ref":"#/components/schemas/AvailableProxyGroups"},"dataRetentionDays":{"type":"integer"},"description":{"type":"string"},"enabledPlatformFeatures":{"items":{"type":"string"},"type":"array"},"id":{"type":"string"},"isEnabled":{"type":"boolean"},"maxActorCount":{"type":"integer"},"maxActorMemoryGbytes":{"type":"number"},"maxActorTaskCount":{"type":"integer"},"maxConcurrentActorRuns":{"type":"integer"},"maxMonthlyActorComputeUnits":{"type":"number"},"maxMonthlyExternalDataTransferGbytes":{"type":"number"},"maxMonthlyProxySerps":{"type":"integer"},"maxMonthlyResidentialProxyGbytes":{"type":"number"},"maxMonthlyUsageUsd":{"type":"number"},"maxScheduleCount":{"type":"integer"},"monthlyBasePriceUsd":{"type":"number"},"monthlyUsageCreditsUsd":{"type":"number"},"planPricing":{"type":"object"},"supportLevel":{"type":"string"},"teamAccountSeatCount":{"type":"integer"},"tier":{"type":"string"},"usageDiscountPercent":{"type":"number"}},"required":["availableProxyGroups"],"type":"object"},"UserPrivateInfo":{"properties":{"createdAt":{"type":"string"},"effectivePlatformFeatures":{"$ref":"#/components/schemas/EffectivePlatformFeatures"},"email":{"type":"string"},"id":{"type":"string"},"isPaying":{"type":"boolean"},"plan":{"$ref":"#/components/schemas/UserPlan"},"profile":{"$ref":"#/components/schemas/UserProfile"},"proxy":{"$ref":"#/components/schemas/ProxyResource"},"username":{"type":"string"}},"required":["username"],"type":"object"},"UserPrivateInfoResponse":{"properties":{"data":{"$ref":"#/components/schemas/UserPrivateInfo"}},"required":["data"],"type":"object"},"UserProfile":{"properties":{"bio":{"type":["string","null"]},"githubUsername":{"type":["string","null"]},"name":{"type":"string"},"pictureUrl":{"type":["string","null"]},"readme":{"type":["string","null"]},"twitterUsername":{"type":["string","null"]},"websiteUrl":{"type":["string","null"]}},"type":"object"},"UserPublicInfo":{"properties":{"profile":{"$ref":"#/components/schemas/UserProfile"},"username":{"type":"string"}},"required":["username"],"type":"object"},"UserPublicInfoResponse":{"properties":{"data":{"$ref":"#/components/schemas/UserPublicInfo"}},"required":["data"],"type":"object"},"Version":{"properties":{"applyEnvVarsToBuild":{"type":["boolean","null"]},"buildTag":{"type":["string","null"]},"envVars":{"anyOf":[{"items":{"$ref":"#/components/schemas/EnvVar"},"type":"array"},{"type":"null"}]},"gitHubGistUrl":{"type":["string","null"]},"gitRepoUrl":{"type":["string","null"]},"sourceFiles":{"$ref":"#/components/schemas/VersionSourceFiles"},"sourceType":{"anyOf":[{"$ref":"#/components/schemas/VersionSourceType"},{"type":"null"}]},"tarballUrl":{"type":["string","null"]},"versionNumber":{"type":"string"}},"required":["versionNumber","sourceType"],"type":"object"},"VersionResponse":{"properties":{"data":{"$ref":"#/components/schemas/Version"}},"required":["data"],"type":"object"},"VersionSourceFiles":{"items":{"anyOf":[{"$ref":"#/components/schemas/SourceCodeFile"},{"$ref":"#/components/schemas/SourceCodeFolder"}]},"type":"array"},"VersionSourceType":{"type":"string"},"WasAlreadyHandled":{"type":"boolean"},"WasAlreadyPresent":{"type":"boolean"},"WebhookCondition":{"properties":{"actorId":{"type":["string","null"]},"actorRunId":{"type":["string","null"]},"actorTaskId":{"type":["string","null"]}},"type":"object"},"WebhookDispatch":{"properties":{"calls":{"items":{"properties":{"errorMessage":{"type":["string","null"]},"finishedAt":{"type":["string","null"]},"responseBody":{"type":["string","null"]},"responseStatus":{"type":["integer","null"]},"startedAt":{"type":["string","null"]}},"type":"object"},"type":"array"},"createdAt":{"type":"string"},"eventData":{"properties":{"actorBuildId":{"type":"string"},"actorId":{"type":"string"},"actorRunId":{"type":"string"},"actorTaskId":{"type":["string","null"]}},"required":["actorId"],"type":["object","null"]},"eventType":{"$ref":"#/components/schemas/WebhookEventType"},"id":{"type":"string"},"status":{"$ref":"#/components/schemas/WebhookDispatchStatus"},"userId":{"type":"string"},"webhook":{"anyOf":[{"$ref":"#/components/schemas/WebhookDispatchWebhookSummary"},{"type":"null"}]},"webhookId":{"type":"string"}},"required":["id","userId","webhookId","createdAt","status","eventType"],"type":"object"},"WebhookDispatchResponse":{"properties":{"data":{"$ref":"#/components/schemas/WebhookDispatch"}},"required":["data"],"type":"object"},"WebhookDispatchStatus":{"type":"string"},"WebhookDispatchWebhookSummary":{"properties":{"actionType":{"type":"string"},"condition":{"$ref":"#/components/schemas/WebhookCondition"},"isAdHoc":{"type":"boolean"},"requestUrl":{"type":["string","null"]}},"type":"object"},"WebhookEventType":{"type":"string"},"WebhookLastDispatch":{"properties":{"finishedAt":{"type":["string","null"]},"removedAt":{"type":["string","null"]},"status":{"$ref":"#/components/schemas/WebhookDispatchStatus"}},"required":["status"],"type":"object"},"WebhookListItem":{"properties":{"actionType":{"type":"string"},"condition":{"$ref":"#/components/schemas/WebhookCondition"},"createdAt":{"type":"string"},"doNotRetry":{"type":"boolean"},"eventTypes":{"items":{"$ref":"#/components/schemas/WebhookEventType"},"type":"array"},"id":{"type":"string"},"ignoreSslErrors":{"type":"boolean"},"isAdHoc":{"type":["boolean","null"]},"isApifyIntegration":{"type":"boolean"},"isEnabled":{"type":"boolean"},"lastDispatch":{"anyOf":[{"$ref":"#/components/schemas/WebhookLastDispatch"},{"type":"null"}]},"modifiedAt":{"type":"string"},"requestUrl":{"type":"string"},"shouldInterpolateStrings":{"type":["boolean","null"]},"stats":{"anyOf":[{"$ref":"#/components/schemas/WebhookStats"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","createdAt","modifiedAt","userId","eventTypes","condition","ignoreSslErrors","doNotRetry","requestUrl"],"type":"object"},"WebhookResource":{"properties":{"condition":{"$ref":"#/components/schemas/WebhookCondition"},"createdAt":{"type":"string"},"description":{"type":["string","null"]},"doNotRetry":{"type":["boolean","null"]},"eventTypes":{"items":{"$ref":"#/components/schemas/WebhookEventType"},"type":"array"},"headersTemplate":{"type":["string","null"]},"id":{"type":"string"},"ignoreSslErrors":{"type":"boolean"},"isAdHoc":{"type":["boolean","null"]},"lastDispatch":{"anyOf":[{"$ref":"#/components/schemas/WebhookLastDispatch"},{"type":"null"}]},"modifiedAt":{"type":"string"},"payloadTemplate":{"type":["string","null"]},"requestUrl":{"type":["string","null"]},"shouldInterpolateStrings":{"type":["boolean","null"]},"stats":{"anyOf":[{"$ref":"#/components/schemas/WebhookStats"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","createdAt","modifiedAt","userId","eventTypes","condition","ignoreSslErrors"],"type":"object"},"WebhookResponse":{"properties":{"data":{"$ref":"#/components/schemas/WebhookResource"}},"required":["data"],"type":"object"},"WebhookStats":{"properties":{"totalDispatches":{"type":"integer"}},"type":"object"}},"specVersion":"v2-2026-10-20T121847Z"} diff --git a/src/Apify.Client/Options/StoreListOptions.cs b/src/Apify.Client/Options/StoreListOptions.cs index bffadf8..f7e3f13 100644 --- a/src/Apify.Client/Options/StoreListOptions.cs +++ b/src/Apify.Client/Options/StoreListOptions.cs @@ -46,20 +46,6 @@ public sealed class StoreListOptions public string? ResponseFormat { get; init; } /// Returns a copy of these options with a new (used by lazy iteration). - internal StoreListOptions WithPage(int? offset, int? limit) => new() - { - Offset = offset, - Limit = limit, - Search = Search, - SortBy = SortBy, - Category = Category, - Username = Username, - PricingModel = PricingModel, - IncludeUnrunnableActors = IncludeUnrunnableActors, - AllowsAgenticUsers = AllowsAgenticUsers, - ResponseFormat = ResponseFormat, - }; - internal void AppendTo(QueryParams q) { q.AddInt("offset", Offset) diff --git a/src/Apify.Client/Resources/ActorClient.cs b/src/Apify.Client/Resources/ActorClient.cs index 8027543..7268942 100644 --- a/src/Apify.Client/Resources/ActorClient.cs +++ b/src/Apify.Client/Resources/ActorClient.cs @@ -148,7 +148,7 @@ public async Task DefaultBuildAsync(int? waitForFinish = null, Canc q.AddInt("waitForFinish", waitForFinish); // Long: waitForFinish is sent unclamped, so the HTTP timeout must cover whatever the caller asks for. var data = await _ctx.GetResourceRequiredAsync("builds/default", q, RequestTier.Long, cancellationToken).ConfigureAwait(false); - var build = new Build(data as JsonObject ?? new JsonObject()); + var build = new Build(data); return new BuildClient(_http, _baseUrl, build.Id ?? string.Empty); } diff --git a/src/Apify.Client/Resources/DatasetClient.cs b/src/Apify.Client/Resources/DatasetClient.cs index 011cb0b..d256632 100644 --- a/src/Apify.Client/Resources/DatasetClient.cs +++ b/src/Apify.Client/Resources/DatasetClient.cs @@ -112,15 +112,7 @@ public async Task UpdateAsync(object newFields, CancellationToken cance while (true) { var q = baseQuery.Copy().Set("offset", offset); - var pageLimit = PageSizing.PageLimit(options.ChunkSize, limit is null ? null : Math.Max(limit.Value - yielded, 0)); - if (pageLimit is not null) - { - q.Set("limit", pageLimit.Value); - } - else if (limit is not null) - { - q.Set("limit", 0); - } + PageSizing.SetPageLimit(q, options.ChunkSize, limit, yielded); var (page, scannedCount) = await FetchItemsPageAsync(q, desc, cancellationToken).ConfigureAwait(false); foreach (var item in page.Items) @@ -129,8 +121,11 @@ public async Task UpdateAsync(object newFields, CancellationToken cance yielded++; } + // Never stop at the reported total: it only reflects the moment its own page was fetched, and the + // dataset can grow while the iteration runs. Like the collection iterators, stop once a page scans + // no rows, which costs one extra request at the end. offset += (int)scannedCount; - if (scannedCount == 0 || offset >= page.Total || (limit is not null && yielded >= limit.Value)) + if (scannedCount == 0 || (limit is not null && yielded >= limit.Value)) { yield break; } diff --git a/src/Apify.Client/Resources/StoreCollectionClient.cs b/src/Apify.Client/Resources/StoreCollectionClient.cs index c1ffbf2..6b79bc8 100644 --- a/src/Apify.Client/Resources/StoreCollectionClient.cs +++ b/src/Apify.Client/Resources/StoreCollectionClient.cs @@ -30,35 +30,18 @@ public Task> ListAsync(StoreListOptions? opti } /// - /// Lazily iterates over all Store Actors matching the options, fetching pages on demand. The page size - /// is the options' ChunkSize, falling back to Limit (which here is the per-page size, not - /// a total cap), falling back to the API default. + /// Lazily iterates over all Store Actors matching the options across pages, fetching each page on + /// demand. /// - /// Optional listing filters; ChunkSize (or else Limit) is the page size. + /// Optional listing filters; Offset/Limit bound where iteration + /// starts and the total number of Actors yielded, and ChunkSize sets the page size. /// A token to cancel the iteration. - public async IAsyncEnumerable IterateAsync( - StoreListOptions? options = null, - [EnumeratorCancellation] CancellationToken cancellationToken = default) + public IAsyncEnumerable IterateAsync(StoreListOptions? options = null, CancellationToken cancellationToken = default) { options ??= new StoreListOptions(); - PageSizing.ValidateChunkSize(options.ChunkSize, nameof(StoreListOptions.ChunkSize)); - var pageSize = options.ChunkSize ?? options.Limit; - var offset = options.Offset ?? 0; - while (true) - { - var page = await ListAsync(options.WithPage(offset, pageSize), cancellationToken).ConfigureAwait(false); - var items = page.Items; - foreach (var item in items) - { - yield return item; - } - - offset += items.Count; - if (items.Count == 0 || offset >= page.Total) - { - yield break; - } - } + var q = new QueryParams(); + options.AppendTo(q); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new ActorStoreListItem(d), RequestTier.Medium, cancellationToken); } /// diff --git a/src/Apify.Client/Resources/UserClient.cs b/src/Apify.Client/Resources/UserClient.cs index a194c06..d0208bf 100644 --- a/src/Apify.Client/Resources/UserClient.cs +++ b/src/Apify.Client/Resources/UserClient.cs @@ -57,8 +57,7 @@ public async Task MonthlyUsageAsync(string? date = null, Cancellatio q.AddString("date", date); } - var data = await _ctx.GetResourceRequiredAsync("usage/monthly", q, RequestTier.Short, cancellationToken).ConfigureAwait(false); - return data as JsonObject ?? new JsonObject(); + return await _ctx.GetResourceRequiredAsync("usage/monthly", q, RequestTier.Short, cancellationToken).ConfigureAwait(false); } /// Fetches the current account's resource limits. Only available for me. @@ -66,8 +65,7 @@ public async Task MonthlyUsageAsync(string? date = null, Cancellatio public async Task LimitsAsync(CancellationToken cancellationToken = default) { RequireMe(); - var data = await _ctx.GetResourceRequiredAsync("limits", new QueryParams(), RequestTier.Short, cancellationToken).ConfigureAwait(false); - return data as JsonObject ?? new JsonObject(); + return await _ctx.GetResourceRequiredAsync("limits", new QueryParams(), RequestTier.Short, cancellationToken).ConfigureAwait(false); } /// Updates the current account's resource limits. Only available for me. diff --git a/tests/Apify.Client.Tests/Integration/StoreIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/StoreIntegrationTests.cs index e9a2727..655c9b9 100644 --- a/tests/Apify.Client.Tests/Integration/StoreIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/StoreIntegrationTests.cs @@ -20,7 +20,7 @@ public async Task IterateStore() { var client = RequireClient(); var count = 0; - await foreach (var item in client.Store().IterateAsync(new StoreListOptions { Limit = 5 })) + await foreach (var item in client.Store().IterateAsync(new StoreListOptions { ChunkSize = 5 })) { Assert.False(string.IsNullOrEmpty(item.Id)); if (++count >= 12) @@ -31,4 +31,18 @@ public async Task IterateStore() Assert.True(count >= 12, "expected to iterate at least 12 store actors"); } + + [SkippableFact] + public async Task IterateStoreStopsAtLimit() + { + var client = RequireClient(); + var count = 0; + await foreach (var unused in client.Store().IterateAsync(new StoreListOptions { Limit = 7, ChunkSize = 3 })) + { + _ = unused; + count++; + } + + Assert.Equal(7, count); + } } diff --git a/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs index 747b970..2ea0f26 100644 --- a/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs +++ b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs @@ -44,9 +44,9 @@ public async Task CollectionIterateWalksAllPagesByOffset() // CollectionIterateDoesNotStopAtAnEarlierPagesStaleTotal), so it always sends one more request // after the last non-empty page to confirm nothing is left. var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"total\":3,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}") - .QueueResponse(200, "{\"data\":{\"total\":3,\"items\":[{\"id\":\"c\"}]}}") - .QueueResponse(200, "{\"data\":{\"total\":3,\"items\":[]}}"); + .QueueResponse(200, SpecFixtures.ActorPage(3, "a", "b")) + .QueueResponse(200, SpecFixtures.ActorPage(3, "c")) + .QueueResponse(200, SpecFixtures.ActorPage(3)); var ids = new List(); await foreach (var actor in Client(transport).Actors().IterateAsync()) @@ -69,11 +69,11 @@ public async Task CollectionIterateDoesNotStopAtAnEarlierPagesStaleTotal() // iteration runs. The iterator must keep paging past an earlier page's `total` and only stop once a // page actually scans no rows. var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"total\":2,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}") + .QueueResponse(200, SpecFixtures.ActorPage(2, "a", "b")) // By the time this page is fetched, a third item was added: total is now 3, not the 2 the first // page reported. - .QueueResponse(200, "{\"data\":{\"total\":3,\"items\":[{\"id\":\"c\"}]}}") - .QueueResponse(200, "{\"data\":{\"total\":3,\"items\":[]}}"); + .QueueResponse(200, SpecFixtures.ActorPage(3, "c")) + .QueueResponse(200, SpecFixtures.ActorPage(3)); var ids = new List(); await foreach (var actor in Client(transport).Actors().IterateAsync()) @@ -89,7 +89,7 @@ public async Task CollectionIterateDoesNotStopAtAnEarlierPagesStaleTotal() public async Task CollectionIterateStopsAtLimit() { var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"total\":10,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}"); + .QueueResponse(200, SpecFixtures.ActorPage(10, "a", "b")); var ids = new List(); await foreach (var actor in Client(transport).Actors().IterateAsync(new ActorListOptions { Limit = 2 })) @@ -108,7 +108,8 @@ public async Task DatasetIterateItemsWalksAllPagesByOffset() { var transport = new MockTransport() .QueueResponse(200, "[{\"i\":1},{\"i\":2}]", Page1Headers) - .QueueResponse(200, "[{\"i\":3}]", Page2Headers); + .QueueResponse(200, "[{\"i\":3}]", Page2Headers) + .QueueResponse(200, "[]"); // a final empty page confirms nothing is left var values = new List(); await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync()) @@ -117,12 +118,33 @@ public async Task DatasetIterateItemsWalksAllPagesByOffset() } Assert.Equal(new[] { 1, 2, 3 }, values); - Assert.Equal(2, transport.CallCount); + Assert.Equal(3, transport.CallCount); Assert.Contains("/datasets/ds1/items", transport.Received[0].Uri, StringComparison.Ordinal); Assert.Contains("offset=0", transport.Received[0].Uri, StringComparison.Ordinal); Assert.Contains("offset=2", transport.Received[1].Uri, StringComparison.Ordinal); } + [Fact] + public async Task DatasetIterateItemsDoesNotStopAtAnEarlierPagesStaleTotal() + { + // The dataset grows while the iteration runs: page 1 reports a total of 2 (and has 2 items), yet a + // third item exists by the time page 2 is fetched. Stopping once offset reached that total would lose it. + var staleTotal = new Dictionary { ["X-Apify-Pagination-Total"] = "2" }; + var grownTotal = new Dictionary { ["X-Apify-Pagination-Total"] = "3" }; + var transport = new MockTransport() + .QueueResponse(200, "[{\"i\":1},{\"i\":2}]", staleTotal) + .QueueResponse(200, "[{\"i\":3}]", grownTotal) + .QueueResponse(200, "[]", grownTotal); + + var values = new List(); + await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync()) + { + values.Add(item!["i"]!.GetValue()); + } + + Assert.Equal(new[] { 1, 2, 3 }, values); + } + [Fact] public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenAPageIsFullyFiltered() { @@ -154,7 +176,8 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenA var transport = new MockTransport() .QueueResponse(200, "[]", page1Headers) // fully filtered: 0 items returned from 2 scanned .QueueResponse(200, "[{\"i\":3}]", page2Headers) // 1 item returned from 2 scanned - .QueueResponse(200, "[{\"i\":5}]", page3Headers); + .QueueResponse(200, "[{\"i\":5}]", page3Headers) + .QueueResponse(200, "[]"); var values = new List(); await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync()) @@ -163,7 +186,7 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenA } Assert.Equal(new[] { 3, 5 }, values); - Assert.Equal(3, transport.CallCount); + Assert.Equal(4, transport.CallCount); Assert.Contains("offset=0", transport.Received[0].Uri, StringComparison.Ordinal); Assert.Contains("offset=2", transport.Received[1].Uri, StringComparison.Ordinal); Assert.Contains("offset=4", transport.Received[2].Uri, StringComparison.Ordinal); @@ -190,7 +213,8 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenU }; var transport = new MockTransport() .QueueResponse(200, "[{\"i\":1},{\"i\":2},{\"i\":3}]", page1Headers) - .QueueResponse(200, "[{\"i\":4}]", page2Headers); + .QueueResponse(200, "[{\"i\":4}]", page2Headers) + .QueueResponse(200, "[]"); var values = new List(); await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync()) @@ -199,7 +223,7 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenU } Assert.Equal(new[] { 1, 2, 3, 4 }, values); - Assert.Equal(2, transport.CallCount); + Assert.Equal(3, transport.CallCount); Assert.Contains("offset=0", transport.Received[0].Uri, StringComparison.Ordinal); Assert.Contains("offset=1", transport.Received[1].Uri, StringComparison.Ordinal); } @@ -208,8 +232,8 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenU public async Task KeyValueStoreIterateKeysWalksAllPagesByExclusiveStartKey() { var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"items\":[{\"key\":\"a\"},{\"key\":\"b\"}],\"limit\":2,\"isTruncated\":true,\"nextExclusiveStartKey\":\"b\"}}") - .QueueResponse(200, "{\"data\":{\"items\":[{\"key\":\"c\"}],\"limit\":2,\"isTruncated\":false}}"); + .QueueResponse(200, SpecFixtures.KeysPage(2, true, "b", "a", "b")) + .QueueResponse(200, SpecFixtures.KeysPage(2, false, null, "c")); var keys = new List(); await foreach (var key in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { MaxPageLimit = 2 })) @@ -231,7 +255,7 @@ public async Task KeyValueStoreIterateKeysWalksAllPagesByExclusiveStartKey() public async Task KeyValueStoreIterateKeysStopsAtLimit() { var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"items\":[{\"key\":\"a\"},{\"key\":\"b\"}],\"limit\":2,\"isTruncated\":true,\"nextExclusiveStartKey\":\"b\"}}"); + .QueueResponse(200, SpecFixtures.KeysPage(2, true, "b", "a", "b")); var keys = new List(); await foreach (var key in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { Limit = 2 })) @@ -299,7 +323,7 @@ public async Task KeyValueStoreIterateKeysStopsOnTruncatedPageWithNullCursor() // reporting `isTruncated=true` with a null/absent cursor must still stop iteration instead of // looping back to the same first page forever and yielding duplicate keys. var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"items\":[{\"key\":\"a\"},{\"key\":\"b\"}],\"limit\":2,\"isTruncated\":true}}"); + .QueueResponse(200, SpecFixtures.KeysPage(2, true, null, "a", "b")); var keys = new List(); await foreach (var key in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { MaxPageLimit = 2 })) diff --git a/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs b/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs index 5ab2a69..f809606 100644 --- a/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs +++ b/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs @@ -20,9 +20,6 @@ public sealed class ChunkSizeTests HttpTransport = transport, }); - private static string Page(params string[] ids) - => "{\"data\":{\"total\":99,\"items\":[" + string.Join(",", Array.ConvertAll(ids, static id => "{\"id\":\"" + id + "\"}")) + "]}}"; - private static async Task DrainAsync(System.Collections.Generic.IAsyncEnumerable items) { await foreach (var unused in items) @@ -35,9 +32,9 @@ private static async Task DrainAsync(System.Collections.Generic.IAsyncEnumera public async Task CollectionIterationRequestsTheSmallerOfChunkSizeAndRemainingLimit() { var transport = new MockTransport() - .QueueResponse(200, Page("a", "b")) - .QueueResponse(200, Page("c", "d")) - .QueueResponse(200, Page("e")); + .QueueResponse(200, SpecFixtures.Page(5, SpecFixtures.TaskItem("a"), SpecFixtures.TaskItem("b"))) + .QueueResponse(200, SpecFixtures.Page(5, SpecFixtures.TaskItem("c"), SpecFixtures.TaskItem("d"))) + .QueueResponse(200, SpecFixtures.Page(5, SpecFixtures.TaskItem("e"))); await DrainAsync(Client(transport).Tasks().IterateAsync(new ListOptions { Limit = 5, ChunkSize = 2 })); @@ -51,7 +48,7 @@ public async Task CollectionIterationRequestsTheSmallerOfChunkSizeAndRemainingLi [Fact] public async Task CollectionIterationWithoutLimitUsesChunkSizeAsPageLimit() { - var transport = new MockTransport().QueueResponse(200, Page("a")).QueueResponse(200, Page()); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.ActorPage(1, "a")).QueueResponse(200, SpecFixtures.ActorPage(1)); await DrainAsync(Client(transport).Actors().IterateAsync(new ActorListOptions { ChunkSize = 7 })); @@ -62,7 +59,7 @@ public async Task CollectionIterationWithoutLimitUsesChunkSizeAsPageLimit() public async Task ZeroLimitDoesNotBeatChunkSize() { // The API reads limit=0 as unset, so it must lose to the chunk size. - var transport = new MockTransport().QueueResponse(200, Page("a")); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(1, SpecFixtures.ScheduleItem("a"))); await DrainAsync(Client(transport).Schedules().IterateAsync(new ListOptions { Limit = 0, ChunkSize = 3 })); @@ -99,7 +96,7 @@ public async Task DatasetItemIterationSizesPagesByChunkSize() [Fact] public async Task KeyIterationHonoursChunkSizeOverItsOlderName() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[],\"isTruncated\":false}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.KeysPage(4, false, null)); await DrainAsync(Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { ChunkSize = 4, MaxPageLimit = 9 })); @@ -109,7 +106,7 @@ public async Task KeyIterationHonoursChunkSizeOverItsOlderName() [Fact] public async Task RequestIterationHonoursChunkSize() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[],\"limit\":5}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.ListRequestsPage(5)); await DrainAsync(Client(transport).RequestQueue("q1").PaginateRequestsAsync(new PaginateRequestsOptions { ChunkSize = 5 })); @@ -117,12 +114,58 @@ public async Task RequestIterationHonoursChunkSize() } [Fact] - public async Task StoreIterationUsesChunkSizeAsPageSize() + public async Task StoreIterationSizesPagesByChunkSizeAndCapsTotalAtLimit() + { + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.Page(9, SpecFixtures.StoreItem("a"), SpecFixtures.StoreItem("b"))) + .QueueResponse(200, SpecFixtures.Page(9, SpecFixtures.StoreItem("c"))); + + var ids = new System.Collections.Generic.List(); + await foreach (var item in Client(transport).Store().IterateAsync(new StoreListOptions { ChunkSize = 2, Limit = 3 })) + { + ids.Add(item.Id); + } + + Assert.Equal(new[] { "a", "b", "c" }, ids); + Assert.Equal(2, transport.CallCount); + Assert.Contains("limit=2", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.Contains("limit=1", transport.Received[1].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task StoreIterationLimitIsATotalCapNotAPageSize() + { + // Limit = 2 is satisfied by the first page, so nothing more may be requested. + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(50, SpecFixtures.StoreItem("a"), SpecFixtures.StoreItem("b"))); + + var count = 0; + await foreach (var unused in Client(transport).Store().IterateAsync(new StoreListOptions { Limit = 2 })) + { + _ = unused; + count++; + } + + Assert.Equal(2, count); + Assert.Equal(1, transport.CallCount); + Assert.Contains("limit=2", transport.Received[0].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task StoreIterationDoesNotStopAtAnEarlierPagesStaleTotal() { - var transport = new MockTransport().QueueResponse(200, Page()); + // The Store grows between pages: page 1 reports a total of 2, but a third item exists on page 2. + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.Page(2, SpecFixtures.StoreItem("a"), SpecFixtures.StoreItem("b"))) + .QueueResponse(200, SpecFixtures.Page(3, SpecFixtures.StoreItem("c"))) + .QueueResponse(200, SpecFixtures.Page(3)); - await DrainAsync(Client(transport).Store().IterateAsync(new StoreListOptions { ChunkSize = 6, Limit = 100 })); + var count = 0; + await foreach (var unused in Client(transport).Store().IterateAsync()) + { + _ = unused; + count++; + } - Assert.Contains("limit=6", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.Equal(3, count); } } diff --git a/tests/Apify.Client.Tests/Unit/HttpClientTests.cs b/tests/Apify.Client.Tests/Unit/HttpClientTests.cs index b0dbbd0..44efef5 100644 --- a/tests/Apify.Client.Tests/Unit/HttpClientTests.cs +++ b/tests/Apify.Client.Tests/Unit/HttpClientTests.cs @@ -20,7 +20,7 @@ public sealed class HttpClientTests [Fact] public async Task AuthAndUserAgentHeadersAreSent() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"abc\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Actor("abc")); await Client(transport).Actor("abc").GetAsync(); Assert.Equal("Bearer test-token", transport.LastRequest.Header("Authorization")); @@ -30,7 +30,7 @@ public async Task AuthAndUserAgentHeadersAreSent() [Fact] public async Task DataEnvelopeIsUnwrapped() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"act1\",\"name\":\"my-actor\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Actor("act1", "my-actor")); var actor = await Client(transport).Actor("act1").GetAsync(); Assert.NotNull(actor); @@ -50,7 +50,7 @@ public async Task ServerErrorsAreRetriedThenSucceed() { var transport = new MockTransport() .QueueResponse(500, "{\"error\":{\"type\":\"server\",\"message\":\"boom\"}}") - .QueueResponse(200, "{\"data\":{\"id\":\"ok\"}}"); + .QueueResponse(200, SpecFixtures.Actor("ok")); var actor = await Client(transport).Actor("x").GetAsync(); Assert.Equal("ok", actor!.Id); @@ -76,7 +76,7 @@ public async Task TransportErrorsAreRetried() { var transport = new MockTransport() .QueueError() - .QueueResponse(200, "{\"data\":{\"id\":\"recovered\"}}"); + .QueueResponse(200, SpecFixtures.Actor("recovered")); var actor = await Client(transport).Actor("x").GetAsync(); Assert.Equal("recovered", actor!.Id); Assert.Equal(2, transport.CallCount); @@ -85,7 +85,7 @@ public async Task TransportErrorsAreRetried() [Fact] public async Task BooleanQueryParamsEncodedAsOneZero() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[],\"total\":0}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(0)); await Client(transport).Actors().ListAsync(new ActorListOptions { My = true, Limit = 5 }); var uri = transport.LastRequest.Uri; @@ -96,8 +96,7 @@ public async Task BooleanQueryParamsEncodedAsOneZero() [Fact] public async Task ListUnwrapsPaginationEnvelope() { - const string body = "{\"data\":{\"total\":2,\"offset\":0,\"limit\":10,\"count\":2,\"desc\":false,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}"; - var transport = new MockTransport().QueueResponse(200, body); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.ActorPage(2, "a", "b")); var page = await Client(transport).Actors().ListAsync(); Assert.Equal(2, page.Total); @@ -111,7 +110,7 @@ public async Task PaginationCountReflectsItemsNotServerMetadata() // Server reports total/count larger than the items actually returned in this page. Count and the // indexer must operate on the item array so `for (i < Count) page[i]` cannot throw; Total keeps the // API's reported total. - const string body = "{\"data\":{\"total\":100,\"offset\":0,\"limit\":2,\"count\":100,\"desc\":false,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}"; + var body = SpecFixtures.ActorPage(2, "a", "b").Replace("\"total\":2", "\"total\":100", StringComparison.Ordinal).Replace("\"count\":2", "\"count\":100", StringComparison.Ordinal); var transport = new MockTransport().QueueResponse(200, body); var page = await Client(transport).Actors().ListAsync(); @@ -151,7 +150,7 @@ public async Task ValidateInputParsesBareObject() [Fact] public async Task SafeIdReplacesFirstSlashWithTilde() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"x\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Actor("x")); await Client(transport).Actor("apify/hello-world").GetAsync(); Assert.Contains("/actors/apify~hello-world", transport.LastRequest.Uri, StringComparison.Ordinal); } @@ -162,7 +161,7 @@ public async Task RateLimitIsRetriedThenSucceeds() // 429 (rate limit) must be retried just like a 5xx, then succeed on the next attempt. var transport = new MockTransport() .QueueResponse(429, "{\"error\":{\"type\":\"rate-limit-exceeded\",\"message\":\"slow down\"}}") - .QueueResponse(200, "{\"data\":{\"id\":\"ok\"}}"); + .QueueResponse(200, SpecFixtures.Actor("ok")); var actor = await Client(transport).Actor("x").GetAsync(); Assert.Equal("ok", actor!.Id); @@ -244,7 +243,7 @@ public async Task ShortTierCallUsesConfiguredShortTimeoutByDefault() { // ActorClient.GetAsync is a Short-tier call (simple metadata read). With no explicit per-call/ // per-context override, its attempt timeout is TimeoutShortSecs, not the overall TimeoutSecs. - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"x\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Actor("x")); var client = new ApifyClient(new ApifyClientOptions { Token = "t", @@ -264,7 +263,7 @@ public async Task ShortTierCallUsesConfiguredShortTimeoutByDefault() public async Task MediumTierCallUsesConfiguredMediumTimeoutByDefault() { // ActorCollectionClient.ListAsync is a Medium-tier call (collection listing). - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"total\":0,\"items\":[]}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(0)); var client = new ApifyClient(new ApifyClientOptions { Token = "t", @@ -335,7 +334,7 @@ await Client(transport) public async Task NullQueryParamsAreOmitted() { // Only Limit is set; every other (null) option must be absent from the query string entirely. - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[],\"total\":0}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(0)); await Client(transport).Actors().ListAsync(new ActorListOptions { Limit = 5 }); var uri = transport.LastRequest.Uri; diff --git a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs index 0c2e8fb..2cdd36a 100644 --- a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs +++ b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs @@ -59,7 +59,7 @@ await Client(transport).Actor("me/act") [Fact] public async Task LastRunRequestQueueForwardsStatusAndOrigin() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[]}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.QueueHead()); await Client(transport).Actor("me/act") .LastRun(new LastRunOptions { Status = "SUCCEEDED", Origin = "API" }) .RequestQueue() @@ -141,7 +141,7 @@ public async Task RunChargeUsesProvidedIdempotencyKey() [Fact] public async Task MetamorphSendsTargetActorIdAndInput() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); await Client(transport).Run("run1").MetamorphAsync("apify/other", new { x = 1 }, new MetamorphOptions { Build = "latest" }); var request = transport.LastRequest; @@ -155,7 +155,7 @@ public async Task MetamorphSendsTargetActorIdAndInput() [Fact] public async Task ResurrectSendsOptions() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); await Client(transport).Run("run1").ResurrectAsync(new RunResurrectOptions { Build = "beta", MemoryMbytes = 1024 }); var uri = transport.LastRequest.Uri; @@ -167,7 +167,7 @@ public async Task ResurrectSendsOptions() [Fact] public async Task RebootPostsToRebootPath() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); await Client(transport).Run("run1").RebootAsync(); var request = transport.LastRequest; @@ -178,7 +178,7 @@ public async Task RebootPostsToRebootPath() [Fact] public async Task AbortSendsGracefullyFlag() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); await Client(transport).Run("run1").AbortAsync(true); Assert.Contains("gracefully=1", transport.LastRequest.Uri, StringComparison.Ordinal); } @@ -186,7 +186,7 @@ public async Task AbortSendsGracefullyFlag() [Fact] public async Task DefaultBuildFetchesBuildsDefault() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"build1\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Build("build1")); await Client(transport).Actor("me~a").DefaultBuildAsync(10); var request = transport.LastRequest; @@ -198,7 +198,7 @@ public async Task DefaultBuildFetchesBuildsDefault() [Fact] public async Task RequestQueueOptionsApplyClientKey() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[]}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.QueueHead()); await Client(transport).RequestQueue("q1", new RequestQueueClientOptions { ClientKey = "ck-123" }).ListHeadAsync(5); Assert.Contains("clientKey=ck-123", transport.LastRequest.Uri, StringComparison.Ordinal); @@ -207,7 +207,7 @@ public async Task RequestQueueOptionsApplyClientKey() [Fact] public async Task RequestQueueOptionsApplyTimeout() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[]}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.QueueHead()); await Client(transport).RequestQueue("q1", new RequestQueueClientOptions { TimeoutSecs = 2.0 }).ListHeadAsync(5); // The per-queue timeout must be threaded down to the transport (first attempt uses it directly). @@ -222,8 +222,8 @@ public async Task WithTimeoutOverridesTheTierDefaultForEveryCallOnThatClient() // as long as it doesn't exceed the overall budget (ApifyClientOptions.TimeoutSecs — see the // dedicated cap test below). var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"id\":\"x\"}}") - .QueueResponse(200, "{\"data\":{\"id\":\"x\"}}"); + .QueueResponse(200, SpecFixtures.Actor("x")) + .QueueResponse(200, SpecFixtures.Actor("x")); var client = new ApifyClient(new ApifyClientOptions { Token = "t", @@ -247,7 +247,7 @@ public async Task WithTimeoutAboveTheOverallBudgetIsCappedAtIt() // Matches the reference client: a requested timeout above timeoutMaxSecs (here, // ApifyClientOptions.TimeoutSecs) is capped at it rather than silently exceeding it — raising the // overall budget itself is how a caller gets a longer per-call timeout. - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"x\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Actor("x")); var client = new ApifyClient(new ApifyClientOptions { Token = "t", @@ -267,7 +267,7 @@ public async Task WithTimeoutZeroMeansNoTimeout() // TimeSpan.Zero is the client's "no timeout" sentinel (HttpClientTransport skips CancelAfter for a // non-positive duration), matching the reference client's 'noTimeout' — which, like here, bypasses // the overall-budget cap entirely rather than being clamped to it. - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"x\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Actor("x")); await Client(transport).Actor("x").WithTimeout(TimeSpan.Zero).GetAsync(); @@ -326,7 +326,7 @@ public async Task PublishPutsIsPublicTrue() { var transport = new MockTransport().QueueResponse( 200, - "{\"data\":{\"id\":\"t1\",\"isPublic\":true,\"publicConfig\":{\"publishedAt\":\"2026-01-01T00:00:00.000Z\",\"seoTitle\":\"My task\"}}}"); + SpecFixtures.Task("t1", "\"isPublic\":true,\"publicConfig\":{\"publishedAt\":\"2026-01-01T00:00:00.000Z\",\"seoTitle\":\"My task\"}")); var task = await Client(transport).Task("t1").PublishAsync(); var request = transport.LastRequest; @@ -341,7 +341,7 @@ public async Task PublishPutsIsPublicTrue() [Fact] public async Task UnpublishPutsIsPublicFalse() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"t1\",\"isPublic\":false}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Task("t1", "\"isPublic\":false")); var task = await Client(transport).Task("t1").UnpublishAsync(); var request = transport.LastRequest; @@ -372,7 +372,7 @@ await Client(transport).Dataset("ds1").ListItemsAsync(new DatasetListItemsOption [Fact] public async Task RunListJoinsStatusAsCsv() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[],\"total\":0}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(0)); await Client(transport).Runs().ListAsync(null, new RunListOptions { Status = new[] { "SUCCEEDED", "RUNNING" } }); Assert.Contains("status=SUCCEEDED%2CRUNNING", transport.LastRequest.Uri, StringComparison.Ordinal); @@ -381,7 +381,7 @@ public async Task RunListJoinsStatusAsCsv() [Fact] public async Task ListRequestsJoinsFilterAsCsv() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[]}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.QueueHead()); await Client(transport).RequestQueue("q1").ListRequestsAsync(new ListRequestsOptions { Filter = new[] { ListRequestsOptions.FilterLocked, ListRequestsOptions.FilterPending }, @@ -398,7 +398,7 @@ public async Task MaxTotalChargeUsdIsFormattedWithInvariantCulture() CultureInfo.CurrentCulture = new CultureInfo("de-DE"); try { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); await Client(transport).Actor("act").StartAsync(null, new ActorStartOptions { MaxTotalChargeUsd = 12.5 }); var uri = transport.LastRequest.Uri; @@ -414,7 +414,7 @@ public async Task MaxTotalChargeUsdIsFormattedWithInvariantCulture() [Fact] public async Task StartAsyncSendsRawBytesInputVerbatimWithoutJsonEncoding() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); var rawInput = new byte[] { 0x00, 0x01, 0xFF, (byte)'{', (byte)'}' }; await Client(transport).Actor("act").StartAsync(rawInput, new ActorStartOptions { ContentType = "application/octet-stream" }); @@ -426,7 +426,7 @@ public async Task StartAsyncSendsRawBytesInputVerbatimWithoutJsonEncoding() [Fact] public async Task MetamorphAsyncSendsRawBytesInputVerbatimWithoutJsonEncoding() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); var rawInput = new byte[] { 0x10, 0x20, 0x30 }; await Client(transport).Run("run1").MetamorphAsync( diff --git a/tests/Apify.Client.Tests/Unit/ResponseDecompressionTests.cs b/tests/Apify.Client.Tests/Unit/ResponseDecompressionTests.cs index d3696d9..359149f 100644 --- a/tests/Apify.Client.Tests/Unit/ResponseDecompressionTests.cs +++ b/tests/Apify.Client.Tests/Unit/ResponseDecompressionTests.cs @@ -25,7 +25,7 @@ public sealed class ResponseDecompressionTests [InlineData("deflate")] public async Task DefaultTransportDecompressesResponse(string encoding) { - const string json = "{\"data\":{\"id\":\"act1\",\"name\":\"compressed-actor\"}}"; + var json = SpecFixtures.Actor("act1", "compressed-actor"); var payload = Compress(Encoding.UTF8.GetBytes(json), encoding); using var server = new LoopbackServer(encoding, payload); diff --git a/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs b/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs new file mode 100644 index 0000000..f86e2a9 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs @@ -0,0 +1,136 @@ +using System; +using System.Linq; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Apify.Client.Exceptions; +using Apify.Client.Internal; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// Verifies the validation of successful responses against the schemas of the Apify OpenAPI +/// specification: required properties, JSON types, nullability, unknown properties and values, and how the +/// failures surface through the resource clients. +/// +[Trait("Category", "Unit")] +public sealed class ResponseSchemaTests +{ + private const string Build = "\"id\":\"b\",\"actId\":\"a\",\"userId\":\"u\",\"startedAt\":\"2026-01-01T00:00:00.000Z\",\"status\":\"SUCCEEDED\",\"meta\":{\"origin\":\"API\"},\"buildNumber\":\"0.0.1\""; + + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + private static string[] Issues(string method, string path, string body) + => ResponseSchemas.Validate(method, path, JsonNode.Parse(body)).ToArray(); + + [Fact] + public void EmbeddedSchemasMatchTheClientsSpecVersion() + { + Assert.Equal(ApifyClientVersion.ApiSpecVersion, ResponseSchemas.SpecVersion); + } + + [Fact] + public void ValidResponseHasNoIssues() + { + Assert.Empty(Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build + "}}")); + } + + [Fact] + public void MissingRequiredPropertyIsReportedWithItsLocation() + { + var issues = Issues("GET", "/v2/actor-builds/b1", "{\"data\":{\"id\":\"b\"}}"); + + Assert.Contains("$.data.actId: required property is missing", issues); + Assert.Contains("$.data.status: required property is missing", issues); + } + + [Fact] + public void WrongJsonTypeIsReported() + { + var issues = Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build.Replace("\"id\":\"b\"", "\"id\":5", StringComparison.Ordinal) + "}}"); + + Assert.Equal(new[] { "$.data.id: expected string but got number" }, issues); + } + + [Fact] + public void NullIsAcceptedWhereTheSpecAllowsIt() + { + // The spec lets a build's actVersion.buildTag be null (apify-docs #3093). + Assert.Empty(Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build + ",\"actVersion\":{\"buildTag\":null}}}")); + } + + [Fact] + public void NullIsRejectedWhereTheSpecDoesNotAllowIt() + { + var issues = Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build + ",\"actVersion\":{\"versionNumber\":null}}}"); + + Assert.Single(issues); + Assert.Contains("actVersion.versionNumber", issues[0], StringComparison.Ordinal); + } + + [Fact] + public void UnknownPropertiesAndUnknownEnumValuesAreAccepted() + { + Assert.Empty(Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build.Replace("SUCCEEDED", "SOME-FUTURE-STATUS", StringComparison.Ordinal) + ",\"brandNewField\":{\"x\":1}}}")); + } + + [Fact] + public void UndocumentedOperationIsNotValidated() + { + Assert.Empty(Issues("GET", "/v2/not-a-documented-endpoint", "[]")); + } + + [Fact] + public void TheMostSpecificPathTemplateWins() + { + // `/runs/last` must be matched by its own template, not by `/runs/{runId}`: both are valid runs, so + // the way to tell them apart is that an invalid body is still reported for either. + Assert.NotEmpty(Issues("GET", "/v2/actors/a1/runs/last?status=SUCCEEDED", "{\"data\":{}}")); + Assert.NotEmpty(Issues("GET", "/v2/actor-runs/r1", "{\"data\":{}}")); + } + + [Fact] + public async Task GetWithArrayDataThrowsInsteadOfLookingLikeNotFound() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":[]}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actor("me~act").GetAsync()); + + Assert.Equal("GET", ex.HttpMethod); + } + + [Fact] + public async Task ListWithArrayDataThrows() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":[]}"); + + await Assert.ThrowsAsync(() => Client(transport).Actors().ListAsync()); + } + + [Fact] + public async Task ResourceMissingRequiredPropertiesThrowsWithTheIssuesInTheMessage() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"b\"}}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Build("b1").GetAsync()); + + Assert.Contains("$.data.actId: required property is missing", ex.Message, StringComparison.Ordinal); + Assert.EndsWith("/actor-builds/b1", ex.Path, StringComparison.Ordinal); + } + + [Fact] + public async Task ValidResourceIsReturned() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{" + Build + "}}"); + + var build = await Client(transport).Build("b1").GetAsync(); + + Assert.Equal("b", build!.Id); + } +} diff --git a/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs b/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs index 465f9b5..3166837 100644 --- a/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs +++ b/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs @@ -16,8 +16,8 @@ namespace Apify.Client.Tests.Unit; [Trait("Category", "Unit")] public sealed class RunIterateDatasetItemsTests { - private const string FinishedRun = "{\"data\":{\"id\":\"run1\",\"status\":\"SUCCEEDED\"}}"; - private const string RunningRun = "{\"data\":{\"id\":\"run1\",\"status\":\"RUNNING\"}}"; + private static readonly string FinishedRun = SpecFixtures.Run("run1"); + private static readonly string RunningRun = SpecFixtures.Run("run1", "RUNNING"); private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions { @@ -27,7 +27,7 @@ public sealed class RunIterateDatasetItemsTests HttpTransport = transport, }); - private static string DatasetWithItemCount(int itemCount) => "{\"data\":{\"id\":\"ds1\",\"itemCount\":" + itemCount + "}}"; + private static string DatasetWithItemCount(int itemCount) => SpecFixtures.Dataset(itemCount); private static string Items(params int[] ids) => "[" + string.Join(",", ids.Select(static i => "{\"n\":" + i + "}")) + "]"; @@ -102,7 +102,7 @@ public async Task LimitBoundsScannedRowsAndStopsWithoutFurtherReads() public async Task LastRunClientPinsToTheRunItFirstResolved() { var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"id\":\"resolvedRun\",\"status\":\"SUCCEEDED\"}}") + .QueueResponse(200, SpecFixtures.Run("resolvedRun")) .QueueResponse(200, DatasetWithItemCount(1)) .QueueResponse(200, Items(1)) .QueueResponse(200, Items()); diff --git a/tests/Apify.Client.Tests/Unit/RunNestedClientTimeoutTests.cs b/tests/Apify.Client.Tests/Unit/RunNestedClientTimeoutTests.cs new file mode 100644 index 0000000..2dc2ab6 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/RunNestedClientTimeoutTests.cs @@ -0,0 +1,92 @@ +using System; +using System.Threading.Tasks; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// A run's and a build's WithTimeout override must reach the requests of the nested storage and log +/// clients obtained from them afterwards, including through a last-run client. +/// +[Trait("Category", "Unit")] +public sealed class RunNestedClientTimeoutTests +{ + private static readonly TimeSpan Override = TimeSpan.FromSeconds(3); + + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + private static string Dataset() => SpecFixtures.Dataset(); + + [Fact] + public async Task RunDatasetInheritsTheTimeout() + { + var transport = new MockTransport().QueueResponse(200, Dataset()); + await Client(transport).Run("r1").WithTimeout(Override).Dataset().GetAsync(); + Assert.Equal(3, transport.Timeouts[0]); + } + + [Fact] + public async Task RunKeyValueStoreInheritsTheTimeout() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"keyValue\":1}}").QueueResponse(200, "x"); + await Client(transport).Run("r1").WithTimeout(Override).KeyValueStore().RecordExistsAsync("k"); + Assert.All(transport.Timeouts, static t => Assert.Equal(3, t)); + } + + [Fact] + public async Task RunRequestQueueInheritsTheTimeout() + { + var transport = new MockTransport().QueueResponse(200, SpecFixtures.QueueHead()); + await Client(transport).Run("r1").WithTimeout(Override).RequestQueue().ListHeadAsync(1); + Assert.Equal(3, transport.Timeouts[0]); + } + + [Fact] + public async Task RunLogInheritsTheTimeout() + { + var transport = new MockTransport().QueueResponse(200, "log text"); + await Client(transport).Run("r1").WithTimeout(Override).Log().GetAsync(); + Assert.Equal(3, transport.Timeouts[0]); + } + + [Fact] + public async Task BuildLogInheritsTheTimeout() + { + var transport = new MockTransport().QueueResponse(200, "log text"); + await Client(transport).Build("b1").WithTimeout(Override).Log().GetAsync(); + Assert.Equal(3, transport.Timeouts[0]); + } + + [Fact] + public async Task LastRunDatasetInheritsTheTimeout() + { + var transport = new MockTransport().QueueResponse(200, Dataset()); + await Client(transport).Actor("me~act").LastRun().WithTimeout(Override).Dataset().GetAsync(); + Assert.Equal(3, transport.Timeouts[0]); + } + + [Fact] + public async Task IteratingALastRunPinnedToAnotherRunKeepsTheTimeout() + { + // The first request resolves `runs/last` to a concrete run; the pinned client used for the dataset + // requests must carry the same override. + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.Run("resolved")) + .QueueResponse(200, Dataset()) + .QueueResponse(200, "[]"); + + await foreach (var unused in Client(transport).Actor("me~act").LastRun().WithTimeout(Override).IterateDatasetItemsAsync()) + { + _ = unused; + } + + Assert.Contains("/actor-runs/resolved/dataset", transport.Received[1].Uri, StringComparison.Ordinal); + Assert.All(transport.Timeouts, static t => Assert.Equal(3, t)); + } +} diff --git a/tests/Apify.Client.Tests/Unit/SpecFixtures.cs b/tests/Apify.Client.Tests/Unit/SpecFixtures.cs new file mode 100644 index 0000000..2f55879 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/SpecFixtures.cs @@ -0,0 +1,95 @@ +using System.Globalization; +using System.Linq; + +namespace Apify.Client.Tests.Unit; + +/// +/// Response bodies that satisfy the schemas of the Apify OpenAPI specification (the client rejects a +/// successful response that does not), built with only the properties the specification requires. +/// +internal static class SpecFixtures +{ + private const string Timestamp = "2026-01-01T00:00:00.000Z"; + + /// A paginated list response ({"data": {"total", "offset", "limit", "desc", "count", "items"}}). + /// The reported total number of items. + /// The raw JSON of each item on this page. + public static string Page(int total, params string[] items) + => string.Format( + CultureInfo.InvariantCulture, + "{{\"data\":{{\"total\":{0},\"offset\":0,\"limit\":1000,\"desc\":false,\"count\":{1},\"items\":[{2}]}}}}", + total, + items.Length, + string.Join(",", items)); + + /// An item of the Actor collection. + public static string ActorItem(string id) + => "{\"id\":\"" + id + "\",\"createdAt\":\"" + Timestamp + "\",\"modifiedAt\":\"" + Timestamp + "\",\"name\":\"n\",\"username\":\"u\"}"; + + /// An item of the Actor task collection. + public static string TaskItem(string id) + => "{\"id\":\"" + id + "\",\"userId\":\"u\",\"actId\":\"a\",\"name\":\"n\",\"createdAt\":\"" + Timestamp + "\",\"modifiedAt\":\"" + Timestamp + "\"}"; + + /// An item of the schedule collection. + public static string ScheduleItem(string id) + => "{\"id\":\"" + id + "\",\"userId\":\"u\",\"name\":\"n\",\"cronExpression\":\"0 0 * * *\",\"timezone\":\"UTC\",\"isEnabled\":false,\"isExclusive\":false,\"createdAt\":\"" + Timestamp + "\",\"modifiedAt\":\"" + Timestamp + "\",\"actions\":[]}"; + + /// An item of the Store collection. + public static string StoreItem(string id) + => "{\"id\":\"" + id + "\",\"title\":\"t\",\"name\":\"n\",\"username\":\"u\",\"stats\":{}}"; + + /// A page of Actors listing the given IDs. + public static string ActorPage(int total, params string[] ids) => Page(total, ids.Select(ActorItem).ToArray()); + + /// An Actor (GET /v2/actors/{actorId} and the Actor update responses). + public static string Actor(string id, string name = "n", bool isPublic = false) + => "{\"data\":{\"id\":\"" + id + "\",\"userId\":\"u\",\"name\":\"" + name + "\",\"username\":\"u\",\"isPublic\":" + (isPublic ? "true" : "false") + + ",\"createdAt\":\"" + Timestamp + "\",\"modifiedAt\":\"" + Timestamp + "\",\"stats\":{},\"versions\":[],\"defaultRunOptions\":{}}}"; + + /// An Actor run. + public static string Run(string id, string status = "SUCCEEDED") + => "{\"data\":" + RunObject(id, status) + "}"; + + /// The bare run object, for tests that compose a larger body. + public static string RunObject(string id, string status = "SUCCEEDED") + => "{\"id\":\"" + id + "\",\"actId\":\"a\",\"userId\":\"u\",\"startedAt\":\"" + Timestamp + "\",\"status\":\"" + status + + "\",\"meta\":{\"origin\":\"API\"},\"stats\":{},\"options\":{\"build\":\"latest\",\"timeoutSecs\":0,\"memoryMbytes\":0,\"diskMbytes\":0},\"buildId\":\"b\"" + + ",\"defaultKeyValueStoreId\":\"kvs\",\"defaultDatasetId\":\"ds\",\"defaultRequestQueueId\":\"rq\"}"; + + /// An Actor build. + public static string Build(string id, string status = "SUCCEEDED") + => "{\"data\":{\"id\":\"" + id + "\",\"actId\":\"a\",\"userId\":\"u\",\"startedAt\":\"" + Timestamp + "\",\"status\":\"" + status + + "\",\"meta\":{\"origin\":\"API\"},\"buildNumber\":\"0.0.1\"}}"; + + /// An Actor task. + /// The task ID. + /// Raw JSON members appended to the task object (e.g. "isPublic":true). + public static string Task(string id, string? extraProperties = null) + { + var item = TaskItem(id); + return "{\"data\":" + (extraProperties is null ? item : item[..^1] + "," + extraProperties + "}") + "}"; + } + + /// A dataset with the given reported item count. + public static string Dataset(int itemCount = 0) + => "{\"data\":{\"id\":\"ds\",\"userId\":\"u\",\"createdAt\":\"" + Timestamp + "\",\"modifiedAt\":\"" + Timestamp + "\",\"accessedAt\":\"" + Timestamp + + "\",\"itemCount\":" + itemCount.ToString(CultureInfo.InvariantCulture) + ",\"cleanItemCount\":0,\"consoleUrl\":\"https://console.apify.com\"}}"; + + /// A page of key-value store keys. + /// The page limit reported by the API. + /// Whether more keys follow. + /// The key to continue after, or null. + /// The keys on this page. + public static string KeysPage(int limit, bool isTruncated, string? nextExclusiveStartKey, params string[] keys) + => "{\"data\":{\"items\":[" + string.Join(",", keys.Select(static k => "{\"key\":\"" + k + "\",\"size\":1,\"recordPublicUrl\":\"https://example.com/" + k + "\"}")) + "],\"count\":" + + keys.Length.ToString(CultureInfo.InvariantCulture) + ",\"limit\":" + limit.ToString(CultureInfo.InvariantCulture) + ",\"isTruncated\":" + (isTruncated ? "true" : "false") + + (nextExclusiveStartKey is null ? string.Empty : ",\"nextExclusiveStartKey\":\"" + nextExclusiveStartKey + "\"") + "}}"; + + /// A request queue head with the given raw request items. + public static string QueueHead(params string[] items) + => "{\"data\":{\"limit\":10,\"queueModifiedAt\":\"" + Timestamp + "\",\"hadMultipleClients\":false,\"items\":[" + string.Join(",", items) + "]}}"; + + /// A page of request queue requests (GET /v2/request-queues/{queueId}/requests) with the given raw request items. + public static string ListRequestsPage(int limit, params string[] items) + => "{\"data\":{\"items\":[" + string.Join(",", items) + "],\"limit\":" + limit.ToString(CultureInfo.InvariantCulture) + "}}"; +} diff --git a/tests/Apify.Client.Tests/Unit/WaitForFinishTests.cs b/tests/Apify.Client.Tests/Unit/WaitForFinishTests.cs index 6c7c439..9005feb 100644 --- a/tests/Apify.Client.Tests/Unit/WaitForFinishTests.cs +++ b/tests/Apify.Client.Tests/Unit/WaitForFinishTests.cs @@ -69,7 +69,7 @@ public async Task WaitForFinishToleratesABriefNotFoundThenSucceeds() // once the resource starts resolving, the run completes normally. var transport = new MockTransport() .QueueResponse(404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"Run was not found\"}}") - .QueueResponse(200, "{\"data\":{\"id\":\"run1\",\"status\":\"SUCCEEDED\"}}"); + .QueueResponse(200, SpecFixtures.Run("run1")); var run = await Client(transport).Run("run1").WaitForFinishAsync(waitSecs: 30); diff --git a/tools/generate_response_schemas.py b/tools/generate_response_schemas.py new file mode 100644 index 0000000..58ea08d --- /dev/null +++ b/tools/generate_response_schemas.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Generates src/Apify.Client/Internal/ResponseSchemas.json from the Apify OpenAPI specification. + +The client validates every successful JSON response against the schema the specification declares for the +operation (see ResponseSchemas.cs), so this file has to be regenerated whenever the client is moved to a new +specification version: + + python3 tools/generate_response_schemas.py [SPEC_URL_OR_PATH] + +The default source is https://docs.apify.com/api/openapi.json. Only what validation needs is kept: types, +required properties, nested properties and items, composition keywords and references. Documentation, +examples and keywords the client deliberately does not enforce (enum, const, pattern, format, bounds) are +dropped, which also keeps the embedded resource small. +""" +import json +import pathlib +import sys +import urllib.request + +DEFAULT_SPEC_URL = "https://docs.apify.com/api/openapi.json" +OUTPUT = pathlib.Path(__file__).resolve().parent.parent / "src" / "Apify.Client" / "Internal" / "ResponseSchemas.json" +HTTP_METHODS = ("get", "post", "put", "delete", "patch") +SCHEMA_REF_PREFIX = "#/components/schemas/" +# Keywords that carry structure the validator evaluates. Everything else is dropped. +SCHEMA_LIST_KEYWORDS = ("allOf", "anyOf", "oneOf") + + +def load_spec(source): + if source.startswith(("http://", "https://")): + with urllib.request.urlopen(source) as response: + return json.load(response) + with open(source, encoding="utf-8") as handle: + return json.load(handle) + + +def slim(schema): + """Reduces a schema to the keywords the client validates.""" + if not isinstance(schema, dict): + return schema + result = {} + if "$ref" in schema: + result["$ref"] = schema["$ref"] + if "type" in schema: + result["type"] = schema["type"] + if "required" in schema: + result["required"] = schema["required"] + if "properties" in schema: + result["properties"] = {name: slim(value) for name, value in schema["properties"].items()} + if "items" in schema: + result["items"] = slim(schema["items"]) + for keyword in SCHEMA_LIST_KEYWORDS: + if keyword in schema: + result[keyword] = [slim(value) for value in schema[keyword]] + return result + + +def collect_refs(node, found): + if isinstance(node, dict): + ref = node.get("$ref") + if isinstance(ref, str): + if not ref.startswith(SCHEMA_REF_PREFIX): + raise SystemExit("unsupported reference: " + ref) + found.add(ref[len(SCHEMA_REF_PREFIX):]) + for value in node.values(): + collect_refs(value, found) + elif isinstance(node, list): + for value in node: + collect_refs(value, found) + + +def success_json_schema(spec, operation): + for status, response in operation.get("responses", {}).items(): + if not status.startswith("2"): + continue + if "$ref" in response: + response = spec["components"]["responses"][response["$ref"].split("/")[-1]] + content = response.get("content", {}).get("application/json") + if content and "schema" in content: + return content["schema"] + return None + + +def main(): + spec = load_spec(sys.argv[1] if len(sys.argv) > 1 else DEFAULT_SPEC_URL) + operations = {} + for path, item in spec["paths"].items(): + for method in HTTP_METHODS: + operation = item.get(method) + schema = success_json_schema(spec, operation) if operation else None + if schema is not None: + operations[method.upper() + " " + path] = slim(schema) + + schemas = {} + pending = set() + collect_refs(operations, pending) + while pending: + name = pending.pop() + if name in schemas: + continue + schemas[name] = slim(spec["components"]["schemas"][name]) + collect_refs(schemas[name], pending) + + document = {"specVersion": spec["info"]["version"], "operations": operations, "schemas": dict(sorted(schemas.items()))} + OUTPUT.write_text(json.dumps(document, separators=(",", ":"), sort_keys=True) + "\n", encoding="utf-8") + print("wrote", OUTPUT, "(%d operations, %d schemas)" % (len(operations), len(schemas))) + + +if __name__ == "__main__": + main() From 56fb6aa19fc167e507d88cb660041ab58fd1140b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 14:16:34 +0000 Subject: [PATCH 5/8] fix: validate formats and ranges, share the reference paging loop, treat limit 0 as unbounded Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01UUMWnU4gH573HweHieMhHi --- .../workflows/dotnet-integration-tests.yml | 3 + CHANGELOG.md | 12 ++- docs/runs.md | 2 +- docs/tasks.md | 4 +- .../Exceptions/ResponseValidationException.cs | 21 +++-- src/Apify.Client/Internal/PageSizing.cs | 23 +++-- src/Apify.Client/Internal/PagedIteration.cs | 74 ++++++++++++++++ src/Apify.Client/Internal/QueryParams.cs | 17 ++++ src/Apify.Client/Internal/ResourceContext.cs | 50 ++++------- src/Apify.Client/Internal/ResponseReader.cs | 8 +- src/Apify.Client/Internal/ResponseSchemas.cs | 84 ++++++++++++++++++- .../Internal/ResponseSchemas.json | 2 +- src/Apify.Client/Options/StoreListOptions.cs | 3 +- src/Apify.Client/Resources/DatasetClient.cs | 54 ++++-------- .../Resources/StoreCollectionClient.cs | 1 - src/Apify.Client/Resources/TaskClient.cs | 2 +- .../Unit/AutoPagingTests.cs | 30 ++++++- .../Apify.Client.Tests/Unit/ChunkSizeTests.cs | 38 ++++++++- .../Unit/ResponseSchemaTests.cs | 47 +++++++++++ tests/Apify.Client.Tests/Unit/SpecFixtures.cs | 2 +- tools/generate_response_schemas.py | 12 ++- 21 files changed, 371 insertions(+), 118 deletions(-) create mode 100644 src/Apify.Client/Internal/PagedIteration.cs diff --git a/.github/workflows/dotnet-integration-tests.yml b/.github/workflows/dotnet-integration-tests.yml index 960447c..db93a32 100644 --- a/.github/workflows/dotnet-integration-tests.yml +++ b/.github/workflows/dotnet-integration-tests.yml @@ -10,6 +10,9 @@ on: - '**/*.cs' - '**/*.csproj' - '**/*.sln' + # The embedded response schemas change the validation of every response, and the script generates them. + - 'src/Apify.Client/Internal/ResponseSchemas.json' + - 'tools/**' - 'Directory.Build.props' - '.editorconfig' # The "Test examples" step validates the in-documentation snippets, so doc changes must re-run diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fcb72b..9f646c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,9 +14,10 @@ the `ApifyApiException` for its status code. - Added `ResponseValidationException` (an `ApifyClientException`), thrown when a successful response does not match the schema the OpenAPI specification declares for its operation (missing `data` envelope, - missing required properties, wrong JSON types, `null` where it is not allowed), instead of silently - returning `null` or an empty object. Unknown properties and enum values are accepted. The schemas are - embedded and generated by `tools/generate_response_schemas.py`. Matches the reference client's + missing required properties, wrong JSON types, `null` where it is not allowed, a value outside the + documented format, pattern or range), instead of silently returning `null` or an empty object. The + exception lists the problems in `Issues`. Unknown properties and enum values are accepted. The schemas + are embedded and generated by `tools/generate_response_schemas.py`. Matches the reference client's `ResponseValidationError`. - Added `ChunkSize` to `ListOptions`, `StorageListOptions`, `ActorListOptions`, `StoreListOptions`, `DatasetListItemsOptions`, `IterateKeysOptions` and `PaginateRequestsOptions`: the page size iterators @@ -27,6 +28,11 @@ yielded (as every other iterator does) instead of as the page size, and made it, like `DatasetClient.IterateItemsAsync`, keep paging until a page is empty instead of stopping at the total an earlier page reported, so items added while iterating are no longer missed. +- Fixed the list, Store and dataset-item iterators to treat a `Limit` of `0` as "no limit" (as the API + does) instead of stopping after the first page, and made dataset-item iteration advance by the larger of + the `X-Apify-Pagination-Count` header and the returned item count (capped at the requested limit), so a + header that lags behind freshly pushed items no longer ends the iteration early. A dataset `Limit` now + caps the rows scanned. - Nested accessors of a run (`Dataset()`, `KeyValueStore()`, `RequestQueue()`, `Log()`) and of a build (`Log()`) now inherit the run's or build's `WithTimeout` override. - Added a documentation test that compiles every C# snippet in the README and `docs/`. diff --git a/docs/runs.md b/docs/runs.md index 5ba78ad..edfbacd 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -9,7 +9,7 @@ Access the account-wide run collection with `client.Runs()`, an Actor's or task' - `IterateAsync(ListOptions? options = null, RunListOptions? filter = null)` → `IAsyncEnumerable` (lazy, all pages). -`ListOptions` fields: `Offset`, `Limit`, `Desc` (standard pagination) and `ChunkSize` (the page size `IterateAsync` requests; the smaller of it and the remaining `Limit` is used per call). `RunListOptions` fields: +`ListOptions` fields: `Offset`, `Limit`, `Desc` (standard pagination; `Limit` `0` or unset means no cap) and `ChunkSize` (the page size `IterateAsync` requests; the smaller of it and the remaining `Limit` is used per call). `RunListOptions` fields: `Status` (`IReadOnlyList?`, filter by one or more run statuses such as `SUCCEEDED`/`RUNNING`), `StartedAfter` and `StartedBefore` (ISO 8601 bounds, honoured only by the Actor- and task-scoped run collections). diff --git a/docs/tasks.md b/docs/tasks.md index 1677ce5..ec2e01f 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -24,8 +24,8 @@ a specific task with `client.Task(id)`. - `StartAsync(object? input = null, TaskStartOptions? options = null)` → `ActorRun`. - `CallAsync(object? input = null, TaskStartOptions? options = null, int? waitSecs = null, Action? log = null)` → `ActorRun` (`log`, if set, redirects the run's live log to that sink for the duration of the wait). -- `GetInputAsync()` → `JsonNode?` (throws if the task does not exist; a JSON `null` means the task has - no stored input) / `UpdateInputAsync(object input)` → `JsonNode?`. +- `GetInputAsync()` → `JsonNode?` (throws if the task does not exist; a task without stored input + yields an empty JSON object) / `UpdateInputAsync(object input)` → `JsonNode?`. - `LastRun(LastRunOptions? options = null)` → `RunClient`; `Runs()` → `RunCollectionClient`. - `Webhooks()` → read-only `NestedWebhookCollectionClient`. diff --git a/src/Apify.Client/Exceptions/ResponseValidationException.cs b/src/Apify.Client/Exceptions/ResponseValidationException.cs index 2962c6c..1b37d0d 100644 --- a/src/Apify.Client/Exceptions/ResponseValidationException.cs +++ b/src/Apify.Client/Exceptions/ResponseValidationException.cs @@ -1,30 +1,35 @@ -using System; +using System.Collections.Generic; namespace Apify.Client.Exceptions; /// -/// Thrown when a successful response is valid JSON but does not have the shape the Apify API -/// specification promises, for example it lacks the data envelope or carries an array where the -/// endpoint returns an object. The response arrived in full, so the request is not retried. Matches the -/// reference client's ResponseValidationError. +/// Thrown when a successful response is valid JSON but does not match the schema the Apify API +/// specification declares for its operation: the data envelope or a required property is missing, a +/// value has the wrong JSON type, is null where that is not allowed, or falls outside the documented +/// format, pattern or range. Unknown properties and unknown enum values are accepted. The response arrived +/// in full, so the request is not retried. Matches the reference client's ResponseValidationError. /// public sealed class ResponseValidationException : ApifyClientException { /// Creates a response-validation exception. - /// A description of how the response deviates from the specification. + /// Each problem found, as a location in the body and what is wrong with it. /// The HTTP status code of the response. /// The HTTP method of the request that produced the response. /// The API endpoint path of the request (URL excluding origin). /// The raw response body that failed validation. - public ResponseValidationException(string message, int statusCode, string httpMethod, string path, string body) - : base(message) + public ResponseValidationException(IReadOnlyList issues, int statusCode, string httpMethod, string path, string body) + : base("response does not match the API specification: " + string.Join("; ", issues)) { + Issues = issues; StatusCode = statusCode; HttpMethod = httpMethod; Path = path; Body = body; } + /// Each problem found, as a location in the body (e.g. $.data.id) and what is wrong with it. + public IReadOnlyList Issues { get; } + /// The HTTP status code of the response. public int StatusCode { get; } diff --git a/src/Apify.Client/Internal/PageSizing.cs b/src/Apify.Client/Internal/PageSizing.cs index 9b0bb64..a438bd4 100644 --- a/src/Apify.Client/Internal/PageSizing.cs +++ b/src/Apify.Client/Internal/PageSizing.cs @@ -21,22 +21,19 @@ public static void ValidateChunkSize(int? chunkSize, string optionName) } /// - /// Sets the limit query parameter of the next page: the smaller of the chunk size and the number - /// of items still wanted under the caller's total limit. A total limit of 0 means unset (the API - /// itself reads limit=0 as unbounded), so it loses to the chunk size, and when neither is set the - /// query is left as the caller built it. + /// The smaller of two page limits, for the limit query parameter. The API reads a limit of + /// 0 as unset, so 0 (like null) means "no limit" here and loses to any other value; + /// the result is null when neither is set. Matches the reference client's minForLimitParam. /// - /// The page query, which already carries the caller's own limit if any. - /// The validated chunk size, or null when unset. - /// The caller's total limit across all pages, or null when unbounded. - /// The number of items already yielded. - public static void SetPageLimit(QueryParams query, int? chunkSize, int? totalLimit, int yielded) + public static int? MinForLimit(int? a, int? b) { - var remaining = totalLimit is null or 0 ? null : (int?)Math.Max(totalLimit.Value - yielded, 0); - var pageLimit = chunkSize is null ? remaining : (remaining is null ? chunkSize : Math.Min(chunkSize.Value, remaining.Value)); - if (pageLimit is not null) + a = a is 0 ? null : a; + b = b is 0 ? null : b; + if (a is null) { - query.Set("limit", pageLimit.Value); + return b; } + + return b is null ? a : Math.Min(a.Value, b.Value); } } diff --git a/src/Apify.Client/Internal/PagedIteration.cs b/src/Apify.Client/Internal/PagedIteration.cs new file mode 100644 index 0000000..c257aee --- /dev/null +++ b/src/Apify.Client/Internal/PagedIteration.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Apify.Client.Internal; + +/// One fetched page: its items and, when the API reports it, the number of rows it scanned. +/// The items on the page. +/// The rows the API scanned to produce the page (null when not reported). +/// The item type. +internal readonly record struct FetchedPage(IReadOnlyList Items, long? ScannedRows); + +/// +/// The offset/limit paging loop shared by every list iterator, ported from the reference client's +/// listPaginatedFromCallback. +/// +internal static class PagedIteration +{ + /// + /// Lazily yields every item from on, fetching pages on demand. + /// + /// + /// caps the number of scanned rows (null or 0 for no cap). + /// The next offset and the stop condition follow the rows a page scanned, not the items it returned: + /// advancing by the item count would re-scan rows after a filter dropped some and skip rows after + /// unwind multiplied them, and stopping at an empty page would end the iteration in front of rows + /// a filter hid. The scanned count is derived from a dataset's item count, which lags a fresh push, so + /// the larger of it and the item count is used, capped at the rows the request asked for so an unwound + /// page does not advance past rows the next request would then never read. The total a page reports is + /// never consulted, because it changes when the listed resource grows during the iteration; iteration + /// stops once a page scans no rows, which costs one extra request at the end. + /// + /// The offset of the first row. + /// The most rows to scan across all pages, or null/0 for no cap. + /// The most rows to request per call, or null for the API default. + /// Fetches one page given its offset and limit (null limit: leave it out). + /// A token to cancel the iteration. + /// The item type. + public static async IAsyncEnumerable IterateAsync( + int startOffset, + int? totalLimit, + int? chunkSize, + Func>> fetchPage, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + PageSizing.ValidateChunkSize(chunkSize, "ChunkSize"); + var offset = Math.Max(startOffset, 0); + int? remaining = totalLimit is > 0 ? totalLimit : null; + + long scanned; + do + { + var requestedLimit = PageSizing.MinForLimit(remaining, chunkSize); + var page = await fetchPage(offset, requestedLimit, cancellationToken).ConfigureAwait(false); + foreach (var item in page.Items) + { + yield return item; + } + + scanned = ScannedRows(page, requestedLimit); + offset += (int)scanned; + remaining -= (int?)scanned; + } + while (scanned > 0 && (remaining is null || remaining > 0)); + } + + private static long ScannedRows(FetchedPage page, int? requestedLimit) + { + var scanned = Math.Max(page.ScannedRows ?? 0, page.Items.Count); + return requestedLimit is > 0 ? Math.Min(scanned, requestedLimit.Value) : scanned; + } +} diff --git a/src/Apify.Client/Internal/QueryParams.cs b/src/Apify.Client/Internal/QueryParams.cs index 796a529..6307fa7 100644 --- a/src/Apify.Client/Internal/QueryParams.cs +++ b/src/Apify.Client/Internal/QueryParams.cs @@ -92,6 +92,23 @@ public QueryParams Set(string key, long value) return this; } + /// + /// Returns a copy with the page's offset and limit set, the latter left out when + /// is null (so a caller's own limit does not leak into every page). + /// + public QueryParams PageQuery(int offset, int? limit) + { + var page = Copy().Set("offset", offset).Remove("limit"); + return limit is null ? page : page.Set("limit", limit.Value); + } + + /// Removes every occurrence of . + public QueryParams Remove(string key) + { + _pairs.RemoveAll(pair => string.Equals(pair.Key, key, StringComparison.Ordinal)); + return this; + } + /// Whether no parameters have been added. public bool IsEmpty => _pairs.Count == 0; diff --git a/src/Apify.Client/Internal/ResourceContext.cs b/src/Apify.Client/Internal/ResourceContext.cs index 5a32911..863f905 100644 --- a/src/Apify.Client/Internal/ResourceContext.cs +++ b/src/Apify.Client/Internal/ResourceContext.cs @@ -214,20 +214,13 @@ public async Task> ListResourceAsync(string subPath, QueryP } /// - /// Lazily iterates every item of an offset/limit-paginated listing, fetching pages on demand. Mirrors - /// the reference client's auto-paging list iterators: it starts at and, - /// if is set, yields at most that many items in total. - /// carries the caller's filters; its offset/limit are overwritten per page. + /// Lazily iterates every item of an offset/limit-paginated listing, fetching pages on demand (see + /// ). It starts at ; + /// caps the total number of items (null or 0 for no cap) and + /// sizes each page request. carries the caller's filters; its + /// offset/limit are replaced per page. /// - /// - /// Paging stops once a page returns no items (or is reached), never by - /// comparing the offset to a page's reported total: the listed resource can grow while the - /// iteration runs (e.g. a dataset collection a running Actor keeps adding to), and a reported total - /// only ever reflects the moment its own page was fetched. Stopping at an early page's total would cut - /// the iteration off before items added later, so — like the reference client — this always issues one - /// more request at the end to confirm there is nothing left, unless is hit first. - /// - public async IAsyncEnumerable IterateListAsync( + public IAsyncEnumerable IterateListAsync( string subPath, QueryParams baseQuery, int startOffset, @@ -235,29 +228,18 @@ public async IAsyncEnumerable IterateListAsync( int? chunkSize, Func hydrate, RequestTier tier, - [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct) + CancellationToken ct) { - PageSizing.ValidateChunkSize(chunkSize, "ChunkSize"); - var offset = Math.Max(startOffset, 0); - var yielded = 0; - while (true) - { - var q = baseQuery.Copy().Set("offset", offset); - PageSizing.SetPageLimit(q, chunkSize, limit, yielded); - - var page = await ListResourceAsync(subPath, q, hydrate, tier, ct).ConfigureAwait(false); - foreach (var item in page.Items) - { - yield return item; - yielded++; - } - - offset += (int)page.Count; - if (page.Count == 0 || (limit is not null && yielded >= limit.Value)) + return PagedIteration.IterateAsync( + startOffset, + limit, + chunkSize, + async (offset, pageLimit, token) => { - yield break; - } - } + var page = await ListResourceAsync(subPath, baseQuery.PageQuery(offset, pageLimit), hydrate, tier, token).ConfigureAwait(false); + return new FetchedPage(page.Items, null); + }, + ct); } /// POST to create a resource with a JSON-serializable body, returning the decoded data. diff --git a/src/Apify.Client/Internal/ResponseReader.cs b/src/Apify.Client/Internal/ResponseReader.cs index de4d4ee..ef6d0c1 100644 --- a/src/Apify.Client/Internal/ResponseReader.cs +++ b/src/Apify.Client/Internal/ResponseReader.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Net.Http; using System.Text.Json; using System.Text.Json.Nodes; @@ -64,7 +65,7 @@ private static void ValidateAgainstSpec(HttpResponseMessage response, string bod var issues = ResponseSchemas.Validate(MethodOf(response), PathOf(response), node); if (issues.Count > 0) { - throw Invalid(response, body, string.Join("; ", issues)); + throw Invalid(response, body, issues); } } @@ -87,8 +88,11 @@ private static void ValidateAgainstSpec(HttpResponseMessage response, string bod } private static ResponseValidationException Invalid(HttpResponseMessage response, string body, string problem) + => Invalid(response, body, new[] { problem }); + + private static ResponseValidationException Invalid(HttpResponseMessage response, string body, IReadOnlyList issues) => new( - "response does not match the API specification: " + problem, + issues, (int)response.StatusCode, MethodOf(response), PathOf(response), diff --git a/src/Apify.Client/Internal/ResponseSchemas.cs b/src/Apify.Client/Internal/ResponseSchemas.cs index 56717d3..034cc6a 100644 --- a/src/Apify.Client/Internal/ResponseSchemas.cs +++ b/src/Apify.Client/Internal/ResponseSchemas.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.IO; @@ -17,10 +18,11 @@ namespace Apify.Client.Internal; /// /// The schemas come from the embedded ResponseSchemas.json, generated from the specification by /// tools/generate_response_schemas.py. Validation covers JSON types, required properties (including -/// nullability), nested properties and array items, and the allOf/anyOf/oneOf -/// combinators. Like the reference client's schemas, it lets unknown properties and unknown enum values -/// through, so an API that grows new fields or values does not break existing clients. A request that -/// matches no documented operation is not validated. +/// nullability), nested properties and array items, the allOf/anyOf/oneOf combinators, +/// and value constraints: the date-time, uri and email formats, pattern, +/// minimum/maximum and minLength/maxLength. Like the reference client's +/// schemas, it lets unknown properties and unknown enum values through, so an API that grows new fields or +/// values does not break existing clients. A request that matches no documented operation is not validated. /// internal static class ResponseSchemas { @@ -31,6 +33,18 @@ internal static class ResponseSchemas /// The most issues reported for one response, so a badly wrong body stays readable. private const int MaxReportedIssues = 10; + /// RFC 3339 date-time: the shape DateTimeOffset.TryParse alone would be too lenient about. + private static readonly Regex DateTimePattern = new( + @"^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}:\d{2})$", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex EmailPattern = new(@"^[^@\s]+@[^@\s]+\.[^@\s]+$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + + /// Bounds the time a schema's pattern may take, so a pathological pattern cannot hang a response read. + private static readonly TimeSpan PatternTimeout = TimeSpan.FromSeconds(1); + + private static readonly ConcurrentDictionary Patterns = new(StringComparer.Ordinal); + private static readonly Lazy Loaded = new(Load); /// The OpenAPI specification version the embedded schemas were generated from. @@ -155,6 +169,11 @@ public void Check(JsonNode? value, JsonNode? schema, string location, ListChecks the value constraints (format, pattern, bounds, length) that apply to a scalar. + private static void CheckConstraints(JsonNode value, JsonObject rules, string location, List issues) + { + switch (value.GetValueKind()) + { + case JsonValueKind.String: + var text = value.GetValue(); + if (rules["format"] is JsonNode format && !MatchesFormat(text, (string)format!)) + { + issues.Add(location + ": is not a valid " + (string)format! + " value"); + } + + if (rules["pattern"] is JsonNode pattern && !PatternFor((string)pattern!).IsMatch(text)) + { + issues.Add(location + ": does not match the pattern " + (string)pattern!); + } + + if (rules["minLength"] is JsonNode minLength && text.Length < (int)minLength!) + { + issues.Add(location + ": is shorter than the minimum length " + ((int)minLength!).ToString(CultureInfo.InvariantCulture)); + } + + if (rules["maxLength"] is JsonNode maxLength && text.Length > (int)maxLength!) + { + issues.Add(location + ": is longer than the maximum length " + ((int)maxLength!).ToString(CultureInfo.InvariantCulture)); + } + + break; + case JsonValueKind.Number: + var number = value.GetValue().GetDouble(); + if (rules["minimum"] is JsonNode minimum && number < (double)minimum!) + { + issues.Add(location + ": is a value outside the documented range (minimum " + ((double)minimum!).ToString(CultureInfo.InvariantCulture) + ")"); + } + + if (rules["maximum"] is JsonNode maximum && number > (double)maximum!) + { + issues.Add(location + ": is a value outside the documented range (maximum " + ((double)maximum!).ToString(CultureInfo.InvariantCulture) + ")"); + } + + break; + default: + break; + } + } + + private static bool MatchesFormat(string text, string format) => format switch + { + "date-time" => DateTimePattern.IsMatch(text) && DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.None, out _), + "uri" => Uri.TryCreate(text, UriKind.Absolute, out _), + "email" => EmailPattern.IsMatch(text), + _ => true, + }; + + private static Regex PatternFor(string pattern) + => Patterns.GetOrAdd(pattern, static p => new Regex(p, RegexOptions.CultureInvariant, PatternTimeout)); + private void CheckObject(JsonObject obj, JsonObject rules, string location, List issues) { if (rules["required"] is JsonArray required) diff --git a/src/Apify.Client/Internal/ResponseSchemas.json b/src/Apify.Client/Internal/ResponseSchemas.json index 577a835..a93ee10 100644 --- a/src/Apify.Client/Internal/ResponseSchemas.json +++ b/src/Apify.Client/Internal/ResponseSchemas.json @@ -1 +1 @@ -{"operations":{"DELETE /v2/actor-runs/{runId}/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/actor-tasks/{actorTaskId}":{"type":"object"},"DELETE /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/actors/{actorId}":{"type":"object"},"DELETE /v2/actors/{actorId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/actors/{actorId}/versions/{versionNumber}":{"type":"object"},"DELETE /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"type":"object"},"DELETE /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"DELETE /v2/request-queues/{queueId}/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/schedules/{scheduleId}":{"type":"object"},"DELETE /v2/webhooks/{webhookId}":{"type":"object"},"GET /v2/actor-builds":{"$ref":"#/components/schemas/ListOfBuildsResponse"},"GET /v2/actor-builds/{buildId}":{"allOf":[{"$ref":"#/components/schemas/BuildResponse"}]},"GET /v2/actor-builds/{buildId}/openapi.json":{"type":"object"},"GET /v2/actor-runs":{"allOf":[{"$ref":"#/components/schemas/ListOfRunsResponse"}]},"GET /v2/actor-runs/{runId}":{"allOf":[{"$ref":"#/components/schemas/RunResponse"}]},"GET /v2/actor-runs/{runId}/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/actor-runs/{runId}/dataset/items":{"items":{"type":"object"},"type":"array"},"GET /v2/actor-runs/{runId}/dataset/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/actor-runs/{runId}/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/actor-runs/{runId}/key-value-store/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/actor-runs/{runId}/key-value-store/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/actor-runs/{runId}/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/actor-runs/{runId}/request-queue/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/actor-runs/{runId}/request-queue/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/actor-runs/{runId}/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/actor-tasks":{"$ref":"#/components/schemas/ListOfTasksResponse"},"GET /v2/actor-tasks/{actorTaskId}":{"properties":{"data":{"$ref":"#/components/schemas/Task"}},"required":["data"],"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/input":{"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/run-sync":{"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/run-sync-get-dataset-items":{"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/runs":{"properties":{"data":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/RunListItem"},"type":"array"}},"required":["items"],"type":"object"}]}},"required":["data"],"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/runs/last":{"properties":{"data":{"$ref":"#/components/schemas/Run"}},"required":["data"],"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/dataset/items":{"items":{"type":"object"},"type":"array"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/dataset/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/actor-tasks/{actorTaskId}/webhooks":{"properties":{"data":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/WebhookResource"},"type":"array"}},"required":["items"],"type":"object"}]}},"required":["data"],"type":"object"},"GET /v2/actors":{"$ref":"#/components/schemas/ListOfActorsResponse"},"GET /v2/actors/{actorId}":{"$ref":"#/components/schemas/ActorResponse"},"GET /v2/actors/{actorId}/builds":{"$ref":"#/components/schemas/ListOfBuildsResponse"},"GET /v2/actors/{actorId}/builds/default":{"$ref":"#/components/schemas/BuildResponse"},"GET /v2/actors/{actorId}/builds/{buildId}":{"$ref":"#/components/schemas/BuildResponse"},"GET /v2/actors/{actorId}/builds/{buildId}/openapi.json":{"type":"object"},"GET /v2/actors/{actorId}/run-sync":{"type":"object"},"GET /v2/actors/{actorId}/run-sync-get-dataset-items":{"items":{"type":"object"},"type":"array"},"GET /v2/actors/{actorId}/runs":{"$ref":"#/components/schemas/ListOfRunsResponse"},"GET /v2/actors/{actorId}/runs/last":{"$ref":"#/components/schemas/RunResponse"},"GET /v2/actors/{actorId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/actors/{actorId}/runs/last/dataset/items":{"items":{"type":"object"},"type":"array"},"GET /v2/actors/{actorId}/runs/last/dataset/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/actors/{actorId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/actors/{actorId}/runs/last/key-value-store/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/actors/{actorId}/runs/last/key-value-store/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/actors/{actorId}/runs/{runId}":{"$ref":"#/components/schemas/RunResponse"},"GET /v2/actors/{actorId}/versions":{"$ref":"#/components/schemas/ListOfVersionsResponse"},"GET /v2/actors/{actorId}/versions/{versionNumber}":{"$ref":"#/components/schemas/VersionResponse"},"GET /v2/actors/{actorId}/versions/{versionNumber}/env-vars":{"$ref":"#/components/schemas/ListOfEnvVarsResponse"},"GET /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"$ref":"#/components/schemas/EnvVarResponse"},"GET /v2/actors/{actorId}/webhooks":{"$ref":"#/components/schemas/ListOfWebhooksResponse"},"GET /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"GET /v2/datasets":{"$ref":"#/components/schemas/ListOfDatasetsResponse"},"GET /v2/datasets/{datasetId}":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/datasets/{datasetId}/items":{"items":{"type":"object"},"type":"array"},"GET /v2/datasets/{datasetId}/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/key-value-stores":{"$ref":"#/components/schemas/ListOfKeyValueStoresResponse"},"GET /v2/key-value-stores/{storeId}":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/key-value-stores/{storeId}/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/key-value-stores/{storeId}/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/request-queues":{"$ref":"#/components/schemas/ListOfRequestQueuesResponse"},"GET /v2/request-queues/{queueId}":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/request-queues/{queueId}/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/request-queues/{queueId}/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/request-queues/{queueId}/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/schedules":{"$ref":"#/components/schemas/ListOfSchedulesResponse"},"GET /v2/schedules/{scheduleId}":{"$ref":"#/components/schemas/ScheduleResponse"},"GET /v2/schedules/{scheduleId}/log":{"$ref":"#/components/schemas/ScheduleLogResponse"},"GET /v2/store":{"$ref":"#/components/schemas/ListOfStoreActorsResponse"},"GET /v2/users/me":{"$ref":"#/components/schemas/UserPrivateInfoResponse"},"GET /v2/users/me/limits":{"$ref":"#/components/schemas/AccountLimitsResponse"},"GET /v2/users/me/usage/monthly":{"$ref":"#/components/schemas/MonthlyUsageResponse"},"GET /v2/users/{userId}":{"$ref":"#/components/schemas/UserPublicInfoResponse"},"GET /v2/webhook-dispatches":{"$ref":"#/components/schemas/ListOfWebhookDispatchesResponse"},"GET /v2/webhook-dispatches/{dispatchId}":{"$ref":"#/components/schemas/WebhookDispatchResponse"},"GET /v2/webhooks":{"$ref":"#/components/schemas/ListOfWebhooksResponse"},"GET /v2/webhooks/{webhookId}":{"$ref":"#/components/schemas/WebhookResponse"},"GET /v2/webhooks/{webhookId}/dispatches":{"$ref":"#/components/schemas/ListOfWebhookDispatchesResponse"},"POST /v2/actor-builds/{buildId}/abort":{"allOf":[{"$ref":"#/components/schemas/BuildResponse"}]},"POST /v2/actor-runs/{runId}/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-runs/{runId}/charge":{"type":"object"},"POST /v2/actor-runs/{runId}/dataset/items":{"type":"object"},"POST /v2/actor-runs/{runId}/key-value-store/records/{recordKey}":{"type":"object"},"POST /v2/actor-runs/{runId}/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-runs/{runId}/reboot":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-runs/{runId}/request-queue/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/actor-runs/{runId}/request-queue/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/actor-runs/{runId}/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/actor-runs/{runId}/request-queue/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/actor-runs/{runId}/resurrect":{"allOf":[{"$ref":"#/components/schemas/RunResponse"}]},"POST /v2/actor-tasks":{"$ref":"#/components/schemas/TaskResponse"},"POST /v2/actor-tasks/{actorTaskId}/run-sync":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/run-sync-get-dataset-items":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs":{"properties":{"data":{"$ref":"#/components/schemas/Run"}},"required":["data"],"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/dataset/items":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/reboot":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/actors":{"$ref":"#/components/schemas/ActorResponse"},"POST /v2/actors/{actorId}/builds":{"$ref":"#/components/schemas/BuildResponse"},"POST /v2/actors/{actorId}/builds/{buildId}/abort":{"$ref":"#/components/schemas/BuildResponse"},"POST /v2/actors/{actorId}/run-sync":{"type":"object"},"POST /v2/actors/{actorId}/run-sync-get-dataset-items":{"items":{"type":"object"},"type":"array"},"POST /v2/actors/{actorId}/runs":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/dataset/items":{"type":"object"},"POST /v2/actors/{actorId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"POST /v2/actors/{actorId}/runs/last/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/reboot":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/actors/{actorId}/runs/{runId}/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/{runId}/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/{runId}/resurrect":{"allOf":[{"$ref":"#/components/schemas/RunResponse"}]},"POST /v2/actors/{actorId}/validate-input":{"properties":{"valid":{"type":"boolean"}},"required":["valid"],"type":"object"},"POST /v2/actors/{actorId}/versions":{"$ref":"#/components/schemas/VersionResponse"},"POST /v2/actors/{actorId}/versions/{versionNumber}":{"$ref":"#/components/schemas/VersionResponse"},"POST /v2/actors/{actorId}/versions/{versionNumber}/env-vars":{"$ref":"#/components/schemas/EnvVarResponse"},"POST /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"$ref":"#/components/schemas/EnvVarResponse"},"POST /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"POST /v2/datasets":{"$ref":"#/components/schemas/DatasetResponse"},"POST /v2/datasets/{datasetId}/items":{"type":"object"},"POST /v2/key-value-stores":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"POST /v2/key-value-stores/{storeId}/records/{recordKey}":{"type":"object"},"POST /v2/request-queues":{"$ref":"#/components/schemas/RequestQueueResponse"},"POST /v2/request-queues/{queueId}/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/request-queues/{queueId}/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/request-queues/{queueId}/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/request-queues/{queueId}/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/schedules":{"$ref":"#/components/schemas/ScheduleResponse"},"POST /v2/tools/decode-and-verify":{"$ref":"#/components/schemas/DecodeAndVerifyResponse"},"POST /v2/tools/encode-and-sign":{"$ref":"#/components/schemas/EncodeAndSignResponse"},"POST /v2/webhooks":{"$ref":"#/components/schemas/WebhookResponse"},"POST /v2/webhooks/{webhookId}/test":{"$ref":"#/components/schemas/WebhookDispatchResponse"},"PUT /v2/actor-runs/{runId}":{"$ref":"#/components/schemas/RunResponse"},"PUT /v2/actor-runs/{runId}/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/actor-runs/{runId}/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/actor-runs/{runId}/key-value-store/records/{recordKey}":{"type":"object"},"PUT /v2/actor-runs/{runId}/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/actor-runs/{runId}/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/actor-runs/{runId}/request-queue/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/actor-tasks/{actorTaskId}":{"properties":{"data":{"$ref":"#/components/schemas/Task"}},"required":["data"],"type":"object"},"PUT /v2/actor-tasks/{actorTaskId}/input":{"type":"object"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/actors/{actorId}":{"$ref":"#/components/schemas/ActorResponse"},"PUT /v2/actors/{actorId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/actors/{actorId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/actors/{actorId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"PUT /v2/actors/{actorId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/actors/{actorId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/actors/{actorId}/runs/last/request-queue/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/actors/{actorId}/versions/{versionNumber}":{"$ref":"#/components/schemas/VersionResponse"},"PUT /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"$ref":"#/components/schemas/EnvVarResponse"},"PUT /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"PUT /v2/datasets/{datasetId}":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/key-value-stores/{storeId}":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/key-value-stores/{storeId}/records/{recordKey}":{"type":"object"},"PUT /v2/request-queues/{queueId}":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/request-queues/{queueId}/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/request-queues/{queueId}/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/schedules/{scheduleId}":{"$ref":"#/components/schemas/ScheduleResponse"},"PUT /v2/users/me/limits":{"type":"object"},"PUT /v2/webhooks/{webhookId}":{"$ref":"#/components/schemas/WebhookResponse"}},"schemas":{"AccountLimits":{"properties":{"current":{"$ref":"#/components/schemas/CurrentUsage"},"limits":{"$ref":"#/components/schemas/Limits"},"monthlyUsageCycle":{"$ref":"#/components/schemas/UsageCycle"}},"required":["monthlyUsageCycle","limits","current"],"type":"object"},"AccountLimitsResponse":{"properties":{"data":{"$ref":"#/components/schemas/AccountLimits"}},"required":["data"],"type":"object"},"ActorDefinition":{"properties":{"actorSpecification":{"type":"integer"},"buildTag":{"type":"string"},"changelog":{"type":["string","null"]},"defaultMemoryMbytes":{"oneOf":[{"type":"string"},{"type":"integer"}]},"dockerContextDir":{"type":"string"},"dockerfile":{"type":"string"},"environmentVariables":{"type":"object"},"input":{"type":["object","null"]},"maxMemoryMbytes":{"type":"integer"},"minMemoryMbytes":{"type":"integer"},"name":{"type":"string"},"readme":{"type":["string","null"]},"storages":{"properties":{"dataset":{"type":"object"}},"type":"object"},"usesStandbyMode":{"type":"boolean"},"version":{"type":"string"}},"type":"object"},"ActorJobStatus":{"type":"string"},"ActorListItem":{"properties":{"createdAt":{"type":"string"},"id":{"type":"string"},"modifiedAt":{"type":"string"},"name":{"type":"string"},"stats":{"anyOf":[{"$ref":"#/components/schemas/ActorStats"},{"type":"null"}]},"title":{"type":"string"},"username":{"type":"string"}},"required":["id","createdAt","modifiedAt","name","username"],"type":"object"},"ActorNotice":{"type":["string","null"]},"ActorPermissionLevel":{"type":"string"},"ActorResource":{"properties":{"actorPermissionLevel":{"$ref":"#/components/schemas/ActorPermissionLevel"},"actorStandby":{"anyOf":[{"$ref":"#/components/schemas/ActorStandby"},{"type":"null"}]},"categories":{"items":{"type":"string"},"type":"array"},"createdAt":{"type":"string"},"defaultRunOptions":{"$ref":"#/components/schemas/DefaultRunOptions"},"deploymentKey":{"type":"string"},"description":{"type":["string","null"]},"exampleRunInput":{"anyOf":[{"$ref":"#/components/schemas/ExampleRunInput"},{"type":"null"}]},"hasNoDataset":{"type":"boolean"},"id":{"type":"string"},"isCritical":{"type":"boolean"},"isDeprecated":{"type":["boolean","null"]},"isGeneric":{"type":"boolean"},"isPublic":{"type":"boolean"},"isSourceCodeHidden":{"type":"boolean"},"modifiedAt":{"type":"string"},"name":{"type":"string"},"notice":{"$ref":"#/components/schemas/ActorNotice"},"pictureUrl":{"type":["string","null"]},"pricingInfos":{"items":{"$ref":"#/components/schemas/ActorRunPricingInfo"},"type":"array"},"readmeSummary":{"type":"string"},"restartOnError":{"type":["boolean","null"]},"seoDescription":{"type":["string","null"]},"seoTitle":{"type":["string","null"]},"standbyUrl":{"type":["string","null"]},"stats":{"$ref":"#/components/schemas/ActorStats"},"taggedBuilds":{"anyOf":[{"$ref":"#/components/schemas/TaggedBuilds"},{"type":"null"}]},"title":{"type":["string","null"]},"userId":{"type":"string"},"username":{"type":"string"},"versions":{"items":{"$ref":"#/components/schemas/Version"},"type":"array"}},"required":["id","userId","name","username","isPublic","createdAt","modifiedAt","stats","versions","defaultRunOptions"],"type":"object"},"ActorResponse":{"properties":{"data":{"$ref":"#/components/schemas/ActorResource"}},"required":["data"],"type":"object"},"ActorRunPricingInfo":{"oneOf":[{"$ref":"#/components/schemas/PayPerEventActorPricingInfo"},{"$ref":"#/components/schemas/PricePerDatasetItemActorPricingInfo"},{"$ref":"#/components/schemas/FlatPricePerMonthActorPricingInfo"},{"$ref":"#/components/schemas/FreeActorPricingInfo"}]},"ActorStandby":{"properties":{"build":{"type":["string","null"]},"desiredRequestsPerActorRun":{"type":["integer","null"]},"disableStandbyFieldsOverride":{"type":["boolean","null"]},"idleTimeoutSecs":{"type":["integer","null"]},"isEnabled":{"type":["boolean","null"]},"maxRequestsPerActorRun":{"type":["integer","null"]},"memoryMbytes":{"type":["integer","null"]},"shouldPassActorInput":{"type":["boolean","null"]}},"type":"object"},"ActorStats":{"properties":{"actorReviewCount":{"type":"integer"},"actorReviewRating":{"type":"number"},"bookmarkCount":{"type":"integer"},"lastRunStartedAt":{"type":"string"},"publicActorRunStats30Days":{"properties":{"ABORTED":{"type":"integer"},"FAILED":{"type":"integer"},"SUCCEEDED":{"type":"integer"},"TIMED-OUT":{"type":"integer"},"TOTAL":{"type":"integer"}},"type":"object"},"totalBuilds":{"type":"integer"},"totalMetamorphs":{"type":"integer"},"totalRuns":{"type":"integer"},"totalUsers":{"type":"integer"},"totalUsers30Days":{"type":"integer"},"totalUsers7Days":{"type":"integer"},"totalUsers90Days":{"type":"integer"}},"type":"object"},"AddRequestResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestRegistration"}},"required":["data"],"type":"object"},"AddedRequest":{"properties":{"requestId":{"$ref":"#/components/schemas/RequestId"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"wasAlreadyHandled":{"$ref":"#/components/schemas/WasAlreadyHandled"},"wasAlreadyPresent":{"$ref":"#/components/schemas/WasAlreadyPresent"}},"required":["requestId","uniqueKey","wasAlreadyHandled","wasAlreadyPresent"],"type":"object"},"AvailableProxyGroups":{"type":"object"},"BatchAddResponse":{"properties":{"data":{"$ref":"#/components/schemas/BatchAddResult"}},"required":["data"],"type":"object"},"BatchAddResult":{"properties":{"processedRequests":{"items":{"$ref":"#/components/schemas/AddedRequest"},"type":"array"},"unprocessedRequests":{"items":{"$ref":"#/components/schemas/UnprocessedRequest"},"type":"array"}},"required":["processedRequests","unprocessedRequests"],"type":"object"},"BatchDeleteResponse":{"properties":{"data":{"$ref":"#/components/schemas/BatchDeleteResult"}},"required":["data"],"type":"object"},"BatchDeleteResult":{"properties":{"processedRequests":{"items":{"$ref":"#/components/schemas/DeletedRequest"},"type":"array"},"unprocessedRequests":{"items":{"$ref":"#/components/schemas/UnprocessedRequest"},"type":"array"}},"required":["processedRequests","unprocessedRequests"],"type":"object"},"BrowserInfoResponse":{"properties":{"bodyLength":{"type":"integer"},"clientIp":{"type":["string","null"]},"countryCode":{"type":["string","null"]},"headers":{"type":"object"},"method":{"type":"string"},"rawHeaders":{"items":{"type":"string"},"type":"array"}},"required":["method","clientIp","countryCode","bodyLength"],"type":"object"},"Build":{"properties":{"actId":{"type":"string"},"actVersion":{"properties":{"buildTag":{"type":["string","null"]},"gitRepoUrl":{"type":"string"},"sourceFiles":{"items":{"$ref":"#/components/schemas/SourceCodeFile"},"type":"array"},"sourceType":{"$ref":"#/components/schemas/VersionSourceType"},"versionNumber":{"type":"string"}},"type":"object"},"actorDefinition":{"anyOf":[{"$ref":"#/components/schemas/ActorDefinition"},{"type":"null"}]},"buildNumber":{"type":"string"},"finishedAt":{"type":["string","null"]},"id":{"type":"string"},"imageDigest":{"type":["string","null"]},"inputSchema":{"type":["string","null"]},"meta":{"$ref":"#/components/schemas/BuildMeta"},"options":{"anyOf":[{"$ref":"#/components/schemas/BuildOptions"},{"type":"null"}]},"readme":{"type":["string","null"]},"startedAt":{"type":"string"},"stats":{"anyOf":[{"$ref":"#/components/schemas/BuildStats"},{"type":"null"}]},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"usage":{"anyOf":[{"$ref":"#/components/schemas/BuildUsage"},{"type":"null"}]},"usageTotalUsd":{"type":["number","null"]},"usageUsd":{"anyOf":[{"$ref":"#/components/schemas/BuildUsage"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","actId","userId","startedAt","status","meta","buildNumber"],"type":"object"},"BuildListItem":{"properties":{"actId":{"type":"string"},"buildNumber":{"type":"string"},"buildNumberInt":{"type":"integer"},"finishedAt":{"type":["string","null"]},"id":{"type":"string"},"meta":{"$ref":"#/components/schemas/BuildMeta"},"startedAt":{"type":"string"},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"usageTotalUsd":{"type":"number"},"userId":{"type":"string"}},"required":["id","status","startedAt","usageTotalUsd","buildNumber"],"type":"object"},"BuildMeta":{"properties":{"clientIp":{"type":["string"]},"origin":{"$ref":"#/components/schemas/RunOrigin"},"userAgent":{"type":["string"]}},"required":["origin"],"type":"object"},"BuildOptions":{"properties":{"betaPackages":{"type":["boolean","null"]},"diskMbytes":{"type":["integer","null"]},"memoryMbytes":{"type":["integer","null"]},"useCache":{"type":["boolean","null"]}},"type":"object"},"BuildResponse":{"properties":{"data":{"$ref":"#/components/schemas/Build"}},"required":["data"],"type":"object"},"BuildStats":{"properties":{"computeUnits":{"type":"number"},"durationMillis":{"type":"integer"},"imageSizeBytes":{"type":["integer","null"]},"runTimeSecs":{"type":"number"}},"type":"object"},"BuildUsage":{"properties":{"ACTOR_COMPUTE_UNITS":{"type":["number","null"]}},"type":"object"},"CommonActorPricingInfo":{"properties":{"apifyMarginPercentage":{"type":"number"},"createdAt":{"type":"string"},"forceContainsSignificantPriceChange":{"type":"boolean"},"isPriceChangeNotificationSuppressed":{"type":"boolean"},"notifiedAboutChangeAt":{"type":["string","null"]},"notifiedAboutFutureChangeAt":{"type":["string","null"]},"reasonForChange":{"type":["string","null"]},"startedAt":{"type":"string"}},"required":["apifyMarginPercentage","createdAt","startedAt"],"type":"object"},"CurrentPricingInfo":{"properties":{"apifyMarginPercentage":{"type":"number"},"createdAt":{"type":"string"},"forceContainsSignificantPriceChange":{"type":"boolean"},"isPPEPlatformUsagePaidByUser":{"type":"boolean"},"isPriceChangeNotificationSuppressed":{"type":"boolean"},"minimalMaxTotalChargeUsd":{"type":["number","null"]},"notifiedAboutChangeAt":{"type":["string","null"]},"notifiedAboutFutureChangeAt":{"type":["string","null"]},"pricingModel":{"type":"string"},"pricingPerEvent":{"type":["object","null"]},"reasonForChange":{"type":["string","null"]},"startedAt":{"type":"string"}},"required":["pricingModel"],"type":"object"},"CurrentUsage":{"properties":{"activeActorJobCount":{"type":"integer"},"actorCount":{"type":"integer"},"actorMemoryGbytes":{"type":"number"},"actorTaskCount":{"type":"integer"},"monthlyActorComputeUnits":{"type":"number"},"monthlyExternalDataTransferGbytes":{"type":"number"},"monthlyProxySerps":{"type":"integer"},"monthlyResidentialProxyGbytes":{"type":"number"},"monthlyUsageUsd":{"type":"number"},"scheduleCount":{"type":"integer"},"teamAccountSeatCount":{"type":"integer"}},"required":["monthlyUsageUsd","monthlyActorComputeUnits","monthlyExternalDataTransferGbytes","monthlyProxySerps","monthlyResidentialProxyGbytes","actorMemoryGbytes","actorCount","actorTaskCount","activeActorJobCount","teamAccountSeatCount"],"type":"object"},"DailyServiceUsage":{"properties":{"date":{"type":"string"},"serviceUsage":{"$ref":"#/components/schemas/ServiceUsage"},"totalUsageCreditsUsd":{"type":"number"}},"required":["date","serviceUsage","totalUsageCreditsUsd"],"type":"object"},"DatasetListItem":{"properties":{"accessedAt":{"type":"string"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"cleanItemCount":{"type":"integer"},"createdAt":{"type":"string"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"id":{"type":"string"},"itemCount":{"type":"integer"},"modifiedAt":{"type":"string"},"name":{"type":"string"},"stats":{"$ref":"#/components/schemas/DatasetStats"},"title":{"type":["string","null"]},"userId":{"type":"string"},"username":{"type":"string"}},"required":["id","name","userId","createdAt","modifiedAt","accessedAt","itemCount","cleanItemCount"],"type":"object"},"DatasetResource":{"properties":{"accessedAt":{"type":"string"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"cleanItemCount":{"type":"integer"},"consoleUrl":{"type":"string"},"createdAt":{"type":"string"},"fields":{"items":{"type":"string"},"type":["array","null"]},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"id":{"type":"string"},"itemCount":{"type":"integer"},"itemsPublicUrl":{"type":"string"},"modifiedAt":{"type":"string"},"name":{"type":["string","null"]},"schema":{"type":["object","null"]},"stats":{"$ref":"#/components/schemas/DatasetStats"},"urlSigningSecretKey":{"type":["string","null"]},"userId":{"type":"string"}},"required":["id","userId","createdAt","modifiedAt","accessedAt","itemCount","cleanItemCount","consoleUrl"],"type":"object"},"DatasetResponse":{"properties":{"data":{"$ref":"#/components/schemas/DatasetResource"}},"required":["data"],"type":"object"},"DatasetStatistics":{"properties":{"fieldStatistics":{"type":["object","null"]}},"type":"object"},"DatasetStatisticsResponse":{"properties":{"data":{"$ref":"#/components/schemas/DatasetStatistics"}},"required":["data"],"type":"object"},"DatasetStats":{"properties":{"inflatedBytes":{"type":"integer"},"readCount":{"type":"integer"},"storageBytes":{"type":"integer"},"writeCount":{"type":"integer"}},"type":"object"},"DecodeAndVerifyResponse":{"properties":{"data":{"$ref":"#/components/schemas/DecodeAndVerifyResult"}},"required":["data"],"type":"object"},"DecodeAndVerifyResult":{"properties":{"decoded":{},"encodedByUserId":{"type":["string","null"]},"isVerifiedUser":{"type":"boolean"}},"required":["decoded","encodedByUserId","isVerifiedUser"],"type":"object"},"DefaultRunOptions":{"properties":{"build":{"type":"string"},"forcePermissionLevel":{"anyOf":[{"$ref":"#/components/schemas/ActorPermissionLevel"},{"type":"null"}]},"maxItems":{"type":["integer","null"]},"memoryMbytes":{"type":"integer"},"restartOnError":{"type":["boolean","null"]},"timeoutSecs":{"type":"integer"}},"type":"object"},"DeletedRequest":{"anyOf":[{"$ref":"#/components/schemas/DeletedRequestById"},{"$ref":"#/components/schemas/DeletedRequestByUniqueKey"}]},"DeletedRequestById":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"}},"required":["id"],"type":"object"},"DeletedRequestByUniqueKey":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"}},"required":["uniqueKey"],"type":"object"},"EffectivePlatformFeature":{"properties":{"disabledReason":{"type":["string","null"]},"disabledReasonType":{"type":["string","null"]},"isEnabled":{"type":"boolean"},"isTrial":{"type":"boolean"},"trialExpirationAt":{"type":["string","null"]}},"required":["isEnabled","disabledReason","disabledReasonType","isTrial","trialExpirationAt"],"type":"object"},"EffectivePlatformFeatures":{"properties":{"ACTORS":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"ACTORS_PUBLIC_ALL":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"ACTORS_PUBLIC_DEVELOPER":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY_EXTERNAL_ACCESS":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY_RESIDENTIAL":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY_SERPS":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"SCHEDULER":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"STORAGE":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"WEBHOOKS":{"$ref":"#/components/schemas/EffectivePlatformFeature"}},"required":["ACTORS","STORAGE","SCHEDULER","PROXY","PROXY_EXTERNAL_ACCESS","PROXY_RESIDENTIAL","PROXY_SERPS","WEBHOOKS","ACTORS_PUBLIC_ALL","ACTORS_PUBLIC_DEVELOPER"],"type":"object"},"EncodeAndSignResponse":{"properties":{"data":{"$ref":"#/components/schemas/EncodeAndSignResult"}},"required":["data"],"type":"object"},"EncodeAndSignResult":{"properties":{"encoded":{"type":"string"}},"required":["encoded"],"type":"object"},"EnvVar":{"properties":{"isSecret":{"type":["boolean","null"]},"name":{"type":"string"},"value":{"type":"string"}},"required":["name"],"type":"object"},"EnvVarResponse":{"properties":{"data":{"$ref":"#/components/schemas/EnvVar"}},"required":["data"],"type":"object"},"ExampleRunInput":{"properties":{"body":{"type":"string"},"contentType":{"type":"string"}},"type":"object"},"FlatPricePerMonthActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"pricePerUnitUsd":{"type":"number"},"pricingModel":{"type":"string"},"trialMinutes":{"type":"integer"}},"required":["pricingModel","pricePerUnitUsd","trialMinutes"],"type":"object"}]},"FreeActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"pricingModel":{"type":"string"}},"required":["pricingModel"],"type":"object"}]},"GeneralAccess":{"type":"string"},"HadMultipleClients":{"type":"boolean"},"HandledRequestCount":{"type":"integer"},"HeadLimit":{"type":"integer"},"HttpMethod":{"type":"string"},"KeyValueStoreKey":{"properties":{"key":{"type":"string"},"recordPublicUrl":{"type":"string"},"size":{"type":"integer"}},"required":["key","size","recordPublicUrl"],"type":"object"},"KeyValueStoreResource":{"properties":{"accessedAt":{"type":"string"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"consoleUrl":{"type":"string"},"createdAt":{"type":"string"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"id":{"type":"string"},"keysPublicUrl":{"type":"string"},"modifiedAt":{"type":"string"},"name":{"type":["string","null"]},"recordsPublicUrl":{"type":"string"},"schema":{"type":["object","null"]},"stats":{"$ref":"#/components/schemas/KeyValueStoreStats"},"urlSigningSecretKey":{"type":["string","null"]},"userId":{"type":["string","null"]},"username":{"type":["string","null"]}},"required":["id","createdAt","modifiedAt","accessedAt"],"type":"object"},"KeyValueStoreResponse":{"properties":{"data":{"$ref":"#/components/schemas/KeyValueStoreResource"}},"required":["data"],"type":"object"},"KeyValueStoreStats":{"properties":{"deleteCount":{"type":"integer"},"listCount":{"type":"integer"},"readCount":{"type":"integer"},"s3StorageBytes":{"type":"integer"},"storageBytes":{"type":"integer"},"writeCount":{"type":"integer"}},"type":"object"},"Limits":{"properties":{"dataRetentionDays":{"type":"integer"},"maxActorCount":{"type":"integer"},"maxActorMemoryGbytes":{"type":"number"},"maxActorTaskCount":{"type":"integer"},"maxConcurrentActorJobs":{"type":"integer"},"maxMonthlyActorComputeUnits":{"type":"number"},"maxMonthlyExternalDataTransferGbytes":{"type":"number"},"maxMonthlyProxySerps":{"type":"integer"},"maxMonthlyResidentialProxyGbytes":{"type":"number"},"maxMonthlyUsageUsd":{"type":"number"},"maxScheduleCount":{"type":"integer"},"maxTeamAccountSeatCount":{"type":"integer"}},"required":["maxMonthlyUsageUsd","maxMonthlyActorComputeUnits","maxMonthlyExternalDataTransferGbytes","maxMonthlyProxySerps","maxMonthlyResidentialProxyGbytes","maxActorMemoryGbytes","maxActorCount","maxActorTaskCount","maxConcurrentActorJobs","maxTeamAccountSeatCount","dataRetentionDays"],"type":"object"},"ListOfActors":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/ActorListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfActorsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfActors"}},"required":["data"],"type":"object"},"ListOfBuilds":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/BuildListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfBuildsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfBuilds"}},"required":["data"],"type":"object"},"ListOfDatasets":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/DatasetListItem"},"type":"array"},"unnamed":{"type":"boolean"}},"required":["items"],"type":"object"}]},"ListOfDatasetsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfDatasets"}},"required":["data"],"type":"object"},"ListOfEnvVars":{"properties":{"items":{"items":{"$ref":"#/components/schemas/EnvVar"},"type":"array"},"total":{"type":"integer"}},"required":["total","items"],"type":"object"},"ListOfEnvVarsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfEnvVars"}},"required":["data"],"type":"object"},"ListOfKeyValueStores":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/KeyValueStoreResource"},"type":"array"},"unnamed":{"type":"boolean"}},"required":["items"],"type":"object"}]},"ListOfKeyValueStoresResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfKeyValueStores"}},"required":["data"],"type":"object"},"ListOfKeys":{"properties":{"count":{"type":"integer"},"exclusiveStartKey":{"type":["string","null"]},"isTruncated":{"type":"boolean"},"items":{"items":{"$ref":"#/components/schemas/KeyValueStoreKey"},"type":"array"},"limit":{"type":"integer"},"nextExclusiveStartKey":{"type":["string","null"]}},"required":["items","count","limit","isTruncated"],"type":"object"},"ListOfKeysResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfKeys"}},"required":["data"],"type":"object"},"ListOfRequestQueues":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/RequestQueueListItem"},"type":"array"},"unnamed":{"type":"boolean"}},"required":["items"],"type":"object"}]},"ListOfRequestQueuesResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfRequestQueues"}},"required":["data"],"type":"object"},"ListOfRequests":{"properties":{"cursor":{"type":"string"},"exclusiveStartId":{"type":"string"},"items":{"items":{"$ref":"#/components/schemas/RequestResource"},"type":"array"},"limit":{"type":"integer"},"nextCursor":{"type":"string"}},"required":["items","limit"],"type":"object"},"ListOfRequestsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfRequests"}},"required":["data"],"type":"object"},"ListOfRuns":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/RunListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfRunsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfRuns"}},"required":["data"],"type":"object"},"ListOfSchedules":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/ScheduleListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfSchedulesResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfSchedules"}},"required":["data"],"type":"object"},"ListOfStoreActors":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/StoreActor"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfStoreActorsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfStoreActors"}},"required":["data"],"type":"object"},"ListOfTasks":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/TaskListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfTasksResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfTasks"}},"required":["data"],"type":"object"},"ListOfVersions":{"properties":{"items":{"items":{"$ref":"#/components/schemas/Version"},"type":"array"},"total":{"type":"integer"}},"required":["total","items"],"type":"object"},"ListOfVersionsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfVersions"}},"required":["data"],"type":"object"},"ListOfWebhookDispatches":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/WebhookDispatch"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfWebhookDispatchesResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfWebhookDispatches"}},"required":["data"],"type":"object"},"ListOfWebhooks":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/WebhookListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfWebhooksResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfWebhooks"}},"required":["data"],"type":"object"},"LockExpiresAt":{"type":"string"},"LockedRequestQueueHead":{"properties":{"clientKey":{"type":"string"},"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"items":{"items":{"$ref":"#/components/schemas/LockedRequestQueueHeadItem"},"type":"array"},"limit":{"$ref":"#/components/schemas/HeadLimit"},"lockSecs":{"type":"integer"},"queueHasLockedRequests":{"type":"boolean"},"queueModifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"}},"required":["limit","queueModifiedAt","hadMultipleClients","lockSecs","items"],"type":"object"},"LockedRequestQueueHeadItem":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"lockExpiresAt":{"$ref":"#/components/schemas/LockExpiresAt"},"method":{"$ref":"#/components/schemas/RequestMethod"},"retryCount":{"$ref":"#/components/schemas/RetryCount"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"}},"required":["id","uniqueKey","url","lockExpiresAt"],"type":"object"},"LockedRequestQueueHeadResponse":{"properties":{"data":{"$ref":"#/components/schemas/LockedRequestQueueHead"}},"required":["data"],"type":"object"},"MonthlyServiceUsage":{"type":"object"},"MonthlyUsage":{"properties":{"dailyServiceUsages":{"items":{"$ref":"#/components/schemas/DailyServiceUsage"},"type":"array"},"monthlyServiceUsage":{"$ref":"#/components/schemas/MonthlyServiceUsage"},"totalUsageCreditsUsdAfterVolumeDiscount":{"type":"number"},"totalUsageCreditsUsdBeforeVolumeDiscount":{"type":"number"},"usageCycle":{"$ref":"#/components/schemas/UsageCycle"}},"required":["usageCycle","monthlyServiceUsage","dailyServiceUsages","totalUsageCreditsUsdBeforeVolumeDiscount","totalUsageCreditsUsdAfterVolumeDiscount"],"type":"object"},"MonthlyUsageResponse":{"properties":{"data":{"$ref":"#/components/schemas/MonthlyUsage"}},"required":["data"],"type":"object"},"PaginationResponse":{"properties":{"count":{"type":"integer"},"desc":{"type":"boolean"},"limit":{"type":"integer"},"offset":{"type":"integer"},"total":{"type":"integer"}},"required":["total","offset","limit","desc","count"],"type":"object"},"PayPerEventActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"minimalMaxTotalChargeUsd":{"type":["number","null"]},"pricingModel":{"type":"string"},"pricingPerEvent":{"properties":{"actorChargeEvents":{"type":"object"}},"type":"object"}},"required":["pricingModel","pricingPerEvent"],"type":"object"}]},"PendingRequestCount":{"type":"integer"},"PricePerDatasetItemActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"pricePerUnitUsd":{"type":"number"},"pricingModel":{"type":"string"},"tieredPricing":{"$ref":"#/components/schemas/TieredPricingPerDatasetItem"},"unitName":{"type":"string"}},"required":["pricingModel","unitName"],"type":"object"}]},"ProlongRequestLockResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestLockInfo"}},"required":["data"],"type":"object"},"ProxyGroup":{"properties":{"availableCount":{"type":"integer"},"description":{"type":["string","null"]},"name":{"type":"string"}},"required":["name","description","availableCount"],"type":"object"},"ProxyResource":{"properties":{"groups":{"items":{"$ref":"#/components/schemas/ProxyGroup"},"type":"array"},"password":{"type":"string"}},"required":["password","groups"],"type":"object"},"QueueAccessedAt":{"type":"string"},"QueueCreatedAt":{"type":"string"},"QueueId":{"type":"string"},"QueueModifiedAt":{"type":"string"},"QueueUserId":{"type":"string"},"RecordResponse":{"type":"object"},"RequestBase":{"properties":{"errorMessages":{"items":{"type":"string"},"type":["array","null"]},"handledAt":{"type":["string","null"]},"headers":{"type":["object","null"]},"loadedUrl":{"type":["string","null"]},"method":{"$ref":"#/components/schemas/RequestMethod"},"noRetry":{"type":["boolean","null"]},"payload":{"type":["string","object","null"]},"retryCount":{"$ref":"#/components/schemas/RetryCount"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"},"userData":{"$ref":"#/components/schemas/RequestUserData"}},"required":["uniqueKey","url"],"type":"object"},"RequestId":{"type":"string"},"RequestLockInfo":{"properties":{"lockExpiresAt":{"$ref":"#/components/schemas/LockExpiresAt"}},"required":["lockExpiresAt"],"type":"object"},"RequestMethod":{"$ref":"#/components/schemas/HttpMethod"},"RequestQueueHead":{"properties":{"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"items":{"items":{"$ref":"#/components/schemas/RequestQueueHeadItem"},"type":"array"},"limit":{"$ref":"#/components/schemas/HeadLimit"},"queueModifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"}},"required":["limit","queueModifiedAt","hadMultipleClients","items"],"type":"object"},"RequestQueueHeadItem":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"method":{"$ref":"#/components/schemas/RequestMethod"},"retryCount":{"$ref":"#/components/schemas/RetryCount"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"}},"required":["id","uniqueKey","url"],"type":"object"},"RequestQueueHeadResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestQueueHead"}},"required":["data"],"type":"object"},"RequestQueueListItem":{"properties":{"accessedAt":{"$ref":"#/components/schemas/QueueAccessedAt"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"createdAt":{"$ref":"#/components/schemas/QueueCreatedAt"},"expireAt":{"type":"string"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"handledRequestCount":{"$ref":"#/components/schemas/HandledRequestCount"},"id":{"$ref":"#/components/schemas/QueueId"},"modifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"},"name":{"type":"string"},"pendingRequestCount":{"$ref":"#/components/schemas/PendingRequestCount"},"stats":{"$ref":"#/components/schemas/RequestQueueStats"},"totalRequestCount":{"$ref":"#/components/schemas/TotalRequestCount"},"userId":{"$ref":"#/components/schemas/QueueUserId"},"username":{"type":"string"}},"required":["id","name","userId","username","createdAt","modifiedAt","accessedAt","totalRequestCount","handledRequestCount","pendingRequestCount","hadMultipleClients"],"type":"object"},"RequestQueueResource":{"properties":{"accessedAt":{"$ref":"#/components/schemas/QueueAccessedAt"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"consoleUrl":{"type":"string"},"createdAt":{"$ref":"#/components/schemas/QueueCreatedAt"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"handledRequestCount":{"$ref":"#/components/schemas/HandledRequestCount"},"id":{"$ref":"#/components/schemas/QueueId"},"modifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"},"name":{"type":["string","null"]},"pendingRequestCount":{"$ref":"#/components/schemas/PendingRequestCount"},"stats":{"$ref":"#/components/schemas/RequestQueueStats"},"totalRequestCount":{"$ref":"#/components/schemas/TotalRequestCount"},"userId":{"$ref":"#/components/schemas/QueueUserId"}},"required":["id","userId","createdAt","modifiedAt","accessedAt","totalRequestCount","handledRequestCount","pendingRequestCount","hadMultipleClients","consoleUrl"],"type":"object"},"RequestQueueResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestQueueResource"}},"required":["data"],"type":"object"},"RequestQueueStats":{"properties":{"deleteCount":{"type":"integer"},"headItemReadCount":{"type":"integer"},"readCount":{"type":"integer"},"storageBytes":{"type":"integer"},"writeCount":{"type":"integer"}},"type":"object"},"RequestRegistration":{"properties":{"requestId":{"$ref":"#/components/schemas/RequestId"},"wasAlreadyHandled":{"$ref":"#/components/schemas/WasAlreadyHandled"},"wasAlreadyPresent":{"$ref":"#/components/schemas/WasAlreadyPresent"}},"required":["requestId","wasAlreadyPresent","wasAlreadyHandled"],"type":"object"},"RequestResource":{"allOf":[{"$ref":"#/components/schemas/RequestBase","required":["id"]},{"properties":{"id":{"$ref":"#/components/schemas/RequestId"}}}]},"RequestResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestResource"}},"required":["data"],"type":"object"},"RequestUrl":{"type":"string"},"RequestUserData":{"type":"object"},"RetryCount":{"type":"integer"},"Run":{"properties":{"actId":{"type":"string"},"actorTaskId":{"type":["string","null"]},"buildId":{"type":"string"},"buildNumber":{"type":["string","null"]},"chargedEventCounts":{"type":"object"},"containerUrl":{"type":"string"},"defaultDatasetId":{"type":"string"},"defaultKeyValueStoreId":{"type":"string"},"defaultRequestQueueId":{"type":"string"},"exitCode":{"type":["integer","null"]},"finishedAt":{"type":["string","null"]},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"gitBranchName":{"type":["string","null"]},"id":{"type":"string"},"isContainerServerReady":{"type":["boolean","null"]},"isStatusMessageTerminal":{"type":["boolean","null"]},"meta":{"$ref":"#/components/schemas/RunMeta"},"metamorphs":{"anyOf":[{"items":{"$ref":"#/components/schemas/RunMetamorphEvent"},"type":"array"},{"type":"null"}]},"options":{"$ref":"#/components/schemas/RunOptions"},"platformUsageBillingModel":{"type":"string"},"pricingInfo":{"$ref":"#/components/schemas/ActorRunPricingInfo"},"startedAt":{"type":"string"},"stats":{"$ref":"#/components/schemas/RunStats"},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"statusMessage":{"type":["string","null"]},"storageIds":{"properties":{"datasets":{"properties":{"default":{"type":"string"}},"type":"object"},"keyValueStores":{"properties":{"default":{"type":"string"}},"type":"object"},"requestQueues":{"properties":{"default":{"type":"string"}},"type":"object"}},"type":"object"},"usage":{"anyOf":[{"$ref":"#/components/schemas/RunUsage"},{"type":"null"}]},"usageTotalUsd":{"type":["number","null"]},"usageUsd":{"anyOf":[{"$ref":"#/components/schemas/RunUsageUsd"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","actId","userId","startedAt","status","meta","stats","options","buildId","defaultKeyValueStoreId","defaultDatasetId","defaultRequestQueueId"],"type":"object"},"RunListItem":{"properties":{"actId":{"type":"string"},"actorTaskId":{"type":["string","null"]},"buildId":{"type":"string"},"buildNumber":{"type":"string"},"buildNumberInt":{"type":"integer"},"defaultDatasetId":{"type":"string"},"defaultKeyValueStoreId":{"type":"string"},"defaultRequestQueueId":{"type":"string"},"finishedAt":{"type":["string","null"]},"id":{"type":"string"},"meta":{"$ref":"#/components/schemas/RunMeta"},"startedAt":{"type":"string"},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"usageTotalUsd":{"type":"number"},"userId":{"type":"string"}},"required":["id","actId","status","startedAt","buildId","meta","usageTotalUsd","defaultKeyValueStoreId","defaultDatasetId","defaultRequestQueueId"],"type":"object"},"RunMeta":{"properties":{"clientIp":{"type":["string","null"]},"origin":{"$ref":"#/components/schemas/RunOrigin"},"scheduleId":{"type":["string","null"]},"scheduledAt":{"type":["string","null"]},"userAgent":{"type":["string","null"]}},"required":["origin"],"type":"object"},"RunMetamorphEvent":{"properties":{"actorId":{"type":"string"},"buildId":{"type":"string"},"createdAt":{"type":"string"},"inputKey":{"type":["string","null"]}},"required":["createdAt","actorId","buildId"],"type":"object"},"RunOptions":{"properties":{"build":{"type":"string"},"diskMbytes":{"type":"integer"},"maxItems":{"type":["integer","null"]},"maxTotalChargeUsd":{"type":["number","null"]},"memoryMbytes":{"type":"integer"},"timeoutSecs":{"type":"integer"}},"required":["build","timeoutSecs","memoryMbytes","diskMbytes"],"type":"object"},"RunOrigin":{"type":"string"},"RunResponse":{"properties":{"data":{"$ref":"#/components/schemas/Run"}},"required":["data"],"type":"object"},"RunStats":{"properties":{"computeUnits":{"type":"number"},"cpuAvgUsage":{"type":"number"},"cpuCurrentUsage":{"type":"number"},"cpuMaxUsage":{"type":"number"},"durationMillis":{"type":"integer"},"inputBodyLen":{"type":["integer","null"]},"memAvgBytes":{"type":"number"},"memCurrentBytes":{"type":"integer"},"memMaxBytes":{"type":"integer"},"metamorph":{"type":"integer"},"migrationCount":{"type":"integer"},"netRxBytes":{"type":"integer"},"netTxBytes":{"type":"integer"},"rebootCount":{"type":"integer"},"restartCount":{"type":"integer"},"resurrectCount":{"type":"integer"},"runTimeSecs":{"type":"number"}},"type":"object"},"RunUsage":{"properties":{"ACTOR_COMPUTE_UNITS":{"type":["number","null"]},"DATASET_READS":{"type":["integer","null"]},"DATASET_WRITES":{"type":["integer","null"]},"DATA_TRANSFER_EXTERNAL_GBYTES":{"type":["number","null"]},"DATA_TRANSFER_INTERNAL_GBYTES":{"type":["number","null"]},"KEY_VALUE_STORE_LISTS":{"type":["integer","null"]},"KEY_VALUE_STORE_READS":{"type":["integer","null"]},"KEY_VALUE_STORE_WRITES":{"type":["integer","null"]},"PROXY_RESIDENTIAL_TRANSFER_GBYTES":{"type":["number","null"]},"PROXY_SERPS":{"type":["integer","null"]},"REQUEST_QUEUE_READS":{"type":["integer","null"]},"REQUEST_QUEUE_WRITES":{"type":["integer","null"]}},"type":"object"},"RunUsageUsd":{"properties":{"ACTOR_COMPUTE_UNITS":{"type":["number","null"]},"DATASET_READS":{"type":["number","null"]},"DATASET_WRITES":{"type":["number","null"]},"DATA_TRANSFER_EXTERNAL_GBYTES":{"type":["number","null"]},"DATA_TRANSFER_INTERNAL_GBYTES":{"type":["number","null"]},"KEY_VALUE_STORE_LISTS":{"type":["number","null"]},"KEY_VALUE_STORE_READS":{"type":["number","null"]},"KEY_VALUE_STORE_WRITES":{"type":["number","null"]},"PROXY_RESIDENTIAL_TRANSFER_GBYTES":{"type":["number","null"]},"PROXY_SERPS":{"type":["number","null"]},"REQUEST_QUEUE_READS":{"type":["number","null"]},"REQUEST_QUEUE_WRITES":{"type":["number","null"]}},"type":"object"},"Schedule":{"allOf":[{"$ref":"#/components/schemas/ScheduleBase"},{"properties":{"actions":{"items":{"$ref":"#/components/schemas/ScheduleAction"},"type":"array"},"description":{"type":["string","null"]},"notifications":{"$ref":"#/components/schemas/ScheduleNotifications"},"title":{"type":["string","null"]}},"required":["actions"],"type":"object"}]},"ScheduleAction":{"oneOf":[{"$ref":"#/components/schemas/ScheduleActionRunActor"},{"$ref":"#/components/schemas/ScheduleActionRunActorTask"}]},"ScheduleActionRunActor":{"properties":{"actorId":{"type":"string"},"id":{"type":"string"},"runInput":{"anyOf":[{"$ref":"#/components/schemas/ScheduleActionRunInput"},{"type":"null"}]},"runOptions":{"anyOf":[{"$ref":"#/components/schemas/TaskOptions"},{"type":"null"}]},"type":{}},"required":["id","type","actorId"],"type":"object"},"ScheduleActionRunActorTask":{"properties":{"actorTaskId":{"type":"string"},"id":{"type":"string"},"input":{"anyOf":[{"type":"object"},{"type":"null"}]},"type":{}},"required":["id","type","actorTaskId"],"type":"object"},"ScheduleActionRunInput":{"properties":{"body":{"type":["string","null"]},"contentType":{"type":["string","null"]}},"type":"object"},"ScheduleBase":{"properties":{"createdAt":{"type":"string"},"cronExpression":{"type":"string"},"id":{"type":"string"},"isEnabled":{"type":"boolean"},"isExclusive":{"type":"boolean"},"lastRunAt":{"type":["string","null"]},"modifiedAt":{"type":"string"},"name":{"type":"string"},"nextRunAt":{"type":["string","null"]},"timezone":{"type":"string"},"userId":{"type":"string"}},"required":["id","userId","name","cronExpression","timezone","isEnabled","isExclusive","createdAt","modifiedAt"],"type":"object"},"ScheduleListItem":{"allOf":[{"$ref":"#/components/schemas/ScheduleBase"},{"properties":{"actions":{"items":{"$ref":"#/components/schemas/ScheduleListItemAction"},"type":"array"}},"required":["actions"],"type":"object"}]},"ScheduleListItemAction":{"oneOf":[{"$ref":"#/components/schemas/ScheduleListItemActionRunActor"},{"$ref":"#/components/schemas/ScheduleListItemActionRunActorTask"}]},"ScheduleListItemActionRunActor":{"properties":{"actorId":{"type":"string"},"id":{"type":"string"},"type":{}},"required":["id","type","actorId"],"type":"object"},"ScheduleListItemActionRunActorTask":{"properties":{"actorTaskId":{"type":"string"},"id":{"type":"string"},"type":{}},"required":["id","type","actorTaskId"],"type":"object"},"ScheduleLogEntry":{"properties":{"createdAt":{"type":"string"},"level":{"type":"string"},"message":{"type":"string"}},"required":["message","level","createdAt"],"type":"object"},"ScheduleLogResponse":{"properties":{"data":{"items":{"$ref":"#/components/schemas/ScheduleLogEntry"},"type":"array"}},"required":["data"],"type":"object"},"ScheduleNotifications":{"properties":{"email":{"type":"boolean"}},"type":"object"},"ScheduleResponse":{"properties":{"data":{"$ref":"#/components/schemas/Schedule"}},"required":["data"],"type":"object"},"ServiceUsage":{"type":"object"},"SourceCodeFile":{"properties":{"content":{"type":"string"},"format":{"$ref":"#/components/schemas/SourceCodeFileFormat"},"name":{"type":"string"}},"required":["name"],"type":"object"},"SourceCodeFileFormat":{"type":"string"},"SourceCodeFolder":{"properties":{"folder":{"type":"boolean"},"name":{"type":"string"}},"required":["name","folder"],"type":"object"},"StoreActor":{"properties":{"actorReviewCount":{"type":"integer"},"actorReviewRating":{"type":"number"},"badge":{"type":["string","null"]},"bookmarkCount":{"type":"integer"},"categories":{"items":{"type":"string"},"type":"array"},"currentPricingInfo":{"$ref":"#/components/schemas/CurrentPricingInfo"},"description":{"type":["string","null"]},"id":{"type":"string"},"isWhiteListedForAgenticPayments":{"type":["boolean","null"]},"name":{"type":"string"},"notice":{"$ref":"#/components/schemas/ActorNotice"},"pictureUrl":{"type":["string","null"]},"readmeSummary":{"type":"string"},"stats":{"$ref":"#/components/schemas/ActorStats"},"title":{"type":"string"},"url":{"type":["string","null"]},"userFullName":{"type":["string","null"]},"userPictureUrl":{"type":["string","null"]},"username":{"type":"string"}},"required":["id","title","name","username","stats"],"type":"object"},"TaggedBuilds":{"type":"object"},"Task":{"properties":{"actId":{"type":"string"},"actorStandby":{"anyOf":[{"$ref":"#/components/schemas/ActorStandby"},{"type":"null"}]},"createdAt":{"type":"string"},"description":{"type":["string","null"]},"id":{"type":"string"},"input":{"anyOf":[{"$ref":"#/components/schemas/TaskInput"},{"items":{"$ref":"#/components/schemas/TaskInput"},"type":"array"},{"type":"null"}]},"isPublic":{"type":"boolean"},"modifiedAt":{"type":"string"},"name":{"type":"string"},"options":{"anyOf":[{"$ref":"#/components/schemas/TaskOptions"},{"type":"null"}]},"publicConfig":{"anyOf":[{"$ref":"#/components/schemas/TaskPublicConfig"},{"type":"null"}]},"removedAt":{"type":["string","null"]},"standbyUrl":{"type":["string","null"]},"stats":{"anyOf":[{"$ref":"#/components/schemas/TaskStats"},{"type":"null"}]},"title":{"type":["string","null"]},"userId":{"type":"string"},"username":{"type":["string","null"]}},"required":["id","userId","actId","name","createdAt","modifiedAt"],"type":"object"},"TaskInput":{"type":"object"},"TaskListItem":{"properties":{"actId":{"type":"string"},"actName":{"type":["string","null"]},"actUsername":{"type":["string","null"]},"createdAt":{"type":"string"},"id":{"type":"string"},"modifiedAt":{"type":"string"},"name":{"type":"string"},"stats":{"anyOf":[{"$ref":"#/components/schemas/TaskStats"},{"type":"null"}]},"userId":{"type":"string"},"username":{"type":["string","null"]}},"required":["id","userId","actId","name","createdAt","modifiedAt"],"type":"object"},"TaskOptions":{"properties":{"build":{"type":["string","null"]},"maxItems":{"type":["integer","null"]},"maxTotalChargeUsd":{"type":["number","null"]},"memoryMbytes":{"type":["integer","null"]},"restartOnError":{"type":["boolean","null"]},"timeoutSecs":{"type":["integer","null"]}},"type":"object"},"TaskPublicConfig":{"properties":{"datasetName":{"type":["string","null"]},"datasetView":{"type":["string","null"]},"inputSchemaFields":{"items":{"type":"string"},"type":["array","null"]},"publishedAt":{"type":["string","null"]},"seoDescription":{"type":["string","null"]},"seoTitle":{"type":["string","null"]}},"type":"object"},"TaskResponse":{"properties":{"data":{"$ref":"#/components/schemas/Task"}},"required":["data"],"type":"object"},"TaskStats":{"properties":{"totalRuns":{"type":"integer"}},"type":"object"},"TieredPricingPerDatasetItem":{"type":"object"},"TotalRequestCount":{"type":"integer"},"UniqueKey":{"type":"string"},"UnlockRequestsResponse":{"properties":{"data":{"$ref":"#/components/schemas/UnlockRequestsResult"}},"required":["data"],"type":"object"},"UnlockRequestsResult":{"properties":{"unlockedCount":{"type":"integer"}},"required":["unlockedCount"],"type":"object"},"UnprocessedRequest":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"method":{"$ref":"#/components/schemas/RequestMethod"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"}},"required":["uniqueKey","url"],"type":"object"},"UpdateRequestResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestRegistration"}},"required":["data"],"type":"object"},"UsageCycle":{"properties":{"endAt":{"type":"string"},"startAt":{"type":"string"}},"required":["startAt","endAt"],"type":"object"},"UserPlan":{"properties":{"apiRateLimitBoosts":{"type":"integer"},"availableAddOns":{"items":{"type":"string"},"type":"array"},"availableProxyGroups":{"$ref":"#/components/schemas/AvailableProxyGroups"},"dataRetentionDays":{"type":"integer"},"description":{"type":"string"},"enabledPlatformFeatures":{"items":{"type":"string"},"type":"array"},"id":{"type":"string"},"isEnabled":{"type":"boolean"},"maxActorCount":{"type":"integer"},"maxActorMemoryGbytes":{"type":"number"},"maxActorTaskCount":{"type":"integer"},"maxConcurrentActorRuns":{"type":"integer"},"maxMonthlyActorComputeUnits":{"type":"number"},"maxMonthlyExternalDataTransferGbytes":{"type":"number"},"maxMonthlyProxySerps":{"type":"integer"},"maxMonthlyResidentialProxyGbytes":{"type":"number"},"maxMonthlyUsageUsd":{"type":"number"},"maxScheduleCount":{"type":"integer"},"monthlyBasePriceUsd":{"type":"number"},"monthlyUsageCreditsUsd":{"type":"number"},"planPricing":{"type":"object"},"supportLevel":{"type":"string"},"teamAccountSeatCount":{"type":"integer"},"tier":{"type":"string"},"usageDiscountPercent":{"type":"number"}},"required":["availableProxyGroups"],"type":"object"},"UserPrivateInfo":{"properties":{"createdAt":{"type":"string"},"effectivePlatformFeatures":{"$ref":"#/components/schemas/EffectivePlatformFeatures"},"email":{"type":"string"},"id":{"type":"string"},"isPaying":{"type":"boolean"},"plan":{"$ref":"#/components/schemas/UserPlan"},"profile":{"$ref":"#/components/schemas/UserProfile"},"proxy":{"$ref":"#/components/schemas/ProxyResource"},"username":{"type":"string"}},"required":["username"],"type":"object"},"UserPrivateInfoResponse":{"properties":{"data":{"$ref":"#/components/schemas/UserPrivateInfo"}},"required":["data"],"type":"object"},"UserProfile":{"properties":{"bio":{"type":["string","null"]},"githubUsername":{"type":["string","null"]},"name":{"type":"string"},"pictureUrl":{"type":["string","null"]},"readme":{"type":["string","null"]},"twitterUsername":{"type":["string","null"]},"websiteUrl":{"type":["string","null"]}},"type":"object"},"UserPublicInfo":{"properties":{"profile":{"$ref":"#/components/schemas/UserProfile"},"username":{"type":"string"}},"required":["username"],"type":"object"},"UserPublicInfoResponse":{"properties":{"data":{"$ref":"#/components/schemas/UserPublicInfo"}},"required":["data"],"type":"object"},"Version":{"properties":{"applyEnvVarsToBuild":{"type":["boolean","null"]},"buildTag":{"type":["string","null"]},"envVars":{"anyOf":[{"items":{"$ref":"#/components/schemas/EnvVar"},"type":"array"},{"type":"null"}]},"gitHubGistUrl":{"type":["string","null"]},"gitRepoUrl":{"type":["string","null"]},"sourceFiles":{"$ref":"#/components/schemas/VersionSourceFiles"},"sourceType":{"anyOf":[{"$ref":"#/components/schemas/VersionSourceType"},{"type":"null"}]},"tarballUrl":{"type":["string","null"]},"versionNumber":{"type":"string"}},"required":["versionNumber","sourceType"],"type":"object"},"VersionResponse":{"properties":{"data":{"$ref":"#/components/schemas/Version"}},"required":["data"],"type":"object"},"VersionSourceFiles":{"items":{"anyOf":[{"$ref":"#/components/schemas/SourceCodeFile"},{"$ref":"#/components/schemas/SourceCodeFolder"}]},"type":"array"},"VersionSourceType":{"type":"string"},"WasAlreadyHandled":{"type":"boolean"},"WasAlreadyPresent":{"type":"boolean"},"WebhookCondition":{"properties":{"actorId":{"type":["string","null"]},"actorRunId":{"type":["string","null"]},"actorTaskId":{"type":["string","null"]}},"type":"object"},"WebhookDispatch":{"properties":{"calls":{"items":{"properties":{"errorMessage":{"type":["string","null"]},"finishedAt":{"type":["string","null"]},"responseBody":{"type":["string","null"]},"responseStatus":{"type":["integer","null"]},"startedAt":{"type":["string","null"]}},"type":"object"},"type":"array"},"createdAt":{"type":"string"},"eventData":{"properties":{"actorBuildId":{"type":"string"},"actorId":{"type":"string"},"actorRunId":{"type":"string"},"actorTaskId":{"type":["string","null"]}},"required":["actorId"],"type":["object","null"]},"eventType":{"$ref":"#/components/schemas/WebhookEventType"},"id":{"type":"string"},"status":{"$ref":"#/components/schemas/WebhookDispatchStatus"},"userId":{"type":"string"},"webhook":{"anyOf":[{"$ref":"#/components/schemas/WebhookDispatchWebhookSummary"},{"type":"null"}]},"webhookId":{"type":"string"}},"required":["id","userId","webhookId","createdAt","status","eventType"],"type":"object"},"WebhookDispatchResponse":{"properties":{"data":{"$ref":"#/components/schemas/WebhookDispatch"}},"required":["data"],"type":"object"},"WebhookDispatchStatus":{"type":"string"},"WebhookDispatchWebhookSummary":{"properties":{"actionType":{"type":"string"},"condition":{"$ref":"#/components/schemas/WebhookCondition"},"isAdHoc":{"type":"boolean"},"requestUrl":{"type":["string","null"]}},"type":"object"},"WebhookEventType":{"type":"string"},"WebhookLastDispatch":{"properties":{"finishedAt":{"type":["string","null"]},"removedAt":{"type":["string","null"]},"status":{"$ref":"#/components/schemas/WebhookDispatchStatus"}},"required":["status"],"type":"object"},"WebhookListItem":{"properties":{"actionType":{"type":"string"},"condition":{"$ref":"#/components/schemas/WebhookCondition"},"createdAt":{"type":"string"},"doNotRetry":{"type":"boolean"},"eventTypes":{"items":{"$ref":"#/components/schemas/WebhookEventType"},"type":"array"},"id":{"type":"string"},"ignoreSslErrors":{"type":"boolean"},"isAdHoc":{"type":["boolean","null"]},"isApifyIntegration":{"type":"boolean"},"isEnabled":{"type":"boolean"},"lastDispatch":{"anyOf":[{"$ref":"#/components/schemas/WebhookLastDispatch"},{"type":"null"}]},"modifiedAt":{"type":"string"},"requestUrl":{"type":"string"},"shouldInterpolateStrings":{"type":["boolean","null"]},"stats":{"anyOf":[{"$ref":"#/components/schemas/WebhookStats"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","createdAt","modifiedAt","userId","eventTypes","condition","ignoreSslErrors","doNotRetry","requestUrl"],"type":"object"},"WebhookResource":{"properties":{"condition":{"$ref":"#/components/schemas/WebhookCondition"},"createdAt":{"type":"string"},"description":{"type":["string","null"]},"doNotRetry":{"type":["boolean","null"]},"eventTypes":{"items":{"$ref":"#/components/schemas/WebhookEventType"},"type":"array"},"headersTemplate":{"type":["string","null"]},"id":{"type":"string"},"ignoreSslErrors":{"type":"boolean"},"isAdHoc":{"type":["boolean","null"]},"lastDispatch":{"anyOf":[{"$ref":"#/components/schemas/WebhookLastDispatch"},{"type":"null"}]},"modifiedAt":{"type":"string"},"payloadTemplate":{"type":["string","null"]},"requestUrl":{"type":["string","null"]},"shouldInterpolateStrings":{"type":["boolean","null"]},"stats":{"anyOf":[{"$ref":"#/components/schemas/WebhookStats"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","createdAt","modifiedAt","userId","eventTypes","condition","ignoreSslErrors"],"type":"object"},"WebhookResponse":{"properties":{"data":{"$ref":"#/components/schemas/WebhookResource"}},"required":["data"],"type":"object"},"WebhookStats":{"properties":{"totalDispatches":{"type":"integer"}},"type":"object"}},"specVersion":"v2-2026-10-20T121847Z"} +{"operations":{"DELETE /v2/actor-runs/{runId}/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/actor-tasks/{actorTaskId}":{"type":"object"},"DELETE /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/actors/{actorId}":{"type":"object"},"DELETE /v2/actors/{actorId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/actors/{actorId}/versions/{versionNumber}":{"type":"object"},"DELETE /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"type":"object"},"DELETE /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"DELETE /v2/request-queues/{queueId}/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/schedules/{scheduleId}":{"type":"object"},"DELETE /v2/webhooks/{webhookId}":{"type":"object"},"GET /v2/actor-builds":{"$ref":"#/components/schemas/ListOfBuildsResponse"},"GET /v2/actor-builds/{buildId}":{"allOf":[{"$ref":"#/components/schemas/BuildResponse"}]},"GET /v2/actor-builds/{buildId}/openapi.json":{"type":"object"},"GET /v2/actor-runs":{"allOf":[{"$ref":"#/components/schemas/ListOfRunsResponse"}]},"GET /v2/actor-runs/{runId}":{"allOf":[{"$ref":"#/components/schemas/RunResponse"}]},"GET /v2/actor-runs/{runId}/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/actor-runs/{runId}/dataset/items":{"items":{"type":"object"},"type":"array"},"GET /v2/actor-runs/{runId}/dataset/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/actor-runs/{runId}/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/actor-runs/{runId}/key-value-store/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/actor-runs/{runId}/key-value-store/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/actor-runs/{runId}/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/actor-runs/{runId}/request-queue/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/actor-runs/{runId}/request-queue/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/actor-runs/{runId}/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/actor-tasks":{"$ref":"#/components/schemas/ListOfTasksResponse"},"GET /v2/actor-tasks/{actorTaskId}":{"properties":{"data":{"$ref":"#/components/schemas/Task"}},"required":["data"],"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/input":{"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/run-sync":{"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/run-sync-get-dataset-items":{"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/runs":{"properties":{"data":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/RunListItem"},"type":"array"}},"required":["items"],"type":"object"}]}},"required":["data"],"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/runs/last":{"properties":{"data":{"$ref":"#/components/schemas/Run"}},"required":["data"],"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/dataset/items":{"items":{"type":"object"},"type":"array"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/dataset/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/actor-tasks/{actorTaskId}/webhooks":{"properties":{"data":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/WebhookResource"},"type":"array"}},"required":["items"],"type":"object"}]}},"required":["data"],"type":"object"},"GET /v2/actors":{"$ref":"#/components/schemas/ListOfActorsResponse"},"GET /v2/actors/{actorId}":{"$ref":"#/components/schemas/ActorResponse"},"GET /v2/actors/{actorId}/builds":{"$ref":"#/components/schemas/ListOfBuildsResponse"},"GET /v2/actors/{actorId}/builds/default":{"$ref":"#/components/schemas/BuildResponse"},"GET /v2/actors/{actorId}/builds/{buildId}":{"$ref":"#/components/schemas/BuildResponse"},"GET /v2/actors/{actorId}/builds/{buildId}/openapi.json":{"type":"object"},"GET /v2/actors/{actorId}/run-sync":{"type":"object"},"GET /v2/actors/{actorId}/run-sync-get-dataset-items":{"items":{"type":"object"},"type":"array"},"GET /v2/actors/{actorId}/runs":{"$ref":"#/components/schemas/ListOfRunsResponse"},"GET /v2/actors/{actorId}/runs/last":{"$ref":"#/components/schemas/RunResponse"},"GET /v2/actors/{actorId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/actors/{actorId}/runs/last/dataset/items":{"items":{"type":"object"},"type":"array"},"GET /v2/actors/{actorId}/runs/last/dataset/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/actors/{actorId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/actors/{actorId}/runs/last/key-value-store/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/actors/{actorId}/runs/last/key-value-store/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/actors/{actorId}/runs/{runId}":{"$ref":"#/components/schemas/RunResponse"},"GET /v2/actors/{actorId}/versions":{"$ref":"#/components/schemas/ListOfVersionsResponse"},"GET /v2/actors/{actorId}/versions/{versionNumber}":{"$ref":"#/components/schemas/VersionResponse"},"GET /v2/actors/{actorId}/versions/{versionNumber}/env-vars":{"$ref":"#/components/schemas/ListOfEnvVarsResponse"},"GET /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"$ref":"#/components/schemas/EnvVarResponse"},"GET /v2/actors/{actorId}/webhooks":{"$ref":"#/components/schemas/ListOfWebhooksResponse"},"GET /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"GET /v2/datasets":{"$ref":"#/components/schemas/ListOfDatasetsResponse"},"GET /v2/datasets/{datasetId}":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/datasets/{datasetId}/items":{"items":{"type":"object"},"type":"array"},"GET /v2/datasets/{datasetId}/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/key-value-stores":{"$ref":"#/components/schemas/ListOfKeyValueStoresResponse"},"GET /v2/key-value-stores/{storeId}":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/key-value-stores/{storeId}/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/key-value-stores/{storeId}/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/request-queues":{"$ref":"#/components/schemas/ListOfRequestQueuesResponse"},"GET /v2/request-queues/{queueId}":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/request-queues/{queueId}/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/request-queues/{queueId}/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/request-queues/{queueId}/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/schedules":{"$ref":"#/components/schemas/ListOfSchedulesResponse"},"GET /v2/schedules/{scheduleId}":{"$ref":"#/components/schemas/ScheduleResponse"},"GET /v2/schedules/{scheduleId}/log":{"$ref":"#/components/schemas/ScheduleLogResponse"},"GET /v2/store":{"$ref":"#/components/schemas/ListOfStoreActorsResponse"},"GET /v2/users/me":{"$ref":"#/components/schemas/UserPrivateInfoResponse"},"GET /v2/users/me/limits":{"$ref":"#/components/schemas/AccountLimitsResponse"},"GET /v2/users/me/usage/monthly":{"$ref":"#/components/schemas/MonthlyUsageResponse"},"GET /v2/users/{userId}":{"$ref":"#/components/schemas/UserPublicInfoResponse"},"GET /v2/webhook-dispatches":{"$ref":"#/components/schemas/ListOfWebhookDispatchesResponse"},"GET /v2/webhook-dispatches/{dispatchId}":{"$ref":"#/components/schemas/WebhookDispatchResponse"},"GET /v2/webhooks":{"$ref":"#/components/schemas/ListOfWebhooksResponse"},"GET /v2/webhooks/{webhookId}":{"$ref":"#/components/schemas/WebhookResponse"},"GET /v2/webhooks/{webhookId}/dispatches":{"$ref":"#/components/schemas/ListOfWebhookDispatchesResponse"},"POST /v2/actor-builds/{buildId}/abort":{"allOf":[{"$ref":"#/components/schemas/BuildResponse"}]},"POST /v2/actor-runs/{runId}/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-runs/{runId}/charge":{"type":"object"},"POST /v2/actor-runs/{runId}/dataset/items":{"type":"object"},"POST /v2/actor-runs/{runId}/key-value-store/records/{recordKey}":{"type":"object"},"POST /v2/actor-runs/{runId}/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-runs/{runId}/reboot":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-runs/{runId}/request-queue/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/actor-runs/{runId}/request-queue/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/actor-runs/{runId}/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/actor-runs/{runId}/request-queue/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/actor-runs/{runId}/resurrect":{"allOf":[{"$ref":"#/components/schemas/RunResponse"}]},"POST /v2/actor-tasks":{"$ref":"#/components/schemas/TaskResponse"},"POST /v2/actor-tasks/{actorTaskId}/run-sync":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/run-sync-get-dataset-items":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs":{"properties":{"data":{"$ref":"#/components/schemas/Run"}},"required":["data"],"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/dataset/items":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/reboot":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/actors":{"$ref":"#/components/schemas/ActorResponse"},"POST /v2/actors/{actorId}/builds":{"$ref":"#/components/schemas/BuildResponse"},"POST /v2/actors/{actorId}/builds/{buildId}/abort":{"$ref":"#/components/schemas/BuildResponse"},"POST /v2/actors/{actorId}/run-sync":{"type":"object"},"POST /v2/actors/{actorId}/run-sync-get-dataset-items":{"items":{"type":"object"},"type":"array"},"POST /v2/actors/{actorId}/runs":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/dataset/items":{"type":"object"},"POST /v2/actors/{actorId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"POST /v2/actors/{actorId}/runs/last/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/reboot":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/actors/{actorId}/runs/{runId}/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/{runId}/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/{runId}/resurrect":{"allOf":[{"$ref":"#/components/schemas/RunResponse"}]},"POST /v2/actors/{actorId}/validate-input":{"properties":{"valid":{"type":"boolean"}},"required":["valid"],"type":"object"},"POST /v2/actors/{actorId}/versions":{"$ref":"#/components/schemas/VersionResponse"},"POST /v2/actors/{actorId}/versions/{versionNumber}":{"$ref":"#/components/schemas/VersionResponse"},"POST /v2/actors/{actorId}/versions/{versionNumber}/env-vars":{"$ref":"#/components/schemas/EnvVarResponse"},"POST /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"$ref":"#/components/schemas/EnvVarResponse"},"POST /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"POST /v2/datasets":{"$ref":"#/components/schemas/DatasetResponse"},"POST /v2/datasets/{datasetId}/items":{"type":"object"},"POST /v2/key-value-stores":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"POST /v2/key-value-stores/{storeId}/records/{recordKey}":{"type":"object"},"POST /v2/request-queues":{"$ref":"#/components/schemas/RequestQueueResponse"},"POST /v2/request-queues/{queueId}/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/request-queues/{queueId}/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/request-queues/{queueId}/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/request-queues/{queueId}/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/schedules":{"$ref":"#/components/schemas/ScheduleResponse"},"POST /v2/tools/decode-and-verify":{"$ref":"#/components/schemas/DecodeAndVerifyResponse"},"POST /v2/tools/encode-and-sign":{"$ref":"#/components/schemas/EncodeAndSignResponse"},"POST /v2/webhooks":{"$ref":"#/components/schemas/WebhookResponse"},"POST /v2/webhooks/{webhookId}/test":{"$ref":"#/components/schemas/WebhookDispatchResponse"},"PUT /v2/actor-runs/{runId}":{"$ref":"#/components/schemas/RunResponse"},"PUT /v2/actor-runs/{runId}/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/actor-runs/{runId}/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/actor-runs/{runId}/key-value-store/records/{recordKey}":{"type":"object"},"PUT /v2/actor-runs/{runId}/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/actor-runs/{runId}/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/actor-runs/{runId}/request-queue/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/actor-tasks/{actorTaskId}":{"properties":{"data":{"$ref":"#/components/schemas/Task"}},"required":["data"],"type":"object"},"PUT /v2/actor-tasks/{actorTaskId}/input":{"type":"object"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/actors/{actorId}":{"$ref":"#/components/schemas/ActorResponse"},"PUT /v2/actors/{actorId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/actors/{actorId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/actors/{actorId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"PUT /v2/actors/{actorId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/actors/{actorId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/actors/{actorId}/runs/last/request-queue/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/actors/{actorId}/versions/{versionNumber}":{"$ref":"#/components/schemas/VersionResponse"},"PUT /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"$ref":"#/components/schemas/EnvVarResponse"},"PUT /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"PUT /v2/datasets/{datasetId}":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/key-value-stores/{storeId}":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/key-value-stores/{storeId}/records/{recordKey}":{"type":"object"},"PUT /v2/request-queues/{queueId}":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/request-queues/{queueId}/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/request-queues/{queueId}/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/schedules/{scheduleId}":{"$ref":"#/components/schemas/ScheduleResponse"},"PUT /v2/users/me/limits":{"type":"object"},"PUT /v2/webhooks/{webhookId}":{"$ref":"#/components/schemas/WebhookResponse"}},"schemas":{"AccountLimits":{"properties":{"current":{"$ref":"#/components/schemas/CurrentUsage"},"limits":{"$ref":"#/components/schemas/Limits"},"monthlyUsageCycle":{"$ref":"#/components/schemas/UsageCycle"}},"required":["monthlyUsageCycle","limits","current"],"type":"object"},"AccountLimitsResponse":{"properties":{"data":{"$ref":"#/components/schemas/AccountLimits"}},"required":["data"],"type":"object"},"ActorDefinition":{"properties":{"actorSpecification":{"type":"integer"},"buildTag":{"type":"string"},"changelog":{"type":["string","null"]},"defaultMemoryMbytes":{"oneOf":[{"type":"string"},{"type":"integer"}]},"dockerContextDir":{"type":"string"},"dockerfile":{"type":"string"},"environmentVariables":{"type":"object"},"input":{"type":["object","null"]},"maxMemoryMbytes":{"minimum":128,"type":"integer"},"minMemoryMbytes":{"minimum":128,"type":"integer"},"name":{"type":"string"},"readme":{"type":["string","null"]},"storages":{"properties":{"dataset":{"type":"object"}},"type":"object"},"usesStandbyMode":{"type":"boolean"},"version":{"pattern":"^[0-9]+(\\.[0-9]+)+$","type":"string"}},"type":"object"},"ActorJobStatus":{"type":"string"},"ActorListItem":{"properties":{"createdAt":{"format":"date-time","type":"string"},"id":{"type":"string"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":"string"},"stats":{"anyOf":[{"$ref":"#/components/schemas/ActorStats"},{"type":"null"}]},"title":{"type":"string"},"username":{"type":"string"}},"required":["id","createdAt","modifiedAt","name","username"],"type":"object"},"ActorNotice":{"type":["string","null"]},"ActorPermissionLevel":{"type":"string"},"ActorResource":{"properties":{"actorPermissionLevel":{"$ref":"#/components/schemas/ActorPermissionLevel"},"actorStandby":{"anyOf":[{"$ref":"#/components/schemas/ActorStandby"},{"type":"null"}]},"categories":{"items":{"type":"string"},"type":"array"},"createdAt":{"format":"date-time","type":"string"},"defaultRunOptions":{"$ref":"#/components/schemas/DefaultRunOptions"},"deploymentKey":{"type":"string"},"description":{"type":["string","null"]},"exampleRunInput":{"anyOf":[{"$ref":"#/components/schemas/ExampleRunInput"},{"type":"null"}]},"hasNoDataset":{"type":"boolean"},"id":{"type":"string"},"isCritical":{"type":"boolean"},"isDeprecated":{"type":["boolean","null"]},"isGeneric":{"type":"boolean"},"isPublic":{"type":"boolean"},"isSourceCodeHidden":{"type":"boolean"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":"string"},"notice":{"$ref":"#/components/schemas/ActorNotice"},"pictureUrl":{"type":["string","null"]},"pricingInfos":{"items":{"$ref":"#/components/schemas/ActorRunPricingInfo"},"type":"array"},"readmeSummary":{"type":"string"},"restartOnError":{"type":["boolean","null"]},"seoDescription":{"type":["string","null"]},"seoTitle":{"type":["string","null"]},"standbyUrl":{"type":["string","null"]},"stats":{"$ref":"#/components/schemas/ActorStats"},"taggedBuilds":{"anyOf":[{"$ref":"#/components/schemas/TaggedBuilds"},{"type":"null"}]},"title":{"type":["string","null"]},"userId":{"type":"string"},"username":{"type":"string"},"versions":{"items":{"$ref":"#/components/schemas/Version"},"type":"array"}},"required":["id","userId","name","username","isPublic","createdAt","modifiedAt","stats","versions","defaultRunOptions"],"type":"object"},"ActorResponse":{"properties":{"data":{"$ref":"#/components/schemas/ActorResource"}},"required":["data"],"type":"object"},"ActorRunPricingInfo":{"oneOf":[{"$ref":"#/components/schemas/PayPerEventActorPricingInfo"},{"$ref":"#/components/schemas/PricePerDatasetItemActorPricingInfo"},{"$ref":"#/components/schemas/FlatPricePerMonthActorPricingInfo"},{"$ref":"#/components/schemas/FreeActorPricingInfo"}]},"ActorStandby":{"properties":{"build":{"type":["string","null"]},"desiredRequestsPerActorRun":{"type":["integer","null"]},"disableStandbyFieldsOverride":{"type":["boolean","null"]},"idleTimeoutSecs":{"type":["integer","null"]},"isEnabled":{"type":["boolean","null"]},"maxRequestsPerActorRun":{"type":["integer","null"]},"memoryMbytes":{"type":["integer","null"]},"shouldPassActorInput":{"type":["boolean","null"]}},"type":"object"},"ActorStats":{"properties":{"actorReviewCount":{"type":"integer"},"actorReviewRating":{"type":"number"},"bookmarkCount":{"type":"integer"},"lastRunStartedAt":{"format":"date-time","type":"string"},"publicActorRunStats30Days":{"properties":{"ABORTED":{"type":"integer"},"FAILED":{"type":"integer"},"SUCCEEDED":{"type":"integer"},"TIMED-OUT":{"type":"integer"},"TOTAL":{"type":"integer"}},"type":"object"},"totalBuilds":{"type":"integer"},"totalMetamorphs":{"type":"integer"},"totalRuns":{"type":"integer"},"totalUsers":{"type":"integer"},"totalUsers30Days":{"type":"integer"},"totalUsers7Days":{"type":"integer"},"totalUsers90Days":{"type":"integer"}},"type":"object"},"AddRequestResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestRegistration"}},"required":["data"],"type":"object"},"AddedRequest":{"properties":{"requestId":{"$ref":"#/components/schemas/RequestId"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"wasAlreadyHandled":{"$ref":"#/components/schemas/WasAlreadyHandled"},"wasAlreadyPresent":{"$ref":"#/components/schemas/WasAlreadyPresent"}},"required":["requestId","uniqueKey","wasAlreadyHandled","wasAlreadyPresent"],"type":"object"},"AvailableProxyGroups":{"type":"object"},"BatchAddResponse":{"properties":{"data":{"$ref":"#/components/schemas/BatchAddResult"}},"required":["data"],"type":"object"},"BatchAddResult":{"properties":{"processedRequests":{"items":{"$ref":"#/components/schemas/AddedRequest"},"type":"array"},"unprocessedRequests":{"items":{"$ref":"#/components/schemas/UnprocessedRequest"},"type":"array"}},"required":["processedRequests","unprocessedRequests"],"type":"object"},"BatchDeleteResponse":{"properties":{"data":{"$ref":"#/components/schemas/BatchDeleteResult"}},"required":["data"],"type":"object"},"BatchDeleteResult":{"properties":{"processedRequests":{"items":{"$ref":"#/components/schemas/DeletedRequest"},"type":"array"},"unprocessedRequests":{"items":{"$ref":"#/components/schemas/UnprocessedRequest"},"type":"array"}},"required":["processedRequests","unprocessedRequests"],"type":"object"},"BrowserInfoResponse":{"properties":{"bodyLength":{"type":"integer"},"clientIp":{"type":["string","null"]},"countryCode":{"type":["string","null"]},"headers":{"type":"object"},"method":{"type":"string"},"rawHeaders":{"items":{"type":"string"},"type":"array"}},"required":["method","clientIp","countryCode","bodyLength"],"type":"object"},"Build":{"properties":{"actId":{"type":"string"},"actVersion":{"properties":{"buildTag":{"type":["string","null"]},"gitRepoUrl":{"type":"string"},"sourceFiles":{"items":{"$ref":"#/components/schemas/SourceCodeFile"},"type":"array"},"sourceType":{"$ref":"#/components/schemas/VersionSourceType"},"versionNumber":{"pattern":"^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])$","type":"string"}},"type":"object"},"actorDefinition":{"anyOf":[{"$ref":"#/components/schemas/ActorDefinition"},{"type":"null"}]},"buildNumber":{"pattern":"^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])(\\.[1-9][0-9]{0,4})$","type":"string"},"finishedAt":{"format":"date-time","type":["string","null"]},"id":{"type":"string"},"imageDigest":{"type":["string","null"]},"inputSchema":{"type":["string","null"]},"meta":{"$ref":"#/components/schemas/BuildMeta"},"options":{"anyOf":[{"$ref":"#/components/schemas/BuildOptions"},{"type":"null"}]},"readme":{"type":["string","null"]},"startedAt":{"format":"date-time","type":"string"},"stats":{"anyOf":[{"$ref":"#/components/schemas/BuildStats"},{"type":"null"}]},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"usage":{"anyOf":[{"$ref":"#/components/schemas/BuildUsage"},{"type":"null"}]},"usageTotalUsd":{"type":["number","null"]},"usageUsd":{"anyOf":[{"$ref":"#/components/schemas/BuildUsage"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","actId","userId","startedAt","status","meta","buildNumber"],"type":"object"},"BuildListItem":{"properties":{"actId":{"type":"string"},"buildNumber":{"pattern":"^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])(\\.[1-9][0-9]{0,4})$","type":"string"},"buildNumberInt":{"type":"integer"},"finishedAt":{"format":"date-time","type":["string","null"]},"id":{"type":"string"},"meta":{"$ref":"#/components/schemas/BuildMeta"},"startedAt":{"format":"date-time","type":"string"},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"usageTotalUsd":{"type":"number"},"userId":{"type":"string"}},"required":["id","status","startedAt","usageTotalUsd","buildNumber"],"type":"object"},"BuildMeta":{"properties":{"clientIp":{"type":["string"]},"origin":{"$ref":"#/components/schemas/RunOrigin"},"userAgent":{"type":["string"]}},"required":["origin"],"type":"object"},"BuildOptions":{"properties":{"betaPackages":{"type":["boolean","null"]},"diskMbytes":{"type":["integer","null"]},"memoryMbytes":{"type":["integer","null"]},"useCache":{"type":["boolean","null"]}},"type":"object"},"BuildResponse":{"properties":{"data":{"$ref":"#/components/schemas/Build"}},"required":["data"],"type":"object"},"BuildStats":{"properties":{"computeUnits":{"type":"number"},"durationMillis":{"type":"integer"},"imageSizeBytes":{"type":["integer","null"]},"runTimeSecs":{"type":"number"}},"type":"object"},"BuildUsage":{"properties":{"ACTOR_COMPUTE_UNITS":{"type":["number","null"]}},"type":"object"},"CommonActorPricingInfo":{"properties":{"apifyMarginPercentage":{"type":"number"},"createdAt":{"format":"date-time","type":"string"},"forceContainsSignificantPriceChange":{"type":"boolean"},"isPriceChangeNotificationSuppressed":{"type":"boolean"},"notifiedAboutChangeAt":{"format":"date-time","type":["string","null"]},"notifiedAboutFutureChangeAt":{"format":"date-time","type":["string","null"]},"reasonForChange":{"type":["string","null"]},"startedAt":{"format":"date-time","type":"string"}},"required":["apifyMarginPercentage","createdAt","startedAt"],"type":"object"},"CurrentPricingInfo":{"properties":{"apifyMarginPercentage":{"type":"number"},"createdAt":{"format":"date-time","type":"string"},"forceContainsSignificantPriceChange":{"type":"boolean"},"isPPEPlatformUsagePaidByUser":{"type":"boolean"},"isPriceChangeNotificationSuppressed":{"type":"boolean"},"minimalMaxTotalChargeUsd":{"type":["number","null"]},"notifiedAboutChangeAt":{"format":"date-time","type":["string","null"]},"notifiedAboutFutureChangeAt":{"format":"date-time","type":["string","null"]},"pricingModel":{"type":"string"},"pricingPerEvent":{"type":["object","null"]},"reasonForChange":{"type":["string","null"]},"startedAt":{"format":"date-time","type":"string"}},"required":["pricingModel"],"type":"object"},"CurrentUsage":{"properties":{"activeActorJobCount":{"type":"integer"},"actorCount":{"type":"integer"},"actorMemoryGbytes":{"type":"number"},"actorTaskCount":{"type":"integer"},"monthlyActorComputeUnits":{"type":"number"},"monthlyExternalDataTransferGbytes":{"type":"number"},"monthlyProxySerps":{"type":"integer"},"monthlyResidentialProxyGbytes":{"type":"number"},"monthlyUsageUsd":{"type":"number"},"scheduleCount":{"type":"integer"},"teamAccountSeatCount":{"type":"integer"}},"required":["monthlyUsageUsd","monthlyActorComputeUnits","monthlyExternalDataTransferGbytes","monthlyProxySerps","monthlyResidentialProxyGbytes","actorMemoryGbytes","actorCount","actorTaskCount","activeActorJobCount","teamAccountSeatCount"],"type":"object"},"DailyServiceUsage":{"properties":{"date":{"format":"date-time","type":"string"},"serviceUsage":{"$ref":"#/components/schemas/ServiceUsage"},"totalUsageCreditsUsd":{"type":"number"}},"required":["date","serviceUsage","totalUsageCreditsUsd"],"type":"object"},"DatasetListItem":{"properties":{"accessedAt":{"format":"date-time","type":"string"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"cleanItemCount":{"type":"integer"},"createdAt":{"format":"date-time","type":"string"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"id":{"type":"string"},"itemCount":{"type":"integer"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":"string"},"stats":{"$ref":"#/components/schemas/DatasetStats"},"title":{"type":["string","null"]},"userId":{"type":"string"},"username":{"type":"string"}},"required":["id","name","userId","createdAt","modifiedAt","accessedAt","itemCount","cleanItemCount"],"type":"object"},"DatasetResource":{"properties":{"accessedAt":{"format":"date-time","type":"string"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"cleanItemCount":{"minimum":0,"type":"integer"},"consoleUrl":{"format":"uri","type":"string"},"createdAt":{"format":"date-time","type":"string"},"fields":{"items":{"type":"string"},"type":["array","null"]},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"id":{"type":"string"},"itemCount":{"minimum":0,"type":"integer"},"itemsPublicUrl":{"format":"uri","type":"string"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":["string","null"]},"schema":{"type":["object","null"]},"stats":{"$ref":"#/components/schemas/DatasetStats"},"urlSigningSecretKey":{"type":["string","null"]},"userId":{"type":"string"}},"required":["id","userId","createdAt","modifiedAt","accessedAt","itemCount","cleanItemCount","consoleUrl"],"type":"object"},"DatasetResponse":{"properties":{"data":{"$ref":"#/components/schemas/DatasetResource"}},"required":["data"],"type":"object"},"DatasetStatistics":{"properties":{"fieldStatistics":{"type":["object","null"]}},"type":"object"},"DatasetStatisticsResponse":{"properties":{"data":{"$ref":"#/components/schemas/DatasetStatistics"}},"required":["data"],"type":"object"},"DatasetStats":{"properties":{"inflatedBytes":{"type":"integer"},"readCount":{"type":"integer"},"storageBytes":{"type":"integer"},"writeCount":{"type":"integer"}},"type":"object"},"DecodeAndVerifyResponse":{"properties":{"data":{"$ref":"#/components/schemas/DecodeAndVerifyResult"}},"required":["data"],"type":"object"},"DecodeAndVerifyResult":{"properties":{"decoded":{},"encodedByUserId":{"type":["string","null"]},"isVerifiedUser":{"type":"boolean"}},"required":["decoded","encodedByUserId","isVerifiedUser"],"type":"object"},"DefaultRunOptions":{"properties":{"build":{"type":"string"},"forcePermissionLevel":{"anyOf":[{"$ref":"#/components/schemas/ActorPermissionLevel"},{"type":"null"}]},"maxItems":{"type":["integer","null"]},"memoryMbytes":{"type":"integer"},"restartOnError":{"type":["boolean","null"]},"timeoutSecs":{"type":"integer"}},"type":"object"},"DeletedRequest":{"anyOf":[{"$ref":"#/components/schemas/DeletedRequestById"},{"$ref":"#/components/schemas/DeletedRequestByUniqueKey"}]},"DeletedRequestById":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"}},"required":["id"],"type":"object"},"DeletedRequestByUniqueKey":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"}},"required":["uniqueKey"],"type":"object"},"EffectivePlatformFeature":{"properties":{"disabledReason":{"type":["string","null"]},"disabledReasonType":{"type":["string","null"]},"isEnabled":{"type":"boolean"},"isTrial":{"type":"boolean"},"trialExpirationAt":{"format":"date-time","type":["string","null"]}},"required":["isEnabled","disabledReason","disabledReasonType","isTrial","trialExpirationAt"],"type":"object"},"EffectivePlatformFeatures":{"properties":{"ACTORS":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"ACTORS_PUBLIC_ALL":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"ACTORS_PUBLIC_DEVELOPER":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY_EXTERNAL_ACCESS":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY_RESIDENTIAL":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY_SERPS":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"SCHEDULER":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"STORAGE":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"WEBHOOKS":{"$ref":"#/components/schemas/EffectivePlatformFeature"}},"required":["ACTORS","STORAGE","SCHEDULER","PROXY","PROXY_EXTERNAL_ACCESS","PROXY_RESIDENTIAL","PROXY_SERPS","WEBHOOKS","ACTORS_PUBLIC_ALL","ACTORS_PUBLIC_DEVELOPER"],"type":"object"},"EncodeAndSignResponse":{"properties":{"data":{"$ref":"#/components/schemas/EncodeAndSignResult"}},"required":["data"],"type":"object"},"EncodeAndSignResult":{"properties":{"encoded":{"type":"string"}},"required":["encoded"],"type":"object"},"EnvVar":{"properties":{"isSecret":{"type":["boolean","null"]},"name":{"type":"string"},"value":{"type":"string"}},"required":["name"],"type":"object"},"EnvVarResponse":{"properties":{"data":{"$ref":"#/components/schemas/EnvVar"}},"required":["data"],"type":"object"},"ExampleRunInput":{"properties":{"body":{"type":"string"},"contentType":{"type":"string"}},"type":"object"},"FlatPricePerMonthActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"pricePerUnitUsd":{"type":"number"},"pricingModel":{"type":"string"},"trialMinutes":{"type":"integer"}},"required":["pricingModel","pricePerUnitUsd","trialMinutes"],"type":"object"}]},"FreeActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"pricingModel":{"type":"string"}},"required":["pricingModel"],"type":"object"}]},"GeneralAccess":{"type":"string"},"HadMultipleClients":{"type":"boolean"},"HandledRequestCount":{"minimum":0,"type":"integer"},"HeadLimit":{"type":"integer"},"HttpMethod":{"type":"string"},"KeyValueStoreKey":{"properties":{"key":{"type":"string"},"recordPublicUrl":{"format":"uri","type":"string"},"size":{"type":"integer"}},"required":["key","size","recordPublicUrl"],"type":"object"},"KeyValueStoreResource":{"properties":{"accessedAt":{"format":"date-time","type":"string"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"consoleUrl":{"format":"uri","type":"string"},"createdAt":{"format":"date-time","type":"string"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"id":{"type":"string"},"keysPublicUrl":{"format":"uri","type":"string"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":["string","null"]},"recordsPublicUrl":{"format":"uri","type":"string"},"schema":{"type":["object","null"]},"stats":{"$ref":"#/components/schemas/KeyValueStoreStats"},"urlSigningSecretKey":{"type":["string","null"]},"userId":{"type":["string","null"]},"username":{"type":["string","null"]}},"required":["id","createdAt","modifiedAt","accessedAt"],"type":"object"},"KeyValueStoreResponse":{"properties":{"data":{"$ref":"#/components/schemas/KeyValueStoreResource"}},"required":["data"],"type":"object"},"KeyValueStoreStats":{"properties":{"deleteCount":{"type":"integer"},"listCount":{"type":"integer"},"readCount":{"type":"integer"},"s3StorageBytes":{"type":"integer"},"storageBytes":{"type":"integer"},"writeCount":{"type":"integer"}},"type":"object"},"Limits":{"properties":{"dataRetentionDays":{"type":"integer"},"maxActorCount":{"type":"integer"},"maxActorMemoryGbytes":{"type":"number"},"maxActorTaskCount":{"type":"integer"},"maxConcurrentActorJobs":{"type":"integer"},"maxMonthlyActorComputeUnits":{"type":"number"},"maxMonthlyExternalDataTransferGbytes":{"type":"number"},"maxMonthlyProxySerps":{"type":"integer"},"maxMonthlyResidentialProxyGbytes":{"type":"number"},"maxMonthlyUsageUsd":{"type":"number"},"maxScheduleCount":{"type":"integer"},"maxTeamAccountSeatCount":{"type":"integer"}},"required":["maxMonthlyUsageUsd","maxMonthlyActorComputeUnits","maxMonthlyExternalDataTransferGbytes","maxMonthlyProxySerps","maxMonthlyResidentialProxyGbytes","maxActorMemoryGbytes","maxActorCount","maxActorTaskCount","maxConcurrentActorJobs","maxTeamAccountSeatCount","dataRetentionDays"],"type":"object"},"ListOfActors":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/ActorListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfActorsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfActors"}},"required":["data"],"type":"object"},"ListOfBuilds":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/BuildListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfBuildsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfBuilds"}},"required":["data"],"type":"object"},"ListOfDatasets":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/DatasetListItem"},"type":"array"},"unnamed":{"type":"boolean"}},"required":["items"],"type":"object"}]},"ListOfDatasetsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfDatasets"}},"required":["data"],"type":"object"},"ListOfEnvVars":{"properties":{"items":{"items":{"$ref":"#/components/schemas/EnvVar"},"type":"array"},"total":{"type":"integer"}},"required":["total","items"],"type":"object"},"ListOfEnvVarsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfEnvVars"}},"required":["data"],"type":"object"},"ListOfKeyValueStores":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/KeyValueStoreResource"},"type":"array"},"unnamed":{"type":"boolean"}},"required":["items"],"type":"object"}]},"ListOfKeyValueStoresResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfKeyValueStores"}},"required":["data"],"type":"object"},"ListOfKeys":{"properties":{"count":{"type":"integer"},"exclusiveStartKey":{"type":["string","null"]},"isTruncated":{"type":"boolean"},"items":{"items":{"$ref":"#/components/schemas/KeyValueStoreKey"},"type":"array"},"limit":{"type":"integer"},"nextExclusiveStartKey":{"type":["string","null"]}},"required":["items","count","limit","isTruncated"],"type":"object"},"ListOfKeysResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfKeys"}},"required":["data"],"type":"object"},"ListOfRequestQueues":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/RequestQueueListItem"},"type":"array"},"unnamed":{"type":"boolean"}},"required":["items"],"type":"object"}]},"ListOfRequestQueuesResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfRequestQueues"}},"required":["data"],"type":"object"},"ListOfRequests":{"properties":{"cursor":{"type":"string"},"exclusiveStartId":{"type":"string"},"items":{"items":{"$ref":"#/components/schemas/RequestResource"},"type":"array"},"limit":{"type":"integer"},"nextCursor":{"type":"string"}},"required":["items","limit"],"type":"object"},"ListOfRequestsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfRequests"}},"required":["data"],"type":"object"},"ListOfRuns":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/RunListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfRunsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfRuns"}},"required":["data"],"type":"object"},"ListOfSchedules":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/ScheduleListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfSchedulesResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfSchedules"}},"required":["data"],"type":"object"},"ListOfStoreActors":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/StoreActor"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfStoreActorsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfStoreActors"}},"required":["data"],"type":"object"},"ListOfTasks":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/TaskListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfTasksResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfTasks"}},"required":["data"],"type":"object"},"ListOfVersions":{"properties":{"items":{"items":{"$ref":"#/components/schemas/Version"},"type":"array"},"total":{"type":"integer"}},"required":["total","items"],"type":"object"},"ListOfVersionsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfVersions"}},"required":["data"],"type":"object"},"ListOfWebhookDispatches":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/WebhookDispatch"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfWebhookDispatchesResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfWebhookDispatches"}},"required":["data"],"type":"object"},"ListOfWebhooks":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/WebhookListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfWebhooksResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfWebhooks"}},"required":["data"],"type":"object"},"LockExpiresAt":{"format":"date-time","type":"string"},"LockedRequestQueueHead":{"properties":{"clientKey":{"type":"string"},"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"items":{"items":{"$ref":"#/components/schemas/LockedRequestQueueHeadItem"},"type":"array"},"limit":{"$ref":"#/components/schemas/HeadLimit"},"lockSecs":{"type":"integer"},"queueHasLockedRequests":{"type":"boolean"},"queueModifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"}},"required":["limit","queueModifiedAt","hadMultipleClients","lockSecs","items"],"type":"object"},"LockedRequestQueueHeadItem":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"lockExpiresAt":{"$ref":"#/components/schemas/LockExpiresAt"},"method":{"$ref":"#/components/schemas/RequestMethod"},"retryCount":{"$ref":"#/components/schemas/RetryCount"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"}},"required":["id","uniqueKey","url","lockExpiresAt"],"type":"object"},"LockedRequestQueueHeadResponse":{"properties":{"data":{"$ref":"#/components/schemas/LockedRequestQueueHead"}},"required":["data"],"type":"object"},"MonthlyServiceUsage":{"type":"object"},"MonthlyUsage":{"properties":{"dailyServiceUsages":{"items":{"$ref":"#/components/schemas/DailyServiceUsage"},"type":"array"},"monthlyServiceUsage":{"$ref":"#/components/schemas/MonthlyServiceUsage"},"totalUsageCreditsUsdAfterVolumeDiscount":{"type":"number"},"totalUsageCreditsUsdBeforeVolumeDiscount":{"type":"number"},"usageCycle":{"$ref":"#/components/schemas/UsageCycle"}},"required":["usageCycle","monthlyServiceUsage","dailyServiceUsages","totalUsageCreditsUsdBeforeVolumeDiscount","totalUsageCreditsUsdAfterVolumeDiscount"],"type":"object"},"MonthlyUsageResponse":{"properties":{"data":{"$ref":"#/components/schemas/MonthlyUsage"}},"required":["data"],"type":"object"},"PaginationResponse":{"properties":{"count":{"minimum":0,"type":"integer"},"desc":{"type":"boolean"},"limit":{"minimum":1,"type":"integer"},"offset":{"minimum":0,"type":"integer"},"total":{"minimum":0,"type":"integer"}},"required":["total","offset","limit","desc","count"],"type":"object"},"PayPerEventActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"minimalMaxTotalChargeUsd":{"type":["number","null"]},"pricingModel":{"type":"string"},"pricingPerEvent":{"properties":{"actorChargeEvents":{"type":"object"}},"type":"object"}},"required":["pricingModel","pricingPerEvent"],"type":"object"}]},"PendingRequestCount":{"type":"integer"},"PricePerDatasetItemActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"pricePerUnitUsd":{"type":"number"},"pricingModel":{"type":"string"},"tieredPricing":{"$ref":"#/components/schemas/TieredPricingPerDatasetItem"},"unitName":{"type":"string"}},"required":["pricingModel","unitName"],"type":"object"}]},"ProlongRequestLockResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestLockInfo"}},"required":["data"],"type":"object"},"ProxyGroup":{"properties":{"availableCount":{"type":"integer"},"description":{"type":["string","null"]},"name":{"type":"string"}},"required":["name","description","availableCount"],"type":"object"},"ProxyResource":{"properties":{"groups":{"items":{"$ref":"#/components/schemas/ProxyGroup"},"type":"array"},"password":{"type":"string"}},"required":["password","groups"],"type":"object"},"QueueAccessedAt":{"format":"date-time","type":"string"},"QueueCreatedAt":{"format":"date-time","type":"string"},"QueueId":{"type":"string"},"QueueModifiedAt":{"format":"date-time","type":"string"},"QueueUserId":{"type":"string"},"RecordResponse":{"type":"object"},"RequestBase":{"properties":{"errorMessages":{"items":{"type":"string"},"type":["array","null"]},"handledAt":{"format":"date-time","type":["string","null"]},"headers":{"type":["object","null"]},"loadedUrl":{"type":["string","null"]},"method":{"$ref":"#/components/schemas/RequestMethod"},"noRetry":{"type":["boolean","null"]},"payload":{"type":["string","object","null"]},"retryCount":{"$ref":"#/components/schemas/RetryCount"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"},"userData":{"$ref":"#/components/schemas/RequestUserData"}},"required":["uniqueKey","url"],"type":"object"},"RequestId":{"type":"string"},"RequestLockInfo":{"properties":{"lockExpiresAt":{"$ref":"#/components/schemas/LockExpiresAt"}},"required":["lockExpiresAt"],"type":"object"},"RequestMethod":{"$ref":"#/components/schemas/HttpMethod"},"RequestQueueHead":{"properties":{"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"items":{"items":{"$ref":"#/components/schemas/RequestQueueHeadItem"},"type":"array"},"limit":{"$ref":"#/components/schemas/HeadLimit"},"queueModifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"}},"required":["limit","queueModifiedAt","hadMultipleClients","items"],"type":"object"},"RequestQueueHeadItem":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"method":{"$ref":"#/components/schemas/RequestMethod"},"retryCount":{"$ref":"#/components/schemas/RetryCount"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"}},"required":["id","uniqueKey","url"],"type":"object"},"RequestQueueHeadResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestQueueHead"}},"required":["data"],"type":"object"},"RequestQueueListItem":{"properties":{"accessedAt":{"$ref":"#/components/schemas/QueueAccessedAt"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"createdAt":{"$ref":"#/components/schemas/QueueCreatedAt"},"expireAt":{"format":"date-time","type":"string"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"handledRequestCount":{"$ref":"#/components/schemas/HandledRequestCount"},"id":{"$ref":"#/components/schemas/QueueId"},"modifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"},"name":{"type":"string"},"pendingRequestCount":{"$ref":"#/components/schemas/PendingRequestCount"},"stats":{"$ref":"#/components/schemas/RequestQueueStats"},"totalRequestCount":{"$ref":"#/components/schemas/TotalRequestCount"},"userId":{"$ref":"#/components/schemas/QueueUserId"},"username":{"type":"string"}},"required":["id","name","userId","username","createdAt","modifiedAt","accessedAt","totalRequestCount","handledRequestCount","pendingRequestCount","hadMultipleClients"],"type":"object"},"RequestQueueResource":{"properties":{"accessedAt":{"$ref":"#/components/schemas/QueueAccessedAt"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"consoleUrl":{"format":"uri","type":"string"},"createdAt":{"$ref":"#/components/schemas/QueueCreatedAt"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"handledRequestCount":{"$ref":"#/components/schemas/HandledRequestCount"},"id":{"$ref":"#/components/schemas/QueueId"},"modifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"},"name":{"type":["string","null"]},"pendingRequestCount":{"$ref":"#/components/schemas/PendingRequestCount"},"stats":{"$ref":"#/components/schemas/RequestQueueStats"},"totalRequestCount":{"$ref":"#/components/schemas/TotalRequestCount"},"userId":{"$ref":"#/components/schemas/QueueUserId"}},"required":["id","userId","createdAt","modifiedAt","accessedAt","totalRequestCount","handledRequestCount","pendingRequestCount","hadMultipleClients","consoleUrl"],"type":"object"},"RequestQueueResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestQueueResource"}},"required":["data"],"type":"object"},"RequestQueueStats":{"properties":{"deleteCount":{"type":"integer"},"headItemReadCount":{"type":"integer"},"readCount":{"type":"integer"},"storageBytes":{"type":"integer"},"writeCount":{"type":"integer"}},"type":"object"},"RequestRegistration":{"properties":{"requestId":{"$ref":"#/components/schemas/RequestId"},"wasAlreadyHandled":{"$ref":"#/components/schemas/WasAlreadyHandled"},"wasAlreadyPresent":{"$ref":"#/components/schemas/WasAlreadyPresent"}},"required":["requestId","wasAlreadyPresent","wasAlreadyHandled"],"type":"object"},"RequestResource":{"allOf":[{"$ref":"#/components/schemas/RequestBase","required":["id"]},{"properties":{"id":{"$ref":"#/components/schemas/RequestId"}}}]},"RequestResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestResource"}},"required":["data"],"type":"object"},"RequestUrl":{"type":"string"},"RequestUserData":{"type":"object"},"RetryCount":{"type":"integer"},"Run":{"properties":{"actId":{"type":"string"},"actorTaskId":{"type":["string","null"]},"buildId":{"type":"string"},"buildNumber":{"pattern":"^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])(\\.[1-9][0-9]{0,4})$","type":["string","null"]},"chargedEventCounts":{"type":"object"},"containerUrl":{"format":"uri","type":"string"},"defaultDatasetId":{"type":"string"},"defaultKeyValueStoreId":{"type":"string"},"defaultRequestQueueId":{"type":"string"},"exitCode":{"type":["integer","null"]},"finishedAt":{"format":"date-time","type":["string","null"]},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"gitBranchName":{"type":["string","null"]},"id":{"type":"string"},"isContainerServerReady":{"type":["boolean","null"]},"isStatusMessageTerminal":{"type":["boolean","null"]},"meta":{"$ref":"#/components/schemas/RunMeta"},"metamorphs":{"anyOf":[{"items":{"$ref":"#/components/schemas/RunMetamorphEvent"},"type":"array"},{"type":"null"}]},"options":{"$ref":"#/components/schemas/RunOptions"},"platformUsageBillingModel":{"type":"string"},"pricingInfo":{"$ref":"#/components/schemas/ActorRunPricingInfo"},"startedAt":{"format":"date-time","type":"string"},"stats":{"$ref":"#/components/schemas/RunStats"},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"statusMessage":{"type":["string","null"]},"storageIds":{"properties":{"datasets":{"properties":{"default":{"type":"string"}},"type":"object"},"keyValueStores":{"properties":{"default":{"type":"string"}},"type":"object"},"requestQueues":{"properties":{"default":{"type":"string"}},"type":"object"}},"type":"object"},"usage":{"anyOf":[{"$ref":"#/components/schemas/RunUsage"},{"type":"null"}]},"usageTotalUsd":{"type":["number","null"]},"usageUsd":{"anyOf":[{"$ref":"#/components/schemas/RunUsageUsd"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","actId","userId","startedAt","status","meta","stats","options","buildId","defaultKeyValueStoreId","defaultDatasetId","defaultRequestQueueId"],"type":"object"},"RunListItem":{"properties":{"actId":{"type":"string"},"actorTaskId":{"type":["string","null"]},"buildId":{"type":"string"},"buildNumber":{"pattern":"^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])(\\.[1-9][0-9]{0,4})$","type":"string"},"buildNumberInt":{"type":"integer"},"defaultDatasetId":{"type":"string"},"defaultKeyValueStoreId":{"type":"string"},"defaultRequestQueueId":{"type":"string"},"finishedAt":{"format":"date-time","type":["string","null"]},"id":{"type":"string"},"meta":{"$ref":"#/components/schemas/RunMeta"},"startedAt":{"format":"date-time","type":"string"},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"usageTotalUsd":{"type":"number"},"userId":{"type":"string"}},"required":["id","actId","status","startedAt","buildId","meta","usageTotalUsd","defaultKeyValueStoreId","defaultDatasetId","defaultRequestQueueId"],"type":"object"},"RunMeta":{"properties":{"clientIp":{"type":["string","null"]},"origin":{"$ref":"#/components/schemas/RunOrigin"},"scheduleId":{"type":["string","null"]},"scheduledAt":{"format":"date-time","type":["string","null"]},"userAgent":{"type":["string","null"]}},"required":["origin"],"type":"object"},"RunMetamorphEvent":{"properties":{"actorId":{"type":"string"},"buildId":{"type":"string"},"createdAt":{"format":"date-time","type":"string"},"inputKey":{"type":["string","null"]}},"required":["createdAt","actorId","buildId"],"type":"object"},"RunOptions":{"properties":{"build":{"type":"string"},"diskMbytes":{"minimum":0,"type":"integer"},"maxItems":{"minimum":0,"type":["integer","null"]},"maxTotalChargeUsd":{"minimum":0,"type":["number","null"]},"memoryMbytes":{"maximum":32768,"minimum":128,"type":"integer"},"timeoutSecs":{"minimum":0,"type":"integer"}},"required":["build","timeoutSecs","memoryMbytes","diskMbytes"],"type":"object"},"RunOrigin":{"type":"string"},"RunResponse":{"properties":{"data":{"$ref":"#/components/schemas/Run"}},"required":["data"],"type":"object"},"RunStats":{"properties":{"computeUnits":{"minimum":0,"type":"number"},"cpuAvgUsage":{"type":"number"},"cpuCurrentUsage":{"type":"number"},"cpuMaxUsage":{"type":"number"},"durationMillis":{"minimum":0,"type":"integer"},"inputBodyLen":{"minimum":0,"type":["integer","null"]},"memAvgBytes":{"type":"number"},"memCurrentBytes":{"minimum":0,"type":"integer"},"memMaxBytes":{"minimum":0,"type":"integer"},"metamorph":{"minimum":0,"type":"integer"},"migrationCount":{"minimum":0,"type":"integer"},"netRxBytes":{"minimum":0,"type":"integer"},"netTxBytes":{"minimum":0,"type":"integer"},"rebootCount":{"minimum":0,"type":"integer"},"restartCount":{"minimum":0,"type":"integer"},"resurrectCount":{"minimum":0,"type":"integer"},"runTimeSecs":{"minimum":0,"type":"number"}},"type":"object"},"RunUsage":{"properties":{"ACTOR_COMPUTE_UNITS":{"type":["number","null"]},"DATASET_READS":{"type":["integer","null"]},"DATASET_WRITES":{"type":["integer","null"]},"DATA_TRANSFER_EXTERNAL_GBYTES":{"type":["number","null"]},"DATA_TRANSFER_INTERNAL_GBYTES":{"type":["number","null"]},"KEY_VALUE_STORE_LISTS":{"type":["integer","null"]},"KEY_VALUE_STORE_READS":{"type":["integer","null"]},"KEY_VALUE_STORE_WRITES":{"type":["integer","null"]},"PROXY_RESIDENTIAL_TRANSFER_GBYTES":{"type":["number","null"]},"PROXY_SERPS":{"type":["integer","null"]},"REQUEST_QUEUE_READS":{"type":["integer","null"]},"REQUEST_QUEUE_WRITES":{"type":["integer","null"]}},"type":"object"},"RunUsageUsd":{"properties":{"ACTOR_COMPUTE_UNITS":{"type":["number","null"]},"DATASET_READS":{"type":["number","null"]},"DATASET_WRITES":{"type":["number","null"]},"DATA_TRANSFER_EXTERNAL_GBYTES":{"type":["number","null"]},"DATA_TRANSFER_INTERNAL_GBYTES":{"type":["number","null"]},"KEY_VALUE_STORE_LISTS":{"type":["number","null"]},"KEY_VALUE_STORE_READS":{"type":["number","null"]},"KEY_VALUE_STORE_WRITES":{"type":["number","null"]},"PROXY_RESIDENTIAL_TRANSFER_GBYTES":{"type":["number","null"]},"PROXY_SERPS":{"type":["number","null"]},"REQUEST_QUEUE_READS":{"type":["number","null"]},"REQUEST_QUEUE_WRITES":{"type":["number","null"]}},"type":"object"},"Schedule":{"allOf":[{"$ref":"#/components/schemas/ScheduleBase"},{"properties":{"actions":{"items":{"$ref":"#/components/schemas/ScheduleAction"},"type":"array"},"description":{"type":["string","null"]},"notifications":{"$ref":"#/components/schemas/ScheduleNotifications"},"title":{"type":["string","null"]}},"required":["actions"],"type":"object"}]},"ScheduleAction":{"oneOf":[{"$ref":"#/components/schemas/ScheduleActionRunActor"},{"$ref":"#/components/schemas/ScheduleActionRunActorTask"}]},"ScheduleActionRunActor":{"properties":{"actorId":{"type":"string"},"id":{"type":"string"},"runInput":{"anyOf":[{"$ref":"#/components/schemas/ScheduleActionRunInput"},{"type":"null"}]},"runOptions":{"anyOf":[{"$ref":"#/components/schemas/TaskOptions"},{"type":"null"}]},"type":{}},"required":["id","type","actorId"],"type":"object"},"ScheduleActionRunActorTask":{"properties":{"actorTaskId":{"type":"string"},"id":{"type":"string"},"input":{"anyOf":[{"type":"object"},{"type":"null"}]},"type":{}},"required":["id","type","actorTaskId"],"type":"object"},"ScheduleActionRunInput":{"properties":{"body":{"type":["string","null"]},"contentType":{"type":["string","null"]}},"type":"object"},"ScheduleBase":{"properties":{"createdAt":{"format":"date-time","type":"string"},"cronExpression":{"type":"string"},"id":{"type":"string"},"isEnabled":{"type":"boolean"},"isExclusive":{"type":"boolean"},"lastRunAt":{"format":"date-time","type":["string","null"]},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":"string"},"nextRunAt":{"format":"date-time","type":["string","null"]},"timezone":{"type":"string"},"userId":{"type":"string"}},"required":["id","userId","name","cronExpression","timezone","isEnabled","isExclusive","createdAt","modifiedAt"],"type":"object"},"ScheduleListItem":{"allOf":[{"$ref":"#/components/schemas/ScheduleBase"},{"properties":{"actions":{"items":{"$ref":"#/components/schemas/ScheduleListItemAction"},"type":"array"}},"required":["actions"],"type":"object"}]},"ScheduleListItemAction":{"oneOf":[{"$ref":"#/components/schemas/ScheduleListItemActionRunActor"},{"$ref":"#/components/schemas/ScheduleListItemActionRunActorTask"}]},"ScheduleListItemActionRunActor":{"properties":{"actorId":{"type":"string"},"id":{"type":"string"},"type":{}},"required":["id","type","actorId"],"type":"object"},"ScheduleListItemActionRunActorTask":{"properties":{"actorTaskId":{"type":"string"},"id":{"type":"string"},"type":{}},"required":["id","type","actorTaskId"],"type":"object"},"ScheduleLogEntry":{"properties":{"createdAt":{"format":"date-time","type":"string"},"level":{"type":"string"},"message":{"type":"string"}},"required":["message","level","createdAt"],"type":"object"},"ScheduleLogResponse":{"properties":{"data":{"items":{"$ref":"#/components/schemas/ScheduleLogEntry"},"type":"array"}},"required":["data"],"type":"object"},"ScheduleNotifications":{"properties":{"email":{"type":"boolean"}},"type":"object"},"ScheduleResponse":{"properties":{"data":{"$ref":"#/components/schemas/Schedule"}},"required":["data"],"type":"object"},"ServiceUsage":{"type":"object"},"SourceCodeFile":{"properties":{"content":{"type":"string"},"format":{"$ref":"#/components/schemas/SourceCodeFileFormat"},"name":{"type":"string"}},"required":["name"],"type":"object"},"SourceCodeFileFormat":{"type":"string"},"SourceCodeFolder":{"properties":{"folder":{"type":"boolean"},"name":{"type":"string"}},"required":["name","folder"],"type":"object"},"StoreActor":{"properties":{"actorReviewCount":{"type":"integer"},"actorReviewRating":{"type":"number"},"badge":{"type":["string","null"]},"bookmarkCount":{"type":"integer"},"categories":{"items":{"type":"string"},"type":"array"},"currentPricingInfo":{"$ref":"#/components/schemas/CurrentPricingInfo"},"description":{"type":["string","null"]},"id":{"type":"string"},"isWhiteListedForAgenticPayments":{"type":["boolean","null"]},"name":{"type":"string"},"notice":{"$ref":"#/components/schemas/ActorNotice"},"pictureUrl":{"type":["string","null"]},"readmeSummary":{"type":"string"},"stats":{"$ref":"#/components/schemas/ActorStats"},"title":{"type":"string"},"url":{"format":"uri","type":["string","null"]},"userFullName":{"type":["string","null"]},"userPictureUrl":{"format":"uri","type":["string","null"]},"username":{"type":"string"}},"required":["id","title","name","username","stats"],"type":"object"},"TaggedBuilds":{"type":"object"},"Task":{"properties":{"actId":{"type":"string"},"actorStandby":{"anyOf":[{"$ref":"#/components/schemas/ActorStandby"},{"type":"null"}]},"createdAt":{"format":"date-time","type":"string"},"description":{"type":["string","null"]},"id":{"type":"string"},"input":{"anyOf":[{"$ref":"#/components/schemas/TaskInput"},{"items":{"$ref":"#/components/schemas/TaskInput"},"type":"array"},{"type":"null"}]},"isPublic":{"type":"boolean"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":"string"},"options":{"anyOf":[{"$ref":"#/components/schemas/TaskOptions"},{"type":"null"}]},"publicConfig":{"anyOf":[{"$ref":"#/components/schemas/TaskPublicConfig"},{"type":"null"}]},"removedAt":{"format":"date-time","type":["string","null"]},"standbyUrl":{"format":"uri","type":["string","null"]},"stats":{"anyOf":[{"$ref":"#/components/schemas/TaskStats"},{"type":"null"}]},"title":{"type":["string","null"]},"userId":{"type":"string"},"username":{"type":["string","null"]}},"required":["id","userId","actId","name","createdAt","modifiedAt"],"type":"object"},"TaskInput":{"type":"object"},"TaskListItem":{"properties":{"actId":{"type":"string"},"actName":{"type":["string","null"]},"actUsername":{"type":["string","null"]},"createdAt":{"format":"date-time","type":"string"},"id":{"type":"string"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":"string"},"stats":{"anyOf":[{"$ref":"#/components/schemas/TaskStats"},{"type":"null"}]},"userId":{"type":"string"},"username":{"type":["string","null"]}},"required":["id","userId","actId","name","createdAt","modifiedAt"],"type":"object"},"TaskOptions":{"properties":{"build":{"type":["string","null"]},"maxItems":{"type":["integer","null"]},"maxTotalChargeUsd":{"type":["number","null"]},"memoryMbytes":{"type":["integer","null"]},"restartOnError":{"type":["boolean","null"]},"timeoutSecs":{"type":["integer","null"]}},"type":"object"},"TaskPublicConfig":{"properties":{"datasetName":{"type":["string","null"]},"datasetView":{"type":["string","null"]},"inputSchemaFields":{"items":{"type":"string"},"type":["array","null"]},"publishedAt":{"format":"date-time","type":["string","null"]},"seoDescription":{"maxLength":160,"type":["string","null"]},"seoTitle":{"maxLength":60,"type":["string","null"]}},"type":"object"},"TaskResponse":{"properties":{"data":{"$ref":"#/components/schemas/Task"}},"required":["data"],"type":"object"},"TaskStats":{"properties":{"totalRuns":{"type":"integer"}},"type":"object"},"TieredPricingPerDatasetItem":{"type":"object"},"TotalRequestCount":{"minimum":0,"type":"integer"},"UniqueKey":{"type":"string"},"UnlockRequestsResponse":{"properties":{"data":{"$ref":"#/components/schemas/UnlockRequestsResult"}},"required":["data"],"type":"object"},"UnlockRequestsResult":{"properties":{"unlockedCount":{"type":"integer"}},"required":["unlockedCount"],"type":"object"},"UnprocessedRequest":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"method":{"$ref":"#/components/schemas/RequestMethod"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"}},"required":["uniqueKey","url"],"type":"object"},"UpdateRequestResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestRegistration"}},"required":["data"],"type":"object"},"UsageCycle":{"properties":{"endAt":{"format":"date-time","type":"string"},"startAt":{"format":"date-time","type":"string"}},"required":["startAt","endAt"],"type":"object"},"UserPlan":{"properties":{"apiRateLimitBoosts":{"type":"integer"},"availableAddOns":{"items":{"type":"string"},"type":"array"},"availableProxyGroups":{"$ref":"#/components/schemas/AvailableProxyGroups"},"dataRetentionDays":{"type":"integer"},"description":{"type":"string"},"enabledPlatformFeatures":{"items":{"type":"string"},"type":"array"},"id":{"type":"string"},"isEnabled":{"type":"boolean"},"maxActorCount":{"type":"integer"},"maxActorMemoryGbytes":{"type":"number"},"maxActorTaskCount":{"type":"integer"},"maxConcurrentActorRuns":{"type":"integer"},"maxMonthlyActorComputeUnits":{"type":"number"},"maxMonthlyExternalDataTransferGbytes":{"type":"number"},"maxMonthlyProxySerps":{"type":"integer"},"maxMonthlyResidentialProxyGbytes":{"type":"number"},"maxMonthlyUsageUsd":{"type":"number"},"maxScheduleCount":{"type":"integer"},"monthlyBasePriceUsd":{"type":"number"},"monthlyUsageCreditsUsd":{"type":"number"},"planPricing":{"type":"object"},"supportLevel":{"type":"string"},"teamAccountSeatCount":{"type":"integer"},"tier":{"type":"string"},"usageDiscountPercent":{"type":"number"}},"required":["availableProxyGroups"],"type":"object"},"UserPrivateInfo":{"properties":{"createdAt":{"format":"date-time","type":"string"},"effectivePlatformFeatures":{"$ref":"#/components/schemas/EffectivePlatformFeatures"},"email":{"format":"email","type":"string"},"id":{"type":"string"},"isPaying":{"type":"boolean"},"plan":{"$ref":"#/components/schemas/UserPlan"},"profile":{"$ref":"#/components/schemas/UserProfile"},"proxy":{"$ref":"#/components/schemas/ProxyResource"},"username":{"type":"string"}},"required":["username"],"type":"object"},"UserPrivateInfoResponse":{"properties":{"data":{"$ref":"#/components/schemas/UserPrivateInfo"}},"required":["data"],"type":"object"},"UserProfile":{"properties":{"bio":{"type":["string","null"]},"githubUsername":{"type":["string","null"]},"name":{"type":"string"},"pictureUrl":{"format":"uri","type":["string","null"]},"readme":{"type":["string","null"]},"twitterUsername":{"type":["string","null"]},"websiteUrl":{"format":"uri","type":["string","null"]}},"type":"object"},"UserPublicInfo":{"properties":{"profile":{"$ref":"#/components/schemas/UserProfile"},"username":{"type":"string"}},"required":["username"],"type":"object"},"UserPublicInfoResponse":{"properties":{"data":{"$ref":"#/components/schemas/UserPublicInfo"}},"required":["data"],"type":"object"},"Version":{"properties":{"applyEnvVarsToBuild":{"type":["boolean","null"]},"buildTag":{"type":["string","null"]},"envVars":{"anyOf":[{"items":{"$ref":"#/components/schemas/EnvVar"},"type":"array"},{"type":"null"}]},"gitHubGistUrl":{"type":["string","null"]},"gitRepoUrl":{"type":["string","null"]},"sourceFiles":{"$ref":"#/components/schemas/VersionSourceFiles"},"sourceType":{"anyOf":[{"$ref":"#/components/schemas/VersionSourceType"},{"type":"null"}]},"tarballUrl":{"type":["string","null"]},"versionNumber":{"pattern":"^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])$","type":"string"}},"required":["versionNumber","sourceType"],"type":"object"},"VersionResponse":{"properties":{"data":{"$ref":"#/components/schemas/Version"}},"required":["data"],"type":"object"},"VersionSourceFiles":{"items":{"anyOf":[{"$ref":"#/components/schemas/SourceCodeFile"},{"$ref":"#/components/schemas/SourceCodeFolder"}]},"type":"array"},"VersionSourceType":{"type":"string"},"WasAlreadyHandled":{"type":"boolean"},"WasAlreadyPresent":{"type":"boolean"},"WebhookCondition":{"properties":{"actorId":{"type":["string","null"]},"actorRunId":{"type":["string","null"]},"actorTaskId":{"type":["string","null"]}},"type":"object"},"WebhookDispatch":{"properties":{"calls":{"items":{"properties":{"errorMessage":{"type":["string","null"]},"finishedAt":{"format":"date-time","type":["string","null"]},"responseBody":{"type":["string","null"]},"responseStatus":{"type":["integer","null"]},"startedAt":{"format":"date-time","type":["string","null"]}},"type":"object"},"type":"array"},"createdAt":{"format":"date-time","type":"string"},"eventData":{"properties":{"actorBuildId":{"type":"string"},"actorId":{"type":"string"},"actorRunId":{"type":"string"},"actorTaskId":{"type":["string","null"]}},"required":["actorId"],"type":["object","null"]},"eventType":{"$ref":"#/components/schemas/WebhookEventType"},"id":{"type":"string"},"status":{"$ref":"#/components/schemas/WebhookDispatchStatus"},"userId":{"type":"string"},"webhook":{"anyOf":[{"$ref":"#/components/schemas/WebhookDispatchWebhookSummary"},{"type":"null"}]},"webhookId":{"type":"string"}},"required":["id","userId","webhookId","createdAt","status","eventType"],"type":"object"},"WebhookDispatchResponse":{"properties":{"data":{"$ref":"#/components/schemas/WebhookDispatch"}},"required":["data"],"type":"object"},"WebhookDispatchStatus":{"type":"string"},"WebhookDispatchWebhookSummary":{"properties":{"actionType":{"type":"string"},"condition":{"$ref":"#/components/schemas/WebhookCondition"},"isAdHoc":{"type":"boolean"},"requestUrl":{"format":"uri","type":["string","null"]}},"type":"object"},"WebhookEventType":{"type":"string"},"WebhookLastDispatch":{"properties":{"finishedAt":{"format":"date-time","type":["string","null"]},"removedAt":{"format":"date-time","type":["string","null"]},"status":{"$ref":"#/components/schemas/WebhookDispatchStatus"}},"required":["status"],"type":"object"},"WebhookListItem":{"properties":{"actionType":{"type":"string"},"condition":{"$ref":"#/components/schemas/WebhookCondition"},"createdAt":{"format":"date-time","type":"string"},"doNotRetry":{"type":"boolean"},"eventTypes":{"items":{"$ref":"#/components/schemas/WebhookEventType"},"type":"array"},"id":{"type":"string"},"ignoreSslErrors":{"type":"boolean"},"isAdHoc":{"type":["boolean","null"]},"isApifyIntegration":{"type":"boolean"},"isEnabled":{"type":"boolean"},"lastDispatch":{"anyOf":[{"$ref":"#/components/schemas/WebhookLastDispatch"},{"type":"null"}]},"modifiedAt":{"format":"date-time","type":"string"},"requestUrl":{"format":"uri","type":"string"},"shouldInterpolateStrings":{"type":["boolean","null"]},"stats":{"anyOf":[{"$ref":"#/components/schemas/WebhookStats"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","createdAt","modifiedAt","userId","eventTypes","condition","ignoreSslErrors","doNotRetry","requestUrl"],"type":"object"},"WebhookResource":{"properties":{"condition":{"$ref":"#/components/schemas/WebhookCondition"},"createdAt":{"format":"date-time","type":"string"},"description":{"type":["string","null"]},"doNotRetry":{"type":["boolean","null"]},"eventTypes":{"items":{"$ref":"#/components/schemas/WebhookEventType"},"type":"array"},"headersTemplate":{"type":["string","null"]},"id":{"type":"string"},"ignoreSslErrors":{"type":"boolean"},"isAdHoc":{"type":["boolean","null"]},"lastDispatch":{"anyOf":[{"$ref":"#/components/schemas/WebhookLastDispatch"},{"type":"null"}]},"modifiedAt":{"format":"date-time","type":"string"},"payloadTemplate":{"type":["string","null"]},"requestUrl":{"format":"uri","type":["string","null"]},"shouldInterpolateStrings":{"type":["boolean","null"]},"stats":{"anyOf":[{"$ref":"#/components/schemas/WebhookStats"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","createdAt","modifiedAt","userId","eventTypes","condition","ignoreSslErrors"],"type":"object"},"WebhookResponse":{"properties":{"data":{"$ref":"#/components/schemas/WebhookResource"}},"required":["data"],"type":"object"},"WebhookStats":{"properties":{"totalDispatches":{"type":"integer"}},"type":"object"}},"specVersion":"v2-2026-10-20T121847Z"} diff --git a/src/Apify.Client/Options/StoreListOptions.cs b/src/Apify.Client/Options/StoreListOptions.cs index f7e3f13..a7a1f64 100644 --- a/src/Apify.Client/Options/StoreListOptions.cs +++ b/src/Apify.Client/Options/StoreListOptions.cs @@ -8,7 +8,7 @@ public sealed class StoreListOptions /// Number of Actors to skip. public int? Offset { get; init; } - /// Maximum number of Actors to return (also the per-page size when iterating). + /// Maximum number of Actors to return (when iterating, in total across all pages). public int? Limit { get; init; } /// @@ -45,7 +45,6 @@ public sealed class StoreListOptions /// The response format (full, agent). public string? ResponseFormat { get; init; } - /// Returns a copy of these options with a new (used by lazy iteration). internal void AppendTo(QueryParams q) { q.AddInt("offset", Offset) diff --git a/src/Apify.Client/Resources/DatasetClient.cs b/src/Apify.Client/Resources/DatasetClient.cs index d256632..eca6478 100644 --- a/src/Apify.Client/Resources/DatasetClient.cs +++ b/src/Apify.Client/Resources/DatasetClient.cs @@ -82,54 +82,36 @@ public async Task UpdateAsync(object newFields, CancellationToken cance /// /// /// The offset for the next page, and whether iteration continues, follow the number of rows the API - /// scanned (X-Apify-Pagination-Count) rather than the number of items a page returned. The + /// scanned (X-Apify-Pagination-Count) rather than the number of items a page returned: the /// // /// filters apply after offset/limit, so a - /// page can scan up to the requested limit of rows while returning fewer of them, or none at all; - /// advancing by the returned count (as intentionally does, since it - /// is public and reports this page's item count) would re-scan rows on the next page or stop iteration - /// in front of rows a filter hid. can conversely leave a page - /// with more items than rows scanned, so the two counts cannot be mixed. The scanned count is read from - /// the response header and falls back to the returned item count only when the header itself is absent - /// (the API always sends it; a missing header means something between the client and the API, such as a - /// proxy, stripped it). + /// page can scan up to the requested limit of rows while returning fewer of them, or none at all, and + /// can return more items than rows scanned. + /// therefore caps the rows scanned (0 or null + /// for no cap). The header can lag behind items that were just pushed, so the larger of it and the + /// returned item count is used, capped at the page's requested limit; see . /// /// Optional item filtering/projection; Offset/Limit bound where - /// iteration starts and the total number of items yielded. + /// iteration starts and the rows scanned, and ChunkSize sets the page size. /// A token to cancel the iteration. - public async IAsyncEnumerable IterateItemsAsync( + public IAsyncEnumerable IterateItemsAsync( DatasetListItemsOptions? options = null, - [EnumeratorCancellation] CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) { options ??= new DatasetListItemsOptions(); - PageSizing.ValidateChunkSize(options.ChunkSize, nameof(DatasetListItemsOptions.ChunkSize)); var desc = options.Desc ?? false; var baseQuery = new QueryParams(); options.AppendTo(baseQuery); - var offset = Math.Max(options.Offset ?? 0, 0); - var limit = options.Limit; - var yielded = 0; - while (true) - { - var q = baseQuery.Copy().Set("offset", offset); - PageSizing.SetPageLimit(q, options.ChunkSize, limit, yielded); - - var (page, scannedCount) = await FetchItemsPageAsync(q, desc, cancellationToken).ConfigureAwait(false); - foreach (var item in page.Items) - { - yield return item; - yielded++; - } - - // Never stop at the reported total: it only reflects the moment its own page was fetched, and the - // dataset can grow while the iteration runs. Like the collection iterators, stop once a page scans - // no rows, which costs one extra request at the end. - offset += (int)scannedCount; - if (scannedCount == 0 || (limit is not null && yielded >= limit.Value)) + return PagedIteration.IterateAsync( + options.Offset ?? 0, + options.Limit, + options.ChunkSize, + async (offset, pageLimit, token) => { - yield break; - } - } + var (page, scannedCount) = await FetchItemsPageAsync(baseQuery.PageQuery(offset, pageLimit), desc, token).ConfigureAwait(false); + return new FetchedPage(page.Items, scannedCount); + }, + cancellationToken); } /// Fetches a single page of items for the given options (see the query overload). diff --git a/src/Apify.Client/Resources/StoreCollectionClient.cs b/src/Apify.Client/Resources/StoreCollectionClient.cs index 6b79bc8..d2cd32d 100644 --- a/src/Apify.Client/Resources/StoreCollectionClient.cs +++ b/src/Apify.Client/Resources/StoreCollectionClient.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Apify.Client.Internal; diff --git a/src/Apify.Client/Resources/TaskClient.cs b/src/Apify.Client/Resources/TaskClient.cs index 27e1a05..2ab2052 100644 --- a/src/Apify.Client/Resources/TaskClient.cs +++ b/src/Apify.Client/Resources/TaskClient.cs @@ -136,7 +136,7 @@ public async Task CallAsync( /// /// Fetches the task's stored input. Throws if the task does not exist (no longer resolves to null /// for that case): the input endpoint has no meaning apart from its parent task, matching the reference - /// client. Still returns a JSON null node if the task exists but has no stored input. + /// client. A task without stored input yields an empty JSON object. /// /// A token to cancel the request. public async Task GetInputAsync(CancellationToken cancellationToken = default) diff --git a/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs index 2ea0f26..a55beca 100644 --- a/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs +++ b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs @@ -196,7 +196,7 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenA public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenUnwindExpandsAPage() { // Regression test: `unwind` can make a page return more items than rows scanned. Advancing by the - // returned item count (3) instead of the scanned count (1) would skip the remaining 4 rows. + // returned item count (3) instead of the rows the page was asked to scan (1) would skip rows. var page1Headers = new Dictionary { ["X-Apify-Pagination-Total"] = "5", @@ -208,8 +208,8 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenU { ["X-Apify-Pagination-Total"] = "5", ["X-Apify-Pagination-Offset"] = "1", - ["X-Apify-Pagination-Limit"] = "4", - ["X-Apify-Pagination-Count"] = "4", + ["X-Apify-Pagination-Limit"] = "1", + ["X-Apify-Pagination-Count"] = "1", }; var transport = new MockTransport() .QueueResponse(200, "[{\"i\":1},{\"i\":2},{\"i\":3}]", page1Headers) @@ -217,7 +217,7 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenU .QueueResponse(200, "[]"); var values = new List(); - await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync()) + await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync(new DatasetListItemsOptions { Unwind = new[] { "a" }, ChunkSize = 1 })) { values.Add(item!["i"]!.GetValue()); } @@ -226,6 +226,28 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenU Assert.Equal(3, transport.CallCount); Assert.Contains("offset=0", transport.Received[0].Uri, StringComparison.Ordinal); Assert.Contains("offset=1", transport.Received[1].Uri, StringComparison.Ordinal); + Assert.Contains("offset=2", transport.Received[2].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task DatasetIterateItemsDoesNotStopWhenTheScannedCountHeaderLagsBehindReturnedItems() + { + // The scanned count is derived from the dataset's item count, which lags a fresh push: a page can + // return items while the header still reports 0. The larger of the two is used, so iteration goes on. + var laggingHeaders = new Dictionary { ["X-Apify-Pagination-Count"] = "0" }; + var transport = new MockTransport() + .QueueResponse(200, "[{\"i\":1},{\"i\":2}]", laggingHeaders) + .QueueResponse(200, "[{\"i\":3}]", laggingHeaders) + .QueueResponse(200, "[]", laggingHeaders); + + var values = new List(); + await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync()) + { + values.Add(item!["i"]!.GetValue()); + } + + Assert.Equal(new[] { 1, 2, 3 }, values); + Assert.Contains("offset=2", transport.Received[1].Uri, StringComparison.Ordinal); } [Fact] diff --git a/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs b/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs index f809606..2f42c9e 100644 --- a/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs +++ b/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs @@ -59,11 +59,45 @@ public async Task CollectionIterationWithoutLimitUsesChunkSizeAsPageLimit() public async Task ZeroLimitDoesNotBeatChunkSize() { // The API reads limit=0 as unset, so it must lose to the chunk size. - var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(1, SpecFixtures.ScheduleItem("a"))); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(1, SpecFixtures.ScheduleItem("a"))).QueueResponse(200, SpecFixtures.Page(1)); await DrainAsync(Client(transport).Schedules().IterateAsync(new ListOptions { Limit = 0, ChunkSize = 3 })); - Assert.Contains("limit=3", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.All(transport.Received, static r => Assert.Contains("limit=3", r.Uri, StringComparison.Ordinal)); + } + + [Fact] + public async Task ZeroLimitMeansNoLimitForCollectionStoreAndDatasetItemIteration() + { + var collection = new MockTransport().QueueResponse(200, SpecFixtures.ActorPage(3, "a", "b")).QueueResponse(200, SpecFixtures.ActorPage(3, "c")).QueueResponse(200, SpecFixtures.ActorPage(3)); + var collected = 0; + await foreach (var unused in Client(collection).Actors().IterateAsync(new ActorListOptions { Limit = 0, ChunkSize = 2 })) + { + _ = unused; + collected++; + } + + Assert.Equal(3, collected); + + var store = new MockTransport().QueueResponse(200, SpecFixtures.Page(2, SpecFixtures.StoreItem("a"))).QueueResponse(200, SpecFixtures.Page(2, SpecFixtures.StoreItem("b"))).QueueResponse(200, SpecFixtures.Page(2)); + var stored = 0; + await foreach (var unused in Client(store).Store().IterateAsync(new StoreListOptions { Limit = 0, ChunkSize = 1 })) + { + _ = unused; + stored++; + } + + Assert.Equal(2, stored); + + var dataset = new MockTransport().QueueResponse(200, "[{\"n\":1},{\"n\":2}]").QueueResponse(200, "[{\"n\":3}]").QueueResponse(200, "[]"); + var items = 0; + await foreach (var unused in Client(dataset).Dataset("ds1").IterateItemsAsync(new DatasetListItemsOptions { Limit = 0, ChunkSize = 2 })) + { + _ = unused; + items++; + } + + Assert.Equal(3, items); } [Theory] diff --git a/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs b/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs index f86e2a9..81b598a 100644 --- a/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs +++ b/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs @@ -74,6 +74,53 @@ public void NullIsRejectedWhereTheSpecDoesNotAllowIt() Assert.Contains("actVersion.versionNumber", issues[0], StringComparison.Ordinal); } + [Fact] + public void DateTimeFormatIsEnforced() + { + var issues = Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build.Replace("2026-01-01T00:00:00.000Z", "yesterday", StringComparison.Ordinal) + "}}"); + + Assert.Equal(new[] { "$.data.startedAt: is not a valid date-time value" }, issues); + } + + [Fact] + public void PatternIsEnforced() + { + var issues = Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build.Replace("0.0.1", "not-a-version", StringComparison.Ordinal) + "}}"); + + Assert.Single(issues); + Assert.Contains("$.data.buildNumber: does not match the pattern", issues[0], StringComparison.Ordinal); + } + + [Fact] + public void NumericBoundsAreEnforced() + { + var below = Issues("GET", "/v2/actor-runs/r1", "{\"data\":" + SpecFixtures.RunObject("r").Replace("\"memoryMbytes\":256", "\"memoryMbytes\":64", StringComparison.Ordinal) + "}"); + var above = Issues("GET", "/v2/actor-runs/r1", "{\"data\":" + SpecFixtures.RunObject("r").Replace("\"memoryMbytes\":256", "\"memoryMbytes\":999999", StringComparison.Ordinal) + "}"); + + Assert.Equal(new[] { "$.data.options.memoryMbytes: is a value outside the documented range (minimum 128)" }, below); + Assert.Equal(new[] { "$.data.options.memoryMbytes: is a value outside the documented range (maximum 32768)" }, above); + } + + [Fact] + public void UriFormatIsEnforced() + { + var issues = Issues("GET", "/v2/key-value-stores/s1/keys", SpecFixtures.KeysPage(1, false, null, "a").Replace("https://example.com/a", "not a url", StringComparison.Ordinal)); + + Assert.Equal(new[] { "$.data.items[0].recordPublicUrl: is not a valid uri value" }, issues); + } + + [Fact] + public async Task ExceptionExposesTheIssuesAsAStructuredList() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"b\"}}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Build("b1").GetAsync()); + + Assert.Contains("$.data.actId: required property is missing", ex.Issues); + Assert.Contains(ex.Issues, static issue => issue.StartsWith("$.data.status", StringComparison.Ordinal)); + Assert.StartsWith("response does not match the API specification: ", ex.Message, StringComparison.Ordinal); + } + [Fact] public void UnknownPropertiesAndUnknownEnumValuesAreAccepted() { diff --git a/tests/Apify.Client.Tests/Unit/SpecFixtures.cs b/tests/Apify.Client.Tests/Unit/SpecFixtures.cs index 2f55879..6cf053d 100644 --- a/tests/Apify.Client.Tests/Unit/SpecFixtures.cs +++ b/tests/Apify.Client.Tests/Unit/SpecFixtures.cs @@ -53,7 +53,7 @@ public static string Run(string id, string status = "SUCCEEDED") /// The bare run object, for tests that compose a larger body. public static string RunObject(string id, string status = "SUCCEEDED") => "{\"id\":\"" + id + "\",\"actId\":\"a\",\"userId\":\"u\",\"startedAt\":\"" + Timestamp + "\",\"status\":\"" + status - + "\",\"meta\":{\"origin\":\"API\"},\"stats\":{},\"options\":{\"build\":\"latest\",\"timeoutSecs\":0,\"memoryMbytes\":0,\"diskMbytes\":0},\"buildId\":\"b\"" + + "\",\"meta\":{\"origin\":\"API\"},\"stats\":{},\"options\":{\"build\":\"latest\",\"timeoutSecs\":0,\"memoryMbytes\":256,\"diskMbytes\":0},\"buildId\":\"b\"" + ",\"defaultKeyValueStoreId\":\"kvs\",\"defaultDatasetId\":\"ds\",\"defaultRequestQueueId\":\"rq\"}"; /// An Actor build. diff --git a/tools/generate_response_schemas.py b/tools/generate_response_schemas.py index 58ea08d..361cee8 100644 --- a/tools/generate_response_schemas.py +++ b/tools/generate_response_schemas.py @@ -8,9 +8,10 @@ python3 tools/generate_response_schemas.py [SPEC_URL_OR_PATH] The default source is https://docs.apify.com/api/openapi.json. Only what validation needs is kept: types, -required properties, nested properties and items, composition keywords and references. Documentation, -examples and keywords the client deliberately does not enforce (enum, const, pattern, format, bounds) are -dropped, which also keeps the embedded resource small. +required properties, nested properties and items, composition keywords, references, and the value +constraints (format, pattern, minimum, maximum, minLength, maxLength). Documentation, examples and keywords the client deliberately does not enforce (enum, const) are dropped, which also keeps the +embedded resource small. Enum values stay open on purpose so that a value the API adds later does not break +existing clients; the reference client's generated schemas do the same. """ import json import pathlib @@ -23,6 +24,8 @@ SCHEMA_REF_PREFIX = "#/components/schemas/" # Keywords that carry structure the validator evaluates. Everything else is dropped. SCHEMA_LIST_KEYWORDS = ("allOf", "anyOf", "oneOf") +# Scalar constraints copied as they are. +CONSTRAINT_KEYWORDS = ("format", "pattern", "minimum", "maximum", "minLength", "maxLength") def load_spec(source): @@ -42,6 +45,9 @@ def slim(schema): result["$ref"] = schema["$ref"] if "type" in schema: result["type"] = schema["type"] + for keyword in CONSTRAINT_KEYWORDS: + if keyword in schema: + result[keyword] = schema[keyword] if "required" in schema: result["required"] = schema["required"] if "properties" in schema: From a9085eee06f6468f0a060d3b2bb3de7e72a4e65a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 14:29:39 +0000 Subject: [PATCH 6/8] fix: advance list iteration by scanned rows, reject negative offset/limit, relax email validation Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01UUMWnU4gH573HweHieMhHi --- CHANGELOG.md | 4 ++ docs/storages.md | 3 +- src/Apify.Client/Internal/PagedIteration.cs | 7 +++- src/Apify.Client/Internal/ResourceContext.cs | 8 +++- src/Apify.Client/Internal/ResponseSchemas.cs | 5 +-- .../Options/DatasetListItemsOptions.cs | 6 ++- .../Apify.Client.Tests/Unit/ChunkSizeTests.cs | 39 +++++++++++++++++++ .../Unit/ResponseSchemaTests.cs | 7 ++++ tests/Apify.Client.Tests/Unit/SpecFixtures.cs | 4 ++ tools/generate_response_schemas.py | 5 ++- 10 files changed, 76 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f646c2..f9d972b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,10 @@ the `X-Apify-Pagination-Count` header and the returned item count (capped at the requested limit), so a header that lags behind freshly pushed items no longer ends the iteration early. A dataset `Limit` now caps the rows scanned. +- Fixed the list and Store iterators to advance by the rows a page scanned (its `count`) instead of the items + it returned, so the Store listing, which drops Actors the caller cannot run after applying offset and + limit, no longer repeats or skips Actors. A negative `Offset` or `Limit` now throws + `ArgumentOutOfRangeException` instead of being silently adjusted. - Nested accessors of a run (`Dataset()`, `KeyValueStore()`, `RequestQueue()`, `Log()`) and of a build (`Log()`) now inherit the run's or build's `WithTimeout` override. - Added a documentation test that compiles every C# snippet in the README and `docs/`. diff --git a/docs/storages.md b/docs/storages.md index c574b55..b73b2cd 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -40,7 +40,8 @@ reached from a run (`client.Run(id).Dataset()`, `.KeyValueStore()`, `.RequestQue - `GetStatisticsAsync()` → `JsonObject` (throws if the dataset does not exist). - `CreateItemsPublicUrlAsync(DatasetListItemsOptions? options = null, int? expiresInSecs = null, DownloadItemsFormat? format = null)` → `string` (a signed public URL; `format` defaults to `Json`). -`DatasetListItemsOptions` selects and reshapes items: `Offset`/`Limit` (pagination), `ChunkSize` (the page +`DatasetListItemsOptions` selects and reshapes items: `Offset`/`Limit` (pagination: `Limit` is the number of +dataset rows read, in total across pages when iterating; `0` or unset means no cap), `ChunkSize` (the page size `IterateItemsAsync` requests, never sent to the API), `Desc` (reverse order), `Fields`/`OutputFields`/`Omit` (choose columns), `Unwind`/`Flatten` (restructure nested fields), `Clean`/`SkipEmpty`/`SkipHidden`/`SkipFailedPages` (drop unwanted rows), `Simplified`, diff --git a/src/Apify.Client/Internal/PagedIteration.cs b/src/Apify.Client/Internal/PagedIteration.cs index c257aee..3352422 100644 --- a/src/Apify.Client/Internal/PagedIteration.cs +++ b/src/Apify.Client/Internal/PagedIteration.cs @@ -22,7 +22,8 @@ internal static class PagedIteration /// Lazily yields every item from on, fetching pages on demand. /// /// - /// caps the number of scanned rows (null or 0 for no cap). + /// caps the number of scanned rows (null or 0 for no cap); a + /// negative offset or limit is rejected. /// The next offset and the stop condition follow the rows a page scanned, not the items it returned: /// advancing by the item count would re-scan rows after a filter dropped some and skip rows after /// unwind multiplied them, and stopping at an empty page would end the iteration in front of rows @@ -46,7 +47,9 @@ public static async IAsyncEnumerable IterateAsync( [EnumeratorCancellation] CancellationToken cancellationToken) { PageSizing.ValidateChunkSize(chunkSize, "ChunkSize"); - var offset = Math.Max(startOffset, 0); + ArgumentOutOfRangeException.ThrowIfNegative(startOffset, "Offset"); + ArgumentOutOfRangeException.ThrowIfNegative(totalLimit ?? 0, "Limit"); + var offset = startOffset; int? remaining = totalLimit is > 0 ? totalLimit : null; long scanned; diff --git a/src/Apify.Client/Internal/ResourceContext.cs b/src/Apify.Client/Internal/ResourceContext.cs index 863f905..3aaaf17 100644 --- a/src/Apify.Client/Internal/ResourceContext.cs +++ b/src/Apify.Client/Internal/ResourceContext.cs @@ -236,8 +236,12 @@ public IAsyncEnumerable IterateListAsync( chunkSize, async (offset, pageLimit, token) => { - var page = await ListResourceAsync(subPath, baseQuery.PageQuery(offset, pageLimit), hydrate, tier, token).ConfigureAwait(false); - return new FetchedPage(page.Items, null); + var data = await GetResourceRequiredAsync(subPath, baseQuery.PageQuery(offset, pageLimit), tier, token).ConfigureAwait(false); + var page = PaginationList.FromData(data, hydrate); + // `count` is the number of rows scanned. It can exceed the items returned: the Store listing + // drops Actors the caller cannot run after applying offset and limit, so advancing by the + // returned items would re-read rows. + return new FetchedPage(page.Items, JsonValues.IntOr(data, "count", 0)); }, ct); } diff --git a/src/Apify.Client/Internal/ResponseSchemas.cs b/src/Apify.Client/Internal/ResponseSchemas.cs index 034cc6a..5f43e24 100644 --- a/src/Apify.Client/Internal/ResponseSchemas.cs +++ b/src/Apify.Client/Internal/ResponseSchemas.cs @@ -19,7 +19,7 @@ namespace Apify.Client.Internal; /// The schemas come from the embedded ResponseSchemas.json, generated from the specification by /// tools/generate_response_schemas.py. Validation covers JSON types, required properties (including /// nullability), nested properties and array items, the allOf/anyOf/oneOf combinators, -/// and value constraints: the date-time, uri and email formats, pattern, +/// and value constraints: the date-time and uri formats, pattern, /// minimum/maximum and minLength/maxLength. Like the reference client's /// schemas, it lets unknown properties and unknown enum values through, so an API that grows new fields or /// values does not break existing clients. A request that matches no documented operation is not validated. @@ -38,8 +38,6 @@ internal static class ResponseSchemas @"^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}:\d{2})$", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex EmailPattern = new(@"^[^@\s]+@[^@\s]+\.[^@\s]+$", RegexOptions.Compiled | RegexOptions.CultureInvariant); - /// Bounds the time a schema's pattern may take, so a pathological pattern cannot hang a response read. private static readonly TimeSpan PatternTimeout = TimeSpan.FromSeconds(1); @@ -237,7 +235,6 @@ private static void CheckConstraints(JsonNode value, JsonObject rules, string lo { "date-time" => DateTimePattern.IsMatch(text) && DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.None, out _), "uri" => Uri.TryCreate(text, UriKind.Absolute, out _), - "email" => EmailPattern.IsMatch(text), _ => true, }; diff --git a/src/Apify.Client/Options/DatasetListItemsOptions.cs b/src/Apify.Client/Options/DatasetListItemsOptions.cs index 5b037d8..ca1adf8 100644 --- a/src/Apify.Client/Options/DatasetListItemsOptions.cs +++ b/src/Apify.Client/Options/DatasetListItemsOptions.cs @@ -12,7 +12,11 @@ public sealed class DatasetListItemsOptions /// Number of items to skip. public int? Offset { get; init; } - /// Maximum number of items to return. + /// + /// Maximum number of dataset rows to read. Filters can return fewer items than rows read and + /// more. When iterating, it caps the rows read in total across all pages; 0 + /// or unset means no cap. + /// public int? Limit { get; init; } /// diff --git a/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs b/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs index 2f42c9e..57f93a8 100644 --- a/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs +++ b/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs @@ -166,6 +166,45 @@ public async Task StoreIterationSizesPagesByChunkSizeAndCapsTotalAtLimit() Assert.Contains("limit=1", transport.Received[1].Uri, StringComparison.Ordinal); } + [Fact] + public async Task StoreIterationAdvancesByTheRowsScannedNotTheItemsReturned() + { + // The Store listing drops Actors the caller cannot run after applying offset and limit, so a page that + // scanned 3 rows can return only 2 Actors. The next page must start after all 3 rows. + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.PageWithCount(9, 3, SpecFixtures.StoreItem("a"), SpecFixtures.StoreItem("b"))) + .QueueResponse(200, SpecFixtures.Page(9)); + + await DrainAsync(Client(transport).Store().IterateAsync(new StoreListOptions { ChunkSize = 3 })); + + Assert.Contains("offset=3", transport.Received[1].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task ScannedRowsAreCappedAtTheRequestedLimit() + { + // Limit = 2 asks for 2 rows; a page claiming 5 scanned rows must not advance past what was asked for. + var transport = new MockTransport().QueueResponse(200, SpecFixtures.PageWithCount(9, 5, SpecFixtures.ActorItem("a"), SpecFixtures.ActorItem("b"))); + + await DrainAsync(Client(transport).Actors().IterateAsync(new ActorListOptions { Limit = 2 })); + + Assert.Equal(1, transport.CallCount); + } + + [Theory] + [InlineData("Offset")] + [InlineData("Limit")] + public async Task NegativeOffsetOrLimitIsRejectedBeforeAnyRequest(string option) + { + var transport = new MockTransport(); + var options = option == "Offset" ? new ActorListOptions { Offset = -1 } : new ActorListOptions { Limit = -1 }; + + var ex = await Assert.ThrowsAsync(() => DrainAsync(Client(transport).Actors().IterateAsync(options))); + + Assert.Equal(option, ex.ParamName); + Assert.Equal(0, transport.CallCount); + } + [Fact] public async Task StoreIterationLimitIsATotalCapNotAPageSize() { diff --git a/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs b/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs index 81b598a..ff39268 100644 --- a/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs +++ b/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs @@ -101,6 +101,13 @@ public void NumericBoundsAreEnforced() Assert.Equal(new[] { "$.data.options.memoryMbytes: is a value outside the documented range (maximum 32768)" }, above); } + [Fact] + public void EmailIsOnlyRequiredToBeAString() + { + // Like the reference client's schemas: an address such as user@localhost is not rejected. + Assert.Empty(Issues("GET", "/v2/users/me", "{\"data\":{\"username\":\"u\",\"email\":\"user@localhost\"}}")); + } + [Fact] public void UriFormatIsEnforced() { diff --git a/tests/Apify.Client.Tests/Unit/SpecFixtures.cs b/tests/Apify.Client.Tests/Unit/SpecFixtures.cs index 6cf053d..bea0c4c 100644 --- a/tests/Apify.Client.Tests/Unit/SpecFixtures.cs +++ b/tests/Apify.Client.Tests/Unit/SpecFixtures.cs @@ -22,6 +22,10 @@ public static string Page(int total, params string[] items) items.Length, string.Join(",", items)); + /// A paginated list response whose reported count (rows scanned) differs from the items returned. + public static string PageWithCount(int total, int count, params string[] items) + => Page(total, items).Replace("\"count\":" + items.Length.ToString(CultureInfo.InvariantCulture), "\"count\":" + count.ToString(CultureInfo.InvariantCulture), System.StringComparison.Ordinal); + /// An item of the Actor collection. public static string ActorItem(string id) => "{\"id\":\"" + id + "\",\"createdAt\":\"" + Timestamp + "\",\"modifiedAt\":\"" + Timestamp + "\",\"name\":\"n\",\"username\":\"u\"}"; diff --git a/tools/generate_response_schemas.py b/tools/generate_response_schemas.py index 361cee8..9637346 100644 --- a/tools/generate_response_schemas.py +++ b/tools/generate_response_schemas.py @@ -9,8 +9,9 @@ The default source is https://docs.apify.com/api/openapi.json. Only what validation needs is kept: types, required properties, nested properties and items, composition keywords, references, and the value -constraints (format, pattern, minimum, maximum, minLength, maxLength). Documentation, examples and keywords the client deliberately does not enforce (enum, const) are dropped, which also keeps the -embedded resource small. Enum values stay open on purpose so that a value the API adds later does not break +constraints (format, pattern, minimum, maximum, minLength, maxLength). Documentation, examples and +keywords the client deliberately does not enforce (enum, const) are dropped, which also keeps the embedded +resource small. Enum values stay open on purpose so that a value the API adds later does not break existing clients; the reference client's generated schemas do the same. """ import json From 812fefb181b40453f9dc9415d15c09a9f6320455 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 14:37:47 +0000 Subject: [PATCH 7/8] fix: Store limit counts Actors yielded, share the list page helper, condense changelog Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01UUMWnU4gH573HweHieMhHi --- CHANGELOG.md | 19 +++------ src/Apify.Client/Internal/PagedIteration.cs | 14 ++++--- src/Apify.Client/Internal/ResourceContext.cs | 21 ++++++---- src/Apify.Client/Resources/DatasetClient.cs | 2 +- .../Apify.Client.Tests/Unit/ChunkSizeTests.cs | 40 +++++++++++++++++++ 5 files changed, 70 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9d972b..2565532 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,19 +24,12 @@ request (the smaller of it and the remaining `Limit`), matching the reference client's `chunkSize`. `MaxPageLimit` stays as the older name of the same option. A non-positive `ChunkSize` throws `ArgumentOutOfRangeException`. -- Fixed `StoreCollectionClient.IterateAsync` to treat `Limit` as a cap on the total number of Actors - yielded (as every other iterator does) instead of as the page size, and made it, like - `DatasetClient.IterateItemsAsync`, keep paging until a page is empty instead of stopping at the total - an earlier page reported, so items added while iterating are no longer missed. -- Fixed the list, Store and dataset-item iterators to treat a `Limit` of `0` as "no limit" (as the API - does) instead of stopping after the first page, and made dataset-item iteration advance by the larger of - the `X-Apify-Pagination-Count` header and the returned item count (capped at the requested limit), so a - header that lags behind freshly pushed items no longer ends the iteration early. A dataset `Limit` now - caps the rows scanned. -- Fixed the list and Store iterators to advance by the rows a page scanned (its `count`) instead of the items - it returned, so the Store listing, which drops Actors the caller cannot run after applying offset and - limit, no longer repeats or skips Actors. A negative `Offset` or `Limit` now throws - `ArgumentOutOfRangeException` instead of being silently adjusted. +- Changed the offset/limit iterators (`IterateAsync`, `IterateItemsAsync`) to follow the reference client: + `Limit` caps the items yielded in total (the rows read, for dataset items), `0` means no limit, a + negative `Offset` or `Limit` throws `ArgumentOutOfRangeException`, and paging follows the rows each page + scanned (its `count`) until a page scans none, instead of the items returned or the reported total. The + Store no longer repeats or skips Actors, and `StoreCollectionClient.IterateAsync` takes `Limit` as a + total cap rather than the page size. - Nested accessors of a run (`Dataset()`, `KeyValueStore()`, `RequestQueue()`, `Log()`) and of a build (`Log()`) now inherit the run's or build's `WithTimeout` override. - Added a documentation test that compiles every C# snippet in the README and `docs/`. diff --git a/src/Apify.Client/Internal/PagedIteration.cs b/src/Apify.Client/Internal/PagedIteration.cs index 3352422..938ff76 100644 --- a/src/Apify.Client/Internal/PagedIteration.cs +++ b/src/Apify.Client/Internal/PagedIteration.cs @@ -9,8 +9,12 @@ namespace Apify.Client.Internal; /// One fetched page: its items and, when the API reports it, the number of rows it scanned. /// The items on the page. /// The rows the API scanned to produce the page (null when not reported). +/// +/// Whether the caller's total limit counts the rows scanned (dataset items) rather than the items returned +/// (listings, where the API can scan more rows than it returns, e.g. the Store dropping unrunnable Actors). +/// /// The item type. -internal readonly record struct FetchedPage(IReadOnlyList Items, long? ScannedRows); +internal readonly record struct FetchedPage(IReadOnlyList Items, long? ScannedRows, bool LimitCountsRows); /// /// The offset/limit paging loop shared by every list iterator, ported from the reference client's @@ -22,8 +26,8 @@ internal static class PagedIteration /// Lazily yields every item from on, fetching pages on demand. /// /// - /// caps the number of scanned rows (null or 0 for no cap); a - /// negative offset or limit is rejected. + /// caps the items yielded, or the rows scanned when the page says the limit + /// counts rows (null or 0 for no cap); a negative offset or limit is rejected. /// The next offset and the stop condition follow the rows a page scanned, not the items it returned: /// advancing by the item count would re-scan rows after a filter dropped some and skip rows after /// unwind multiplied them, and stopping at an empty page would end the iteration in front of rows @@ -34,7 +38,7 @@ internal static class PagedIteration /// stops once a page scans no rows, which costs one extra request at the end. /// /// The offset of the first row. - /// The most rows to scan across all pages, or null/0 for no cap. + /// The most items (or rows, see ) across all pages, or null/0 for no cap. /// The most rows to request per call, or null for the API default. /// Fetches one page given its offset and limit (null limit: leave it out). /// A token to cancel the iteration. @@ -64,7 +68,7 @@ public static async IAsyncEnumerable IterateAsync( scanned = ScannedRows(page, requestedLimit); offset += (int)scanned; - remaining -= (int?)scanned; + remaining -= (int?)(page.LimitCountsRows ? scanned : page.Items.Count); } while (scanned > 0 && (remaining is null || remaining > 0)); } diff --git a/src/Apify.Client/Internal/ResourceContext.cs b/src/Apify.Client/Internal/ResourceContext.cs index 3aaaf17..eba8b26 100644 --- a/src/Apify.Client/Internal/ResourceContext.cs +++ b/src/Apify.Client/Internal/ResourceContext.cs @@ -208,9 +208,20 @@ public async Task DeleteResourceAsync(string subPath, RequestTier tier, Cancella /// GET a paginated listing and build a with each item hydrated. public async Task> ListResourceAsync(string subPath, QueryParams p, Func hydrate, RequestTier tier, CancellationToken ct) + { + var (page, _) = await ListPageAsync(subPath, p, hydrate, tier, ct).ConfigureAwait(false); + return page; + } + + /// + /// Fetches one page of a listing together with the API's count of the rows it scanned to produce + /// the page. That can exceed the number of items returned: the Store listing drops Actors the caller + /// cannot run after applying offset and limit, so advancing by the returned items would re-read rows. + /// + private async Task<(PaginationList Page, long RowsScanned)> ListPageAsync(string subPath, QueryParams p, Func hydrate, RequestTier tier, CancellationToken ct) { var data = await GetResourceRequiredAsync(subPath, p, tier, ct).ConfigureAwait(false); - return PaginationList.FromData(data, hydrate); + return (PaginationList.FromData(data, hydrate), JsonValues.IntOr(data, "count", 0)); } /// @@ -236,12 +247,8 @@ public IAsyncEnumerable IterateListAsync( chunkSize, async (offset, pageLimit, token) => { - var data = await GetResourceRequiredAsync(subPath, baseQuery.PageQuery(offset, pageLimit), tier, token).ConfigureAwait(false); - var page = PaginationList.FromData(data, hydrate); - // `count` is the number of rows scanned. It can exceed the items returned: the Store listing - // drops Actors the caller cannot run after applying offset and limit, so advancing by the - // returned items would re-read rows. - return new FetchedPage(page.Items, JsonValues.IntOr(data, "count", 0)); + var (page, rowsScanned) = await ListPageAsync(subPath, baseQuery.PageQuery(offset, pageLimit), hydrate, tier, token).ConfigureAwait(false); + return new FetchedPage(page.Items, rowsScanned, LimitCountsRows: false); }, ct); } diff --git a/src/Apify.Client/Resources/DatasetClient.cs b/src/Apify.Client/Resources/DatasetClient.cs index eca6478..ece574a 100644 --- a/src/Apify.Client/Resources/DatasetClient.cs +++ b/src/Apify.Client/Resources/DatasetClient.cs @@ -109,7 +109,7 @@ public async Task UpdateAsync(object newFields, CancellationToken cance async (offset, pageLimit, token) => { var (page, scannedCount) = await FetchItemsPageAsync(baseQuery.PageQuery(offset, pageLimit), desc, token).ConfigureAwait(false); - return new FetchedPage(page.Items, scannedCount); + return new FetchedPage(page.Items, scannedCount, LimitCountsRows: true); }, cancellationToken); } diff --git a/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs b/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs index 57f93a8..13df3e3 100644 --- a/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs +++ b/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs @@ -180,6 +180,46 @@ public async Task StoreIterationAdvancesByTheRowsScannedNotTheItemsReturned() Assert.Contains("offset=3", transport.Received[1].Uri, StringComparison.Ordinal); } + [Fact] + public async Task StoreIterationContinuesPastAPageThatScannedRowsButReturnedNoItems() + { + // Every Actor of the first page was dropped (count 1, no items); the next page still has one. + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.PageWithCount(9, 1)) + .QueueResponse(200, SpecFixtures.Page(9, SpecFixtures.StoreItem("a"))) + .QueueResponse(200, SpecFixtures.Page(9)); + + var ids = new System.Collections.Generic.List(); + await foreach (var item in Client(transport).Store().IterateAsync(new StoreListOptions { ChunkSize = 1 })) + { + ids.Add(item.Id); + } + + Assert.Equal(new[] { "a" }, ids); + Assert.Contains("offset=1", transport.Received[1].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task StoreLimitCountsTheActorsYieldedEvenWhenAPageDropsSome() + { + // Limit = 3 asks for three Actors. The first page scanned 3 rows but returned only 2 (one was + // dropped), so one more Actor is still wanted, from the row after the 3 scanned. + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.PageWithCount(9, 3, SpecFixtures.StoreItem("a"), SpecFixtures.StoreItem("b"))) + .QueueResponse(200, SpecFixtures.Page(9, SpecFixtures.StoreItem("c"))); + + var ids = new System.Collections.Generic.List(); + await foreach (var item in Client(transport).Store().IterateAsync(new StoreListOptions { Limit = 3, ChunkSize = 3 })) + { + ids.Add(item.Id); + } + + Assert.Equal(new[] { "a", "b", "c" }, ids); + Assert.Equal(2, transport.CallCount); + Assert.Contains("offset=3", transport.Received[1].Uri, StringComparison.Ordinal); + Assert.Contains("limit=1", transport.Received[1].Uri, StringComparison.Ordinal); + } + [Fact] public async Task ScannedRowsAreCappedAtTheRequestedLimit() { From 1b442637a49f7743ee6554472a1ce1785e5c4351 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 14:45:56 +0000 Subject: [PATCH 8/8] fix: make limitCountsRows a loop parameter, test dataset limit, correct changelog Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01UUMWnU4gH573HweHieMhHi --- CHANGELOG.md | 16 ++++++++------ src/Apify.Client/Internal/PagedIteration.cs | 21 +++++++++++-------- src/Apify.Client/Internal/ResourceContext.cs | 3 ++- src/Apify.Client/Resources/DatasetClient.cs | 3 ++- .../Unit/AutoPagingTests.cs | 19 +++++++++++++++++ 5 files changed, 45 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2565532..2b3970d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,12 +24,16 @@ request (the smaller of it and the remaining `Limit`), matching the reference client's `chunkSize`. `MaxPageLimit` stays as the older name of the same option. A non-positive `ChunkSize` throws `ArgumentOutOfRangeException`. -- Changed the offset/limit iterators (`IterateAsync`, `IterateItemsAsync`) to follow the reference client: - `Limit` caps the items yielded in total (the rows read, for dataset items), `0` means no limit, a - negative `Offset` or `Limit` throws `ArgumentOutOfRangeException`, and paging follows the rows each page - scanned (its `count`) until a page scans none, instead of the items returned or the reported total. The - Store no longer repeats or skips Actors, and `StoreCollectionClient.IterateAsync` takes `Limit` as a - total cap rather than the page size. +- Changed `DatasetClient.IterateItemsAsync`: `Limit` now caps the dataset rows read instead of the items + yielded (so with `Clean`, `SkipEmpty` or `Unwind` the item count differs), matching the reference + client. Paging advances by the larger of the `X-Apify-Pagination-Count` header and the items returned, + so a header lagging behind freshly pushed items no longer ends the iteration, and it no longer stops at + the total an earlier page reported. +- Changed `StoreCollectionClient.IterateAsync`: `Limit` is now a cap on the total number of Actors rather + than the page size, and, unlike the reference client, paging advances by the response's `count`, so the + Store dropping Actors the caller cannot run no longer makes it repeat or skip Actors. +- A `Limit` of `0` now means no limit in all offset/limit iterators, and a negative `Offset` or `Limit` + throws `ArgumentOutOfRangeException` instead of being silently adjusted. - Nested accessors of a run (`Dataset()`, `KeyValueStore()`, `RequestQueue()`, `Log()`) and of a build (`Log()`) now inherit the run's or build's `WithTimeout` override. - Added a documentation test that compiles every C# snippet in the README and `docs/`. diff --git a/src/Apify.Client/Internal/PagedIteration.cs b/src/Apify.Client/Internal/PagedIteration.cs index 938ff76..4cf97a6 100644 --- a/src/Apify.Client/Internal/PagedIteration.cs +++ b/src/Apify.Client/Internal/PagedIteration.cs @@ -9,12 +9,8 @@ namespace Apify.Client.Internal; /// One fetched page: its items and, when the API reports it, the number of rows it scanned. /// The items on the page. /// The rows the API scanned to produce the page (null when not reported). -/// -/// Whether the caller's total limit counts the rows scanned (dataset items) rather than the items returned -/// (listings, where the API can scan more rows than it returns, e.g. the Store dropping unrunnable Actors). -/// /// The item type. -internal readonly record struct FetchedPage(IReadOnlyList Items, long? ScannedRows, bool LimitCountsRows); +internal readonly record struct FetchedPage(IReadOnlyList Items, long? ScannedRows); /// /// The offset/limit paging loop shared by every list iterator, ported from the reference client's @@ -26,8 +22,9 @@ internal static class PagedIteration /// Lazily yields every item from on, fetching pages on demand. /// /// - /// caps the items yielded, or the rows scanned when the page says the limit - /// counts rows (null or 0 for no cap); a negative offset or limit is rejected. + /// caps the items yielded, or the rows scanned when + /// is set (null or 0 for no cap); a negative offset or + /// limit is rejected. /// The next offset and the stop condition follow the rows a page scanned, not the items it returned: /// advancing by the item count would re-scan rows after a filter dropped some and skip rows after /// unwind multiplied them, and stopping at an empty page would end the iteration in front of rows @@ -38,7 +35,12 @@ internal static class PagedIteration /// stops once a page scans no rows, which costs one extra request at the end. /// /// The offset of the first row. - /// The most items (or rows, see ) across all pages, or null/0 for no cap. + /// The most items (or rows, see ) across all pages, or null/0 for no cap. + /// + /// Whether counts the rows scanned (dataset items) rather than the items + /// returned (listings, where the API can scan more rows than it returns, e.g. the Store dropping Actors + /// the caller cannot run). + /// /// The most rows to request per call, or null for the API default. /// Fetches one page given its offset and limit (null limit: leave it out). /// A token to cancel the iteration. @@ -47,6 +49,7 @@ public static async IAsyncEnumerable IterateAsync( int startOffset, int? totalLimit, int? chunkSize, + bool limitCountsRows, Func>> fetchPage, [EnumeratorCancellation] CancellationToken cancellationToken) { @@ -68,7 +71,7 @@ public static async IAsyncEnumerable IterateAsync( scanned = ScannedRows(page, requestedLimit); offset += (int)scanned; - remaining -= (int?)(page.LimitCountsRows ? scanned : page.Items.Count); + remaining -= (int?)(limitCountsRows ? scanned : page.Items.Count); } while (scanned > 0 && (remaining is null || remaining > 0)); } diff --git a/src/Apify.Client/Internal/ResourceContext.cs b/src/Apify.Client/Internal/ResourceContext.cs index eba8b26..2f59647 100644 --- a/src/Apify.Client/Internal/ResourceContext.cs +++ b/src/Apify.Client/Internal/ResourceContext.cs @@ -245,10 +245,11 @@ public IAsyncEnumerable IterateListAsync( startOffset, limit, chunkSize, + limitCountsRows: false, async (offset, pageLimit, token) => { var (page, rowsScanned) = await ListPageAsync(subPath, baseQuery.PageQuery(offset, pageLimit), hydrate, tier, token).ConfigureAwait(false); - return new FetchedPage(page.Items, rowsScanned, LimitCountsRows: false); + return new FetchedPage(page.Items, rowsScanned); }, ct); } diff --git a/src/Apify.Client/Resources/DatasetClient.cs b/src/Apify.Client/Resources/DatasetClient.cs index ece574a..ea41489 100644 --- a/src/Apify.Client/Resources/DatasetClient.cs +++ b/src/Apify.Client/Resources/DatasetClient.cs @@ -106,10 +106,11 @@ public async Task UpdateAsync(object newFields, CancellationToken cance options.Offset ?? 0, options.Limit, options.ChunkSize, + limitCountsRows: true, async (offset, pageLimit, token) => { var (page, scannedCount) = await FetchItemsPageAsync(baseQuery.PageQuery(offset, pageLimit), desc, token).ConfigureAwait(false); - return new FetchedPage(page.Items, scannedCount, LimitCountsRows: true); + return new FetchedPage(page.Items, scannedCount); }, cancellationToken); } diff --git a/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs index a55beca..cc59bc0 100644 --- a/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs +++ b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs @@ -229,6 +229,25 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenU Assert.Contains("offset=2", transport.Received[2].Uri, StringComparison.Ordinal); } + [Fact] + public async Task DatasetIterateItemsLimitCapsTheRowsReadNotTheItemsYielded() + { + // With a filter, a page can scan its whole limit while returning nothing. Limit = 2 allows reading 2 + // rows only, so iteration ends after this page instead of reading on until 2 items were yielded. + var scanned = new Dictionary { ["X-Apify-Pagination-Count"] = "2" }; + var transport = new MockTransport().QueueResponse(200, "[]", scanned); + + var count = 0; + await foreach (var unused in Client(transport).Dataset("ds1").IterateItemsAsync(new DatasetListItemsOptions { SkipEmpty = true, Limit = 2 })) + { + _ = unused; + count++; + } + + Assert.Equal(0, count); + Assert.Equal(1, transport.CallCount); + } + [Fact] public async Task DatasetIterateItemsDoesNotStopWhenTheScannedCountHeaderLagsBehindReturnedItems() {