diff --git a/.editorconfig b/.editorconfig index 42ce52a..a36b2b0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -12,10 +12,10 @@ indent_size = 4 # Use file-scoped namespaces throughout. csharp_style_namespace_declarations = file_scoped:warning -# CA1711: the model type `RequestQueue` deliberately mirrors the Apify API resource name (and the -# JS/PHP/Java/Go/Rust sibling clients). Renaming it away from the reserved "Queue" suffix would break -# cross-client naming consistency, which the client requirements mandate. Justified suppression. -dotnet_diagnostic.CA1711.severity = none +# CA1711: the `RequestQueue*` types deliberately mirror the Apify API resource name and the reference +# client. Renaming them away from the "Queue" suffix would break cross-client naming consistency, which +# the client requirements mandate. The rule stays on for every other suffix. +dotnet_code_quality.CA1711.allowed_suffixes = Queue [*.{csproj,props,targets}] indent_size = 2 diff --git a/.github/workflows/dotnet-integration-tests.yml b/.github/workflows/dotnet-integration-tests.yml index 2e80494..db93a32 100644 --- a/.github/workflows/dotnet-integration-tests.yml +++ b/.github/workflows/dotnet-integration-tests.yml @@ -10,6 +10,9 @@ on: - '**/*.cs' - '**/*.csproj' - '**/*.sln' + # The embedded response schemas change the validation of every response, and the script generates them. + - 'src/Apify.Client/Internal/ResponseSchemas.json' + - 'tools/**' - 'Directory.Build.props' - '.editorconfig' # The "Test examples" step validates the in-documentation snippets, so doc changes must re-run @@ -69,8 +72,8 @@ jobs: APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} run: dotnet test --configuration Release --no-build --filter Category=Integration - # Standalone CI step that verifies the documentation examples work end-to-end against the live - # API (the Examples suite runs each example). + # Standalone CI step that verifies the documentation examples: the Examples suite runs each example + # end-to-end against the live API and compiles every C# snippet in the README and docs/. - name: Test examples env: APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 8406474..2b3970d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,51 @@ # Changelog +## 0.7.0 + +- Bumped `ApifyClientVersion.ApiSpecVersion` to the Apify OpenAPI spec `v2-2026-10-20T121847Z` and + the project version to `0.7.0`. +- Added `RequestId` to `RunChargeOptions`, sent as `requestId` in the charge request body; it identifies + the Server Actor request being charged (required for multi-tenant Server Actor runs). +- Documented that the rental pricing model (`FLAT_PRICE_PER_MONTH`) is no longer supported and is not an + accepted `StoreListOptions.PricingModel` filter value. +- Added `InvalidResponseBodyException` (an `ApifyClientException`), thrown when a successful response's + body is not valid JSON, instead of a raw `System.Text.Json.JsonException`. It is not retried and exposes + the status code, HTTP method, path and raw body. An error status with an unparsable body keeps throwing + the `ApifyApiException` for its status code. +- Added `ResponseValidationException` (an `ApifyClientException`), thrown when a successful response does + not match the schema the OpenAPI specification declares for its operation (missing `data` envelope, + missing required properties, wrong JSON types, `null` where it is not allowed, a value outside the + documented format, pattern or range), instead of silently returning `null` or an empty object. The + exception lists the problems in `Issues`. Unknown properties and enum values are accepted. The schemas + are embedded and generated by `tools/generate_response_schemas.py`. Matches the reference client's + `ResponseValidationError`. +- Added `ChunkSize` to `ListOptions`, `StorageListOptions`, `ActorListOptions`, `StoreListOptions`, + `DatasetListItemsOptions`, `IterateKeysOptions` and `PaginateRequestsOptions`: the page size iterators + request (the smaller of it and the remaining `Limit`), matching the reference client's `chunkSize`. + `MaxPageLimit` stays as the older name of the same option. A non-positive `ChunkSize` throws + `ArgumentOutOfRangeException`. +- Changed `DatasetClient.IterateItemsAsync`: `Limit` now caps the dataset rows read instead of the items + yielded (so with `Clean`, `SkipEmpty` or `Unwind` the item count differs), matching the reference + client. Paging advances by the larger of the `X-Apify-Pagination-Count` header and the items returned, + so a header lagging behind freshly pushed items no longer ends the iteration, and it no longer stops at + the total an earlier page reported. +- Changed `StoreCollectionClient.IterateAsync`: `Limit` is now a cap on the total number of Actors rather + than the page size, and, unlike the reference client, paging advances by the response's `count`, so the + Store dropping Actors the caller cannot run no longer makes it repeat or skip Actors. +- A `Limit` of `0` now means no limit in all offset/limit iterators, and a negative `Offset` or `Limit` + throws `ArgumentOutOfRangeException` instead of being silently adjusted. +- Nested accessors of a run (`Dataset()`, `KeyValueStore()`, `RequestQueue()`, `Log()`) and of a build + (`Log()`) now inherit the run's or build's `WithTimeout` override. +- Added a documentation test that compiles every C# snippet in the README and `docs/`. +- Added `RunClient.IterateDatasetItemsAsync` (with `RunIterateDatasetItemsOptions`), which iterates a run's + default dataset while the run is still producing items, matching the reference client's + `iterateDatasetItems()`. +- Fixed the webhook integration tests: the API now requires a webhook condition to reference an existing + Actor ID, so they use the ID of the public `apify/hello-world` Actor instead of a made-up run ID. +- Added integration tests for Actor and task webhook listing, request-queue request update/list and + dataset item iteration, and narrowed the CA1711 analyzer suppression to the `Queue` suffix. +- Removed the experimental/AI notice from the NuGet package description (it stays in the top-level README). + ## 0.6.0 - Bumped `ApifyClientVersion.ApiSpecVersion` to the Apify OpenAPI spec `v2-2026-10-19T133704Z` and diff --git a/docs/README.md b/docs/README.md index 81a0b60..26a22b9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -74,7 +74,7 @@ the `using` directives for whichever ones a file references: | `Apify.Client.Resources` | Every resource client the entry point returns — `ActorClient`, `RunClient`, `BuildClient`, `DatasetClient`, `KeyValueStoreClient`, `RequestQueueClient`, `TaskClient`, `ScheduleClient`, `LogClient`, `UserClient`, `ActorVersionClient`, `ActorEnvVarClient`, the `…CollectionClient` types (including `ActorVersionCollectionClient`, `ActorEnvVarCollectionClient`, `NestedWebhookCollectionClient`, and `WebhookDispatchCollectionClient`), etc. | | `Apify.Client.Models` | Data models returned by the clients — `Actor`, `ActorRun`, `Build`, `Dataset`, `RequestQueueRequest`, `ActorEnvVar`, `PaginationList`, and so on. | | `Apify.Client.Options` | The option/request objects passed into methods — `ActorStartOptions`, `ActorBuildOptions`, `ActorListOptions`, `RunListOptions`, `RunResurrectOptions`, `RunChargeOptions`, `MetamorphOptions`, `LastRunOptions`, `DatasetListItemsOptions`, `DatasetDownloadOptions`, `DownloadItemsFormat`, `GetRecordOptions`, `SetRecordOptions`, `ListKeysOptions`, `IterateKeysOptions`, `ListRequestsOptions`, `PaginateRequestsOptions`, `BatchAddRequestsOptions`, `RequestQueueClientOptions`, `TaskStartOptions`, `ValidateInputOptions`, `LogOptions`, `ListOptions`, `StorageListOptions`, `StoreListOptions`. | -| `Apify.Client.Exceptions` | `ApifyClientException` (common base), `ApifyApiException` (and its per-status subclasses) and `ApifyTransportException`. | +| `Apify.Client.Exceptions` | `ApifyClientException` (common base), `ApifyApiException` (and its per-status subclasses), `ApifyTransportException`, `InvalidResponseBodyException` and `ResponseValidationException`. | | `Apify.Client.Http` | The replaceable transport: `IHttpTransport` and the default `HttpClientTransport`. | | `System.Text.Json.Nodes` (BCL) | Not an Apify namespace, but required whenever you name the JSON escape-hatch types the client returns/accepts — `JsonObject`/`JsonNode` (e.g. dataset items, `GetInputAsync`/`UpdateInputAsync`, `GetStatisticsAsync`, `GetOpenApiDefinitionAsync`, `MonthlyUsageAsync`/`LimitsAsync`, `RequestQueueRequest.UserData`). Add `using System.Text.Json.Nodes;`. | @@ -205,11 +205,30 @@ catch (ApifyApiException e) `ApifyTransportException` is thrown instead when the request never reaches the API (network failure, timeout, or DNS error) after all retries are exhausted. -Both `ApifyApiException` and `ApifyTransportException` extend the common `ApifyClientException`, so -`catch (ApifyClientException)` catches every error this client raises for a failed call without naming -either concrete type: +`InvalidResponseBodyException` is thrown when a successful response's body is not valid JSON, and +`ResponseValidationException` when it is valid JSON but does not match the schema the API specification +declares for that operation (for example the `data` envelope or a required property is missing, or a +property has the wrong JSON type). Unknown properties and enum values are accepted, so new API fields do +not break the client. The response arrived in full, so neither is +retried. Both expose `StatusCode`, `HttpMethod`, `Path` and the raw `Body`. An error status with an +unparsable body throws the `ApifyApiException` for its status code, with the raw body in its message. + +`ApifyApiException`, `ApifyTransportException`, `InvalidResponseBodyException` and +`ResponseValidationException` extend the common +`ApifyClientException`, so `catch (ApifyClientException)` catches every error this client raises for a +failed call without naming the concrete types: ```csharp +using System; +using Apify.Client; +using Apify.Client.Exceptions; + +var client = new ApifyClient("my-api-token"); +try +{ + var user = await client.Me().GetAsync(); + Console.WriteLine(user?.Username); +} catch (ApifyClientException e) { Console.WriteLine("client call failed: " + e.Message); diff --git a/docs/actors.md b/docs/actors.md index 5d5f695..c55ee7c 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -20,6 +20,7 @@ Access the Actor collection with `client.Actors()` and a specific Actor with `cl | `Desc` | `bool?` | Sort newest-first when `true`. | | `My` | `bool?` | Return only Actors owned by the current account when `true`. | | `SortBy` | `string?` | Field to sort by (e.g. `createdAt`, `lastRunStartedAt`). | +| `ChunkSize` | `int?` | Page size used by `IterateAsync` (positive; the smaller of this and the remaining `Limit` is requested per call). Never sent to the API. | ```csharp using System; diff --git a/docs/builds.md b/docs/builds.md index cfb39c7..33fb81f 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -5,7 +5,7 @@ Access the account-wide build collection with `client.Builds()`, an Actor's buil ## Collection -- `ListAsync(ListOptions? options = null)` → `PaginationList` (`Offset`, `Limit`, `Desc`). +- `ListAsync(ListOptions? options = null)` → `PaginationList` (`Offset`, `Limit`, `Desc`; `ChunkSize` sizes the pages of `IterateAsync`). - `IterateAsync(ListOptions? options = null)` → `IAsyncEnumerable` (lazy, all pages). ## Single build — `client.Build(buildId)` diff --git a/docs/misc.md b/docs/misc.md index 3081f9d..8871884 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -9,19 +9,20 @@ Browse public Actors in the [Apify Store](https://apify.com/store). - `ListAsync(StoreListOptions? options = null)` → `PaginationList` (one page). - `IterateAsync(StoreListOptions? options = null)` → `IAsyncEnumerable` (lazy, all - pages; `Limit` is the page size). + pages; `Limit` caps the total number of Actors yielded and `ChunkSize` sets the page size). `StoreListOptions` fields: | Field | Type | Description | |---|---|---| | `Offset` | `int?` | Number of Actors to skip from the start. | -| `Limit` | `int?` | Maximum number of Actors to return in the page. | +| `Limit` | `int?` | Maximum number of Actors to return (for `IterateAsync`, in total across all pages). | +| `ChunkSize` | `int?` | Page size used by `IterateAsync` (positive; the smaller of this and the remaining `Limit` is requested per call). Never sent to the API. | | `Search` | `string?` | Full-text search string to filter Actors by. | | `SortBy` | `string?` | Field to sort by (e.g. `popularity`, `newest`). | | `Category` | `string?` | Restrict results to a Store category. | | `Username` | `string?` | Restrict results to a given owner's Actors. | -| `PricingModel` | `string?` | Filter by pricing model (`FREE`, `FLAT_PRICE_PER_MONTH`, `PRICE_PER_DATASET_ITEM`, …). | +| `PricingModel` | `string?` | Filter by pricing model (`FREE`, `PRICE_PER_DATASET_ITEM`, `PAY_PER_EVENT`; the rental model `FLAT_PRICE_PER_MONTH` is no longer supported). | | `IncludeUnrunnableActors` | `bool?` | Include Actors that cannot currently be run. | | `AllowsAgenticUsers` | `bool?` | Only Actors that permit agentic (automated) users. | | `ResponseFormat` | `string?` | Requested response format. | diff --git a/docs/runs.md b/docs/runs.md index 75be828..edfbacd 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -9,7 +9,7 @@ Access the account-wide run collection with `client.Runs()`, an Actor's or task' - `IterateAsync(ListOptions? options = null, RunListOptions? filter = null)` → `IAsyncEnumerable` (lazy, all pages). -`ListOptions` fields: `Offset`, `Limit`, `Desc` (standard pagination). `RunListOptions` fields: +`ListOptions` fields: `Offset`, `Limit`, `Desc` (standard pagination; `Limit` `0` or unset means no cap) and `ChunkSize` (the page size `IterateAsync` requests; the smaller of it and the remaining `Limit` is used per call). `RunListOptions` fields: `Status` (`IReadOnlyList?`, filter by one or more run statuses such as `SUCCEEDED`/`RUNNING`), `StartedAfter` and `StartedBefore` (ISO 8601 bounds, honoured only by the Actor- and task-scoped run collections). @@ -27,6 +27,10 @@ collections). - `WaitForFinishAsync(int? waitSecs = null)` → `ActorRun` — client-side polling until terminal (`null` waits indefinitely). A run id that keeps answering 404 for 3 seconds straight (replica lag resolves faster than that) throws `NotFoundException` instead of polling for the rest of `waitSecs`. +- `IterateDatasetItemsAsync(RunIterateDatasetItemsOptions? options = null)` → `IAsyncEnumerable` — + lazily iterates the run's default dataset while the run is still producing items: it reads the rows + below the dataset's `itemCount`, waits for the run to finish between polls, and after the run finishes + reads any remaining rows. On a `LastRun()` client it sticks to the run its first request resolved to. - `Dataset()`, `KeyValueStore()`, `RequestQueue()` — the run's default storages. Their own `GetAsync()`/`DeleteAsync()` throw on a 404 instead of resolving `null`/no-op, since the run itself (not just the storage) could be what's missing; a record/request looked up by its own key or id @@ -45,6 +49,26 @@ collections). `MemoryMbytes` (`int?`), `TimeoutSecs` (`int?`), `MaxItems` (`int?`), `MaxTotalChargeUsd` (`double?`), and `RestartOnError` (`bool?`). See [Actors](actors.md) for each field's meaning. +`RunIterateDatasetItemsOptions` fields (all optional): `Offset` (rows to skip), `Limit` (maximum rows to +scan; `null` or `0` for no limit), `ChunkSize` (rows per API call, default 1000), `PollIntervalSecs` +(seconds to wait for the run to finish between polls, default 5), and the item filters `Clean`, `Fields`, +`Omit`, `Unwind`, `SkipEmpty` and `SkipHidden` (same meaning as in `DatasetListItemsOptions`). + +```csharp +using System; +using Apify.Client; +using Apify.Client.Options; + +var client = new ApifyClient("my-api-token"); +var run = await client.Actor("apify/hello-world").StartAsync(); + +// Items are yielded as the run produces them, until the run has finished and its dataset is drained. +await foreach (var item in client.Run(run.Id!).IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { ChunkSize = 100 })) +{ + Console.WriteLine(item); +} +``` + `RunChargeOptions` describes a pay-per-event charge and is built via its constructor `RunChargeOptions(string eventName, int? count = null, string? idempotencyKey = null)`: @@ -53,6 +77,7 @@ and `RestartOnError` (`bool?`). See [Actors](actors.md) for each field's meaning | `EventName` | `string` | Name of the pay-per-event event to charge for (required, non-empty). | | `Count` | `int?` | Number of event occurrences to charge (defaults to 1 server-side). | | `IdempotencyKey` | `string?` | Key that deduplicates retried charges; auto-generated when omitted. | +| `RequestId` | `string?` | ID of the Server Actor request being charged (the `X-Actor-Request-ID` header of the incoming request); required for multi-tenant Server Actor runs. Set via object initializer. | ```csharp using Apify.Client; diff --git a/docs/storages.md b/docs/storages.md index 890fe30..b73b2cd 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -19,6 +19,7 @@ reached from a run (`client.Run(id).Dataset()`, `.KeyValueStore()`, `.RequestQue | `Desc` | `bool?` | Sort newest-first when `true`. | | `Unnamed` | `bool?` | Include unnamed storages when `true` (they are excluded by default). | | `Ownership` | `string?` | Filter by ownership (e.g. only storages owned by the current account). | +| `ChunkSize` | `int?` | Page size used by `IterateAsync` (positive; the smaller of this and the remaining `Limit` is requested per call). Never sent to the API. | ## Datasets @@ -39,7 +40,9 @@ reached from a run (`client.Run(id).Dataset()`, `.KeyValueStore()`, `.RequestQue - `GetStatisticsAsync()` → `JsonObject` (throws if the dataset does not exist). - `CreateItemsPublicUrlAsync(DatasetListItemsOptions? options = null, int? expiresInSecs = null, DownloadItemsFormat? format = null)` → `string` (a signed public URL; `format` defaults to `Json`). -`DatasetListItemsOptions` selects and reshapes items: `Offset`/`Limit` (pagination), `Desc` (reverse +`DatasetListItemsOptions` selects and reshapes items: `Offset`/`Limit` (pagination: `Limit` is the number of +dataset rows read, in total across pages when iterating; `0` or unset means no cap), `ChunkSize` (the page +size `IterateItemsAsync` requests, never sent to the API), `Desc` (reverse order), `Fields`/`OutputFields`/`Omit` (choose columns), `Unwind`/`Flatten` (restructure nested fields), `Clean`/`SkipEmpty`/`SkipHidden`/`SkipFailedPages` (drop unwanted rows), `Simplified`, `View`, and `Signature` (for signed public URLs). @@ -99,7 +102,7 @@ await csvStream.CopyToAsync(file); - `GetAsync()` → `KeyValueStore?`; `UpdateAsync(object newFields)` → `KeyValueStore`; `DeleteAsync()`. - `ListKeysAsync(ListKeysOptions? options = null)` → `KeyValueStoreKeysPage` (one page). - `IterateKeysAsync(IterateKeysOptions? options = null)` → `IAsyncEnumerable` — lazily - iterate every key across pages, fetching each page (sized by `MaxPageLimit`) on demand. + iterate every key across pages, fetching each page (sized by `ChunkSize`) on demand. - `RecordExistsAsync(string key)` → `bool`. - `GetRecordAsync(string key, GetRecordOptions? options = null)` → `KeyValueStoreRecord?`. `KeyValueStoreRecord.Value` is a `byte[]` of the record's raw bytes (so binary records survive intact); @@ -114,8 +117,8 @@ await csvStream.CopyToAsync(file); `ListKeysOptions` fields: `Limit` (page size), `ExclusiveStartKey` (start after this key), `Prefix` (only keys with this prefix), `Collection` (a named record collection), and `Signature` (for signed public URLs). `IterateKeysOptions` fields: `Limit` (total across all pages), -`MaxPageLimit` (page size; client-side only, never sent to the API — defaults to 1000), -`ExclusiveStartKey`, `Collection`, and `Prefix`. `GetRecordOptions` fields: `Attachment` (request a +`ChunkSize` (page size; client-side only, never sent to the API — defaults to 1000; `MaxPageLimit` is its +older name, and `ChunkSize` wins if both are set), `ExclusiveStartKey`, `Collection`, and `Prefix`. `GetRecordOptions` fields: `Attachment` (request a download disposition) and `Signature`. `SetRecordOptions` fields: `TimeoutSecs`, `DoNotRetryTimeouts`, and `ContentEncoding` (upload a value that is already compressed — see below). @@ -210,7 +213,7 @@ leaves unprocessed), `MaxParallel` (how many chunks are sent concurrently), and `MinDelayBetweenUnprocessedRequestsRetriesMillis` (backoff before retrying unprocessed requests). `ListRequestsOptions`/`PaginateRequestsOptions` fields: `Limit`, `ExclusiveStartId`, `Cursor`, and `Filter` (an `IReadOnlyList?` — one or more of `"pending"`/`"locked"`, so several states can be -requested at once); `PaginateRequestsOptions` also has `MaxPageLimit`. +requested at once); `PaginateRequestsOptions` also has `ChunkSize` (page size, defaults to 1000; `MaxPageLimit` is its older name). ```csharp using System; diff --git a/docs/tasks.md b/docs/tasks.md index 1677ce5..ec2e01f 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -24,8 +24,8 @@ a specific task with `client.Task(id)`. - `StartAsync(object? input = null, TaskStartOptions? options = null)` → `ActorRun`. - `CallAsync(object? input = null, TaskStartOptions? options = null, int? waitSecs = null, Action? log = null)` → `ActorRun` (`log`, if set, redirects the run's live log to that sink for the duration of the wait). -- `GetInputAsync()` → `JsonNode?` (throws if the task does not exist; a JSON `null` means the task has - no stored input) / `UpdateInputAsync(object input)` → `JsonNode?`. +- `GetInputAsync()` → `JsonNode?` (throws if the task does not exist; a task without stored input + yields an empty JSON object) / `UpdateInputAsync(object input)` → `JsonNode?`. - `LastRun(LastRunOptions? options = null)` → `RunClient`; `Runs()` → `RunCollectionClient`. - `Webhooks()` → read-only `NestedWebhookCollectionClient`. diff --git a/docs/webhooks.md b/docs/webhooks.md index 11a2aa6..6243eb8 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -26,10 +26,12 @@ using Apify.Client; using Apify.Client.Options; var client = new ApifyClient("my-api-token"); +// The webhook condition needs the Actor's ID, not its "username/name" slug. +var actor = await client.Actor("apify/hello-world").GetAsync(); var webhook = await client.Webhooks().CreateAsync(new { eventTypes = new[] { "ACTOR.RUN.SUCCEEDED" }, - condition = new { actorId = "apify/hello-world" }, + condition = new { actorId = actor!.Id }, requestUrl = "https://example.com/webhook", }); diff --git a/src/Apify.Client/Apify.Client.csproj b/src/Apify.Client/Apify.Client.csproj index 2c789f3..2776f9c 100644 --- a/src/Apify.Client/Apify.Client.csproj +++ b/src/Apify.Client/Apify.Client.csproj @@ -6,11 +6,11 @@ Apify.Client - 0.6.0 + 0.7.0 Apify Apify Apify API client for .NET - Official, experimental (AI-generated and AI-maintained) Apify API client for .NET: run Actors, manage storages (datasets, key-value stores, request queues), schedules, webhooks and more. + Apify API client for .NET: run Actors, manage storages (datasets, key-value stores, request queues), schedules, webhooks and more. apify;api;client;actor;scraping;crawler;automation Apache-2.0 https://github.com/apify/apify-client-dotnet @@ -25,6 +25,11 @@ + + + + + diff --git a/src/Apify.Client/ApifyClient.cs b/src/Apify.Client/ApifyClient.cs index 5450648..a3ff367 100644 --- a/src/Apify.Client/ApifyClient.cs +++ b/src/Apify.Client/ApifyClient.cs @@ -141,7 +141,7 @@ private static string ToApiBaseUrl(string url) /// A client for a specific Actor run. Must not be empty. /// The run ID. - public RunClient Run(string id) => new(_http, _baseUrl, "actor-runs", RequireNonEmptyId(id, nameof(id))); + public RunClient Run(string id) => new(this, _http, _baseUrl, "actor-runs", RequireNonEmptyId(id, nameof(id))); // ----- Dataset accessors --------------------------------------------------- diff --git a/src/Apify.Client/ApifyClientVersion.cs b/src/Apify.Client/ApifyClientVersion.cs index 4fea34c..3e1945e 100644 --- a/src/Apify.Client/ApifyClientVersion.cs +++ b/src/Apify.Client/ApifyClientVersion.cs @@ -14,11 +14,11 @@ public static class ApifyClientVersion /// The semantic version of this client library (see https://semver.org/). Changes to the public /// interface other than additive ones are considered breaking changes. /// - public const string ClientVersion = "0.6.0"; + public const string ClientVersion = "0.7.0"; /// /// The version of the Apify OpenAPI specification this client was generated and verified against. /// Corresponds to the info.version field of the Apify OpenAPI document. /// - public const string ApiSpecVersion = "v2-2026-10-19T133704Z"; + public const string ApiSpecVersion = "v2-2026-10-20T121847Z"; } diff --git a/src/Apify.Client/Exceptions/ApifyClientException.cs b/src/Apify.Client/Exceptions/ApifyClientException.cs index 8a12787..9a65b6a 100644 --- a/src/Apify.Client/Exceptions/ApifyClientException.cs +++ b/src/Apify.Client/Exceptions/ApifyClientException.cs @@ -4,7 +4,8 @@ namespace Apify.Client.Exceptions; /// /// Base class of every error this client throws for a failed API call: -/// (and its per-status subclasses) and . A single +/// (and its per-status subclasses), , +/// and . A single /// catch (ApifyClientException) catches all of them, matching the reference client's /// ApifyClientError. /// diff --git a/src/Apify.Client/Exceptions/InvalidResponseBodyException.cs b/src/Apify.Client/Exceptions/InvalidResponseBodyException.cs new file mode 100644 index 0000000..712455d --- /dev/null +++ b/src/Apify.Client/Exceptions/InvalidResponseBodyException.cs @@ -0,0 +1,40 @@ +using System; + +namespace Apify.Client.Exceptions; + +/// +/// Thrown when the body of a successful response does not parse as JSON, for example a proxy's HTML page +/// served with a success status. The response arrived in full, so the request is not retried. An error +/// status with an unparsable body throws the for its status code instead. +/// Matches the reference client's InvalidResponseBodyError. +/// +public sealed class InvalidResponseBodyException : ApifyClientException +{ + /// Creates an invalid-response-body exception. + /// A description of the failure. + /// The HTTP status code of the response. + /// The HTTP method of the request that produced the response. + /// The API endpoint path of the request (URL excluding origin). + /// The raw response body that failed to parse. + /// The JSON parse error, if any. + public InvalidResponseBodyException(string message, int statusCode, string httpMethod, string path, string body, Exception? innerException = null) + : base(message, innerException) + { + StatusCode = statusCode; + HttpMethod = httpMethod; + Path = path; + Body = body; + } + + /// The HTTP status code of the response. + public int StatusCode { get; } + + /// The HTTP method of the request that produced the response (e.g. GET). + public string HttpMethod { get; } + + /// The API endpoint path of the request (URL excluding origin). + public string Path { get; } + + /// The raw response body that failed to parse. + public string Body { get; } +} diff --git a/src/Apify.Client/Exceptions/ResponseValidationException.cs b/src/Apify.Client/Exceptions/ResponseValidationException.cs new file mode 100644 index 0000000..1b37d0d --- /dev/null +++ b/src/Apify.Client/Exceptions/ResponseValidationException.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; + +namespace Apify.Client.Exceptions; + +/// +/// Thrown when a successful response is valid JSON but does not match the schema the Apify API +/// specification declares for its operation: the data envelope or a required property is missing, a +/// value has the wrong JSON type, is null where that is not allowed, or falls outside the documented +/// format, pattern or range. Unknown properties and unknown enum values are accepted. The response arrived +/// in full, so the request is not retried. Matches the reference client's ResponseValidationError. +/// +public sealed class ResponseValidationException : ApifyClientException +{ + /// Creates a response-validation exception. + /// Each problem found, as a location in the body and what is wrong with it. + /// The HTTP status code of the response. + /// The HTTP method of the request that produced the response. + /// The API endpoint path of the request (URL excluding origin). + /// The raw response body that failed validation. + public ResponseValidationException(IReadOnlyList issues, int statusCode, string httpMethod, string path, string body) + : base("response does not match the API specification: " + string.Join("; ", issues)) + { + Issues = issues; + StatusCode = statusCode; + HttpMethod = httpMethod; + Path = path; + Body = body; + } + + /// Each problem found, as a location in the body (e.g. $.data.id) and what is wrong with it. + public IReadOnlyList Issues { get; } + + /// The HTTP status code of the response. + public int StatusCode { get; } + + /// The HTTP method of the request that produced the response (e.g. GET). + public string HttpMethod { get; } + + /// The API endpoint path of the request (URL excluding origin). + public string Path { get; } + + /// The raw response body that failed validation. + public string Body { get; } +} diff --git a/src/Apify.Client/Internal/HttpClientCore.cs b/src/Apify.Client/Internal/HttpClientCore.cs index ce7c851..9bb4858 100644 --- a/src/Apify.Client/Internal/HttpClientCore.cs +++ b/src/Apify.Client/Internal/HttpClientCore.cs @@ -169,6 +169,9 @@ public async Task CallAsync( var status = (int)response.StatusCode; if (status < MaxSuccessStatus) { + // The sent request is disposed by now; keep a body-less copy of it so response readers can + // report the method and path in the errors they raise. + response.RequestMessage = new HttpRequestMessage(method, url); return response; } diff --git a/src/Apify.Client/Internal/Json.cs b/src/Apify.Client/Internal/Json.cs index ddd8f7f..89e0764 100644 --- a/src/Apify.Client/Internal/Json.cs +++ b/src/Apify.Client/Internal/Json.cs @@ -25,7 +25,11 @@ internal static class Json public static string Encode(JsonNode? value) => value?.ToJsonString(SerializerOptions) ?? "null"; - /// Decodes a JSON string into a (null for an empty body). + /// + /// Decodes a JSON string into a (null for an empty body). Throws + /// when it is not valid JSON; response bodies go through + /// , which turns that into an . + /// public static JsonNode? Decode(string body) { if (string.IsNullOrEmpty(body)) @@ -36,20 +40,6 @@ public static string Encode(JsonNode? value) => return JsonNode.Parse(body); } - /// - /// Decodes a JSON response body wrapped in a {"data": ...} envelope, returning the unwrapped - /// data value (or null if it is absent/null). - /// - public static JsonNode? DecodeData(string body) - { - if (Decode(body) is JsonObject obj && obj.TryGetPropertyValue("data", out var data)) - { - return data; - } - - return null; - } - /// Attempts to decode a body, returning null on any parse error. public static JsonNode? TryDecode(string body) { diff --git a/src/Apify.Client/Internal/PageSizing.cs b/src/Apify.Client/Internal/PageSizing.cs new file mode 100644 index 0000000..a438bd4 --- /dev/null +++ b/src/Apify.Client/Internal/PageSizing.cs @@ -0,0 +1,39 @@ +using System; + +namespace Apify.Client.Internal; + +/// +/// Shared handling of the client-side ChunkSize iteration option: the maximum number of items +/// requested per API call while iterating across pages. It never reaches the query string itself; it only +/// sizes each page's limit, matching the reference client's chunkSize. +/// +internal static class PageSizing +{ + /// Rejects a zero or negative chunk size, which the reference client's options schema also refuses. + /// The chunk size from the caller's options, or null when unset. + /// The option's public name, used in the error message. + public static void ValidateChunkSize(int? chunkSize, string optionName) + { + if (chunkSize is <= 0) + { + throw new ArgumentOutOfRangeException(optionName, chunkSize, optionName + " must be positive."); + } + } + + /// + /// The smaller of two page limits, for the limit query parameter. The API reads a limit of + /// 0 as unset, so 0 (like null) means "no limit" here and loses to any other value; + /// the result is null when neither is set. Matches the reference client's minForLimitParam. + /// + public static int? MinForLimit(int? a, int? b) + { + a = a is 0 ? null : a; + b = b is 0 ? null : b; + if (a is null) + { + return b; + } + + return b is null ? a : Math.Min(a.Value, b.Value); + } +} diff --git a/src/Apify.Client/Internal/PagedIteration.cs b/src/Apify.Client/Internal/PagedIteration.cs new file mode 100644 index 0000000..4cf97a6 --- /dev/null +++ b/src/Apify.Client/Internal/PagedIteration.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Apify.Client.Internal; + +/// One fetched page: its items and, when the API reports it, the number of rows it scanned. +/// The items on the page. +/// The rows the API scanned to produce the page (null when not reported). +/// The item type. +internal readonly record struct FetchedPage(IReadOnlyList Items, long? ScannedRows); + +/// +/// The offset/limit paging loop shared by every list iterator, ported from the reference client's +/// listPaginatedFromCallback. +/// +internal static class PagedIteration +{ + /// + /// Lazily yields every item from on, fetching pages on demand. + /// + /// + /// caps the items yielded, or the rows scanned when + /// is set (null or 0 for no cap); a negative offset or + /// limit is rejected. + /// The next offset and the stop condition follow the rows a page scanned, not the items it returned: + /// advancing by the item count would re-scan rows after a filter dropped some and skip rows after + /// unwind multiplied them, and stopping at an empty page would end the iteration in front of rows + /// a filter hid. The scanned count is derived from a dataset's item count, which lags a fresh push, so + /// the larger of it and the item count is used, capped at the rows the request asked for so an unwound + /// page does not advance past rows the next request would then never read. The total a page reports is + /// never consulted, because it changes when the listed resource grows during the iteration; iteration + /// stops once a page scans no rows, which costs one extra request at the end. + /// + /// The offset of the first row. + /// The most items (or rows, see ) across all pages, or null/0 for no cap. + /// + /// Whether counts the rows scanned (dataset items) rather than the items + /// returned (listings, where the API can scan more rows than it returns, e.g. the Store dropping Actors + /// the caller cannot run). + /// + /// The most rows to request per call, or null for the API default. + /// Fetches one page given its offset and limit (null limit: leave it out). + /// A token to cancel the iteration. + /// The item type. + public static async IAsyncEnumerable IterateAsync( + int startOffset, + int? totalLimit, + int? chunkSize, + bool limitCountsRows, + Func>> fetchPage, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + PageSizing.ValidateChunkSize(chunkSize, "ChunkSize"); + ArgumentOutOfRangeException.ThrowIfNegative(startOffset, "Offset"); + ArgumentOutOfRangeException.ThrowIfNegative(totalLimit ?? 0, "Limit"); + var offset = startOffset; + int? remaining = totalLimit is > 0 ? totalLimit : null; + + long scanned; + do + { + var requestedLimit = PageSizing.MinForLimit(remaining, chunkSize); + var page = await fetchPage(offset, requestedLimit, cancellationToken).ConfigureAwait(false); + foreach (var item in page.Items) + { + yield return item; + } + + scanned = ScannedRows(page, requestedLimit); + offset += (int)scanned; + remaining -= (int?)(limitCountsRows ? scanned : page.Items.Count); + } + while (scanned > 0 && (remaining is null || remaining > 0)); + } + + private static long ScannedRows(FetchedPage page, int? requestedLimit) + { + var scanned = Math.Max(page.ScannedRows ?? 0, page.Items.Count); + return requestedLimit is > 0 ? Math.Min(scanned, requestedLimit.Value) : scanned; + } +} diff --git a/src/Apify.Client/Internal/QueryParams.cs b/src/Apify.Client/Internal/QueryParams.cs index 796a529..6307fa7 100644 --- a/src/Apify.Client/Internal/QueryParams.cs +++ b/src/Apify.Client/Internal/QueryParams.cs @@ -92,6 +92,23 @@ public QueryParams Set(string key, long value) return this; } + /// + /// Returns a copy with the page's offset and limit set, the latter left out when + /// is null (so a caller's own limit does not leak into every page). + /// + public QueryParams PageQuery(int offset, int? limit) + { + var page = Copy().Set("offset", offset).Remove("limit"); + return limit is null ? page : page.Set("limit", limit.Value); + } + + /// Removes every occurrence of . + public QueryParams Remove(string key) + { + _pairs.RemoveAll(pair => string.Equals(pair.Key, key, StringComparison.Ordinal)); + return this; + } + /// Whether no parameters have been added. public bool IsEmpty => _pairs.Count == 0; diff --git a/src/Apify.Client/Internal/ResourceContext.cs b/src/Apify.Client/Internal/ResourceContext.cs index d5398fe..2f59647 100644 --- a/src/Apify.Client/Internal/ResourceContext.cs +++ b/src/Apify.Client/Internal/ResourceContext.cs @@ -148,7 +148,7 @@ public string PublicUrl(string subPath) /// resource or the nested one is missing) and propagates instead. A non-empty /// (e.g. a request looked up by its own id) is never ambiguous regardless of how this context was built. /// - public async Task GetResourceAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) + public async Task GetResourceAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) { if (_ambiguousNotFound && subPath.Length == 0) { @@ -165,13 +165,12 @@ public string PublicUrl(string subPath) } } - /// GET a single resource, returning its decoded data (propagates errors). - public async Task GetResourceRequiredAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) + /// GET a single resource, returning its decoded data object (propagates errors). + public async Task GetResourceRequiredAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) { var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); using var response = await Http.CallAsync(HttpMethod.Get, url, timeout: _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); - var body = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); - return Json.DecodeData(body); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); } /// PUT to update a resource with a JSON-serializable body, returning the decoded data. @@ -179,7 +178,7 @@ public async Task UpdateResourceAsync(string subPath, object? body, { var url = MergedParams(new QueryParams()).ApplyToUrl(SubUrl(subPath)); using var response = await Http.CallAsync(HttpMethod.Put, url, Json.Encode(body), ContentTypeJson, _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); - return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); } /// @@ -209,57 +208,50 @@ public async Task DeleteResourceAsync(string subPath, RequestTier tier, Cancella /// GET a paginated listing and build a with each item hydrated. public async Task> ListResourceAsync(string subPath, QueryParams p, Func hydrate, RequestTier tier, CancellationToken ct) + { + var (page, _) = await ListPageAsync(subPath, p, hydrate, tier, ct).ConfigureAwait(false); + return page; + } + + /// + /// Fetches one page of a listing together with the API's count of the rows it scanned to produce + /// the page. That can exceed the number of items returned: the Store listing drops Actors the caller + /// cannot run after applying offset and limit, so advancing by the returned items would re-read rows. + /// + private async Task<(PaginationList Page, long RowsScanned)> ListPageAsync(string subPath, QueryParams p, Func hydrate, RequestTier tier, CancellationToken ct) { var data = await GetResourceRequiredAsync(subPath, p, tier, ct).ConfigureAwait(false); - return PaginationList.FromData(data, hydrate); + return (PaginationList.FromData(data, hydrate), JsonValues.IntOr(data, "count", 0)); } /// - /// Lazily iterates every item of an offset/limit-paginated listing, fetching pages on demand. Mirrors - /// the reference client's auto-paging list iterators: it starts at and, - /// if is set, yields at most that many items in total. - /// carries the caller's filters; its offset/limit are overwritten per page. + /// Lazily iterates every item of an offset/limit-paginated listing, fetching pages on demand (see + /// ). It starts at ; + /// caps the total number of items (null or 0 for no cap) and + /// sizes each page request. carries the caller's filters; its + /// offset/limit are replaced per page. /// - /// - /// Paging stops once a page returns no items (or is reached), never by - /// comparing the offset to a page's reported total: the listed resource can grow while the - /// iteration runs (e.g. a dataset collection a running Actor keeps adding to), and a reported total - /// only ever reflects the moment its own page was fetched. Stopping at an early page's total would cut - /// the iteration off before items added later, so — like the reference client — this always issues one - /// more request at the end to confirm there is nothing left, unless is hit first. - /// - public async IAsyncEnumerable IterateListAsync( + public IAsyncEnumerable IterateListAsync( string subPath, QueryParams baseQuery, int startOffset, int? limit, + int? chunkSize, Func hydrate, RequestTier tier, - [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct) + CancellationToken ct) { - var offset = Math.Max(startOffset, 0); - var yielded = 0; - while (true) - { - var q = baseQuery.Copy().Set("offset", offset); - if (limit is not null) + return PagedIteration.IterateAsync( + startOffset, + limit, + chunkSize, + limitCountsRows: false, + async (offset, pageLimit, token) => { - q.Set("limit", Math.Max(limit.Value - yielded, 0)); - } - - var page = await ListResourceAsync(subPath, q, hydrate, tier, ct).ConfigureAwait(false); - foreach (var item in page.Items) - { - yield return item; - yielded++; - } - - offset += (int)page.Count; - if (page.Count == 0 || (limit is not null && yielded >= limit.Value)) - { - yield break; - } - } + var (page, rowsScanned) = await ListPageAsync(subPath, baseQuery.PageQuery(offset, pageLimit), hydrate, tier, token).ConfigureAwait(false); + return new FetchedPage(page.Items, rowsScanned); + }, + ct); } /// POST to create a resource with a JSON-serializable body, returning the decoded data. @@ -267,7 +259,7 @@ public async Task CreateResourceAsync(QueryParams p, object? body, R { var url = MergedParams(p).ApplyToUrl(SubUrl(string.Empty)); using var response = await Http.CallAsync(HttpMethod.Post, url, Json.Encode(body), ContentTypeJson, _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); - return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); } /// @@ -286,7 +278,7 @@ public async Task GetOrCreateNamedAsync(string? name, JsonNode? sche using var response = schema is not null ? await Http.CallAsync(HttpMethod.Post, url, Json.Encode(new JsonObject { ["schema"] = schema.DeepClone() }), ContentTypeJson, _requestTimeout, tier: RequestTier.Short, cancellationToken: ct).ConfigureAwait(false) : await Http.CallAsync(HttpMethod.Post, url, timeout: _requestTimeout, tier: RequestTier.Short, cancellationToken: ct).ConfigureAwait(false); - return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); } /// @@ -298,7 +290,7 @@ public async Task PostWithBodyAsync(string subPath, QueryParams p, s { var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); using var response = await Http.CallAsync(HttpMethod.Post, url, body, contentType, _requestTimeout, bodyBytes: bodyBytes, tier: tier, cancellationToken: ct).ConfigureAwait(false); - return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); } /// @@ -311,7 +303,7 @@ public async Task PostWithBodyAsync(string subPath, QueryParams p, s { var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); using var response = await Http.CallAsync(HttpMethod.Post, url, body, contentType, _requestTimeout, bodyBytes: bodyBytes, tier: tier, cancellationToken: ct).ConfigureAwait(false); - return Json.Decode(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false)); + return await response.ReadJsonAsync(ct).ConfigureAwait(false); } /// @@ -340,7 +332,7 @@ public async Task DeleteWithBodyAsync(string subPath, QueryParams p, { var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); using var response = await Http.CallAsync(HttpMethod.Delete, url, Json.Encode(body), ContentTypeJson, _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); - return AsObject(Json.DecodeData(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false))); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); } /// @@ -373,6 +365,40 @@ public async Task GetRawRequiredAsync(string subPath, QueryParams p, Req return await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); } + /// + /// GET returning a JSON body that is not wrapped in a data envelope; propagates a 404 (see + /// for why). Returns null for an empty body. + /// + public async Task GetJsonRequiredAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); + using var response = await Http.CallAsync(HttpMethod.Get, url, timeout: _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); + return await response.ReadJsonAsync(ct).ConfigureAwait(false); + } + + /// + /// GET returning a JSON body that is not wrapped in a data envelope. Returns null on not-found. + /// + public async Task GetJsonAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) + { + try + { + return await GetJsonRequiredAsync(subPath, p, tier, ct).ConfigureAwait(false); + } + catch (ApifyApiException e) when (HttpClientCore.IsNotFound(e)) + { + return null; + } + } + + /// GET returning a data-enveloped object; propagates a 404 (see for why). + public async Task GetDataObjectRequiredAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); + using var response = await Http.CallAsync(HttpMethod.Get, url, timeout: _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); + return await response.ReadDataObjectAsync(ct).ConfigureAwait(false); + } + /// /// GET returning the raw response body as bytes (no data envelope, no UTF-8 decoding — useful when the /// body is not necessarily text). Returns null on not-found. @@ -538,13 +564,6 @@ public async Task WaitForFinishAsync(int? waitSecs, string resourceN $"waiting for {resourceName} to finish failed: cannot fetch {resourceName} details from the server"); } - /// - /// Coerces a decoded value to a JSON object. Endpoints that return a resource object always decode to - /// an object; a non-object (e.g. an unexpected null data field) becomes an empty object so model - /// construction stays type-safe. - /// - private static JsonObject AsObject(JsonNode? value) => value as JsonObject ?? new JsonObject(); - // ---- URL / id helpers ----------------------------------------------------- /// diff --git a/src/Apify.Client/Internal/ResponseReader.cs b/src/Apify.Client/Internal/ResponseReader.cs new file mode 100644 index 0000000..ef6d0c1 --- /dev/null +++ b/src/Apify.Client/Internal/ResponseReader.cs @@ -0,0 +1,105 @@ +using System.Collections.Generic; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Apify.Client.Exceptions; + +namespace Apify.Client.Internal; + +/// +/// Reads a successful JSON response into a and reports a body that is unparsable +/// () or does not match the schema the API specification +/// declares for the operation (, see ), +/// with the request's method and path as context. Every successful JSON response is decoded through here, +/// matching the reference client's parseResponse. +/// +internal static class ResponseReader +{ + private const string DataEnvelopeKey = "data"; + + /// Reads a JSON body that is not wrapped in a data envelope (null for an empty body). + public static async Task ReadJsonAsync(this HttpResponseMessage response, CancellationToken cancellationToken) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var node = Parse(response, body); + ValidateAgainstSpec(response, body, node); + return node; + } + + /// Reads a bare (not enveloped) JSON array body, such as a dataset's items. + public static async Task ReadJsonArrayAsync(this HttpResponseMessage response, CancellationToken cancellationToken) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var node = Parse(response, body); + ValidateAgainstSpec(response, body, node); + return node as JsonArray ?? throw Invalid(response, body, "the response is not a JSON array"); + } + + /// Reads a data-enveloped body whose data must be an object. + public static async Task ReadDataObjectAsync(this HttpResponseMessage response, CancellationToken cancellationToken) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + return UnwrapData(response, body) as JsonObject ?? throw Invalid(response, body, "the response's `data` is not an object"); + } + + /// + /// Parses an enveloped body, validates it against the specification's schema for the operation, and + /// returns the unwrapped non-null data. + /// + private static JsonNode UnwrapData(HttpResponseMessage response, string body) + { + var node = Parse(response, body); + ValidateAgainstSpec(response, body, node); + if (node is not JsonObject envelope || !envelope.TryGetPropertyValue(DataEnvelopeKey, out var data)) + { + throw Invalid(response, body, "the response is missing the `data` envelope"); + } + + return data ?? throw Invalid(response, body, "the response's `data` is null"); + } + + private static void ValidateAgainstSpec(HttpResponseMessage response, string body, JsonNode? node) + { + var issues = ResponseSchemas.Validate(MethodOf(response), PathOf(response), node); + if (issues.Count > 0) + { + throw Invalid(response, body, issues); + } + } + + private static JsonNode? Parse(HttpResponseMessage response, string body) + { + try + { + return Json.Decode(body); + } + catch (JsonException ex) + { + throw new InvalidResponseBodyException( + "response body is not valid JSON: " + ex.Message, + (int)response.StatusCode, + MethodOf(response), + PathOf(response), + body, + ex); + } + } + + private static ResponseValidationException Invalid(HttpResponseMessage response, string body, string problem) + => Invalid(response, body, new[] { problem }); + + private static ResponseValidationException Invalid(HttpResponseMessage response, string body, IReadOnlyList issues) + => new( + issues, + (int)response.StatusCode, + MethodOf(response), + PathOf(response), + body); + + private static string MethodOf(HttpResponseMessage response) => response.RequestMessage?.Method.Method ?? string.Empty; + + private static string PathOf(HttpResponseMessage response) + => response.RequestMessage?.RequestUri is { } uri ? HttpClientCore.ExtractPath(uri.ToString()) : string.Empty; +} diff --git a/src/Apify.Client/Internal/ResponseSchemas.cs b/src/Apify.Client/Internal/ResponseSchemas.cs new file mode 100644 index 0000000..5f43e24 --- /dev/null +++ b/src/Apify.Client/Internal/ResponseSchemas.cs @@ -0,0 +1,362 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; + +namespace Apify.Client.Internal; + +/// +/// Validates a successful JSON response against the schema the Apify OpenAPI specification declares for +/// its operation, the way the reference client's parseResponse does with its generated schemas. +/// +/// +/// The schemas come from the embedded ResponseSchemas.json, generated from the specification by +/// tools/generate_response_schemas.py. Validation covers JSON types, required properties (including +/// nullability), nested properties and array items, the allOf/anyOf/oneOf combinators, +/// and value constraints: the date-time and uri formats, pattern, +/// minimum/maximum and minLength/maxLength. Like the reference client's +/// schemas, it lets unknown properties and unknown enum values through, so an API that grows new fields or +/// values does not break existing clients. A request that matches no documented operation is not validated. +/// +internal static class ResponseSchemas +{ + private const string ResourceName = "Apify.Client.ResponseSchemas.json"; + private const string ApiVersionSegment = "/v2/"; + private const string RefPrefix = "#/components/schemas/"; + + /// The most issues reported for one response, so a badly wrong body stays readable. + private const int MaxReportedIssues = 10; + + /// RFC 3339 date-time: the shape DateTimeOffset.TryParse alone would be too lenient about. + private static readonly Regex DateTimePattern = new( + @"^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}:\d{2})$", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + /// Bounds the time a schema's pattern may take, so a pathological pattern cannot hang a response read. + private static readonly TimeSpan PatternTimeout = TimeSpan.FromSeconds(1); + + private static readonly ConcurrentDictionary Patterns = new(StringComparer.Ordinal); + + private static readonly Lazy Loaded = new(Load); + + /// The OpenAPI specification version the embedded schemas were generated from. + public static string SpecVersion => Loaded.Value.SpecVersion; + + /// + /// Validates against the schema documented for the operation identified by + /// and (the request path, optionally with a query). + /// Returns the problems found, empty when the body is valid or the operation is not documented. + /// + public static IReadOnlyList Validate(string method, string path, JsonNode? body) + { + var set = Loaded.Value; + var operation = set.Find(method, path); + if (operation is null) + { + return Array.Empty(); + } + + var issues = new List(); + set.Check(body, operation, "$", issues); + return issues; + } + + private static SchemaSet Load() + { + using var stream = typeof(ResponseSchemas).Assembly.GetManifestResourceStream(ResourceName) + ?? throw new InvalidOperationException("The embedded resource " + ResourceName + " is missing."); + var document = (JsonObject)JsonNode.Parse(stream)!; + return new SchemaSet( + (string)document["specVersion"]!, + document["operations"]!.AsObject(), + document["schemas"]!.AsObject()); + } + + private sealed class SchemaSet + { + private readonly JsonObject _schemas; + private readonly Dictionary> _operationsByMethod = new(StringComparer.Ordinal); + + public SchemaSet(string specVersion, JsonObject operations, JsonObject schemas) + { + SpecVersion = specVersion; + _schemas = schemas; + foreach (var (key, schema) in operations) + { + var space = key.IndexOf(' ', StringComparison.Ordinal); + var method = key[..space]; + var template = key[(space + 1)..]; + if (!_operationsByMethod.TryGetValue(method, out var list)) + { + list = new List(); + _operationsByMethod[method] = list; + } + + list.Add(new OperationSchema(template, schema!)); + } + + // The most specific template wins (e.g. `/runs/last` over `/runs/{runId}`): fewest parameters first. + foreach (var list in _operationsByMethod.Values) + { + list.Sort(static (a, b) => a.ParameterCount.CompareTo(b.ParameterCount)); + } + } + + public string SpecVersion { get; } + + public JsonNode? Find(string method, string path) + { + if (!_operationsByMethod.TryGetValue(method.ToUpperInvariant(), out var candidates)) + { + return null; + } + + var normalized = NormalizePath(path); + foreach (var candidate in candidates) + { + if (candidate.Pattern.IsMatch(normalized)) + { + return candidate.Schema; + } + } + + return null; + } + + /// Appends a message to for each way violates . + public void Check(JsonNode? value, JsonNode? schema, string location, List issues) + { + if (schema is not JsonObject rules || issues.Count >= MaxReportedIssues) + { + return; + } + + if (rules["$ref"] is JsonNode reference) + { + Check(value, Resolve((string)reference!), location, issues); + } + + if (rules["allOf"] is JsonArray all) + { + foreach (var part in all) + { + Check(value, part, location, issues); + } + } + + if (rules["anyOf"] is JsonArray any) + { + CheckAny(value, any, location, issues); + } + + if (rules["oneOf"] is JsonArray one) + { + // A value satisfying several alternatives is accepted too, like the reference client's unions. + CheckAny(value, one, location, issues); + } + + if (rules["type"] is JsonNode typeNode && !MatchesType(value, typeNode)) + { + issues.Add(location + ": expected " + DescribeType(typeNode) + " but got " + DescribeKind(value)); + return; + } + + if (value is not null) + { + CheckConstraints(value, rules, location, issues); + } + + if (value is JsonObject obj) + { + CheckObject(obj, rules, location, issues); + } + else if (value is JsonArray array && rules["items"] is JsonNode items) + { + for (var i = 0; i < array.Count; i++) + { + Check(array[i], items, location + "[" + i.ToString(CultureInfo.InvariantCulture) + "]", issues); + } + } + } + + /// Checks the value constraints (format, pattern, bounds, length) that apply to a scalar. + private static void CheckConstraints(JsonNode value, JsonObject rules, string location, List issues) + { + switch (value.GetValueKind()) + { + case JsonValueKind.String: + var text = value.GetValue(); + if (rules["format"] is JsonNode format && !MatchesFormat(text, (string)format!)) + { + issues.Add(location + ": is not a valid " + (string)format! + " value"); + } + + if (rules["pattern"] is JsonNode pattern && !PatternFor((string)pattern!).IsMatch(text)) + { + issues.Add(location + ": does not match the pattern " + (string)pattern!); + } + + if (rules["minLength"] is JsonNode minLength && text.Length < (int)minLength!) + { + issues.Add(location + ": is shorter than the minimum length " + ((int)minLength!).ToString(CultureInfo.InvariantCulture)); + } + + if (rules["maxLength"] is JsonNode maxLength && text.Length > (int)maxLength!) + { + issues.Add(location + ": is longer than the maximum length " + ((int)maxLength!).ToString(CultureInfo.InvariantCulture)); + } + + break; + case JsonValueKind.Number: + var number = value.GetValue().GetDouble(); + if (rules["minimum"] is JsonNode minimum && number < (double)minimum!) + { + issues.Add(location + ": is a value outside the documented range (minimum " + ((double)minimum!).ToString(CultureInfo.InvariantCulture) + ")"); + } + + if (rules["maximum"] is JsonNode maximum && number > (double)maximum!) + { + issues.Add(location + ": is a value outside the documented range (maximum " + ((double)maximum!).ToString(CultureInfo.InvariantCulture) + ")"); + } + + break; + default: + break; + } + } + + private static bool MatchesFormat(string text, string format) => format switch + { + "date-time" => DateTimePattern.IsMatch(text) && DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.None, out _), + "uri" => Uri.TryCreate(text, UriKind.Absolute, out _), + _ => true, + }; + + private static Regex PatternFor(string pattern) + => Patterns.GetOrAdd(pattern, static p => new Regex(p, RegexOptions.CultureInvariant, PatternTimeout)); + + private void CheckObject(JsonObject obj, JsonObject rules, string location, List issues) + { + if (rules["required"] is JsonArray required) + { + foreach (var name in required) + { + if (!obj.ContainsKey((string)name!)) + { + issues.Add(location + "." + (string)name! + ": required property is missing"); + } + } + } + + if (rules["properties"] is JsonObject properties) + { + foreach (var (name, propertySchema) in properties) + { + // An absent optional property is fine; an absent required one was reported above. + if (obj.TryGetPropertyValue(name, out var propertyValue)) + { + Check(propertyValue, propertySchema, location + "." + name, issues); + } + } + } + } + + private void CheckAny(JsonNode? value, JsonArray alternatives, string location, List issues) + { + foreach (var alternative in alternatives) + { + var attempt = new List(); + Check(value, alternative, location, attempt); + if (attempt.Count == 0) + { + return; + } + } + + issues.Add(location + ": matches none of the allowed variants (got " + DescribeKind(value) + ")"); + } + + private JsonNode? Resolve(string reference) + { + if (!reference.StartsWith(RefPrefix, StringComparison.Ordinal)) + { + throw new InvalidOperationException("Unsupported schema reference: " + reference); + } + + return _schemas[reference[RefPrefix.Length..]]; + } + + private static bool MatchesType(JsonNode? value, JsonNode typeNode) + => typeNode is JsonArray types + ? types.Any(t => MatchesSingleType(value, (string)t!)) + : MatchesSingleType(value, (string)typeNode!); + + private static bool MatchesSingleType(JsonNode? value, string type) + { + if (value is null) + { + return type == "null"; + } + + return type switch + { + "object" => value is JsonObject, + "array" => value is JsonArray, + "string" => value.GetValueKind() == JsonValueKind.String, + "boolean" => value.GetValueKind() is JsonValueKind.True or JsonValueKind.False, + "number" => value.GetValueKind() == JsonValueKind.Number, + "integer" => IsInteger(value), + _ => false, + }; + } + + private static bool IsInteger(JsonNode value) + { + if (value.GetValueKind() != JsonValueKind.Number) + { + return false; + } + + var number = value.GetValue(); + return number.TryGetInt64(out _) || (number.TryGetDouble(out var d) && d == Math.Floor(d) && !double.IsInfinity(d)); + } + + private static string DescribeType(JsonNode typeNode) + => typeNode is JsonArray types ? string.Join(" or ", types.Select(static t => (string)t!)) : (string)typeNode!; + + private static string DescribeKind(JsonNode? value) => value is null ? "null" : value.GetValueKind().ToString().ToLowerInvariant(); + + private static string NormalizePath(string path) + { + var query = path.IndexOf('?', StringComparison.Ordinal); + var withoutQuery = query >= 0 ? path[..query] : path; + var version = withoutQuery.IndexOf(ApiVersionSegment, StringComparison.Ordinal); + return (version >= 0 ? withoutQuery[version..] : withoutQuery).TrimEnd('/'); + } + } + + private sealed class OperationSchema + { + /// Matches a path parameter ({name}) in a template after escaped its opening brace. + private static readonly Regex Parameter = new(@"\\\{[^}/]+\}", RegexOptions.Compiled); + + public OperationSchema(string template, JsonNode schema) + { + Schema = schema; + ParameterCount = template.Count(static c => c == '{'); + Pattern = new Regex("^" + Parameter.Replace(Regex.Escape(template), "[^/]+") + "$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + } + + public JsonNode Schema { get; } + + public int ParameterCount { get; } + + public Regex Pattern { get; } + } +} diff --git a/src/Apify.Client/Internal/ResponseSchemas.json b/src/Apify.Client/Internal/ResponseSchemas.json new file mode 100644 index 0000000..a93ee10 --- /dev/null +++ b/src/Apify.Client/Internal/ResponseSchemas.json @@ -0,0 +1 @@ +{"operations":{"DELETE /v2/actor-runs/{runId}/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/actor-tasks/{actorTaskId}":{"type":"object"},"DELETE /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/actors/{actorId}":{"type":"object"},"DELETE /v2/actors/{actorId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/actors/{actorId}/versions/{versionNumber}":{"type":"object"},"DELETE /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"type":"object"},"DELETE /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"DELETE /v2/request-queues/{queueId}/requests/batch":{"$ref":"#/components/schemas/BatchDeleteResponse"},"DELETE /v2/schedules/{scheduleId}":{"type":"object"},"DELETE /v2/webhooks/{webhookId}":{"type":"object"},"GET /v2/actor-builds":{"$ref":"#/components/schemas/ListOfBuildsResponse"},"GET /v2/actor-builds/{buildId}":{"allOf":[{"$ref":"#/components/schemas/BuildResponse"}]},"GET /v2/actor-builds/{buildId}/openapi.json":{"type":"object"},"GET /v2/actor-runs":{"allOf":[{"$ref":"#/components/schemas/ListOfRunsResponse"}]},"GET /v2/actor-runs/{runId}":{"allOf":[{"$ref":"#/components/schemas/RunResponse"}]},"GET /v2/actor-runs/{runId}/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/actor-runs/{runId}/dataset/items":{"items":{"type":"object"},"type":"array"},"GET /v2/actor-runs/{runId}/dataset/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/actor-runs/{runId}/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/actor-runs/{runId}/key-value-store/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/actor-runs/{runId}/key-value-store/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/actor-runs/{runId}/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/actor-runs/{runId}/request-queue/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/actor-runs/{runId}/request-queue/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/actor-runs/{runId}/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/actor-tasks":{"$ref":"#/components/schemas/ListOfTasksResponse"},"GET /v2/actor-tasks/{actorTaskId}":{"properties":{"data":{"$ref":"#/components/schemas/Task"}},"required":["data"],"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/input":{"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/run-sync":{"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/run-sync-get-dataset-items":{"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/runs":{"properties":{"data":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/RunListItem"},"type":"array"}},"required":["items"],"type":"object"}]}},"required":["data"],"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/runs/last":{"properties":{"data":{"$ref":"#/components/schemas/Run"}},"required":["data"],"type":"object"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/dataset/items":{"items":{"type":"object"},"type":"array"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/dataset/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/actor-tasks/{actorTaskId}/webhooks":{"properties":{"data":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/WebhookResource"},"type":"array"}},"required":["items"],"type":"object"}]}},"required":["data"],"type":"object"},"GET /v2/actors":{"$ref":"#/components/schemas/ListOfActorsResponse"},"GET /v2/actors/{actorId}":{"$ref":"#/components/schemas/ActorResponse"},"GET /v2/actors/{actorId}/builds":{"$ref":"#/components/schemas/ListOfBuildsResponse"},"GET /v2/actors/{actorId}/builds/default":{"$ref":"#/components/schemas/BuildResponse"},"GET /v2/actors/{actorId}/builds/{buildId}":{"$ref":"#/components/schemas/BuildResponse"},"GET /v2/actors/{actorId}/builds/{buildId}/openapi.json":{"type":"object"},"GET /v2/actors/{actorId}/run-sync":{"type":"object"},"GET /v2/actors/{actorId}/run-sync-get-dataset-items":{"items":{"type":"object"},"type":"array"},"GET /v2/actors/{actorId}/runs":{"$ref":"#/components/schemas/ListOfRunsResponse"},"GET /v2/actors/{actorId}/runs/last":{"$ref":"#/components/schemas/RunResponse"},"GET /v2/actors/{actorId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/actors/{actorId}/runs/last/dataset/items":{"items":{"type":"object"},"type":"array"},"GET /v2/actors/{actorId}/runs/last/dataset/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/actors/{actorId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/actors/{actorId}/runs/last/key-value-store/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/actors/{actorId}/runs/last/key-value-store/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/actors/{actorId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/actors/{actorId}/runs/{runId}":{"$ref":"#/components/schemas/RunResponse"},"GET /v2/actors/{actorId}/versions":{"$ref":"#/components/schemas/ListOfVersionsResponse"},"GET /v2/actors/{actorId}/versions/{versionNumber}":{"$ref":"#/components/schemas/VersionResponse"},"GET /v2/actors/{actorId}/versions/{versionNumber}/env-vars":{"$ref":"#/components/schemas/ListOfEnvVarsResponse"},"GET /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"$ref":"#/components/schemas/EnvVarResponse"},"GET /v2/actors/{actorId}/webhooks":{"$ref":"#/components/schemas/ListOfWebhooksResponse"},"GET /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"GET /v2/datasets":{"$ref":"#/components/schemas/ListOfDatasetsResponse"},"GET /v2/datasets/{datasetId}":{"$ref":"#/components/schemas/DatasetResponse"},"GET /v2/datasets/{datasetId}/items":{"items":{"type":"object"},"type":"array"},"GET /v2/datasets/{datasetId}/statistics":{"$ref":"#/components/schemas/DatasetStatisticsResponse"},"GET /v2/key-value-stores":{"$ref":"#/components/schemas/ListOfKeyValueStoresResponse"},"GET /v2/key-value-stores/{storeId}":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"GET /v2/key-value-stores/{storeId}/keys":{"$ref":"#/components/schemas/ListOfKeysResponse"},"GET /v2/key-value-stores/{storeId}/records/{recordKey}":{"$ref":"#/components/schemas/RecordResponse"},"GET /v2/request-queues":{"$ref":"#/components/schemas/ListOfRequestQueuesResponse"},"GET /v2/request-queues/{queueId}":{"$ref":"#/components/schemas/RequestQueueResponse"},"GET /v2/request-queues/{queueId}/head":{"$ref":"#/components/schemas/RequestQueueHeadResponse"},"GET /v2/request-queues/{queueId}/requests":{"$ref":"#/components/schemas/ListOfRequestsResponse"},"GET /v2/request-queues/{queueId}/requests/{requestId}":{"$ref":"#/components/schemas/RequestResponse"},"GET /v2/schedules":{"$ref":"#/components/schemas/ListOfSchedulesResponse"},"GET /v2/schedules/{scheduleId}":{"$ref":"#/components/schemas/ScheduleResponse"},"GET /v2/schedules/{scheduleId}/log":{"$ref":"#/components/schemas/ScheduleLogResponse"},"GET /v2/store":{"$ref":"#/components/schemas/ListOfStoreActorsResponse"},"GET /v2/users/me":{"$ref":"#/components/schemas/UserPrivateInfoResponse"},"GET /v2/users/me/limits":{"$ref":"#/components/schemas/AccountLimitsResponse"},"GET /v2/users/me/usage/monthly":{"$ref":"#/components/schemas/MonthlyUsageResponse"},"GET /v2/users/{userId}":{"$ref":"#/components/schemas/UserPublicInfoResponse"},"GET /v2/webhook-dispatches":{"$ref":"#/components/schemas/ListOfWebhookDispatchesResponse"},"GET /v2/webhook-dispatches/{dispatchId}":{"$ref":"#/components/schemas/WebhookDispatchResponse"},"GET /v2/webhooks":{"$ref":"#/components/schemas/ListOfWebhooksResponse"},"GET /v2/webhooks/{webhookId}":{"$ref":"#/components/schemas/WebhookResponse"},"GET /v2/webhooks/{webhookId}/dispatches":{"$ref":"#/components/schemas/ListOfWebhookDispatchesResponse"},"POST /v2/actor-builds/{buildId}/abort":{"allOf":[{"$ref":"#/components/schemas/BuildResponse"}]},"POST /v2/actor-runs/{runId}/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-runs/{runId}/charge":{"type":"object"},"POST /v2/actor-runs/{runId}/dataset/items":{"type":"object"},"POST /v2/actor-runs/{runId}/key-value-store/records/{recordKey}":{"type":"object"},"POST /v2/actor-runs/{runId}/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-runs/{runId}/reboot":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-runs/{runId}/request-queue/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/actor-runs/{runId}/request-queue/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/actor-runs/{runId}/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/actor-runs/{runId}/request-queue/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/actor-runs/{runId}/resurrect":{"allOf":[{"$ref":"#/components/schemas/RunResponse"}]},"POST /v2/actor-tasks":{"$ref":"#/components/schemas/TaskResponse"},"POST /v2/actor-tasks/{actorTaskId}/run-sync":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/run-sync-get-dataset-items":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs":{"properties":{"data":{"$ref":"#/components/schemas/Run"}},"required":["data"],"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/dataset/items":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/reboot":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/actors":{"$ref":"#/components/schemas/ActorResponse"},"POST /v2/actors/{actorId}/builds":{"$ref":"#/components/schemas/BuildResponse"},"POST /v2/actors/{actorId}/builds/{buildId}/abort":{"$ref":"#/components/schemas/BuildResponse"},"POST /v2/actors/{actorId}/run-sync":{"type":"object"},"POST /v2/actors/{actorId}/run-sync-get-dataset-items":{"items":{"type":"object"},"type":"array"},"POST /v2/actors/{actorId}/runs":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/dataset/items":{"type":"object"},"POST /v2/actors/{actorId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"POST /v2/actors/{actorId}/runs/last/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/reboot":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/actors/{actorId}/runs/last/request-queue/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/actors/{actorId}/runs/{runId}/abort":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/{runId}/metamorph":{"$ref":"#/components/schemas/RunResponse"},"POST /v2/actors/{actorId}/runs/{runId}/resurrect":{"allOf":[{"$ref":"#/components/schemas/RunResponse"}]},"POST /v2/actors/{actorId}/validate-input":{"properties":{"valid":{"type":"boolean"}},"required":["valid"],"type":"object"},"POST /v2/actors/{actorId}/versions":{"$ref":"#/components/schemas/VersionResponse"},"POST /v2/actors/{actorId}/versions/{versionNumber}":{"$ref":"#/components/schemas/VersionResponse"},"POST /v2/actors/{actorId}/versions/{versionNumber}/env-vars":{"$ref":"#/components/schemas/EnvVarResponse"},"POST /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"$ref":"#/components/schemas/EnvVarResponse"},"POST /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"POST /v2/datasets":{"$ref":"#/components/schemas/DatasetResponse"},"POST /v2/datasets/{datasetId}/items":{"type":"object"},"POST /v2/key-value-stores":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"POST /v2/key-value-stores/{storeId}/records/{recordKey}":{"type":"object"},"POST /v2/request-queues":{"$ref":"#/components/schemas/RequestQueueResponse"},"POST /v2/request-queues/{queueId}/head/lock":{"$ref":"#/components/schemas/LockedRequestQueueHeadResponse"},"POST /v2/request-queues/{queueId}/requests":{"$ref":"#/components/schemas/AddRequestResponse"},"POST /v2/request-queues/{queueId}/requests/batch":{"$ref":"#/components/schemas/BatchAddResponse"},"POST /v2/request-queues/{queueId}/requests/unlock":{"$ref":"#/components/schemas/UnlockRequestsResponse"},"POST /v2/schedules":{"$ref":"#/components/schemas/ScheduleResponse"},"POST /v2/tools/decode-and-verify":{"$ref":"#/components/schemas/DecodeAndVerifyResponse"},"POST /v2/tools/encode-and-sign":{"$ref":"#/components/schemas/EncodeAndSignResponse"},"POST /v2/webhooks":{"$ref":"#/components/schemas/WebhookResponse"},"POST /v2/webhooks/{webhookId}/test":{"$ref":"#/components/schemas/WebhookDispatchResponse"},"PUT /v2/actor-runs/{runId}":{"$ref":"#/components/schemas/RunResponse"},"PUT /v2/actor-runs/{runId}/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/actor-runs/{runId}/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/actor-runs/{runId}/key-value-store/records/{recordKey}":{"type":"object"},"PUT /v2/actor-runs/{runId}/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/actor-runs/{runId}/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/actor-runs/{runId}/request-queue/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/actor-tasks/{actorTaskId}":{"properties":{"data":{"$ref":"#/components/schemas/Task"}},"required":["data"],"type":"object"},"PUT /v2/actor-tasks/{actorTaskId}/input":{"type":"object"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/actor-tasks/{actorTaskId}/runs/last/request-queue/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/actors/{actorId}":{"$ref":"#/components/schemas/ActorResponse"},"PUT /v2/actors/{actorId}/runs/last/dataset":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/actors/{actorId}/runs/last/key-value-store":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/actors/{actorId}/runs/last/key-value-store/records/{recordKey}":{"type":"object"},"PUT /v2/actors/{actorId}/runs/last/request-queue":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/actors/{actorId}/runs/last/request-queue/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/actors/{actorId}/runs/last/request-queue/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/actors/{actorId}/versions/{versionNumber}":{"$ref":"#/components/schemas/VersionResponse"},"PUT /v2/actors/{actorId}/versions/{versionNumber}/env-vars/{envVarName}":{"$ref":"#/components/schemas/EnvVarResponse"},"PUT /v2/browser-info":{"$ref":"#/components/schemas/BrowserInfoResponse"},"PUT /v2/datasets/{datasetId}":{"$ref":"#/components/schemas/DatasetResponse"},"PUT /v2/key-value-stores/{storeId}":{"$ref":"#/components/schemas/KeyValueStoreResponse"},"PUT /v2/key-value-stores/{storeId}/records/{recordKey}":{"type":"object"},"PUT /v2/request-queues/{queueId}":{"$ref":"#/components/schemas/RequestQueueResponse"},"PUT /v2/request-queues/{queueId}/requests/{requestId}":{"$ref":"#/components/schemas/UpdateRequestResponse"},"PUT /v2/request-queues/{queueId}/requests/{requestId}/lock":{"$ref":"#/components/schemas/ProlongRequestLockResponse"},"PUT /v2/schedules/{scheduleId}":{"$ref":"#/components/schemas/ScheduleResponse"},"PUT /v2/users/me/limits":{"type":"object"},"PUT /v2/webhooks/{webhookId}":{"$ref":"#/components/schemas/WebhookResponse"}},"schemas":{"AccountLimits":{"properties":{"current":{"$ref":"#/components/schemas/CurrentUsage"},"limits":{"$ref":"#/components/schemas/Limits"},"monthlyUsageCycle":{"$ref":"#/components/schemas/UsageCycle"}},"required":["monthlyUsageCycle","limits","current"],"type":"object"},"AccountLimitsResponse":{"properties":{"data":{"$ref":"#/components/schemas/AccountLimits"}},"required":["data"],"type":"object"},"ActorDefinition":{"properties":{"actorSpecification":{"type":"integer"},"buildTag":{"type":"string"},"changelog":{"type":["string","null"]},"defaultMemoryMbytes":{"oneOf":[{"type":"string"},{"type":"integer"}]},"dockerContextDir":{"type":"string"},"dockerfile":{"type":"string"},"environmentVariables":{"type":"object"},"input":{"type":["object","null"]},"maxMemoryMbytes":{"minimum":128,"type":"integer"},"minMemoryMbytes":{"minimum":128,"type":"integer"},"name":{"type":"string"},"readme":{"type":["string","null"]},"storages":{"properties":{"dataset":{"type":"object"}},"type":"object"},"usesStandbyMode":{"type":"boolean"},"version":{"pattern":"^[0-9]+(\\.[0-9]+)+$","type":"string"}},"type":"object"},"ActorJobStatus":{"type":"string"},"ActorListItem":{"properties":{"createdAt":{"format":"date-time","type":"string"},"id":{"type":"string"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":"string"},"stats":{"anyOf":[{"$ref":"#/components/schemas/ActorStats"},{"type":"null"}]},"title":{"type":"string"},"username":{"type":"string"}},"required":["id","createdAt","modifiedAt","name","username"],"type":"object"},"ActorNotice":{"type":["string","null"]},"ActorPermissionLevel":{"type":"string"},"ActorResource":{"properties":{"actorPermissionLevel":{"$ref":"#/components/schemas/ActorPermissionLevel"},"actorStandby":{"anyOf":[{"$ref":"#/components/schemas/ActorStandby"},{"type":"null"}]},"categories":{"items":{"type":"string"},"type":"array"},"createdAt":{"format":"date-time","type":"string"},"defaultRunOptions":{"$ref":"#/components/schemas/DefaultRunOptions"},"deploymentKey":{"type":"string"},"description":{"type":["string","null"]},"exampleRunInput":{"anyOf":[{"$ref":"#/components/schemas/ExampleRunInput"},{"type":"null"}]},"hasNoDataset":{"type":"boolean"},"id":{"type":"string"},"isCritical":{"type":"boolean"},"isDeprecated":{"type":["boolean","null"]},"isGeneric":{"type":"boolean"},"isPublic":{"type":"boolean"},"isSourceCodeHidden":{"type":"boolean"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":"string"},"notice":{"$ref":"#/components/schemas/ActorNotice"},"pictureUrl":{"type":["string","null"]},"pricingInfos":{"items":{"$ref":"#/components/schemas/ActorRunPricingInfo"},"type":"array"},"readmeSummary":{"type":"string"},"restartOnError":{"type":["boolean","null"]},"seoDescription":{"type":["string","null"]},"seoTitle":{"type":["string","null"]},"standbyUrl":{"type":["string","null"]},"stats":{"$ref":"#/components/schemas/ActorStats"},"taggedBuilds":{"anyOf":[{"$ref":"#/components/schemas/TaggedBuilds"},{"type":"null"}]},"title":{"type":["string","null"]},"userId":{"type":"string"},"username":{"type":"string"},"versions":{"items":{"$ref":"#/components/schemas/Version"},"type":"array"}},"required":["id","userId","name","username","isPublic","createdAt","modifiedAt","stats","versions","defaultRunOptions"],"type":"object"},"ActorResponse":{"properties":{"data":{"$ref":"#/components/schemas/ActorResource"}},"required":["data"],"type":"object"},"ActorRunPricingInfo":{"oneOf":[{"$ref":"#/components/schemas/PayPerEventActorPricingInfo"},{"$ref":"#/components/schemas/PricePerDatasetItemActorPricingInfo"},{"$ref":"#/components/schemas/FlatPricePerMonthActorPricingInfo"},{"$ref":"#/components/schemas/FreeActorPricingInfo"}]},"ActorStandby":{"properties":{"build":{"type":["string","null"]},"desiredRequestsPerActorRun":{"type":["integer","null"]},"disableStandbyFieldsOverride":{"type":["boolean","null"]},"idleTimeoutSecs":{"type":["integer","null"]},"isEnabled":{"type":["boolean","null"]},"maxRequestsPerActorRun":{"type":["integer","null"]},"memoryMbytes":{"type":["integer","null"]},"shouldPassActorInput":{"type":["boolean","null"]}},"type":"object"},"ActorStats":{"properties":{"actorReviewCount":{"type":"integer"},"actorReviewRating":{"type":"number"},"bookmarkCount":{"type":"integer"},"lastRunStartedAt":{"format":"date-time","type":"string"},"publicActorRunStats30Days":{"properties":{"ABORTED":{"type":"integer"},"FAILED":{"type":"integer"},"SUCCEEDED":{"type":"integer"},"TIMED-OUT":{"type":"integer"},"TOTAL":{"type":"integer"}},"type":"object"},"totalBuilds":{"type":"integer"},"totalMetamorphs":{"type":"integer"},"totalRuns":{"type":"integer"},"totalUsers":{"type":"integer"},"totalUsers30Days":{"type":"integer"},"totalUsers7Days":{"type":"integer"},"totalUsers90Days":{"type":"integer"}},"type":"object"},"AddRequestResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestRegistration"}},"required":["data"],"type":"object"},"AddedRequest":{"properties":{"requestId":{"$ref":"#/components/schemas/RequestId"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"wasAlreadyHandled":{"$ref":"#/components/schemas/WasAlreadyHandled"},"wasAlreadyPresent":{"$ref":"#/components/schemas/WasAlreadyPresent"}},"required":["requestId","uniqueKey","wasAlreadyHandled","wasAlreadyPresent"],"type":"object"},"AvailableProxyGroups":{"type":"object"},"BatchAddResponse":{"properties":{"data":{"$ref":"#/components/schemas/BatchAddResult"}},"required":["data"],"type":"object"},"BatchAddResult":{"properties":{"processedRequests":{"items":{"$ref":"#/components/schemas/AddedRequest"},"type":"array"},"unprocessedRequests":{"items":{"$ref":"#/components/schemas/UnprocessedRequest"},"type":"array"}},"required":["processedRequests","unprocessedRequests"],"type":"object"},"BatchDeleteResponse":{"properties":{"data":{"$ref":"#/components/schemas/BatchDeleteResult"}},"required":["data"],"type":"object"},"BatchDeleteResult":{"properties":{"processedRequests":{"items":{"$ref":"#/components/schemas/DeletedRequest"},"type":"array"},"unprocessedRequests":{"items":{"$ref":"#/components/schemas/UnprocessedRequest"},"type":"array"}},"required":["processedRequests","unprocessedRequests"],"type":"object"},"BrowserInfoResponse":{"properties":{"bodyLength":{"type":"integer"},"clientIp":{"type":["string","null"]},"countryCode":{"type":["string","null"]},"headers":{"type":"object"},"method":{"type":"string"},"rawHeaders":{"items":{"type":"string"},"type":"array"}},"required":["method","clientIp","countryCode","bodyLength"],"type":"object"},"Build":{"properties":{"actId":{"type":"string"},"actVersion":{"properties":{"buildTag":{"type":["string","null"]},"gitRepoUrl":{"type":"string"},"sourceFiles":{"items":{"$ref":"#/components/schemas/SourceCodeFile"},"type":"array"},"sourceType":{"$ref":"#/components/schemas/VersionSourceType"},"versionNumber":{"pattern":"^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])$","type":"string"}},"type":"object"},"actorDefinition":{"anyOf":[{"$ref":"#/components/schemas/ActorDefinition"},{"type":"null"}]},"buildNumber":{"pattern":"^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])(\\.[1-9][0-9]{0,4})$","type":"string"},"finishedAt":{"format":"date-time","type":["string","null"]},"id":{"type":"string"},"imageDigest":{"type":["string","null"]},"inputSchema":{"type":["string","null"]},"meta":{"$ref":"#/components/schemas/BuildMeta"},"options":{"anyOf":[{"$ref":"#/components/schemas/BuildOptions"},{"type":"null"}]},"readme":{"type":["string","null"]},"startedAt":{"format":"date-time","type":"string"},"stats":{"anyOf":[{"$ref":"#/components/schemas/BuildStats"},{"type":"null"}]},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"usage":{"anyOf":[{"$ref":"#/components/schemas/BuildUsage"},{"type":"null"}]},"usageTotalUsd":{"type":["number","null"]},"usageUsd":{"anyOf":[{"$ref":"#/components/schemas/BuildUsage"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","actId","userId","startedAt","status","meta","buildNumber"],"type":"object"},"BuildListItem":{"properties":{"actId":{"type":"string"},"buildNumber":{"pattern":"^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])(\\.[1-9][0-9]{0,4})$","type":"string"},"buildNumberInt":{"type":"integer"},"finishedAt":{"format":"date-time","type":["string","null"]},"id":{"type":"string"},"meta":{"$ref":"#/components/schemas/BuildMeta"},"startedAt":{"format":"date-time","type":"string"},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"usageTotalUsd":{"type":"number"},"userId":{"type":"string"}},"required":["id","status","startedAt","usageTotalUsd","buildNumber"],"type":"object"},"BuildMeta":{"properties":{"clientIp":{"type":["string"]},"origin":{"$ref":"#/components/schemas/RunOrigin"},"userAgent":{"type":["string"]}},"required":["origin"],"type":"object"},"BuildOptions":{"properties":{"betaPackages":{"type":["boolean","null"]},"diskMbytes":{"type":["integer","null"]},"memoryMbytes":{"type":["integer","null"]},"useCache":{"type":["boolean","null"]}},"type":"object"},"BuildResponse":{"properties":{"data":{"$ref":"#/components/schemas/Build"}},"required":["data"],"type":"object"},"BuildStats":{"properties":{"computeUnits":{"type":"number"},"durationMillis":{"type":"integer"},"imageSizeBytes":{"type":["integer","null"]},"runTimeSecs":{"type":"number"}},"type":"object"},"BuildUsage":{"properties":{"ACTOR_COMPUTE_UNITS":{"type":["number","null"]}},"type":"object"},"CommonActorPricingInfo":{"properties":{"apifyMarginPercentage":{"type":"number"},"createdAt":{"format":"date-time","type":"string"},"forceContainsSignificantPriceChange":{"type":"boolean"},"isPriceChangeNotificationSuppressed":{"type":"boolean"},"notifiedAboutChangeAt":{"format":"date-time","type":["string","null"]},"notifiedAboutFutureChangeAt":{"format":"date-time","type":["string","null"]},"reasonForChange":{"type":["string","null"]},"startedAt":{"format":"date-time","type":"string"}},"required":["apifyMarginPercentage","createdAt","startedAt"],"type":"object"},"CurrentPricingInfo":{"properties":{"apifyMarginPercentage":{"type":"number"},"createdAt":{"format":"date-time","type":"string"},"forceContainsSignificantPriceChange":{"type":"boolean"},"isPPEPlatformUsagePaidByUser":{"type":"boolean"},"isPriceChangeNotificationSuppressed":{"type":"boolean"},"minimalMaxTotalChargeUsd":{"type":["number","null"]},"notifiedAboutChangeAt":{"format":"date-time","type":["string","null"]},"notifiedAboutFutureChangeAt":{"format":"date-time","type":["string","null"]},"pricingModel":{"type":"string"},"pricingPerEvent":{"type":["object","null"]},"reasonForChange":{"type":["string","null"]},"startedAt":{"format":"date-time","type":"string"}},"required":["pricingModel"],"type":"object"},"CurrentUsage":{"properties":{"activeActorJobCount":{"type":"integer"},"actorCount":{"type":"integer"},"actorMemoryGbytes":{"type":"number"},"actorTaskCount":{"type":"integer"},"monthlyActorComputeUnits":{"type":"number"},"monthlyExternalDataTransferGbytes":{"type":"number"},"monthlyProxySerps":{"type":"integer"},"monthlyResidentialProxyGbytes":{"type":"number"},"monthlyUsageUsd":{"type":"number"},"scheduleCount":{"type":"integer"},"teamAccountSeatCount":{"type":"integer"}},"required":["monthlyUsageUsd","monthlyActorComputeUnits","monthlyExternalDataTransferGbytes","monthlyProxySerps","monthlyResidentialProxyGbytes","actorMemoryGbytes","actorCount","actorTaskCount","activeActorJobCount","teamAccountSeatCount"],"type":"object"},"DailyServiceUsage":{"properties":{"date":{"format":"date-time","type":"string"},"serviceUsage":{"$ref":"#/components/schemas/ServiceUsage"},"totalUsageCreditsUsd":{"type":"number"}},"required":["date","serviceUsage","totalUsageCreditsUsd"],"type":"object"},"DatasetListItem":{"properties":{"accessedAt":{"format":"date-time","type":"string"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"cleanItemCount":{"type":"integer"},"createdAt":{"format":"date-time","type":"string"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"id":{"type":"string"},"itemCount":{"type":"integer"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":"string"},"stats":{"$ref":"#/components/schemas/DatasetStats"},"title":{"type":["string","null"]},"userId":{"type":"string"},"username":{"type":"string"}},"required":["id","name","userId","createdAt","modifiedAt","accessedAt","itemCount","cleanItemCount"],"type":"object"},"DatasetResource":{"properties":{"accessedAt":{"format":"date-time","type":"string"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"cleanItemCount":{"minimum":0,"type":"integer"},"consoleUrl":{"format":"uri","type":"string"},"createdAt":{"format":"date-time","type":"string"},"fields":{"items":{"type":"string"},"type":["array","null"]},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"id":{"type":"string"},"itemCount":{"minimum":0,"type":"integer"},"itemsPublicUrl":{"format":"uri","type":"string"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":["string","null"]},"schema":{"type":["object","null"]},"stats":{"$ref":"#/components/schemas/DatasetStats"},"urlSigningSecretKey":{"type":["string","null"]},"userId":{"type":"string"}},"required":["id","userId","createdAt","modifiedAt","accessedAt","itemCount","cleanItemCount","consoleUrl"],"type":"object"},"DatasetResponse":{"properties":{"data":{"$ref":"#/components/schemas/DatasetResource"}},"required":["data"],"type":"object"},"DatasetStatistics":{"properties":{"fieldStatistics":{"type":["object","null"]}},"type":"object"},"DatasetStatisticsResponse":{"properties":{"data":{"$ref":"#/components/schemas/DatasetStatistics"}},"required":["data"],"type":"object"},"DatasetStats":{"properties":{"inflatedBytes":{"type":"integer"},"readCount":{"type":"integer"},"storageBytes":{"type":"integer"},"writeCount":{"type":"integer"}},"type":"object"},"DecodeAndVerifyResponse":{"properties":{"data":{"$ref":"#/components/schemas/DecodeAndVerifyResult"}},"required":["data"],"type":"object"},"DecodeAndVerifyResult":{"properties":{"decoded":{},"encodedByUserId":{"type":["string","null"]},"isVerifiedUser":{"type":"boolean"}},"required":["decoded","encodedByUserId","isVerifiedUser"],"type":"object"},"DefaultRunOptions":{"properties":{"build":{"type":"string"},"forcePermissionLevel":{"anyOf":[{"$ref":"#/components/schemas/ActorPermissionLevel"},{"type":"null"}]},"maxItems":{"type":["integer","null"]},"memoryMbytes":{"type":"integer"},"restartOnError":{"type":["boolean","null"]},"timeoutSecs":{"type":"integer"}},"type":"object"},"DeletedRequest":{"anyOf":[{"$ref":"#/components/schemas/DeletedRequestById"},{"$ref":"#/components/schemas/DeletedRequestByUniqueKey"}]},"DeletedRequestById":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"}},"required":["id"],"type":"object"},"DeletedRequestByUniqueKey":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"}},"required":["uniqueKey"],"type":"object"},"EffectivePlatformFeature":{"properties":{"disabledReason":{"type":["string","null"]},"disabledReasonType":{"type":["string","null"]},"isEnabled":{"type":"boolean"},"isTrial":{"type":"boolean"},"trialExpirationAt":{"format":"date-time","type":["string","null"]}},"required":["isEnabled","disabledReason","disabledReasonType","isTrial","trialExpirationAt"],"type":"object"},"EffectivePlatformFeatures":{"properties":{"ACTORS":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"ACTORS_PUBLIC_ALL":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"ACTORS_PUBLIC_DEVELOPER":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY_EXTERNAL_ACCESS":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY_RESIDENTIAL":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"PROXY_SERPS":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"SCHEDULER":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"STORAGE":{"$ref":"#/components/schemas/EffectivePlatformFeature"},"WEBHOOKS":{"$ref":"#/components/schemas/EffectivePlatformFeature"}},"required":["ACTORS","STORAGE","SCHEDULER","PROXY","PROXY_EXTERNAL_ACCESS","PROXY_RESIDENTIAL","PROXY_SERPS","WEBHOOKS","ACTORS_PUBLIC_ALL","ACTORS_PUBLIC_DEVELOPER"],"type":"object"},"EncodeAndSignResponse":{"properties":{"data":{"$ref":"#/components/schemas/EncodeAndSignResult"}},"required":["data"],"type":"object"},"EncodeAndSignResult":{"properties":{"encoded":{"type":"string"}},"required":["encoded"],"type":"object"},"EnvVar":{"properties":{"isSecret":{"type":["boolean","null"]},"name":{"type":"string"},"value":{"type":"string"}},"required":["name"],"type":"object"},"EnvVarResponse":{"properties":{"data":{"$ref":"#/components/schemas/EnvVar"}},"required":["data"],"type":"object"},"ExampleRunInput":{"properties":{"body":{"type":"string"},"contentType":{"type":"string"}},"type":"object"},"FlatPricePerMonthActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"pricePerUnitUsd":{"type":"number"},"pricingModel":{"type":"string"},"trialMinutes":{"type":"integer"}},"required":["pricingModel","pricePerUnitUsd","trialMinutes"],"type":"object"}]},"FreeActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"pricingModel":{"type":"string"}},"required":["pricingModel"],"type":"object"}]},"GeneralAccess":{"type":"string"},"HadMultipleClients":{"type":"boolean"},"HandledRequestCount":{"minimum":0,"type":"integer"},"HeadLimit":{"type":"integer"},"HttpMethod":{"type":"string"},"KeyValueStoreKey":{"properties":{"key":{"type":"string"},"recordPublicUrl":{"format":"uri","type":"string"},"size":{"type":"integer"}},"required":["key","size","recordPublicUrl"],"type":"object"},"KeyValueStoreResource":{"properties":{"accessedAt":{"format":"date-time","type":"string"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"consoleUrl":{"format":"uri","type":"string"},"createdAt":{"format":"date-time","type":"string"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"id":{"type":"string"},"keysPublicUrl":{"format":"uri","type":"string"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":["string","null"]},"recordsPublicUrl":{"format":"uri","type":"string"},"schema":{"type":["object","null"]},"stats":{"$ref":"#/components/schemas/KeyValueStoreStats"},"urlSigningSecretKey":{"type":["string","null"]},"userId":{"type":["string","null"]},"username":{"type":["string","null"]}},"required":["id","createdAt","modifiedAt","accessedAt"],"type":"object"},"KeyValueStoreResponse":{"properties":{"data":{"$ref":"#/components/schemas/KeyValueStoreResource"}},"required":["data"],"type":"object"},"KeyValueStoreStats":{"properties":{"deleteCount":{"type":"integer"},"listCount":{"type":"integer"},"readCount":{"type":"integer"},"s3StorageBytes":{"type":"integer"},"storageBytes":{"type":"integer"},"writeCount":{"type":"integer"}},"type":"object"},"Limits":{"properties":{"dataRetentionDays":{"type":"integer"},"maxActorCount":{"type":"integer"},"maxActorMemoryGbytes":{"type":"number"},"maxActorTaskCount":{"type":"integer"},"maxConcurrentActorJobs":{"type":"integer"},"maxMonthlyActorComputeUnits":{"type":"number"},"maxMonthlyExternalDataTransferGbytes":{"type":"number"},"maxMonthlyProxySerps":{"type":"integer"},"maxMonthlyResidentialProxyGbytes":{"type":"number"},"maxMonthlyUsageUsd":{"type":"number"},"maxScheduleCount":{"type":"integer"},"maxTeamAccountSeatCount":{"type":"integer"}},"required":["maxMonthlyUsageUsd","maxMonthlyActorComputeUnits","maxMonthlyExternalDataTransferGbytes","maxMonthlyProxySerps","maxMonthlyResidentialProxyGbytes","maxActorMemoryGbytes","maxActorCount","maxActorTaskCount","maxConcurrentActorJobs","maxTeamAccountSeatCount","dataRetentionDays"],"type":"object"},"ListOfActors":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/ActorListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfActorsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfActors"}},"required":["data"],"type":"object"},"ListOfBuilds":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/BuildListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfBuildsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfBuilds"}},"required":["data"],"type":"object"},"ListOfDatasets":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/DatasetListItem"},"type":"array"},"unnamed":{"type":"boolean"}},"required":["items"],"type":"object"}]},"ListOfDatasetsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfDatasets"}},"required":["data"],"type":"object"},"ListOfEnvVars":{"properties":{"items":{"items":{"$ref":"#/components/schemas/EnvVar"},"type":"array"},"total":{"type":"integer"}},"required":["total","items"],"type":"object"},"ListOfEnvVarsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfEnvVars"}},"required":["data"],"type":"object"},"ListOfKeyValueStores":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/KeyValueStoreResource"},"type":"array"},"unnamed":{"type":"boolean"}},"required":["items"],"type":"object"}]},"ListOfKeyValueStoresResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfKeyValueStores"}},"required":["data"],"type":"object"},"ListOfKeys":{"properties":{"count":{"type":"integer"},"exclusiveStartKey":{"type":["string","null"]},"isTruncated":{"type":"boolean"},"items":{"items":{"$ref":"#/components/schemas/KeyValueStoreKey"},"type":"array"},"limit":{"type":"integer"},"nextExclusiveStartKey":{"type":["string","null"]}},"required":["items","count","limit","isTruncated"],"type":"object"},"ListOfKeysResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfKeys"}},"required":["data"],"type":"object"},"ListOfRequestQueues":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/RequestQueueListItem"},"type":"array"},"unnamed":{"type":"boolean"}},"required":["items"],"type":"object"}]},"ListOfRequestQueuesResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfRequestQueues"}},"required":["data"],"type":"object"},"ListOfRequests":{"properties":{"cursor":{"type":"string"},"exclusiveStartId":{"type":"string"},"items":{"items":{"$ref":"#/components/schemas/RequestResource"},"type":"array"},"limit":{"type":"integer"},"nextCursor":{"type":"string"}},"required":["items","limit"],"type":"object"},"ListOfRequestsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfRequests"}},"required":["data"],"type":"object"},"ListOfRuns":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/RunListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfRunsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfRuns"}},"required":["data"],"type":"object"},"ListOfSchedules":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/ScheduleListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfSchedulesResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfSchedules"}},"required":["data"],"type":"object"},"ListOfStoreActors":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/StoreActor"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfStoreActorsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfStoreActors"}},"required":["data"],"type":"object"},"ListOfTasks":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/TaskListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfTasksResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfTasks"}},"required":["data"],"type":"object"},"ListOfVersions":{"properties":{"items":{"items":{"$ref":"#/components/schemas/Version"},"type":"array"},"total":{"type":"integer"}},"required":["total","items"],"type":"object"},"ListOfVersionsResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfVersions"}},"required":["data"],"type":"object"},"ListOfWebhookDispatches":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/WebhookDispatch"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfWebhookDispatchesResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfWebhookDispatches"}},"required":["data"],"type":"object"},"ListOfWebhooks":{"allOf":[{"$ref":"#/components/schemas/PaginationResponse"},{"properties":{"items":{"items":{"$ref":"#/components/schemas/WebhookListItem"},"type":"array"}},"required":["items"],"type":"object"}]},"ListOfWebhooksResponse":{"properties":{"data":{"$ref":"#/components/schemas/ListOfWebhooks"}},"required":["data"],"type":"object"},"LockExpiresAt":{"format":"date-time","type":"string"},"LockedRequestQueueHead":{"properties":{"clientKey":{"type":"string"},"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"items":{"items":{"$ref":"#/components/schemas/LockedRequestQueueHeadItem"},"type":"array"},"limit":{"$ref":"#/components/schemas/HeadLimit"},"lockSecs":{"type":"integer"},"queueHasLockedRequests":{"type":"boolean"},"queueModifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"}},"required":["limit","queueModifiedAt","hadMultipleClients","lockSecs","items"],"type":"object"},"LockedRequestQueueHeadItem":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"lockExpiresAt":{"$ref":"#/components/schemas/LockExpiresAt"},"method":{"$ref":"#/components/schemas/RequestMethod"},"retryCount":{"$ref":"#/components/schemas/RetryCount"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"}},"required":["id","uniqueKey","url","lockExpiresAt"],"type":"object"},"LockedRequestQueueHeadResponse":{"properties":{"data":{"$ref":"#/components/schemas/LockedRequestQueueHead"}},"required":["data"],"type":"object"},"MonthlyServiceUsage":{"type":"object"},"MonthlyUsage":{"properties":{"dailyServiceUsages":{"items":{"$ref":"#/components/schemas/DailyServiceUsage"},"type":"array"},"monthlyServiceUsage":{"$ref":"#/components/schemas/MonthlyServiceUsage"},"totalUsageCreditsUsdAfterVolumeDiscount":{"type":"number"},"totalUsageCreditsUsdBeforeVolumeDiscount":{"type":"number"},"usageCycle":{"$ref":"#/components/schemas/UsageCycle"}},"required":["usageCycle","monthlyServiceUsage","dailyServiceUsages","totalUsageCreditsUsdBeforeVolumeDiscount","totalUsageCreditsUsdAfterVolumeDiscount"],"type":"object"},"MonthlyUsageResponse":{"properties":{"data":{"$ref":"#/components/schemas/MonthlyUsage"}},"required":["data"],"type":"object"},"PaginationResponse":{"properties":{"count":{"minimum":0,"type":"integer"},"desc":{"type":"boolean"},"limit":{"minimum":1,"type":"integer"},"offset":{"minimum":0,"type":"integer"},"total":{"minimum":0,"type":"integer"}},"required":["total","offset","limit","desc","count"],"type":"object"},"PayPerEventActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"minimalMaxTotalChargeUsd":{"type":["number","null"]},"pricingModel":{"type":"string"},"pricingPerEvent":{"properties":{"actorChargeEvents":{"type":"object"}},"type":"object"}},"required":["pricingModel","pricingPerEvent"],"type":"object"}]},"PendingRequestCount":{"type":"integer"},"PricePerDatasetItemActorPricingInfo":{"allOf":[{"$ref":"#/components/schemas/CommonActorPricingInfo"},{"properties":{"pricePerUnitUsd":{"type":"number"},"pricingModel":{"type":"string"},"tieredPricing":{"$ref":"#/components/schemas/TieredPricingPerDatasetItem"},"unitName":{"type":"string"}},"required":["pricingModel","unitName"],"type":"object"}]},"ProlongRequestLockResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestLockInfo"}},"required":["data"],"type":"object"},"ProxyGroup":{"properties":{"availableCount":{"type":"integer"},"description":{"type":["string","null"]},"name":{"type":"string"}},"required":["name","description","availableCount"],"type":"object"},"ProxyResource":{"properties":{"groups":{"items":{"$ref":"#/components/schemas/ProxyGroup"},"type":"array"},"password":{"type":"string"}},"required":["password","groups"],"type":"object"},"QueueAccessedAt":{"format":"date-time","type":"string"},"QueueCreatedAt":{"format":"date-time","type":"string"},"QueueId":{"type":"string"},"QueueModifiedAt":{"format":"date-time","type":"string"},"QueueUserId":{"type":"string"},"RecordResponse":{"type":"object"},"RequestBase":{"properties":{"errorMessages":{"items":{"type":"string"},"type":["array","null"]},"handledAt":{"format":"date-time","type":["string","null"]},"headers":{"type":["object","null"]},"loadedUrl":{"type":["string","null"]},"method":{"$ref":"#/components/schemas/RequestMethod"},"noRetry":{"type":["boolean","null"]},"payload":{"type":["string","object","null"]},"retryCount":{"$ref":"#/components/schemas/RetryCount"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"},"userData":{"$ref":"#/components/schemas/RequestUserData"}},"required":["uniqueKey","url"],"type":"object"},"RequestId":{"type":"string"},"RequestLockInfo":{"properties":{"lockExpiresAt":{"$ref":"#/components/schemas/LockExpiresAt"}},"required":["lockExpiresAt"],"type":"object"},"RequestMethod":{"$ref":"#/components/schemas/HttpMethod"},"RequestQueueHead":{"properties":{"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"items":{"items":{"$ref":"#/components/schemas/RequestQueueHeadItem"},"type":"array"},"limit":{"$ref":"#/components/schemas/HeadLimit"},"queueModifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"}},"required":["limit","queueModifiedAt","hadMultipleClients","items"],"type":"object"},"RequestQueueHeadItem":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"method":{"$ref":"#/components/schemas/RequestMethod"},"retryCount":{"$ref":"#/components/schemas/RetryCount"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"}},"required":["id","uniqueKey","url"],"type":"object"},"RequestQueueHeadResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestQueueHead"}},"required":["data"],"type":"object"},"RequestQueueListItem":{"properties":{"accessedAt":{"$ref":"#/components/schemas/QueueAccessedAt"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"createdAt":{"$ref":"#/components/schemas/QueueCreatedAt"},"expireAt":{"format":"date-time","type":"string"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"handledRequestCount":{"$ref":"#/components/schemas/HandledRequestCount"},"id":{"$ref":"#/components/schemas/QueueId"},"modifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"},"name":{"type":"string"},"pendingRequestCount":{"$ref":"#/components/schemas/PendingRequestCount"},"stats":{"$ref":"#/components/schemas/RequestQueueStats"},"totalRequestCount":{"$ref":"#/components/schemas/TotalRequestCount"},"userId":{"$ref":"#/components/schemas/QueueUserId"},"username":{"type":"string"}},"required":["id","name","userId","username","createdAt","modifiedAt","accessedAt","totalRequestCount","handledRequestCount","pendingRequestCount","hadMultipleClients"],"type":"object"},"RequestQueueResource":{"properties":{"accessedAt":{"$ref":"#/components/schemas/QueueAccessedAt"},"actId":{"type":["string","null"]},"actRunId":{"type":["string","null"]},"consoleUrl":{"format":"uri","type":"string"},"createdAt":{"$ref":"#/components/schemas/QueueCreatedAt"},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"hadMultipleClients":{"$ref":"#/components/schemas/HadMultipleClients"},"handledRequestCount":{"$ref":"#/components/schemas/HandledRequestCount"},"id":{"$ref":"#/components/schemas/QueueId"},"modifiedAt":{"$ref":"#/components/schemas/QueueModifiedAt"},"name":{"type":["string","null"]},"pendingRequestCount":{"$ref":"#/components/schemas/PendingRequestCount"},"stats":{"$ref":"#/components/schemas/RequestQueueStats"},"totalRequestCount":{"$ref":"#/components/schemas/TotalRequestCount"},"userId":{"$ref":"#/components/schemas/QueueUserId"}},"required":["id","userId","createdAt","modifiedAt","accessedAt","totalRequestCount","handledRequestCount","pendingRequestCount","hadMultipleClients","consoleUrl"],"type":"object"},"RequestQueueResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestQueueResource"}},"required":["data"],"type":"object"},"RequestQueueStats":{"properties":{"deleteCount":{"type":"integer"},"headItemReadCount":{"type":"integer"},"readCount":{"type":"integer"},"storageBytes":{"type":"integer"},"writeCount":{"type":"integer"}},"type":"object"},"RequestRegistration":{"properties":{"requestId":{"$ref":"#/components/schemas/RequestId"},"wasAlreadyHandled":{"$ref":"#/components/schemas/WasAlreadyHandled"},"wasAlreadyPresent":{"$ref":"#/components/schemas/WasAlreadyPresent"}},"required":["requestId","wasAlreadyPresent","wasAlreadyHandled"],"type":"object"},"RequestResource":{"allOf":[{"$ref":"#/components/schemas/RequestBase","required":["id"]},{"properties":{"id":{"$ref":"#/components/schemas/RequestId"}}}]},"RequestResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestResource"}},"required":["data"],"type":"object"},"RequestUrl":{"type":"string"},"RequestUserData":{"type":"object"},"RetryCount":{"type":"integer"},"Run":{"properties":{"actId":{"type":"string"},"actorTaskId":{"type":["string","null"]},"buildId":{"type":"string"},"buildNumber":{"pattern":"^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])(\\.[1-9][0-9]{0,4})$","type":["string","null"]},"chargedEventCounts":{"type":"object"},"containerUrl":{"format":"uri","type":"string"},"defaultDatasetId":{"type":"string"},"defaultKeyValueStoreId":{"type":"string"},"defaultRequestQueueId":{"type":"string"},"exitCode":{"type":["integer","null"]},"finishedAt":{"format":"date-time","type":["string","null"]},"generalAccess":{"anyOf":[{"$ref":"#/components/schemas/GeneralAccess"},{"type":"null"}]},"gitBranchName":{"type":["string","null"]},"id":{"type":"string"},"isContainerServerReady":{"type":["boolean","null"]},"isStatusMessageTerminal":{"type":["boolean","null"]},"meta":{"$ref":"#/components/schemas/RunMeta"},"metamorphs":{"anyOf":[{"items":{"$ref":"#/components/schemas/RunMetamorphEvent"},"type":"array"},{"type":"null"}]},"options":{"$ref":"#/components/schemas/RunOptions"},"platformUsageBillingModel":{"type":"string"},"pricingInfo":{"$ref":"#/components/schemas/ActorRunPricingInfo"},"startedAt":{"format":"date-time","type":"string"},"stats":{"$ref":"#/components/schemas/RunStats"},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"statusMessage":{"type":["string","null"]},"storageIds":{"properties":{"datasets":{"properties":{"default":{"type":"string"}},"type":"object"},"keyValueStores":{"properties":{"default":{"type":"string"}},"type":"object"},"requestQueues":{"properties":{"default":{"type":"string"}},"type":"object"}},"type":"object"},"usage":{"anyOf":[{"$ref":"#/components/schemas/RunUsage"},{"type":"null"}]},"usageTotalUsd":{"type":["number","null"]},"usageUsd":{"anyOf":[{"$ref":"#/components/schemas/RunUsageUsd"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","actId","userId","startedAt","status","meta","stats","options","buildId","defaultKeyValueStoreId","defaultDatasetId","defaultRequestQueueId"],"type":"object"},"RunListItem":{"properties":{"actId":{"type":"string"},"actorTaskId":{"type":["string","null"]},"buildId":{"type":"string"},"buildNumber":{"pattern":"^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])(\\.[1-9][0-9]{0,4})$","type":"string"},"buildNumberInt":{"type":"integer"},"defaultDatasetId":{"type":"string"},"defaultKeyValueStoreId":{"type":"string"},"defaultRequestQueueId":{"type":"string"},"finishedAt":{"format":"date-time","type":["string","null"]},"id":{"type":"string"},"meta":{"$ref":"#/components/schemas/RunMeta"},"startedAt":{"format":"date-time","type":"string"},"status":{"$ref":"#/components/schemas/ActorJobStatus"},"usageTotalUsd":{"type":"number"},"userId":{"type":"string"}},"required":["id","actId","status","startedAt","buildId","meta","usageTotalUsd","defaultKeyValueStoreId","defaultDatasetId","defaultRequestQueueId"],"type":"object"},"RunMeta":{"properties":{"clientIp":{"type":["string","null"]},"origin":{"$ref":"#/components/schemas/RunOrigin"},"scheduleId":{"type":["string","null"]},"scheduledAt":{"format":"date-time","type":["string","null"]},"userAgent":{"type":["string","null"]}},"required":["origin"],"type":"object"},"RunMetamorphEvent":{"properties":{"actorId":{"type":"string"},"buildId":{"type":"string"},"createdAt":{"format":"date-time","type":"string"},"inputKey":{"type":["string","null"]}},"required":["createdAt","actorId","buildId"],"type":"object"},"RunOptions":{"properties":{"build":{"type":"string"},"diskMbytes":{"minimum":0,"type":"integer"},"maxItems":{"minimum":0,"type":["integer","null"]},"maxTotalChargeUsd":{"minimum":0,"type":["number","null"]},"memoryMbytes":{"maximum":32768,"minimum":128,"type":"integer"},"timeoutSecs":{"minimum":0,"type":"integer"}},"required":["build","timeoutSecs","memoryMbytes","diskMbytes"],"type":"object"},"RunOrigin":{"type":"string"},"RunResponse":{"properties":{"data":{"$ref":"#/components/schemas/Run"}},"required":["data"],"type":"object"},"RunStats":{"properties":{"computeUnits":{"minimum":0,"type":"number"},"cpuAvgUsage":{"type":"number"},"cpuCurrentUsage":{"type":"number"},"cpuMaxUsage":{"type":"number"},"durationMillis":{"minimum":0,"type":"integer"},"inputBodyLen":{"minimum":0,"type":["integer","null"]},"memAvgBytes":{"type":"number"},"memCurrentBytes":{"minimum":0,"type":"integer"},"memMaxBytes":{"minimum":0,"type":"integer"},"metamorph":{"minimum":0,"type":"integer"},"migrationCount":{"minimum":0,"type":"integer"},"netRxBytes":{"minimum":0,"type":"integer"},"netTxBytes":{"minimum":0,"type":"integer"},"rebootCount":{"minimum":0,"type":"integer"},"restartCount":{"minimum":0,"type":"integer"},"resurrectCount":{"minimum":0,"type":"integer"},"runTimeSecs":{"minimum":0,"type":"number"}},"type":"object"},"RunUsage":{"properties":{"ACTOR_COMPUTE_UNITS":{"type":["number","null"]},"DATASET_READS":{"type":["integer","null"]},"DATASET_WRITES":{"type":["integer","null"]},"DATA_TRANSFER_EXTERNAL_GBYTES":{"type":["number","null"]},"DATA_TRANSFER_INTERNAL_GBYTES":{"type":["number","null"]},"KEY_VALUE_STORE_LISTS":{"type":["integer","null"]},"KEY_VALUE_STORE_READS":{"type":["integer","null"]},"KEY_VALUE_STORE_WRITES":{"type":["integer","null"]},"PROXY_RESIDENTIAL_TRANSFER_GBYTES":{"type":["number","null"]},"PROXY_SERPS":{"type":["integer","null"]},"REQUEST_QUEUE_READS":{"type":["integer","null"]},"REQUEST_QUEUE_WRITES":{"type":["integer","null"]}},"type":"object"},"RunUsageUsd":{"properties":{"ACTOR_COMPUTE_UNITS":{"type":["number","null"]},"DATASET_READS":{"type":["number","null"]},"DATASET_WRITES":{"type":["number","null"]},"DATA_TRANSFER_EXTERNAL_GBYTES":{"type":["number","null"]},"DATA_TRANSFER_INTERNAL_GBYTES":{"type":["number","null"]},"KEY_VALUE_STORE_LISTS":{"type":["number","null"]},"KEY_VALUE_STORE_READS":{"type":["number","null"]},"KEY_VALUE_STORE_WRITES":{"type":["number","null"]},"PROXY_RESIDENTIAL_TRANSFER_GBYTES":{"type":["number","null"]},"PROXY_SERPS":{"type":["number","null"]},"REQUEST_QUEUE_READS":{"type":["number","null"]},"REQUEST_QUEUE_WRITES":{"type":["number","null"]}},"type":"object"},"Schedule":{"allOf":[{"$ref":"#/components/schemas/ScheduleBase"},{"properties":{"actions":{"items":{"$ref":"#/components/schemas/ScheduleAction"},"type":"array"},"description":{"type":["string","null"]},"notifications":{"$ref":"#/components/schemas/ScheduleNotifications"},"title":{"type":["string","null"]}},"required":["actions"],"type":"object"}]},"ScheduleAction":{"oneOf":[{"$ref":"#/components/schemas/ScheduleActionRunActor"},{"$ref":"#/components/schemas/ScheduleActionRunActorTask"}]},"ScheduleActionRunActor":{"properties":{"actorId":{"type":"string"},"id":{"type":"string"},"runInput":{"anyOf":[{"$ref":"#/components/schemas/ScheduleActionRunInput"},{"type":"null"}]},"runOptions":{"anyOf":[{"$ref":"#/components/schemas/TaskOptions"},{"type":"null"}]},"type":{}},"required":["id","type","actorId"],"type":"object"},"ScheduleActionRunActorTask":{"properties":{"actorTaskId":{"type":"string"},"id":{"type":"string"},"input":{"anyOf":[{"type":"object"},{"type":"null"}]},"type":{}},"required":["id","type","actorTaskId"],"type":"object"},"ScheduleActionRunInput":{"properties":{"body":{"type":["string","null"]},"contentType":{"type":["string","null"]}},"type":"object"},"ScheduleBase":{"properties":{"createdAt":{"format":"date-time","type":"string"},"cronExpression":{"type":"string"},"id":{"type":"string"},"isEnabled":{"type":"boolean"},"isExclusive":{"type":"boolean"},"lastRunAt":{"format":"date-time","type":["string","null"]},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":"string"},"nextRunAt":{"format":"date-time","type":["string","null"]},"timezone":{"type":"string"},"userId":{"type":"string"}},"required":["id","userId","name","cronExpression","timezone","isEnabled","isExclusive","createdAt","modifiedAt"],"type":"object"},"ScheduleListItem":{"allOf":[{"$ref":"#/components/schemas/ScheduleBase"},{"properties":{"actions":{"items":{"$ref":"#/components/schemas/ScheduleListItemAction"},"type":"array"}},"required":["actions"],"type":"object"}]},"ScheduleListItemAction":{"oneOf":[{"$ref":"#/components/schemas/ScheduleListItemActionRunActor"},{"$ref":"#/components/schemas/ScheduleListItemActionRunActorTask"}]},"ScheduleListItemActionRunActor":{"properties":{"actorId":{"type":"string"},"id":{"type":"string"},"type":{}},"required":["id","type","actorId"],"type":"object"},"ScheduleListItemActionRunActorTask":{"properties":{"actorTaskId":{"type":"string"},"id":{"type":"string"},"type":{}},"required":["id","type","actorTaskId"],"type":"object"},"ScheduleLogEntry":{"properties":{"createdAt":{"format":"date-time","type":"string"},"level":{"type":"string"},"message":{"type":"string"}},"required":["message","level","createdAt"],"type":"object"},"ScheduleLogResponse":{"properties":{"data":{"items":{"$ref":"#/components/schemas/ScheduleLogEntry"},"type":"array"}},"required":["data"],"type":"object"},"ScheduleNotifications":{"properties":{"email":{"type":"boolean"}},"type":"object"},"ScheduleResponse":{"properties":{"data":{"$ref":"#/components/schemas/Schedule"}},"required":["data"],"type":"object"},"ServiceUsage":{"type":"object"},"SourceCodeFile":{"properties":{"content":{"type":"string"},"format":{"$ref":"#/components/schemas/SourceCodeFileFormat"},"name":{"type":"string"}},"required":["name"],"type":"object"},"SourceCodeFileFormat":{"type":"string"},"SourceCodeFolder":{"properties":{"folder":{"type":"boolean"},"name":{"type":"string"}},"required":["name","folder"],"type":"object"},"StoreActor":{"properties":{"actorReviewCount":{"type":"integer"},"actorReviewRating":{"type":"number"},"badge":{"type":["string","null"]},"bookmarkCount":{"type":"integer"},"categories":{"items":{"type":"string"},"type":"array"},"currentPricingInfo":{"$ref":"#/components/schemas/CurrentPricingInfo"},"description":{"type":["string","null"]},"id":{"type":"string"},"isWhiteListedForAgenticPayments":{"type":["boolean","null"]},"name":{"type":"string"},"notice":{"$ref":"#/components/schemas/ActorNotice"},"pictureUrl":{"type":["string","null"]},"readmeSummary":{"type":"string"},"stats":{"$ref":"#/components/schemas/ActorStats"},"title":{"type":"string"},"url":{"format":"uri","type":["string","null"]},"userFullName":{"type":["string","null"]},"userPictureUrl":{"format":"uri","type":["string","null"]},"username":{"type":"string"}},"required":["id","title","name","username","stats"],"type":"object"},"TaggedBuilds":{"type":"object"},"Task":{"properties":{"actId":{"type":"string"},"actorStandby":{"anyOf":[{"$ref":"#/components/schemas/ActorStandby"},{"type":"null"}]},"createdAt":{"format":"date-time","type":"string"},"description":{"type":["string","null"]},"id":{"type":"string"},"input":{"anyOf":[{"$ref":"#/components/schemas/TaskInput"},{"items":{"$ref":"#/components/schemas/TaskInput"},"type":"array"},{"type":"null"}]},"isPublic":{"type":"boolean"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":"string"},"options":{"anyOf":[{"$ref":"#/components/schemas/TaskOptions"},{"type":"null"}]},"publicConfig":{"anyOf":[{"$ref":"#/components/schemas/TaskPublicConfig"},{"type":"null"}]},"removedAt":{"format":"date-time","type":["string","null"]},"standbyUrl":{"format":"uri","type":["string","null"]},"stats":{"anyOf":[{"$ref":"#/components/schemas/TaskStats"},{"type":"null"}]},"title":{"type":["string","null"]},"userId":{"type":"string"},"username":{"type":["string","null"]}},"required":["id","userId","actId","name","createdAt","modifiedAt"],"type":"object"},"TaskInput":{"type":"object"},"TaskListItem":{"properties":{"actId":{"type":"string"},"actName":{"type":["string","null"]},"actUsername":{"type":["string","null"]},"createdAt":{"format":"date-time","type":"string"},"id":{"type":"string"},"modifiedAt":{"format":"date-time","type":"string"},"name":{"type":"string"},"stats":{"anyOf":[{"$ref":"#/components/schemas/TaskStats"},{"type":"null"}]},"userId":{"type":"string"},"username":{"type":["string","null"]}},"required":["id","userId","actId","name","createdAt","modifiedAt"],"type":"object"},"TaskOptions":{"properties":{"build":{"type":["string","null"]},"maxItems":{"type":["integer","null"]},"maxTotalChargeUsd":{"type":["number","null"]},"memoryMbytes":{"type":["integer","null"]},"restartOnError":{"type":["boolean","null"]},"timeoutSecs":{"type":["integer","null"]}},"type":"object"},"TaskPublicConfig":{"properties":{"datasetName":{"type":["string","null"]},"datasetView":{"type":["string","null"]},"inputSchemaFields":{"items":{"type":"string"},"type":["array","null"]},"publishedAt":{"format":"date-time","type":["string","null"]},"seoDescription":{"maxLength":160,"type":["string","null"]},"seoTitle":{"maxLength":60,"type":["string","null"]}},"type":"object"},"TaskResponse":{"properties":{"data":{"$ref":"#/components/schemas/Task"}},"required":["data"],"type":"object"},"TaskStats":{"properties":{"totalRuns":{"type":"integer"}},"type":"object"},"TieredPricingPerDatasetItem":{"type":"object"},"TotalRequestCount":{"minimum":0,"type":"integer"},"UniqueKey":{"type":"string"},"UnlockRequestsResponse":{"properties":{"data":{"$ref":"#/components/schemas/UnlockRequestsResult"}},"required":["data"],"type":"object"},"UnlockRequestsResult":{"properties":{"unlockedCount":{"type":"integer"}},"required":["unlockedCount"],"type":"object"},"UnprocessedRequest":{"properties":{"id":{"$ref":"#/components/schemas/RequestId"},"method":{"$ref":"#/components/schemas/RequestMethod"},"uniqueKey":{"$ref":"#/components/schemas/UniqueKey"},"url":{"$ref":"#/components/schemas/RequestUrl"}},"required":["uniqueKey","url"],"type":"object"},"UpdateRequestResponse":{"properties":{"data":{"$ref":"#/components/schemas/RequestRegistration"}},"required":["data"],"type":"object"},"UsageCycle":{"properties":{"endAt":{"format":"date-time","type":"string"},"startAt":{"format":"date-time","type":"string"}},"required":["startAt","endAt"],"type":"object"},"UserPlan":{"properties":{"apiRateLimitBoosts":{"type":"integer"},"availableAddOns":{"items":{"type":"string"},"type":"array"},"availableProxyGroups":{"$ref":"#/components/schemas/AvailableProxyGroups"},"dataRetentionDays":{"type":"integer"},"description":{"type":"string"},"enabledPlatformFeatures":{"items":{"type":"string"},"type":"array"},"id":{"type":"string"},"isEnabled":{"type":"boolean"},"maxActorCount":{"type":"integer"},"maxActorMemoryGbytes":{"type":"number"},"maxActorTaskCount":{"type":"integer"},"maxConcurrentActorRuns":{"type":"integer"},"maxMonthlyActorComputeUnits":{"type":"number"},"maxMonthlyExternalDataTransferGbytes":{"type":"number"},"maxMonthlyProxySerps":{"type":"integer"},"maxMonthlyResidentialProxyGbytes":{"type":"number"},"maxMonthlyUsageUsd":{"type":"number"},"maxScheduleCount":{"type":"integer"},"monthlyBasePriceUsd":{"type":"number"},"monthlyUsageCreditsUsd":{"type":"number"},"planPricing":{"type":"object"},"supportLevel":{"type":"string"},"teamAccountSeatCount":{"type":"integer"},"tier":{"type":"string"},"usageDiscountPercent":{"type":"number"}},"required":["availableProxyGroups"],"type":"object"},"UserPrivateInfo":{"properties":{"createdAt":{"format":"date-time","type":"string"},"effectivePlatformFeatures":{"$ref":"#/components/schemas/EffectivePlatformFeatures"},"email":{"format":"email","type":"string"},"id":{"type":"string"},"isPaying":{"type":"boolean"},"plan":{"$ref":"#/components/schemas/UserPlan"},"profile":{"$ref":"#/components/schemas/UserProfile"},"proxy":{"$ref":"#/components/schemas/ProxyResource"},"username":{"type":"string"}},"required":["username"],"type":"object"},"UserPrivateInfoResponse":{"properties":{"data":{"$ref":"#/components/schemas/UserPrivateInfo"}},"required":["data"],"type":"object"},"UserProfile":{"properties":{"bio":{"type":["string","null"]},"githubUsername":{"type":["string","null"]},"name":{"type":"string"},"pictureUrl":{"format":"uri","type":["string","null"]},"readme":{"type":["string","null"]},"twitterUsername":{"type":["string","null"]},"websiteUrl":{"format":"uri","type":["string","null"]}},"type":"object"},"UserPublicInfo":{"properties":{"profile":{"$ref":"#/components/schemas/UserProfile"},"username":{"type":"string"}},"required":["username"],"type":"object"},"UserPublicInfoResponse":{"properties":{"data":{"$ref":"#/components/schemas/UserPublicInfo"}},"required":["data"],"type":"object"},"Version":{"properties":{"applyEnvVarsToBuild":{"type":["boolean","null"]},"buildTag":{"type":["string","null"]},"envVars":{"anyOf":[{"items":{"$ref":"#/components/schemas/EnvVar"},"type":"array"},{"type":"null"}]},"gitHubGistUrl":{"type":["string","null"]},"gitRepoUrl":{"type":["string","null"]},"sourceFiles":{"$ref":"#/components/schemas/VersionSourceFiles"},"sourceType":{"anyOf":[{"$ref":"#/components/schemas/VersionSourceType"},{"type":"null"}]},"tarballUrl":{"type":["string","null"]},"versionNumber":{"pattern":"^([0-9]|[1-9][0-9])\\.([0-9]|[1-9][0-9])$","type":"string"}},"required":["versionNumber","sourceType"],"type":"object"},"VersionResponse":{"properties":{"data":{"$ref":"#/components/schemas/Version"}},"required":["data"],"type":"object"},"VersionSourceFiles":{"items":{"anyOf":[{"$ref":"#/components/schemas/SourceCodeFile"},{"$ref":"#/components/schemas/SourceCodeFolder"}]},"type":"array"},"VersionSourceType":{"type":"string"},"WasAlreadyHandled":{"type":"boolean"},"WasAlreadyPresent":{"type":"boolean"},"WebhookCondition":{"properties":{"actorId":{"type":["string","null"]},"actorRunId":{"type":["string","null"]},"actorTaskId":{"type":["string","null"]}},"type":"object"},"WebhookDispatch":{"properties":{"calls":{"items":{"properties":{"errorMessage":{"type":["string","null"]},"finishedAt":{"format":"date-time","type":["string","null"]},"responseBody":{"type":["string","null"]},"responseStatus":{"type":["integer","null"]},"startedAt":{"format":"date-time","type":["string","null"]}},"type":"object"},"type":"array"},"createdAt":{"format":"date-time","type":"string"},"eventData":{"properties":{"actorBuildId":{"type":"string"},"actorId":{"type":"string"},"actorRunId":{"type":"string"},"actorTaskId":{"type":["string","null"]}},"required":["actorId"],"type":["object","null"]},"eventType":{"$ref":"#/components/schemas/WebhookEventType"},"id":{"type":"string"},"status":{"$ref":"#/components/schemas/WebhookDispatchStatus"},"userId":{"type":"string"},"webhook":{"anyOf":[{"$ref":"#/components/schemas/WebhookDispatchWebhookSummary"},{"type":"null"}]},"webhookId":{"type":"string"}},"required":["id","userId","webhookId","createdAt","status","eventType"],"type":"object"},"WebhookDispatchResponse":{"properties":{"data":{"$ref":"#/components/schemas/WebhookDispatch"}},"required":["data"],"type":"object"},"WebhookDispatchStatus":{"type":"string"},"WebhookDispatchWebhookSummary":{"properties":{"actionType":{"type":"string"},"condition":{"$ref":"#/components/schemas/WebhookCondition"},"isAdHoc":{"type":"boolean"},"requestUrl":{"format":"uri","type":["string","null"]}},"type":"object"},"WebhookEventType":{"type":"string"},"WebhookLastDispatch":{"properties":{"finishedAt":{"format":"date-time","type":["string","null"]},"removedAt":{"format":"date-time","type":["string","null"]},"status":{"$ref":"#/components/schemas/WebhookDispatchStatus"}},"required":["status"],"type":"object"},"WebhookListItem":{"properties":{"actionType":{"type":"string"},"condition":{"$ref":"#/components/schemas/WebhookCondition"},"createdAt":{"format":"date-time","type":"string"},"doNotRetry":{"type":"boolean"},"eventTypes":{"items":{"$ref":"#/components/schemas/WebhookEventType"},"type":"array"},"id":{"type":"string"},"ignoreSslErrors":{"type":"boolean"},"isAdHoc":{"type":["boolean","null"]},"isApifyIntegration":{"type":"boolean"},"isEnabled":{"type":"boolean"},"lastDispatch":{"anyOf":[{"$ref":"#/components/schemas/WebhookLastDispatch"},{"type":"null"}]},"modifiedAt":{"format":"date-time","type":"string"},"requestUrl":{"format":"uri","type":"string"},"shouldInterpolateStrings":{"type":["boolean","null"]},"stats":{"anyOf":[{"$ref":"#/components/schemas/WebhookStats"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","createdAt","modifiedAt","userId","eventTypes","condition","ignoreSslErrors","doNotRetry","requestUrl"],"type":"object"},"WebhookResource":{"properties":{"condition":{"$ref":"#/components/schemas/WebhookCondition"},"createdAt":{"format":"date-time","type":"string"},"description":{"type":["string","null"]},"doNotRetry":{"type":["boolean","null"]},"eventTypes":{"items":{"$ref":"#/components/schemas/WebhookEventType"},"type":"array"},"headersTemplate":{"type":["string","null"]},"id":{"type":"string"},"ignoreSslErrors":{"type":"boolean"},"isAdHoc":{"type":["boolean","null"]},"lastDispatch":{"anyOf":[{"$ref":"#/components/schemas/WebhookLastDispatch"},{"type":"null"}]},"modifiedAt":{"format":"date-time","type":"string"},"payloadTemplate":{"type":["string","null"]},"requestUrl":{"format":"uri","type":["string","null"]},"shouldInterpolateStrings":{"type":["boolean","null"]},"stats":{"anyOf":[{"$ref":"#/components/schemas/WebhookStats"},{"type":"null"}]},"userId":{"type":"string"}},"required":["id","createdAt","modifiedAt","userId","eventTypes","condition","ignoreSslErrors"],"type":"object"},"WebhookResponse":{"properties":{"data":{"$ref":"#/components/schemas/WebhookResource"}},"required":["data"],"type":"object"},"WebhookStats":{"properties":{"totalDispatches":{"type":"integer"}},"type":"object"}},"specVersion":"v2-2026-10-20T121847Z"} diff --git a/src/Apify.Client/Options/ActorListOptions.cs b/src/Apify.Client/Options/ActorListOptions.cs index 3acf7a8..442b30c 100644 --- a/src/Apify.Client/Options/ActorListOptions.cs +++ b/src/Apify.Client/Options/ActorListOptions.cs @@ -11,6 +11,13 @@ public sealed class ActorListOptions /// Maximum number of Actors to return. public int? Limit { get; init; } + /// + /// Maximum number of items requested per API call when iterating across pages (must be positive). Only + /// iteration uses it, as the page size; it is never sent to the API. The effective page size is the + /// smaller of this and the remaining Limit. Defaults to the API's own page size. + /// + public int? ChunkSize { get; init; } + /// If true, return Actors newest-first. public bool? Desc { get; init; } diff --git a/src/Apify.Client/Options/DatasetListItemsOptions.cs b/src/Apify.Client/Options/DatasetListItemsOptions.cs index c8f4edf..ca1adf8 100644 --- a/src/Apify.Client/Options/DatasetListItemsOptions.cs +++ b/src/Apify.Client/Options/DatasetListItemsOptions.cs @@ -12,9 +12,20 @@ public sealed class DatasetListItemsOptions /// Number of items to skip. public int? Offset { get; init; } - /// Maximum number of items to return. + /// + /// Maximum number of dataset rows to read. Filters can return fewer items than rows read and + /// more. When iterating, it caps the rows read in total across all pages; 0 + /// or unset means no cap. + /// public int? Limit { get; init; } + /// + /// Maximum number of items requested per API call when iterating across pages (must be positive). Only + /// iteration uses it, as the page size; it is never sent to the API. The effective page size is the + /// smaller of this and the remaining Limit. Defaults to the API's own page size. + /// + public int? ChunkSize { get; init; } + /// Return items newest-first. public bool? Desc { get; init; } diff --git a/src/Apify.Client/Options/IterateKeysOptions.cs b/src/Apify.Client/Options/IterateKeysOptions.cs index e560b5f..8385dc6 100644 --- a/src/Apify.Client/Options/IterateKeysOptions.cs +++ b/src/Apify.Client/Options/IterateKeysOptions.cs @@ -14,8 +14,15 @@ public sealed class IterateKeysOptions public int? Limit { get; init; } /// - /// Maximum number of keys fetched per page (defaults to ). Never sent - /// to the API as-is; it only sizes each page request, matching the reference client's chunkSize. + /// Maximum number of keys requested per API call (must be positive; defaults to + /// ). Never sent to the API as-is; it only sizes each page request. + /// Matches the reference client's chunkSize. + /// + public int? ChunkSize { get; init; } + + /// + /// The earlier name of , kept so existing callers keep working. + /// takes precedence when both are set. /// public int? MaxPageLimit { get; init; } diff --git a/src/Apify.Client/Options/ListOptions.cs b/src/Apify.Client/Options/ListOptions.cs index e028c4d..521899c 100644 --- a/src/Apify.Client/Options/ListOptions.cs +++ b/src/Apify.Client/Options/ListOptions.cs @@ -15,6 +15,13 @@ public sealed class ListOptions /// Maximum number of items to return. public int? Limit { get; init; } + /// + /// Maximum number of items requested per API call when iterating across pages (must be positive). Only + /// iteration uses it, as the page size; it is never sent to the API. The effective page size is the + /// smaller of this and the remaining Limit. Defaults to the API's own page size. + /// + public int? ChunkSize { get; init; } + /// If true, return items newest-first. public bool? Desc { get; init; } diff --git a/src/Apify.Client/Options/PaginateRequestsOptions.cs b/src/Apify.Client/Options/PaginateRequestsOptions.cs index 6db4d7f..3a3f008 100644 --- a/src/Apify.Client/Options/PaginateRequestsOptions.cs +++ b/src/Apify.Client/Options/PaginateRequestsOptions.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using Apify.Client.Internal; namespace Apify.Client.Options; @@ -23,7 +24,16 @@ public sealed class PaginateRequestsOptions /// Maximum total number of requests to iterate across all pages (null for no bound). public int? Limit { get; init; } - /// Maximum number of requests fetched per page (defaults to ). + /// + /// Maximum number of requests fetched per API call (must be positive; defaults to + /// ). Matches the reference client's chunkSize. + /// + public int? ChunkSize { get; init; } + + /// + /// The earlier name of , kept so existing callers keep working. + /// takes precedence when both are set. + /// public int? MaxPageLimit { get; init; } /// Start iterating after this request ID (first page only; mutually exclusive with cursor). @@ -41,6 +51,9 @@ public sealed class PaginateRequestsOptions /// Validates the options for API-level constraints. internal void Validate() { + PageSizing.ValidateChunkSize(ChunkSize, nameof(ChunkSize)); + PageSizing.ValidateChunkSize(MaxPageLimit, nameof(MaxPageLimit)); + if (ExclusiveStartId is not null && Cursor is not null) { throw new ArgumentException("PaginateRequestsOptions: ExclusiveStartId and Cursor are mutually exclusive"); diff --git a/src/Apify.Client/Options/RunChargeOptions.cs b/src/Apify.Client/Options/RunChargeOptions.cs index 3c6d829..3e82fc0 100644 --- a/src/Apify.Client/Options/RunChargeOptions.cs +++ b/src/Apify.Client/Options/RunChargeOptions.cs @@ -26,6 +26,13 @@ public RunChargeOptions(string eventName, int? count = null, string? idempotency /// A key that deduplicates the charge across retries. public string? IdempotencyKey { get; } + /// + /// The ID of the Server Actor request being charged, taken from the X-Actor-Request-ID header of + /// the incoming request. Required for multi-tenant Server Actor runs, where it identifies the request + /// the charge belongs to. + /// + public string? RequestId { get; init; } + /// The count to send, defaulting to 1. internal int CountValue() => Count ?? 1; } diff --git a/src/Apify.Client/Options/RunIterateDatasetItemsOptions.cs b/src/Apify.Client/Options/RunIterateDatasetItemsOptions.cs new file mode 100644 index 0000000..abd5bd7 --- /dev/null +++ b/src/Apify.Client/Options/RunIterateDatasetItemsOptions.cs @@ -0,0 +1,56 @@ +using System.Collections.Generic; + +namespace Apify.Client.Options; + +/// +/// Configures , which iterates over +/// the items of a run's default dataset while the run is still producing them. Mirrors the reference +/// client's RunIterateDatasetItemsOptions. All fields are optional. +/// +public sealed class RunIterateDatasetItemsOptions +{ + /// Default maximum number of dataset rows requested per API call. + public const int DefaultChunkSize = 1000; + + /// Default time, in seconds, to wait for the run to finish between polls. + public const int DefaultPollIntervalSecs = 5; + + /// Number of dataset rows to skip from the beginning (defaults to 0). + public int? Offset { get; init; } + + /// + /// Maximum number of dataset rows to scan (null or 0 for no limit). Fewer items are + /// yielded when filters drop some, more when splits a row into several. + /// + public int? Limit { get; init; } + + /// + /// Maximum number of dataset rows requested per API call (must be positive; defaults to + /// ). + /// + public int? ChunkSize { get; init; } + + /// How long to wait for the run to finish between polls, in seconds (defaults to ). + public int? PollIntervalSecs { get; init; } + + /// Return only non-empty items and skip hidden fields. + public bool? Clean { get; init; } + + /// Restrict the output to these fields. + public IReadOnlyList? Fields { get; init; } + + /// Exclude these fields from the output. + public IReadOnlyList? Omit { get; init; } + + /// Expand these fields (each array element becomes a separate item). + public IReadOnlyList? Unwind { get; init; } + + /// Skip empty items. + public bool? SkipEmpty { get; init; } + + /// Skip hidden fields (those starting with #). + public bool? SkipHidden { get; init; } + + /// Whether any filter that can drop or split rows (, , ) is set. + internal bool HasRowFilters => Clean == true || SkipEmpty == true || Unwind is { Count: > 0 }; +} diff --git a/src/Apify.Client/Options/StorageListOptions.cs b/src/Apify.Client/Options/StorageListOptions.cs index 1428a11..259162e 100644 --- a/src/Apify.Client/Options/StorageListOptions.cs +++ b/src/Apify.Client/Options/StorageListOptions.cs @@ -15,6 +15,13 @@ public sealed class StorageListOptions /// Maximum number of items to return. public int? Limit { get; init; } + /// + /// Maximum number of items requested per API call when iterating across pages (must be positive). Only + /// iteration uses it, as the page size; it is never sent to the API. The effective page size is the + /// smaller of this and the remaining Limit. Defaults to the API's own page size. + /// + public int? ChunkSize { get; init; } + /// If true, return items newest-first. public bool? Desc { get; init; } diff --git a/src/Apify.Client/Options/StoreListOptions.cs b/src/Apify.Client/Options/StoreListOptions.cs index 5b8070b..a7a1f64 100644 --- a/src/Apify.Client/Options/StoreListOptions.cs +++ b/src/Apify.Client/Options/StoreListOptions.cs @@ -8,9 +8,16 @@ public sealed class StoreListOptions /// Number of Actors to skip. public int? Offset { get; init; } - /// Maximum number of Actors to return (also the per-page size when iterating). + /// Maximum number of Actors to return (when iterating, in total across all pages). public int? Limit { get; init; } + /// + /// Maximum number of items requested per API call when iterating across pages (must be positive). Only + /// iteration uses it, as the page size; it is never sent to the API. The effective page size is the + /// smaller of this and the remaining Limit. Defaults to the API's own page size. + /// + public int? ChunkSize { get; init; } + /// Full-text search query. public string? Search { get; init; } @@ -24,8 +31,8 @@ public sealed class StoreListOptions public string? Username { get; init; } /// - /// Filter Actors by pricing model (FREE, FLAT_PRICE_PER_MONTH, - /// PRICE_PER_DATASET_ITEM, PAY_PER_EVENT). + /// Filter Actors by pricing model (FREE, PRICE_PER_DATASET_ITEM, + /// PAY_PER_EVENT). The rental model (FLAT_PRICE_PER_MONTH) is no longer supported. /// public string? PricingModel { get; init; } @@ -38,21 +45,6 @@ public sealed class StoreListOptions /// The response format (full, agent). public string? ResponseFormat { get; init; } - /// Returns a copy of these options with a new (used by lazy iteration). - internal StoreListOptions WithOffset(int? offset) => new() - { - Offset = offset, - Limit = Limit, - Search = Search, - SortBy = SortBy, - Category = Category, - Username = Username, - PricingModel = PricingModel, - IncludeUnrunnableActors = IncludeUnrunnableActors, - AllowsAgenticUsers = AllowsAgenticUsers, - ResponseFormat = ResponseFormat, - }; - internal void AppendTo(QueryParams q) { q.AddInt("offset", Offset) diff --git a/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs b/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs index 93a3b6f..91470ca 100644 --- a/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs +++ b/src/Apify.Client/Resources/AbstractWebhookCollectionClient.cs @@ -41,7 +41,7 @@ public IAsyncEnumerable IterateAsync(ListOptions? options = null, Cance options ??= new ListOptions(); var q = new QueryParams(); options.AppendTo(q); - return Ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Webhook(d), RequestTier.Medium, cancellationToken); + return Ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new Webhook(d), RequestTier.Medium, cancellationToken); } } diff --git a/src/Apify.Client/Resources/ActorClient.cs b/src/Apify.Client/Resources/ActorClient.cs index 5a8e61a..7268942 100644 --- a/src/Apify.Client/Resources/ActorClient.cs +++ b/src/Apify.Client/Resources/ActorClient.cs @@ -148,7 +148,7 @@ public async Task DefaultBuildAsync(int? waitForFinish = null, Canc q.AddInt("waitForFinish", waitForFinish); // Long: waitForFinish is sent unclamped, so the HTTP timeout must cover whatever the caller asks for. var data = await _ctx.GetResourceRequiredAsync("builds/default", q, RequestTier.Long, cancellationToken).ConfigureAwait(false); - var build = new Build(data as JsonObject ?? new JsonObject()); + var build = new Build(data); return new BuildClient(_http, _baseUrl, build.Id ?? string.Empty); } @@ -156,7 +156,7 @@ public async Task DefaultBuildAsync(int? waitForFinish = null, Canc /// Optional last-run filters. public RunClient LastRun(LastRunOptions? options = null) { - var client = new RunClient(_http, _ctx.SubUrl(""), "runs", "last"); + var client = new RunClient(_root, _http, _ctx.SubUrl(""), "runs", "last"); client.SetLastRunParams(options ?? new LastRunOptions()); return client; } diff --git a/src/Apify.Client/Resources/ActorCollectionClient.cs b/src/Apify.Client/Resources/ActorCollectionClient.cs index 1c88039..cb3b01a 100644 --- a/src/Apify.Client/Resources/ActorCollectionClient.cs +++ b/src/Apify.Client/Resources/ActorCollectionClient.cs @@ -39,7 +39,7 @@ public IAsyncEnumerable IterateAsync(ActorListOptions? options = null, Ca options ??= new ActorListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Actor(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new Actor(d), RequestTier.Medium, cancellationToken); } /// Creates a new Actor. diff --git a/src/Apify.Client/Resources/BuildClient.cs b/src/Apify.Client/Resources/BuildClient.cs index 3b58c6e..6c35e95 100644 --- a/src/Apify.Client/Resources/BuildClient.cs +++ b/src/Apify.Client/Resources/BuildClient.cs @@ -63,12 +63,11 @@ public async Task WaitForFinishAsync(int? waitSecs = null, CancellationTo /// A token to cancel the request. public async Task GetOpenApiDefinitionAsync(CancellationToken cancellationToken = default) { - var body = await _ctx.GetRawAsync("openapi.json", new QueryParams(), RequestTier.Medium, cancellationToken).ConfigureAwait(false); - return body is null ? null : Json.Decode(body) as JsonObject; + return await _ctx.GetJsonAsync("openapi.json", new QueryParams(), RequestTier.Medium, cancellationToken).ConfigureAwait(false) as JsonObject; } /// A client for accessing this build's log. - public LogClient Log() => LogClient.Nested(_http, _ctx.SubUrl("")); + public LogClient Log() => LogClient.Nested(_http, _ctx.SubUrl(""), timeout: _ctx.RequestTimeout); /// /// Returns this client with every subsequent call's timeout set to , diff --git a/src/Apify.Client/Resources/BuildCollectionClient.cs b/src/Apify.Client/Resources/BuildCollectionClient.cs index 55c3b9d..ed355cd 100644 --- a/src/Apify.Client/Resources/BuildCollectionClient.cs +++ b/src/Apify.Client/Resources/BuildCollectionClient.cs @@ -40,7 +40,7 @@ public IAsyncEnumerable IterateAsync(ListOptions? options = null, Cancell options ??= new ListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Build(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new Build(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/DatasetClient.cs b/src/Apify.Client/Resources/DatasetClient.cs index 48c6b19..ea41489 100644 --- a/src/Apify.Client/Resources/DatasetClient.cs +++ b/src/Apify.Client/Resources/DatasetClient.cs @@ -28,8 +28,8 @@ private DatasetClient(HttpClientCore http, ResourceContext ctx) internal static DatasetClient ForId(HttpClientCore http, string baseUrl, string id) => new(http, ResourceContext.Single(http, baseUrl, "datasets", id)); - internal static DatasetClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null) - => new(http, ResourceContext.NestedSingleton(http, baseUrl, subPath, inheritedParams)); + internal static DatasetClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null, TimeSpan? timeout = null) + => new(http, ResourceContext.NestedSingleton(http, baseUrl, subPath, inheritedParams).WithTimeout(timeout)); internal DatasetClient WithPublicBase(string publicBaseUrl) { @@ -82,53 +82,45 @@ public async Task UpdateAsync(object newFields, CancellationToken cance /// /// /// The offset for the next page, and whether iteration continues, follow the number of rows the API - /// scanned (X-Apify-Pagination-Count) rather than the number of items a page returned. The + /// scanned (X-Apify-Pagination-Count) rather than the number of items a page returned: the /// // /// filters apply after offset/limit, so a - /// page can scan up to the requested limit of rows while returning fewer of them, or none at all; - /// advancing by the returned count (as intentionally does, since it - /// is public and reports this page's item count) would re-scan rows on the next page or stop iteration - /// in front of rows a filter hid. can conversely leave a page - /// with more items than rows scanned, so the two counts cannot be mixed. The scanned count is read from - /// the response header and falls back to the returned item count only when the header itself is absent - /// (the API always sends it; a missing header means something between the client and the API, such as a - /// proxy, stripped it). + /// page can scan up to the requested limit of rows while returning fewer of them, or none at all, and + /// can return more items than rows scanned. + /// therefore caps the rows scanned (0 or null + /// for no cap). The header can lag behind items that were just pushed, so the larger of it and the + /// returned item count is used, capped at the page's requested limit; see . /// /// Optional item filtering/projection; Offset/Limit bound where - /// iteration starts and the total number of items yielded. + /// iteration starts and the rows scanned, and ChunkSize sets the page size. /// A token to cancel the iteration. - public async IAsyncEnumerable IterateItemsAsync( + public IAsyncEnumerable IterateItemsAsync( DatasetListItemsOptions? options = null, - [EnumeratorCancellation] CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) { options ??= new DatasetListItemsOptions(); var desc = options.Desc ?? false; var baseQuery = new QueryParams(); options.AppendTo(baseQuery); - var offset = Math.Max(options.Offset ?? 0, 0); - var limit = options.Limit; - var yielded = 0; - while (true) - { - var q = baseQuery.Copy().Set("offset", offset); - if (limit is not null) - { - q.Set("limit", Math.Max(limit.Value - yielded, 0)); - } - - var (page, scannedCount) = await FetchItemsPageAsync(q, desc, cancellationToken).ConfigureAwait(false); - foreach (var item in page.Items) + return PagedIteration.IterateAsync( + options.Offset ?? 0, + options.Limit, + options.ChunkSize, + limitCountsRows: true, + async (offset, pageLimit, token) => { - yield return item; - yielded++; - } + var (page, scannedCount) = await FetchItemsPageAsync(baseQuery.PageQuery(offset, pageLimit), desc, token).ConfigureAwait(false); + return new FetchedPage(page.Items, scannedCount); + }, + cancellationToken); + } - offset += (int)scannedCount; - if (scannedCount == 0 || offset >= page.Total || (limit is not null && yielded >= limit.Value)) - { - yield break; - } - } + /// Fetches a single page of items for the given options (see the query overload). + internal Task<(PaginationList Page, long ScannedCount)> FetchItemsPageAsync(DatasetListItemsOptions options, CancellationToken cancellationToken) + { + var q = new QueryParams(); + options.AppendTo(q); + return FetchItemsPageAsync(q, options.Desc ?? false, cancellationToken); } /// @@ -141,15 +133,10 @@ public async Task UpdateAsync(object newFields, CancellationToken cance { var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("items")); using var response = await _http.CallAsync(HttpMethod.Get, url, timeout: _ctx.RequestTimeout, tier: RequestTier.Long, cancellationToken: cancellationToken).ConfigureAwait(false); - var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - var items = new List(); - if (Json.Decode(body) is JsonArray array) + foreach (var item in await response.ReadJsonArrayAsync(cancellationToken).ConfigureAwait(false)) { - foreach (var item in array) - { - items.Add(item); - } + items.Add(item); } var count = items.Count; @@ -241,8 +228,7 @@ public async Task PushItemsAsync(object items, CancellationToken cancellationTok /// A token to cancel the request. public async Task GetStatisticsAsync(CancellationToken cancellationToken = default) { - var body = await _ctx.GetRawRequiredAsync("statistics", new QueryParams(), RequestTier.Short, cancellationToken).ConfigureAwait(false); - return Json.DecodeData(body) as JsonObject ?? new JsonObject(); + return await _ctx.GetDataObjectRequiredAsync("statistics", new QueryParams(), RequestTier.Short, cancellationToken).ConfigureAwait(false); } /// diff --git a/src/Apify.Client/Resources/DatasetCollectionClient.cs b/src/Apify.Client/Resources/DatasetCollectionClient.cs index 577807c..de136ac 100644 --- a/src/Apify.Client/Resources/DatasetCollectionClient.cs +++ b/src/Apify.Client/Resources/DatasetCollectionClient.cs @@ -51,7 +51,7 @@ public IAsyncEnumerable IterateAsync(StorageListOptions? options = null options ??= new StorageListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Dataset(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new Dataset(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/KeyValueStoreClient.cs b/src/Apify.Client/Resources/KeyValueStoreClient.cs index 03afe09..8b3c3a2 100644 --- a/src/Apify.Client/Resources/KeyValueStoreClient.cs +++ b/src/Apify.Client/Resources/KeyValueStoreClient.cs @@ -26,8 +26,8 @@ private KeyValueStoreClient(HttpClientCore http, ResourceContext ctx) internal static KeyValueStoreClient ForId(HttpClientCore http, string baseUrl, string id) => new(http, ResourceContext.Single(http, baseUrl, "key-value-stores", id)); - internal static KeyValueStoreClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null) - => new(http, ResourceContext.NestedSingleton(http, baseUrl, subPath, inheritedParams)); + internal static KeyValueStoreClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null, TimeSpan? timeout = null) + => new(http, ResourceContext.NestedSingleton(http, baseUrl, subPath, inheritedParams).WithTimeout(timeout)); internal KeyValueStoreClient WithPublicBase(string publicBaseUrl) { @@ -74,13 +74,13 @@ public async Task ListKeysAsync(ListKeysOptions? options /// until the store is exhausted. Limit caps the total number of keys yielded across all pages — /// 0 yields none, without issuing any request (the API itself reads a literal limit=0 as /// unbounded, not zero, so this is checked before building the request rather than left to the API), and - /// must not be negative. MaxPageLimit caps the page size (never sent to the API itself) and must - /// be positive, and ExclusiveStartKey chooses the starting point (first page only). + /// must not be negative. ChunkSize caps the page size (never sent to the API itself) and must + /// be positive (MaxPageLimit is its older name), and ExclusiveStartKey chooses the starting point (first page only). /// /// Optional iteration options. /// A token to cancel the iteration. /// - /// is zero or negative, or + /// or is zero or negative, or /// is negative. /// public async IAsyncEnumerable IterateKeysAsync( @@ -88,17 +88,15 @@ public async IAsyncEnumerable IterateKeysAsync( [EnumeratorCancellation] CancellationToken cancellationToken = default) { options ??= new IterateKeysOptions(); - if (options.MaxPageLimit is <= 0) - { - throw new ArgumentException("IterateKeysOptions.MaxPageLimit must be positive.", nameof(options)); - } + PageSizing.ValidateChunkSize(options.ChunkSize, nameof(IterateKeysOptions.ChunkSize)); + PageSizing.ValidateChunkSize(options.MaxPageLimit, nameof(IterateKeysOptions.MaxPageLimit)); if (options.Limit is < 0) { throw new ArgumentException("IterateKeysOptions.Limit must not be negative.", nameof(options)); } - var maxPageLimit = options.MaxPageLimit ?? IterateKeysOptions.DefaultMaxPageLimit; + var maxPageLimit = options.ChunkSize ?? options.MaxPageLimit ?? IterateKeysOptions.DefaultMaxPageLimit; var limit = options.Limit; // total across all pages; null = unbounded var nextExclusiveStartKey = options.ExclusiveStartKey; var iterated = 0; diff --git a/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs b/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs index d2bf650..def2ad5 100644 --- a/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs +++ b/src/Apify.Client/Resources/KeyValueStoreCollectionClient.cs @@ -50,7 +50,7 @@ public IAsyncEnumerable IterateAsync(StorageListOptions? options options ??= new StorageListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new KeyValueStore(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new KeyValueStore(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/LogClient.cs b/src/Apify.Client/Resources/LogClient.cs index 35ffceb..dec89a2 100644 --- a/src/Apify.Client/Resources/LogClient.cs +++ b/src/Apify.Client/Resources/LogClient.cs @@ -35,8 +35,8 @@ internal static LogClient ForId(HttpClientCore http, string baseUrl, string id) /// missing — so and throw instead of resolving /// null, matching the reference client's catchNotFoundForResourceOrThrow(). /// - internal static LogClient Nested(HttpClientCore http, string baseUrl, QueryParams? inheritedParams = null) - => new(http, ResourceContext.Collection(http, baseUrl, "log", inheritedParams), ambiguousNotFound: true); + internal static LogClient Nested(HttpClientCore http, string baseUrl, QueryParams? inheritedParams = null, TimeSpan? timeout = null) + => new(http, ResourceContext.Collection(http, baseUrl, "log", inheritedParams).WithTimeout(timeout), ambiguousNotFound: true); /// /// Fetches the log as text. For a log addressed by its own id (), a 404 diff --git a/src/Apify.Client/Resources/RequestQueueClient.cs b/src/Apify.Client/Resources/RequestQueueClient.cs index 412ef26..9036740 100644 --- a/src/Apify.Client/Resources/RequestQueueClient.cs +++ b/src/Apify.Client/Resources/RequestQueueClient.cs @@ -49,8 +49,8 @@ internal static RequestQueueClient ForId(HttpClientCore http, string baseUrl, st return new RequestQueueClient(http, ctx, options?.ClientKey, timeout); } - internal static RequestQueueClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null) - => new(http, ResourceContext.NestedSingleton(http, baseUrl, subPath, inheritedParams), null, null); + internal static RequestQueueClient Nested(HttpClientCore http, string baseUrl, string subPath, QueryParams? inheritedParams = null, TimeSpan? timeout = null) + => new(http, ResourceContext.NestedSingleton(http, baseUrl, subPath, inheritedParams).WithTimeout(timeout), null, timeout); /// /// Returns a copy of the client that identifies its requests with . A @@ -130,8 +130,7 @@ public async Task UpdateRequestAsync(RequestQueueRequ ApplyClientKey(q); var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("requests/" + ResourceContext.EncodePathSegment(request.Id ?? string.Empty))); using var response = await _http.CallAsync(HttpMethod.Put, url, Json.Encode(request.ToJsonObject()), ResourceContext.ContentTypeJson, _timeout, tier: RequestTier.Medium, cancellationToken: cancellationToken).ConfigureAwait(false); - var data = Json.DecodeData(await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)); - return new RequestQueueOperationInfo(data as JsonObject ?? new JsonObject()); + return new RequestQueueOperationInfo(await response.ReadDataObjectAsync(cancellationToken).ConfigureAwait(false)); } /// Deletes a request by ID. @@ -483,8 +482,7 @@ public async Task ProlongRequestLockAsync(string id, int lockSe ApplyClientKey(q); var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("requests/" + ResourceContext.EncodePathSegment(id) + "/lock")); using var response = await _http.CallAsync(HttpMethod.Put, url, null, "", _timeout, tier: RequestTier.Medium, cancellationToken: cancellationToken).ConfigureAwait(false); - var data = Json.DecodeData(await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)); - return RequestLockInfo.FromData(data); + return RequestLockInfo.FromData(await response.ReadDataObjectAsync(cancellationToken).ConfigureAwait(false)); } /// @@ -526,7 +524,7 @@ public async Task UnlockRequestsAsync(CancellationToken ca /// /// With no options it fetches pages of up to /// requests until the queue is exhausted. The options mirror the reference client: Limit caps the - /// total number of requests yielded across all pages, MaxPageLimit caps the page size, + /// total number of requests yielded across all pages, ChunkSize (or its older name MaxPageLimit) caps the page size, /// ExclusiveStartId/Cursor choose the starting point (first page only), and Filter /// restricts to locked/pending requests. /// @@ -539,7 +537,7 @@ public async IAsyncEnumerable PaginateRequestsAsync( options ??= new PaginateRequestsOptions(); options.Validate(); - var maxPageLimit = options.MaxPageLimit ?? PaginateRequestsOptions.DefaultMaxPageLimit; + var maxPageLimit = options.ChunkSize ?? options.MaxPageLimit ?? PaginateRequestsOptions.DefaultMaxPageLimit; var limit = options.Limit; // total across all pages; null = unbounded var nextCursor = options.Cursor; var nextExclusiveStartId = options.ExclusiveStartId; // used for the first page only diff --git a/src/Apify.Client/Resources/RequestQueueCollectionClient.cs b/src/Apify.Client/Resources/RequestQueueCollectionClient.cs index 298a649..bb865cd 100644 --- a/src/Apify.Client/Resources/RequestQueueCollectionClient.cs +++ b/src/Apify.Client/Resources/RequestQueueCollectionClient.cs @@ -48,7 +48,7 @@ public IAsyncEnumerable IterateAsync(StorageListOptions? options = options ??= new StorageListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new RequestQueue(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new RequestQueue(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/RunClient.cs b/src/Apify.Client/Resources/RunClient.cs index 8babaa6..d42ca04 100644 --- a/src/Apify.Client/Resources/RunClient.cs +++ b/src/Apify.Client/Resources/RunClient.cs @@ -3,6 +3,7 @@ using System.Globalization; using System.IO; using System.Net.Http; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using System.Text.Json.Nodes; @@ -25,12 +26,14 @@ public sealed class RunClient /// Header the API uses to deduplicate charge requests. private const string ChargeIdempotencyHeader = "idempotency-key"; + private readonly ApifyClient _root; private readonly HttpClientCore _http; private readonly ResourceContext _ctx; private readonly string _id; - internal RunClient(HttpClientCore http, string baseUrl, string resourcePath, string id) + internal RunClient(ApifyClient root, HttpClientCore http, string baseUrl, string resourcePath, string id) { + _root = root; _http = http; _id = id; _ctx = ResourceContext.Single(http, baseUrl, resourcePath, id); @@ -166,6 +169,11 @@ public async Task ChargeAsync(RunChargeOptions options, CancellationToken cancel ["eventName"] = options.EventName, ["count"] = options.CountValue(), }; + if (!string.IsNullOrEmpty(options.RequestId)) + { + body["requestId"] = options.RequestId; + } + using var response = await _http.CallAsync( HttpMethod.Post, _ctx.SubUrl("charge"), @@ -227,20 +235,150 @@ internal async Task WaitForFinishWithLogAsync(int? waitSecs, Action + /// Lazily iterates over the items of the run's default dataset while the run is still producing them. + /// Mirrors the reference client's iterateDatasetItems(). + /// + /// + /// While the run has not finished, each poll yields the rows below the dataset's itemCount and + /// then waits up to for the run to finish, + /// so the last rows are read as soon as it does. Each page is requested with a limit that ends at + /// itemCount, so it covers exactly the rows it asks for, whatever the filters or + /// do to the items. itemCount lags a few + /// seconds behind the pushed items, so once the run reaches a terminal status the rows past it are read + /// a page at a time until none are left. On a last-run client, the iteration sticks to the run its + /// first request resolves to, so a newer run cannot swap the dataset mid-iteration. + /// + /// Optional pagination, filtering and polling options. + /// A token to cancel the iteration. + public async IAsyncEnumerable IterateDatasetItemsAsync( + RunIterateDatasetItemsOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + options ??= new RunIterateDatasetItemsOptions(); + PageSizing.ValidateChunkSize(options.ChunkSize, nameof(RunIterateDatasetItemsOptions.ChunkSize)); + var chunkSize = options.ChunkSize ?? RunIterateDatasetItemsOptions.DefaultChunkSize; + var pollIntervalSecs = options.PollIntervalSecs ?? RunIterateDatasetItemsOptions.DefaultPollIntervalSecs; + ArgumentOutOfRangeException.ThrowIfNegative(pollIntervalSecs, nameof(RunIterateDatasetItemsOptions.PollIntervalSecs)); + ArgumentOutOfRangeException.ThrowIfNegative(options.Offset ?? 0, nameof(RunIterateDatasetItemsOptions.Offset)); + ArgumentOutOfRangeException.ThrowIfNegative(options.Limit ?? 0, nameof(RunIterateDatasetItemsOptions.Limit)); + + var position = options.Offset ?? 0; + // Wider than int: Offset + Limit can exceed int.MaxValue (e.g. Limit = int.MaxValue as "everything"). + long? end = options.Limit is > 0 ? (long)position + options.Limit.Value : null; + + var run = await GetAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + // A last-run client resolves `runs/last` on each request, so a newer run would swap the dataset + // mid-iteration: pin to the run the first request resolved to. + var runClient = this; + if (run?.Id is { } resolvedId && resolvedId != _id) + { + runClient = _root.Run(resolvedId); + if (_ctx.RequestTimeout is { } timeout) + { + runClient.WithTimeout(timeout); + } + } + + var datasetClient = runClient.Dataset(); + + while (true) + { + var isFinished = run is null || run.IsTerminal; + var dataset = await datasetClient.GetAsync(cancellationToken).ConfigureAwait(false); + var itemCount = dataset?.ItemCount ?? 0; + if (end is not null) + { + itemCount = Math.Min(itemCount, end.Value); + } + + while (position < itemCount) + { + var pageLimit = (int)Math.Min(chunkSize, itemCount - position); + var (page, _) = await datasetClient.FetchItemsPageAsync(ItemsPageOptions(options, position, pageLimit), cancellationToken).ConfigureAwait(false); + foreach (var item in page.Items) + { + yield return item; + } + + position += pageLimit; + } + + if (end is not null && position >= end) + { + yield break; + } + + if (isFinished) + { + break; + } + + run = await runClient.WaitForFinishAsync(pollIntervalSecs, cancellationToken).ConfigureAwait(false); + } + + // The run is finished, but `itemCount` lags the pushed items: read the remaining rows page by page. + while (true) + { + var pageLimit = end is not null ? (int)Math.Min(chunkSize, end.Value - position) : chunkSize; + var (page, scannedCount) = await datasetClient.FetchItemsPageAsync(ItemsPageOptions(options, position, pageLimit), cancellationToken).ConfigureAwait(false); + foreach (var item in page.Items) + { + yield return item; + } + + // Only a page that returned nothing and scanned no rows marks the end: filters can shorten a + // full page, or empty one that still scanned rows. When a filter could have hidden every row + // past a lagging `itemCount`, a plain one-row read confirms nothing is left. + if (page.Items.Count == 0 && scannedCount == 0 + && (!options.HasRowFilters || await IsDatasetExhaustedAsync(datasetClient, position, cancellationToken).ConfigureAwait(false))) + { + yield break; + } + + position += pageLimit; + if (end is not null && position >= end) + { + yield break; + } + } + } + + /// Builds the dataset-items options for one page of . + private static DatasetListItemsOptions ItemsPageOptions(RunIterateDatasetItemsOptions options, int offset, int limit) => new() + { + Offset = offset, + Limit = limit, + Clean = options.Clean, + Fields = options.Fields, + Omit = options.Omit, + Unwind = options.Unwind, + SkipEmpty = options.SkipEmpty, + SkipHidden = options.SkipHidden, + }; + + /// Checks, without any filter, whether the dataset has no row at or after . + private static async Task IsDatasetExhaustedAsync(DatasetClient datasetClient, int position, CancellationToken cancellationToken) + { + var plain = new DatasetListItemsOptions { Offset = position, Limit = 1 }; + var (page, _) = await datasetClient.FetchItemsPageAsync(plain, cancellationToken).ConfigureAwait(false); + return page.Items.Count == 0; + } + + // Nested accessors inherit this client's params, so last-run status/origin filters (see SetLastRunParams) + // resolve the intended run's storage/log rather than the latest run's, and its WithTimeout override. /// A client for this run's default dataset. - public DatasetClient Dataset() => DatasetClient.Nested(_http, _ctx.SubUrl(""), "dataset", _ctx.BaseParams); + public DatasetClient Dataset() => DatasetClient.Nested(_http, _ctx.SubUrl(""), "dataset", _ctx.BaseParams, _ctx.RequestTimeout); /// A client for this run's default key-value store. - public KeyValueStoreClient KeyValueStore() => KeyValueStoreClient.Nested(_http, _ctx.SubUrl(""), "key-value-store", _ctx.BaseParams); + public KeyValueStoreClient KeyValueStore() => KeyValueStoreClient.Nested(_http, _ctx.SubUrl(""), "key-value-store", _ctx.BaseParams, _ctx.RequestTimeout); /// A client for this run's default request queue. - public RequestQueueClient RequestQueue() => RequestQueueClient.Nested(_http, _ctx.SubUrl(""), "request-queue", _ctx.BaseParams); + public RequestQueueClient RequestQueue() => RequestQueueClient.Nested(_http, _ctx.SubUrl(""), "request-queue", _ctx.BaseParams, _ctx.RequestTimeout); /// A client for accessing this run's log. - public LogClient Log() => LogClient.Nested(_http, _ctx.SubUrl(""), _ctx.BaseParams); + public LogClient Log() => LogClient.Nested(_http, _ctx.SubUrl(""), _ctx.BaseParams, _ctx.RequestTimeout); /// /// Opens a live stream of this run's raw log bytes. The caller reads (and disposes) the returned diff --git a/src/Apify.Client/Resources/RunCollectionClient.cs b/src/Apify.Client/Resources/RunCollectionClient.cs index c1f63de..23f3309 100644 --- a/src/Apify.Client/Resources/RunCollectionClient.cs +++ b/src/Apify.Client/Resources/RunCollectionClient.cs @@ -50,7 +50,7 @@ public IAsyncEnumerable IterateAsync( var q = new QueryParams(); options.AppendTo(q); (filter ?? new RunListOptions()).AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new ActorRun(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new ActorRun(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/ScheduleCollectionClient.cs b/src/Apify.Client/Resources/ScheduleCollectionClient.cs index ff26c93..e7985e9 100644 --- a/src/Apify.Client/Resources/ScheduleCollectionClient.cs +++ b/src/Apify.Client/Resources/ScheduleCollectionClient.cs @@ -45,7 +45,7 @@ public IAsyncEnumerable IterateAsync(ListOptions? options = null, Canc options ??= new ListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new Schedule(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new Schedule(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/StoreCollectionClient.cs b/src/Apify.Client/Resources/StoreCollectionClient.cs index 4b32a8c..d2cd32d 100644 --- a/src/Apify.Client/Resources/StoreCollectionClient.cs +++ b/src/Apify.Client/Resources/StoreCollectionClient.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Apify.Client.Internal; @@ -30,32 +29,18 @@ public Task> ListAsync(StoreListOptions? opti } /// - /// Lazily iterates over all Store Actors matching the options, fetching pages on demand. The options' - /// Limit (if set) is used as the per-page size. + /// Lazily iterates over all Store Actors matching the options across pages, fetching each page on + /// demand. /// - /// Optional listing filters; Limit is used as the page size. + /// Optional listing filters; Offset/Limit bound where iteration + /// starts and the total number of Actors yielded, and ChunkSize sets the page size. /// A token to cancel the iteration. - public async IAsyncEnumerable IterateAsync( - StoreListOptions? options = null, - [EnumeratorCancellation] CancellationToken cancellationToken = default) + public IAsyncEnumerable IterateAsync(StoreListOptions? options = null, CancellationToken cancellationToken = default) { options ??= new StoreListOptions(); - var offset = options.Offset ?? 0; - while (true) - { - var page = await ListAsync(options.WithOffset(offset), cancellationToken).ConfigureAwait(false); - var items = page.Items; - foreach (var item in items) - { - yield return item; - } - - offset += items.Count; - if (items.Count == 0 || offset >= page.Total) - { - yield break; - } - } + var q = new QueryParams(); + options.AppendTo(q); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new ActorStoreListItem(d), RequestTier.Medium, cancellationToken); } /// diff --git a/src/Apify.Client/Resources/TaskClient.cs b/src/Apify.Client/Resources/TaskClient.cs index 220d349..2ab2052 100644 --- a/src/Apify.Client/Resources/TaskClient.cs +++ b/src/Apify.Client/Resources/TaskClient.cs @@ -136,13 +136,12 @@ public async Task CallAsync( /// /// Fetches the task's stored input. Throws if the task does not exist (no longer resolves to null /// for that case): the input endpoint has no meaning apart from its parent task, matching the reference - /// client. Still returns a JSON null node if the task exists but has no stored input. + /// client. A task without stored input yields an empty JSON object. /// /// A token to cancel the request. public async Task GetInputAsync(CancellationToken cancellationToken = default) { - var body = await _ctx.GetRawRequiredAsync("input", new QueryParams(), RequestTier.Short, cancellationToken).ConfigureAwait(false); - return Json.Decode(body); + return await _ctx.GetJsonRequiredAsync("input", new QueryParams(), RequestTier.Short, cancellationToken).ConfigureAwait(false); } /// Replaces the task's stored input and returns the updated input. @@ -158,14 +157,14 @@ public async Task CallAsync( timeout: _ctx.RequestTimeout, tier: RequestTier.Short, cancellationToken: cancellationToken).ConfigureAwait(false); - return Json.Decode(await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)); + return await response.ReadJsonAsync(cancellationToken).ConfigureAwait(false); } /// Returns a client for the last run of this task, optionally filtered by status and/or origin. /// Optional last-run filters. public RunClient LastRun(LastRunOptions? options = null) { - var client = new RunClient(_http, _ctx.SubUrl(""), "runs", "last"); + var client = new RunClient(_root, _http, _ctx.SubUrl(""), "runs", "last"); client.SetLastRunParams(options ?? new LastRunOptions()); return client; } diff --git a/src/Apify.Client/Resources/TaskCollectionClient.cs b/src/Apify.Client/Resources/TaskCollectionClient.cs index 288a8e5..76410f9 100644 --- a/src/Apify.Client/Resources/TaskCollectionClient.cs +++ b/src/Apify.Client/Resources/TaskCollectionClient.cs @@ -45,7 +45,7 @@ public IAsyncEnumerable IterateAsync(ListOptions? options = null, Can options ??= new ListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new ActorTask(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new ActorTask(d), RequestTier.Medium, cancellationToken); } diff --git a/src/Apify.Client/Resources/UserClient.cs b/src/Apify.Client/Resources/UserClient.cs index a194c06..d0208bf 100644 --- a/src/Apify.Client/Resources/UserClient.cs +++ b/src/Apify.Client/Resources/UserClient.cs @@ -57,8 +57,7 @@ public async Task MonthlyUsageAsync(string? date = null, Cancellatio q.AddString("date", date); } - var data = await _ctx.GetResourceRequiredAsync("usage/monthly", q, RequestTier.Short, cancellationToken).ConfigureAwait(false); - return data as JsonObject ?? new JsonObject(); + return await _ctx.GetResourceRequiredAsync("usage/monthly", q, RequestTier.Short, cancellationToken).ConfigureAwait(false); } /// Fetches the current account's resource limits. Only available for me. @@ -66,8 +65,7 @@ public async Task MonthlyUsageAsync(string? date = null, Cancellatio public async Task LimitsAsync(CancellationToken cancellationToken = default) { RequireMe(); - var data = await _ctx.GetResourceRequiredAsync("limits", new QueryParams(), RequestTier.Short, cancellationToken).ConfigureAwait(false); - return data as JsonObject ?? new JsonObject(); + return await _ctx.GetResourceRequiredAsync("limits", new QueryParams(), RequestTier.Short, cancellationToken).ConfigureAwait(false); } /// Updates the current account's resource limits. Only available for me. diff --git a/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs b/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs index c5a5e7b..7134177 100644 --- a/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs +++ b/src/Apify.Client/Resources/WebhookDispatchCollectionClient.cs @@ -40,7 +40,7 @@ public IAsyncEnumerable IterateAsync(ListOptions? options = nul options ??= new ListOptions(); var q = new QueryParams(); options.AppendTo(q); - return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, static d => new WebhookDispatch(d), RequestTier.Medium, cancellationToken); + return _ctx.IterateListAsync("", q, options.Offset ?? 0, options.Limit, options.ChunkSize, static d => new WebhookDispatch(d), RequestTier.Medium, cancellationToken); } diff --git a/tests/Apify.Client.Tests/Apify.Client.Tests.csproj b/tests/Apify.Client.Tests/Apify.Client.Tests.csproj index 4c62faf..9f2b502 100644 --- a/tests/Apify.Client.Tests/Apify.Client.Tests.csproj +++ b/tests/Apify.Client.Tests/Apify.Client.Tests.csproj @@ -14,6 +14,8 @@ + + diff --git a/tests/Apify.Client.Tests/Examples/DocumentationSnippetTests.cs b/tests/Apify.Client.Tests/Examples/DocumentationSnippetTests.cs new file mode 100644 index 0000000..b73e846 --- /dev/null +++ b/tests/Apify.Client.Tests/Examples/DocumentationSnippetTests.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Xunit; + +namespace Apify.Client.Tests.Examples; + +/// +/// Compiles every csharp code block of the top-level README and of docs/*.md against the +/// library, so a snippet that no longer compiles (a renamed method, a missing argument, a bare fragment) +/// fails CI. The runnable scenarios in docs/examples.md are additionally executed by +/// . +/// +/// +/// A block that starts with using directives is a complete program (top-level statements). Any +/// other block is a fragment that continues the most recent complete block of the same file, so it is +/// compiled after that block's directives and declarations. +/// +[Trait("Category", "Examples")] +public sealed class DocumentationSnippetTests +{ + private static readonly Regex CodeBlock = new(@"```csharp\r?\n(?.*?)```", RegexOptions.Singleline | RegexOptions.Compiled); + + public static IEnumerable Snippets() + { + var root = FindRepositoryRoot(); + var files = new[] { Path.Combine(root, "README.md") } + .Concat(Directory.GetFiles(Path.Combine(root, "docs"), "*.md").OrderBy(static f => f, StringComparer.Ordinal)); + foreach (var file in files) + { + var relative = Path.GetRelativePath(root, file).Replace('\\', '/'); + var lastComplete = string.Empty; + var index = 0; + foreach (Match match in CodeBlock.Matches(File.ReadAllText(file))) + { + index++; + var code = match.Groups["code"].Value; + var isComplete = code.TrimStart().StartsWith("using ", StringComparison.Ordinal); + var program = isComplete ? code : lastComplete + "\n" + code; + if (isComplete) + { + lastComplete = code; + } + + yield return new object[] { relative + " snippet #" + index, program }; + } + } + } + + [Theory] + [MemberData(nameof(Snippets))] + public void SnippetCompiles(string name, string program) + { + var tree = CSharpSyntaxTree.ParseText(program, new CSharpParseOptions(LanguageVersion.Latest)); + var compilation = CSharpCompilation.Create( + "DocSnippet", + new[] { tree }, + References(), + new CSharpCompilationOptions(OutputKind.ConsoleApplication, nullableContextOptions: NullableContextOptions.Enable)); + + var errors = compilation.GetDiagnostics() + .Where(static d => d.Severity == DiagnosticSeverity.Error) + .Select(static d => d.ToString()) + .ToList(); + + Assert.True(errors.Count == 0, name + " does not compile:\n" + string.Join("\n", errors)); + } + + private static IEnumerable References() + { + var trusted = (string)AppContext.GetData("TRUSTED_PLATFORM_ASSEMBLIES")!; + foreach (var path in trusted.Split(Path.PathSeparator)) + { + yield return MetadataReference.CreateFromFile(path); + } + + yield return MetadataReference.CreateFromFile(typeof(ApifyClient).Assembly.Location); + } + + private static string FindRepositoryRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null && !File.Exists(Path.Combine(directory.FullName, "Apify.Client.sln"))) + { + directory = directory.Parent; + } + + return directory?.FullName ?? throw new InvalidOperationException("Could not locate the repository root (Apify.Client.sln)."); + } +} diff --git a/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs index 24a889b..8c08d68 100644 --- a/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs @@ -1,3 +1,5 @@ +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Apify.Client.Options; using Xunit; @@ -38,6 +40,40 @@ public async Task RunActorAndReadOutputs() await client.Run(run.Id!).KeyValueStore().GetRecordAsync("OUTPUT"); } + [SkippableFact] + public async Task IterateRunDatasetItems() + { + var client = RequireClient(); + var run = await client.Actor("apify/hello-world").CallAsync(null, null, 120); + + var iterated = new List(); + await foreach (var item in client.Run(run.Id!).IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { ChunkSize = 1 })) + { + Assert.NotNull(item); + iterated.Add(item!.ToJsonString()); + } + + // hello-world pushes an item, so an iterator that yields nothing is a failure. + Assert.NotEmpty(iterated); + + // The run is finished, so its dataset is complete. The dataset listing lags the pushed items by a few + // seconds (which is why the iterator drains past the reported item count), so it must catch up to + // exactly the items the iterator already read. + var listed = new List(); + Assert.True( + await PollUntilAsync( + EventualConsistencyAttempts, + EventualConsistencyBackoff, + async () => + { + var page = await client.Run(run.Id!).Dataset().ListItemsAsync(new DatasetListItemsOptions()); + listed = page.Items.Select(static i => i!.ToJsonString()).ToList(); + return listed.Count == iterated.Count; + }), + "expected the dataset listing to catch up with the iterated items"); + Assert.Equal(listed, iterated); + } + [SkippableFact] public async Task UpdateAndDeleteRun() { diff --git a/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs b/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs index c1bb2e6..46efb42 100644 --- a/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs +++ b/tests/Apify.Client.Tests/Integration/IntegrationTestBase.cs @@ -26,14 +26,14 @@ public abstract class IntegrationTestBase /// Retry budget for : how many times a freshly created /// resource's collection listing is re-scanned before giving up on it having propagated. /// - private const int EventualConsistencyAttempts = 16; + protected const int EventualConsistencyAttempts = 16; /// /// Delay between retries in . Combined with /// , the total wait budget is /// (EventualConsistencyAttempts - 1) * EventualConsistencyBackoff = ~15s. /// - private static readonly TimeSpan EventualConsistencyBackoff = TimeSpan.FromSeconds(1); + protected static readonly TimeSpan EventualConsistencyBackoff = TimeSpan.FromSeconds(1); /// /// Derives the client base URL from an optional APIFY_API_URL. The variable includes the @@ -114,15 +114,22 @@ protected static string UniqueName(string prefix) actions = Array.Empty(), }; - /// A minimal ad-hoc webhook definition targeting a condition that never actually fires. - protected static object MinimalWebhook(string requestUrl) => new + /// + /// A minimal webhook definition that fires on runs of the given Actor. The API requires the condition to + /// reference an existing Actor by its ID (not its username/name slug), see + /// . + /// + protected static object MinimalWebhook(string requestUrl, string actorId) => new { - isAdHoc = true, eventTypes = new[] { "ACTOR.RUN.SUCCEEDED" }, - condition = new { actorRunId = "ZZZZZZZZZZZZZZZZZ" }, + condition = new { actorId }, requestUrl, }; + /// Resolves the ID of the public apify/hello-world Actor, for use as a webhook condition. + protected static async Task ResolveHelloWorldActorIdAsync(ApifyClient client) + => (await client.Actor("apify/hello-world").GetAsync())!.Id!; + /// /// Retries up to times, sleeping /// between attempts, until it returns true. Used to tolerate diff --git a/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs index d44015f..082ab4a 100644 --- a/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Apify.Client.Models; using Apify.Client.Options; @@ -7,29 +8,15 @@ namespace Apify.Client.Tests.Integration; /// -/// Iteration coverage for the 12 collection-of-resources IterateAsync clients that had no -/// dedicated test before this suite was added (Actors, Actor versions, Actor environment variables, -/// datasets, key-value stores, request queues, tasks, schedules, webhooks, builds, runs, and webhook -/// dispatches — exercised by the 11 test methods below, since Actor versions and Actor environment -/// variables share one). already covers the thirteenth -/// (StoreCollectionClient), so it is not duplicated here. +/// Live iteration tests for the collection-of-resources IterateAsync clients, plus dataset item +/// iteration. Where creation is cheap, a test creates a couple of uniquely-named resources and asserts that +/// iterating the collection (newest-first) eventually surfaces every one of them. Builds, runs and webhook +/// dispatches are slow to create on demand, so those tests drain a Limit-bounded slice of what +/// already exists on the account and check the iterator terminates, respects the cap and yields +/// well-formed items. covers the store collection; key +/// iteration and request iteration have their own tests below and in +/// RequestQueueIntegrationTests.RequestQueuePaginateMultiplePages. /// -/// -/// This suite is about iterating a collection of resources (e.g. "every dataset"), not a resource's own -/// contents: dataset item iteration (DatasetClient.IterateItemsAsync) and key-value-store key -/// iteration (KeyValueStoreClient.IterateKeysAsync) are covered by unit tests (AutoPagingTests) -/// against a mocked transport, since they need to exercise multi-page paging logic deterministically; -/// below covers the same iterator against the real API. -/// Request-queue request iteration has its own integration test -/// (RequestQueueIntegrationTests.RequestQueuePaginateMultiplePages). -/// -/// Where creation is cheap, each test below creates a couple of uniquely-named resources and asserts that -/// iterating the collection (newest-first) eventually surfaces every one of them — exercising the real -/// async generator, not just a single ListAsync call. Builds, runs, and webhook dispatches are -/// expensive or slow to create on demand, so those tests instead drain a Limit-bounded slice of -/// whatever already exists on the test account and assert the iterator behaves (terminates, respects the -/// cap, yields well-formed items). -/// [Trait("Category", "Integration")] public sealed class IterationIntegrationTests : IntegrationTestBase { @@ -178,11 +165,12 @@ public async Task IterateWebhooks() { var client = RequireClient(); var ids = new HashSet(); + var actorId = await ResolveHelloWorldActorIdAsync(client); try { for (var i = 0; i < 2; i++) { - ids.Add((await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/it-wh-" + i))).Id!); + ids.Add((await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/it-wh-" + i, actorId))).Id!); } Assert.True( @@ -349,4 +337,48 @@ public async Task IterateWebhookDispatchesBounded() Assert.True(count <= 5, "the total-cap limit must bound iteration; got " + count); } + + [SkippableFact] + public async Task IterateDatasetItemsMultiplePages() + { + var client = RequireClient(); + var ds = await client.Datasets().GetOrCreateAsync(UniqueName("ds-iter")); + try + { + const int total = 5; + var dataset = client.Dataset(ds.Id!); + await dataset.PushItemsAsync(Enumerable.Range(0, total).Select(static n => (object)new { n }).ToArray()); + + // Dataset item counts propagate with a delay, so retry until every pushed item is readable. + var seen = new List(); + Assert.True( + await PollUntilAsync( + EventualConsistencyAttempts, + EventualConsistencyBackoff, + async () => + { + seen.Clear(); + await foreach (var item in dataset.IterateItemsAsync(new DatasetListItemsOptions { ChunkSize = 2 })) + { + seen.Add(item!["n"]!.GetValue()); + } + + return seen.Count == total; + }), + "expected iteration to eventually return every pushed item"); + Assert.Equal(Enumerable.Range(0, total), seen); + + var limited = new List(); + await foreach (var item in dataset.IterateItemsAsync(new DatasetListItemsOptions { Offset = 1, Limit = 3, ChunkSize = 2 })) + { + limited.Add(item!["n"]!.GetValue()); + } + + Assert.Equal(new[] { 1, 2, 3 }, limited); + } + finally + { + await client.Dataset(ds.Id!).DeleteAsync(); + } + } } diff --git a/tests/Apify.Client.Tests/Integration/RequestQueueIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/RequestQueueIntegrationTests.cs index 9f53fcc..946448f 100644 --- a/tests/Apify.Client.Tests/Integration/RequestQueueIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/RequestQueueIntegrationTests.cs @@ -55,6 +55,14 @@ public async Task RequestQueueCrudFlow() Assert.Equal("https://example.com", got!.Url); Assert.NotEmpty((await queue.ListHeadAsync(10)).Items); + + got.Method = "POST"; + await queue.UpdateRequestAsync(got); + Assert.Equal("POST", (await queue.GetRequestAsync(info.RequestId!))!.Method); + Assert.Contains( + (await queue.ListRequestsAsync(new ListRequestsOptions { Limit = 10 })).Items, + r => r.Id == info.RequestId); + await queue.UpdateAsync(new { name = UniqueName("rq-renamed") }); await queue.DeleteRequestAsync(info.RequestId!); } diff --git a/tests/Apify.Client.Tests/Integration/StoreIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/StoreIntegrationTests.cs index e9a2727..655c9b9 100644 --- a/tests/Apify.Client.Tests/Integration/StoreIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/StoreIntegrationTests.cs @@ -20,7 +20,7 @@ public async Task IterateStore() { var client = RequireClient(); var count = 0; - await foreach (var item in client.Store().IterateAsync(new StoreListOptions { Limit = 5 })) + await foreach (var item in client.Store().IterateAsync(new StoreListOptions { ChunkSize = 5 })) { Assert.False(string.IsNullOrEmpty(item.Id)); if (++count >= 12) @@ -31,4 +31,18 @@ public async Task IterateStore() Assert.True(count >= 12, "expected to iterate at least 12 store actors"); } + + [SkippableFact] + public async Task IterateStoreStopsAtLimit() + { + var client = RequireClient(); + var count = 0; + await foreach (var unused in client.Store().IterateAsync(new StoreListOptions { Limit = 7, ChunkSize = 3 })) + { + _ = unused; + count++; + } + + Assert.Equal(7, count); + } } diff --git a/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs index 61ba64f..84b8ff3 100644 --- a/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/WebhookIntegrationTests.cs @@ -31,7 +31,7 @@ public async Task ListWebhookDispatches() public async Task GetWebhook() { var client = RequireClient(); - var wh = await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook")); + var wh = await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook", await ResolveHelloWorldActorIdAsync(client))); try { var got = await client.Webhook(wh.Id!).GetAsync(); @@ -48,7 +48,7 @@ public async Task GetWebhook() public async Task GetWebhookDispatch() { var client = RequireClient(); - var wh = await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook")); + var wh = await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook", await ResolveHelloWorldActorIdAsync(client))); try { var dispatch = await client.Webhook(wh.Id!).TestAsync(); @@ -66,7 +66,7 @@ public async Task GetWebhookDispatch() public async Task WebhookCrudFlow() { var client = RequireClient(); - var wh = await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook")); + var wh = await client.Webhooks().CreateAsync(MinimalWebhook("https://example.com/webhook", await ResolveHelloWorldActorIdAsync(client))); try { var webhook = client.Webhook(wh.Id!); @@ -81,4 +81,40 @@ public async Task WebhookCrudFlow() await client.Webhook(wh.Id!).DeleteAsync(); } } + + [SkippableFact] + public async Task ListActorWebhooks() + { + var client = RequireClient(); + var actor = await client.Actors().CreateAsync(MinimalActor(UniqueName("wh-act"))); + try + { + var page = await client.Actor(actor.Id!).Webhooks().ListAsync(new ListOptions { Limit = 5 }); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); + } + finally + { + await client.Actor(actor.Id!).DeleteAsync(); + } + } + + [SkippableFact] + public async Task ListTaskWebhooks() + { + var client = RequireClient(); + var task = await client.Tasks().CreateAsync(MinimalTask(UniqueName("wh-task"))); + try + { + var page = await client.Task(task.Id!).Webhooks().ListAsync(new ListOptions { Limit = 5 }); + Assert.True(page.Items.Count <= 5); + Assert.Equal(page.Items.Count, (int)page.Count); + Assert.True(page.Total >= page.Items.Count); + } + finally + { + await client.Task(task.Id!).DeleteAsync(); + } + } } diff --git a/tests/Apify.Client.Tests/Unit/ApifyClientExceptionTests.cs b/tests/Apify.Client.Tests/Unit/ApifyClientExceptionTests.cs index 9b7413f..533cddf 100644 --- a/tests/Apify.Client.Tests/Unit/ApifyClientExceptionTests.cs +++ b/tests/Apify.Client.Tests/Unit/ApifyClientExceptionTests.cs @@ -47,6 +47,25 @@ public async Task ApifyTransportExceptionIsAnApifyClientException() Assert.IsAssignableFrom(ex); } + [Fact] + public async Task SuccessWithUnparsableBodyThrowsInvalidResponseBodyWithoutRetry() + { + var transport = new MockTransport().QueueResponse(200, "not json").QueueResponse(200, "{\"data\":{\"id\":\"a\"}}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actor("x").GetAsync()); + Assert.IsAssignableFrom(ex); + Assert.Equal(1, transport.CallCount); + } + + [Fact] + public async Task ErrorStatusWithUnparsableBodyThrowsApiExceptionForItsStatus() + { + var transport = new MockTransport().QueueResponse(400, "Bad Request"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actors().CreateAsync(new { name = "x" })); + Assert.Contains("Bad Request", ex.Message, StringComparison.Ordinal); + } + [Fact] public async Task CatchingApifyClientExceptionCatchesBothFamilies() { diff --git a/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs index a7d0187..cc59bc0 100644 --- a/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs +++ b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs @@ -44,9 +44,9 @@ public async Task CollectionIterateWalksAllPagesByOffset() // CollectionIterateDoesNotStopAtAnEarlierPagesStaleTotal), so it always sends one more request // after the last non-empty page to confirm nothing is left. var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"total\":3,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}") - .QueueResponse(200, "{\"data\":{\"total\":3,\"items\":[{\"id\":\"c\"}]}}") - .QueueResponse(200, "{\"data\":{\"total\":3,\"items\":[]}}"); + .QueueResponse(200, SpecFixtures.ActorPage(3, "a", "b")) + .QueueResponse(200, SpecFixtures.ActorPage(3, "c")) + .QueueResponse(200, SpecFixtures.ActorPage(3)); var ids = new List(); await foreach (var actor in Client(transport).Actors().IterateAsync()) @@ -69,11 +69,11 @@ public async Task CollectionIterateDoesNotStopAtAnEarlierPagesStaleTotal() // iteration runs. The iterator must keep paging past an earlier page's `total` and only stop once a // page actually scans no rows. var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"total\":2,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}") + .QueueResponse(200, SpecFixtures.ActorPage(2, "a", "b")) // By the time this page is fetched, a third item was added: total is now 3, not the 2 the first // page reported. - .QueueResponse(200, "{\"data\":{\"total\":3,\"items\":[{\"id\":\"c\"}]}}") - .QueueResponse(200, "{\"data\":{\"total\":3,\"items\":[]}}"); + .QueueResponse(200, SpecFixtures.ActorPage(3, "c")) + .QueueResponse(200, SpecFixtures.ActorPage(3)); var ids = new List(); await foreach (var actor in Client(transport).Actors().IterateAsync()) @@ -89,7 +89,7 @@ public async Task CollectionIterateDoesNotStopAtAnEarlierPagesStaleTotal() public async Task CollectionIterateStopsAtLimit() { var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"total\":10,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}"); + .QueueResponse(200, SpecFixtures.ActorPage(10, "a", "b")); var ids = new List(); await foreach (var actor in Client(transport).Actors().IterateAsync(new ActorListOptions { Limit = 2 })) @@ -108,7 +108,8 @@ public async Task DatasetIterateItemsWalksAllPagesByOffset() { var transport = new MockTransport() .QueueResponse(200, "[{\"i\":1},{\"i\":2}]", Page1Headers) - .QueueResponse(200, "[{\"i\":3}]", Page2Headers); + .QueueResponse(200, "[{\"i\":3}]", Page2Headers) + .QueueResponse(200, "[]"); // a final empty page confirms nothing is left var values = new List(); await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync()) @@ -117,12 +118,33 @@ public async Task DatasetIterateItemsWalksAllPagesByOffset() } Assert.Equal(new[] { 1, 2, 3 }, values); - Assert.Equal(2, transport.CallCount); + Assert.Equal(3, transport.CallCount); Assert.Contains("/datasets/ds1/items", transport.Received[0].Uri, StringComparison.Ordinal); Assert.Contains("offset=0", transport.Received[0].Uri, StringComparison.Ordinal); Assert.Contains("offset=2", transport.Received[1].Uri, StringComparison.Ordinal); } + [Fact] + public async Task DatasetIterateItemsDoesNotStopAtAnEarlierPagesStaleTotal() + { + // The dataset grows while the iteration runs: page 1 reports a total of 2 (and has 2 items), yet a + // third item exists by the time page 2 is fetched. Stopping once offset reached that total would lose it. + var staleTotal = new Dictionary { ["X-Apify-Pagination-Total"] = "2" }; + var grownTotal = new Dictionary { ["X-Apify-Pagination-Total"] = "3" }; + var transport = new MockTransport() + .QueueResponse(200, "[{\"i\":1},{\"i\":2}]", staleTotal) + .QueueResponse(200, "[{\"i\":3}]", grownTotal) + .QueueResponse(200, "[]", grownTotal); + + var values = new List(); + await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync()) + { + values.Add(item!["i"]!.GetValue()); + } + + Assert.Equal(new[] { 1, 2, 3 }, values); + } + [Fact] public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenAPageIsFullyFiltered() { @@ -154,7 +176,8 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenA var transport = new MockTransport() .QueueResponse(200, "[]", page1Headers) // fully filtered: 0 items returned from 2 scanned .QueueResponse(200, "[{\"i\":3}]", page2Headers) // 1 item returned from 2 scanned - .QueueResponse(200, "[{\"i\":5}]", page3Headers); + .QueueResponse(200, "[{\"i\":5}]", page3Headers) + .QueueResponse(200, "[]"); var values = new List(); await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync()) @@ -163,7 +186,7 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenA } Assert.Equal(new[] { 3, 5 }, values); - Assert.Equal(3, transport.CallCount); + Assert.Equal(4, transport.CallCount); Assert.Contains("offset=0", transport.Received[0].Uri, StringComparison.Ordinal); Assert.Contains("offset=2", transport.Received[1].Uri, StringComparison.Ordinal); Assert.Contains("offset=4", transport.Received[2].Uri, StringComparison.Ordinal); @@ -173,7 +196,7 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenA public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenUnwindExpandsAPage() { // Regression test: `unwind` can make a page return more items than rows scanned. Advancing by the - // returned item count (3) instead of the scanned count (1) would skip the remaining 4 rows. + // returned item count (3) instead of the rows the page was asked to scan (1) would skip rows. var page1Headers = new Dictionary { ["X-Apify-Pagination-Total"] = "5", @@ -185,31 +208,73 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenU { ["X-Apify-Pagination-Total"] = "5", ["X-Apify-Pagination-Offset"] = "1", - ["X-Apify-Pagination-Limit"] = "4", - ["X-Apify-Pagination-Count"] = "4", + ["X-Apify-Pagination-Limit"] = "1", + ["X-Apify-Pagination-Count"] = "1", }; var transport = new MockTransport() .QueueResponse(200, "[{\"i\":1},{\"i\":2},{\"i\":3}]", page1Headers) - .QueueResponse(200, "[{\"i\":4}]", page2Headers); + .QueueResponse(200, "[{\"i\":4}]", page2Headers) + .QueueResponse(200, "[]"); var values = new List(); - await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync()) + await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync(new DatasetListItemsOptions { Unwind = new[] { "a" }, ChunkSize = 1 })) { values.Add(item!["i"]!.GetValue()); } Assert.Equal(new[] { 1, 2, 3, 4 }, values); - Assert.Equal(2, transport.CallCount); + Assert.Equal(3, transport.CallCount); Assert.Contains("offset=0", transport.Received[0].Uri, StringComparison.Ordinal); Assert.Contains("offset=1", transport.Received[1].Uri, StringComparison.Ordinal); + Assert.Contains("offset=2", transport.Received[2].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task DatasetIterateItemsLimitCapsTheRowsReadNotTheItemsYielded() + { + // With a filter, a page can scan its whole limit while returning nothing. Limit = 2 allows reading 2 + // rows only, so iteration ends after this page instead of reading on until 2 items were yielded. + var scanned = new Dictionary { ["X-Apify-Pagination-Count"] = "2" }; + var transport = new MockTransport().QueueResponse(200, "[]", scanned); + + var count = 0; + await foreach (var unused in Client(transport).Dataset("ds1").IterateItemsAsync(new DatasetListItemsOptions { SkipEmpty = true, Limit = 2 })) + { + _ = unused; + count++; + } + + Assert.Equal(0, count); + Assert.Equal(1, transport.CallCount); + } + + [Fact] + public async Task DatasetIterateItemsDoesNotStopWhenTheScannedCountHeaderLagsBehindReturnedItems() + { + // The scanned count is derived from the dataset's item count, which lags a fresh push: a page can + // return items while the header still reports 0. The larger of the two is used, so iteration goes on. + var laggingHeaders = new Dictionary { ["X-Apify-Pagination-Count"] = "0" }; + var transport = new MockTransport() + .QueueResponse(200, "[{\"i\":1},{\"i\":2}]", laggingHeaders) + .QueueResponse(200, "[{\"i\":3}]", laggingHeaders) + .QueueResponse(200, "[]", laggingHeaders); + + var values = new List(); + await foreach (var item in Client(transport).Dataset("ds1").IterateItemsAsync()) + { + values.Add(item!["i"]!.GetValue()); + } + + Assert.Equal(new[] { 1, 2, 3 }, values); + Assert.Contains("offset=2", transport.Received[1].Uri, StringComparison.Ordinal); } [Fact] public async Task KeyValueStoreIterateKeysWalksAllPagesByExclusiveStartKey() { var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"items\":[{\"key\":\"a\"},{\"key\":\"b\"}],\"limit\":2,\"isTruncated\":true,\"nextExclusiveStartKey\":\"b\"}}") - .QueueResponse(200, "{\"data\":{\"items\":[{\"key\":\"c\"}],\"limit\":2,\"isTruncated\":false}}"); + .QueueResponse(200, SpecFixtures.KeysPage(2, true, "b", "a", "b")) + .QueueResponse(200, SpecFixtures.KeysPage(2, false, null, "c")); var keys = new List(); await foreach (var key in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { MaxPageLimit = 2 })) @@ -231,7 +296,7 @@ public async Task KeyValueStoreIterateKeysWalksAllPagesByExclusiveStartKey() public async Task KeyValueStoreIterateKeysStopsAtLimit() { var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"items\":[{\"key\":\"a\"},{\"key\":\"b\"}],\"limit\":2,\"isTruncated\":true,\"nextExclusiveStartKey\":\"b\"}}"); + .QueueResponse(200, SpecFixtures.KeysPage(2, true, "b", "a", "b")); var keys = new List(); await foreach (var key in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { Limit = 2 })) @@ -268,7 +333,7 @@ public async Task KeyValueStoreIterateKeysRejectsNonPositiveMaxPageLimit(int max { var transport = new MockTransport(); - await Assert.ThrowsAsync(async () => + await Assert.ThrowsAsync(async () => { await foreach (var _ in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { MaxPageLimit = maxPageLimit })) { @@ -299,7 +364,7 @@ public async Task KeyValueStoreIterateKeysStopsOnTruncatedPageWithNullCursor() // reporting `isTruncated=true` with a null/absent cursor must still stop iteration instead of // looping back to the same first page forever and yielding duplicate keys. var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"items\":[{\"key\":\"a\"},{\"key\":\"b\"}],\"limit\":2,\"isTruncated\":true}}"); + .QueueResponse(200, SpecFixtures.KeysPage(2, true, null, "a", "b")); var keys = new List(); await foreach (var key in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { MaxPageLimit = 2 })) diff --git a/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs b/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs new file mode 100644 index 0000000..13df3e3 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/ChunkSizeTests.cs @@ -0,0 +1,284 @@ +using System; +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// Verifies the ChunkSize iteration option, which sizes each page's limit (the smaller of the +/// chunk size and the remaining total Limit) and is never sent as a query parameter itself. +/// +[Trait("Category", "Unit")] +public sealed class ChunkSizeTests +{ + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + private static async Task DrainAsync(System.Collections.Generic.IAsyncEnumerable items) + { + await foreach (var unused in items) + { + _ = unused; + } + } + + [Fact] + public async Task CollectionIterationRequestsTheSmallerOfChunkSizeAndRemainingLimit() + { + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.Page(5, SpecFixtures.TaskItem("a"), SpecFixtures.TaskItem("b"))) + .QueueResponse(200, SpecFixtures.Page(5, SpecFixtures.TaskItem("c"), SpecFixtures.TaskItem("d"))) + .QueueResponse(200, SpecFixtures.Page(5, SpecFixtures.TaskItem("e"))); + + await DrainAsync(Client(transport).Tasks().IterateAsync(new ListOptions { Limit = 5, ChunkSize = 2 })); + + Assert.Equal(3, transport.CallCount); + Assert.Contains("limit=2", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.Contains("limit=2", transport.Received[1].Uri, StringComparison.Ordinal); + Assert.Contains("limit=1", transport.Received[2].Uri, StringComparison.Ordinal); + Assert.DoesNotContain("chunkSize", transport.Received[0].Uri, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task CollectionIterationWithoutLimitUsesChunkSizeAsPageLimit() + { + var transport = new MockTransport().QueueResponse(200, SpecFixtures.ActorPage(1, "a")).QueueResponse(200, SpecFixtures.ActorPage(1)); + + await DrainAsync(Client(transport).Actors().IterateAsync(new ActorListOptions { ChunkSize = 7 })); + + Assert.All(transport.Received, static r => Assert.Contains("limit=7", r.Uri, StringComparison.Ordinal)); + } + + [Fact] + public async Task ZeroLimitDoesNotBeatChunkSize() + { + // The API reads limit=0 as unset, so it must lose to the chunk size. + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(1, SpecFixtures.ScheduleItem("a"))).QueueResponse(200, SpecFixtures.Page(1)); + + await DrainAsync(Client(transport).Schedules().IterateAsync(new ListOptions { Limit = 0, ChunkSize = 3 })); + + Assert.All(transport.Received, static r => Assert.Contains("limit=3", r.Uri, StringComparison.Ordinal)); + } + + [Fact] + public async Task ZeroLimitMeansNoLimitForCollectionStoreAndDatasetItemIteration() + { + var collection = new MockTransport().QueueResponse(200, SpecFixtures.ActorPage(3, "a", "b")).QueueResponse(200, SpecFixtures.ActorPage(3, "c")).QueueResponse(200, SpecFixtures.ActorPage(3)); + var collected = 0; + await foreach (var unused in Client(collection).Actors().IterateAsync(new ActorListOptions { Limit = 0, ChunkSize = 2 })) + { + _ = unused; + collected++; + } + + Assert.Equal(3, collected); + + var store = new MockTransport().QueueResponse(200, SpecFixtures.Page(2, SpecFixtures.StoreItem("a"))).QueueResponse(200, SpecFixtures.Page(2, SpecFixtures.StoreItem("b"))).QueueResponse(200, SpecFixtures.Page(2)); + var stored = 0; + await foreach (var unused in Client(store).Store().IterateAsync(new StoreListOptions { Limit = 0, ChunkSize = 1 })) + { + _ = unused; + stored++; + } + + Assert.Equal(2, stored); + + var dataset = new MockTransport().QueueResponse(200, "[{\"n\":1},{\"n\":2}]").QueueResponse(200, "[{\"n\":3}]").QueueResponse(200, "[]"); + var items = 0; + await foreach (var unused in Client(dataset).Dataset("ds1").IterateItemsAsync(new DatasetListItemsOptions { Limit = 0, ChunkSize = 2 })) + { + _ = unused; + items++; + } + + Assert.Equal(3, items); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public async Task NonPositiveChunkSizeIsRejectedBeforeAnyRequest(int chunkSize) + { + var transport = new MockTransport(); + + var ex = await Assert.ThrowsAsync(() => DrainAsync(Client(transport).Runs().IterateAsync(new ListOptions { ChunkSize = chunkSize }))); + + Assert.Equal("ChunkSize", ex.ParamName); + Assert.Equal(0, transport.CallCount); + } + + [Fact] + public async Task DatasetItemIterationSizesPagesByChunkSize() + { + var headers = new System.Collections.Generic.Dictionary { ["X-Apify-Pagination-Total"] = "3" }; + var transport = new MockTransport() + .QueueResponse(200, "[{\"n\":1},{\"n\":2}]", headers) + .QueueResponse(200, "[{\"n\":3}]", headers); + + await DrainAsync(Client(transport).Dataset("ds1").IterateItemsAsync(new DatasetListItemsOptions { Limit = 3, ChunkSize = 2 })); + + Assert.Contains("limit=2", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.Contains("limit=1", transport.Received[1].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task KeyIterationHonoursChunkSizeOverItsOlderName() + { + var transport = new MockTransport().QueueResponse(200, SpecFixtures.KeysPage(4, false, null)); + + await DrainAsync(Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { ChunkSize = 4, MaxPageLimit = 9 })); + + Assert.Contains("limit=4", transport.Received[0].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task RequestIterationHonoursChunkSize() + { + var transport = new MockTransport().QueueResponse(200, SpecFixtures.ListRequestsPage(5)); + + await DrainAsync(Client(transport).RequestQueue("q1").PaginateRequestsAsync(new PaginateRequestsOptions { ChunkSize = 5 })); + + Assert.Contains("limit=5", transport.Received[0].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task StoreIterationSizesPagesByChunkSizeAndCapsTotalAtLimit() + { + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.Page(9, SpecFixtures.StoreItem("a"), SpecFixtures.StoreItem("b"))) + .QueueResponse(200, SpecFixtures.Page(9, SpecFixtures.StoreItem("c"))); + + var ids = new System.Collections.Generic.List(); + await foreach (var item in Client(transport).Store().IterateAsync(new StoreListOptions { ChunkSize = 2, Limit = 3 })) + { + ids.Add(item.Id); + } + + Assert.Equal(new[] { "a", "b", "c" }, ids); + Assert.Equal(2, transport.CallCount); + Assert.Contains("limit=2", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.Contains("limit=1", transport.Received[1].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task StoreIterationAdvancesByTheRowsScannedNotTheItemsReturned() + { + // The Store listing drops Actors the caller cannot run after applying offset and limit, so a page that + // scanned 3 rows can return only 2 Actors. The next page must start after all 3 rows. + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.PageWithCount(9, 3, SpecFixtures.StoreItem("a"), SpecFixtures.StoreItem("b"))) + .QueueResponse(200, SpecFixtures.Page(9)); + + await DrainAsync(Client(transport).Store().IterateAsync(new StoreListOptions { ChunkSize = 3 })); + + Assert.Contains("offset=3", transport.Received[1].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task StoreIterationContinuesPastAPageThatScannedRowsButReturnedNoItems() + { + // Every Actor of the first page was dropped (count 1, no items); the next page still has one. + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.PageWithCount(9, 1)) + .QueueResponse(200, SpecFixtures.Page(9, SpecFixtures.StoreItem("a"))) + .QueueResponse(200, SpecFixtures.Page(9)); + + var ids = new System.Collections.Generic.List(); + await foreach (var item in Client(transport).Store().IterateAsync(new StoreListOptions { ChunkSize = 1 })) + { + ids.Add(item.Id); + } + + Assert.Equal(new[] { "a" }, ids); + Assert.Contains("offset=1", transport.Received[1].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task StoreLimitCountsTheActorsYieldedEvenWhenAPageDropsSome() + { + // Limit = 3 asks for three Actors. The first page scanned 3 rows but returned only 2 (one was + // dropped), so one more Actor is still wanted, from the row after the 3 scanned. + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.PageWithCount(9, 3, SpecFixtures.StoreItem("a"), SpecFixtures.StoreItem("b"))) + .QueueResponse(200, SpecFixtures.Page(9, SpecFixtures.StoreItem("c"))); + + var ids = new System.Collections.Generic.List(); + await foreach (var item in Client(transport).Store().IterateAsync(new StoreListOptions { Limit = 3, ChunkSize = 3 })) + { + ids.Add(item.Id); + } + + Assert.Equal(new[] { "a", "b", "c" }, ids); + Assert.Equal(2, transport.CallCount); + Assert.Contains("offset=3", transport.Received[1].Uri, StringComparison.Ordinal); + Assert.Contains("limit=1", transport.Received[1].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task ScannedRowsAreCappedAtTheRequestedLimit() + { + // Limit = 2 asks for 2 rows; a page claiming 5 scanned rows must not advance past what was asked for. + var transport = new MockTransport().QueueResponse(200, SpecFixtures.PageWithCount(9, 5, SpecFixtures.ActorItem("a"), SpecFixtures.ActorItem("b"))); + + await DrainAsync(Client(transport).Actors().IterateAsync(new ActorListOptions { Limit = 2 })); + + Assert.Equal(1, transport.CallCount); + } + + [Theory] + [InlineData("Offset")] + [InlineData("Limit")] + public async Task NegativeOffsetOrLimitIsRejectedBeforeAnyRequest(string option) + { + var transport = new MockTransport(); + var options = option == "Offset" ? new ActorListOptions { Offset = -1 } : new ActorListOptions { Limit = -1 }; + + var ex = await Assert.ThrowsAsync(() => DrainAsync(Client(transport).Actors().IterateAsync(options))); + + Assert.Equal(option, ex.ParamName); + Assert.Equal(0, transport.CallCount); + } + + [Fact] + public async Task StoreIterationLimitIsATotalCapNotAPageSize() + { + // Limit = 2 is satisfied by the first page, so nothing more may be requested. + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(50, SpecFixtures.StoreItem("a"), SpecFixtures.StoreItem("b"))); + + var count = 0; + await foreach (var unused in Client(transport).Store().IterateAsync(new StoreListOptions { Limit = 2 })) + { + _ = unused; + count++; + } + + Assert.Equal(2, count); + Assert.Equal(1, transport.CallCount); + Assert.Contains("limit=2", transport.Received[0].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task StoreIterationDoesNotStopAtAnEarlierPagesStaleTotal() + { + // The Store grows between pages: page 1 reports a total of 2, but a third item exists on page 2. + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.Page(2, SpecFixtures.StoreItem("a"), SpecFixtures.StoreItem("b"))) + .QueueResponse(200, SpecFixtures.Page(3, SpecFixtures.StoreItem("c"))) + .QueueResponse(200, SpecFixtures.Page(3)); + + var count = 0; + await foreach (var unused in Client(transport).Store().IterateAsync()) + { + _ = unused; + count++; + } + + Assert.Equal(3, count); + } +} diff --git a/tests/Apify.Client.Tests/Unit/HttpClientTests.cs b/tests/Apify.Client.Tests/Unit/HttpClientTests.cs index b0dbbd0..44efef5 100644 --- a/tests/Apify.Client.Tests/Unit/HttpClientTests.cs +++ b/tests/Apify.Client.Tests/Unit/HttpClientTests.cs @@ -20,7 +20,7 @@ public sealed class HttpClientTests [Fact] public async Task AuthAndUserAgentHeadersAreSent() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"abc\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Actor("abc")); await Client(transport).Actor("abc").GetAsync(); Assert.Equal("Bearer test-token", transport.LastRequest.Header("Authorization")); @@ -30,7 +30,7 @@ public async Task AuthAndUserAgentHeadersAreSent() [Fact] public async Task DataEnvelopeIsUnwrapped() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"act1\",\"name\":\"my-actor\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Actor("act1", "my-actor")); var actor = await Client(transport).Actor("act1").GetAsync(); Assert.NotNull(actor); @@ -50,7 +50,7 @@ public async Task ServerErrorsAreRetriedThenSucceed() { var transport = new MockTransport() .QueueResponse(500, "{\"error\":{\"type\":\"server\",\"message\":\"boom\"}}") - .QueueResponse(200, "{\"data\":{\"id\":\"ok\"}}"); + .QueueResponse(200, SpecFixtures.Actor("ok")); var actor = await Client(transport).Actor("x").GetAsync(); Assert.Equal("ok", actor!.Id); @@ -76,7 +76,7 @@ public async Task TransportErrorsAreRetried() { var transport = new MockTransport() .QueueError() - .QueueResponse(200, "{\"data\":{\"id\":\"recovered\"}}"); + .QueueResponse(200, SpecFixtures.Actor("recovered")); var actor = await Client(transport).Actor("x").GetAsync(); Assert.Equal("recovered", actor!.Id); Assert.Equal(2, transport.CallCount); @@ -85,7 +85,7 @@ public async Task TransportErrorsAreRetried() [Fact] public async Task BooleanQueryParamsEncodedAsOneZero() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[],\"total\":0}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(0)); await Client(transport).Actors().ListAsync(new ActorListOptions { My = true, Limit = 5 }); var uri = transport.LastRequest.Uri; @@ -96,8 +96,7 @@ public async Task BooleanQueryParamsEncodedAsOneZero() [Fact] public async Task ListUnwrapsPaginationEnvelope() { - const string body = "{\"data\":{\"total\":2,\"offset\":0,\"limit\":10,\"count\":2,\"desc\":false,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}"; - var transport = new MockTransport().QueueResponse(200, body); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.ActorPage(2, "a", "b")); var page = await Client(transport).Actors().ListAsync(); Assert.Equal(2, page.Total); @@ -111,7 +110,7 @@ public async Task PaginationCountReflectsItemsNotServerMetadata() // Server reports total/count larger than the items actually returned in this page. Count and the // indexer must operate on the item array so `for (i < Count) page[i]` cannot throw; Total keeps the // API's reported total. - const string body = "{\"data\":{\"total\":100,\"offset\":0,\"limit\":2,\"count\":100,\"desc\":false,\"items\":[{\"id\":\"a\"},{\"id\":\"b\"}]}}"; + var body = SpecFixtures.ActorPage(2, "a", "b").Replace("\"total\":2", "\"total\":100", StringComparison.Ordinal).Replace("\"count\":2", "\"count\":100", StringComparison.Ordinal); var transport = new MockTransport().QueueResponse(200, body); var page = await Client(transport).Actors().ListAsync(); @@ -151,7 +150,7 @@ public async Task ValidateInputParsesBareObject() [Fact] public async Task SafeIdReplacesFirstSlashWithTilde() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"x\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Actor("x")); await Client(transport).Actor("apify/hello-world").GetAsync(); Assert.Contains("/actors/apify~hello-world", transport.LastRequest.Uri, StringComparison.Ordinal); } @@ -162,7 +161,7 @@ public async Task RateLimitIsRetriedThenSucceeds() // 429 (rate limit) must be retried just like a 5xx, then succeed on the next attempt. var transport = new MockTransport() .QueueResponse(429, "{\"error\":{\"type\":\"rate-limit-exceeded\",\"message\":\"slow down\"}}") - .QueueResponse(200, "{\"data\":{\"id\":\"ok\"}}"); + .QueueResponse(200, SpecFixtures.Actor("ok")); var actor = await Client(transport).Actor("x").GetAsync(); Assert.Equal("ok", actor!.Id); @@ -244,7 +243,7 @@ public async Task ShortTierCallUsesConfiguredShortTimeoutByDefault() { // ActorClient.GetAsync is a Short-tier call (simple metadata read). With no explicit per-call/ // per-context override, its attempt timeout is TimeoutShortSecs, not the overall TimeoutSecs. - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"x\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Actor("x")); var client = new ApifyClient(new ApifyClientOptions { Token = "t", @@ -264,7 +263,7 @@ public async Task ShortTierCallUsesConfiguredShortTimeoutByDefault() public async Task MediumTierCallUsesConfiguredMediumTimeoutByDefault() { // ActorCollectionClient.ListAsync is a Medium-tier call (collection listing). - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"total\":0,\"items\":[]}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(0)); var client = new ApifyClient(new ApifyClientOptions { Token = "t", @@ -335,7 +334,7 @@ await Client(transport) public async Task NullQueryParamsAreOmitted() { // Only Limit is set; every other (null) option must be absent from the query string entirely. - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[],\"total\":0}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(0)); await Client(transport).Actors().ListAsync(new ActorListOptions { Limit = 5 }); var uri = transport.LastRequest.Uri; diff --git a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs index c1eb7c2..2cdd36a 100644 --- a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs +++ b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs @@ -59,7 +59,7 @@ await Client(transport).Actor("me/act") [Fact] public async Task LastRunRequestQueueForwardsStatusAndOrigin() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[]}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.QueueHead()); await Client(transport).Actor("me/act") .LastRun(new LastRunOptions { Status = "SUCCEEDED", Origin = "API" }) .RequestQueue() @@ -117,6 +117,19 @@ public async Task RunChargeSendsBodyAndIdempotencyKey() Assert.Equal(3, body["count"]!.GetValue()); } + [Fact] + public async Task RunChargeSendsRequestIdOnlyWhenSet() + { + var transport = new MockTransport().QueueResponse(200, string.Empty).QueueResponse(200, string.Empty); + var run = Client(transport).Run("run1"); + + await run.ChargeAsync(new RunChargeOptions("e") { RequestId = "req-1" }); + Assert.Equal("req-1", JsonNode.Parse(transport.LastRequest.Body)!["requestId"]!.GetValue()); + + await run.ChargeAsync(new RunChargeOptions("e")); + Assert.Null(JsonNode.Parse(transport.LastRequest.Body)!.AsObject()["requestId"]); + } + [Fact] public async Task RunChargeUsesProvidedIdempotencyKey() { @@ -128,7 +141,7 @@ public async Task RunChargeUsesProvidedIdempotencyKey() [Fact] public async Task MetamorphSendsTargetActorIdAndInput() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); await Client(transport).Run("run1").MetamorphAsync("apify/other", new { x = 1 }, new MetamorphOptions { Build = "latest" }); var request = transport.LastRequest; @@ -142,7 +155,7 @@ public async Task MetamorphSendsTargetActorIdAndInput() [Fact] public async Task ResurrectSendsOptions() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); await Client(transport).Run("run1").ResurrectAsync(new RunResurrectOptions { Build = "beta", MemoryMbytes = 1024 }); var uri = transport.LastRequest.Uri; @@ -154,7 +167,7 @@ public async Task ResurrectSendsOptions() [Fact] public async Task RebootPostsToRebootPath() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); await Client(transport).Run("run1").RebootAsync(); var request = transport.LastRequest; @@ -165,7 +178,7 @@ public async Task RebootPostsToRebootPath() [Fact] public async Task AbortSendsGracefullyFlag() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); await Client(transport).Run("run1").AbortAsync(true); Assert.Contains("gracefully=1", transport.LastRequest.Uri, StringComparison.Ordinal); } @@ -173,7 +186,7 @@ public async Task AbortSendsGracefullyFlag() [Fact] public async Task DefaultBuildFetchesBuildsDefault() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"build1\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Build("build1")); await Client(transport).Actor("me~a").DefaultBuildAsync(10); var request = transport.LastRequest; @@ -185,7 +198,7 @@ public async Task DefaultBuildFetchesBuildsDefault() [Fact] public async Task RequestQueueOptionsApplyClientKey() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[]}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.QueueHead()); await Client(transport).RequestQueue("q1", new RequestQueueClientOptions { ClientKey = "ck-123" }).ListHeadAsync(5); Assert.Contains("clientKey=ck-123", transport.LastRequest.Uri, StringComparison.Ordinal); @@ -194,7 +207,7 @@ public async Task RequestQueueOptionsApplyClientKey() [Fact] public async Task RequestQueueOptionsApplyTimeout() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[]}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.QueueHead()); await Client(transport).RequestQueue("q1", new RequestQueueClientOptions { TimeoutSecs = 2.0 }).ListHeadAsync(5); // The per-queue timeout must be threaded down to the transport (first attempt uses it directly). @@ -209,8 +222,8 @@ public async Task WithTimeoutOverridesTheTierDefaultForEveryCallOnThatClient() // as long as it doesn't exceed the overall budget (ApifyClientOptions.TimeoutSecs — see the // dedicated cap test below). var transport = new MockTransport() - .QueueResponse(200, "{\"data\":{\"id\":\"x\"}}") - .QueueResponse(200, "{\"data\":{\"id\":\"x\"}}"); + .QueueResponse(200, SpecFixtures.Actor("x")) + .QueueResponse(200, SpecFixtures.Actor("x")); var client = new ApifyClient(new ApifyClientOptions { Token = "t", @@ -234,7 +247,7 @@ public async Task WithTimeoutAboveTheOverallBudgetIsCappedAtIt() // Matches the reference client: a requested timeout above timeoutMaxSecs (here, // ApifyClientOptions.TimeoutSecs) is capped at it rather than silently exceeding it — raising the // overall budget itself is how a caller gets a longer per-call timeout. - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"x\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Actor("x")); var client = new ApifyClient(new ApifyClientOptions { Token = "t", @@ -254,7 +267,7 @@ public async Task WithTimeoutZeroMeansNoTimeout() // TimeSpan.Zero is the client's "no timeout" sentinel (HttpClientTransport skips CancelAfter for a // non-positive duration), matching the reference client's 'noTimeout' — which, like here, bypasses // the overall-budget cap entirely rather than being clamped to it. - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"x\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Actor("x")); await Client(transport).Actor("x").WithTimeout(TimeSpan.Zero).GetAsync(); @@ -313,7 +326,7 @@ public async Task PublishPutsIsPublicTrue() { var transport = new MockTransport().QueueResponse( 200, - "{\"data\":{\"id\":\"t1\",\"isPublic\":true,\"publicConfig\":{\"publishedAt\":\"2026-01-01T00:00:00.000Z\",\"seoTitle\":\"My task\"}}}"); + SpecFixtures.Task("t1", "\"isPublic\":true,\"publicConfig\":{\"publishedAt\":\"2026-01-01T00:00:00.000Z\",\"seoTitle\":\"My task\"}")); var task = await Client(transport).Task("t1").PublishAsync(); var request = transport.LastRequest; @@ -328,7 +341,7 @@ public async Task PublishPutsIsPublicTrue() [Fact] public async Task UnpublishPutsIsPublicFalse() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"t1\",\"isPublic\":false}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Task("t1", "\"isPublic\":false")); var task = await Client(transport).Task("t1").UnpublishAsync(); var request = transport.LastRequest; @@ -359,7 +372,7 @@ await Client(transport).Dataset("ds1").ListItemsAsync(new DatasetListItemsOption [Fact] public async Task RunListJoinsStatusAsCsv() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[],\"total\":0}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Page(0)); await Client(transport).Runs().ListAsync(null, new RunListOptions { Status = new[] { "SUCCEEDED", "RUNNING" } }); Assert.Contains("status=SUCCEEDED%2CRUNNING", transport.LastRequest.Uri, StringComparison.Ordinal); @@ -368,7 +381,7 @@ public async Task RunListJoinsStatusAsCsv() [Fact] public async Task ListRequestsJoinsFilterAsCsv() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"items\":[]}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.QueueHead()); await Client(transport).RequestQueue("q1").ListRequestsAsync(new ListRequestsOptions { Filter = new[] { ListRequestsOptions.FilterLocked, ListRequestsOptions.FilterPending }, @@ -385,7 +398,7 @@ public async Task MaxTotalChargeUsdIsFormattedWithInvariantCulture() CultureInfo.CurrentCulture = new CultureInfo("de-DE"); try { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); await Client(transport).Actor("act").StartAsync(null, new ActorStartOptions { MaxTotalChargeUsd = 12.5 }); var uri = transport.LastRequest.Uri; @@ -401,7 +414,7 @@ public async Task MaxTotalChargeUsdIsFormattedWithInvariantCulture() [Fact] public async Task StartAsyncSendsRawBytesInputVerbatimWithoutJsonEncoding() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); var rawInput = new byte[] { 0x00, 0x01, 0xFF, (byte)'{', (byte)'}' }; await Client(transport).Actor("act").StartAsync(rawInput, new ActorStartOptions { ContentType = "application/octet-stream" }); @@ -413,7 +426,7 @@ public async Task StartAsyncSendsRawBytesInputVerbatimWithoutJsonEncoding() [Fact] public async Task MetamorphAsyncSendsRawBytesInputVerbatimWithoutJsonEncoding() { - var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"r\"}}"); + var transport = new MockTransport().QueueResponse(200, SpecFixtures.Run("r")); var rawInput = new byte[] { 0x10, 0x20, 0x30 }; await Client(transport).Run("run1").MetamorphAsync( diff --git a/tests/Apify.Client.Tests/Unit/ResponseDecompressionTests.cs b/tests/Apify.Client.Tests/Unit/ResponseDecompressionTests.cs index d3696d9..359149f 100644 --- a/tests/Apify.Client.Tests/Unit/ResponseDecompressionTests.cs +++ b/tests/Apify.Client.Tests/Unit/ResponseDecompressionTests.cs @@ -25,7 +25,7 @@ public sealed class ResponseDecompressionTests [InlineData("deflate")] public async Task DefaultTransportDecompressesResponse(string encoding) { - const string json = "{\"data\":{\"id\":\"act1\",\"name\":\"compressed-actor\"}}"; + var json = SpecFixtures.Actor("act1", "compressed-actor"); var payload = Compress(Encoding.UTF8.GetBytes(json), encoding); using var server = new LoopbackServer(encoding, payload); diff --git a/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs b/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs new file mode 100644 index 0000000..ff39268 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/ResponseSchemaTests.cs @@ -0,0 +1,190 @@ +using System; +using System.Linq; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Apify.Client.Exceptions; +using Apify.Client.Internal; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// Verifies the validation of successful responses against the schemas of the Apify OpenAPI +/// specification: required properties, JSON types, nullability, unknown properties and values, and how the +/// failures surface through the resource clients. +/// +[Trait("Category", "Unit")] +public sealed class ResponseSchemaTests +{ + private const string Build = "\"id\":\"b\",\"actId\":\"a\",\"userId\":\"u\",\"startedAt\":\"2026-01-01T00:00:00.000Z\",\"status\":\"SUCCEEDED\",\"meta\":{\"origin\":\"API\"},\"buildNumber\":\"0.0.1\""; + + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + private static string[] Issues(string method, string path, string body) + => ResponseSchemas.Validate(method, path, JsonNode.Parse(body)).ToArray(); + + [Fact] + public void EmbeddedSchemasMatchTheClientsSpecVersion() + { + Assert.Equal(ApifyClientVersion.ApiSpecVersion, ResponseSchemas.SpecVersion); + } + + [Fact] + public void ValidResponseHasNoIssues() + { + Assert.Empty(Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build + "}}")); + } + + [Fact] + public void MissingRequiredPropertyIsReportedWithItsLocation() + { + var issues = Issues("GET", "/v2/actor-builds/b1", "{\"data\":{\"id\":\"b\"}}"); + + Assert.Contains("$.data.actId: required property is missing", issues); + Assert.Contains("$.data.status: required property is missing", issues); + } + + [Fact] + public void WrongJsonTypeIsReported() + { + var issues = Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build.Replace("\"id\":\"b\"", "\"id\":5", StringComparison.Ordinal) + "}}"); + + Assert.Equal(new[] { "$.data.id: expected string but got number" }, issues); + } + + [Fact] + public void NullIsAcceptedWhereTheSpecAllowsIt() + { + // The spec lets a build's actVersion.buildTag be null (apify-docs #3093). + Assert.Empty(Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build + ",\"actVersion\":{\"buildTag\":null}}}")); + } + + [Fact] + public void NullIsRejectedWhereTheSpecDoesNotAllowIt() + { + var issues = Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build + ",\"actVersion\":{\"versionNumber\":null}}}"); + + Assert.Single(issues); + Assert.Contains("actVersion.versionNumber", issues[0], StringComparison.Ordinal); + } + + [Fact] + public void DateTimeFormatIsEnforced() + { + var issues = Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build.Replace("2026-01-01T00:00:00.000Z", "yesterday", StringComparison.Ordinal) + "}}"); + + Assert.Equal(new[] { "$.data.startedAt: is not a valid date-time value" }, issues); + } + + [Fact] + public void PatternIsEnforced() + { + var issues = Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build.Replace("0.0.1", "not-a-version", StringComparison.Ordinal) + "}}"); + + Assert.Single(issues); + Assert.Contains("$.data.buildNumber: does not match the pattern", issues[0], StringComparison.Ordinal); + } + + [Fact] + public void NumericBoundsAreEnforced() + { + var below = Issues("GET", "/v2/actor-runs/r1", "{\"data\":" + SpecFixtures.RunObject("r").Replace("\"memoryMbytes\":256", "\"memoryMbytes\":64", StringComparison.Ordinal) + "}"); + var above = Issues("GET", "/v2/actor-runs/r1", "{\"data\":" + SpecFixtures.RunObject("r").Replace("\"memoryMbytes\":256", "\"memoryMbytes\":999999", StringComparison.Ordinal) + "}"); + + Assert.Equal(new[] { "$.data.options.memoryMbytes: is a value outside the documented range (minimum 128)" }, below); + Assert.Equal(new[] { "$.data.options.memoryMbytes: is a value outside the documented range (maximum 32768)" }, above); + } + + [Fact] + public void EmailIsOnlyRequiredToBeAString() + { + // Like the reference client's schemas: an address such as user@localhost is not rejected. + Assert.Empty(Issues("GET", "/v2/users/me", "{\"data\":{\"username\":\"u\",\"email\":\"user@localhost\"}}")); + } + + [Fact] + public void UriFormatIsEnforced() + { + var issues = Issues("GET", "/v2/key-value-stores/s1/keys", SpecFixtures.KeysPage(1, false, null, "a").Replace("https://example.com/a", "not a url", StringComparison.Ordinal)); + + Assert.Equal(new[] { "$.data.items[0].recordPublicUrl: is not a valid uri value" }, issues); + } + + [Fact] + public async Task ExceptionExposesTheIssuesAsAStructuredList() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"b\"}}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Build("b1").GetAsync()); + + Assert.Contains("$.data.actId: required property is missing", ex.Issues); + Assert.Contains(ex.Issues, static issue => issue.StartsWith("$.data.status", StringComparison.Ordinal)); + Assert.StartsWith("response does not match the API specification: ", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void UnknownPropertiesAndUnknownEnumValuesAreAccepted() + { + Assert.Empty(Issues("GET", "/v2/actor-builds/b1", "{\"data\":{" + Build.Replace("SUCCEEDED", "SOME-FUTURE-STATUS", StringComparison.Ordinal) + ",\"brandNewField\":{\"x\":1}}}")); + } + + [Fact] + public void UndocumentedOperationIsNotValidated() + { + Assert.Empty(Issues("GET", "/v2/not-a-documented-endpoint", "[]")); + } + + [Fact] + public void TheMostSpecificPathTemplateWins() + { + // `/runs/last` must be matched by its own template, not by `/runs/{runId}`: both are valid runs, so + // the way to tell them apart is that an invalid body is still reported for either. + Assert.NotEmpty(Issues("GET", "/v2/actors/a1/runs/last?status=SUCCEEDED", "{\"data\":{}}")); + Assert.NotEmpty(Issues("GET", "/v2/actor-runs/r1", "{\"data\":{}}")); + } + + [Fact] + public async Task GetWithArrayDataThrowsInsteadOfLookingLikeNotFound() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":[]}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actor("me~act").GetAsync()); + + Assert.Equal("GET", ex.HttpMethod); + } + + [Fact] + public async Task ListWithArrayDataThrows() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":[]}"); + + await Assert.ThrowsAsync(() => Client(transport).Actors().ListAsync()); + } + + [Fact] + public async Task ResourceMissingRequiredPropertiesThrowsWithTheIssuesInTheMessage() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"id\":\"b\"}}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Build("b1").GetAsync()); + + Assert.Contains("$.data.actId: required property is missing", ex.Message, StringComparison.Ordinal); + Assert.EndsWith("/actor-builds/b1", ex.Path, StringComparison.Ordinal); + } + + [Fact] + public async Task ValidResourceIsReturned() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{" + Build + "}}"); + + var build = await Client(transport).Build("b1").GetAsync(); + + Assert.Equal("b", build!.Id); + } +} diff --git a/tests/Apify.Client.Tests/Unit/ResponseValidationTests.cs b/tests/Apify.Client.Tests/Unit/ResponseValidationTests.cs new file mode 100644 index 0000000..fb2752f --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/ResponseValidationTests.cs @@ -0,0 +1,87 @@ +using System; +using System.Threading.Tasks; +using Apify.Client.Exceptions; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// Verifies how a successful (2xx) response that is not what the API specification promises is reported: +/// unparsable JSON as , a missing data envelope or a +/// wrongly-typed data as , both with the request context. +/// +[Trait("Category", "Unit")] +public sealed class ResponseValidationTests +{ + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + [Fact] + public async Task UnparsableBodyExceptionCarriesStatusMethodPathAndBody() + { + var transport = new MockTransport().QueueResponse(200, "oops"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actor("me~act").GetAsync()); + + Assert.Equal(200, ex.StatusCode); + Assert.Equal("GET", ex.HttpMethod); + Assert.EndsWith("/actors/me~act", ex.Path, StringComparison.Ordinal); + Assert.Equal("oops", ex.Body); + } + + [Fact] + public async Task MissingDataEnvelopeOnGetThrowsInsteadOfLookingLikeNotFound() + { + var transport = new MockTransport().QueueResponse(200, "{}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actor("me~act").GetAsync()); + + Assert.IsAssignableFrom(ex); + Assert.Equal(200, ex.StatusCode); + Assert.Equal("GET", ex.HttpMethod); + Assert.Equal("{}", ex.Body); + Assert.Equal(1, transport.CallCount); + } + + [Fact] + public async Task NullDataOnGetThrows() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":null}"); + + await Assert.ThrowsAsync(() => Client(transport).Actor("me~act").GetAsync()); + } + + [Fact] + public async Task ArrayDataWhereAnObjectIsExpectedThrowsOnUpdate() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":[]}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actor("me~act").UpdateAsync(new { title = "x" })); + + Assert.Equal("PUT", ex.HttpMethod); + } + + [Fact] + public async Task MissingDataEnvelopeOnCreateThrows() + { + var transport = new MockTransport().QueueResponse(201, "{\"id\":\"a\"}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actors().CreateAsync(new { name = "x" })); + + Assert.Equal("POST", ex.HttpMethod); + Assert.Equal(201, ex.StatusCode); + } + + [Fact] + public async Task NonArrayDatasetItemsBodyThrows() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":[]}"); + + await Assert.ThrowsAsync(() => Client(transport).Dataset("ds1").ListItemsAsync()); + } +} diff --git a/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs b/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs new file mode 100644 index 0000000..3166837 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/RunIterateDatasetItemsTests.cs @@ -0,0 +1,190 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// Verifies RunClient.IterateDatasetItemsAsync: it reads a run's dataset while the run is still +/// producing items, pages by chunkSize up to itemCount, and drains rows past a lagging +/// itemCount once the run is finished. +/// +[Trait("Category", "Unit")] +public sealed class RunIterateDatasetItemsTests +{ + private static readonly string FinishedRun = SpecFixtures.Run("run1"); + private static readonly string RunningRun = SpecFixtures.Run("run1", "RUNNING"); + + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + private static string DatasetWithItemCount(int itemCount) => SpecFixtures.Dataset(itemCount); + + private static string Items(params int[] ids) => "[" + string.Join(",", ids.Select(static i => "{\"n\":" + i + "}")) + "]"; + + private static async Task> CollectAsync(IAsyncEnumerable items) + { + var result = new List(); + await foreach (var item in items) + { + result.Add(item!["n"]!.GetValue()); + } + + return result; + } + + [Fact] + public async Task FinishedRunPagesByChunkSizeAndDrainsRowsPastLaggingItemCount() + { + var transport = new MockTransport() + .QueueResponse(200, FinishedRun) + .QueueResponse(200, DatasetWithItemCount(3)) + .QueueResponse(200, Items(1, 2)) + .QueueResponse(200, Items(3)) + .QueueResponse(200, Items(4)) // past itemCount: the count lags the pushed items + .QueueResponse(200, Items()); + + var items = await CollectAsync(Client(transport).Run("run1").IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { ChunkSize = 2 })); + + Assert.Equal(new[] { 1, 2, 3, 4 }, items); + var uris = transport.Received.Select(static r => r.Uri).ToList(); + Assert.Contains("offset=0", uris[2], StringComparison.Ordinal); + Assert.Contains("limit=2", uris[2], StringComparison.Ordinal); + Assert.Contains("offset=2", uris[3], StringComparison.Ordinal); + Assert.Contains("limit=1", uris[3], StringComparison.Ordinal); + Assert.Contains("offset=3", uris[4], StringComparison.Ordinal); + } + + [Fact] + public async Task RunningRunPollsForFinishAndReadsRowsAsTheyAppear() + { + var transport = new MockTransport() + .QueueResponse(200, RunningRun) + .QueueResponse(200, DatasetWithItemCount(1)) + .QueueResponse(200, Items(1)) + .QueueResponse(200, FinishedRun) // waitForFinish poll + .QueueResponse(200, DatasetWithItemCount(2)) + .QueueResponse(200, Items(2)) + .QueueResponse(200, Items()); + + var items = await CollectAsync(Client(transport).Run("run1").IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { PollIntervalSecs = 1 })); + + Assert.Equal(new[] { 1, 2 }, items); + Assert.Contains("waitForFinish=", transport.Received[3].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task LimitBoundsScannedRowsAndStopsWithoutFurtherReads() + { + var transport = new MockTransport() + .QueueResponse(200, FinishedRun) + .QueueResponse(200, DatasetWithItemCount(10)) + .QueueResponse(200, Items(2, 3)); + + var items = await CollectAsync(Client(transport).Run("run1").IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { Offset = 1, Limit = 2 })); + + Assert.Equal(new[] { 2, 3 }, items); + Assert.Equal(3, transport.CallCount); + Assert.Contains("offset=1", transport.LastRequest.Uri, StringComparison.Ordinal); + Assert.Contains("limit=2", transport.LastRequest.Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task LastRunClientPinsToTheRunItFirstResolved() + { + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.Run("resolvedRun")) + .QueueResponse(200, DatasetWithItemCount(1)) + .QueueResponse(200, Items(1)) + .QueueResponse(200, Items()); + + var items = await CollectAsync(Client(transport).Actor("me~act").LastRun().IterateDatasetItemsAsync()); + + Assert.Equal(new[] { 1 }, items); + Assert.Contains("/actors/me~act/runs/last", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.Contains("/actor-runs/resolvedRun/dataset", transport.Received[1].Uri, StringComparison.Ordinal); + Assert.Contains("/actor-runs/resolvedRun/dataset/items", transport.Received[2].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task FilteredEmptyTailIsConfirmedWithAPlainRead() + { + var transport = new MockTransport() + .QueueResponse(200, FinishedRun) + .QueueResponse(200, DatasetWithItemCount(0)) + .QueueResponse(200, Items()) // filtered page: empty, nothing scanned + .QueueResponse(200, "[{}]") // plain read: a row the filter dropped, so iteration must go on + .QueueResponse(200, Items()) // next filtered page + .QueueResponse(200, Items()); // plain read: nothing left + + var items = await CollectAsync(Client(transport).Run("run1").IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { SkipEmpty = true })); + + Assert.Empty(items); + Assert.Equal(6, transport.CallCount); + Assert.Contains("skipEmpty=1", transport.Received[2].Uri, StringComparison.Ordinal); + Assert.Contains("offset=0", transport.Received[3].Uri, StringComparison.Ordinal); + Assert.DoesNotContain("skipEmpty", transport.Received[3].Uri, StringComparison.Ordinal); + Assert.Contains("offset=1000", transport.Received[4].Uri, StringComparison.Ordinal); + Assert.Contains("offset=1000", transport.Received[5].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task OffsetPlusLimitBeyondIntRangeStillIteratesAllRows() + { + var transport = new MockTransport() + .QueueResponse(200, FinishedRun) + .QueueResponse(200, DatasetWithItemCount(3)) + .QueueResponse(200, Items(2, 3)) + .QueueResponse(200, Items()); + + var items = await CollectAsync(Client(transport).Run("run1").IterateDatasetItemsAsync(new RunIterateDatasetItemsOptions { Offset = 1, Limit = int.MaxValue })); + + Assert.Equal(new[] { 2, 3 }, items); + } + + [Fact] + public async Task RunTimeoutAppliesToTheDatasetRequests() + { + var transport = new MockTransport() + .QueueResponse(200, FinishedRun) + .QueueResponse(200, DatasetWithItemCount(1)) + .QueueResponse(200, Items(1)) + .QueueResponse(200, Items()); + + await CollectAsync(Client(transport).Run("run1").WithTimeout(TimeSpan.FromSeconds(3)).IterateDatasetItemsAsync()); + + Assert.All(transport.Timeouts, static t => Assert.Equal(3, t)); + } + + [Theory] + [InlineData("Offset", -1)] + [InlineData("Limit", -1)] + [InlineData("PollIntervalSecs", -1)] + [InlineData("ChunkSize", 0)] + [InlineData("ChunkSize", -1)] + public async Task InvalidArgumentsAreRejectedNamingTheOption(string option, int value) + { + var options = option switch + { + "Offset" => new RunIterateDatasetItemsOptions { Offset = value }, + "Limit" => new RunIterateDatasetItemsOptions { Limit = value }, + "PollIntervalSecs" => new RunIterateDatasetItemsOptions { PollIntervalSecs = value }, + _ => new RunIterateDatasetItemsOptions { ChunkSize = value }, + }; + var transport = new MockTransport(); + + var ex = await Assert.ThrowsAsync(() => CollectAsync(Client(transport).Run("run1").IterateDatasetItemsAsync(options))); + + Assert.Equal(option, ex.ParamName); + Assert.Equal(0, transport.CallCount); + } +} diff --git a/tests/Apify.Client.Tests/Unit/RunNestedClientTimeoutTests.cs b/tests/Apify.Client.Tests/Unit/RunNestedClientTimeoutTests.cs new file mode 100644 index 0000000..2dc2ab6 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/RunNestedClientTimeoutTests.cs @@ -0,0 +1,92 @@ +using System; +using System.Threading.Tasks; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// A run's and a build's WithTimeout override must reach the requests of the nested storage and log +/// clients obtained from them afterwards, including through a last-run client. +/// +[Trait("Category", "Unit")] +public sealed class RunNestedClientTimeoutTests +{ + private static readonly TimeSpan Override = TimeSpan.FromSeconds(3); + + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + private static string Dataset() => SpecFixtures.Dataset(); + + [Fact] + public async Task RunDatasetInheritsTheTimeout() + { + var transport = new MockTransport().QueueResponse(200, Dataset()); + await Client(transport).Run("r1").WithTimeout(Override).Dataset().GetAsync(); + Assert.Equal(3, transport.Timeouts[0]); + } + + [Fact] + public async Task RunKeyValueStoreInheritsTheTimeout() + { + var transport = new MockTransport().QueueResponse(200, "{\"data\":{\"keyValue\":1}}").QueueResponse(200, "x"); + await Client(transport).Run("r1").WithTimeout(Override).KeyValueStore().RecordExistsAsync("k"); + Assert.All(transport.Timeouts, static t => Assert.Equal(3, t)); + } + + [Fact] + public async Task RunRequestQueueInheritsTheTimeout() + { + var transport = new MockTransport().QueueResponse(200, SpecFixtures.QueueHead()); + await Client(transport).Run("r1").WithTimeout(Override).RequestQueue().ListHeadAsync(1); + Assert.Equal(3, transport.Timeouts[0]); + } + + [Fact] + public async Task RunLogInheritsTheTimeout() + { + var transport = new MockTransport().QueueResponse(200, "log text"); + await Client(transport).Run("r1").WithTimeout(Override).Log().GetAsync(); + Assert.Equal(3, transport.Timeouts[0]); + } + + [Fact] + public async Task BuildLogInheritsTheTimeout() + { + var transport = new MockTransport().QueueResponse(200, "log text"); + await Client(transport).Build("b1").WithTimeout(Override).Log().GetAsync(); + Assert.Equal(3, transport.Timeouts[0]); + } + + [Fact] + public async Task LastRunDatasetInheritsTheTimeout() + { + var transport = new MockTransport().QueueResponse(200, Dataset()); + await Client(transport).Actor("me~act").LastRun().WithTimeout(Override).Dataset().GetAsync(); + Assert.Equal(3, transport.Timeouts[0]); + } + + [Fact] + public async Task IteratingALastRunPinnedToAnotherRunKeepsTheTimeout() + { + // The first request resolves `runs/last` to a concrete run; the pinned client used for the dataset + // requests must carry the same override. + var transport = new MockTransport() + .QueueResponse(200, SpecFixtures.Run("resolved")) + .QueueResponse(200, Dataset()) + .QueueResponse(200, "[]"); + + await foreach (var unused in Client(transport).Actor("me~act").LastRun().WithTimeout(Override).IterateDatasetItemsAsync()) + { + _ = unused; + } + + Assert.Contains("/actor-runs/resolved/dataset", transport.Received[1].Uri, StringComparison.Ordinal); + Assert.All(transport.Timeouts, static t => Assert.Equal(3, t)); + } +} diff --git a/tests/Apify.Client.Tests/Unit/SpecFixtures.cs b/tests/Apify.Client.Tests/Unit/SpecFixtures.cs new file mode 100644 index 0000000..bea0c4c --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/SpecFixtures.cs @@ -0,0 +1,99 @@ +using System.Globalization; +using System.Linq; + +namespace Apify.Client.Tests.Unit; + +/// +/// Response bodies that satisfy the schemas of the Apify OpenAPI specification (the client rejects a +/// successful response that does not), built with only the properties the specification requires. +/// +internal static class SpecFixtures +{ + private const string Timestamp = "2026-01-01T00:00:00.000Z"; + + /// A paginated list response ({"data": {"total", "offset", "limit", "desc", "count", "items"}}). + /// The reported total number of items. + /// The raw JSON of each item on this page. + public static string Page(int total, params string[] items) + => string.Format( + CultureInfo.InvariantCulture, + "{{\"data\":{{\"total\":{0},\"offset\":0,\"limit\":1000,\"desc\":false,\"count\":{1},\"items\":[{2}]}}}}", + total, + items.Length, + string.Join(",", items)); + + /// A paginated list response whose reported count (rows scanned) differs from the items returned. + public static string PageWithCount(int total, int count, params string[] items) + => Page(total, items).Replace("\"count\":" + items.Length.ToString(CultureInfo.InvariantCulture), "\"count\":" + count.ToString(CultureInfo.InvariantCulture), System.StringComparison.Ordinal); + + /// An item of the Actor collection. + public static string ActorItem(string id) + => "{\"id\":\"" + id + "\",\"createdAt\":\"" + Timestamp + "\",\"modifiedAt\":\"" + Timestamp + "\",\"name\":\"n\",\"username\":\"u\"}"; + + /// An item of the Actor task collection. + public static string TaskItem(string id) + => "{\"id\":\"" + id + "\",\"userId\":\"u\",\"actId\":\"a\",\"name\":\"n\",\"createdAt\":\"" + Timestamp + "\",\"modifiedAt\":\"" + Timestamp + "\"}"; + + /// An item of the schedule collection. + public static string ScheduleItem(string id) + => "{\"id\":\"" + id + "\",\"userId\":\"u\",\"name\":\"n\",\"cronExpression\":\"0 0 * * *\",\"timezone\":\"UTC\",\"isEnabled\":false,\"isExclusive\":false,\"createdAt\":\"" + Timestamp + "\",\"modifiedAt\":\"" + Timestamp + "\",\"actions\":[]}"; + + /// An item of the Store collection. + public static string StoreItem(string id) + => "{\"id\":\"" + id + "\",\"title\":\"t\",\"name\":\"n\",\"username\":\"u\",\"stats\":{}}"; + + /// A page of Actors listing the given IDs. + public static string ActorPage(int total, params string[] ids) => Page(total, ids.Select(ActorItem).ToArray()); + + /// An Actor (GET /v2/actors/{actorId} and the Actor update responses). + public static string Actor(string id, string name = "n", bool isPublic = false) + => "{\"data\":{\"id\":\"" + id + "\",\"userId\":\"u\",\"name\":\"" + name + "\",\"username\":\"u\",\"isPublic\":" + (isPublic ? "true" : "false") + + ",\"createdAt\":\"" + Timestamp + "\",\"modifiedAt\":\"" + Timestamp + "\",\"stats\":{},\"versions\":[],\"defaultRunOptions\":{}}}"; + + /// An Actor run. + public static string Run(string id, string status = "SUCCEEDED") + => "{\"data\":" + RunObject(id, status) + "}"; + + /// The bare run object, for tests that compose a larger body. + public static string RunObject(string id, string status = "SUCCEEDED") + => "{\"id\":\"" + id + "\",\"actId\":\"a\",\"userId\":\"u\",\"startedAt\":\"" + Timestamp + "\",\"status\":\"" + status + + "\",\"meta\":{\"origin\":\"API\"},\"stats\":{},\"options\":{\"build\":\"latest\",\"timeoutSecs\":0,\"memoryMbytes\":256,\"diskMbytes\":0},\"buildId\":\"b\"" + + ",\"defaultKeyValueStoreId\":\"kvs\",\"defaultDatasetId\":\"ds\",\"defaultRequestQueueId\":\"rq\"}"; + + /// An Actor build. + public static string Build(string id, string status = "SUCCEEDED") + => "{\"data\":{\"id\":\"" + id + "\",\"actId\":\"a\",\"userId\":\"u\",\"startedAt\":\"" + Timestamp + "\",\"status\":\"" + status + + "\",\"meta\":{\"origin\":\"API\"},\"buildNumber\":\"0.0.1\"}}"; + + /// An Actor task. + /// The task ID. + /// Raw JSON members appended to the task object (e.g. "isPublic":true). + public static string Task(string id, string? extraProperties = null) + { + var item = TaskItem(id); + return "{\"data\":" + (extraProperties is null ? item : item[..^1] + "," + extraProperties + "}") + "}"; + } + + /// A dataset with the given reported item count. + public static string Dataset(int itemCount = 0) + => "{\"data\":{\"id\":\"ds\",\"userId\":\"u\",\"createdAt\":\"" + Timestamp + "\",\"modifiedAt\":\"" + Timestamp + "\",\"accessedAt\":\"" + Timestamp + + "\",\"itemCount\":" + itemCount.ToString(CultureInfo.InvariantCulture) + ",\"cleanItemCount\":0,\"consoleUrl\":\"https://console.apify.com\"}}"; + + /// A page of key-value store keys. + /// The page limit reported by the API. + /// Whether more keys follow. + /// The key to continue after, or null. + /// The keys on this page. + public static string KeysPage(int limit, bool isTruncated, string? nextExclusiveStartKey, params string[] keys) + => "{\"data\":{\"items\":[" + string.Join(",", keys.Select(static k => "{\"key\":\"" + k + "\",\"size\":1,\"recordPublicUrl\":\"https://example.com/" + k + "\"}")) + "],\"count\":" + + keys.Length.ToString(CultureInfo.InvariantCulture) + ",\"limit\":" + limit.ToString(CultureInfo.InvariantCulture) + ",\"isTruncated\":" + (isTruncated ? "true" : "false") + + (nextExclusiveStartKey is null ? string.Empty : ",\"nextExclusiveStartKey\":\"" + nextExclusiveStartKey + "\"") + "}}"; + + /// A request queue head with the given raw request items. + public static string QueueHead(params string[] items) + => "{\"data\":{\"limit\":10,\"queueModifiedAt\":\"" + Timestamp + "\",\"hadMultipleClients\":false,\"items\":[" + string.Join(",", items) + "]}}"; + + /// A page of request queue requests (GET /v2/request-queues/{queueId}/requests) with the given raw request items. + public static string ListRequestsPage(int limit, params string[] items) + => "{\"data\":{\"items\":[" + string.Join(",", items) + "],\"limit\":" + limit.ToString(CultureInfo.InvariantCulture) + "}}"; +} diff --git a/tests/Apify.Client.Tests/Unit/WaitForFinishTests.cs b/tests/Apify.Client.Tests/Unit/WaitForFinishTests.cs index 6c7c439..9005feb 100644 --- a/tests/Apify.Client.Tests/Unit/WaitForFinishTests.cs +++ b/tests/Apify.Client.Tests/Unit/WaitForFinishTests.cs @@ -69,7 +69,7 @@ public async Task WaitForFinishToleratesABriefNotFoundThenSucceeds() // once the resource starts resolving, the run completes normally. var transport = new MockTransport() .QueueResponse(404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"Run was not found\"}}") - .QueueResponse(200, "{\"data\":{\"id\":\"run1\",\"status\":\"SUCCEEDED\"}}"); + .QueueResponse(200, SpecFixtures.Run("run1")); var run = await Client(transport).Run("run1").WaitForFinishAsync(waitSecs: 30); diff --git a/tools/generate_response_schemas.py b/tools/generate_response_schemas.py new file mode 100644 index 0000000..9637346 --- /dev/null +++ b/tools/generate_response_schemas.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Generates src/Apify.Client/Internal/ResponseSchemas.json from the Apify OpenAPI specification. + +The client validates every successful JSON response against the schema the specification declares for the +operation (see ResponseSchemas.cs), so this file has to be regenerated whenever the client is moved to a new +specification version: + + python3 tools/generate_response_schemas.py [SPEC_URL_OR_PATH] + +The default source is https://docs.apify.com/api/openapi.json. Only what validation needs is kept: types, +required properties, nested properties and items, composition keywords, references, and the value +constraints (format, pattern, minimum, maximum, minLength, maxLength). Documentation, examples and +keywords the client deliberately does not enforce (enum, const) are dropped, which also keeps the embedded +resource small. Enum values stay open on purpose so that a value the API adds later does not break +existing clients; the reference client's generated schemas do the same. +""" +import json +import pathlib +import sys +import urllib.request + +DEFAULT_SPEC_URL = "https://docs.apify.com/api/openapi.json" +OUTPUT = pathlib.Path(__file__).resolve().parent.parent / "src" / "Apify.Client" / "Internal" / "ResponseSchemas.json" +HTTP_METHODS = ("get", "post", "put", "delete", "patch") +SCHEMA_REF_PREFIX = "#/components/schemas/" +# Keywords that carry structure the validator evaluates. Everything else is dropped. +SCHEMA_LIST_KEYWORDS = ("allOf", "anyOf", "oneOf") +# Scalar constraints copied as they are. +CONSTRAINT_KEYWORDS = ("format", "pattern", "minimum", "maximum", "minLength", "maxLength") + + +def load_spec(source): + if source.startswith(("http://", "https://")): + with urllib.request.urlopen(source) as response: + return json.load(response) + with open(source, encoding="utf-8") as handle: + return json.load(handle) + + +def slim(schema): + """Reduces a schema to the keywords the client validates.""" + if not isinstance(schema, dict): + return schema + result = {} + if "$ref" in schema: + result["$ref"] = schema["$ref"] + if "type" in schema: + result["type"] = schema["type"] + for keyword in CONSTRAINT_KEYWORDS: + if keyword in schema: + result[keyword] = schema[keyword] + if "required" in schema: + result["required"] = schema["required"] + if "properties" in schema: + result["properties"] = {name: slim(value) for name, value in schema["properties"].items()} + if "items" in schema: + result["items"] = slim(schema["items"]) + for keyword in SCHEMA_LIST_KEYWORDS: + if keyword in schema: + result[keyword] = [slim(value) for value in schema[keyword]] + return result + + +def collect_refs(node, found): + if isinstance(node, dict): + ref = node.get("$ref") + if isinstance(ref, str): + if not ref.startswith(SCHEMA_REF_PREFIX): + raise SystemExit("unsupported reference: " + ref) + found.add(ref[len(SCHEMA_REF_PREFIX):]) + for value in node.values(): + collect_refs(value, found) + elif isinstance(node, list): + for value in node: + collect_refs(value, found) + + +def success_json_schema(spec, operation): + for status, response in operation.get("responses", {}).items(): + if not status.startswith("2"): + continue + if "$ref" in response: + response = spec["components"]["responses"][response["$ref"].split("/")[-1]] + content = response.get("content", {}).get("application/json") + if content and "schema" in content: + return content["schema"] + return None + + +def main(): + spec = load_spec(sys.argv[1] if len(sys.argv) > 1 else DEFAULT_SPEC_URL) + operations = {} + for path, item in spec["paths"].items(): + for method in HTTP_METHODS: + operation = item.get(method) + schema = success_json_schema(spec, operation) if operation else None + if schema is not None: + operations[method.upper() + " " + path] = slim(schema) + + schemas = {} + pending = set() + collect_refs(operations, pending) + while pending: + name = pending.pop() + if name in schemas: + continue + schemas[name] = slim(spec["components"]["schemas"][name]) + collect_refs(schemas[name], pending) + + document = {"specVersion": spec["info"]["version"], "operations": operations, "schemas": dict(sorted(schemas.items()))} + OUTPUT.write_text(json.dumps(document, separators=(",", ":"), sort_keys=True) + "\n", encoding="utf-8") + print("wrote", OUTPUT, "(%d operations, %d schemas)" % (len(operations), len(schemas))) + + +if __name__ == "__main__": + main()