diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cf4c37..8406474 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 0.6.0 + +- Bumped `ApifyClientVersion.ApiSpecVersion` to the Apify OpenAPI spec `v2-2026-10-19T133704Z` and + the project version to `0.6.0`. +- Added `ApifyClientException`, a common base class that `ApifyApiException` (and its per-status + subclasses) and `ApifyTransportException` now extend, matching the reference client's + `ApifyClientError`. A single `catch (ApifyClientException)` now catches every error this client + raises for a failed call. +- Added `ContentEncoding` to `SetRecordOptions`, for uploading a key-value-store record that is already + compressed (`gzip`, `br`, `deflate`): the client sends the bytes as-is and skips its own compression, + matching the reference client's `setRecord` `contentEncoding` option. +- Added `DatasetClient.StreamItemsAsync`, which downloads a dataset item export as a `Stream` instead + of buffering it into a `byte[]`, matching the reference client's `downloadItems` `stream` option. +- Added `LogClient.GetBytesAsync`, which fetches a log as raw bytes instead of a decoded string, + matching the reference client's `log.get` `buffer` option. +- Added `KeyValueStoreClient.IterateKeysAsync` (with a new `IterateKeysOptions`, whose `MaxPageLimit` + caps the per-page size), an auto-paging iterator over a store's keys, matching the reference client's + chunked `listKeys()` iteration. `Limit = 0` yields no keys without issuing any request (the API itself + reads a literal `limit=0` as unbounded, not zero); `Limit` must not be negative and `MaxPageLimit` must + be positive. Pages continue only while the API returns a non-null `nextExclusiveStartKey`, matching the + reference client, rather than on `isTruncated` alone. + ## 0.5.0 - Bumped `ApifyClientVersion.ApiSpecVersion` to the Apify OpenAPI spec `v2-2026-10-08T073338Z` and diff --git a/docs/README.md b/docs/README.md index fe13d85..81a0b60 100644 --- a/docs/README.md +++ b/docs/README.md @@ -73,8 +73,8 @@ the `using` directives for whichever ones a file references: | `Apify.Client` | The entry point (`ApifyClient`), `ApifyClientOptions`, `ApifyClientVersion`, and the log-redirection helper `StreamedLog`. | | `Apify.Client.Resources` | Every resource client the entry point returns — `ActorClient`, `RunClient`, `BuildClient`, `DatasetClient`, `KeyValueStoreClient`, `RequestQueueClient`, `TaskClient`, `ScheduleClient`, `LogClient`, `UserClient`, `ActorVersionClient`, `ActorEnvVarClient`, the `…CollectionClient` types (including `ActorVersionCollectionClient`, `ActorEnvVarCollectionClient`, `NestedWebhookCollectionClient`, and `WebhookDispatchCollectionClient`), etc. | | `Apify.Client.Models` | Data models returned by the clients — `Actor`, `ActorRun`, `Build`, `Dataset`, `RequestQueueRequest`, `ActorEnvVar`, `PaginationList`, and so on. | -| `Apify.Client.Options` | The option/request objects passed into methods — `ActorStartOptions`, `ActorBuildOptions`, `ActorListOptions`, `RunListOptions`, `RunResurrectOptions`, `RunChargeOptions`, `MetamorphOptions`, `LastRunOptions`, `DatasetListItemsOptions`, `DatasetDownloadOptions`, `DownloadItemsFormat`, `GetRecordOptions`, `SetRecordOptions`, `ListKeysOptions`, `ListRequestsOptions`, `PaginateRequestsOptions`, `BatchAddRequestsOptions`, `RequestQueueClientOptions`, `TaskStartOptions`, `ValidateInputOptions`, `LogOptions`, `ListOptions`, `StorageListOptions`, `StoreListOptions`. | -| `Apify.Client.Exceptions` | `ApifyApiException` and `ApifyTransportException`. | +| `Apify.Client.Options` | The option/request objects passed into methods — `ActorStartOptions`, `ActorBuildOptions`, `ActorListOptions`, `RunListOptions`, `RunResurrectOptions`, `RunChargeOptions`, `MetamorphOptions`, `LastRunOptions`, `DatasetListItemsOptions`, `DatasetDownloadOptions`, `DownloadItemsFormat`, `GetRecordOptions`, `SetRecordOptions`, `ListKeysOptions`, `IterateKeysOptions`, `ListRequestsOptions`, `PaginateRequestsOptions`, `BatchAddRequestsOptions`, `RequestQueueClientOptions`, `TaskStartOptions`, `ValidateInputOptions`, `LogOptions`, `ListOptions`, `StorageListOptions`, `StoreListOptions`. | +| `Apify.Client.Exceptions` | `ApifyClientException` (common base), `ApifyApiException` (and its per-status subclasses) and `ApifyTransportException`. | | `Apify.Client.Http` | The replaceable transport: `IHttpTransport` and the default `HttpClientTransport`. | | `System.Text.Json.Nodes` (BCL) | Not an Apify namespace, but required whenever you name the JSON escape-hatch types the client returns/accepts — `JsonObject`/`JsonNode` (e.g. dataset items, `GetInputAsync`/`UpdateInputAsync`, `GetStatisticsAsync`, `GetOpenApiDefinitionAsync`, `MonthlyUsageAsync`/`LimitsAsync`, `RequestQueueRequest.UserData`). Add `using System.Text.Json.Nodes;`. | @@ -205,6 +205,17 @@ catch (ApifyApiException e) `ApifyTransportException` is thrown instead when the request never reaches the API (network failure, timeout, or DNS error) after all retries are exhausted. +Both `ApifyApiException` and `ApifyTransportException` extend the common `ApifyClientException`, so +`catch (ApifyClientException)` catches every error this client raises for a failed call without naming +either concrete type: + +```csharp +catch (ApifyClientException e) +{ + Console.WriteLine("client call failed: " + e.Message); +} +``` + ## Versioning - `Apify.Client.ApifyClientVersion.ClientVersion` — the semantic version of this library. diff --git a/docs/misc.md b/docs/misc.md index 4d4f299..3081f9d 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -63,13 +63,15 @@ Console.WriteLine(otherUser?.Username); ## Logs — `client.Log(buildOrRunId)` - `GetAsync(LogOptions? options = null)` → `string?` (buffered; `null` if the log does not exist). +- `GetBytesAsync(LogOptions? options = null)` → `byte[]?` — the same buffered fetch as `GetAsync`, as raw + bytes instead of a decoded string (for a log that is not necessarily valid UTF-8 text). - `StreamAsync(LogOptions? options = null)` → `Stream?` (live; `null` if the log does not exist). Also `client.Run(id).GetStreamedLogAsync()`. Addressed by its own id (`client.Log(buildOrRunId)`) as above, a missing log resolves to `null`. Reached through a run or build instead (`client.Run(id).Log()`, `client.Build(id).Log()`), a 404 is ambiguous — -it could mean the run/build itself or its log is missing — so both methods throw an `ApifyApiException` -there instead. +it could mean the run/build itself or its log is missing — so `GetAsync`/`GetBytesAsync` throw an +`ApifyApiException` there instead. `LogOptions` fields: `Raw` (`bool?`, return the unprocessed log rather than the parsed form) and `Download` (`bool?`, request a download `Content-Disposition`). diff --git a/docs/storages.md b/docs/storages.md index b0b34d2..890fe30 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -31,6 +31,10 @@ reached from a run (`client.Run(id).Dataset()`, `.KeyValueStore()`, `.RequestQue item across pages, fetching each page on demand. - `DownloadItemsAsync(DownloadItemsFormat format, DatasetDownloadOptions? options = null)` → serialized items as `byte[]` (raw bytes, so binary formats like `Xlsx` are not corrupted; decode text formats yourself). +- `StreamItemsAsync(DownloadItemsFormat format, DatasetDownloadOptions? options = null)` → the same export as a + `Stream`, so a large export is processed incrementally instead of buffered into memory. Unlike + `DownloadItemsAsync`, this is a single attempt (not retried); dispose the stream when done, which also + releases the underlying connection. - `PushItemsAsync(object items)` — push one object or an array of objects. - `GetStatisticsAsync()` → `JsonObject` (throws if the dataset does not exist). - `CreateItemsPublicUrlAsync(DatasetListItemsOptions? options = null, int? expiresInSecs = null, DownloadItemsFormat? format = null)` → `string` (a signed public URL; `format` defaults to `Json`). @@ -69,6 +73,7 @@ API's export formats: ```csharp using System; +using System.IO; using System.Text; using Apify.Client; using Apify.Client.Options; @@ -80,6 +85,11 @@ var page = await client.Dataset(dataset.Id!).ListItemsAsync(new DatasetListItems Console.WriteLine(page.Count); // items in this page; page.Total is the count across all pages var csvBytes = await client.Dataset(dataset.Id!).DownloadItemsAsync(DownloadItemsFormat.Csv); Console.WriteLine(Encoding.UTF8.GetString(csvBytes)); // CSV is text; decode the raw bytes + +// Stream a large export to a file instead of buffering it all in memory. +await using var csvStream = await client.Dataset(dataset.Id!).StreamItemsAsync(DownloadItemsFormat.Csv); +await using var file = File.Create("items.csv"); +await csvStream.CopyToAsync(file); ``` ## Key-value stores @@ -87,7 +97,9 @@ Console.WriteLine(Encoding.UTF8.GetString(csvBytes)); // CSV is text; decode the `client.KeyValueStores()` / `client.KeyValueStore(id)`. - `GetAsync()` → `KeyValueStore?`; `UpdateAsync(object newFields)` → `KeyValueStore`; `DeleteAsync()`. -- `ListKeysAsync(ListKeysOptions? options = null)` → `KeyValueStoreKeysPage`. +- `ListKeysAsync(ListKeysOptions? options = null)` → `KeyValueStoreKeysPage` (one page). +- `IterateKeysAsync(IterateKeysOptions? options = null)` → `IAsyncEnumerable` — lazily + iterate every key across pages, fetching each page (sized by `MaxPageLimit`) on demand. - `RecordExistsAsync(string key)` → `bool`. - `GetRecordAsync(string key, GetRecordOptions? options = null)` → `KeyValueStoreRecord?`. `KeyValueStoreRecord.Value` is a `byte[]` of the record's raw bytes (so binary records survive intact); @@ -101,8 +113,11 @@ Console.WriteLine(Encoding.UTF8.GetString(csvBytes)); // CSV is text; decode the `ListKeysOptions` fields: `Limit` (page size), `ExclusiveStartKey` (start after this key), `Prefix` (only keys with this prefix), `Collection` (a named record collection), and `Signature` -(for signed public URLs). `GetRecordOptions` fields: `Attachment` (request a download disposition) -and `Signature`. `SetRecordOptions` fields: `TimeoutSecs` and `DoNotRetryTimeouts`. +(for signed public URLs). `IterateKeysOptions` fields: `Limit` (total across all pages), +`MaxPageLimit` (page size; client-side only, never sent to the API — defaults to 1000), +`ExclusiveStartKey`, `Collection`, and `Prefix`. `GetRecordOptions` fields: `Attachment` (request a +download disposition) and `Signature`. `SetRecordOptions` fields: `TimeoutSecs`, `DoNotRetryTimeouts`, +and `ContentEncoding` (upload a value that is already compressed — see below). ```csharp using System; @@ -128,6 +143,40 @@ if (record is not null) await client.KeyValueStore(store.Id!).SetRecordAsync("blob", new byte[] { 0x00, 0xFF }, "application/octet-stream"); ``` +### Uploading a pre-compressed record + +`SetRecordOptions.ContentEncoding` uploads a value you already compressed yourself (`gzip`, `br`, +`deflate`, or `identity`), sent verbatim as the `Content-Encoding` header: the client sends the bytes +as-is and skips its own compression, so they are never compressed twice. + +```csharp +using System.IO; +using System.IO.Compression; +using Apify.Client; +using Apify.Client.Options; + +var client = new ApifyClient("my-api-token"); +var store = await client.KeyValueStores().GetOrCreateAsync("my-store"); + +byte[] Gzip(byte[] data) +{ + using var output = new MemoryStream(); + using (var gzip = new GZipStream(output, CompressionMode.Compress)) + { + gzip.Write(data, 0, data.Length); + } + + return output.ToArray(); +} + +var gzippedReport = Gzip(System.Text.Encoding.UTF8.GetBytes("{\"rows\":1000000}")); +await client.KeyValueStore(store.Id!).SetRecordAsync( + "report.json", + gzippedReport, + "application/json", + new SetRecordOptions { ContentEncoding = "gzip" }); +``` + ## Request queues `client.RequestQueues()` / `client.RequestQueue(id, RequestQueueClientOptions? options = null)`. The diff --git a/src/Apify.Client/Apify.Client.csproj b/src/Apify.Client/Apify.Client.csproj index 157148d..2c789f3 100644 --- a/src/Apify.Client/Apify.Client.csproj +++ b/src/Apify.Client/Apify.Client.csproj @@ -6,7 +6,7 @@ Apify.Client - 0.5.0 + 0.6.0 Apify Apify Apify API client for .NET diff --git a/src/Apify.Client/ApifyClientVersion.cs b/src/Apify.Client/ApifyClientVersion.cs index dd0bf82..4fea34c 100644 --- a/src/Apify.Client/ApifyClientVersion.cs +++ b/src/Apify.Client/ApifyClientVersion.cs @@ -14,11 +14,11 @@ public static class ApifyClientVersion /// The semantic version of this client library (see https://semver.org/). Changes to the public /// interface other than additive ones are considered breaking changes. /// - public const string ClientVersion = "0.5.0"; + public const string ClientVersion = "0.6.0"; /// /// The version of the Apify OpenAPI specification this client was generated and verified against. /// Corresponds to the info.version field of the Apify OpenAPI document. /// - public const string ApiSpecVersion = "v2-2026-10-08T073338Z"; + public const string ApiSpecVersion = "v2-2026-10-19T133704Z"; } diff --git a/src/Apify.Client/Exceptions/ApifyApiException.cs b/src/Apify.Client/Exceptions/ApifyApiException.cs index d4dd36c..5a0628d 100644 --- a/src/Apify.Client/Exceptions/ApifyApiException.cs +++ b/src/Apify.Client/Exceptions/ApifyApiException.cs @@ -13,7 +13,7 @@ namespace Apify.Client.Exceptions; /// , the human-readable , the HTTP , /// the number of the final , and the request /. /// -public class ApifyApiException : Exception +public class ApifyApiException : ApifyClientException { /// Creates an API exception with the parsed error details. /// The HTTP status code of the error response. diff --git a/src/Apify.Client/Exceptions/ApifyClientException.cs b/src/Apify.Client/Exceptions/ApifyClientException.cs new file mode 100644 index 0000000..8a12787 --- /dev/null +++ b/src/Apify.Client/Exceptions/ApifyClientException.cs @@ -0,0 +1,24 @@ +using System; + +namespace Apify.Client.Exceptions; + +/// +/// Base class of every error this client throws for a failed API call: +/// (and its per-status subclasses) and . A single +/// catch (ApifyClientException) catches all of them, matching the reference client's +/// ApifyClientError. +/// +/// +/// An invalid argument throws a standard / +/// before any request is sent; neither of those extends this class. +/// +public abstract class ApifyClientException : Exception +{ + /// Creates a client exception with the given message. + /// A description of the failure. + /// The underlying exception, if any. + protected ApifyClientException(string message, Exception? innerException = null) + : base(message, innerException) + { + } +} diff --git a/src/Apify.Client/Exceptions/ApifyTransportException.cs b/src/Apify.Client/Exceptions/ApifyTransportException.cs index 8190e57..c2ab392 100644 --- a/src/Apify.Client/Exceptions/ApifyTransportException.cs +++ b/src/Apify.Client/Exceptions/ApifyTransportException.cs @@ -9,7 +9,7 @@ namespace Apify.Client.Exceptions; /// A custom should throw this for connection, DNS and /// timeout failures; a non-2xx HTTP status must be returned as a normal response instead. /// -public sealed class ApifyTransportException : Exception +public sealed class ApifyTransportException : ApifyClientException { /// Creates a transport exception. /// A description of the failure. diff --git a/src/Apify.Client/Internal/HttpClientCore.cs b/src/Apify.Client/Internal/HttpClientCore.cs index 6767fe9..ce7c851 100644 --- a/src/Apify.Client/Internal/HttpClientCore.cs +++ b/src/Apify.Client/Internal/HttpClientCore.cs @@ -135,6 +135,7 @@ public async Task CallAsync( byte[]? bodyBytes = null, IReadOnlyDictionary? extraHeaders = null, RequestTier tier = RequestTier.Long, + string? explicitContentEncoding = null, CancellationToken cancellationToken = default) { var delayMillis = _retry.MinDelayMillis; @@ -148,8 +149,12 @@ public async Task CallAsync( var baseTimeout = timeout ?? TimeSpan.FromSeconds(_retry.TierSecs(tier)); var overallCap = TimeSpan.FromSeconds(_retry.TimeoutSecs); // Normalize (and, when large enough, compress) the body once up front so retries reuse the same - // prepared payload instead of re-encoding and re-compressing on every attempt. - var prepared = PrepareBody(body, bodyBytes, contentType); + // prepared payload instead of re-encoding and re-compressing on every attempt. A caller-supplied + // explicitContentEncoding (e.g. SetRecordAsync's contentEncoding for an already-compressed value) + // skips this: the bytes are already in their final, encoded form and must reach the API unchanged. + var prepared = explicitContentEncoding is not null + ? new PreparedBody(bodyBytes ?? (body is not null ? Encoding.UTF8.GetBytes(body) : null), contentType, explicitContentEncoding) + : PrepareBody(body, bodyBytes, contentType); Exception? lastError = null; for (var attempt = 1; attempt <= maxAttempts; attempt++) diff --git a/src/Apify.Client/Internal/ResourceContext.cs b/src/Apify.Client/Internal/ResourceContext.cs index 61f99d3..d5398fe 100644 --- a/src/Apify.Client/Internal/ResourceContext.cs +++ b/src/Apify.Client/Internal/ResourceContext.cs @@ -373,6 +373,36 @@ public async Task GetRawRequiredAsync(string subPath, QueryParams p, Req return await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); } + /// + /// GET returning the raw response body as bytes (no data envelope, no UTF-8 decoding — useful when the + /// body is not necessarily text). Returns null on not-found. + /// + public async Task GetRawBytesAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); + try + { + using var response = await Http.CallAsync(HttpMethod.Get, url, timeout: _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); + return await response.Content.ReadAsByteArrayAsync(ct).ConfigureAwait(false); + } + catch (ApifyApiException e) when (HttpClientCore.IsNotFound(e)) + { + return null; + } + } + + /// + /// GET returning the raw response body as bytes; propagates a 404 instead of swallowing it. The byte + /// counterpart of (see its remarks on when a missing parent should + /// surface as an error rather than an ambiguous empty result). + /// + public async Task GetRawBytesRequiredAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) + { + var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); + using var response = await Http.CallAsync(HttpMethod.Get, url, timeout: _requestTimeout, tier: tier, cancellationToken: ct).ConfigureAwait(false); + return await response.Content.ReadAsByteArrayAsync(ct).ConfigureAwait(false); + } + /// HEAD request; returns whether the resource exists. public async Task HeadExistsAsync(string subPath, QueryParams p, RequestTier tier, CancellationToken ct) { @@ -388,11 +418,15 @@ public async Task HeadExistsAsync(string subPath, QueryParams p, RequestTi } } - /// PUT with raw bytes and a content type, with an explicit per-request timeout and retry control. - public async Task PutRawAsync(string subPath, QueryParams p, byte[] body, string contentType, TimeSpan? timeout, bool doNotRetryTimeouts, RequestTier tier, CancellationToken ct) + /// + /// PUT with raw bytes and a content type, with an explicit per-request timeout and retry control. + /// , when given, is sent verbatim as the Content-Encoding + /// header and skips the client's own compression — used for a value that is already compressed. + /// + public async Task PutRawAsync(string subPath, QueryParams p, byte[] body, string contentType, TimeSpan? timeout, bool doNotRetryTimeouts, RequestTier tier, CancellationToken ct, string? contentEncoding = null) { var url = MergedParams(p).ApplyToUrl(SubUrl(subPath)); - using var response = await Http.CallAsync(HttpMethod.Put, url, null, contentType, timeout ?? _requestTimeout, doNotRetryTimeouts, bodyBytes: body, tier: tier, cancellationToken: ct).ConfigureAwait(false); + using var response = await Http.CallAsync(HttpMethod.Put, url, null, contentType, timeout ?? _requestTimeout, doNotRetryTimeouts, bodyBytes: body, tier: tier, explicitContentEncoding: contentEncoding, cancellationToken: ct).ConfigureAwait(false); } // ---- Wait-for-finish ------------------------------------------------------ diff --git a/src/Apify.Client/Options/IterateKeysOptions.cs b/src/Apify.Client/Options/IterateKeysOptions.cs new file mode 100644 index 0000000..e560b5f --- /dev/null +++ b/src/Apify.Client/Options/IterateKeysOptions.cs @@ -0,0 +1,30 @@ +namespace Apify.Client.Options; + +/// +/// Configures lazy iteration over a key-value store's keys +/// (), mirroring the reference +/// client's chunked listKeys() iteration (limit, chunkSize, ...). +/// +public sealed class IterateKeysOptions +{ + /// Default maximum number of keys fetched per page (matches the API's own default/maximum). + public const int DefaultMaxPageLimit = 1000; + + /// Maximum total number of keys to iterate across all pages (null for no bound). + public int? Limit { get; init; } + + /// + /// Maximum number of keys fetched per page (defaults to ). Never sent + /// to the API as-is; it only sizes each page request, matching the reference client's chunkSize. + /// + public int? MaxPageLimit { get; init; } + + /// Start iterating after this key (first page only). + public string? ExclusiveStartKey { get; init; } + + /// Filter keys by collection name. + public string? Collection { get; init; } + + /// Only iterate keys with this prefix. + public string? Prefix { get; init; } +} diff --git a/src/Apify.Client/Options/SetRecordOptions.cs b/src/Apify.Client/Options/SetRecordOptions.cs index 55892bc..41acf18 100644 --- a/src/Apify.Client/Options/SetRecordOptions.cs +++ b/src/Apify.Client/Options/SetRecordOptions.cs @@ -15,4 +15,12 @@ public sealed class SetRecordOptions /// If true, do not retry the upload when it fails with a request timeout. public bool DoNotRetryTimeouts { get; init; } + + /// + /// Encoding already applied to the record value, sent verbatim as the Content-Encoding header. + /// Set it to upload a pre-compressed value (gzip, br, deflate, or identity): + /// the client then sends the bytes as they are and skips its own compression, so they are never + /// compressed twice. Must not be empty or whitespace-only. + /// + public string? ContentEncoding { get; init; } } diff --git a/src/Apify.Client/Resources/DatasetClient.cs b/src/Apify.Client/Resources/DatasetClient.cs index e1b18b0..48c6b19 100644 --- a/src/Apify.Client/Resources/DatasetClient.cs +++ b/src/Apify.Client/Resources/DatasetClient.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.IO; using System.Net.Http; using System.Runtime.CompilerServices; using System.Threading; @@ -182,6 +183,39 @@ public async Task DownloadItemsAsync(DownloadItemsFormat format, Dataset return await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); } + /// + /// Downloads dataset items serialized in the given format as a stream, so a large export is processed + /// incrementally without holding it all in memory. Takes the same arguments as + /// ; only how the response body is consumed differs. + /// + /// + /// Unlike , this bypasses the buffered/retrying transport so the body + /// can be read incrementally as it arrives (a single attempt, not retried, matching + /// ). The caller must dispose the returned stream; disposing it also + /// releases the underlying HTTP response/connection. + /// + /// The output format. + /// Optional format-specific and filtering options. + /// A token to cancel the request. + public async Task StreamItemsAsync(DownloadItemsFormat format, DatasetDownloadOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + q.AddString("format", format.ToWireValue()); + (options ?? new DatasetDownloadOptions()).AppendTo(q); + var url = _ctx.MergedParams(q).ApplyToUrl(_ctx.SubUrl("items")); + + var response = await _http.StreamAsync(url, cancellationToken).ConfigureAwait(false); + var status = (int)response.StatusCode; + if (status >= HttpClientCore.MaxSuccessStatus) + { + var errorBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + response.Dispose(); + throw HttpClientCore.BuildApiError(status, errorBody, 1, "GET", HttpClientCore.ExtractPath(url)); + } + + return await ResponseOwningStream.CreateAsync(response, cancellationToken).ConfigureAwait(false); + } + /// Pushes one or more items to the dataset. /// Must serialize to a JSON object or an array of objects. /// A token to cancel the request. diff --git a/src/Apify.Client/Resources/KeyValueStoreClient.cs b/src/Apify.Client/Resources/KeyValueStoreClient.cs index 81f2cec..03afe09 100644 --- a/src/Apify.Client/Resources/KeyValueStoreClient.cs +++ b/src/Apify.Client/Resources/KeyValueStoreClient.cs @@ -1,5 +1,7 @@ using System; +using System.Collections.Generic; using System.Net.Http; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Apify.Client.Exceptions; @@ -63,6 +65,94 @@ public async Task ListKeysAsync(ListKeysOptions? options return KeyValueStoreKeysPage.FromData(await _ctx.GetResourceRequiredAsync("keys", q, RequestTier.Medium, cancellationToken).ConfigureAwait(false)); } + /// + /// Lazily iterates over the store's keys across pages, fetching each page on demand. Mirrors the + /// reference client's chunked listKeys() iteration. + /// + /// + /// With no options it fetches pages of up to keys + /// until the store is exhausted. Limit caps the total number of keys yielded across all pages — + /// 0 yields none, without issuing any request (the API itself reads a literal limit=0 as + /// unbounded, not zero, so this is checked before building the request rather than left to the API), and + /// must not be negative. MaxPageLimit caps the page size (never sent to the API itself) and must + /// be positive, and ExclusiveStartKey chooses the starting point (first page only). + /// + /// Optional iteration options. + /// A token to cancel the iteration. + /// + /// is zero or negative, or + /// is negative. + /// + public async IAsyncEnumerable IterateKeysAsync( + IterateKeysOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + options ??= new IterateKeysOptions(); + if (options.MaxPageLimit is <= 0) + { + throw new ArgumentException("IterateKeysOptions.MaxPageLimit must be positive.", nameof(options)); + } + + if (options.Limit is < 0) + { + throw new ArgumentException("IterateKeysOptions.Limit must not be negative.", nameof(options)); + } + + var maxPageLimit = options.MaxPageLimit ?? IterateKeysOptions.DefaultMaxPageLimit; + var limit = options.Limit; // total across all pages; null = unbounded + var nextExclusiveStartKey = options.ExclusiveStartKey; + var iterated = 0; + + while (true) + { + // Checked before building the request, not only after: the API reads a literal `limit=0` as + // unset (unbounded) rather than "zero", so a Limit of 0 (or one already exhausted by a prior + // page) must stop here instead of reaching the wire as 0 and fetching a full page regardless. + if (limit is not null && iterated >= limit.Value) + { + yield break; + } + + var pageLimit = limit is not null ? Math.Min(maxPageLimit, limit.Value - iterated) : maxPageLimit; + + var page = await ListKeysAsync( + new ListKeysOptions + { + Limit = pageLimit, + ExclusiveStartKey = nextExclusiveStartKey, + Collection = options.Collection, + Prefix = options.Prefix, + }, + cancellationToken).ConfigureAwait(false); + + if (page.Items.Count == 0) + { + yield break; + } + + foreach (var item in page.Items) + { + yield return item; + } + + iterated += page.Items.Count; + + // Continue only while there is a next cursor to follow — mirrors the reference client + // (apify-client-js `key_value_store.ts` listKeys iteration), which continues on + // `nextExclusiveStartKey != null` rather than on `isTruncated` alone. The API answers with + // `null` on the last page and omits the field entirely for a listing that was not truncated, + // but `isTruncated` alone is not a safe continuation signal: if it were ever `true` alongside a + // null/absent cursor, looping on it would re-fetch the same first page forever (no + // `exclusiveStartKey` reaches the wire for a null cursor), yielding duplicate keys. + if (page.NextExclusiveStartKey is null || (limit is not null && iterated >= limit.Value)) + { + yield break; + } + + nextExclusiveStartKey = page.NextExclusiveStartKey; + } + } + /// Reports whether a record with the given key exists. /// The record key. /// A token to cancel the request. @@ -98,15 +188,21 @@ public Task RecordExistsAsync(string key, CancellationToken cancellationTo /// /// Stores a record with raw bytes and the given content type, honoring the given write options - /// (TimeoutSecs, DoNotRetryTimeouts). + /// (TimeoutSecs, DoNotRetryTimeouts, ContentEncoding). /// /// The record key. - /// The raw record bytes. + /// + /// The raw record bytes. With set, these must already be + /// encoded that way (e.g. already gzipped) — the client sends them as-is and does not compress them + /// again. + /// /// /// The record's MIME type. Worth setting precisely for media and archives: the client skips /// compressing a body whose content type already carries its own compression (images, audio, video, /// archives, office/ZIP formats, web fonts). application/octet-stream — the usual choice for /// data of an unknown or mixed binary type — is treated as compressible, since it could hold anything. + /// This content-type-based skip does not apply when is + /// set: that always skips the client's own compression, regardless of content type. /// /// Optional write options. /// A token to cancel the request. @@ -114,6 +210,16 @@ public Task SetRecordAsync(string key, byte[] value, string contentType, SetReco { options ??= new SetRecordOptions(); var timeout = options.TimeoutSecs is not null ? TimeSpan.FromSeconds(options.TimeoutSecs.Value) : (TimeSpan?)null; + string? contentEncoding = null; + if (options.ContentEncoding is not null) + { + contentEncoding = options.ContentEncoding.Trim(); + if (contentEncoding.Length == 0) + { + throw new ArgumentException("ContentEncoding must not be empty or whitespace-only.", nameof(options)); + } + } + return _ctx.PutRawAsync( "records/" + ResourceContext.EncodePathSegment(key), new QueryParams(), @@ -122,7 +228,8 @@ public Task SetRecordAsync(string key, byte[] value, string contentType, SetReco timeout, options.DoNotRetryTimeouts, RequestTier.Long, - cancellationToken); + cancellationToken, + contentEncoding); } /// Stores a record holding the JSON serialization of . diff --git a/src/Apify.Client/Resources/LogClient.cs b/src/Apify.Client/Resources/LogClient.cs index b5890e7..35ffceb 100644 --- a/src/Apify.Client/Resources/LogClient.cs +++ b/src/Apify.Client/Resources/LogClient.cs @@ -58,6 +58,26 @@ internal static LogClient Nested(HttpClientCore http, string baseUrl, QueryParam private async Task GetRequiredAsync(QueryParams q, CancellationToken cancellationToken) => await _ctx.GetRawRequiredAsync("", q, RequestTier.Long, cancellationToken).ConfigureAwait(false); + /// + /// Fetches the log as raw bytes instead of a decoded string — the counterpart of + /// for a log that is not (or might not be) valid UTF-8 text. It does not change how the log is fetched, + /// only how the body is returned; the whole log is downloaded in one response either way. To read it + /// incrementally, use . + /// + /// Optional log-content options. + /// A token to cancel the request. + public Task GetBytesAsync(LogOptions? options = null, CancellationToken cancellationToken = default) + { + var q = new QueryParams(); + (options ?? new LogOptions()).AppendTo(q); + return _ambiguousNotFound + ? GetRequiredBytesAsync(q, cancellationToken) + : _ctx.GetRawBytesAsync("", q, RequestTier.Long, cancellationToken); + } + + private async Task GetRequiredBytesAsync(QueryParams q, CancellationToken cancellationToken) + => await _ctx.GetRawBytesRequiredAsync("", q, RequestTier.Long, cancellationToken).ConfigureAwait(false); + /// /// Opens a live, streaming connection to the log and returns a stream over the log bytes. For a log /// addressed by its own id, a 404 resolves to null (matching ); for a log diff --git a/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs index 8a04fa2..24a889b 100644 --- a/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/ActorRunIntegrationTests.cs @@ -30,6 +30,10 @@ public async Task RunActorAndReadOutputs() Assert.NotNull(log); Assert.NotEqual(string.Empty, log); + var logBytes = await client.Run(run.Id!).Log().GetBytesAsync(); + Assert.NotNull(logBytes); + Assert.Equal(log, System.Text.Encoding.UTF8.GetString(logBytes!)); + await client.Run(run.Id!).Dataset().ListItemsAsync(new DatasetListItemsOptions()); await client.Run(run.Id!).KeyValueStore().GetRecordAsync("OUTPUT"); } diff --git a/tests/Apify.Client.Tests/Integration/DatasetIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/DatasetIntegrationTests.cs index e73473c..c64dba5 100644 --- a/tests/Apify.Client.Tests/Integration/DatasetIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/DatasetIntegrationTests.cs @@ -1,3 +1,4 @@ +using System.IO; using System.Threading.Tasks; using Apify.Client.Options; using Xunit; @@ -60,6 +61,16 @@ await dataset.PushItemsAsync(new object[] Assert.NotEmpty(csvBytes); Assert.Contains("url", System.Text.Encoding.UTF8.GetString(csvBytes), System.StringComparison.Ordinal); + // Streaming the same export must yield the same bytes as the buffered download. Compared as raw + // bytes (not decoded text) so a leading UTF-8 BOM — which StreamReader would silently strip — + // is still caught by a mismatch. + using (var stream = await dataset.StreamItemsAsync(DownloadItemsFormat.Csv, new DatasetDownloadOptions { Bom = true })) + using (var copy = new MemoryStream()) + { + await stream.CopyToAsync(copy); + Assert.Equal(csvBytes, copy.ToArray()); + } + // XLSX is a binary (ZIP-based) export; verify the raw bytes are returned uncorrupted by checking // the ZIP local-file-header magic (PK\x03\x04). A string-based download would mangle these bytes. var xlsxBytes = await dataset.DownloadItemsAsync(DownloadItemsFormat.Xlsx); diff --git a/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs index 7e30556..d44015f 100644 --- a/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/IterationIntegrationTests.cs @@ -16,11 +16,12 @@ namespace Apify.Client.Tests.Integration; /// /// /// This suite is about iterating a collection of resources (e.g. "every dataset"), not a resource's own -/// contents: dataset item iteration (DatasetClient.IterateItemsAsync) is covered by a unit test -/// (AutoPagingTests) against a mocked transport, since it needs to exercise multi-page paging logic -/// deterministically; request-queue request iteration has its own integration test -/// (RequestQueueIntegrationTests.RequestQueuePaginateMultiplePages). Key-value stores expose no item -/// iterator (only ListKeysAsync), so there is nothing to cover there. +/// contents: dataset item iteration (DatasetClient.IterateItemsAsync) and key-value-store key +/// iteration (KeyValueStoreClient.IterateKeysAsync) are covered by unit tests (AutoPagingTests) +/// against a mocked transport, since they need to exercise multi-page paging logic deterministically; +/// below covers the same iterator against the real API. +/// Request-queue request iteration has its own integration test +/// (RequestQueueIntegrationTests.RequestQueuePaginateMultiplePages). /// /// Where creation is cheap, each test below creates a couple of uniquely-named resources and asserts that /// iterating the collection (newest-first) eventually surfaces every one of them — exercising the real @@ -272,6 +273,39 @@ public async Task IterateActorVersionsAndEnvVars() } } + [SkippableFact] + public async Task IterateKeyValueStoreKeysMultiplePages() + { + var client = RequireClient(); + var store = await client.KeyValueStores().GetOrCreateAsync(UniqueName("it-kvs-keys")); + try + { + var kvs = client.KeyValueStore(store.Id!); + const int total = 5; + for (var i = 0; i < total; i++) + { + await kvs.SetRecordJsonAsync("it-key-" + i, new { n = i }); + } + + // MaxPageLimit forces several pages (5 keys, 2 per page) so the exclusiveStartKey-driven paging + // is actually exercised against the real API, not satisfied by a single response. + var seen = new HashSet(); + await foreach (var key in kvs.IterateKeysAsync(new IterateKeysOptions { MaxPageLimit = 2 })) + { + seen.Add(key.Key!); + } + + for (var i = 0; i < total; i++) + { + Assert.Contains("it-key-" + i, seen); + } + } + finally + { + await client.KeyValueStore(store.Id!).DeleteAsync(); + } + } + [SkippableFact] public async Task IterateBuildsBounded() { diff --git a/tests/Apify.Client.Tests/Integration/KeyValueStoreIntegrationTests.cs b/tests/Apify.Client.Tests/Integration/KeyValueStoreIntegrationTests.cs index b86c0b3..89d302f 100644 --- a/tests/Apify.Client.Tests/Integration/KeyValueStoreIntegrationTests.cs +++ b/tests/Apify.Client.Tests/Integration/KeyValueStoreIntegrationTests.cs @@ -1,4 +1,5 @@ using System; +using System.IO.Compression; using System.Net.Http; using System.Threading.Tasks; using Apify.Client.Options; @@ -107,6 +108,42 @@ public async Task BinaryRecordRoundTripPreservesBytes() } } + [SkippableFact] + public async Task PreCompressedRecordRoundTrips() + { + var client = RequireClient(); + var store = await client.KeyValueStores().GetOrCreateAsync(UniqueName("kvs-gzip")); + try + { + var kvs = client.KeyValueStore(store.Id!); + var plaintext = System.Text.Encoding.UTF8.GetBytes("{\"hello\":\"pre-compressed world\"}"); + byte[] gzipped; + using (var output = new System.IO.MemoryStream()) + { + using (var gzip = new GZipStream(output, CompressionMode.Compress)) + { + gzip.Write(plaintext, 0, plaintext.Length); + } + + gzipped = output.ToArray(); + } + + await kvs.SetRecordAsync("pre-compressed", gzipped, "application/json", new SetRecordOptions { ContentEncoding = "gzip" }); + + // The default transport auto-decompresses the response body per its own Content-Encoding, so + // the record reads back as the original plaintext regardless of how it was uploaded. + var record = await kvs.GetRecordAsync("pre-compressed"); + Assert.NotNull(record); + Assert.Equal(plaintext, record!.Value); + + await kvs.DeleteRecordAsync("pre-compressed"); + } + finally + { + await client.KeyValueStore(store.Id!).DeleteAsync(); + } + } + [SkippableFact] public async Task RecordPublicUrlIsFetchable() { diff --git a/tests/Apify.Client.Tests/Unit/ApifyClientExceptionTests.cs b/tests/Apify.Client.Tests/Unit/ApifyClientExceptionTests.cs new file mode 100644 index 0000000..9b7413f --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/ApifyClientExceptionTests.cs @@ -0,0 +1,79 @@ +using System; +using System.Threading.Tasks; +using Apify.Client.Exceptions; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +/// +/// Verifies the common base lets every error this client raises for a +/// failed API call be caught with a single catch (ApifyClientException), matching the reference +/// client's ApifyClientError. +/// +[Trait("Category", "Unit")] +public sealed class ApifyClientExceptionTests +{ + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + [Fact] + public async Task ApifyApiExceptionIsAnApifyClientException() + { + var transport = new MockTransport().QueueResponse(400, "{\"error\":{\"type\":\"bad-input\",\"message\":\"invalid\"}}"); + + var ex = await Assert.ThrowsAsync(() => Client(transport).Actors().CreateAsync(new { name = "x" })); + Assert.IsAssignableFrom(ex); + Assert.IsAssignableFrom(ex); + } + + [Fact] + public async Task ApifyTransportExceptionIsAnApifyClientException() + { + var options = new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + MaxRetries = 0, + HttpTransport = new MockTransport().QueueError(), + }; + + var ex = await Assert.ThrowsAsync(() => new ApifyClient(options).Actor("x").GetAsync()); + Assert.IsAssignableFrom(ex); + } + + [Fact] + public async Task CatchingApifyClientExceptionCatchesBothFamilies() + { + // A single `catch (ApifyClientException)` must catch an API error... + var apiTransport = new MockTransport().QueueResponse(404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"gone\"}}"); + await AssertThrowsApifyClientExceptionAsync(() => Client(apiTransport).Actor("missing").UpdateAsync(new { title = "x" })); + + // ...and a transport failure, without naming either concrete subclass. + var transportOptions = new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + MaxRetries = 0, + HttpTransport = new MockTransport().QueueError(), + }; + await AssertThrowsApifyClientExceptionAsync(() => new ApifyClient(transportOptions).Actor("x").GetAsync()); + } + + /// + /// Asserts throws something assignable to , + /// the way application code catching the common base would — without xUnit's exact-type + /// ThrowsAsync<T>, which would reject the concrete subclasses this test exercises. + /// + private static async Task AssertThrowsApifyClientExceptionAsync(Func action) + { + var ex = await Record.ExceptionAsync(action); + Assert.IsAssignableFrom(ex); + } +} diff --git a/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs index 0176199..a7d0187 100644 --- a/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs +++ b/tests/Apify.Client.Tests/Unit/AutoPagingTests.cs @@ -203,4 +203,111 @@ public async Task DatasetIterateItemsAdvancesByScannedCountNotReturnedItemsWhenU Assert.Contains("offset=0", transport.Received[0].Uri, StringComparison.Ordinal); Assert.Contains("offset=1", transport.Received[1].Uri, StringComparison.Ordinal); } + + [Fact] + public async Task KeyValueStoreIterateKeysWalksAllPagesByExclusiveStartKey() + { + var transport = new MockTransport() + .QueueResponse(200, "{\"data\":{\"items\":[{\"key\":\"a\"},{\"key\":\"b\"}],\"limit\":2,\"isTruncated\":true,\"nextExclusiveStartKey\":\"b\"}}") + .QueueResponse(200, "{\"data\":{\"items\":[{\"key\":\"c\"}],\"limit\":2,\"isTruncated\":false}}"); + + var keys = new List(); + await foreach (var key in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { MaxPageLimit = 2 })) + { + keys.Add(key.Key); + } + + Assert.Equal(new[] { "a", "b", "c" }, keys); + Assert.Equal(2, transport.CallCount); + Assert.Contains("limit=2", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.DoesNotContain("exclusiveStartKey=", transport.Received[0].Uri, StringComparison.Ordinal); + Assert.Contains("exclusiveStartKey=b", transport.Received[1].Uri, StringComparison.Ordinal); + // chunkSize/maxPageLimit is client-side only and must never reach the API query string. + Assert.DoesNotContain("maxPageLimit", transport.Received[1].Uri, StringComparison.Ordinal); + Assert.DoesNotContain("chunkSize", transport.Received[1].Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task KeyValueStoreIterateKeysStopsAtLimit() + { + var transport = new MockTransport() + .QueueResponse(200, "{\"data\":{\"items\":[{\"key\":\"a\"},{\"key\":\"b\"}],\"limit\":2,\"isTruncated\":true,\"nextExclusiveStartKey\":\"b\"}}"); + + var keys = new List(); + await foreach (var key in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { Limit = 2 })) + { + keys.Add(key.Key); + } + + // Limit=2 is satisfied by the first page, so no second page is fetched even though isTruncated=true. + Assert.Equal(new[] { "a", "b" }, keys); + Assert.Equal(1, transport.CallCount); + } + + [Fact] + public async Task KeyValueStoreIterateKeysWithZeroLimitYieldsNoneAndMakesNoRequest() + { + // Regression test: the API reads a literal limit=0 as unbounded, not zero, so Limit=0 must stop + // before ever building a request rather than send 0 and yield whatever the API returns anyway. + var transport = new MockTransport(); + + var keys = new List(); + await foreach (var key in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { Limit = 0 })) + { + keys.Add(key.Key); + } + + Assert.Empty(keys); + Assert.Equal(0, transport.CallCount); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public async Task KeyValueStoreIterateKeysRejectsNonPositiveMaxPageLimit(int maxPageLimit) + { + var transport = new MockTransport(); + + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { MaxPageLimit = maxPageLimit })) + { + } + }); + Assert.Equal(0, transport.CallCount); + } + + [Fact] + public async Task KeyValueStoreIterateKeysRejectsNegativeLimit() + { + var transport = new MockTransport(); + + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { Limit = -1 })) + { + } + }); + Assert.Equal(0, transport.CallCount); + } + + [Fact] + public async Task KeyValueStoreIterateKeysStopsOnTruncatedPageWithNullCursor() + { + // Regression test: the continuation decision must follow `nextExclusiveStartKey`, not + // `isTruncated` alone (mirrors apify-client-js `key_value_store.ts` listKeys iteration). A page + // reporting `isTruncated=true` with a null/absent cursor must still stop iteration instead of + // looping back to the same first page forever and yielding duplicate keys. + var transport = new MockTransport() + .QueueResponse(200, "{\"data\":{\"items\":[{\"key\":\"a\"},{\"key\":\"b\"}],\"limit\":2,\"isTruncated\":true}}"); + + var keys = new List(); + await foreach (var key in Client(transport).KeyValueStore("s1").IterateKeysAsync(new IterateKeysOptions { MaxPageLimit = 2 })) + { + keys.Add(key.Key); + } + + Assert.Equal(new[] { "a", "b" }, keys); + Assert.Equal(1, transport.CallCount); + } } diff --git a/tests/Apify.Client.Tests/Unit/DatasetClientTests.cs b/tests/Apify.Client.Tests/Unit/DatasetClientTests.cs new file mode 100644 index 0000000..b3e66a6 --- /dev/null +++ b/tests/Apify.Client.Tests/Unit/DatasetClientTests.cs @@ -0,0 +1,46 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Apify.Client.Exceptions; +using Apify.Client.Options; +using Xunit; + +namespace Apify.Client.Tests.Unit; + +[Trait("Category", "Unit")] +public sealed class DatasetClientTests +{ + private static ApifyClient Client(MockTransport transport) => new(new ApifyClientOptions + { + Token = "t", + MinDelayBetweenRetriesMillis = 1, + TimeoutSecs = 5, + HttpTransport = transport, + }); + + [Fact] + public async Task StreamItemsUsesFormatQueryAndReturnsReadableStream() + { + var transport = new MockTransport().QueueResponse(200, "a,b\n1,2\n"); + using var stream = await Client(transport).Dataset("ds1").StreamItemsAsync(DownloadItemsFormat.Csv, new DatasetDownloadOptions { Bom = true }); + + var uri = transport.LastRequest.Uri; + Assert.Contains("/datasets/ds1/items", uri, StringComparison.Ordinal); + Assert.Contains("format=csv", uri, StringComparison.Ordinal); + Assert.Contains("bom=1", uri, StringComparison.Ordinal); + + using var reader = new StreamReader(stream); + Assert.Equal("a,b\n1,2\n", await reader.ReadToEndAsync()); + } + + [Fact] + public async Task StreamItemsThrowsApifyApiExceptionOnError() + { + var transport = new MockTransport().QueueResponse(404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"no dataset\"}}"); + var ex = await Assert.ThrowsAsync( + () => Client(transport).Dataset("missing").StreamItemsAsync(DownloadItemsFormat.Json)); + + // Still catchable through the common base, matching every other client-raised error. + Assert.IsAssignableFrom(ex); + } +} diff --git a/tests/Apify.Client.Tests/Unit/LogClientTests.cs b/tests/Apify.Client.Tests/Unit/LogClientTests.cs index 5ec0655..ec1a6b1 100644 --- a/tests/Apify.Client.Tests/Unit/LogClientTests.cs +++ b/tests/Apify.Client.Tests/Unit/LogClientTests.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Threading.Tasks; +using Apify.Client.Exceptions; using Apify.Client.Options; using Xunit; @@ -35,6 +36,33 @@ public async Task MissingLogReturnsNull() Assert.Null(await Client(transport).Log("missing").GetAsync()); } + [Fact] + public async Task GetBytesByIdReturnsRawBytes() + { + var transport = new MockTransport().QueueResponse(200, "line1\nline2\n"); + var bytes = await Client(transport).Log("run1").GetBytesAsync(); + + Assert.NotNull(bytes); + Assert.Equal("line1\nline2\n", System.Text.Encoding.UTF8.GetString(bytes!)); + Assert.Equal("GET", transport.LastRequest.Method); + Assert.Contains("/logs/run1", transport.LastRequest.Uri, StringComparison.Ordinal); + } + + [Fact] + public async Task MissingLogGetBytesReturnsNull() + { + var transport = new MockTransport().QueueResponse(404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"no log\"}}"); + Assert.Null(await Client(transport).Log("missing").GetBytesAsync()); + } + + [Fact] + public async Task RunNestedLogGetBytesThrowsOnAmbiguousNotFound() + { + var transport = new MockTransport().QueueResponse(404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"no run\"}}"); + await Assert.ThrowsAsync( + () => Client(transport).Run("run1").Log().GetBytesAsync()); + } + [Fact] public async Task RunNestedLogGet() { diff --git a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs index 21c8f55..c1eb7c2 100644 --- a/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs +++ b/tests/Apify.Client.Tests/Unit/RequestShapeTests.cs @@ -1,5 +1,8 @@ using System; using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Text; using System.Threading.Tasks; using System.Text.Json.Nodes; using Apify.Client.Models; @@ -687,4 +690,62 @@ public async Task BatchDeleteRequestsSendsIdentifiersAndDecodesResult() Assert.Equal("r1", Assert.Single(result.ProcessedRequests).Id); Assert.Equal("k2", Assert.Single(result.UnprocessedRequests).UniqueKey); } + + [Fact] + public async Task SetRecordWithContentEncodingSendsItVerbatimAndSkipsClientCompression() + { + // A value already gzip-compressed by the caller, well above the auto-compression threshold: if the + // client applied its own (brotli) compression on top, the bytes on the wire would no longer be + // valid single-pass gzip and the header would read "br", not "gzip". + var plaintext = Encoding.UTF8.GetBytes(new string('x', 2048)); + var preCompressed = Gzip(plaintext); + + var transport = new MockTransport().QueueResponse(200, string.Empty); + await Client(transport).KeyValueStore("s1").SetRecordAsync( + "report.json", + preCompressed, + "application/json", + new SetRecordOptions { ContentEncoding = "gzip" }); + + var request = transport.LastRequest; + Assert.Equal("gzip", request.Header("Content-Encoding")); + // Sent as-is (not re-compressed): the raw bytes on the wire are exactly the pre-compressed input. + Assert.Equal(preCompressed, request.BodyBytes); + } + + [Fact] + public async Task SetRecordWithContentEncodingIsTrimmed() + { + var transport = new MockTransport().QueueResponse(200, string.Empty); + await Client(transport).KeyValueStore("s1").SetRecordAsync( + "k", + new byte[] { 1, 2, 3 }, + "application/octet-stream", + new SetRecordOptions { ContentEncoding = " identity " }); + + Assert.Equal("identity", transport.LastRequest.Header("Content-Encoding")); + } + + [Fact] + public async Task SetRecordWithWhitespaceOnlyContentEncodingThrowsWithoutSendingRequest() + { + var transport = new MockTransport(); + await Assert.ThrowsAsync(() => Client(transport).KeyValueStore("s1").SetRecordAsync( + "k", + new byte[] { 1, 2, 3 }, + "application/octet-stream", + new SetRecordOptions { ContentEncoding = " " })); + Assert.Equal(0, transport.CallCount); + } + + private static byte[] Gzip(byte[] data) + { + using var output = new MemoryStream(); + using (var gzip = new GZipStream(output, CompressionMode.Compress)) + { + gzip.Write(data, 0, data.Length); + } + + return output.ToArray(); + } }