diff --git a/CHANGELOG.md b/CHANGELOG.md index df2869c..f45ae8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,49 @@ All notable changes to the Apify Java client are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.5.0] - 2026-07-23 + +### Changed + +- **Breaking:** migrated from Jackson 2 to Jackson 3. Dependency `groupId` changes from + `com.fasterxml.jackson.core` to `tools.jackson.core` for `jackson-databind`; + `jackson-datatype-jsr310` is no longer a dependency (Java-time (de)serialization is now built into + `jackson-databind`). Any public type coming from `com.fasterxml.jackson.databind` (e.g. `JsonNode` + on `DatasetClient`/`TaskClient`/`ActorClient`/...) now comes from `tools.jackson.databind` instead; + `com.fasterxml.jackson.annotation` (the annotation types) is unchanged. +- **Breaking:** every network-calling method on `ApifyClient` and its resource clients is now + asynchronous and non-blocking, returning a `java.util.concurrent.CompletableFuture` instead of + blocking the calling thread and returning its result directly (e.g. `Optional get()` is now + `CompletableFuture> get()`; a `void`-returning call is now `CompletableFuture`). +- **Breaking:** every paginated `iterate(...)`/`paginateRequests(...)` method now returns a + `java.util.concurrent.Flow.Publisher` instead of a blocking `java.util.Iterator`, fetching pages + only as the subscriber signals demand (`Flow.Subscription#request(long)`). +- **Breaking:** `HttpTransport.send`/`sendStreamingResponse` replaced with non-blocking + `sendAsync`/`sendStreamingAsync`, both returning `CompletableFuture>`; + `DefaultHttpTransport` is now backed by the JDK `HttpClient`'s own `sendAsync`. +- **Breaking:** `HttpTimeoutException` no longer extends `java.io.IOException`; it is now a plain + unchecked `RuntimeException`, matching the transport contract no longer declaring any checked + exception. +- Retry backoff and the `waitForFinish`/`batchAddRequests` polling delays are now scheduled timers + instead of `Thread.sleep`, so no thread is blocked while a call is being retried or polled. +- Bumped `Version.API_SPEC_VERSION` to `v2-2026-07-22T122437Z`. + +### Added + +- `com.apify.client.Publishers.collect(Flow.Publisher)`: a convenience that subscribes with + unbounded demand and collects a publisher's items into a `CompletableFuture>`, for callers + that do not need backpressure. + +### Fixed + +- `Webhook.getRequestUrl()`/`WebhookDispatchWebhookInfo.getRequestUrl()` javadoc now documents that + the field is `null` for a hook action other than the conventional HTTP case (e.g. a Slack or email + notification), matching the OpenAPI specification. +- A JSON `null` for a primitive numeric model field (e.g. `ActorRunStats` fields before a run's + stats are fully populated) no longer fails deserialization; Jackson 3 defaults this stricter than + Jackson 2 did, so the mapper now explicitly restores the previous, lenient (`null` -> zero) + behavior. + ## [0.4.0] - 2026-07-20 ### Changed diff --git a/README.md b/README.md index c733167..92fc960 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Maven (Maven Central is a default repository, so no extra configuration is neede com.apify apify-client - 0.4.0 + 0.5.0 ``` @@ -35,7 +35,7 @@ repositories { } dependencies { - implementation 'com.apify:apify-client:0.4.0' + implementation 'com.apify:apify-client:0.5.0' } ``` @@ -52,7 +52,7 @@ class HelloApify { public static void main(String[] args) { // Your API token from https://console.apify.com/settings/integrations ApifyClient client = ApifyClient.create(System.getenv("APIFY_TOKEN")); - ActorRun run = client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L); + ActorRun run = client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L).join(); System.out.println("Run " + run.getId() + " finished with status " + run.getStatus()); } } @@ -69,13 +69,14 @@ enumerates the model/option-type packages. The Quick start example above is a co program (imports, class, `main`); every other snippet in this file, from here on, is a fragment that assumes a configured `client` and the correct imports for the types it uses — not, by itself, a complete program — the [resource pages](docs/README.md) show the same kind of fragment per -method. Reading items from a run's default dataset: +method. Reading items from a run's default dataset, chaining the two asynchronous calls: ```java -ActorRun run = client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L); -PaginationList items = - client.dataset(run.getDefaultDatasetId()).listItems(new DatasetListItemsOptions()); -System.out.println("Items in this page: " + items.getCount()); +client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L) + .thenCompose(run -> client.dataset(run.getDefaultDatasetId()) + .listItems(new DatasetListItemsOptions())) + .thenAccept(items -> System.out.println("Items in this page: " + items.getCount())) + .join(); ``` The types used above — `PaginationList` (root package), `DatasetListItemsOptions` @@ -86,6 +87,41 @@ in the same style (build-and-run, storages, log redirection, and more); the comp programs live under [`src/test/java/com/apify/client/examples/`](https://github.com/apify/apify-client-java/tree/master/src/test/java/com/apify/client/examples). +## Asynchronous API + +Every network-calling method returns a +[`CompletableFuture`](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/concurrent/CompletableFuture.html) +(`java.util.concurrent.CompletableFuture`) instead of blocking the calling thread on the HTTP +exchange, and every paginated collection's `iterate(...)` returns a +[`Flow.Publisher`](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/concurrent/Flow.Publisher.html) +(`java.util.concurrent.Flow.Publisher`) — the JDK's built-in Reactive Streams type — instead of a +blocking `Iterator`, so following a large collection never blocks a thread waiting on the next +page's network round-trip. + +- **Chain calls** with `thenApply`/`thenCompose`/`thenAccept` (see the dataset example above) to + keep a pipeline fully asynchronous end to end. +- **Block for a result** with `future.join()` (unchecked) or `future.get()` (checked, + `InterruptedException`/`ExecutionException`) when you just want a synchronous-looking call site + (e.g. a `main` method, a test, or a simple script) — every snippet in this documentation that + needs a value calls `.join()` for exactly that reason. +- **Consume an `iterate(...)` publisher** either by implementing `Flow.Subscriber` yourself + (`onSubscribe` → `request(n)` to pull `n` items, `onNext` per item, `onComplete`/`onError` to + finish) for real backpressure, or with the small convenience bridge + `com.apify.client.Publishers.collect(publisher)`, which subscribes with unbounded demand and + returns a `CompletableFuture>` — useful when you just want "give me everything" and don't + need backpressure: + + ```java + List actors = Publishers.collect(client.actors().iterate(new ActorListOptions())).join(); + ``` + +A future/publisher's failure mode is unchanged from before: any `ApifyApiException`/ +`ApifyTransportException` this client throws now arrives as the future's exceptional completion +(surfaced by `.join()`/`.get()` as a `CompletionException`/`ExecutionException` wrapping the +original, still-unchecked exception) instead of a direct `throw` — see +[Error handling](#error-handling) below for the full exception hierarchy, which is otherwise +unchanged. + ## Configuration Use `ApifyClient.builder()` for non-default settings: @@ -160,38 +196,48 @@ exponential backoff and jitter on `429`, `5xx` and network errors. #### `HttpTransport` contract -A custom implementation (`com.apify.client.http.HttpTransport`) must provide both methods. -`HttpRequest` and `HttpResponse` below are the JDK's `java.net.http.HttpRequest`/ +A custom implementation (`com.apify.client.http.HttpTransport`) must provide both methods, both +non-blocking. `HttpRequest` and `HttpResponse` below are the JDK's `java.net.http.HttpRequest`/ `java.net.http.HttpResponse` (not custom client types) — same as `Duration` above: | Method | Signature | Contract | |---|---|---| -| `send` | `HttpResponse send(HttpRequest request) throws IOException, InterruptedException` | Sends the single, fully-prepared request (auth, `User-Agent` and any other headers are already set by the client) and returns the response with its body fully buffered as `byte[]`. Used for every non-streaming call. | -| `sendStreamingResponse` | `HttpResponse sendStreamingResponse(HttpRequest request) throws IOException, InterruptedException` | Sends the request and returns the response body as a live `InputStream` for incremental consumption, used by log streaming (`LogClient.stream(...)`). The *caller* (the client) is responsible for closing the returned stream; an implementation must not close it itself before returning. | +| `sendAsync` | `CompletableFuture> sendAsync(HttpRequest request)` | Sends the single, fully-prepared request (auth, `User-Agent` and any other headers are already set by the client) and completes the future with the response with its body fully buffered as `byte[]`. Used for every non-streaming call. | +| `sendStreamingAsync` | `CompletableFuture> sendStreamingAsync(HttpRequest request)` | Sends the request and completes the future with the response body as a live `InputStream` for incremental consumption, used by log streaming (`LogClient.stream(...)`). The *caller* (the client) is responsible for closing the returned stream; an implementation must not close it itself before completing the future. | Both methods perform exactly one network round-trip each — no retrying, no request mutation -(retries, auth, and the `User-Agent` header are handled one layer up, by the client itself). A -non-2xx HTTP status is **not** a transport-level error: return it as a normal `HttpResponse` with -its actual status code. Only throw for a genuine transport failure — connection refused, DNS -resolution failure, or a timeout. Any thrown `IOException` (or `InterruptedException`, with the -interrupt flag restored) is translated by the client into an `ApifyTransportException`; throw +(retries, auth, and the `User-Agent` header are handled one layer up, by the client itself), and +neither may block the calling thread waiting on the network. A non-2xx HTTP status is **not** a +transport-level error: complete the future normally with it, as a normal `HttpResponse` carrying its +actual status code. Only fail the future for a genuine transport failure — connection refused, DNS +resolution failure, or a timeout. Any exception the future is completed exceptionally with is +translated by the client into an `ApifyTransportException`; complete exceptionally with `com.apify.client.http.HttpTimeoutException` specifically for a timeout so -`ApifyTransportException.isTimeout()` reports it correctly. +`ApifyTransportException.isTimeout()` reports it correctly. `DefaultHttpTransport` implements both +methods directly on top of the JDK `HttpClient`'s own `sendAsync`. ## Fetching single resources -Methods that fetch a single resource return an `Optional`: a missing resource is reported by an -empty `Optional` rather than an exception. +Methods that fetch a single resource complete with an `Optional`: a missing resource is reported +by an empty `Optional` rather than an exception. ```java -client.actor("apify/hello-world").get().ifPresent(actor -> System.out.println(actor.getTitle())); +client.actor("apify/hello-world").get() + .thenAccept(actor -> actor.ifPresent(a -> System.out.println(a.getTitle()))) + .join(); ``` ## Error handling Every exception this client throws for a request/transport failure is an unchecked `com.apify.client.http.ApifyClientException`. It has two concrete subtypes, both also in -`com.apify.client.http`: +`com.apify.client.http`. Since every network-calling method is asynchronous (see +[Asynchronous API](#asynchronous-api) above), these exceptions arrive as the returned +`CompletableFuture`'s exceptional completion rather than a direct `throw`; `.join()` re-throws the +original exception wrapped in an unchecked `CompletionException` (`.get()` wraps it in a checked +`ExecutionException` instead) — unwrap `getCause()` to recover the original +`ApifyApiException`/`ApifyTransportException`, or use `.handle(...)`/`.exceptionally(...)` to react +to it without unwrapping at all: - `ApifyApiException` — the request reached the API, which answered with a non-success status. - `ApifyTransportException` — the request never produced an API response at all (connection @@ -206,22 +252,27 @@ Every exception this client throws for a request/transport failure is an uncheck Catch `ApifyClientException` to handle both failure modes uniformly, or catch a specific subtype to handle one of them differently. `ApifyApiException` is imported from `com.apify.client.http`: -A few methods validate their own preconditions before making any request and throw a plain JDK -exception instead, not an `ApifyClientException` subtype: `ApifyClient.setStatusMessage(message, -options)` throws `IllegalStateException` if the `ACTOR_RUN_ID` environment variable is not set, -`ApifyClient.me()`'s limits methods (`limits()`, `updateLimits(...)`, `monthlyUsage(...)`) throw the -same `IllegalStateException` if called on a `UserClient` that is not `me()`, and -`ApifyClientBuilder`'s setters throw `IllegalArgumentException` on an invalid configuration value -(e.g. a negative retry count). These are local, no-request-sent failures, not something the API -responded with or the transport failed to deliver, which is why they stay outside the -request/transport exception hierarchy above — catch `IllegalStateException`/`IllegalArgumentException` -separately if you call any of them. +A few methods validate their own preconditions synchronously, before ever building a request or +future, and throw a plain JDK exception directly from the call site instead, not an +`ApifyClientException` subtype (so no `.join()`/`.get()` unwrapping is involved for these): +`ApifyClient.setStatusMessage(message, options)` throws `IllegalStateException` if the +`ACTOR_RUN_ID` environment variable is not set, `ApifyClient.me()`'s limits methods (`limits()`, +`updateLimits(...)`, `monthlyUsage(...)`) throw the same `IllegalStateException` if called on a +`UserClient` that is not `me()`, and `ApifyClientBuilder`'s setters throw `IllegalArgumentException` +on an invalid configuration value (e.g. a negative retry count). These are local, no-request-sent +failures, not something the API responded with or the transport failed to deliver, which is why +they stay outside the request/transport exception hierarchy above — catch +`IllegalStateException`/`IllegalArgumentException` separately if you call any of them. ```java try { - client.actor("does/not-exist").update(Map.of("title", "x")); -} catch (ApifyApiException e) { - System.out.println("status=" + e.getStatusCode() + " type=" + e.getType()); + client.actor("does/not-exist").update(Map.of("title", "x")).join(); +} catch (CompletionException e) { + if (e.getCause() instanceof ApifyApiException apiError) { + System.out.println("status=" + apiError.getStatusCode() + " type=" + apiError.getType()); + } else { + throw e; + } } ``` @@ -242,10 +293,10 @@ try { The public `com.apify.client.Version` class (`import com.apify.client.Version;`) exposes two constants: -- `Version.CLIENT_VERSION` — the semantic version of this client (`0.4.0`). +- `Version.CLIENT_VERSION` — the semantic version of this client (`0.5.0`). - `Version.API_SPEC_VERSION` — the version of the [Apify OpenAPI specification](https://docs.apify.com/api/openapi.json) (its `info.version` field) that this client's endpoints, parameters and models were last generated - and checked against (`v2-2026-07-20T094852Z`). It is a snapshot, not a live compatibility + and checked against (`v2-2026-07-22T122437Z`). It is a snapshot, not a live compatibility guarantee: the client keeps working against newer, backward-compatible spec revisions, but a feature added to the API after this snapshot has no corresponding method here yet. diff --git a/docs/README.md b/docs/README.md index 525c6a7..6a2e645 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,8 +22,11 @@ user-agent suffix, custom HTTP transport). Get your API token from the [Apify Console → Settings → API & Integrations](https://console.apify.com/settings/integrations). -Methods that fetch a single resource return an `Optional`: a missing resource is reported by an -empty `Optional` rather than an exception. API failures are thrown as `ApifyApiException` (see +Every network-calling method returns a `CompletableFuture` (`java.util.concurrent.CompletableFuture`) +rather than blocking the calling thread — see [Asynchronous API](../README.md#asynchronous-api) in +the top-level README. Methods that fetch a single resource complete with an `Optional`: a missing +resource is reported by an empty `Optional` rather than an exception. API failures surface as the +future's exceptional completion, wrapping an `ApifyApiException` (see [error handling](../README.md#error-handling)). ## Imports and dependencies @@ -53,16 +56,19 @@ option types, so import each resource you use from its own package: For example, the [top-level README's dataset snippet](../README.md#quick-start) needs `com.apify.client.ApifyClient`, `com.apify.client.PaginationList`, `com.apify.client.dataset.DatasetListItemsOptions`, `com.apify.client.run.ActorRun`, -`com.apify.client.actor.ActorStartOptions` and `com.fasterxml.jackson.databind.JsonNode` — five -different packages for one four-line snippet. +`com.apify.client.actor.ActorStartOptions` and `tools.jackson.databind.JsonNode` — five +different packages for one short snippet. Snippets in these docs also assume the standard-library types they use are imported (`java.util.List`, `java.util.ArrayList`, `java.util.Map`, `java.util.Optional`, -`java.util.Iterator`, `java.util.UUID`, `java.util.function.Consumer`, `java.time.Duration`, -`java.time.Instant`, `java.io.InputStream`). `java.time.Instant` is returned by many model getters -(e.g. `createdAt`, `modifiedAt`, `nextRunAt`, `lockExpiresAt`). - -Raw-JSON return values use Jackson's `com.fasterxml.jackson.databind.JsonNode`. Jackson is a +`java.util.UUID`, `java.util.function.Consumer`, `java.util.concurrent.CompletableFuture`, +`java.util.concurrent.Flow`, `java.util.concurrent.CountDownLatch`, `java.time.Duration`, +`java.time.Instant`, `java.io.InputStream`). +`java.time.Instant` is returned by many model getters (e.g. `createdAt`, `modifiedAt`, `nextRunAt`, +`lockExpiresAt`). + +Raw-JSON return values use Jackson's `tools.jackson.databind.JsonNode`. Jackson (3.x; note the +`tools.jackson.*` package, not the older `com.fasterxml.jackson.databind.*` from Jackson 2) is a transitive dependency of this client, so it is already on your classpath. ## Raw JSON values @@ -110,7 +116,7 @@ fields later. For example a `me()` `User`'s private account details (email, plan …) are available via `getExtra()`, since `User` models only its most commonly used fields. ```java -User me = client.me().get().orElseThrow(); +User me = client.me().get().join().orElseThrow(); Object plan = me.getExtra().get("plan"); ``` @@ -123,7 +129,7 @@ Object plan = me.getExtra().get("plan"); the message as final. ```java -client.setStatusMessage("half way there", new SetStatusMessageOptions().isStatusMessageTerminal(false)); +client.setStatusMessage("half way there", new SetStatusMessageOptions().isStatusMessageTerminal(false)).join(); ``` ## Optional option fields @@ -133,7 +139,7 @@ default". Leave a setter uncalled to omit that parameter. ```java ActorListOptions options = new ActorListOptions().my(true).limit(10L); -PaginationList page = client.actors().list(options); +PaginationList page = client.actors().list(options).join(); ``` ## Common list options — `ListOptions` @@ -149,7 +155,7 @@ Most `list` methods (builds, tasks, schedules, webhooks, Actor versions) take th | `desc(Boolean)` | `Boolean` | If `true`, return items newest-first. | ```java -PaginationList builds = client.builds().list(new ListOptions().limit(50L).desc(true)); +PaginationList builds = client.builds().list(new ListOptions().limit(50L).desc(true)).join(); ``` ## Pagination — `PaginationList` @@ -160,18 +166,28 @@ their own page/head containers instead. ## Iteration — `iterate` / `iterateItems` / `iterateKeys` -Each paginated collection also offers a lazy `Iterator` that fetches pages on demand: `iterate(...)` -on the collection clients, `DatasetClient.iterateItems(...)`, and `KeyValueStoreClient.iterateKeys(...)` -(request-queue requests use `RequestQueueClient.paginateRequests(...)`). The options' `limit` caps the -**total** number of items yielded; `null`/unset — or a non-positive value such as `0` — means no cap, -so every item is yielded. (This differs from `list(...)`, which sends `limit=0` to the server -verbatim rather than treating it as unbounded — the iteration behavior matches the reference JS -client.) The per-request page size is an optional -trailing `chunkSize` argument: the per-resource tables below show the `chunkSize` form, and each -iterator also has an overload that omits it (using the server's default page size). The page size -does not change which items a collection iterator yields; note the one exception in -[Storages](storages.md) — `iterateItems` combined with server-side item filters, where the page size -can affect the result. +Each paginated collection also offers a lazy `Flow.Publisher` (`java.util.concurrent.Flow.Publisher`, +the JDK's built-in Reactive Streams type) that fetches pages only as the subscriber signals demand: +`iterate(...)` on the collection clients, `DatasetClient.iterateItems(...)`, and +`KeyValueStoreClient.iterateKeys(...)` (request-queue requests use +`RequestQueueClient.paginateRequests(...)`). The options' `limit` caps the **total** number of items +yielded; `null`/unset — or a non-positive value such as `0` — means no cap, so every item is +yielded. (This differs from `list(...)`, which sends `limit=0` to the server verbatim rather than +treating it as unbounded — the iteration behavior matches the reference JS client.) The per-request +page size is an optional trailing `chunkSize` argument: the per-resource tables below show the +`chunkSize` form, and each publisher also has an overload that omits it (using the server's default +page size). The page size does not change which items a collection publisher yields; note the one +exception in [Storages](storages.md) — `iterateItems` combined with server-side item filters, where +the page size can affect the result. + +Consume a publisher either by implementing `Flow.Subscriber` yourself (see +[Asynchronous API](../README.md#asynchronous-api) in the top-level README for the shape), or with +the convenience bridge `com.apify.client.Publishers.collect(publisher)`, which subscribes with +unbounded demand and returns a `CompletableFuture>`: + +```java +List actors = Publishers.collect(client.actors().iterate(new ActorListOptions())).join(); +``` ## Resource pages diff --git a/docs/actors.md b/docs/actors.md index a1d6ea9..122c02d 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -7,15 +7,15 @@ where `id` is an Actor ID or `username~name` (a `/` in the id is accepted and no | Method | Description | |---|---| -| `list(ActorListOptions)` | List the account's Actors. Returns `PaginationList`. | -| `iterate(ActorListOptions, Long chunkSize)` | Lazy `Iterator` over all matches; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the page size (`null` = server default). | -| `create(Object)` | Create a new Actor from a JSON-serializable definition. Returns `Actor`. | +| `list(ActorListOptions)` | List the account's Actors. Completes with `PaginationList`. | +| `iterate(ActorListOptions, Long chunkSize)` | Lazy `Flow.Publisher` over all matches; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the page size (`null` = server default). | +| `create(Object)` | Create a new Actor from a JSON-serializable definition. Completes with `Actor`. | `ActorListOptions` adds `my(Boolean)` (only Actors owned by the current user) and `sortBy(String)` (e.g. `createdAt`, `stats.lastRunStartedAt`) on top of the standard offset/limit/desc. ```java -PaginationList mine = client.actors().list(new ActorListOptions().my(true).limit(5L)); +PaginationList mine = client.actors().list(new ActorListOptions().my(true).limit(5L)).join(); for (Actor actor : mine.getItems()) { System.out.println(actor.getName()); } @@ -36,22 +36,22 @@ Actor created = client.actors().create(Map.of( Map.of("name", "Dockerfile", "format", "TEXT", "content", "FROM apify/actor-node:20\nCOPY . ./\nCMD node main.js"), Map.of("name", "main.js", "format", "TEXT", - "content", "console.log('hi');")))))); + "content", "console.log('hi');")))))).join(); ``` ## `ActorClient` | Method | Description | |---|---| -| `get()` | Fetch the Actor. Returns `Optional`. | -| `update(Object)` | Update the Actor with the given fields. Returns `Actor`. | +| `get()` | Fetch the Actor. Completes with `Optional`. | +| `update(Object)` | Update the Actor with the given fields. Completes with `Actor`. | | `delete()` | Delete the Actor. | -| `start(Object input, ActorStartOptions)` | Start a run, returning immediately. Returns `ActorRun`. | -| `call(Object input, ActorStartOptions, Long waitSecs)` | Start a run and poll until it finishes (`null` waits indefinitely); does **not** stream the run's log. Returns `ActorRun`. | +| `start(Object input, ActorStartOptions)` | Start a run, returning immediately. Completes with `ActorRun`. | +| `call(Object input, ActorStartOptions, Long waitSecs)` | Start a run and poll until it finishes (`null` waits indefinitely); does **not** stream the run's log. Completes with `ActorRun`. | | `call(Object input, ActorCallOptions, Long waitSecs)` | As above, additionally streaming the run's log for the duration of the wait by default (matching the reference client's `call` defaulting `options.log` to `'default'`). Use `ActorCallOptions.disableLogStreaming()` to opt out, or `logOptions(StreamedLogOptions)` for a custom destination. | -| `validateInput(Object input)` / `validateInput(Object input, ValidateInputOptions)` | Validate an input against the Actor's input schema. Returns `boolean`. `ValidateInputOptions` fields (optional): `build` (`String`), `contentType` (`String`). | -| `build(String versionNumber, ActorBuildOptions)` | Build a version. Returns `Build`. | -| `defaultBuild(Long waitForFinish)` | Resolve the default build. Returns `BuildClient`. | +| `validateInput(Object input)` / `validateInput(Object input, ValidateInputOptions)` | Validate an input against the Actor's input schema. Completes with `boolean`. `ValidateInputOptions` fields (optional): `build` (`String`), `contentType` (`String`). | +| `build(String versionNumber, ActorBuildOptions)` | Build a version. Completes with `Build`. | +| `defaultBuild(Long waitForFinish)` | Resolve the default build. Completes with `BuildClient`. | | `lastRun(String status)` / `lastRun(LastRunOptions)` | A `RunClient` for the last run. | | `builds()` / `runs()` / `versions()` | Nested collection clients. | | `webhooks()` | Read-only nested webhook collection (`NestedWebhookCollectionClient`, `list` + `iterate`, no `create`). | @@ -82,12 +82,13 @@ filter. `LastRunOptions` has fluent setters `status(String)` (e.g. `SUCCEEDED`, ```java Optional last = - client.actor("apify/hello-world").lastRun(new LastRunOptions().status("SUCCEEDED").origin("API")).get(); + client.actor("apify/hello-world").lastRun(new LastRunOptions().status("SUCCEEDED").origin("API")).get().join(); ``` ```java ActorRun run = client.actor("apify/hello-world") - .call(Map.of("greeting", "hi"), new ActorStartOptions().memoryMbytes(512L), 120L); + .call(Map.of("greeting", "hi"), new ActorStartOptions().memoryMbytes(512L), 120L) + .join(); System.out.println(run.getStatus()); ``` @@ -115,22 +116,22 @@ and deletes a single version and exposes its environment variables. | Method | Description | |---|---| -| `list(ListOptions)` | List the Actor's versions. Returns `PaginationList`. The endpoint takes no query parameters; `options` (`offset`/`limit`/`desc`) is accepted only for signature consistency with every other `list(ListOptions)` and has no effect — pass `null`. | -| `iterate(ListOptions)` | Lazy `Iterator` over all versions; `limit` caps the total (`null`/unset or non-positive = all), applied client-side after the single fetch. `offset` has no effect and there is no page size to tune. | -| `create(Object version)` | Create a version. Returns `ActorVersion`. | +| `list(ListOptions)` | List the Actor's versions. Completes with `PaginationList`. The endpoint takes no query parameters; `options` (`offset`/`limit`/`desc`) is accepted only for signature consistency with every other `list(ListOptions)` and has no effect — pass `null`. | +| `iterate(ListOptions)` | Lazy `Flow.Publisher` over all versions; `limit` caps the total (`null`/unset or non-positive = all), applied client-side after the single fetch. `offset` has no effect and there is no page size to tune. | +| `create(Object version)` | Create a version. Completes with `ActorVersion`. | ### `ActorVersionClient` — `client.actor(id).version(v)` | Method | Description | |---|---| -| `get()` | Fetch the version. Returns `Optional`. | -| `update(Object)` | Update the version. Returns `ActorVersion`. | +| `get()` | Fetch the version. Completes with `Optional`. | +| `update(Object)` | Update the version. Completes with `ActorVersion`. | | `delete()` | Delete the version. | | `envVars()` | An `ActorEnvVarCollectionClient` for the version's environment variables. | | `envVar(String name)` | An `ActorEnvVarClient` for a single environment variable. | ```java -ActorVersion version = client.actor("me/my-actor").version("0.0").get().orElseThrow(); +ActorVersion version = client.actor("me/my-actor").version("0.0").get().join().orElseThrow(); System.out.println(version.getSourceType()); ``` @@ -147,20 +148,21 @@ encrypted), and matching getters `getName()`, `getValue()`, `getIsSecret()`. | Method | Description | |---|---| -| `list()` | List the version's environment variables. Returns `PaginationList`. | -| `iterate()` | `Iterator` over the variables. The env-var collection is not paginated (all variables are returned at once), so this iterates a single fetched page; provided for API consistency. | -| `create(ActorEnvVar)` | Create an environment variable. Returns `ActorEnvVar`. | +| `list()` | List the version's environment variables. Completes with `PaginationList`. | +| `iterate()` | `Flow.Publisher` over the variables. The env-var collection is not paginated (all variables are returned at once), so this iterates a single fetched page; provided for API consistency. | +| `create(ActorEnvVar)` | Create an environment variable. Completes with `ActorEnvVar`. | ### `ActorEnvVarClient` — `client.actor(id).version(v).envVar(name)` | Method | Description | |---|---| -| `get()` | Fetch the environment variable. Returns `Optional`. | -| `update(ActorEnvVar)` | Update the environment variable. Returns `ActorEnvVar`. | +| `get()` | Fetch the environment variable. Completes with `Optional`. | +| `update(ActorEnvVar)` | Update the environment variable. Completes with `ActorEnvVar`. | | `delete()` | Delete the environment variable. | ```java client.actor("me/my-actor").version("0.0").envVars() - .create(new ActorEnvVar("API_KEY", "secret").setIsSecret(true)); -Optional ev = client.actor("me/my-actor").version("0.0").envVar("API_KEY").get(); + .create(new ActorEnvVar("API_KEY", "secret").setIsSecret(true)) + .join(); +Optional ev = client.actor("me/my-actor").version("0.0").envVar("API_KEY").get().join(); ``` diff --git a/docs/builds.md b/docs/builds.md index a2c9b1e..88a7ace 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -7,19 +7,19 @@ builds) and a single build with `client.build(id)`. | Method | Description | |---|---| -| `list(ListOptions)` | List builds. Returns `PaginationList`. | -| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all builds; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | +| `list(ListOptions)` | List builds. Completes with `PaginationList`. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Flow.Publisher` over all builds; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | ## `BuildClient` | Method | Description | |---|---| -| `get()` | Fetch the build. Returns `Optional`. | -| `getWithWait(Long waitForFinishSecs)` | Fetch, optionally waiting server-side for the build to finish (clamped to the request timeout; the API caps server-side waiting at 60s). Returns `Optional`. | -| `abort()` | Abort the build. Returns `Build`. | +| `get()` | Fetch the build. Completes with `Optional`. | +| `getWithWait(Long waitForFinishSecs)` | Fetch, optionally waiting server-side for the build to finish (clamped to the request timeout; the API caps server-side waiting at 60s). Completes with `Optional`. | +| `abort()` | Abort the build. Completes with `Build`. | | `delete()` | Delete the build. | -| `waitForFinish(Long waitSecs)` | Poll until the build finishes (`null` waits indefinitely). Returns `Build`. | -| `getOpenApiDefinition()` | The build's OpenAPI definition. Returns `Optional`. | +| `waitForFinish(Long waitSecs)` | Poll until the build finishes (`null` waits indefinitely). Completes with `Build`. | +| `getOpenApiDefinition()` | The build's OpenAPI definition. Completes with `Optional`. | | `log()` | A `LogClient` for the build's log. | `Build` fields: `getId()`, `getActId()`, `getUserId()`, `getStatus()`, `getStartedAt()`, @@ -34,8 +34,8 @@ Every nested model above (`BuildMeta`, `BuildStats`, `BuildOptions`, `BuildUsage `getExtra()`, so a field not yet listed here is still reachable rather than dropped. ```java -Build build = client.actor("me/my-actor").build("0.0", new ActorBuildOptions().tag("latest")); -Build finished = client.build(build.getId()).waitForFinish(300L); +Build build = client.actor("me/my-actor").build("0.0", new ActorBuildOptions().tag("latest")).join(); +Build finished = client.build(build.getId()).waitForFinish(300L).join(); if (finished.isTerminal()) { System.out.println("built " + finished.getBuildNumber()); } @@ -56,14 +56,14 @@ before the API responds, max 60 — see `defaultBuild` below for the same wait m `ActorClient.defaultBuild(Long waitForFinish)` resolves the Actor's currently-tagged default build (the build behind the `"latest"`/`"default"` tag, i.e. what a plain `start`/`call` with no explicit -`build` would run) and returns a `BuildClient` handle for it — it does not itself return the `Build` -object. `waitForFinish` only bounds how long the *resolving GET* waits server-side for that build to +`build` would run) and completes with a `BuildClient` handle for it — it does not itself return the +`Build` object. `waitForFinish` only bounds how long the *resolving GET* waits server-side for that build to finish before responding (`null`/`0` returns immediately with whatever state the build is currently in); it does not make `defaultBuild` block until the build is done. To actually observe the finished build, call `.get()` (or `.waitForFinish(...)` for client-side polling) on the returned handle: ```java -BuildClient defaultBuild = client.actor("me/my-actor").defaultBuild(30L); -Build finished = defaultBuild.waitForFinish(300L); +BuildClient defaultBuild = client.actor("me/my-actor").defaultBuild(30L).join(); +Build finished = defaultBuild.waitForFinish(300L).join(); ``` diff --git a/docs/examples.md b/docs/examples.md index 0e1a417..625b3db 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -11,9 +11,9 @@ rendered docs as well as the repository. ## Run a store Actor and read its default dataset ```java -ActorRun run = client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L); +ActorRun run = client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L).join(); PaginationList items = - client.dataset(run.getDefaultDatasetId()).listItems(new DatasetListItemsOptions()); + client.dataset(run.getDefaultDatasetId()).listItems(new DatasetListItemsOptions()).join(); System.out.println("Items in this page: " + items.getCount()); ``` @@ -26,40 +26,43 @@ System.out.println("Items in this page: " + items.getCount()); String suffix = UUID.randomUUID().toString().substring(0, 8); // Dataset: create, push items, read them back. -Dataset dataset = client.datasets().getOrCreate("java-example-ds-" + suffix); +Dataset dataset = client.datasets().getOrCreate("java-example-ds-" + suffix).join(); try { - client.dataset(dataset.getId()).pushItems(List.of(Map.of("hello", "world"))); - PaginationList items = client.dataset(dataset.getId()).listItems(new DatasetListItemsOptions()); + client.dataset(dataset.getId()).pushItems(List.of(Map.of("hello", "world"))).join(); + PaginationList items = + client.dataset(dataset.getId()).listItems(new DatasetListItemsOptions()).join(); System.out.println("Dataset items: " + items.getItems()); } finally { - client.dataset(dataset.getId()).delete(); + client.dataset(dataset.getId()).delete().join(); } // Key-value store: create, set a record, read it back. -KeyValueStore store = client.keyValueStores().getOrCreate("java-example-kvs-" + suffix); +KeyValueStore store = client.keyValueStores().getOrCreate("java-example-kvs-" + suffix).join(); try { - client.keyValueStore(store.getId()).setRecordJson("OUTPUT", Map.of("answer", 42)); - Optional record = client.keyValueStore(store.getId()).getRecord("OUTPUT"); + client.keyValueStore(store.getId()).setRecordJson("OUTPUT", Map.of("answer", 42)).join(); + Optional record = client.keyValueStore(store.getId()).getRecord("OUTPUT").join(); record.ifPresent(r -> System.out.println("KVS record bytes: " + r.getValue().length)); } finally { - client.keyValueStore(store.getId()).delete(); + client.keyValueStore(store.getId()).delete().join(); } // Request queue: create, add a request, read the head. -RequestQueue queue = client.requestQueues().getOrCreate("java-example-rq-" + suffix); +RequestQueue queue = client.requestQueues().getOrCreate("java-example-rq-" + suffix).join(); try { - client.requestQueue(queue.getId()).addRequest(new RequestQueueRequest("https://example.com", "example"), false); - RequestQueueHead head = client.requestQueue(queue.getId()).listHead(10L); + client.requestQueue(queue.getId()) + .addRequest(new RequestQueueRequest("https://example.com", "example"), false) + .join(); + RequestQueueHead head = client.requestQueue(queue.getId()).listHead(10L).join(); System.out.println("Request queue head size: " + head.getItems().size()); } finally { - client.requestQueue(queue.getId()).delete(); + client.requestQueue(queue.getId()).delete().join(); } ``` ## Get own account details ```java -Optional user = client.me().get(); +Optional user = client.me().get().join(); user.ifPresent(u -> { System.out.println("Account ID: " + u.getId()); System.out.println("Username: " + u.getUsername()); @@ -82,55 +85,81 @@ Actor created = client.actors().create(Map.of( "sourceFiles", List.of( Map.of("name", "Dockerfile", "format", "TEXT", "content", "FROM apify/actor-node:20\nCOPY . ./\nCMD node main.js"), - Map.of("name", "main.js", "format", "TEXT", "content", "console.log('hi');")))))); + Map.of("name", "main.js", "format", "TEXT", "content", "console.log('hi');")))))).join(); try { - Build build = client.actor(created.getId()).build("0.0", new ActorBuildOptions()); - client.build(build.getId()).waitForFinish(300L); - ActorRun run = client.actor(created.getId()).call(null, new ActorStartOptions(), 120L); - Optional log = client.run(run.getId()).log().get(); + Build build = client.actor(created.getId()).build("0.0", new ActorBuildOptions()).join(); + client.build(build.getId()).waitForFinish(300L).join(); + ActorRun run = client.actor(created.getId()).call(null, new ActorStartOptions(), 120L).join(); + Optional log = client.run(run.getId()).log().get().join(); log.ifPresent(System.out::println); } finally { - client.actor(created.getId()).delete(); + client.actor(created.getId()).delete().join(); } ``` ## Start a run, wait, then fetch the Actor's last run and its storages ```java -client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L); -Optional last = client.actor("apify/hello-world").lastRun("SUCCEEDED").get(); +client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L).join(); +Optional last = client.actor("apify/hello-world").lastRun("SUCCEEDED").get().join(); if (last.isPresent()) { ActorRun run = last.get(); - client.dataset(run.getDefaultDatasetId()).listItems(new DatasetListItemsOptions()); - client.keyValueStore(run.getDefaultKeyValueStoreId()).getRecord("OUTPUT"); + client.dataset(run.getDefaultDatasetId()).listItems(new DatasetListItemsOptions()).join(); + client.keyValueStore(run.getDefaultKeyValueStoreId()).getRecord("OUTPUT").join(); } ``` ## Lazy iteration of Store Actors ```java -Iterator it = client.store().iterate(new StoreListOptions().limit(10L), 10L); -int count = 0; -while (count < 5 && it.hasNext()) { - ActorStoreListItem item = it.next(); - System.out.println((count + 1) + ". " + item.getUsername() + "/" + item.getName()); - count++; -} +// Requests one item at a time (real backpressure) and cancels once 5 have been seen, so no more +// pages are fetched than needed. See IterateStore.java in the examples for the full Flow.Subscriber. +CountDownLatch done = new CountDownLatch(1); +client.store().iterate(new StoreListOptions().limit(10L), 10L) + .subscribe(new Flow.Subscriber() { + private Flow.Subscription subscription; + private int count; + + public void onSubscribe(Flow.Subscription subscription) { + this.subscription = subscription; + subscription.request(1); + } + + public void onNext(ActorStoreListItem item) { + count++; + System.out.println(count + ". " + item.getUsername() + "/" + item.getName()); + if (count >= 5) { + subscription.cancel(); + done.countDown(); + } else { + subscription.request(1); + } + } + + public void onError(Throwable throwable) { + done.countDown(); + } + + public void onComplete() { + done.countDown(); + } + }); +done.await(); ``` ## Run an Actor with log redirection -`getStreamedLog()` returns a [`StreamedLog`](runs.md#streamed-log-redirection) that follows the +`getStreamedLog()` completes with a [`StreamedLog`](runs.md#streamed-log-redirection) that follows the run's live log and redirects each message to a destination (by default a per-run prefixed logger). It is `AutoCloseable`, so a try-with-resources block stops redirection when the run finishes. The complete program lives in [`LogRedirection.java`](https://github.com/apify/apify-client-java/blob/master/src/test/java/com/apify/client/examples/LogRedirection.java). ```java -ActorRun run = client.actor("apify/hello-world").start(null, new ActorStartOptions()); +ActorRun run = client.actor("apify/hello-world").start(null, new ActorStartOptions()).join(); RunClient runClient = client.run(run.getId()); -try (StreamedLog streamedLog = runClient.getStreamedLog()) { - streamedLog.start(); - runClient.waitForFinish(120L); +try (StreamedLog streamedLog = runClient.getStreamedLog().join()) { + streamedLog.start().join(); + runClient.waitForFinish(120L).join(); } ``` diff --git a/docs/misc.md b/docs/misc.md index a7dd063..33bf9d9 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -6,8 +6,8 @@ Browse public Actors in the Apify Store. `client.store()` returns a `StoreCollec | Method | Description | |---|---| -| `list(StoreListOptions)` | A page of Store Actors. Returns `PaginationList`. | -| `iterate(StoreListOptions, Long chunkSize)` | A lazy `Iterator` over all matches, fetching pages on demand. The options' `limit` caps the total number of Actors yielded (`null`/unset or non-positive = all); `chunkSize` is the per-request page size (`null` = server default). | +| `list(StoreListOptions)` | A page of Store Actors. Completes with `PaginationList`. | +| `iterate(StoreListOptions, Long chunkSize)` | A lazy `Flow.Publisher` over all matches, fetching pages on demand. The options' `limit` caps the total number of Actors yielded (`null`/unset or non-positive = all); `chunkSize` is the per-request page size (`null` = server default). | `StoreListOptions` fields: `offset` (number of Actors to skip), `limit` (maximum number of Actors to return), `search` (full-text search query), `sortBy` (the sort field, e.g. `popularity`, `newest`), @@ -17,14 +17,13 @@ return), `search` (full-text search query), `sortBy` (the sort field, e.g. `popu Actors that allow agentic users), `responseFormat` (the response shape: `full` or `agent`). ```java -Iterator it = - client.store().iterate(new StoreListOptions().search("crawler").limit(20L), 10L); -int shown = 0; -while (shown < 5 && it.hasNext()) { - ActorStoreListItem item = it.next(); - System.out.println(item.getUsername() + "/" + item.getName()); - shown++; -} +// Publishers.collect() drains the whole publisher; fine here since `limit` already caps it. +// To stop early without fetching further pages, subscribe with your own Flow.Subscriber instead +// (request(1) at a time, cancel() once you have enough) — see IterateStore.java in the examples. +List items = + Publishers.collect(client.store().iterate(new StoreListOptions().search("crawler").limit(20L), 10L)) + .join(); +items.forEach(item -> System.out.println(item.getUsername() + "/" + item.getName())); ``` `ActorStoreListItem` fields: `getId()`, `getName()`, `getUsername()`, `getTitle()`, @@ -38,16 +37,16 @@ still available via the inherited `getExtra()` (see | Method | Description | |---|---| -| `get()` | Fetch the user. Returns `Optional` (private details for `me()` via `getExtra()`). | -| `monthlyUsage()` / `monthlyUsage(String date)` | Account monthly usage (`me()` only). `date` (any day within the target month, formatted `YYYY-MM-DD`) reports that month; `null`/empty reports the current month. Returns `JsonNode`. | -| `limits()` | Account resource limits (`me()` only). Returns `JsonNode`. | -| `updateLimits(Object)` | Update account limits (`me()` only). No return value. | +| `get()` | Fetch the user. Completes with `Optional` (private details for `me()` via `getExtra()`). | +| `monthlyUsage()` / `monthlyUsage(String date)` | Account monthly usage (`me()` only). `date` (any day within the target month, formatted `YYYY-MM-DD`) reports that month; `null`/empty reports the current month. Completes with `JsonNode`. | +| `limits()` | Account resource limits (`me()` only). Completes with `JsonNode`. | +| `updateLimits(Object)` | Update account limits (`me()` only). Completes with no value (`CompletableFuture`). | The usage/limits methods are only available for `me()`; calling them on `user(id)` throws `IllegalStateException`. ```java -Optional me = client.me().get(); +Optional me = client.me().get().join(); me.ifPresent( u -> { System.out.println("Account: " + u.getId()); @@ -55,7 +54,7 @@ me.ifPresent( // remaining account field not modelled directly. System.out.println("Email: " + u.getEmail()); }); -JsonNode usage = client.me().monthlyUsage(); +JsonNode usage = client.me().monthlyUsage().join(); ``` `User` fields: `getId()`, `getUsername()`, `getProfile()` (`UserProfile` — `getBio()`, `getName()`, @@ -79,12 +78,12 @@ Access a build's or run's log directly, or via `client.run(id).log()` / `client. | Method | Description | |---|---| -| `get()` / `get(LogOptions)` | The whole log as text. Returns `Optional`. | -| `stream()` / `stream(LogOptions)` | A live `InputStream` over the log (for redirection). | +| `get()` / `get(LogOptions)` | The whole log as text. Completes with `Optional`. | +| `stream()` / `stream(LogOptions)` | A live `InputStream` over the log (for redirection). Completes with `InputStream`. | `LogOptions` fields: `raw(Boolean)`, `download(Boolean)`. ```java -Optional log = client.log("RUN_OR_BUILD_ID").get(); +Optional log = client.log("RUN_OR_BUILD_ID").get().join(); log.ifPresent(System.out::println); ``` diff --git a/docs/runs.md b/docs/runs.md index 4a891d5..70cfac5 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -7,8 +7,8 @@ Access the run collection with `client.runs()` (or `client.actor(id).runs()` / | Method | Description | |---|---| -| `list(ListOptions, RunListOptions)` | List runs. Returns `PaginationList`. | -| `iterate(ListOptions, RunListOptions, Long chunkSize)` | Lazy `Iterator` over all matching runs; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | +| `list(ListOptions, RunListOptions)` | List runs. Completes with `PaginationList`. | +| `iterate(ListOptions, RunListOptions, Long chunkSize)` | Lazy `Flow.Publisher` over all matching runs; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | `RunListOptions` adds `status(List)` (e.g. `SUCCEEDED`, `RUNNING`; sent comma-separated), declared by every scope's endpoint (`client.runs()`, `client.actor(id).runs()`, @@ -21,26 +21,26 @@ declares compose (e.g. a status filter combined with a start-time window on `cli ```java PaginationList runs = client.runs().list( new ListOptions().limit(10L), - new RunListOptions().status(List.of("SUCCEEDED"))); + new RunListOptions().status(List.of("SUCCEEDED"))).join(); ``` ## `RunClient` | Method | Description | |---|---| -| `get()` | Fetch the run. Returns `Optional`. | -| `getWithWait(Long waitForFinishSecs)` | Fetch, optionally waiting server-side for the run to finish (clamped to the request timeout; the API caps server-side waiting at 60s). Returns `Optional`. | -| `update(Object)` | Update the run. Returns `ActorRun`. | +| `get()` | Fetch the run. Completes with `Optional`. | +| `getWithWait(Long waitForFinishSecs)` | Fetch, optionally waiting server-side for the run to finish (clamped to the request timeout; the API caps server-side waiting at 60s). Completes with `Optional`. | +| `update(Object)` | Update the run. Completes with `ActorRun`. | | `delete()` | Delete the run. | -| `abort(Boolean gracefully)` | Abort the run (`null` = server default). Returns `ActorRun`. | -| `metamorph(String targetActorId, Object input, MetamorphOptions)` | Metamorph into another Actor. Returns `ActorRun`. | -| `reboot()` | Reboot the run. Returns `ActorRun`. | -| `resurrect(RunResurrectOptions)` | Resurrect a finished run. Returns `ActorRun`. | -| `charge(RunChargeOptions)` | Charge a pay-per-event run for a named event. No return value. | -| `waitForFinish(Long waitSecs)` | Poll until the run finishes (`null` waits indefinitely). Returns `ActorRun`. | +| `abort(Boolean gracefully)` | Abort the run (`null` = server default). Completes with `ActorRun`. | +| `metamorph(String targetActorId, Object input, MetamorphOptions)` | Metamorph into another Actor. Completes with `ActorRun`. | +| `reboot()` | Reboot the run. Completes with `ActorRun`. | +| `resurrect(RunResurrectOptions)` | Resurrect a finished run. Completes with `ActorRun`. | +| `charge(RunChargeOptions)` | Charge a pay-per-event run for a named event. Completes with no value (`CompletableFuture`). | +| `waitForFinish(Long waitSecs)` | Poll until the run finishes (`null` waits indefinitely). Completes with `ActorRun`. | | `dataset()` / `keyValueStore()` / `requestQueue()` | Clients for the run's default storages. | | `log()` | A `LogClient` for the run's log (see [Store, users & logs](misc.md#logs--clientlogid)). | -| `getStreamedLog()` | A `StreamedLog` that redirects the live log to a default per-run logger. | +| `getStreamedLog()` | Completes with a `StreamedLog` that redirects the live log to a default per-run logger. | | `getStreamedLog(StreamedLogOptions)` | As above, with a custom destination / options. | If `waitSecs` elapses before the run reaches a terminal state, `waitForFinish` returns the run's @@ -53,9 +53,10 @@ run finishes. ### Streamed log redirection -`getStreamedLog()` returns a `StreamedLog`: a helper that follows the run's live raw log in a +`getStreamedLog()` completes with a `StreamedLog`: a helper that follows the run's live raw log in a background thread and redirects each complete, timestamped message to a destination. It is -`AutoCloseable` — call `start()` to begin redirection and `stop()` (or `close()`, via +`AutoCloseable` — call `start()` (itself asynchronous; it completes once the live stream is open and +the background reader thread is running) to begin redirection and `stop()` (or `close()`, via try-with-resources) to end it. With no options, messages go to an SLF4J `Logger` at `INFO` level, prefixed with the @@ -71,9 +72,9 @@ Actor name and run id (looked up automatically). `StreamedLogOptions` customizes RunClient runClient = client.run("RUN_ID"); List collected = new ArrayList<>(); try (StreamedLog streamedLog = - runClient.getStreamedLog(new StreamedLogOptions().toLog(collected::add).fromStart(true))) { - streamedLog.start(); - runClient.waitForFinish(120L); + runClient.getStreamedLog(new StreamedLogOptions().toLog(collected::add).fromStart(true)).join()) { + streamedLog.start().join(); + runClient.waitForFinish(120L).join(); } ``` @@ -86,14 +87,14 @@ To update the status message of a run other than the current one, call `update() `RunClient` directly: ```java -client.run("RUN_ID").update(Map.of("statusMessage", "half way there", "isStatusMessageTerminal", false)); +client.run("RUN_ID").update(Map.of("statusMessage", "half way there", "isStatusMessageTerminal", false)).join(); ``` > **`lastRun()` clients are for reads.** A `RunClient` obtained via `actor(id).lastRun(...)` / > `task(id).lastRun(...)` targets the `runs/last` path and is intended for reading (`get`, > `dataset()`/`keyValueStore()`/`requestQueue()`, `log()`). The mutating methods (`abort`, `reboot`, > `resurrect`, `metamorph`, `charge`, `update`, `delete`) require a concrete run and have no -> `runs/last` endpoint — resolve the id first (`lastRun(...).get()` then `client.run(id)`). The type +> `runs/last` endpoint — resolve the id first (`lastRun(...).get().join()` then `client.run(id)`). The type > exposes them uniformly (matching the reference client's shared `RunClient`), but calling them on a > last-run client will fail server-side. @@ -145,7 +146,7 @@ nullable), `getMemoryMbytes()` (`Long`, nullable), `getDiskMbytes()` (`Long`, nu applied at most once. ```java -client.run("RUN_ID").charge(new RunChargeOptions("my-event").count(3L)); +client.run("RUN_ID").charge(new RunChargeOptions("my-event").count(3L)).join(); ``` `MetamorphOptions` uses plain values `build(String)` and `contentType(String)`. diff --git a/docs/schedules.md b/docs/schedules.md index 5e8372b..6c5f8ce 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -7,9 +7,9 @@ Schedules automatically start Actor or task runs at specified times. Access the | Method | Description | |---|---| -| `list(ListOptions)` | List schedules. Returns `PaginationList`. | -| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all schedules; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | -| `create(Object)` | Create a schedule from a JSON-serializable definition. Returns `Schedule`. | +| `list(ListOptions)` | List schedules. Completes with `PaginationList`. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Flow.Publisher` over all schedules; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | +| `create(Object)` | Create a schedule from a JSON-serializable definition. Completes with `Schedule`. | The `actions` array describes what the schedule runs (e.g. a `RUN_ACTOR` action): @@ -21,7 +21,7 @@ Schedule schedule = client.schedules().create(Map.of( "isExclusive", true, "actions", List.of(Map.of( "type", "RUN_ACTOR", - "actorId", "apify/hello-world")))); + "actorId", "apify/hello-world")))).join(); ``` ## `ScheduleClient` @@ -29,10 +29,10 @@ Schedule schedule = client.schedules().create(Map.of( | Method | Description | |---|---| | `get()` / `update(Object)` / `delete()` | CRUD. | -| `getLog()` | The schedule's invocation log. Returns `Optional`. | +| `getLog()` | The schedule's invocation log. Completes with `Optional`. | ```java -Optional s = client.schedule("SCHEDULE_ID").get(); +Optional s = client.schedule("SCHEDULE_ID").get().join(); s.ifPresent(sched -> System.out.println(sched.getCronExpression())); ``` diff --git a/docs/storages.md b/docs/storages.md index 22e6d13..6ae6511 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -27,10 +27,10 @@ any field not covered by a typed getter above is still available via the inherit | Method | Description | |---|---| -| `list(StorageListOptions)` | List datasets. Returns `PaginationList`. | -| `iterate(StorageListOptions, Long chunkSize)` | Lazy `Iterator` over all datasets; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | -| `getOrCreate(String name)` | Get or create a named dataset (empty name → unnamed). Returns `Dataset`. | -| `getOrCreate(String name, Object schema)` | As above, sending a creation-time dataset `schema` when a new dataset is created. Returns `Dataset`. | +| `list(StorageListOptions)` | List datasets. Completes with `PaginationList`. | +| `iterate(StorageListOptions, Long chunkSize)` | Lazy `Flow.Publisher` over all datasets; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | +| `getOrCreate(String name)` | Get or create a named dataset (empty name → unnamed). Completes with `Dataset`. | +| `getOrCreate(String name, Object schema)` | As above, sending a creation-time dataset `schema` when a new dataset is created. Completes with `Dataset`. | `StorageListOptions` adds `unnamed(Boolean)` and `ownership(String)` on top of offset/limit/desc. @@ -39,14 +39,14 @@ any field not covered by a typed getter above is still available via the inherit | Method | Description | |---|---| | `get()` / `update(Object)` / `delete()` | Metadata CRUD. | -| `listItems(DatasetListItemsOptions)` | List items as `PaginationList`. | -| `listItems(DatasetListItemsOptions, Class)` | List items decoded into `T`. Returns `PaginationList`. | -| `iterateItems(DatasetListItemsOptions)` / `iterateItems(DatasetListItemsOptions, Long chunkSize)` | Lazy `Iterator` over all items; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), the optional `chunkSize` sets the per-request page size (omitted/`null` = server default). | -| `iterateItems(DatasetListItemsOptions, Long chunkSize, Class)` | As above, decoded into `T`. Returns `Iterator`. For typed iteration at the server-default page size, pass a `null` chunk size: `iterateItems(opts, null, T.class)`. | -| `downloadItems(DownloadItemsFormat, DatasetDownloadOptions)` | Serialized bytes. `DownloadItemsFormat` is one of `JSON`, `JSONL`, `CSV`, `XLSX`, `XML`, `RSS`, `HTML`. | -| `pushItems(Object)` | Push a single item or a list of items. No return value. | -| `getStatistics()` | Dataset statistics. Returns `Optional`. | -| `createItemsPublicUrl(DatasetListItemsOptions, Long expiresInSecs)` | A public (optionally signed) items URL. | +| `listItems(DatasetListItemsOptions)` | List items. Completes with `PaginationList`. | +| `listItems(DatasetListItemsOptions, Class)` | List items decoded into `T`. Completes with `PaginationList`. | +| `iterateItems(DatasetListItemsOptions)` / `iterateItems(DatasetListItemsOptions, Long chunkSize)` | A lazy `Flow.Publisher` over all items; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), the optional `chunkSize` sets the per-request page size (omitted/`null` = server default). | +| `iterateItems(DatasetListItemsOptions, Long chunkSize, Class)` | As above, decoded into `T`. Returns `Flow.Publisher`. For typed iteration at the server-default page size, pass a `null` chunk size: `iterateItems(opts, null, T.class)`. | +| `downloadItems(DownloadItemsFormat, DatasetDownloadOptions)` | Serialized bytes. `DownloadItemsFormat` is one of `JSON`, `JSONL`, `CSV`, `XLSX`, `XML`, `RSS`, `HTML`. Completes with `byte[]`. | +| `pushItems(Object)` | Push a single item or a list of items. Completes with no value (`CompletableFuture`). | +| `getStatistics()` | Dataset statistics. Completes with `Optional`. | +| `createItemsPublicUrl(DatasetListItemsOptions, Long expiresInSecs)` | A public (optionally signed) items URL. Completes with `String`. | > **Server-side item filters and iteration.** The dataset-items endpoint applies `offset`/`limit` to > the raw items and then drops those removed by a server-side filter (`skipEmpty`, `skipHidden`, @@ -59,10 +59,10 @@ any field not covered by a typed getter above is still available via the inherit > iterating, or fetch pages explicitly with `listItems` and filter client-side. ```java -Dataset ds = client.datasets().getOrCreate("my-dataset"); -client.dataset(ds.getId()).pushItems(List.of(Map.of("url", "https://a.com"))); -PaginationList page = client.dataset(ds.getId()).listItems(new DatasetListItemsOptions().limit(100L)); -byte[] csv = client.dataset(ds.getId()).downloadItems(DownloadItemsFormat.CSV, new DatasetDownloadOptions().bom(true)); +Dataset ds = client.datasets().getOrCreate("my-dataset").join(); +client.dataset(ds.getId()).pushItems(List.of(Map.of("url", "https://a.com"))).join(); +PaginationList page = client.dataset(ds.getId()).listItems(new DatasetListItemsOptions().limit(100L)).join(); +byte[] csv = client.dataset(ds.getId()).downloadItems(DownloadItemsFormat.CSV, new DatasetDownloadOptions().bom(true)).join(); ``` `DatasetListItemsOptions` fields: `offset` (`Long`, number of items to skip), `limit` (`Long`, @@ -83,10 +83,10 @@ adds `attachment` (`Boolean`), `bom` (`Boolean`), `delimiter` (`String`), `skipH (`Boolean`), `xmlRoot` (`String`), `xmlRow` (`String`), `feedTitle` (`String`), `feedDescription` (`String`). -`createItemsPublicUrl(DatasetListItemsOptions, Long expiresInSecs)` returns a `String` URL. If the -dataset is private, the client fetches it, reads its URL-signing secret, and appends an HMAC-SHA256 -signature (bounded by `expiresInSecs`, or non-expiring when `null`); for public datasets the URL is -unsigned. +`createItemsPublicUrl(DatasetListItemsOptions, Long expiresInSecs)` completes with a `String` URL. +If the dataset is private, the client fetches it, reads its URL-signing secret, and appends an +HMAC-SHA256 signature (bounded by `expiresInSecs`, or non-expiring when `null`); for public datasets +the URL is unsigned. > **Public-URL limitation (matches the JavaScript reference client).** The public-URL builders > (`createItemsPublicUrl`, and the key-value-store `getRecordPublicUrl` / `createKeysPublicUrl`) @@ -109,16 +109,16 @@ datasets. | Method | Description | |---|---| | `get()` / `update(Object)` / `delete()` | Metadata CRUD. | -| `listKeys(ListKeysOptions)` | List keys. Returns `KeyValueStoreKeysPage`. | -| `iterateKeys(ListKeysOptions)` / `iterateKeys(ListKeysOptions, Long chunkSize)` | Lazy `Iterator` over all keys, paging with the cursor (`exclusiveStartKey`). Note: here the options' `limit` caps the **total** number of keys yielded (`null`/unset or non-positive = all), whereas for `listKeys`/`createKeysPublicUrl` the same `ListKeysOptions.limit` is a single-request page size. `chunkSize` sets the per-request page size (`null` = server default). | -| `recordExists(String key)` | Whether a record exists. | -| `getRecord(String key)` / `getRecord(String key, GetRecordOptions)` | Fetch a record. Returns `Optional`. | -| `setRecord(String key, byte[] value, String contentType)` | Store raw bytes. No return value. | -| `setRecord(String key, byte[] value, String contentType, SetRecordOptions)` | Store raw bytes with write options (`SetRecordOptions`: `timeoutSecs` (`Long`), `doNotRetryTimeouts` (`Boolean`)). No return value. | -| `setRecordJson(String key, Object value)` | Store JSON. No return value. | -| `deleteRecord(String key)` | Delete a record. No return value. | -| `getRecordPublicUrl(String key)` | A public (optionally signed) record URL. | -| `createKeysPublicUrl(Long expiresInSecs)` | A public (optionally signed) key-list URL. | +| `listKeys(ListKeysOptions)` | List keys. Completes with `KeyValueStoreKeysPage`. | +| `iterateKeys(ListKeysOptions)` / `iterateKeys(ListKeysOptions, Long chunkSize)` | A lazy `Flow.Publisher` over all keys, paging with the cursor (`exclusiveStartKey`). Note: here the options' `limit` caps the **total** number of keys yielded (`null`/unset or non-positive = all), whereas for `listKeys`/`createKeysPublicUrl` the same `ListKeysOptions.limit` is a single-request page size. `chunkSize` sets the per-request page size (`null` = server default). | +| `recordExists(String key)` | Whether a record exists. Completes with `boolean`. | +| `getRecord(String key)` / `getRecord(String key, GetRecordOptions)` | Fetch a record. Completes with `Optional`. | +| `setRecord(String key, byte[] value, String contentType)` | Store raw bytes. Completes with no value (`CompletableFuture`). | +| `setRecord(String key, byte[] value, String contentType, SetRecordOptions)` | Store raw bytes with write options (`SetRecordOptions`: `timeoutSecs` (`Long`), `doNotRetryTimeouts` (`Boolean`)). Completes with no value. | +| `setRecordJson(String key, Object value)` | Store JSON. Completes with no value. | +| `deleteRecord(String key)` | Delete a record. Completes with no value. | +| `getRecordPublicUrl(String key)` | A public (optionally signed) record URL. Completes with `String`. | +| `createKeysPublicUrl(Long expiresInSecs)` | A public (optionally signed) key-list URL. Completes with `String`. | | `createKeysPublicUrl(ListKeysOptions, Long expiresInSecs)` | As above, forwarding key-listing filters (`limit`, `prefix`, `collection`, `exclusiveStartKey`). | `ListKeysOptions` fields (all optional): `limit(Long)`, `exclusiveStartKey(String)`, @@ -127,9 +127,9 @@ keys), `signature(String)` (a pre-shared URL signature granting access without a by `createKeysPublicUrl`). ```java -KeyValueStore store = client.keyValueStores().getOrCreate("my-store"); -client.keyValueStore(store.getId()).setRecordJson("OUTPUT", Map.of("answer", 42)); -Optional rec = client.keyValueStore(store.getId()).getRecord("OUTPUT"); +KeyValueStore store = client.keyValueStores().getOrCreate("my-store").join(); +client.keyValueStore(store.getId()).setRecordJson("OUTPUT", Map.of("answer", 42)).join(); +Optional rec = client.keyValueStore(store.getId()).getRecord("OUTPUT").join(); rec.ifPresent(r -> System.out.println(new String(r.getValue()))); ``` @@ -140,7 +140,7 @@ rec.ifPresent(r -> System.out.println(new String(r.getValue()))); `KeyValueStoreKeysPage` exposes `getItems()` (a list of `KeyValueStoreKey` with `getKey()`/`getSize()`), `isTruncated()`, `getExclusiveStartKey()` and `getNextExclusiveStartKey()`. -Both `getRecordPublicUrl` and `createKeysPublicUrl` return a `String` URL, signed for private stores +Both `getRecordPublicUrl` and `createKeysPublicUrl` complete with a `String` URL, signed for private stores and unsigned for public ones. They differ because they sign different things: a single record URL signs the record key directly (there is no expiry to bound), while a key-list URL uses an expiry-aware storage-content signature — hence only `createKeysPublicUrl` takes an `expiresInSecs` @@ -160,20 +160,20 @@ overload here — the request-queue creation endpoint does not accept a creation |---|---| | `get()` / `update(Object)` / `delete()` | Metadata CRUD. | | `withClientKey(String)` | A copy that identifies its requests with a stable client key (required for lock operations). | -| `listHead(Long limit)` | Requests at the head. Returns `RequestQueueHead`. | -| `addRequest(RequestQueueRequest, boolean forefront)` | Add a request. Returns `RequestQueueOperationInfo`. | -| `getRequest(String id)` | Fetch a request. Returns `Optional`. | -| `updateRequest(RequestQueueRequest, boolean forefront)` | Update a request. Returns `RequestQueueOperationInfo`. | -| `deleteRequest(String id)` | Delete a request. No return value. | -| `batchAddRequests(List, boolean forefront)` | Add many (auto-chunked at 25 requests *and* by the API's ~9 MiB payload-size limit; unprocessed requests retried). Returns `BatchAddResult`. | -| `batchAddRequests(List, boolean forefront, BatchAddRequestsOptions)` | As above, tuning `maxUnprocessedRequestsRetries`, `maxParallel` and `minDelayBetweenUnprocessedRequestsRetriesMillis`. | -| `batchDeleteRequests(Object)` | Delete many. Returns `BatchDeleteResult`. | -| `listAndLockHead(long lockSecs, Long limit)` | Atomically lock the head. Returns `LockedRequestQueueHead`. | -| `listRequests(ListRequestsOptions)` | List requests. Returns `RequestsList`. | -| `prolongRequestLock(String id, long lockSecs, boolean forefront)` | Extend a lock. Returns `RequestLockInfo`. | -| `deleteRequestLock(String id, boolean forefront)` | Release a lock. No return value. | -| `unlockRequests()` | Release all the client's locks. Returns `UnlockRequestsResult`. | -| `paginateRequests(Long pageLimit)` | A lazy `Iterator` over all requests, paging with the queue's forward cursor. Equivalent to `paginateRequests(null, pageLimit, null)`. | +| `listHead(Long limit)` | Requests at the head. Completes with `RequestQueueHead`. | +| `addRequest(RequestQueueRequest, boolean forefront)` | Add a request. Completes with `RequestQueueOperationInfo`. | +| `getRequest(String id)` | Fetch a request. Completes with `Optional`. | +| `updateRequest(RequestQueueRequest, boolean forefront)` | Update a request. Completes with `RequestQueueOperationInfo`. | +| `deleteRequest(String id)` | Delete a request. Completes with no value (`CompletableFuture`). | +| `batchAddRequests(List, boolean forefront)` | Add many (auto-chunked at 25 requests *and* by the API's ~9 MiB payload-size limit; unprocessed requests retried). Completes with `BatchAddResult`. | +| `batchAddRequests(List, boolean forefront, BatchAddRequestsOptions)` | As above, tuning `maxUnprocessedRequestsRetries`, `maxParallel` (bounds how many chunk requests are in flight concurrently) and `minDelayBetweenUnprocessedRequestsRetriesMillis`. | +| `batchDeleteRequests(Object)` | Delete many. Completes with `BatchDeleteResult`. | +| `listAndLockHead(long lockSecs, Long limit)` | Atomically lock the head. Completes with `LockedRequestQueueHead`. | +| `listRequests(ListRequestsOptions)` | List requests. Completes with `RequestsList`. | +| `prolongRequestLock(String id, long lockSecs, boolean forefront)` | Extend a lock. Completes with `RequestLockInfo`. | +| `deleteRequestLock(String id, boolean forefront)` | Release a lock. Completes with no value. | +| `unlockRequests()` | Release all the client's locks. Completes with `UnlockRequestsResult`. | +| `paginateRequests(Long pageLimit)` | A lazy `Flow.Publisher` over all requests, paging with the queue's forward cursor. Equivalent to `paginateRequests(null, pageLimit, null)`. | | `paginateRequests(Long totalLimit, Long chunkSize, List filter)` | As above, with a cap on the total number yielded (`null`/non-positive = unbounded) and an optional state `filter` (`ListRequestsOptions.FILTER_LOCKED`/`FILTER_PENDING`), matching `listRequests`'s filter. | > **The `forefront` parameter.** `addRequest`, `updateRequest`, `batchAddRequests`, @@ -186,25 +186,23 @@ overload here — the request-queue creation endpoint does not accept a creation ```java // Retry a failed request ahead of everything else already queued. RequestQueueClient queue = client.requestQueue("QUEUE_ID"); -RequestQueueRequest failed = queue.getRequest("REQUEST_ID").orElseThrow(); -queue.updateRequest(failed.setNoRetry(false), true); +RequestQueueRequest failed = queue.getRequest("REQUEST_ID").join().orElseThrow(); +queue.updateRequest(failed.setNoRetry(false), true).join(); ``` > **Naming exception.** Request-queue *requests* are iterated with `paginateRequests(...)` — not an > `iterate(...)` method — because the request-queue listing is cursor-based rather than > offset/limit; it always starts from the beginning of the queue (resuming from an explicit -> `exclusiveStartId`/`cursor` is not supported by the iterator — use `listRequests(ListRequestsOptions)` +> `exclusiveStartId`/`cursor` is not supported by the publisher — use `listRequests(ListRequestsOptions)` > directly for that single-page use case). Every other resource uses the > `iterate`/`iterateItems`/`iterateKeys` family. ```java -RequestQueue rq = client.requestQueues().getOrCreate("my-queue"); +RequestQueue rq = client.requestQueues().getOrCreate("my-queue").join(); RequestQueueClient queue = client.requestQueue(rq.getId()); -queue.addRequest(new RequestQueueRequest("https://example.com", "example"), false); -Iterator it = queue.paginateRequests(100L); -while (it.hasNext()) { - System.out.println(it.next().getUrl()); -} +queue.addRequest(new RequestQueueRequest("https://example.com", "example"), false).join(); +List requests = Publishers.collect(queue.paginateRequests(100L)).join(); +requests.forEach(r -> System.out.println(r.getUrl())); ``` > **Lock lifecycle for distributed crawling.** `withClientKey(String)` returns a copy of the client @@ -217,23 +215,23 @@ while (it.hasNext()) { RequestQueueClient queue = client.requestQueue("QUEUE_ID").withClientKey("worker-1"); // Atomically reserve up to 10 requests from the head, locked for 60s. -LockedRequestQueueHead locked = queue.listAndLockHead(60L, 10L); +LockedRequestQueueHead locked = queue.listAndLockHead(60L, 10L).join(); for (RequestQueueRequest request : locked.getItems()) { boolean handledSuccessfully = true; // replace with your own processing logic/result if (handledSuccessfully) { - queue.deleteRequest(request.getId()); + queue.deleteRequest(request.getId()).join(); } else { // Give up on this one: release its lock so another worker can pick it up instead. - queue.deleteRequestLock(request.getId(), false); + queue.deleteRequestLock(request.getId(), false).join(); } } // Still working on a request past its lock's expiry? Extend it instead of losing it mid-process. -queue.prolongRequestLock(locked.getItems().get(0).getId(), 60L, false); +queue.prolongRequestLock(locked.getItems().get(0).getId(), 60L, false).join(); // On shutdown, release every lock this client still holds so other workers are not blocked // waiting for locks this worker will never finish processing. -queue.unlockRequests(); +queue.unlockRequests().join(); ``` `ListRequestsOptions` fields: `limit`, `exclusiveStartId`, `cursor` (mutually exclusive with diff --git a/docs/tasks.md b/docs/tasks.md index cd433ec..179b931 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -7,28 +7,28 @@ Tasks are pre-configured Actor runs with stored input. Access the task collectio | Method | Description | |---|---| -| `list(ListOptions)` | List tasks. Returns `PaginationList`. | -| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all tasks; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | -| `create(Object)` | Create a task from a JSON-serializable definition. Returns `Task`. | +| `list(ListOptions)` | List tasks. Completes with `PaginationList`. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Flow.Publisher` over all tasks; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | +| `create(Object)` | Create a task from a JSON-serializable definition. Completes with `Task`. | ```java Task task = client.tasks().create(Map.of( "actId", "apify/hello-world", "name", "my-task", "options", Map.of("build", "latest", "memoryMbytes", 256, "timeoutSecs", 60), - "input", Map.of("message", "hello"))); + "input", Map.of("message", "hello"))).join(); ``` ## `TaskClient` | Method | Description | |---|---| -| `get()` / `update(Object)` / `delete()` | CRUD. Return `Optional` / `Task` / `void`. | -| `start(Object input, TaskStartOptions)` | Start a task run (input overrides stored input; `null` uses it). Returns `ActorRun`. | -| `call(Object input, TaskStartOptions, Long waitSecs)` | Start and poll until finished; does **not** stream the run's log. Returns `ActorRun`. | +| `get()` / `update(Object)` / `delete()` | CRUD. Complete with `Optional` / `Task` / no value. | +| `start(Object input, TaskStartOptions)` | Start a task run (input overrides stored input; `null` uses it). Completes with `ActorRun`. | +| `call(Object input, TaskStartOptions, Long waitSecs)` | Start and poll until finished; does **not** stream the run's log. Completes with `ActorRun`. | | `call(Object input, TaskCallOptions, Long waitSecs)` | As above, additionally streaming the run's log for the duration of the wait by default (matching the reference client's `call` defaulting `options.log` to `'default'`). Use `TaskCallOptions.disableLogStreaming()` to opt out, or `logOptions(StreamedLogOptions)` for a custom destination. | -| `getInput()` | The stored input. Returns `Optional`. | -| `updateInput(Object)` | Replace the stored input. Returns `JsonNode`. | +| `getInput()` | The stored input. Completes with `Optional`. | +| `updateInput(Object)` | Replace the stored input. Completes with `JsonNode`. | | `lastRun(String status)` / `lastRun(LastRunOptions)` | A `RunClient` for the last run (see [`LastRunOptions`](actors.md#actorclient)). | | `runs()` | Nested run collection client. | | `webhooks()` | Read-only nested webhook collection (`NestedWebhookCollectionClient`, `list` + `iterate`, no `create`). | @@ -41,11 +41,12 @@ open server-side while the run finishes, which is redundant with (and wastes a r to) `call`'s own client-side `waitSecs` polling. ```java -ActorRun run = client.task("TASK_ID").call(null, new TaskStartOptions().memoryMbytes(512L), 120L); +ActorRun run = client.task("TASK_ID").call(null, new TaskStartOptions().memoryMbytes(512L), 120L).join(); System.out.println(run.getStatus()); // Streams the run's log to a default per-run logger for the duration of the wait. -ActorRun streamed = client.task("TASK_ID").call(null, new TaskCallOptions().memoryMbytes(512L), 120L); +ActorRun streamed = + client.task("TASK_ID").call(null, new TaskCallOptions().memoryMbytes(512L), 120L).join(); ``` `Task` fields: `getId()`, `getActId()`, `getUserId()`, `getName()`, `getTitle()`, diff --git a/docs/webhooks.md b/docs/webhooks.md index ee431ef..4a8b0cb 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -11,9 +11,9 @@ The account-wide collection supports both listing and creation. | Method | Description | |---|---| -| `list(ListOptions)` | List webhooks. Returns `PaginationList`. | -| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all webhooks; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). Also available on the read-only nested collections. | -| `create(Object)` | Create a webhook. Returns `Webhook`. | +| `list(ListOptions)` | List webhooks. Completes with `PaginationList`. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Flow.Publisher` over all webhooks; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). Also available on the read-only nested collections. | +| `create(Object)` | Create a webhook. Completes with `Webhook`. | ### Nested webhook collections (read-only) @@ -34,7 +34,7 @@ Webhook webhook = client.webhooks().create(Map.of( "isAdHoc", false, "eventTypes", List.of("ACTOR.RUN.SUCCEEDED", "ACTOR.RUN.FAILED"), "condition", Map.of("actorId", "apify/hello-world"), - "requestUrl", "https://example.com/webhook")); + "requestUrl", "https://example.com/webhook")).join(); ``` ## `WebhookClient` @@ -42,11 +42,11 @@ Webhook webhook = client.webhooks().create(Map.of( | Method | Description | |---|---| | `get()` / `update(Object)` / `delete()` | CRUD. | -| `test()` | Dispatch the webhook immediately. Returns `WebhookDispatch`. | +| `test()` | Dispatch the webhook immediately. Completes with `WebhookDispatch`. | | `dispatches()` | This webhook's dispatch collection. | ```java -WebhookDispatch dispatch = client.webhook("WEBHOOK_ID").test(); +WebhookDispatch dispatch = client.webhook("WEBHOOK_ID").test().join(); System.out.println(dispatch.getId()); ``` @@ -67,9 +67,9 @@ inherited `getExtra()` (see [the docs index](README.md#model-fields-and-unmodele | Method | Description | |---|---| -| `webhookDispatches().list(ListOptions)` | List dispatches. Returns `PaginationList`. | -| `webhookDispatches().iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all dispatches; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | -| `webhookDispatch(id).get()` | Fetch a dispatch. Returns `Optional`. | +| `webhookDispatches().list(ListOptions)` | List dispatches. Completes with `PaginationList`. | +| `webhookDispatches().iterate(ListOptions, Long chunkSize)` | Lazy `Flow.Publisher` over all dispatches; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | +| `webhookDispatch(id).get()` | Fetch a dispatch. Completes with `Optional`. | `WebhookDispatch` fields: `getId()`, `getUserId()`, `getWebhookId()`, `getCreatedAt()` (`Instant`), `getStatus()` (one of `"ACTIVE"`/`"SUCCEEDED"`/`"FAILED"`), `getEventType()` (the event that diff --git a/pom.xml b/pom.xml index bd3f1d3..7ba097d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.apify apify-client - 0.4.0 + 0.5.0 jar Apify Java Client @@ -39,23 +39,22 @@ 17 UTF-8 - 2.17.2 + 3.2.1 5.10.2 1.23.0 2.0.13 + - com.fasterxml.jackson.core + tools.jackson.core jackson-databind ${jackson.version} - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - ${jackson.version} - @@ -55,6 +62,10 @@ + + + + diff --git a/src/main/java/com/apify/client/ApifyClient.java b/src/main/java/com/apify/client/ApifyClient.java index ea349a2..731513f 100644 --- a/src/main/java/com/apify/client/ApifyClient.java +++ b/src/main/java/com/apify/client/ApifyClient.java @@ -31,6 +31,7 @@ import com.apify.client.webhook.WebhookDispatchCollectionClient; import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; /** * The entry point for interacting with the Apify API. @@ -39,13 +40,20 @@ * resource clients via the accessor methods, e.g. {@link #actor(String)}, {@link #dataset(String)}, * {@link #run(String)}. It is safe for concurrent use. * + *

Every network-calling method is asynchronous and non-blocking: it returns a {@link + * java.util.concurrent.CompletableFuture} (or, for a paginated collection's lazy iteration, a + * {@link java.util.concurrent.Flow.Publisher}) instead of blocking the calling thread on the HTTP + * exchange. + * *

Quick start

* *
{@code
  * ApifyClient client = ApifyClient.create("my-api-token");
- * ActorRun run = client.actor("apify/hello-world").call(null, new ActorStartOptions(), null);
- * PaginationList items = client.dataset(run.getDefaultDatasetId())
- *     .listItems(new DatasetListItemsOptions());
+ * client.actor("apify/hello-world").call(null, new ActorStartOptions(), null)
+ *     .thenCompose(run -> client.dataset(run.getDefaultDatasetId())
+ *         .listItems(new DatasetListItemsOptions()))
+ *     .thenAccept(items -> System.out.println(items.getItems()))
+ *     .join();
  * }
* *

Architecture

@@ -55,7 +63,7 @@ *
  • Replaceable transport: the {@link HttpTransport} interface, with a default {@link * DefaultHttpTransport}; swap it via {@link ApifyClientBuilder#httpTransport(HttpTransport)}. *
  • Cross-cutting behaviour (auth, User-Agent, retries with exponential backoff, timeouts) - * lives in the internal HTTP client and is applied to every request. + * lives in the internal HTTP client and is applied to every request, non-blocking end to end. * * *

    Every resource client's constructor is public, but that is an artifact of each living in its @@ -256,7 +264,8 @@ public UserClient user(String id) { * IllegalStateException} if {@code ACTOR_RUN_ID} is not set. Matches the reference client's * top-level {@code setStatusMessage}. */ - public ActorRun setStatusMessage(String message, SetStatusMessageOptions options) { + public CompletableFuture setStatusMessage( + String message, SetStatusMessageOptions options) { String runId = System.getenv(ACTOR_RUN_ID_ENV_VAR); if (runId == null || runId.isEmpty()) { throw new IllegalStateException(ACTOR_RUN_ID_ENV_VAR + " environment variable is not set"); diff --git a/src/main/java/com/apify/client/Publishers.java b/src/main/java/com/apify/client/Publishers.java new file mode 100644 index 0000000..cf10cfa --- /dev/null +++ b/src/main/java/com/apify/client/Publishers.java @@ -0,0 +1,64 @@ +package com.apify.client; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Flow; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * A small bridge from this client's async {@link Flow.Publisher}-based iteration helpers to a plain + * {@link List}, for callers who want "give me everything" rather than driving a {@link + * Flow.Subscriber} themselves. + * + *

    Every {@code iterate(...)} method across the resource clients (e.g. {@code + * ActorCollectionClient#iterate}) returns a {@link Flow.Publisher} - the JDK's built-in Reactive + * Streams type - so a large paginated collection can be followed without blocking a thread on each + * page's network round-trip, with the subscriber controlling how many items are in flight at once + * via {@link Flow.Subscription#request(long)}. {@link #collect(Flow.Publisher)} is a convenience + * subscriber that requests everything unbounded and collects it into a {@link CompletableFuture} of + * a {@link List}, for the common case where backpressure is not a concern (e.g. because the caller + * knows the total is bounded, or a test simply wants the full list to assert on). + */ +public final class Publishers { + + private Publishers() {} + + /** + * Subscribes to {@code publisher} with unbounded demand and collects every emitted item into a + * {@link List}, in emission order. The returned future completes with the full list once the + * publisher signals completion, or exceptionally with whatever error the publisher signalled. + */ + public static CompletableFuture> collect(Flow.Publisher publisher) { + CompletableFuture> result = new CompletableFuture<>(); + ConcurrentLinkedQueue items = new ConcurrentLinkedQueue<>(); + AtomicBoolean done = new AtomicBoolean(); + publisher.subscribe( + new Flow.Subscriber<>() { + @Override + public void onSubscribe(Flow.Subscription subscription) { + subscription.request(Long.MAX_VALUE); + } + + @Override + public void onNext(T item) { + items.add(item); + } + + @Override + public void onError(Throwable throwable) { + if (done.compareAndSet(false, true)) { + result.completeExceptionally(throwable); + } + } + + @Override + public void onComplete() { + if (done.compareAndSet(false, true)) { + result.complete(List.copyOf(items)); + } + } + }); + return result; + } +} diff --git a/src/main/java/com/apify/client/Version.java b/src/main/java/com/apify/client/Version.java index 5ef75df..61a6b16 100644 --- a/src/main/java/com/apify/client/Version.java +++ b/src/main/java/com/apify/client/Version.java @@ -13,13 +13,13 @@ public final class Version { * The semantic version of this client library (see SemVer). * Changes to the public interface other than additive ones are considered breaking changes. */ - public static final String CLIENT_VERSION = "0.4.0"; + public static final String CLIENT_VERSION = "0.5.0"; /** * The version of the Apify OpenAPI specification this client was generated and verified against. * Corresponds to the {@code info.version} field of the Apify OpenAPI document. */ - public static final String API_SPEC_VERSION = "v2-2026-07-20T094852Z"; + public static final String API_SPEC_VERSION = "v2-2026-07-22T122437Z"; private Version() {} } diff --git a/src/main/java/com/apify/client/actor/Actor.java b/src/main/java/com/apify/client/actor/Actor.java index 1895944..52bc6be 100644 --- a/src/main/java/com/apify/client/actor/Actor.java +++ b/src/main/java/com/apify/client/actor/Actor.java @@ -1,10 +1,10 @@ package com.apify.client.actor; import com.apify.client.ApifyResource; -import com.fasterxml.jackson.databind.JsonNode; import java.time.Instant; import java.util.Collections; import java.util.List; +import tools.jackson.databind.JsonNode; /** An Actor on the Apify platform. */ public final class Actor extends ApifyResource { diff --git a/src/main/java/com/apify/client/actor/ActorClient.java b/src/main/java/com/apify/client/actor/ActorClient.java index 078f6f7..5f601fb 100644 --- a/src/main/java/com/apify/client/actor/ActorClient.java +++ b/src/main/java/com/apify/client/actor/ActorClient.java @@ -15,8 +15,9 @@ import com.apify.client.run.RunClient; import com.apify.client.run.RunCollectionClient; import com.apify.client.webhook.NestedWebhookCollectionClient; -import com.fasterxml.jackson.databind.JsonNode; import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import tools.jackson.databind.JsonNode; /** * A client for a specific Actor. @@ -45,54 +46,54 @@ public String getId() { } /** Fetches the Actor object, or empty if it does not exist. */ - public Optional get() { + public CompletableFuture> get() { return ctx.getResource("", new QueryParams(), Actor.class); } /** Updates the Actor with the given fields and returns the updated object. */ - public Actor update(Object newFields) { + public CompletableFuture update(Object newFields) { return ctx.updateResource("", newFields, Actor.class); } /** Deletes the Actor. */ - public void delete() { - ctx.deleteResource(""); + public CompletableFuture delete() { + return ctx.deleteResource(""); } /** - * Starts the Actor and returns immediately with the created run. {@code input} is any - * JSON-serializable value (or {@code null} for no input). + * Starts the Actor and completes with the created run as soon as it exists (no waiting). {@code + * input} is any JSON-serializable value (or {@code null} for no input). */ - public ActorRun start(Object input, ActorStartOptions options) { + public CompletableFuture start(Object input, ActorStartOptions options) { return RunStartSupport.start(ctx, input, options::apply, options.contentTypeOrDefault()); } /** - * Starts the Actor and waits (client-side polling) for it to finish. {@code waitSecs} bounds the - * wait; {@code null} waits indefinitely. Returns the finished run (or the still-running run if - * the wait budget was exhausted). + * Starts the Actor and waits (non-blocking, polling) for it to finish. {@code waitSecs} bounds + * the wait; {@code null} waits indefinitely. Completes with the finished run (or the + * still-running run if the wait budget was exhausted). * *

    This overload does not stream the run's log; use {@link #call(Object, ActorCallOptions, * Long)} for that (matching the reference client's default {@code call} behavior). */ - public ActorRun call(Object input, ActorStartOptions options, Long waitSecs) { + public CompletableFuture call(Object input, ActorStartOptions options, Long waitSecs) { return RunStartSupport.call( root, ctx, input, options::apply, options.contentTypeOrDefault(), waitSecs); } /** - * Starts the Actor and waits (client-side polling) for it to finish, additionally streaming the + * Starts the Actor and waits (non-blocking, polling) for it to finish, additionally streaming the * run's log for the duration of the wait — matching the reference client's {@code call}, whose * {@code options.log} defaults to {@code 'default'}. {@code waitSecs} bounds the wait; {@code - * null} waits indefinitely. Returns the finished run (or the still-running run if the wait budget - * was exhausted). + * null} waits indefinitely. Completes with the finished run (or the still-running run if the wait + * budget was exhausted). * *

    Log streaming is best-effort: if starting it fails (e.g. the log is not yet available), the * run still starts and is still waited for, just without redirected log output. Use {@link * ActorCallOptions#disableLogStreaming()} to opt out entirely, or {@link * ActorCallOptions#logOptions(com.apify.client.log.StreamedLogOptions)} for a custom destination. */ - public ActorRun call(Object input, ActorCallOptions options, Long waitSecs) { + public CompletableFuture call(Object input, ActorCallOptions options, Long waitSecs) { ActorCallOptions opts = options != null ? options : new ActorCallOptions(); ActorStartOptions startOptions = opts.toStartOptions(); return RunStartSupport.callWithLogStreaming( @@ -107,31 +108,33 @@ public ActorRun call(Object input, ActorCallOptions options, Long waitSecs) { } /** Validates the given input against the Actor's default-build input schema. */ - public boolean validateInput(Object input) { + public CompletableFuture validateInput(Object input) { return validateInput(input, new ValidateInputOptions()); } /** - * Validates {@code input} against the Actor's input schema and returns whether it is valid. - * {@code input} is any JSON-serializable value (or {@code null}). {@code options} may pin the - * build whose schema is used and the content type of the input body. + * Validates {@code input} against the Actor's input schema and completes with whether it is + * valid. {@code input} is any JSON-serializable value (or {@code null}). {@code options} may pin + * the build whose schema is used and the content type of the input body. */ - public boolean validateInput(Object input, ValidateInputOptions options) { + public CompletableFuture validateInput(Object input, ValidateInputOptions options) { ValidateInputOptions opts = options == null ? new ValidateInputOptions() : options; QueryParams params = new QueryParams(); opts.apply(params); byte[] body = input == null ? null : Json.toBytes(input); // The validate-input endpoint returns a bare {"valid": } object, not the standard // {"data": ...} envelope, so parse it without unwrapping. - JsonNode result = - ctx.postWithBodyNoEnvelope( - "validate-input", params, body, opts.contentTypeOrDefault(), JsonNode.class); - JsonNode valid = result == null ? null : result.get("valid"); - return valid != null && valid.asBoolean(); + return ctx.postWithBodyNoEnvelope( + "validate-input", params, body, opts.contentTypeOrDefault(), JsonNode.class) + .thenApply( + result -> { + JsonNode valid = result == null ? null : result.get("valid"); + return valid != null && valid.asBoolean(); + }); } - /** Builds the given version of the Actor and returns the created build. */ - public Build build(String versionNumber, ActorBuildOptions options) { + /** Builds the given version of the Actor and completes with the created build. */ + public CompletableFuture build(String versionNumber, ActorBuildOptions options) { if (options == null) { throw new IllegalArgumentException("options is required and must not be null"); } @@ -142,17 +145,17 @@ public Build build(String versionNumber, ActorBuildOptions options) { } /** - * Resolves the Actor's default build and returns a client for it. {@code waitForFinish} + * Resolves the Actor's default build and completes with a client for it. {@code waitForFinish} * optionally bounds how long (in seconds) the API waits for the build to finish before * responding. */ - public BuildClient defaultBuild(Long waitForFinish) { + public CompletableFuture defaultBuild(Long waitForFinish) { QueryParams params = new QueryParams(); // Clamp like the getWithWait twins so a large wait paired with a short client timeout cannot // abort every attempt and burn the retry budget (the API caps server-side waiting at 60s). params.addLong("waitForFinish", ctx.clampServerWait(waitForFinish)); - Build build = ctx.getResourceRequired("builds/default", params, Build.class); - return new BuildClient(http, baseUrl, build.getId()); + return ctx.getResourceRequired("builds/default", params, Build.class) + .thenApply(build -> new BuildClient(http, baseUrl, build.getId())); } /** diff --git a/src/main/java/com/apify/client/actor/ActorCollectionClient.java b/src/main/java/com/apify/client/actor/ActorCollectionClient.java index a1735e2..9a829c2 100644 --- a/src/main/java/com/apify/client/actor/ActorCollectionClient.java +++ b/src/main/java/com/apify/client/actor/ActorCollectionClient.java @@ -5,6 +5,7 @@ import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; +import java.util.concurrent.CompletableFuture; /** A client for the Actor collection ({@code GET/POST /v2/actors}). */ public final class ActorCollectionClient extends AbstractCollectionClient { @@ -17,7 +18,7 @@ public ActorCollectionClient(HttpClientCore http, String baseUrl) { } /** Creates a new Actor. {@code actor} is any JSON-serializable Actor definition. */ - public Actor create(Object actor) { + public CompletableFuture create(Object actor) { return ctx.createResource(new QueryParams(), actor, Actor.class); } } diff --git a/src/main/java/com/apify/client/actor/ActorEnvVarClient.java b/src/main/java/com/apify/client/actor/ActorEnvVarClient.java index 745661d..aabab48 100644 --- a/src/main/java/com/apify/client/actor/ActorEnvVarClient.java +++ b/src/main/java/com/apify/client/actor/ActorEnvVarClient.java @@ -4,6 +4,7 @@ import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; import java.util.Optional; +import java.util.concurrent.CompletableFuture; /** * A client for a single environment variable ({@code GET/PUT/DELETE @@ -17,17 +18,17 @@ public final class ActorEnvVarClient { } /** Fetches the environment variable, or empty if it does not exist. */ - public Optional get() { + public CompletableFuture> get() { return ctx.getResource("", new QueryParams(), ActorEnvVar.class); } /** Updates the environment variable and returns the updated object. */ - public ActorEnvVar update(ActorEnvVar envVar) { + public CompletableFuture update(ActorEnvVar envVar) { return ctx.updateResource("", envVar, ActorEnvVar.class); } /** Deletes the environment variable. */ - public void delete() { - ctx.deleteResource(""); + public CompletableFuture delete() { + return ctx.deleteResource(""); } } diff --git a/src/main/java/com/apify/client/actor/ActorEnvVarCollectionClient.java b/src/main/java/com/apify/client/actor/ActorEnvVarCollectionClient.java index 871cf19..009bda3 100644 --- a/src/main/java/com/apify/client/actor/ActorEnvVarCollectionClient.java +++ b/src/main/java/com/apify/client/actor/ActorEnvVarCollectionClient.java @@ -2,9 +2,11 @@ import com.apify.client.PaginationList; import com.apify.client.internal.HttpClientCore; +import com.apify.client.internal.ListPublisher; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; -import java.util.Iterator; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; /** * A client for an Actor version's environment variable collection ({@code GET/POST @@ -18,21 +20,21 @@ public final class ActorEnvVarCollectionClient { } /** Lists the version's environment variables. */ - public PaginationList list() { + public CompletableFuture> list() { return ctx.listResource("", new QueryParams(), ActorEnvVar.class); } /** - * Returns an iterator over the version's environment variables. The env-var collection is not - * paginated (the API returns every variable in one response), so this iterates a single fetched + * Returns a publisher over the version's environment variables. The env-var collection is not + * paginated (the API returns every variable in one response), so this publishes a single fetched * page; the method exists for API consistency with the other collection clients. */ - public Iterator iterate() { - return list().getItems().iterator(); + public Flow.Publisher iterate() { + return new ListPublisher<>(list().thenApply(PaginationList::getItems)); } /** Creates a new environment variable. */ - public ActorEnvVar create(ActorEnvVar envVar) { + public CompletableFuture create(ActorEnvVar envVar) { return ctx.createResource(new QueryParams(), envVar, ActorEnvVar.class); } } diff --git a/src/main/java/com/apify/client/actor/ActorVersionClient.java b/src/main/java/com/apify/client/actor/ActorVersionClient.java index 0d0c60b..e48b30c 100644 --- a/src/main/java/com/apify/client/actor/ActorVersionClient.java +++ b/src/main/java/com/apify/client/actor/ActorVersionClient.java @@ -4,6 +4,7 @@ import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; import java.util.Optional; +import java.util.concurrent.CompletableFuture; /** * A client for a specific Actor version ({@code GET/PUT/DELETE @@ -21,18 +22,18 @@ public final class ActorVersionClient { } /** Fetches the version, or empty if it does not exist. */ - public Optional get() { + public CompletableFuture> get() { return ctx.getResource("", new QueryParams(), ActorVersion.class); } /** Updates the version with the given fields and returns the updated object. */ - public ActorVersion update(Object newFields) { + public CompletableFuture update(Object newFields) { return ctx.updateResource("", newFields, ActorVersion.class); } /** Deletes the version. */ - public void delete() { - ctx.deleteResource(""); + public CompletableFuture delete() { + return ctx.deleteResource(""); } /** A client for a specific environment variable of this version. */ diff --git a/src/main/java/com/apify/client/actor/ActorVersionCollectionClient.java b/src/main/java/com/apify/client/actor/ActorVersionCollectionClient.java index 0ea9ce9..4b18c98 100644 --- a/src/main/java/com/apify/client/actor/ActorVersionCollectionClient.java +++ b/src/main/java/com/apify/client/actor/ActorVersionCollectionClient.java @@ -3,10 +3,12 @@ import com.apify.client.ListOptions; import com.apify.client.PaginationList; import com.apify.client.internal.HttpClientCore; +import com.apify.client.internal.ListPublisher; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; -import java.util.Iterator; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; /** A client for an Actor's version collection ({@code GET/POST /v2/actors/{actorId}/versions}). */ public final class ActorVersionCollectionClient { @@ -24,33 +26,38 @@ public final class ActorVersionCollectionClient { * offset}/{@code limit}/{@code desc}) is sent to the server. Pass {@code null} for the common * case; use {@link #iterate(ListOptions)}'s {@code limit} to cap the number returned client-side. */ - public PaginationList list(ListOptions options) { + public CompletableFuture> list(ListOptions options) { return ctx.listResource("", new QueryParams(), ActorVersion.class); } /** - * Returns an iterator over the Actor's versions. Unlike the paginated collection iterators, this - * fetches eagerly: the single request runs when {@code iterate} is called, not on first {@code - * next()}. That's because {@code GET /v2/actors/{actorId}/versions} is not offset/limit paginated - * at all — it takes no pagination parameters and always returns the full version list in one - * {@code {total, items}} response, so routing it through the offset/limit paging engine would - * loop forever (the server returns the same non-empty page at every offset; this is why the - * sibling non-paginated {@code env-vars} collection is also a single-fetch iterator). The - * options' {@code limit} still caps the number yielded ({@code null} or non-positive = all); - * {@code offset} has no effect and there is no page size to tune. + * Returns a publisher over the Actor's versions. Unlike the paginated collection publishers, this + * fetches eagerly: the single request runs when {@code iterate} is called, not on first demand. + * That's because {@code GET /v2/actors/{actorId}/versions} is not offset/limit paginated at all — + * it takes no pagination parameters and always returns the full version list in one {@code + * {total, items}} response, so routing it through the offset/limit paging engine would loop + * forever (the server returns the same non-empty page at every offset; this is why the sibling + * non-paginated {@code env-vars} collection is also a single-fetch publisher). The options' + * {@code limit} still caps the number yielded ({@code null} or non-positive = all); {@code + * offset} has no effect and there is no page size to tune. */ - public Iterator iterate(ListOptions options) { + public Flow.Publisher iterate(ListOptions options) { ListOptions opts = options != null ? options : new ListOptions(); - List items = list(opts).getItems(); Long limit = opts.limitValue(); - if (limit != null && limit > 0 && items.size() > limit) { - items = items.subList(0, (int) (long) limit); - } - return items.iterator(); + CompletableFuture> items = + list(opts) + .thenApply( + page -> { + List all = page.getItems(); + return (limit != null && limit > 0 && all.size() > limit) + ? all.subList(0, (int) (long) limit) + : all; + }); + return new ListPublisher<>(items); } /** Creates a new Actor version. {@code version} is any JSON-serializable version definition. */ - public ActorVersion create(Object version) { + public CompletableFuture create(Object version) { return ctx.createResource(new QueryParams(), version, ActorVersion.class); } } diff --git a/src/main/java/com/apify/client/build/BuildClient.java b/src/main/java/com/apify/client/build/BuildClient.java index ea3f651..5a22197 100644 --- a/src/main/java/com/apify/client/build/BuildClient.java +++ b/src/main/java/com/apify/client/build/BuildClient.java @@ -1,14 +1,14 @@ package com.apify.client.build; -import com.apify.client.http.ApiResponse; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.Json; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; import com.apify.client.log.LogClient; -import com.fasterxml.jackson.databind.JsonNode; import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import tools.jackson.databind.JsonNode; /** A client for a specific Actor build ({@code /v2/actor-builds/{buildId}}). */ public final class BuildClient { @@ -21,7 +21,7 @@ public BuildClient(HttpClientCore http, String baseUrl, String id) { } /** Fetches the build object, or empty if it does not exist. */ - public Optional get() { + public CompletableFuture> get() { return getWithWait(null); } @@ -31,7 +31,7 @@ public Optional get() { * before the client's per-request timeout; the API itself caps server-side waiting at 60 seconds. * Pass {@code null} for an immediate fetch. */ - public Optional getWithWait(Long waitForFinishSecs) { + public CompletableFuture> getWithWait(Long waitForFinishSecs) { QueryParams params = new QueryParams(); // Clamp to the client's per-request timeout so a short custom timeout doesn't abort the call. params.addLong("waitForFinish", ctx.clampServerWait(waitForFinishSecs)); @@ -39,20 +39,20 @@ public Optional getWithWait(Long waitForFinishSecs) { } /** Aborts the build and returns its updated state. */ - public Build abort() { + public CompletableFuture abort() { return ctx.postWithBody("abort", new QueryParams(), null, "", Build.class); } /** Deletes the build. */ - public void delete() { - ctx.deleteResource(""); + public CompletableFuture delete() { + return ctx.deleteResource(""); } /** * Polls until the build reaches a terminal state or {@code waitSecs} elapses ({@code null} waits * indefinitely). Returns the latest build. */ - public Build waitForFinish(Long waitSecs) { + public CompletableFuture waitForFinish(Long waitSecs) { return ctx.waitForFinish(waitSecs, "build", Json.type(Build.class), Build::isTerminal); } @@ -60,12 +60,13 @@ public Build waitForFinish(Long waitSecs) { * Returns the OpenAPI definition generated for the build, or empty if it is not available. The * result is the raw OpenAPI document. */ - public Optional getOpenApiDefinition() { - ApiResponse resp = ctx.getRaw("openapi.json", new QueryParams()); - if (resp == null) { - return Optional.empty(); - } - return Optional.of(Json.parse(resp.body(), JsonNode.class)); + public CompletableFuture> getOpenApiDefinition() { + return ctx.getRaw("openapi.json", new QueryParams()) + .thenApply( + resp -> + resp == null + ? Optional.empty() + : Optional.of(Json.parse(resp.body(), JsonNode.class))); } /** A client for accessing this build's log. */ diff --git a/src/main/java/com/apify/client/dataset/DatasetClient.java b/src/main/java/com/apify/client/dataset/DatasetClient.java index b644149..71053ac 100644 --- a/src/main/java/com/apify/client/dataset/DatasetClient.java +++ b/src/main/java/com/apify/client/dataset/DatasetClient.java @@ -3,18 +3,19 @@ import com.apify.client.PaginationList; import com.apify.client.http.ApiResponse; import com.apify.client.internal.ApiPaths; +import com.apify.client.internal.AsyncPaginatedPublisher; import com.apify.client.internal.Extras; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.Json; -import com.apify.client.internal.PaginatedIterator; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; import com.apify.client.internal.Signatures; -import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.JsonNode; -import java.util.Iterator; import java.util.List; import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; +import tools.jackson.databind.JavaType; +import tools.jackson.databind.JsonNode; /** A client for a specific dataset (and run-nested variants). */ public final class DatasetClient { @@ -65,25 +66,25 @@ public static DatasetClient nested( } /** Fetches the dataset metadata, or empty if it does not exist. */ - public Optional get() { + public CompletableFuture> get() { return ctx.getResource("", new QueryParams(), Dataset.class); } /** Updates the dataset metadata (e.g. name, title) and returns the updated object. */ - public Dataset update(Object newFields) { + public CompletableFuture update(Object newFields) { return ctx.updateResource("", newFields, Dataset.class); } /** Deletes the dataset. */ - public void delete() { - ctx.deleteResource(""); + public CompletableFuture delete() { + return ctx.deleteResource(""); } /** * Lists items from the dataset, decoding each into a generic {@link JsonNode}. For typed decoding * use {@link #listItems(DatasetListItemsOptions, Class)}. */ - public PaginationList listItems(DatasetListItemsOptions options) { + public CompletableFuture> listItems(DatasetListItemsOptions options) { return listItems(options, JsonNode.class); } @@ -94,30 +95,32 @@ public PaginationList listItems(DatasetListItemsOptions options) { * pagination via {@code X-Apify-Pagination-*} headers, surfaced in the returned {@link * PaginationList}. */ - public PaginationList listItems(DatasetListItemsOptions options, Class itemClass) { + public CompletableFuture> listItems( + DatasetListItemsOptions options, Class itemClass) { QueryParams params = new QueryParams(); options.apply(params); return fetchItemsPage(params, options.descValue(), itemClass); } /** - * Returns a lazy iterator over the dataset's items, decoding each into a generic {@link - * JsonNode}. For typed decoding use {@link #iterateItems(DatasetListItemsOptions, Long, Class)}. + * Returns a lazy, backpressure-aware publisher over the dataset's items, decoding each into a + * generic {@link JsonNode}. For typed decoding use {@link #iterateItems(DatasetListItemsOptions, + * Long, Class)}. */ - public Iterator iterateItems(DatasetListItemsOptions options, Long chunkSize) { + public Flow.Publisher iterateItems(DatasetListItemsOptions options, Long chunkSize) { return iterateItems(options, chunkSize, JsonNode.class); } /** As {@link #iterateItems(DatasetListItemsOptions, Long)} with the server-default page size. */ - public Iterator iterateItems(DatasetListItemsOptions options) { + public Flow.Publisher iterateItems(DatasetListItemsOptions options) { return iterateItems(options, null, JsonNode.class); } /** - * Returns a lazy iterator over the dataset's items, decoding each into {@code itemClass}, - * fetching pages on demand. The options' {@code limit} caps the total number of items yielded - * ({@code null} or non-positive = all); {@code chunkSize} is the per-request page size ({@code - * null} = server default). + * Returns a lazy, backpressure-aware publisher over the dataset's items, decoding each into + * {@code itemClass}, fetching pages on demand. The options' {@code limit} caps the total number + * of items yielded ({@code null} or non-positive = all); {@code chunkSize} is the per-request + * page size ({@code null} = server default). * *

    Note: server-side item filters ({@code skipEmpty}, {@code skipHidden}, {@code clean}, {@code * simplified}) are applied after {@code offset}/{@code limit}, so a page can return fewer items @@ -126,7 +129,7 @@ public Iterator iterateItems(DatasetListItemsOptions options) { * iteration early — silently skipping the remaining items. Iterate without server-side item * filters, or page explicitly with {@link #listItems} and filter client-side. */ - public Iterator iterateItems( + public Flow.Publisher iterateItems( DatasetListItemsOptions options, Long chunkSize, Class itemClass) { DatasetListItemsOptions opts = options != null ? options : new DatasetListItemsOptions(); // Snapshot the filters/offset/limit/desc once so mutating the options mid-iteration cannot @@ -134,7 +137,7 @@ public Iterator iterateItems( QueryParams filters = new QueryParams(); opts.applyFilters(filters); Boolean desc = opts.descValue(); - return new PaginatedIterator<>( + return new AsyncPaginatedPublisher<>( opts.limitValue(), chunkSize, opts.offsetValue(), @@ -150,25 +153,27 @@ public Iterator iterateItems( * items endpoint returns a bare JSON array (not a data envelope) and reports pagination via * {@code X-Apify-Pagination-*} headers, surfaced in the returned {@link PaginationList}. */ - private PaginationList fetchItemsPage( + private CompletableFuture> fetchItemsPage( QueryParams params, Boolean desc, Class itemClass) { String url = ctx.mergedParams(params).applyToUrl(ctx.subUrl("items")); - ApiResponse resp = http.call("GET", url, null, "", http.baseRequestTimeout()); + return http.call("GET", url, null, "", http.baseRequestTimeout()) + .thenApply( + resp -> { + JavaType listType = Json.parametric(List.class, Json.type(itemClass)); + List items = Json.parse(resp.body(), listType); + long count = items.size(); - JavaType listType = Json.parametric(List.class, Json.type(itemClass)); - List items = Json.parse(resp.body(), listType); - long count = items.size(); - - PaginationList result = new PaginationList<>(); - result.setItems(items); - result.setCount(count); - result.setTotal(headerLong(resp, HEADER_PAGINATION_TOTAL, count)); - result.setOffset(headerLong(resp, HEADER_PAGINATION_OFFSET, 0)); - result.setLimit(headerLong(resp, HEADER_PAGINATION_LIMIT, count)); - if (desc != null) { - result.setDesc(desc); - } - return result; + PaginationList result = new PaginationList<>(); + result.setItems(items); + result.setCount(count); + result.setTotal(headerLong(resp, HEADER_PAGINATION_TOTAL, count)); + result.setOffset(headerLong(resp, HEADER_PAGINATION_OFFSET, 0)); + result.setLimit(headerLong(resp, HEADER_PAGINATION_LIMIT, count)); + if (desc != null) { + result.setDesc(desc); + } + return result; + }); } /** @@ -176,38 +181,40 @@ private PaginationList fetchItemsPage( * #listItems} (parsed items), this returns the items already serialized to JSON, CSV, XLSX, XML, * RSS or HTML — useful for exporting. */ - public byte[] downloadItems(DownloadItemsFormat format, DatasetDownloadOptions options) { + public CompletableFuture downloadItems( + DownloadItemsFormat format, DatasetDownloadOptions options) { QueryParams params = new QueryParams(); params.addString("format", format.wireValue()); options.apply(params); String url = ctx.mergedParams(params).applyToUrl(ctx.subUrl("items")); - ApiResponse resp = http.call("GET", url, null, "", http.baseRequestTimeout()); - return resp.body(); + return http.call("GET", url, null, "", http.baseRequestTimeout()).thenApply(ApiResponse::body); } /** * Pushes one or more items to the dataset. {@code items} must serialize to a JSON object or an * array of objects. */ - public void pushItems(Object items) { + public CompletableFuture pushItems(Object items) { // Route through mergedParams like every sibling method, so a context seeded with pinned filters // (e.g. actor(id).lastRun(...).dataset()) targets the same run's dataset on write as on read. String url = ctx.mergedParams(new QueryParams()).applyToUrl(ctx.subUrl("items")); - http.call( - "POST", - url, - Json.toBytes(items), - ResourceContext.CONTENT_TYPE_JSON_CHARSET, - http.baseRequestTimeout()); + return http.call( + "POST", + url, + Json.toBytes(items), + ResourceContext.CONTENT_TYPE_JSON_CHARSET, + http.baseRequestTimeout()) + .thenApply(resp -> null); } /** Returns statistical information about the dataset, or empty if unavailable. */ - public Optional getStatistics() { - ApiResponse resp = ctx.getRaw("statistics", new QueryParams()); - if (resp == null) { - return Optional.empty(); - } - return Optional.of(Json.parseData(resp.body(), JsonNode.class)); + public CompletableFuture> getStatistics() { + return ctx.getRaw("statistics", new QueryParams()) + .thenApply( + resp -> + resp == null + ? Optional.empty() + : Optional.of(Json.parseData(resp.body(), JsonNode.class))); } /** @@ -218,22 +225,29 @@ public Optional getStatistics() { * {@code expiresInSecs} optionally bounds the validity of a signed URL ({@code null} for * non-expiring). The URL is built from the configured public base URL. */ - public String createItemsPublicUrl(DatasetListItemsOptions options, Long expiresInSecs) { + public CompletableFuture createItemsPublicUrl( + DatasetListItemsOptions options, Long expiresInSecs) { QueryParams params = new QueryParams(); options.apply(params); - Optional dataset = get(); - if (dataset.isPresent()) { - String secret = Extras.extractString(dataset.get().getExtra(), "urlSigningSecretKey"); - if (secret != null) { - String sig = Signatures.signStorageContent(secret, dataset.get().getId(), expiresInSecs); - params.addString("signature", sig); - } - } - // Mirror the JS reference: the public URL is this client's resource path plus the signature - // (over the resolved concrete dataset id) and the explicit options — the seeded status/origin - // filters are deliberately not carried. On a last-run-nested client this keeps the unpinned - // ".../runs/last/dataset" path; see docs/storages.md for that limitation. - return params.applyToUrl(ctx.publicUrl("items")); + return get() + .thenApply( + dataset -> { + if (dataset.isPresent()) { + String secret = + Extras.extractString(dataset.get().getExtra(), "urlSigningSecretKey"); + if (secret != null) { + String sig = + Signatures.signStorageContent(secret, dataset.get().getId(), expiresInSecs); + params.addString("signature", sig); + } + } + // Mirror the JS reference: the public URL is this client's resource path plus the + // signature (over the resolved concrete dataset id) and the explicit options — the + // seeded status/origin filters are deliberately not carried. On a last-run-nested + // client this keeps the unpinned ".../runs/last/dataset" path; see docs/storages.md + // for that limitation. + return params.applyToUrl(ctx.publicUrl("items")); + }); } private static long headerLong(ApiResponse resp, String name, long fallback) { diff --git a/src/main/java/com/apify/client/dataset/DatasetCollectionClient.java b/src/main/java/com/apify/client/dataset/DatasetCollectionClient.java index d2392bd..50e2fa8 100644 --- a/src/main/java/com/apify/client/dataset/DatasetCollectionClient.java +++ b/src/main/java/com/apify/client/dataset/DatasetCollectionClient.java @@ -5,6 +5,7 @@ import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.ResourceContext; +import java.util.concurrent.CompletableFuture; /** A client for the dataset collection ({@code GET/POST /v2/datasets}). */ public final class DatasetCollectionClient @@ -21,7 +22,7 @@ public DatasetCollectionClient(HttpClientCore http, String baseUrl) { * Gets the dataset with the given name, creating it if it does not exist. An empty/{@code null} * name creates a new unnamed dataset. */ - public Dataset getOrCreate(String name) { + public CompletableFuture getOrCreate(String name) { return getOrCreate(name, null); } @@ -30,7 +31,7 @@ public Dataset getOrCreate(String name) { * {@code schema} on creation. Mirrors the reference client's {@code getOrCreate(name, {schema})}; * a {@code null} schema behaves like {@link #getOrCreate(String)}. */ - public Dataset getOrCreate(String name, Object schema) { + public CompletableFuture getOrCreate(String name, Object schema) { return ctx.getOrCreateNamedWithSchema(name, schema, Dataset.class); } } diff --git a/src/main/java/com/apify/client/http/DefaultHttpTransport.java b/src/main/java/com/apify/client/http/DefaultHttpTransport.java index ff076a5..325da87 100644 --- a/src/main/java/com/apify/client/http/DefaultHttpTransport.java +++ b/src/main/java/com/apify/client/http/DefaultHttpTransport.java @@ -1,14 +1,16 @@ package com.apify.client.http; -import java.io.IOException; import java.io.InputStream; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; /** - * The default {@link HttpTransport}, backed by the JDK's {@link java.net.http.HttpClient}. + * The default {@link HttpTransport}, backed by the JDK's {@link java.net.http.HttpClient} and its + * native non-blocking {@code sendAsync}. * *

    The per-attempt timeout is applied to each {@link HttpRequest} by the orchestrating client, so * this transport sets only a connection timeout of its own. It follows normal redirects and reuses @@ -51,21 +53,35 @@ public DefaultHttpTransport(HttpClient client) { } @Override - public HttpResponse send(HttpRequest request) throws IOException, InterruptedException { - try { - return client.send(request, HttpResponse.BodyHandlers.ofByteArray()); - } catch (java.net.http.HttpTimeoutException e) { - throw new HttpTimeoutException(e.getMessage(), e); - } + public CompletableFuture> sendAsync(HttpRequest request) { + return unwrapTimeout(client.sendAsync(request, HttpResponse.BodyHandlers.ofByteArray())); } @Override - public HttpResponse sendStreamingResponse(HttpRequest request) - throws IOException, InterruptedException { - try { - return client.send(request, HttpResponse.BodyHandlers.ofInputStream()); - } catch (java.net.http.HttpTimeoutException e) { - throw new HttpTimeoutException(e.getMessage(), e); - } + public CompletableFuture> sendStreamingAsync(HttpRequest request) { + return unwrapTimeout(client.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream())); + } + + /** + * Rewrites a {@link java.net.http.HttpTimeoutException} the JDK client fails the future with into + * this contract's transport-agnostic {@link HttpTimeoutException}, unwrapping the {@link + * CompletionException} {@code sendAsync} wraps it in. + */ + private static CompletableFuture unwrapTimeout(CompletableFuture future) { + CompletableFuture result = new CompletableFuture<>(); + future.whenComplete( + (value, error) -> { + if (error == null) { + result.complete(value); + return; + } + Throwable cause = error instanceof CompletionException ? error.getCause() : error; + if (cause instanceof java.net.http.HttpTimeoutException timeout) { + result.completeExceptionally(new HttpTimeoutException(timeout.getMessage(), timeout)); + } else { + result.completeExceptionally(cause); + } + }); + return result; } } diff --git a/src/main/java/com/apify/client/http/HttpTimeoutException.java b/src/main/java/com/apify/client/http/HttpTimeoutException.java index e446d79..f4860c0 100644 --- a/src/main/java/com/apify/client/http/HttpTimeoutException.java +++ b/src/main/java/com/apify/client/http/HttpTimeoutException.java @@ -1,18 +1,22 @@ package com.apify.client.http; -import java.io.IOException; - /** * Signals that an {@link HttpTransport} gave up on a request because it exceeded its timeout, as * opposed to some other transport failure (connection refused, DNS, ...). * *

    Part of the {@link HttpTransport} contract, not tied to any specific transport implementation: - * an implementation that can distinguish "timed out" from other I/O failures should throw this - * (wrapping or in place of the underlying exception) so the client can apply {@code - * doNotRetryTimeouts} correctly. A transport implementation that cannot tell the two apart may - * throw a plain {@link IOException} instead; the call is then simply always eligible for a retry. + * an implementation that can distinguish "timed out" from other failures should complete its future + * exceptionally with this (wrapping or in place of the underlying exception) so the client can + * apply {@code doNotRetryTimeouts} correctly. A transport implementation that cannot tell the two + * apart may fail with a plain {@link RuntimeException} instead; the call is then simply always + * eligible for a retry. + * + *

    Unchecked, matching the rest of the client's async transport contract: a {@link + * java.util.concurrent.CompletableFuture} can only be completed exceptionally with a {@link + * Throwable}, and every other exception this client throws is already unchecked (see {@link + * ApifyClientException}). */ -public class HttpTimeoutException extends IOException { +public class HttpTimeoutException extends RuntimeException { private static final long serialVersionUID = 1L; diff --git a/src/main/java/com/apify/client/http/HttpTransport.java b/src/main/java/com/apify/client/http/HttpTransport.java index 2ece3d3..074f3b1 100644 --- a/src/main/java/com/apify/client/http/HttpTransport.java +++ b/src/main/java/com/apify/client/http/HttpTransport.java @@ -1,34 +1,44 @@ package com.apify.client.http; import com.apify.client.ApifyClientBuilder; -import java.io.IOException; import java.io.InputStream; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.util.concurrent.CompletableFuture; /** - * The replaceable transport contract of the client. + * The replaceable, non-blocking transport contract of the client. * *

    Implementations are responsible only for sending a single, fully-prepared request and - * returning the raw response. Authentication, the {@code User-Agent} header, retries and + * completing the returned future with the raw response — asynchronously, never blocking the calling + * thread on the network round-trip. Authentication, the {@code User-Agent} header, retries and * (de)serialization are handled by the client, so a transport implementation only needs to perform - * one network round-trip. + * one non-blocking network round-trip. * - *

    A non-2xx HTTP status is not an error at this layer — return it as a normal {@link - * HttpResponse}. Only transport-level failures (connection refused, DNS, timeout) should be thrown. + *

    A non-2xx HTTP status is not an error at this layer — complete the future normally with + * that {@link HttpResponse}. Only a transport-level failure (connection refused, DNS, timeout) + * should complete the future exceptionally. * *

    Swap the default implementation via {@link ApifyClientBuilder#httpTransport(HttpTransport)} to * share a connection pool, customize TLS/proxy settings, or inject a mock in tests. */ public interface HttpTransport { - /** Sends a single request and buffers the whole response body as bytes. */ - HttpResponse send(HttpRequest request) throws IOException, InterruptedException; + /** + * Sends a request and buffers the whole response body as bytes, completing the returned future + * once the exchange finishes. Completes exceptionally with an {@link HttpTimeoutException} on a + * timeout; any other transport-level failure (connection refused, DNS, ...) completes it with + * whatever unchecked {@code Throwable} the transport implementation fails with - for {@link + * DefaultHttpTransport}, that is typically an {@code IOException} the JDK {@link + * java.net.http.HttpClient} itself failed with, since this contract has no checked-exception + * requirement of its own. + */ + CompletableFuture> sendAsync(HttpRequest request); /** - * Sends a single request and returns the response body as a live {@link InputStream}, for - * incremental consumption (used by log streaming). The caller closes the stream. + * Sends a request and completes the returned future with the response body as a live {@link + * InputStream}, for incremental consumption (used by log streaming). The caller closes the + * stream. Completes exceptionally the same way as {@link #sendAsync(HttpRequest)}. */ - HttpResponse sendStreamingResponse(HttpRequest request) - throws IOException, InterruptedException; + CompletableFuture> sendStreamingAsync(HttpRequest request); } diff --git a/src/main/java/com/apify/client/internal/AbstractCollectionClient.java b/src/main/java/com/apify/client/internal/AbstractCollectionClient.java index 650c7ee..ef679b9 100644 --- a/src/main/java/com/apify/client/internal/AbstractCollectionClient.java +++ b/src/main/java/com/apify/client/internal/AbstractCollectionClient.java @@ -1,7 +1,8 @@ package com.apify.client.internal; import com.apify.client.PaginationList; -import java.util.Iterator; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; import java.util.function.Consumer; import java.util.function.Supplier; @@ -34,7 +35,7 @@ protected AbstractCollectionClient( } /** Lists the collection's items for one page. {@code options} may be {@code null} (defaults). */ - public PaginationList list(O options) { + public CompletableFuture> list(O options) { O opts = options != null ? options : defaultOptions.get(); QueryParams params = new QueryParams(); opts.apply(params); @@ -42,16 +43,16 @@ public PaginationList list(O options) { } /** - * Returns a lazy iterator over the collection. The options' {@code limit} caps the total number - * yielded ({@code null} or non-positive = all); {@code chunkSize} is the per-request page size - * ({@code null} = server default). + * Returns a lazy, backpressure-aware {@link Flow.Publisher} over the collection. The options' + * {@code limit} caps the total number yielded ({@code null} or non-positive = all); {@code + * chunkSize} is the per-request page size ({@code null} = server default). */ - public Iterator iterate(O options) { + public Flow.Publisher iterate(O options) { return iterate(options, null); } /** As {@link #iterate(ListOptionsLike)}, but {@code chunkSize} sets the per-request page size. */ - public Iterator iterate(O options, Long chunkSize) { + public Flow.Publisher iterate(O options, Long chunkSize) { O opts = options != null ? options : defaultOptions.get(); return iterateWithFilters(opts.limitValue(), chunkSize, opts.offsetValue(), opts::applyFilters); } @@ -61,15 +62,15 @@ public Iterator iterate(O options, Long chunkSize) { * subclasses (e.g. {@code RunCollectionClient}) that need to merge in an extra filter ({@code * options} alone cannot express) before delegating to the same {@code ctx}/item-class. */ - protected final PaginationList listWithParams(QueryParams params) { + protected final CompletableFuture> listWithParams(QueryParams params) { return ctx.listResource("", params, itemClass); } /** * As {@link #listWithParams}, but for lazy iteration: {@code applyFilters} supplies every filter - * except {@code offset}/{@code limit}, which the iterator drives per page. + * except {@code offset}/{@code limit}, which the publisher drives per page. */ - protected final Iterator iterateWithFilters( + protected final Flow.Publisher iterateWithFilters( Long limit, Long chunkSize, Long offset, Consumer applyFilters) { return ctx.iterateResource("", limit, chunkSize, offset, applyFilters, itemClass); } diff --git a/src/main/java/com/apify/client/internal/Async.java b/src/main/java/com/apify/client/internal/Async.java new file mode 100644 index 0000000..e773c8e --- /dev/null +++ b/src/main/java/com/apify/client/internal/Async.java @@ -0,0 +1,42 @@ +package com.apify.client.internal; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +/** + * Shared scheduling primitive for the client's non-blocking retry backoff and wait-for-finish + * polling. Internal to the client. + * + *

    A single daemon-thread {@link ScheduledExecutorService} backs every delay: this is a client + * that never blocks a caller's thread waiting on the network, so a retry/poll delay is expressed as + * a scheduled {@link CompletableFuture} completion rather than {@code Thread.sleep}. The scheduler + * thread only ever fires a timer (sub-millisecond of work); the actual HTTP call that follows each + * delay runs on the JDK {@link java.net.http.HttpClient}'s own async executor, so this single + * thread is never a throughput bottleneck. + */ +public final class Async { + + private static final ThreadFactory DAEMON_THREAD_FACTORY = + runnable -> { + Thread t = new Thread(runnable, "apify-client-scheduler"); + t.setDaemon(true); + return t; + }; + + private static final ScheduledExecutorService SCHEDULER = + Executors.newSingleThreadScheduledExecutor(DAEMON_THREAD_FACTORY); + + private Async() {} + + /** Returns a future that completes after {@code delay} has elapsed (never negative/blocking). */ + public static CompletableFuture delay(Duration delay) { + CompletableFuture future = new CompletableFuture<>(); + long millis = Math.max(0, delay.toMillis()); + SCHEDULER.schedule(() -> future.complete(null), millis, TimeUnit.MILLISECONDS); + return future; + } +} diff --git a/src/main/java/com/apify/client/internal/AsyncPaginatedPublisher.java b/src/main/java/com/apify/client/internal/AsyncPaginatedPublisher.java new file mode 100644 index 0000000..39ae2a0 --- /dev/null +++ b/src/main/java/com/apify/client/internal/AsyncPaginatedPublisher.java @@ -0,0 +1,277 @@ +package com.apify.client.internal; + +import com.apify.client.PaginationList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A lazy, backpressure-aware {@link Flow.Publisher} over an offset/limit-paginated list endpoint, + * fetching one page at a time as demand allows. Internal reusable engine shared by every + * offset/limit-paginated collection's {@code iterate} method, so the paging arithmetic lives in one + * place (DRY). (The cursor-based {@code iterateKeys} and the single-fetch {@code versions}/{@code + * env-vars} iterators do not use it.) + * + *

    This is the client's "reactive" iteration primitive: a {@link java.util.concurrent.Flow} + * ({@code java.util.concurrent.Flow}, the JDK's built-in Reactive Streams interfaces) publisher, + * chosen over a blocking {@link java.util.Iterator} so that following a large paginated collection + * never blocks a thread waiting on the next page's network round-trip. Pages are fetched only as + * the subscriber signals demand via {@link Flow.Subscription#request(long)}. + * + *

    Behaviour, mirroring the end-user contract of the reference JS client's iterable {@code + * list()}: + * + *

      + *
    • {@code totalLimit} caps the total number of items yielded across all pages ({@code + * null} = yield everything). + *
    • {@code chunkSize} is the per-request page size ({@code null} = let the API choose). Each + * page requests {@code min(remaining-under-cap, chunkSize)} items so the cap is never + * overshot. + *
    + * + *

    Each page advances the offset by the number of items actually returned, and iteration stops + * only when the cap is reached or the API returns an empty page. It deliberately does not + * stop on a short page or a reported {@code total}: the API clamps an over-large requested page + * size to its own maximum (so a "short" page is common mid-collection), and some endpoints report a + * {@code total} of {@code 0} or a value that lags right after a write (e.g. dataset items) — either + * signal would truncate iteration. Terminating on an empty page costs one extra request at the end + * but yields the complete result in every case. + * + *

    Supports a single subscriber (as permitted by the Reactive Streams specification); subscribing + * twice completes the second subscriber's subscription with an error instead of sharing state + * between them. + */ +public final class AsyncPaginatedPublisher implements Flow.Publisher { + + /** Fetches a single page starting at {@code offset}, requesting at most {@code limit} items. */ + @FunctionalInterface + public interface PageFetcher { + CompletableFuture> fetch(long offset, Long limit); + } + + private final PageFetcher fetcher; + private final Long totalLimit; + private final Long chunkSize; + private final Long startOffset; + private final AtomicBoolean subscribed = new AtomicBoolean(); + + public AsyncPaginatedPublisher( + Long totalLimit, Long chunkSize, Long startOffset, PageFetcher fetcher) { + this.totalLimit = totalLimit != null && totalLimit > 0 ? totalLimit : null; + this.chunkSize = chunkSize != null && chunkSize > 0 ? chunkSize : null; + this.startOffset = startOffset != null && startOffset > 0 ? startOffset : 0; + this.fetcher = fetcher; + } + + @Override + public void subscribe(Flow.Subscriber subscriber) { + if (!subscribed.compareAndSet(false, true)) { + subscriber.onSubscribe(NoopSubscription.INSTANCE); + subscriber.onError( + new IllegalStateException( + "This publisher supports only a single subscriber (already subscribed)")); + return; + } + Session session = new Session(subscriber); + subscriber.onSubscribe(session); + } + + /** + * A subscription with no demand, used to reject a second subscriber per Reactive Streams §1.9. + */ + private enum NoopSubscription implements Flow.Subscription { + INSTANCE; + + @Override + public void request(long n) {} + + @Override + public void cancel() {} + } + + /** + * Per-subscriber paging state and the {@link Flow.Subscription} driving it. Not shared across + * subscribers - each {@link #subscribe} call gets its own instance. + */ + private final class Session implements Flow.Subscription { + private final Flow.Subscriber subscriber; + private final AtomicLong requested = new AtomicLong(); + private final AtomicBoolean draining = new AtomicBoolean(); + private final AtomicBoolean cancelled = new AtomicBoolean(); + private final AtomicBoolean terminated = new AtomicBoolean(); + + // Only ever touched from within a `draining`-guarded section, which - by construction below - + // never runs concurrently with itself (the flag admits only one drain loop at a time, and every + // continuation after an async page fetch re-enters through the same guarded path), so no + // separate synchronization on these fields is needed. + private List buffer = List.of(); + private int pos; + private long offset; + private long yielded; + private boolean exhausted; + private boolean fetchInFlight; + + Session(Flow.Subscriber subscriber) { + this.subscriber = subscriber; + this.offset = startOffset; + } + + @Override + public void request(long n) { + if (n <= 0) { + // Deliberately does not go through terminate(): request() is called from outside the + // drain loop (it does not hold the `draining` guard), so - unlike the drain loop's own + // terminal paths - it must not release a guard it may not own. Releasing it here would + // let a concurrent request() elsewhere win the drain() CAS and start a second drain loop + // in parallel with a possibly still-running one, racing on the unsynchronized paging + // fields (mirrors ListPublisher.Session's identical n<=0 handling). + if (terminated.compareAndSet(false, true)) { + subscriber.onError( + new IllegalArgumentException( + "Reactive Streams violation: requested amount must be positive, was " + n)); + } + return; + } + requested.updateAndGet(current -> addSaturating(current, n)); + drain(); + } + + @Override + public void cancel() { + cancelled.set(true); + } + + /** Enters the drain loop unless another drain (of this same session) is already running. */ + private void drain() { + if (draining.compareAndSet(false, true)) { + drainLoop(); + } + } + + /** + * Emits buffered items while there is demand, fetches the next page when the buffer runs dry + * (or completes the subscriber when the collection is exhausted), and always releases the + * {@code draining} flag exactly once per entry - either directly, or (when a page fetch is + * started) from that fetch's completion callback, which re-enters this same method. + */ + private void drainLoop() { + while (!cancelled.get() && !terminated.get() && requested.get() > 0 && pos < buffer.size()) { + T item = buffer.get(pos++); + requested.decrementAndGet(); + subscriber.onNext(item); + } + if (cancelled.get() || terminated.get()) { + // `terminated` here means a reentrant request(n<=0) fired onError from within the onNext + // call just above (a subscriber violating Reactive Streams §3.9); stop emitting immediately + // instead of continuing to drain the rest of the buffered demand into a subscriber that has + // already received a terminal signal (RS §1.7). + draining.set(false); + return; + } + if (pos < buffer.size()) { + // Demand exhausted (requested == 0) with items still buffered: stop until more is + // requested. + draining.set(false); + // Re-check demand: a request() on another thread may have arrived between the loop's last + // check and this flag release, which would otherwise be a missed wakeup. + if (requested.get() > 0 && pos < buffer.size()) { + drain(); + } + return; + } + if (exhausted) { + complete(); + return; + } + if (fetchInFlight) { + // A fetch from an earlier drain() is still outstanding; its completion will resume this + // loop. With a well-behaved subscriber this is unreachable (the flag is only set/cleared + // within this draining-guarded section, so no concurrent drain can normally observe it + // true here) - but it is a real safety net, not dead code: request(n<=0) no longer + // releases `draining` on its own (see the comment there), yet a subscriber that itself + // races two concurrent request() calls could still reach this branch, and it correctly + // defers to the in-flight fetch's own completion instead of starting a second one. + draining.set(false); + return; + } + fetchNextPage(); + } + + private void fetchNextPage() { + fetchInFlight = true; + Long capRemaining = totalLimit != null ? totalLimit - yielded : null; + Long pageLimit = minForLimit(capRemaining, chunkSize); + fetcher + .fetch(offset, pageLimit) + .whenComplete( + (page, error) -> { + fetchInFlight = false; + if (cancelled.get()) { + draining.set(false); + return; + } + if (error != null) { + terminate(() -> subscriber.onError(HttpClientCore.unwrapCompletion(error))); + return; + } + applyPage(page); + drainLoop(); + }); + } + + private void applyPage(PaginationList page) { + buffer = page.getItems(); + pos = 0; + int count = buffer.size(); + offset += count; + yielded += count; + // Defensively trim the last page to the cap in case the server returned more than requested, + // so the subscriber never sees more than `totalLimit` items. + if (totalLimit != null && yielded > totalLimit) { + buffer = buffer.subList(0, count - (int) (yielded - totalLimit)); + yielded = totalLimit; + } + // Stop on the cap or an empty page; never on a short page (the API clamps large page sizes) + // or the reported total (unreliable on some endpoints — see class doc). + if (count == 0 || (totalLimit != null && yielded >= totalLimit)) { + exhausted = true; + } + } + + private void complete() { + terminate(subscriber::onComplete); + } + + /** Runs a terminal subscriber callback at most once, then releases the drain guard. */ + private void terminate(Runnable callback) { + if (terminated.compareAndSet(false, true)) { + callback.run(); + } + draining.set(false); + } + } + + /** + * Returns the smaller of the two per-page bounds, or {@code null} (server default) when neither + * is set. Both inputs are already positive-or-{@code null}: {@code chunkSize} is normalized in + * the constructor and {@code capRemaining} is only ever {@code > 0} here (the {@code exhausted} + * guard blocks fetching once the cap is reached). + */ + private static Long minForLimit(Long a, Long b) { + if (a == null) { + return b; + } + if (b == null) { + return a; + } + return Math.min(a, b); + } + + /** Adds {@code b} to {@code a}, saturating at {@link Long#MAX_VALUE} instead of overflowing. */ + private static long addSaturating(long a, long b) { + long sum = a + b; + return sum < 0 ? Long.MAX_VALUE : sum; + } +} diff --git a/src/main/java/com/apify/client/internal/HttpClientCore.java b/src/main/java/com/apify/client/internal/HttpClientCore.java index dd44ce9..be91d15 100644 --- a/src/main/java/com/apify/client/internal/HttpClientCore.java +++ b/src/main/java/com/apify/client/internal/HttpClientCore.java @@ -17,6 +17,9 @@ import java.time.Duration; import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadLocalRandom; import java.util.zip.GZIPOutputStream; @@ -24,6 +27,11 @@ * The orchestrating HTTP client shared by every resource client. It owns the transport, the * optional API token, the {@code User-Agent}, and the retry/timeout policy, and applies them to * every request. Internal to the client; safe for concurrent use. + * + *

    Every call is non-blocking end to end: a request never occupies the calling thread while + * waiting on the network, a retryable failure's backoff delay is a scheduled timer (see {@link + * Async}) rather than {@code Thread.sleep}, and each retry attempt is chained via {@link + * CompletableFuture#thenCompose} rather than looped over synchronously. */ public final class HttpClientCore { @@ -120,7 +128,7 @@ public Duration baseRequestTimeout() { } /** Sends a request with auth, User-Agent and the retry policy applied. */ - public ApiResponse call( + public CompletableFuture call( String method, String url, byte[] body, String contentType, Duration baseTimeout) { return call(method, url, body, contentType, null, baseTimeout, false); } @@ -131,7 +139,7 @@ public ApiResponse call( * rather than being retried (other transport/network errors and retryable statuses are still * retried). */ - public ApiResponse call( + public CompletableFuture call( String method, String url, byte[] body, @@ -145,7 +153,7 @@ public ApiResponse call( * Like {@link #call(String, String, byte[], String, Duration)} but additionally sets the given * extra headers on every attempt. */ - public ApiResponse call( + public CompletableFuture call( String method, String url, byte[] body, @@ -156,7 +164,7 @@ public ApiResponse call( } /** The canonical overload every other {@code call} convenience overload delegates to. */ - public ApiResponse call( + public CompletableFuture call( String method, String url, byte[] body, @@ -175,41 +183,109 @@ public ApiResponse call( headers.put(CONTENT_ENCODING_HEADER, compressed.encoding()); } - Duration delay = retry.minDelayBetweenRetries; - int maxAttempts = retry.maxRetries + 1; String path = extractPath(url); - RuntimeException lastError = null; - - for (int attempt = 1; attempt <= maxAttempts; attempt++) { - boolean retryable; - try { - ApiResponse resp = - doAttempt( - method, - url, - requestBody, - contentType, - headers, - attemptTimeout(baseTimeout, attempt)); - if (resp.statusCode() < MAX_SUCCESS_STATUS) { - return resp; - } - lastError = buildApiError(resp.statusCode(), resp.body(), attempt, method, path); - retryable = isStatusRetryable(resp.statusCode()); - } catch (ApifyTransportException e) { - lastError = e; - // Network/timeout failures are retryable, unless the caller opted out of retrying timeouts. - retryable = !(doNotRetryTimeouts && e.isTimeout()); - } + int maxAttempts = retry.maxRetries + 1; + return attemptWithRetry( + method, + url, + requestBody, + contentType, + headers, + baseTimeout, + doNotRetryTimeouts, + path, + 1, + maxAttempts, + retry.minDelayBetweenRetries); + } - if (!retryable || attempt == maxAttempts) { - throw lastError; - } + /** + * Runs one attempt and, on a retryable failure that has not exhausted {@code maxAttempts}, + * schedules the next one after a backoff delay - chained via {@link + * CompletableFuture#thenCompose} rather than a blocking loop, so no thread is ever parked waiting + * on the network or the delay. + */ + private CompletableFuture attemptWithRetry( + String method, + String url, + byte[] body, + String contentType, + Map headers, + Duration baseTimeout, + boolean doNotRetryTimeouts, + String path, + int attempt, + int maxAttempts, + Duration delay) { + Duration timeout = attemptTimeout(baseTimeout, attempt); + return attempt( + method, url, body, contentType, headers, timeout, doNotRetryTimeouts, attempt, path) + .thenCompose( + outcome -> { + if (outcome.success()) { + return CompletableFuture.completedFuture(outcome.response()); + } + if (!outcome.retryable() || attempt == maxAttempts) { + return failedFuture(outcome.error()); + } + Duration sleepFor = randomizedDelay(delay); + Duration nextDelay = + minDuration(delay.multipliedBy(BACKOFF_FACTOR), retry.maxDelayBetweenRetries); + return Async.delay(sleepFor) + .thenCompose( + unused -> + attemptWithRetry( + method, + url, + body, + contentType, + headers, + baseTimeout, + doNotRetryTimeouts, + path, + attempt + 1, + maxAttempts, + nextDelay)); + }); + } - sleep(randomizedDelay(delay)); - delay = minDuration(delay.multipliedBy(BACKOFF_FACTOR), retry.maxDelayBetweenRetries); - } - throw lastError; // unreachable in practice (maxAttempts >= 1), defensive + /** The outcome of a single attempt: either a success response, or a retryable/terminal error. */ + private record AttemptOutcome( + ApiResponse response, boolean success, boolean retryable, RuntimeException error) {} + + /** Sends one attempt and classifies the result (success / retryable error / terminal error). */ + private CompletableFuture attempt( + String method, + String url, + byte[] body, + String contentType, + Map extraHeaders, + Duration timeout, + boolean doNotRetryTimeouts, + int attemptNum, + String path) { + return doAttempt(method, url, body, contentType, extraHeaders, timeout) + .handle( + (resp, err) -> { + if (err == null) { + if (resp.statusCode() < MAX_SUCCESS_STATUS) { + return new AttemptOutcome(resp, true, false, null); + } + RuntimeException apiError = + buildApiError(resp.statusCode(), resp.body(), attemptNum, method, path); + return new AttemptOutcome( + null, false, isStatusRetryable(resp.statusCode()), apiError); + } + Throwable cause = unwrapCompletion(err); + ApifyTransportException transportError = + cause instanceof ApifyTransportException apifyTransportException + ? apifyTransportException + : new ApifyTransportException(cause); + // Network/timeout failures are retryable, unless the caller opted out of retrying + // timeouts. + boolean retryable = !(doNotRetryTimeouts && transportError.isTimeout()); + return new AttemptOutcome(null, false, retryable, transportError); + }); } /** @@ -241,7 +317,7 @@ HttpRequest buildRequest( return b.build(); } - private ApiResponse doAttempt( + private CompletableFuture doAttempt( String method, String url, byte[] body, @@ -249,15 +325,9 @@ private ApiResponse doAttempt( Map extraHeaders, Duration timeout) { HttpRequest request = buildRequest(method, url, body, contentType, extraHeaders, timeout); - try { - HttpResponse resp = transport.send(request); - return new ApiResponse(resp.statusCode(), resp.headers(), resp.body()); - } catch (IOException e) { - throw new ApifyTransportException(e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new ApifyTransportException(e); - } + return transport + .sendAsync(request) + .thenApply(resp -> new ApiResponse(resp.statusCode(), resp.headers(), resp.body())); } /** @@ -362,23 +432,34 @@ private static Duration randomizedDelay(Duration delay) { return Duration.ofMillis(millis + ThreadLocalRandom.current().nextLong(millis)); } - private static void sleep(Duration d) { - try { - Thread.sleep(Math.max(0, d.toMillis())); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new ApifyTransportException(e); - } - } - private static Duration minDuration(Duration a, Duration b) { return a.compareTo(b) < 0 ? a : b; } + private static CompletableFuture failedFuture(RuntimeException error) { + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(error); + return future; + } + + /** + * Unwraps the {@link CompletionException}/{@link ExecutionException} layers {@link + * CompletableFuture} plumbing may add around the original failure, so callers always classify + * (and wrap) the real cause rather than the wrapper. + */ + public static Throwable unwrapCompletion(Throwable t) { + Throwable current = t; + while ((current instanceof CompletionException || current instanceof ExecutionException) + && current.getCause() != null) { + current = current.getCause(); + } + return current; + } + /** * The {@code {"error": {...}}} envelope the API sends on a non-success response, mapped directly * by Jackson rather than navigated field-by-field as a raw {@link - * com.fasterxml.jackson.databind.JsonNode} tree. + * tools.jackson.databind.JsonNode} tree. */ private static final class ErrorEnvelope { public ErrorBody error; @@ -425,16 +506,25 @@ public static String extractPath(String url) { } } - /** Opens a live streaming response (single attempt, no retry). Used by log streaming. */ - public HttpResponse stream(String url) { + /** + * Opens a live streaming response (single attempt, no retry). Used by log streaming. The returned + * future completes exceptionally with an {@link ApifyTransportException} on a transport failure; + * a non-2xx status is returned as a normal response for the caller (log streaming) to inspect, + * matching {@link HttpTransport}'s contract. + */ + public CompletableFuture> streamAsync(String url) { HttpRequest request = buildRequest("GET", url, null, null, null, retry.timeout); - try { - return transport.sendStreamingResponse(request); - } catch (IOException e) { - throw new ApifyTransportException(e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new ApifyTransportException(e); - } + CompletableFuture> result = new CompletableFuture<>(); + transport + .sendStreamingAsync(request) + .whenComplete( + (resp, err) -> { + if (err == null) { + result.complete(resp); + } else { + result.completeExceptionally(new ApifyTransportException(unwrapCompletion(err))); + } + }); + return result; } } diff --git a/src/main/java/com/apify/client/internal/Json.java b/src/main/java/com/apify/client/internal/Json.java index bf0aecf..2933554 100644 --- a/src/main/java/com/apify/client/internal/Json.java +++ b/src/main/java/com/apify/client/internal/Json.java @@ -2,15 +2,13 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import java.io.IOException; -import java.io.UncheckedIOException; +import tools.jackson.core.JacksonException; +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.DeserializationFeature; +import tools.jackson.databind.JavaType; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.cfg.DateTimeFeature; +import tools.jackson.databind.json.JsonMapper; /** * Shared JSON (de)serialization for the client, centered on a single configured {@link @@ -19,48 +17,56 @@ *

    The mapper ignores unknown properties (models also collect them in an {@code extra} map for * forward compatibility), renders dates as ISO-8601 strings, and omits {@code null} fields when * serializing request bodies. + * + *

    Built with Jackson 3's immutable {@link JsonMapper.Builder}: unlike Jackson 2's mutable {@code + * ObjectMapper}, every setting is fixed at construction time, so the resulting mapper needs no + * further synchronization to be safely shared across threads. Java-time (de)serialization ({@code + * Instant}, {@code Duration}, ...) is built into jackson-databind itself in Jackson 3, so (unlike + * Jackson 2) no separate {@code JavaTimeModule} registration is needed. */ public final class Json { static final ObjectMapper MAPPER = - new ObjectMapper() - .registerModule(new JavaTimeModule()) + JsonMapper.builder() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false) - .setSerializationInclusion(JsonInclude.Include.NON_NULL) + // Several model classes use primitive numeric fields (long/double) for API values that + // are usually present but can legitimately be absent/null early in a resource's + // lifecycle (e.g. ActorRunStats fields before a run's stats are fully populated). + // Jackson 2 coerced a JSON null into a primitive's zero value by default; Jackson 3 does + // not, so this is set explicitly to keep that lenient, non-throwing behavior against the + // live API instead of every such field having to be widened to a boxed type. + .configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false) + .configure(DateTimeFeature.WRITE_DATES_AS_TIMESTAMPS, false) + .changeDefaultPropertyInclusion( + incl -> incl.withValueInclusion(JsonInclude.Include.NON_NULL)) // Bind directly to (private) fields so models need no setters — getters remain the // public read surface. Any-getters/setters are still honored for the `extra` map. - .setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY) - .setVisibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.NONE) - .setVisibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE); + .changeDefaultVisibility( + vc -> + vc.withFieldVisibility(JsonAutoDetect.Visibility.ANY) + .withGetterVisibility(JsonAutoDetect.Visibility.NONE) + .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE)) + .build(); private Json() {} - /** Serializes a value to JSON bytes. */ + /** + * Serializes a value to JSON bytes. + * + * @throws JacksonException (unchecked, per Jackson 3) if serialization fails + */ public static byte[] toBytes(Object value) { - try { - return MAPPER.writeValueAsBytes(value); - } catch (IOException e) { - throw new UncheckedIOException("failed to serialize request body", e); - } + return MAPPER.writeValueAsBytes(value); } /** Parses JSON bytes into the given type. */ public static T parse(byte[] body, JavaType type) { - try { - return MAPPER.readValue(body, type); - } catch (IOException e) { - throw new UncheckedIOException("failed to parse API response", e); - } + return MAPPER.readValue(body, type); } /** Parses JSON bytes into the given class. */ public static T parse(byte[] body, Class type) { - try { - return MAPPER.readValue(body, type); - } catch (IOException e) { - throw new UncheckedIOException("failed to parse API response", e); - } + return MAPPER.readValue(body, type); } /** @@ -74,18 +80,14 @@ static T tryParse(byte[] body, Class type) { } try { return MAPPER.readValue(body, type); - } catch (IOException e) { + } catch (JacksonException e) { return null; } } /** Parses JSON bytes into the given {@link TypeReference} (for generic types). */ public static T parse(byte[] body, TypeReference type) { - try { - return MAPPER.readValue(body, type); - } catch (IOException e) { - throw new UncheckedIOException("failed to parse API response", e); - } + return MAPPER.readValue(body, type); } /** Constructs a {@link JavaType} for a raw class. */ diff --git a/src/main/java/com/apify/client/internal/ListPublisher.java b/src/main/java/com/apify/client/internal/ListPublisher.java new file mode 100644 index 0000000..05f38a3 --- /dev/null +++ b/src/main/java/com/apify/client/internal/ListPublisher.java @@ -0,0 +1,142 @@ +package com.apify.client.internal; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A backpressure-aware {@link Flow.Publisher} over an already-in-flight, single-fetch {@link List} + * - for the handful of "collections" (Actor version env-vars, Actor versions) that are not actually + * offset/limit-paginated on the server: it always returns every item in one response, so routing it + * through {@link AsyncPaginatedPublisher}'s repeat-until-empty-page paging engine would fetch (and + * re-emit) the same full list forever. This fetches once, then drains the resulting list to the + * subscriber according to its signalled demand. + * + *

    Supports a single subscriber, like {@link AsyncPaginatedPublisher}. + */ +public final class ListPublisher implements Flow.Publisher { + + private final CompletableFuture> source; + private final AtomicBoolean subscribed = new AtomicBoolean(); + + public ListPublisher(CompletableFuture> source) { + this.source = source; + } + + @Override + public void subscribe(Flow.Subscriber subscriber) { + if (!subscribed.compareAndSet(false, true)) { + subscriber.onSubscribe(NoopSubscription.INSTANCE); + subscriber.onError( + new IllegalStateException( + "This publisher supports only a single subscriber (already subscribed)")); + return; + } + subscriber.onSubscribe(new Session(subscriber)); + } + + private enum NoopSubscription implements Flow.Subscription { + INSTANCE; + + @Override + public void request(long n) {} + + @Override + public void cancel() {} + } + + private final class Session implements Flow.Subscription { + private final Flow.Subscriber subscriber; + private final AtomicLong requested = new AtomicLong(); + private final AtomicBoolean draining = new AtomicBoolean(); + private final AtomicBoolean cancelled = new AtomicBoolean(); + private final AtomicBoolean terminated = new AtomicBoolean(); + + // Only touched from within the draining-guarded section; see AsyncPaginatedPublisher.Session. + private List items; + private int pos; + + Session(Flow.Subscriber subscriber) { + this.subscriber = subscriber; + } + + @Override + public void request(long n) { + if (n <= 0) { + if (terminated.compareAndSet(false, true)) { + subscriber.onError( + new IllegalArgumentException( + "Reactive Streams violation: requested amount must be positive, was " + n)); + } + return; + } + requested.updateAndGet(current -> addSaturating(current, n)); + drain(); + } + + @Override + public void cancel() { + cancelled.set(true); + } + + private void drain() { + if (draining.compareAndSet(false, true)) { + if (items == null) { + source.whenComplete( + (list, error) -> { + if (cancelled.get()) { + draining.set(false); + return; + } + if (error != null) { + terminate(() -> subscriber.onError(HttpClientCore.unwrapCompletion(error))); + return; + } + items = list; + drainLoop(); + }); + } else { + drainLoop(); + } + } + } + + private void drainLoop() { + while (!cancelled.get() && !terminated.get() && requested.get() > 0 && pos < items.size()) { + T item = items.get(pos++); + requested.decrementAndGet(); + subscriber.onNext(item); + } + if (cancelled.get() || terminated.get()) { + // `terminated` here means a reentrant request(n<=0) fired onError from within the onNext + // call just above (a subscriber violating Reactive Streams §3.9); stop emitting immediately + // instead of continuing to drain the rest of the buffered demand into a subscriber that has + // already received a terminal signal (RS §1.7). Mirrors AsyncPaginatedPublisher.Session. + draining.set(false); + return; + } + if (pos >= items.size()) { + terminate(subscriber::onComplete); + return; + } + draining.set(false); + if (requested.get() > 0 && pos < items.size()) { + drain(); + } + } + + private void terminate(Runnable callback) { + if (terminated.compareAndSet(false, true)) { + callback.run(); + } + draining.set(false); + } + } + + private static long addSaturating(long a, long b) { + long sum = a + b; + return sum < 0 ? Long.MAX_VALUE : sum; + } +} diff --git a/src/main/java/com/apify/client/internal/PaginatedIterator.java b/src/main/java/com/apify/client/internal/PaginatedIterator.java deleted file mode 100644 index f9af796..0000000 --- a/src/main/java/com/apify/client/internal/PaginatedIterator.java +++ /dev/null @@ -1,115 +0,0 @@ -package com.apify.client.internal; - -import com.apify.client.PaginationList; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; - -/** - * A lazy {@link Iterator} over an offset/limit-paginated list endpoint, fetching one page at a - * time. Internal reusable engine shared by every offset/limit-paginated collection's {@code - * iterate} method, so the paging arithmetic lives in one place (DRY). (The cursor-based {@code - * iterateKeys} and the single-fetch {@code versions}/{@code env-vars} iterators do not use it.) - * - *

    Behaviour, mirroring the end-user contract of the reference JS client's iterable {@code - * list()}: - * - *

      - *
    • {@code totalLimit} caps the total number of items yielded across all pages ({@code - * null} = yield everything). - *
    • {@code chunkSize} is the per-request page size ({@code null} = let the API choose). Each - * page requests {@code min(remaining-under-cap, chunkSize)} items so the cap is never - * overshot. - *
    - * - *

    Each page advances the offset by the number of items actually returned, and iteration stops - * only when the cap is reached or the API returns an empty page. It deliberately does not - * stop on a short page or a reported {@code total}: the API clamps an over-large requested page - * size to its own maximum (so a "short" page is common mid-collection), and some endpoints report a - * {@code total} of {@code 0} or a value that lags right after a write (e.g. dataset items) — either - * signal would truncate iteration. Terminating on an empty page costs one extra request at the end - * but yields the complete result in every case. - */ -public final class PaginatedIterator implements Iterator { - - /** Fetches a single page starting at {@code offset}, requesting at most {@code limit} items. */ - @FunctionalInterface - public interface PageFetcher { - PaginationList fetch(long offset, Long limit); - } - - private final PageFetcher fetcher; - private final Long totalLimit; - private final Long chunkSize; - - private List buffer = List.of(); - private int pos; - private long offset; - private long yielded; - private boolean exhausted; - - public PaginatedIterator( - Long totalLimit, Long chunkSize, Long startOffset, PageFetcher fetcher) { - this.totalLimit = totalLimit != null && totalLimit > 0 ? totalLimit : null; - this.chunkSize = chunkSize != null && chunkSize > 0 ? chunkSize : null; - this.offset = startOffset != null && startOffset > 0 ? startOffset : 0; - this.fetcher = fetcher; - } - - @Override - public boolean hasNext() { - while (pos >= buffer.size()) { - if (exhausted) { - return false; - } - fetchNextPage(); - } - return true; - } - - @Override - public T next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - return buffer.get(pos++); - } - - private void fetchNextPage() { - Long capRemaining = totalLimit != null ? totalLimit - yielded : null; - Long pageLimit = minForLimit(capRemaining, chunkSize); - PaginationList page = fetcher.fetch(offset, pageLimit); - buffer = page.getItems(); - pos = 0; - int count = buffer.size(); - offset += count; - yielded += count; - // Defensively trim the last page to the cap in case the server returned more than requested, so - // the caller never sees more than `totalLimit` items. - if (totalLimit != null && yielded > totalLimit) { - buffer = buffer.subList(0, count - (int) (yielded - totalLimit)); - yielded = totalLimit; - } - // Stop on the caller's cap or an empty page; never on a short page (the API clamps large page - // sizes) or the reported total (unreliable on some endpoints — see class doc). - if (count == 0 || (totalLimit != null && yielded >= totalLimit)) { - exhausted = true; - } - } - - /** - * Returns the smaller of the two per-page bounds, or {@code null} (server default) when neither - * is set. Both inputs are already positive-or-{@code null}: {@code chunkSize} is normalized in - * the constructor and {@code capRemaining} is only ever {@code > 0} here (the {@code exhausted} - * guard blocks fetching once the cap is reached). - */ - private static Long minForLimit(Long a, Long b) { - if (a == null) { - return b; - } - if (b == null) { - return a; - } - return Math.min(a, b); - } -} diff --git a/src/main/java/com/apify/client/internal/ResourceContext.java b/src/main/java/com/apify/client/internal/ResourceContext.java index b43fe97..9b521ff 100644 --- a/src/main/java/com/apify/client/internal/ResourceContext.java +++ b/src/main/java/com/apify/client/internal/ResourceContext.java @@ -4,20 +4,25 @@ import com.apify.client.http.ApiResponse; import com.apify.client.http.ApifyApiException; import com.apify.client.http.ApifyTransportException; -import com.fasterxml.jackson.databind.JavaType; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.Iterator; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; import java.util.function.Consumer; import java.util.function.Predicate; +import tools.jackson.databind.JavaType; /** * The resolved context for a resource client: its base URL and the shared HTTP client. The methods * here implement the CRUD primitives once, so each resource client stays small and consistent * (DRY). Internal to the client. + * + *

    Every primitive is non-blocking: it returns a {@link CompletableFuture} (or, for a paginated + * collection's lazy iteration, a {@link Flow.Publisher}) rather than blocking the calling thread on + * the underlying HTTP exchange. */ public final class ResourceContext { @@ -160,65 +165,81 @@ public ResourceContext seedParams(QueryParams inherited) { // ---- CRUD primitives ------------------------------------------------------ - public Optional getResource(String subPath, QueryParams params, JavaType dataType) { - try { - // ofNullable, not of: an HTTP 200 with body {"data": null} unwraps to null, which is a valid - // "no resource" answer rather than a programming error — never surface it as a raw NPE. - return Optional.ofNullable(getResourceRequired(subPath, params, dataType)); - } catch (ApifyApiException e) { - if (isNotFound(e)) { - return Optional.empty(); - } - throw e; - } - } - - public Optional getResource(String subPath, QueryParams params, Class dataClass) { + public CompletableFuture> getResource( + String subPath, QueryParams params, JavaType dataType) { + // ofNullable, not of: an HTTP 200 with body {"data": null} unwraps to null, which is a valid + // "no resource" answer rather than a programming error — never surface it as a raw NPE. + // Explicit witness: getResourceRequired is the *receiver* of .handle below (not an argument + // to another generic call), so unlike a direct `return`, its type parameter cannot be inferred + // from the surrounding target type and must be pinned here. + return this.getResourceRequired(subPath, params, dataType) + .handle( + (value, error) -> { + if (error == null) { + return Optional.ofNullable(value); + } + Throwable cause = HttpClientCore.unwrapCompletion(error); + if (cause instanceof ApifyApiException apiError && isNotFound(apiError)) { + return Optional.empty(); + } + throw asRuntimeException(cause); + }); + } + + public CompletableFuture> getResource( + String subPath, QueryParams params, Class dataClass) { return getResource(subPath, params, Json.type(dataClass)); } - public T getResourceRequired(String subPath, QueryParams params, JavaType dataType) { + public CompletableFuture getResourceRequired( + String subPath, QueryParams params, JavaType dataType) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - ApiResponse resp = http.call("GET", u, null, "", http.baseRequestTimeout()); - return Json.parseData(resp.body(), dataType); + return http.call("GET", u, null, "", http.baseRequestTimeout()) + .thenApply(resp -> Json.parseData(resp.body(), dataType)); } - public T getResourceRequired(String subPath, QueryParams params, Class dataClass) { + public CompletableFuture getResourceRequired( + String subPath, QueryParams params, Class dataClass) { return getResourceRequired(subPath, params, Json.type(dataClass)); } - public T updateResource(String subPath, Object body, Class dataClass) { + public CompletableFuture updateResource(String subPath, Object body, Class dataClass) { String u = mergedParams(new QueryParams()).applyToUrl(subUrl(subPath)); - ApiResponse resp = - http.call("PUT", u, Json.toBytes(body), CONTENT_TYPE_JSON, http.baseRequestTimeout()); - return Json.parseData(resp.body(), dataClass); + return http.call("PUT", u, Json.toBytes(body), CONTENT_TYPE_JSON, http.baseRequestTimeout()) + .thenApply(resp -> Json.parseData(resp.body(), dataClass)); } /** Performs a DELETE; a not-found is treated as a successful no-op. */ - public void deleteResource(String subPath) { + public CompletableFuture deleteResource(String subPath) { String u = mergedParams(new QueryParams()).applyToUrl(subUrl(subPath)); - try { - http.call("DELETE", u, null, "", http.baseRequestTimeout()); - } catch (ApifyApiException e) { - if (!isNotFound(e)) { - throw e; - } - } - } - - public PaginationList listResource( + return http.call("DELETE", u, null, "", http.baseRequestTimeout()) + .handle( + (resp, error) -> { + if (error == null) { + return null; + } + Throwable cause = HttpClientCore.unwrapCompletion(error); + if (cause instanceof ApifyApiException apiError && isNotFound(apiError)) { + return null; + } + throw asRuntimeException(cause); + }); + } + + public CompletableFuture> listResource( String subPath, QueryParams params, Class itemClass) { JavaType listType = Json.parametric(PaginationList.class, Json.type(itemClass)); return getResourceRequired(subPath, params, listType); } /** - * Builds a lazy iterator over an offset/limit-paginated endpoint. {@code applyFilters} adds the - * per-endpoint filter params (everything except {@code offset}/{@code limit}, which the iterator - * drives per page). {@code totalLimit} caps the total items yielded; {@code chunkSize} is the - * page size (both {@code null} meaning "unbounded" / "server default"). + * Builds a lazy, backpressure-aware {@link Flow.Publisher} over an offset/limit-paginated + * endpoint. {@code applyFilters} adds the per-endpoint filter params (everything except {@code + * offset}/{@code limit}, which the publisher drives per page). {@code totalLimit} caps the total + * items yielded; {@code chunkSize} is the page size (both {@code null} meaning "unbounded" / + * "server default"). */ - public Iterator iterateResource( + public Flow.Publisher iterateResource( String subPath, Long totalLimit, Long chunkSize, @@ -226,10 +247,10 @@ public Iterator iterateResource( Consumer applyFilters, Class itemClass) { // Snapshot the caller's filters once, so mutating the options object mid-iteration cannot leak - // into later pages (the iterator owns an independent copy of every filter, offset and limit). + // into later pages (the publisher owns an independent copy of every filter, offset and limit). QueryParams filters = new QueryParams(); applyFilters.accept(filters); - return new PaginatedIterator<>( + return new AsyncPaginatedPublisher<>( totalLimit, chunkSize, startOffset, @@ -240,15 +261,15 @@ public Iterator iterateResource( }); } - public T createResource(QueryParams params, Object body, Class dataClass) { + public CompletableFuture createResource( + QueryParams params, Object body, Class dataClass) { String u = mergedParams(params).applyToUrl(subUrl("")); - ApiResponse resp = - http.call("POST", u, Json.toBytes(body), CONTENT_TYPE_JSON, http.baseRequestTimeout()); - return Json.parseData(resp.body(), dataClass); + return http.call("POST", u, Json.toBytes(body), CONTENT_TYPE_JSON, http.baseRequestTimeout()) + .thenApply(resp -> Json.parseData(resp.body(), dataClass)); } /** POST that gets-or-creates a named resource ({@code POST {collection}?name=...}). */ - public T getOrCreateNamed(String name, Class dataClass) { + public CompletableFuture getOrCreateNamed(String name, Class dataClass) { return getOrCreateNamed(name, null, dataClass); } @@ -259,7 +280,8 @@ public T getOrCreateNamed(String name, Class dataClass) { * #getOrCreateNamed(String, Class)}). Shared by {@code DatasetCollectionClient}/{@code * KeyValueStoreCollectionClient} so neither duplicates the wrapping (DRY). */ - public T getOrCreateNamedWithSchema(String name, Object schema, Class dataClass) { + public CompletableFuture getOrCreateNamedWithSchema( + String name, Object schema, Class dataClass) { Object body = schema == null ? null : Map.of("schema", schema); return getOrCreateNamed(name, body, dataClass); } @@ -268,30 +290,29 @@ public T getOrCreateNamedWithSchema(String name, Object schema, Class dat * POST that gets-or-creates a named resource, optionally sending a JSON request body (e.g. a * storage {@code schema}). A {@code null} body sends no body, matching the plain get-or-create. */ - public T getOrCreateNamed(String name, Object body, Class dataClass) { + public CompletableFuture getOrCreateNamed(String name, Object body, Class dataClass) { QueryParams params = new QueryParams(); if (name != null && !name.isEmpty()) { params.addString("name", name); } String u = params.applyToUrl(subUrl("")); byte[] bodyBytes = body == null ? null : Json.toBytes(body); - ApiResponse resp = - http.call( - "POST", u, bodyBytes, body == null ? "" : CONTENT_TYPE_JSON, http.baseRequestTimeout()); - return Json.parseData(resp.body(), dataClass); + return http.call( + "POST", u, bodyBytes, body == null ? "" : CONTENT_TYPE_JSON, http.baseRequestTimeout()) + .thenApply(resp -> Json.parseData(resp.body(), dataClass)); } /** POST with a raw body (optional) and content type, unwrapping the data envelope. */ - public T postWithBody( + public CompletableFuture postWithBody( String subPath, QueryParams params, byte[] body, String contentType, Class dataClass) { return postWithBody(subPath, params, body, contentType, Json.type(dataClass)); } - public T postWithBody( + public CompletableFuture postWithBody( String subPath, QueryParams params, byte[] body, String contentType, JavaType dataType) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - ApiResponse resp = http.call("POST", u, body, contentType, http.baseRequestTimeout()); - return Json.parseData(resp.body(), dataType); + return http.call("POST", u, body, contentType, http.baseRequestTimeout()) + .thenApply(resp -> Json.parseData(resp.body(), dataType)); } /** @@ -299,19 +320,19 @@ public T postWithBody( * {"data": ...}} envelope. Used by endpoints (e.g. actor input validation) whose response is a * plain object rather than the standard data envelope. */ - public T postWithBodyNoEnvelope( + public CompletableFuture postWithBodyNoEnvelope( String subPath, QueryParams params, byte[] body, String contentType, Class dataClass) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - ApiResponse resp = http.call("POST", u, body, contentType, http.baseRequestTimeout()); - return Json.parse(resp.body(), dataClass); + return http.call("POST", u, body, contentType, http.baseRequestTimeout()) + .thenApply(resp -> Json.parse(resp.body(), dataClass)); } /** PUT with a raw body (optional) and content type, unwrapping the data envelope. */ - public T putWithBody( + public CompletableFuture putWithBody( String subPath, QueryParams params, byte[] body, String contentType, Class dataClass) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - ApiResponse resp = http.call("PUT", u, body, contentType, http.baseRequestTimeout()); - return Json.parseData(resp.body(), dataClass); + return http.call("PUT", u, body, contentType, http.baseRequestTimeout()) + .thenApply(resp -> Json.parseData(resp.body(), dataClass)); } /** @@ -319,59 +340,70 @@ public T putWithBody( * unwrapping a {@code {"data": ...}} envelope. Used by endpoints (e.g. actor task input) whose * response is a plain object rather than the standard data envelope. */ - public T putWithBodyNoEnvelope( + public CompletableFuture putWithBodyNoEnvelope( String subPath, QueryParams params, byte[] body, String contentType, Class dataClass) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - ApiResponse resp = http.call("PUT", u, body, contentType, http.baseRequestTimeout()); - return Json.parse(resp.body(), dataClass); + return http.call("PUT", u, body, contentType, http.baseRequestTimeout()) + .thenApply(resp -> Json.parse(resp.body(), dataClass)); } /** DELETE with a JSON body (used for batch request deletion), unwrapping the data envelope. */ - public T deleteWithBody(String subPath, QueryParams params, Object body, Class dataClass) { + public CompletableFuture deleteWithBody( + String subPath, QueryParams params, Object body, Class dataClass) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - ApiResponse resp = - http.call("DELETE", u, Json.toBytes(body), CONTENT_TYPE_JSON, http.baseRequestTimeout()); - return Json.parseData(resp.body(), dataClass); + return http.call("DELETE", u, Json.toBytes(body), CONTENT_TYPE_JSON, http.baseRequestTimeout()) + .thenApply(resp -> Json.parseData(resp.body(), dataClass)); } - /** GET returning the raw response (no data envelope). Returns {@code null} on not-found. */ - public ApiResponse getRaw(String subPath, QueryParams params) { + /** + * GET returning the raw response (no data envelope). Completes with {@code null} on not-found. + */ + public CompletableFuture getRaw(String subPath, QueryParams params) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - try { - return http.call("GET", u, null, "", http.baseRequestTimeout()); - } catch (ApifyApiException e) { - if (isNotFound(e)) { - return null; - } - throw e; - } - } - - /** HEAD request; returns whether the resource exists. */ - public boolean headExists(String subPath, QueryParams params) { + return http.call("GET", u, null, "", http.baseRequestTimeout()) + .handle( + (resp, error) -> { + if (error == null) { + return resp; + } + Throwable cause = HttpClientCore.unwrapCompletion(error); + if (cause instanceof ApifyApiException apiError && isNotFound(apiError)) { + return null; + } + throw asRuntimeException(cause); + }); + } + + /** HEAD request; completes with whether the resource exists. */ + public CompletableFuture headExists(String subPath, QueryParams params) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - try { - http.call("HEAD", u, null, "", http.baseRequestTimeout()); - return true; - } catch (ApifyApiException e) { - if (isNotFound(e)) { - return false; - } - throw e; - } + return http.call("HEAD", u, null, "", http.baseRequestTimeout()) + .handle( + (resp, error) -> { + if (error == null) { + return true; + } + Throwable cause = HttpClientCore.unwrapCompletion(error); + if (cause instanceof ApifyApiException apiError && isNotFound(apiError)) { + return false; + } + throw asRuntimeException(cause); + }); } /** PUT with raw bytes and a content type (used for key-value-store record uploads). */ - public void putRaw(String subPath, QueryParams params, byte[] body, String contentType) { - putRaw(subPath, params, body, contentType, http.baseRequestTimeout(), false); + public CompletableFuture putRaw( + String subPath, QueryParams params, byte[] body, String contentType) { + return putRaw(subPath, params, body, contentType, http.baseRequestTimeout(), false); } /** - * PUT with raw bytes and a content type, with an explicit per-request {@code timeout} and control - * over whether transport timeouts are retried. Used by key-value-store record uploads that expose - * the reference client's {@code timeoutSecs}/{@code doNotRetryTimeouts} write options. + * As {@link #putRaw(String, QueryParams, byte[], String)}, with an explicit per-request {@code + * timeout} and control over whether transport timeouts are retried. Used by key-value-store + * record uploads that expose the reference client's {@code timeoutSecs}/{@code + * doNotRetryTimeouts} write options. */ - public void putRaw( + public CompletableFuture putRaw( String subPath, QueryParams params, byte[] body, @@ -379,7 +411,8 @@ public void putRaw( Duration timeout, boolean doNotRetryTimeouts) { String u = mergedParams(params).applyToUrl(subUrl(subPath)); - http.call("PUT", u, body, contentType, timeout, doNotRetryTimeouts); + return http.call("PUT", u, body, contentType, timeout, doNotRetryTimeouts) + .thenApply(resp -> null); } /** @@ -410,13 +443,16 @@ public Long clampServerWait(Long waitForFinishSecs) { /** * Polls a GET endpoint with {@code waitForFinish} until the resource reaches a terminal state or * the wait budget elapses. {@code waitSecs == null} means "wait indefinitely", implemented as a - * finite but very large bound so the loop always terminates. + * finite but very large bound so the polling always terminates. * *

    The budget is a pure time bound, evaluated independently of whether the resource is * currently present: a just-started run/build can transiently return 404 (database-replica lag), * which is treated as "not yet available". + * + *

    Each poll is chained via {@link CompletableFuture#thenCompose}, and the inter-poll delay is + * a scheduled timer (see {@link Async}), so waiting never blocks a thread. */ - public T waitForFinish( + public CompletableFuture waitForFinish( Long waitSecs, String resourceName, JavaType dataType, Predicate isTerminal) { // Clamp to MAX_WAIT_FOR_FINISH_SECS so a pathological waitSecs near Long.MAX_VALUE cannot // overflow budgetMillis into a negative value (which would degrade the wait into a single @@ -427,58 +463,64 @@ public T waitForFinish( : MAX_WAIT_FOR_FINISH_SECS; long budgetMillis = effectiveWaitSecs * 1000L; long start = System.currentTimeMillis(); - - // Never ask the server to hold the connection longer than the client's own per-request timeout, - // or a short configured timeout would abort every poll (HttpTimeoutException) and exhaust the - // retry budget on an otherwise-healthy run. A value of 0 disables server-side waiting and falls - // back to pure client-side polling. long serverWaitCap = serverWaitCapSecs(); + return pollOnce(dataType, isTerminal, resourceName, start, budgetMillis, serverWaitCap, null); + } - T resource = null; - boolean present = false; - - while (true) { - long elapsed = System.currentTimeMillis() - start; - long remainingSecs = (budgetMillis - elapsed) / 1000L; - long requestSecs = - Math.min(Math.min(Math.max(remainingSecs, 0), WAIT_REQUEST_SECS), serverWaitCap); - - QueryParams params = new QueryParams(); - params.addLong("waitForFinish", requestSecs); - - Optional res = getResource("", params, dataType); - if (res.isPresent()) { - resource = res.get(); - present = true; - if (isTerminal.test(resource)) { - return resource; - } - } - - if (System.currentTimeMillis() - start >= budgetMillis) { - break; - } - sleep(WAIT_POLL_INTERVAL); - } + /** + * One poll of the {@code waitForFinish} loop. {@code lastSeen} carries the most recently observed + * resource across recursive calls (there is no shared mutable state between polls, so each + * recursive step gets everything it needs as a parameter). + */ + private CompletableFuture pollOnce( + JavaType dataType, + Predicate isTerminal, + String resourceName, + long start, + long budgetMillis, + long serverWaitCap, + T lastSeen) { + long elapsed = System.currentTimeMillis() - start; + long remainingSecs = (budgetMillis - elapsed) / 1000L; + long requestSecs = + Math.min(Math.min(Math.max(remainingSecs, 0), WAIT_REQUEST_SECS), serverWaitCap); - if (present) { - return resource; - } - throw new IllegalStateException( - "waiting for " - + resourceName - + " to finish failed: cannot fetch " - + resourceName - + " details from the server"); - } - - private static void sleep(Duration d) { - try { - Thread.sleep(d.toMillis()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new ApifyTransportException(e); - } + QueryParams params = new QueryParams(); + params.addLong("waitForFinish", requestSecs); + + return this.getResource("", params, dataType) + .thenCompose( + res -> { + T seen = res.orElse(lastSeen); + if (res.isPresent() && isTerminal.test(res.get())) { + return CompletableFuture.completedFuture(res.get()); + } + if (System.currentTimeMillis() - start >= budgetMillis) { + if (seen != null) { + return CompletableFuture.completedFuture(seen); + } + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally( + new IllegalStateException( + "waiting for " + + resourceName + + " to finish failed: cannot fetch " + + resourceName + + " details from the server")); + return failed; + } + return Async.delay(WAIT_POLL_INTERVAL) + .thenCompose( + unused -> + pollOnce( + dataType, + isTerminal, + resourceName, + start, + budgetMillis, + serverWaitCap, + seen)); + }); } // ---- URL / id helpers ----------------------------------------------------- @@ -494,6 +536,16 @@ public static boolean isNotFound(ApifyApiException e) { || "HEAD".equals(e.getHttpMethod()); } + /** Rethrows a cause classified from a future's completion as an unchecked exception, as-is. */ + private static RuntimeException asRuntimeException(Throwable cause) { + if (cause instanceof RuntimeException runtimeException) { + return runtimeException; + } + // Every failure this client's async pipeline produces is already a RuntimeException + // (ApifyApiException/ApifyTransportException); wrapping here is a defensive fallback only. + return new ApifyTransportException(cause); + } + /** * Encodes a resource id so it is safe to embed in a URL path. Apify uses the {@code * username~resourcename} form, so the first {@code /} of an id is replaced with {@code ~}. diff --git a/src/main/java/com/apify/client/internal/RunStartSupport.java b/src/main/java/com/apify/client/internal/RunStartSupport.java index 2264de3..bc8b8a4 100644 --- a/src/main/java/com/apify/client/internal/RunStartSupport.java +++ b/src/main/java/com/apify/client/internal/RunStartSupport.java @@ -5,6 +5,7 @@ import com.apify.client.log.StreamedLogOptions; import com.apify.client.run.ActorRun; import com.apify.client.run.RunClient; +import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; /** @@ -28,12 +29,13 @@ public final class RunStartSupport { private RunStartSupport() {} /** - * Starts a run on {@code ctx}'s {@code runs} sub-resource and returns immediately with the - * created run. {@code input} is any JSON-serializable value, or {@code null} for no input. {@code - * applyParams} applies every start-option query parameter (build, memory, timeout, ...) the - * caller's options type configures; {@code contentType} is the input body's content type. + * Starts a run on {@code ctx}'s {@code runs} sub-resource and completes with the created run as + * soon as it exists (no waiting). {@code input} is any JSON-serializable value, or {@code null} + * for no input. {@code applyParams} applies every start-option query parameter (build, memory, + * timeout, ...) the caller's options type configures; {@code contentType} is the input body's + * content type. */ - public static ActorRun start( + public static CompletableFuture start( ResourceContext ctx, Object input, Consumer applyParams, String contentType) { QueryParams params = new QueryParams(); applyParams.accept(params); @@ -42,19 +44,19 @@ public static ActorRun start( } /** - * Starts a run and waits (client-side polling) for it to finish, without log streaming. + * Starts a run and waits (non-blocking, polling) for it to finish, without log streaming. * * @param waitSecs bounds the wait; {@code null} waits indefinitely */ - public static ActorRun call( + public static CompletableFuture call( ApifyClient root, ResourceContext ctx, Object input, Consumer applyParams, String contentType, Long waitSecs) { - ActorRun run = start(ctx, input, applyParams, contentType); - return root.run(run.getId()).waitForFinish(waitSecs); + return start(ctx, input, applyParams, contentType) + .thenCompose(run -> root.run(run.getId()).waitForFinish(waitSecs)); } /** @@ -71,7 +73,7 @@ public static ActorRun call( * @param logOptions a custom log-streaming destination, or {@code null} for the default * @param waitSecs bounds the wait; {@code null} waits indefinitely */ - public static ActorRun callWithLogStreaming( + public static CompletableFuture callWithLogStreaming( ApifyClient root, ResourceContext ctx, Object input, @@ -80,35 +82,40 @@ public static ActorRun callWithLogStreaming( boolean logStreamingEnabled, StreamedLogOptions logOptions, Long waitSecs) { - ActorRun run = start(ctx, input, applyParams, contentType); - RunClient runClient = root.run(run.getId()); - - StreamedLog streamedLog = null; - if (logStreamingEnabled) { - streamedLog = startStreamedLogQuietly(runClient, logOptions); - } - try { - return runClient.waitForFinish(waitSecs); - } finally { - if (streamedLog != null) { - streamedLog.close(); - } - } + return start(ctx, input, applyParams, contentType) + .thenCompose( + run -> { + RunClient runClient = root.run(run.getId()); + CompletableFuture streamedLogFuture = + logStreamingEnabled + ? startStreamedLogQuietly(runClient, logOptions) + : CompletableFuture.completedFuture(null); + return streamedLogFuture.thenCompose( + streamedLog -> + runClient + .waitForFinish(waitSecs) + .whenComplete( + (result, error) -> { + if (streamedLog != null) { + streamedLog.close(); + } + })); + }); } /** * Starts {@code call}'s default log streaming, swallowing (rather than propagating) any failure * to open the live log stream, so a transient log-endpoint issue cannot abort the run itself. + * Completes with the started {@link StreamedLog}, or {@code null} if it could not be started. */ - private static StreamedLog startStreamedLogQuietly( + private static CompletableFuture startStreamedLogQuietly( RunClient runClient, StreamedLogOptions logOptions) { - try { - StreamedLog streamedLog = - logOptions != null ? runClient.getStreamedLog(logOptions) : runClient.getStreamedLog(); - streamedLog.start(); - return streamedLog; - } catch (RuntimeException e) { - return null; - } + CompletableFuture created = + logOptions != null ? runClient.getStreamedLog(logOptions) : runClient.getStreamedLog(); + return created + .thenCompose( + streamedLog -> + streamedLog.start().handle((unused, error) -> error == null ? streamedLog : null)) + .exceptionally(error -> null); } } diff --git a/src/main/java/com/apify/client/keyvalue/KeyValueStoreClient.java b/src/main/java/com/apify/client/keyvalue/KeyValueStoreClient.java index 9a6a85c..fb651f6 100644 --- a/src/main/java/com/apify/client/keyvalue/KeyValueStoreClient.java +++ b/src/main/java/com/apify/client/keyvalue/KeyValueStoreClient.java @@ -1,6 +1,5 @@ package com.apify.client.keyvalue; -import com.apify.client.http.ApiResponse; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.Extras; import com.apify.client.internal.HttpClientCore; @@ -10,10 +9,12 @@ import com.apify.client.internal.ResourceContext; import com.apify.client.internal.Signatures; import java.time.Duration; -import java.util.Iterator; import java.util.List; -import java.util.NoSuchElementException; import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; /** A client for a specific key-value store (and run-nested variants). */ public final class KeyValueStoreClient { @@ -51,35 +52,36 @@ public static KeyValueStoreClient nested( } /** Fetches the store metadata, or empty if it does not exist. */ - public Optional get() { + public CompletableFuture> get() { return ctx.getResource("", new QueryParams(), KeyValueStore.class); } /** Updates the store metadata (e.g. name) and returns the updated object. */ - public KeyValueStore update(Object newFields) { + public CompletableFuture update(Object newFields) { return ctx.updateResource("", newFields, KeyValueStore.class); } /** Deletes the store. */ - public void delete() { - ctx.deleteResource(""); + public CompletableFuture delete() { + return ctx.deleteResource(""); } /** Lists the keys stored in this key-value store. */ - public KeyValueStoreKeysPage listKeys(ListKeysOptions options) { + public CompletableFuture listKeys(ListKeysOptions options) { QueryParams params = new QueryParams(); options.apply(params); return ctx.getResourceRequired("keys", params, KeyValueStoreKeysPage.class); } /** - * Returns a lazy iterator over this store's keys, fetching pages on demand via the cursor-based - * ({@code exclusiveStartKey}) listing endpoint. The options' {@code limit} caps the total number - * of keys yielded ({@code null} or non-positive = all); any {@code exclusiveStartKey} sets the - * starting point. The per-request page size is left to the server (bounded by any {@code limit} - * cap); use {@link #iterateKeys(ListKeysOptions, Long)} to set it explicitly. + * Returns a lazy, backpressure-aware publisher over this store's keys, fetching pages on demand + * via the cursor-based ({@code exclusiveStartKey}) listing endpoint. The options' {@code limit} + * caps the total number of keys yielded ({@code null} or non-positive = all); any {@code + * exclusiveStartKey} sets the starting point. The per-request page size is left to the server + * (bounded by any {@code limit} cap); use {@link #iterateKeys(ListKeysOptions, Long)} to set it + * explicitly. */ - public Iterator iterateKeys(ListKeysOptions options) { + public Flow.Publisher iterateKeys(ListKeysOptions options) { return iterateKeys(options, null); } @@ -88,91 +90,181 @@ public Iterator iterateKeys(ListKeysOptions options) { * ({@code null} = server default). Provided for consistency with the collection {@code iterate} * helpers; key listing is cursor-based, so the options' {@code limit} remains a total-items cap. */ - public Iterator iterateKeys(ListKeysOptions options, Long chunkSize) { - return new KeysIterator(options != null ? options : new ListKeysOptions(), chunkSize); + public Flow.Publisher iterateKeys(ListKeysOptions options, Long chunkSize) { + return new KeysPublisher(options != null ? options : new ListKeysOptions(), chunkSize); } /** - * Lazily iterates over a store's keys via the cursor-based ({@code exclusiveStartKey}) listing. + * Lazily publishes a store's keys via the cursor-based ({@code exclusiveStartKey}) listing, + * fetching a page only once the subscriber has signalled demand for it. Supports a single + * subscriber, like {@link com.apify.client.internal.AsyncPaginatedPublisher}, whose draining + * design this mirrors (cursor pagination instead of offset/limit is the only real difference). */ - private final class KeysIterator implements Iterator { + private final class KeysPublisher implements Flow.Publisher { private final Long chunkSize; private final QueryParams filters; - private List buffer = List.of(); - private int pos; - private String cursor; - private Long remaining; - private boolean exhausted; + private final String initialCursor; + private final Long initialRemaining; + private final AtomicBoolean subscribed = new AtomicBoolean(); - KeysIterator(ListKeysOptions options, Long chunkSize) { + KeysPublisher(ListKeysOptions options, Long chunkSize) { this.chunkSize = chunkSize != null && chunkSize > 0 ? chunkSize : null; - this.cursor = options.exclusiveStartKeyValue(); + this.initialCursor = options.exclusiveStartKeyValue(); Long limit = options.limitValue(); - this.remaining = limit != null && limit > 0 ? limit : null; - // Snapshot the filters once so mutating the options mid-iteration cannot leak into later - // pages. + this.initialRemaining = limit != null && limit > 0 ? limit : null; this.filters = new QueryParams(); options.applyFilters(this.filters); } @Override - public boolean hasNext() { - while (pos >= buffer.size()) { - if (exhausted) { - return false; - } - fetchPage(); + public void subscribe(Flow.Subscriber subscriber) { + if (!subscribed.compareAndSet(false, true)) { + subscriber.onSubscribe( + new Flow.Subscription() { + @Override + public void request(long n) {} + + @Override + public void cancel() {} + }); + subscriber.onError( + new IllegalStateException( + "This publisher supports only a single subscriber (already subscribed)")); + return; } - return true; + subscriber.onSubscribe(new Session(subscriber)); } - @Override - public KeyValueStoreKey next() { - if (!hasNext()) { - throw new NoSuchElementException(); + private final class Session implements Flow.Subscription { + private final Flow.Subscriber subscriber; + private final AtomicLong requested = new AtomicLong(); + private final AtomicBoolean draining = new AtomicBoolean(); + private final AtomicBoolean cancelled = new AtomicBoolean(); + private final AtomicBoolean terminated = new AtomicBoolean(); + + // Only touched from within the draining-guarded section; see AsyncPaginatedPublisher.Session. + private List buffer = List.of(); + private int pos; + private String cursor; + private Long remaining; + private boolean exhausted; + + Session(Flow.Subscriber subscriber) { + this.subscriber = subscriber; + this.cursor = initialCursor; + this.remaining = initialRemaining; } - return buffer.get(pos++); - } - private void fetchPage() { - QueryParams params = new QueryParams(); - // Request the smaller of the remaining cap and the page size, so the last page never - // overshoots the caller's total cap; a null limit lets the server choose its default page. - Long pageLimit = remaining; - if (chunkSize != null && (pageLimit == null || chunkSize < pageLimit)) { - pageLimit = chunkSize; + @Override + public void request(long n) { + if (n <= 0) { + if (terminated.compareAndSet(false, true)) { + subscriber.onError( + new IllegalArgumentException( + "Reactive Streams violation: requested amount must be positive, was " + n)); + } + return; + } + requested.updateAndGet(current -> current + n < 0 ? Long.MAX_VALUE : current + n); + drain(); + } + + @Override + public void cancel() { + cancelled.set(true); } - params.addLong("limit", pageLimit); - params.addString("exclusiveStartKey", cursor); - params.extend(filters); - KeyValueStoreKeysPage page = - ctx.getResourceRequired("keys", params, KeyValueStoreKeysPage.class); - buffer = page.getItems(); - pos = 0; - cursor = page.getNextExclusiveStartKey(); - boolean truncated = page.isTruncated(); - if (remaining != null) { - // Defensively trim the last page to the cap in case the server returned more than - // requested. - if (buffer.size() > remaining) { - buffer = buffer.subList(0, remaining.intValue()); + + private void drain() { + if (draining.compareAndSet(false, true)) { + drainLoop(); } - remaining -= buffer.size(); } - // Stop when the API reports the listing is not truncated (no more keys), there is no next - // cursor, the page is empty, or the total cap is reached. - if (!truncated - || cursor == null - || cursor.isEmpty() - || buffer.isEmpty() - || (remaining != null && remaining <= 0)) { - exhausted = true; + + private void drainLoop() { + while (!cancelled.get() && requested.get() > 0 && pos < buffer.size()) { + KeyValueStoreKey item = buffer.get(pos++); + requested.decrementAndGet(); + subscriber.onNext(item); + } + if (cancelled.get()) { + draining.set(false); + return; + } + if (pos < buffer.size()) { + draining.set(false); + if (requested.get() > 0 && pos < buffer.size()) { + drain(); + } + return; + } + if (exhausted) { + if (terminated.compareAndSet(false, true)) { + subscriber.onComplete(); + } + draining.set(false); + return; + } + fetchPage(); + } + + private void fetchPage() { + QueryParams params = new QueryParams(); + // Request the smaller of the remaining cap and the page size, so the last page never + // overshoots the caller's total cap; a null limit lets the server choose its default page. + Long pageLimit = remaining; + if (chunkSize != null && (pageLimit == null || chunkSize < pageLimit)) { + pageLimit = chunkSize; + } + params.addLong("limit", pageLimit); + params.addString("exclusiveStartKey", cursor); + params.extend(filters); + ctx.getResourceRequired("keys", params, KeyValueStoreKeysPage.class) + .whenComplete( + (page, error) -> { + if (cancelled.get()) { + draining.set(false); + return; + } + if (error != null) { + if (terminated.compareAndSet(false, true)) { + subscriber.onError(HttpClientCore.unwrapCompletion(error)); + } + draining.set(false); + return; + } + applyPage(page); + drainLoop(); + }); + } + + private void applyPage(KeyValueStoreKeysPage page) { + buffer = page.getItems(); + pos = 0; + cursor = page.getNextExclusiveStartKey(); + boolean truncated = page.isTruncated(); + if (remaining != null) { + // Defensively trim the last page to the cap in case the server returned more than + // requested. + if (buffer.size() > remaining) { + buffer = buffer.subList(0, remaining.intValue()); + } + remaining -= buffer.size(); + } + // Stop when the API reports the listing is not truncated (no more keys), there is no next + // cursor, the page is empty, or the total cap is reached. + if (!truncated + || cursor == null + || cursor.isEmpty() + || buffer.isEmpty() + || (remaining != null && remaining <= 0)) { + exhausted = true; + } } } } /** Reports whether a record with the given key exists. */ - public boolean recordExists(String key) { + public CompletableFuture recordExists(String key) { return ctx.headExists("records/" + ResourceContext.encodePathSegment(key), new QueryParams()); } @@ -180,37 +272,42 @@ public boolean recordExists(String key) { * Fetches a record by key, or empty if it does not exist. Like the reference client, it requests * the record as an attachment so the API returns the raw bytes directly rather than redirecting. */ - public Optional getRecord(String key) { + public CompletableFuture> getRecord(String key) { return getRecord(key, new GetRecordOptions().attachment(true)); } /** Fetches a record with explicit options (attachment, signature). */ - public Optional getRecord(String key, GetRecordOptions options) { + public CompletableFuture> getRecord( + String key, GetRecordOptions options) { QueryParams params = new QueryParams(); options.apply(params); - ApiResponse resp = ctx.getRaw("records/" + ResourceContext.encodePathSegment(key), params); - if (resp == null) { - return Optional.empty(); - } - String contentType = resp.headers().firstValue(HttpHeaders.CONTENT_TYPE).orElse(null); - return Optional.of(new KeyValueStoreRecord(key, resp.body(), contentType)); + return ctx.getRaw("records/" + ResourceContext.encodePathSegment(key), params) + .thenApply( + resp -> { + if (resp == null) { + return Optional.empty(); + } + String contentType = resp.headers().firstValue(HttpHeaders.CONTENT_TYPE).orElse(null); + return Optional.of(new KeyValueStoreRecord(key, resp.body(), contentType)); + }); } /** Stores a record with raw bytes and the given content type. */ - public void setRecord(String key, byte[] value, String contentType) { - setRecord(key, value, contentType, new SetRecordOptions()); + public CompletableFuture setRecord(String key, byte[] value, String contentType) { + return setRecord(key, value, contentType, new SetRecordOptions()); } /** * Stores a record with raw bytes and the given content type, honoring the given write options * ({@code timeoutSecs}, {@code doNotRetryTimeouts}). */ - public void setRecord(String key, byte[] value, String contentType, SetRecordOptions options) { + public CompletableFuture setRecord( + String key, byte[] value, String contentType, SetRecordOptions options) { Duration timeout = options.timeoutSecsValue() != null ? Duration.ofSeconds(options.timeoutSecsValue()) : ctx.http.baseRequestTimeout(); - ctx.putRaw( + return ctx.putRaw( "records/" + ResourceContext.encodePathSegment(key), new QueryParams(), value, @@ -220,13 +317,13 @@ public void setRecord(String key, byte[] value, String contentType, SetRecordOpt } /** Stores a record holding the JSON serialization of {@code value}. */ - public void setRecordJson(String key, Object value) { - setRecord(key, Json.toBytes(value), ResourceContext.CONTENT_TYPE_JSON_CHARSET); + public CompletableFuture setRecordJson(String key, Object value) { + return setRecord(key, Json.toBytes(value), ResourceContext.CONTENT_TYPE_JSON_CHARSET); } /** Deletes a record by key. */ - public void deleteRecord(String key) { - ctx.deleteResource("records/" + ResourceContext.encodePathSegment(key)); + public CompletableFuture deleteRecord(String key) { + return ctx.deleteResource("records/" + ResourceContext.encodePathSegment(key)); } /** @@ -234,19 +331,23 @@ public void deleteRecord(String key) { * exposes a URL-signing secret key (i.e. it is private), appends an HMAC-SHA256 signature so the * URL grants access without an API token. The URL is built from the configured public base URL. */ - public String getRecordPublicUrl(String key) { + public CompletableFuture getRecordPublicUrl(String key) { QueryParams params = new QueryParams(); - Optional store = get(); - if (store.isPresent()) { - String secret = Extras.extractString(store.get().getExtra(), "urlSigningSecretKey"); - if (secret != null) { - params.addString("signature", Signatures.createHmacSignature(secret, key)); - } - } - // Public URL = resource path + signature only, matching the JS reference. - // Seeded filters are not carried; see DatasetClient.createItemsPublicUrl and - // docs/storages.md for the last-run caveat. - return params.applyToUrl(ctx.publicUrl("records/" + ResourceContext.encodePathSegment(key))); + return get() + .thenApply( + store -> { + if (store.isPresent()) { + String secret = Extras.extractString(store.get().getExtra(), "urlSigningSecretKey"); + if (secret != null) { + params.addString("signature", Signatures.createHmacSignature(secret, key)); + } + } + // Public URL = resource path + signature only, matching the JS reference. + // Seeded filters are not carried; see DatasetClient.createItemsPublicUrl and + // docs/storages.md for the last-run caveat. + return params.applyToUrl( + ctx.publicUrl("records/" + ResourceContext.encodePathSegment(key))); + }); } /** @@ -254,7 +355,7 @@ public String getRecordPublicUrl(String key) { * signature is appended for private stores. {@code expiresInSecs} optionally bounds the validity * of a signed URL ({@code null} for non-expiring). */ - public String createKeysPublicUrl(Long expiresInSecs) { + public CompletableFuture createKeysPublicUrl(Long expiresInSecs) { return createKeysPublicUrl(new ListKeysOptions(), expiresInSecs); } @@ -265,22 +366,27 @@ public String createKeysPublicUrl(Long expiresInSecs) { * already supplied one. {@code expiresInSecs} optionally bounds the validity of a signed URL * ({@code null} for non-expiring). */ - public String createKeysPublicUrl(ListKeysOptions options, Long expiresInSecs) { + public CompletableFuture createKeysPublicUrl( + ListKeysOptions options, Long expiresInSecs) { QueryParams params = new QueryParams(); options.apply(params); - if (options.signatureValue() == null) { - Optional store = get(); - if (store.isPresent()) { - String secret = Extras.extractString(store.get().getExtra(), "urlSigningSecretKey"); - if (secret != null) { - params.addString( - "signature", - Signatures.signStorageContent(secret, store.get().getId(), expiresInSecs)); - } - } + if (options.signatureValue() != null) { + return CompletableFuture.completedFuture(params.applyToUrl(ctx.publicUrl("keys"))); } - // Public URL = resource path + the explicit key-listing options + signature, matching the JS - // reference (see the last-run caveat documented in docs/storages.md). - return params.applyToUrl(ctx.publicUrl("keys")); + return get() + .thenApply( + store -> { + if (store.isPresent()) { + String secret = Extras.extractString(store.get().getExtra(), "urlSigningSecretKey"); + if (secret != null) { + params.addString( + "signature", + Signatures.signStorageContent(secret, store.get().getId(), expiresInSecs)); + } + } + // Public URL = resource path + the explicit key-listing options + signature, matching + // the JS reference (see the last-run caveat documented in docs/storages.md). + return params.applyToUrl(ctx.publicUrl("keys")); + }); } } diff --git a/src/main/java/com/apify/client/keyvalue/KeyValueStoreCollectionClient.java b/src/main/java/com/apify/client/keyvalue/KeyValueStoreCollectionClient.java index 7c421ac..7766807 100644 --- a/src/main/java/com/apify/client/keyvalue/KeyValueStoreCollectionClient.java +++ b/src/main/java/com/apify/client/keyvalue/KeyValueStoreCollectionClient.java @@ -5,6 +5,7 @@ import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.ResourceContext; +import java.util.concurrent.CompletableFuture; /** A client for the key-value store collection ({@code GET/POST /v2/key-value-stores}). */ public final class KeyValueStoreCollectionClient @@ -21,7 +22,7 @@ public KeyValueStoreCollectionClient(HttpClientCore http, String baseUrl) { * Gets the store with the given name, creating it if it does not exist. An empty/{@code null} * name creates a new unnamed store. */ - public KeyValueStore getOrCreate(String name) { + public CompletableFuture getOrCreate(String name) { return getOrCreate(name, null); } @@ -30,7 +31,7 @@ public KeyValueStore getOrCreate(String name) { * schema} on creation. Mirrors the reference client's {@code getOrCreate(name, {schema})}; a * {@code null} schema behaves like {@link #getOrCreate(String)}. */ - public KeyValueStore getOrCreate(String name, Object schema) { + public CompletableFuture getOrCreate(String name, Object schema) { return ctx.getOrCreateNamedWithSchema(name, schema, KeyValueStore.class); } } diff --git a/src/main/java/com/apify/client/log/LogClient.java b/src/main/java/com/apify/client/log/LogClient.java index 5ed3f54..45b28d8 100644 --- a/src/main/java/com/apify/client/log/LogClient.java +++ b/src/main/java/com/apify/client/log/LogClient.java @@ -1,15 +1,14 @@ package com.apify.client.log; -import com.apify.client.http.ApiResponse; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; import java.io.IOException; import java.io.InputStream; -import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; import java.util.Optional; +import java.util.concurrent.CompletableFuture; /** * A client for accessing the log of an Actor build or run ({@code /v2/logs/{buildOrRunId}}, or the @@ -37,47 +36,52 @@ public static LogClient nested(HttpClientCore http, String base, QueryParams inh } /** Fetches the entire log as text, or empty if the log does not exist. */ - public Optional get() { + public CompletableFuture> get() { return get(new LogOptions()); } /** Fetches the log with explicit options (raw, download). */ - public Optional get(LogOptions options) { + public CompletableFuture> get(LogOptions options) { QueryParams params = new QueryParams(); options.apply(params); - ApiResponse resp = ctx.getRaw("", params); - if (resp == null) { - return Optional.empty(); - } - return Optional.of(new String(resp.body(), StandardCharsets.UTF_8)); + return ctx.getRaw("", params) + .thenApply( + resp -> + resp == null + ? Optional.empty() + : Optional.of(new String(resp.body(), StandardCharsets.UTF_8))); } /** - * Opens a live, streaming connection to the log and returns a stream over the log bytes. The - * caller is responsible for closing the returned {@link InputStream}. + * Opens a live, streaming connection to the log and completes with a stream over the log bytes. + * The caller is responsible for closing the returned {@link InputStream}. * *

    Unlike {@link #get()}, this bypasses the buffered/retrying transport so the log can be * followed in real time as the run produces it (the {@code stream=1} query parameter). Because * the response is consumed incrementally, it is not retried. */ - public InputStream stream() { + public CompletableFuture stream() { return stream(new LogOptions()); } /** Opens a live log stream with explicit options (raw, download). */ - public InputStream stream(LogOptions options) { + public CompletableFuture stream(LogOptions options) { QueryParams params = new QueryParams(); params.addBool("stream", true); options.apply(params); String url = ctx.mergedParams(params).applyToUrl(ctx.subUrl("")); - HttpResponse resp = ctx.http.stream(url); - if (resp.statusCode() >= HttpClientCore.MAX_SUCCESS_STATUS) { - byte[] body = drain(resp.body()); - throw HttpClientCore.buildApiError( - resp.statusCode(), body, 1, "GET", HttpClientCore.extractPath(url)); - } - return resp.body(); + return ctx.http + .streamAsync(url) + .thenApply( + resp -> { + if (resp.statusCode() >= HttpClientCore.MAX_SUCCESS_STATUS) { + byte[] body = drain(resp.body()); + throw HttpClientCore.buildApiError( + resp.statusCode(), body, 1, "GET", HttpClientCore.extractPath(url)); + } + return resp.body(); + }); } private static byte[] drain(InputStream in) { diff --git a/src/main/java/com/apify/client/log/StreamedLog.java b/src/main/java/com/apify/client/log/StreamedLog.java index 1a92d18..9188352 100644 --- a/src/main/java/com/apify/client/log/StreamedLog.java +++ b/src/main/java/com/apify/client/log/StreamedLog.java @@ -1,6 +1,7 @@ package com.apify.client.log; import com.apify.client.http.ApifyApiException; +import com.apify.client.internal.HttpClientCore; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -8,6 +9,7 @@ import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -73,6 +75,13 @@ public final class StreamedLog implements AutoCloseable { private volatile InputStream activeStream; private Thread streamingThread; + /** + * Set for the duration between {@link #start()} being called and the async stream-open completing + * (success or failure), closing the window where a concurrent {@link #start()} call would + * otherwise see {@code streamingThread == null} and race the first call's setup. + */ + private boolean starting; + public StreamedLog(LogClient logClient, Consumer destination, boolean fromStart) { this.logClient = logClient; this.destination = destination; @@ -82,30 +91,46 @@ public StreamedLog(LogClient logClient, Consumer destination, boolean fr /** * Starts redirecting the log in a background daemon thread. * - *

    The live log stream is opened synchronously before the thread is launched, so this call - * blocks briefly on the HTTP round-trip and surfaces a failure to open the stream to the caller - * rather than on the background thread. + *

    Opening the live log stream is asynchronous (this call returns immediately); the returned + * future completes once the stream is open and the background reader thread has been launched, or + * exceptionally if the stream could not be opened. If it were opened inside the reader thread + * instead, a {@link #stop()} that ran during the HTTP round-trip would close a still-null stream, + * leaving the reader blocked on a live read that never returns and {@code join()} hanging + * forever. * - * @throws IllegalStateException if redirection is already running - * @throws ApifyApiException if the log stream cannot be opened (the API returns a non-2xx - * status); other transport failures propagate as their own runtime exception + * @throws IllegalStateException if redirection is already running (or a previous {@link #start()} + * call's stream-open is still in flight) + * @implNote the returned future completes exceptionally with an {@link ApifyApiException} if the + * log stream cannot be opened (the API returns a non-2xx status); other transport failures + * complete it with their own runtime exception. */ - public synchronized void start() { - if (streamingThread != null) { + public synchronized CompletableFuture start() { + if (streamingThread != null || starting) { throw new IllegalStateException("Streaming task already active"); } + starting = true; stopLogging = false; forwardingFailed = false; - // Open the live stream here, before launching the reader thread, so activeStream is guaranteed - // to be set once the thread exists. If it were opened inside the thread, a stop() that ran - // during the HTTP round-trip would close a still-null stream, leaving the reader blocked on a - // live read() that never returns and join() hanging forever. Opening it up front also lets a - // failed connection surface to the caller of start() instead of a background-thread warning. - activeStream = logClient.stream(new LogOptions().raw(true)); - Thread thread = new Thread(this::streamLog, "apify-streamed-log"); - thread.setDaemon(true); - streamingThread = thread; - thread.start(); + return logClient.stream(new LogOptions().raw(true)) + .handle( + (stream, error) -> { + synchronized (StreamedLog.this) { + starting = false; + if (error != null) { + Throwable cause = HttpClientCore.unwrapCompletion(error); + if (cause instanceof RuntimeException runtimeException) { + throw runtimeException; + } + throw new IllegalStateException("failed to open log stream", cause); + } + activeStream = stream; + Thread thread = new Thread(this::streamLog, "apify-streamed-log"); + thread.setDaemon(true); + streamingThread = thread; + thread.start(); + } + return null; + }); } /** diff --git a/src/main/java/com/apify/client/requestqueue/RequestQueueClient.java b/src/main/java/com/apify/client/requestqueue/RequestQueueClient.java index 5d0b9d3..dd46483 100644 --- a/src/main/java/com/apify/client/requestqueue/RequestQueueClient.java +++ b/src/main/java/com/apify/client/requestqueue/RequestQueueClient.java @@ -1,25 +1,25 @@ package com.apify.client.requestqueue; import com.apify.client.http.ApifyApiException; -import com.apify.client.http.ApifyClientException; import com.apify.client.http.ApifyTransportException; import com.apify.client.internal.ApiPaths; +import com.apify.client.internal.Async; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.Json; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; +import java.time.Duration; import java.util.ArrayList; import java.util.HashSet; -import java.util.Iterator; import java.util.List; -import java.util.NoSuchElementException; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; /** A client for a specific request queue (and run-nested variants). */ public final class RequestQueueClient { @@ -95,25 +95,25 @@ private QueryParams withClientKey(QueryParams params) { } /** Fetches the queue metadata, or empty if it does not exist. */ - public Optional get() { + public CompletableFuture> get() { return ctx.getResource("", new QueryParams(), RequestQueue.class); } /** Updates the queue metadata (e.g. name) and returns the updated object. */ - public RequestQueue update(Object newFields) { + public CompletableFuture update(Object newFields) { return ctx.updateResource("", newFields, RequestQueue.class); } /** Deletes the queue. */ - public void delete() { - ctx.deleteResource(""); + public CompletableFuture delete() { + return ctx.deleteResource(""); } /** * Returns the requests at the head (front) of the queue, up to {@code limit} ({@code null} for * the server default). */ - public RequestQueueHead listHead(Long limit) { + public CompletableFuture listHead(Long limit) { QueryParams params = new QueryParams(); params.addLong("limit", limit); withClientKey(params); @@ -121,7 +121,8 @@ public RequestQueueHead listHead(Long limit) { } /** Adds a request to the queue. If {@code forefront} is true, it is added to the front. */ - public RequestQueueOperationInfo addRequest(RequestQueueRequest request, boolean forefront) { + public CompletableFuture addRequest( + RequestQueueRequest request, boolean forefront) { QueryParams params = new QueryParams(); params.addBool("forefront", forefront); withClientKey(params); @@ -134,7 +135,7 @@ public RequestQueueOperationInfo addRequest(RequestQueueRequest request, boolean } /** Fetches a request by ID, or empty if it does not exist. */ - public Optional getRequest(String id) { + public CompletableFuture> getRequest(String id) { return ctx.getResource( "requests/" + ResourceContext.encodePathSegment(id), new QueryParams(), @@ -145,7 +146,8 @@ public Optional getRequest(String id) { * Updates an existing request (identified by its ID field) and returns the operation info. If * {@code forefront} is true, the request is moved to the front of the queue. */ - public RequestQueueOperationInfo updateRequest(RequestQueueRequest request, boolean forefront) { + public CompletableFuture updateRequest( + RequestQueueRequest request, boolean forefront) { QueryParams params = new QueryParams(); params.addBool("forefront", forefront); withClientKey(params); @@ -158,18 +160,24 @@ public RequestQueueOperationInfo updateRequest(RequestQueueRequest request, bool } /** Deletes a request by ID. */ - public void deleteRequest(String id) { + public CompletableFuture deleteRequest(String id) { QueryParams params = withClientKey(new QueryParams()); String url = ctx.mergedParams(params) .applyToUrl(ctx.subUrl("requests/" + ResourceContext.encodePathSegment(id))); - try { - http.call("DELETE", url, null, "", http.baseRequestTimeout()); - } catch (ApifyApiException e) { - if (!ResourceContext.isNotFound(e)) { - throw e; - } - } + return http.call("DELETE", url, null, "", http.baseRequestTimeout()) + .handle( + (resp, error) -> { + if (error == null) { + return null; + } + Throwable cause = HttpClientCore.unwrapCompletion(error); + if (cause instanceof ApifyApiException apiError + && ResourceContext.isNotFound(apiError)) { + return null; + } + throw rethrow(cause); + }); } /** @@ -178,7 +186,7 @@ public void deleteRequest(String id) { * via {@link #deleteRequestLock}. This is the primary method used by distributed crawlers to * coordinate work across multiple workers. */ - public LockedRequestQueueHead listAndLockHead(long lockSecs, Long limit) { + public CompletableFuture listAndLockHead(long lockSecs, Long limit) { QueryParams params = new QueryParams(); params.addLong("lockSecs", lockSecs).addLong("limit", limit); withClientKey(params); @@ -189,7 +197,8 @@ public LockedRequestQueueHead listAndLockHead(long lockSecs, Long limit) { * Adds multiple requests to the queue with the default batch options. If {@code forefront} is * true, they are added to the front. */ - public BatchAddResult batchAddRequests(List requests, boolean forefront) { + public CompletableFuture batchAddRequests( + List requests, boolean forefront) { return batchAddRequests(requests, forefront, new BatchAddRequestsOptions()); } @@ -201,16 +210,17 @@ public BatchAddResult batchAddRequests(List requests, boole * #MAX_PAYLOAD_SIZE_BYTES}): a chunk of up to 25 requests is further split by cumulative * JSON-encoded byte size, so a batch of individually large requests (e.g. sizeable {@code * userData}) cannot 413. Chunks are sent using up to {@link BatchAddRequestsOptions#maxParallel} - * parallel API calls, and any requests the API leaves unprocessed (typically rate-limited) are - * retried with exponential backoff up to {@link + * concurrently in-flight chunk requests, and any requests the API leaves unprocessed (typically + * rate-limited) are retried with exponential backoff up to {@link * BatchAddRequestsOptions#maxUnprocessedRequestsRetries} times. The per-chunk results are merged * into a single {@link BatchAddResult}. * - *

    This method never throws for an API error, matching the reference client's contract: any - * request that could not be confirmed processed — whether due to persistent rate-limiting, server - * errors, or a non-retryable client error (e.g. an invalid token or insufficient permissions) — - * is returned in {@link BatchAddResult#getUnprocessedRequests()} instead. A single request whose - * own JSON encoding already exceeds the payload-size limit is rejected up front with {@link + *

    This method's returned future never completes exceptionally due to an API/transport failure, + * matching the reference client's contract: any request that could not be confirmed processed — + * whether due to persistent rate-limiting, server errors, or a non-retryable client error (e.g. + * an invalid token or insufficient permissions) — is returned in {@link + * BatchAddResult#getUnprocessedRequests()} instead. A single request whose own JSON encoding + * already exceeds the payload-size limit is rejected up front with {@link * IllegalArgumentException}, since no chunk size could ever fit it. * *

    Caveat: processed-vs-unprocessed reconciliation after a retry matches requests by @@ -220,7 +230,7 @@ public BatchAddResult batchAddRequests(List requests, boole * caller-supplied key to match it back against the API's per-attempt response. Callers that rely * on {@code batchAddRequests}' return value should set {@code uniqueKey} explicitly. */ - public BatchAddResult batchAddRequests( + public CompletableFuture batchAddRequests( List requests, boolean forefront, BatchAddRequestsOptions options) { long payloadSizeLimitBytes = MAX_PAYLOAD_SIZE_BYTES @@ -235,32 +245,53 @@ public BatchAddResult batchAddRequests( start += chunk.size(); } - BatchAddResult merged = new BatchAddResult(); if (chunks.isEmpty()) { - return merged; + return CompletableFuture.completedFuture(new BatchAddResult()); } - int maxParallel = options.maxParallelValue(); - if (maxParallel <= 1 || chunks.size() == 1) { - for (List chunk : chunks) { - merged.merge(batchAddChunkWithRetries(chunk, forefront, options)); - } - return merged; + int maxParallel = Math.max(1, options.maxParallelValue()); + int workerCount = Math.min(maxParallel, chunks.size()); + BatchAddResult[] resultsByChunk = new BatchAddResult[chunks.size()]; + AtomicInteger nextChunkIndex = new AtomicInteger(0); + + List> workers = new ArrayList<>(workerCount); + for (int w = 0; w < workerCount; w++) { + workers.add(runBatchAddWorker(chunks, nextChunkIndex, resultsByChunk, forefront, options)); } + return CompletableFuture.allOf(workers.toArray(CompletableFuture[]::new)) + .thenApply( + unused -> { + BatchAddResult merged = new BatchAddResult(); + for (BatchAddResult chunkResult : resultsByChunk) { + merged.merge(chunkResult); + } + return merged; + }); + } - ExecutorService pool = Executors.newFixedThreadPool(Math.min(maxParallel, chunks.size())); - try { - List> futures = new ArrayList<>(); - for (List chunk : chunks) { - futures.add(pool.submit(() -> batchAddChunkWithRetries(chunk, forefront, options))); - } - for (Future future : futures) { - merged.merge(awaitResult(future)); - } - } finally { - pool.shutdownNow(); + /** + * One "worker": repeatedly claims the next not-yet-started chunk (via {@code nextChunkIndex}) and + * processes it, until every chunk has been claimed. Running {@code workerCount} of these + * concurrently bounds how many chunk requests (including their own retries) are in flight at + * once, matching {@link BatchAddRequestsOptions#maxParallel} without needing a thread pool - each + * worker is a chain of async continuations, not a blocked thread. + */ + private CompletableFuture runBatchAddWorker( + List> chunks, + AtomicInteger nextChunkIndex, + BatchAddResult[] resultsByChunk, + boolean forefront, + BatchAddRequestsOptions options) { + int index = nextChunkIndex.getAndIncrement(); + if (index >= chunks.size()) { + return CompletableFuture.completedFuture(null); } - return merged; + return batchAddChunkWithRetries(chunks.get(index), forefront, options) + .thenCompose( + result -> { + resultsByChunk[index] = result; + return runBatchAddWorker(chunks, nextChunkIndex, resultsByChunk, forefront, options); + }); } /** @@ -317,59 +348,79 @@ private static long jsonByteLength(Object value) { return Json.toBytes(value).length; } - private static BatchAddResult awaitResult(Future future) { - try { - return future.get(); - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - if (cause instanceof RuntimeException) { - throw (RuntimeException) cause; - } - throw new IllegalStateException("batch add request failed", cause); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new ApifyTransportException(e); - } - } - /** * Adds one chunk (already sized to the API limit), retrying requests the API leaves unprocessed - * with exponential backoff. Never throws regardless of the failure cause, matching the reference - * client's {@code _batchAddRequestsWithRetries}: on any failure — a non-retryable 4xx/5xx API - * response, or a transport-level failure (connection error, timeout) that never produced a - * response at all — the remaining requests in the chunk are simply returned as unprocessed rather - * than surfaced as an exception, so the method keeps a single, uniform never-throws contract - * across both {@link ApifyApiException} and {@link ApifyTransportException}. + * with exponential backoff (a scheduled delay, not a blocked thread; see {@link Async}). The + * returned future never completes exceptionally, matching the reference client's {@code + * _batchAddRequestsWithRetries}: on any failure — a non-retryable 4xx/5xx API response, or a + * transport-level failure (connection error, timeout) that never produced a response at all — the + * remaining requests in the chunk are simply reported as unprocessed rather than surfaced as an + * exception, so the method keeps a single, uniform never-fails contract across both {@link + * ApifyApiException} and {@link com.apify.client.http.ApifyTransportException}. */ - private BatchAddResult batchAddChunkWithRetries( + private CompletableFuture batchAddChunkWithRetries( List chunk, boolean forefront, BatchAddRequestsOptions options) { int maxRetries = options.maxUnprocessedRequestsRetriesValue(); long minDelayMillis = options.minDelayBetweenUnprocessedRequestsRetriesMillisValue(); - - List remaining = chunk; - List processed = new ArrayList<>(); - - for (int attempt = 0; attempt <= maxRetries; attempt++) { - try { - BatchAddResult response = batchAddChunk(remaining, forefront); - processed.addAll(response.getProcessedRequests()); - remaining = requestsNotYetProcessed(chunk, processed); - if (remaining.isEmpty()) { - break; - } - } catch (ApifyClientException ignored) { - // Any client-level failure — an API error response (rate-limit, server error, or a hard - // client error such as a bad token) or a transport failure that never reached the API at - // all (timeout, connection reset) — stops retrying this chunk immediately; whatever has - // not been confirmed processed is reported as unprocessed below, never thrown — matching - // the reference client's contract. - break; - } - if (attempt < maxRetries) { - sleepBackoff(attempt, minDelayMillis); - } - } - + return batchAddAttempt(chunk, chunk, List.of(), forefront, 0, maxRetries, minDelayMillis); + } + + /** One attempt of the retry loop backing {@link #batchAddChunkWithRetries}. */ + private CompletableFuture batchAddAttempt( + List chunk, + List toSend, + List processedSoFar, + boolean forefront, + int attempt, + int maxRetries, + long minDelayMillis) { + return batchAddChunk(toSend, forefront) + .handle( + (response, error) -> { + if (error == null) { + return response; + } + Throwable cause = HttpClientCore.unwrapCompletion(error); + if (cause instanceof ApifyApiException || cause instanceof ApifyTransportException) { + // A non-retryable API error or a transport failure: stop retrying this chunk, with + // whatever was already confirmed processed. Any other (unexpected) exception is + // rethrown rather than swallowed. + return null; + } + throw rethrow(cause); + }) + .thenCompose( + response -> { + List processed = new ArrayList<>(processedSoFar); + if (response != null) { + processed.addAll(response.getProcessedRequests()); + } + List stillRemaining = requestsNotYetProcessed(chunk, processed); + // Any client-level failure (response == null, i.e. the batchAddChunk call above + // failed) stops retrying this chunk immediately; whatever has not been confirmed + // processed is reported as unprocessed below, never thrown - matching the reference + // client's contract. + boolean stop = response == null || stillRemaining.isEmpty() || attempt >= maxRetries; + if (stop) { + return CompletableFuture.completedFuture(finish(chunk, processed)); + } + return Async.delay(backoffDelay(attempt, minDelayMillis)) + .thenCompose( + unused -> + batchAddAttempt( + chunk, + stillRemaining, + processed, + forefront, + attempt + 1, + maxRetries, + minDelayMillis)); + }) + .exceptionally(error -> finish(chunk, processedSoFar)); + } + + private static BatchAddResult finish( + List chunk, List processed) { BatchAddResult result = new BatchAddResult(); result.setProcessedRequests(processed); // Unprocessed = everything sent minus everything acknowledged processed. Computing it here @@ -395,25 +446,22 @@ private static List requestsNotYetProcessed( return remaining; } - private static void sleepBackoff(int attempt, long minDelayMillis) { + /** + * {@code (1 + random) * 2^attempt * minDelay} — exponential backoff with jitter, matching the + * reference. The exponent is capped so a pathologically large retry count cannot overflow the + * delay into an absurd (or negative) duration. + */ + private static Duration backoffDelay(int attempt, long minDelayMillis) { if (minDelayMillis <= 0) { - return; + return Duration.ZERO; } - // (1 + random) * 2^attempt * minDelay — exponential backoff with jitter, matching the - // reference. The exponent is capped so a pathologically large retry count cannot overflow the - // delay into an absurd (or negative, after the long cast) sleep. int cappedAttempt = Math.min(attempt, MAX_BACKOFF_EXPONENT); double factor = (1 + ThreadLocalRandom.current().nextDouble()) * Math.pow(2, cappedAttempt); - long delayMillis = (long) Math.floor(factor * minDelayMillis); - try { - Thread.sleep(delayMillis); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new ApifyTransportException(e); - } + return Duration.ofMillis((long) Math.floor(factor * minDelayMillis)); } - private BatchAddResult batchAddChunk(List requests, boolean forefront) { + private CompletableFuture batchAddChunk( + List requests, boolean forefront) { QueryParams params = new QueryParams(); params.addBool("forefront", forefront); withClientKey(params); @@ -429,13 +477,13 @@ private BatchAddResult batchAddChunk(List requests, boolean * Deletes multiple requests in a single call. Each entry identifies a request (e.g. by id or * uniqueKey). */ - public BatchDeleteResult batchDeleteRequests(Object requests) { + public CompletableFuture batchDeleteRequests(Object requests) { QueryParams params = withClientKey(new QueryParams()); return ctx.deleteWithBody("requests/batch", params, requests, BatchDeleteResult.class); } /** Lists the queue's requests with pagination. */ - public RequestsList listRequests(ListRequestsOptions options) { + public CompletableFuture listRequests(ListRequestsOptions options) { options.validate(); QueryParams params = new QueryParams(); options.apply(params); @@ -447,7 +495,8 @@ public RequestsList listRequests(ListRequestsOptions options) { * Extends the lock on a request by {@code lockSecs} seconds. If {@code forefront} is true, the * request is moved to the front when its lock expires. */ - public RequestLockInfo prolongRequestLock(String id, long lockSecs, boolean forefront) { + public CompletableFuture prolongRequestLock( + String id, long lockSecs, boolean forefront) { QueryParams params = new QueryParams(); params.addLong("lockSecs", lockSecs).addBool("forefront", forefront); withClientKey(params); @@ -463,43 +512,49 @@ public RequestLockInfo prolongRequestLock(String id, long lockSecs, boolean fore * Releases the lock on a request. If {@code forefront} is true, the request is moved to the front * of the queue. */ - public void deleteRequestLock(String id, boolean forefront) { + public CompletableFuture deleteRequestLock(String id, boolean forefront) { QueryParams params = new QueryParams(); params.addBool("forefront", forefront); withClientKey(params); String url = ctx.mergedParams(params) .applyToUrl(ctx.subUrl("requests/" + ResourceContext.encodePathSegment(id) + "/lock")); - try { - http.call("DELETE", url, null, "", http.baseRequestTimeout()); - } catch (ApifyApiException e) { - if (!ResourceContext.isNotFound(e)) { - throw e; - } - } + return http.call("DELETE", url, null, "", http.baseRequestTimeout()) + .handle( + (resp, error) -> { + if (error == null) { + return null; + } + Throwable cause = HttpClientCore.unwrapCompletion(error); + if (cause instanceof ApifyApiException apiError + && ResourceContext.isNotFound(apiError)) { + return null; + } + throw rethrow(cause); + }); } /** Releases all locks the client holds on this queue's requests. */ - public UnlockRequestsResult unlockRequests() { + public CompletableFuture unlockRequests() { QueryParams params = withClientKey(new QueryParams()); return ctx.postWithBody("requests/unlock", params, null, "", UnlockRequestsResult.class); } /** - * Returns a lazy iterator over all requests in the queue, fetching pages of up to {@code - * pageLimit} requests at a time ({@code null} for the server default). Equivalent to {@link - * #paginateRequests(Long, Long, List) paginateRequests(null, pageLimit, null)} (no total cap, no - * state filter). + * Returns a lazy, backpressure-aware publisher over all requests in the queue, fetching pages of + * up to {@code pageLimit} requests at a time ({@code null} for the server default). Equivalent to + * {@link #paginateRequests(Long, Long, List) paginateRequests(null, pageLimit, null)} (no total + * cap, no state filter). */ - public Iterator paginateRequests(Long pageLimit) { + public Flow.Publisher paginateRequests(Long pageLimit) { return paginateRequests(null, pageLimit, null); } /** - * Returns a lazy iterator over the queue's requests via the cursor-based listing endpoint. {@code - * totalLimit} caps the total number of requests yielded across all pages ({@code - * null}/non-positive = unbounded); {@code chunkSize} is the per-request page size ({@code null} = - * server default); {@code filter} restricts to requests in the given states ({@link + * Returns a lazy, backpressure-aware publisher over the queue's requests via the cursor-based + * listing endpoint. {@code totalLimit} caps the total number of requests yielded across all pages + * ({@code null}/non-positive = unbounded); {@code chunkSize} is the per-request page size ({@code + * null} = server default); {@code filter} restricts to requests in the given states ({@link * ListRequestsOptions#FILTER_LOCKED}/{@link ListRequestsOptions#FILTER_PENDING}, {@code null} = * no filter), matching {@link #listRequests(ListRequestsOptions)}'s filter. * @@ -507,79 +562,182 @@ public Iterator paginateRequests(Long pageLimit) { * exclusiveStartId}/{@code cursor} is not supported here (use {@link * #listRequests(ListRequestsOptions)} directly for that single-page use case). */ - public Iterator paginateRequests( + public Flow.Publisher paginateRequests( Long totalLimit, Long chunkSize, List filter) { - return new RequestsIterator(totalLimit, chunkSize, filter); + return new RequestsPublisher(totalLimit, chunkSize, filter); } - /** Lazily iterates over a request queue's requests via the cursor-based listing endpoint. */ - private final class RequestsIterator implements Iterator { + /** + * Lazily publishes a request queue's requests via the cursor-based listing endpoint, fetching a + * page only once the subscriber has signalled demand. Supports a single subscriber, like {@link + * com.apify.client.internal.AsyncPaginatedPublisher}, whose draining design this mirrors (cursor + * pagination instead of offset/limit, plus the "not yet started" distinction the cursor-based + * termination check needs, are the only real differences). + */ + private final class RequestsPublisher implements Flow.Publisher { private final Long totalLimit; private final Long chunkSize; private final List filter; - private List buffer = List.of(); - private int pos; - private String nextCursor; - private long yielded; - private boolean started; - private boolean exhausted; - - RequestsIterator(Long totalLimit, Long chunkSize, List filter) { + private final AtomicBoolean subscribed = new AtomicBoolean(); + + RequestsPublisher(Long totalLimit, Long chunkSize, List filter) { this.totalLimit = totalLimit != null && totalLimit > 0 ? totalLimit : null; this.chunkSize = chunkSize; this.filter = filter; } @Override - public boolean hasNext() { - while (pos >= buffer.size()) { - if (exhausted || (started && (nextCursor == null || nextCursor.isEmpty()))) { - return false; + public void subscribe(Flow.Subscriber subscriber) { + if (!subscribed.compareAndSet(false, true)) { + subscriber.onSubscribe( + new Flow.Subscription() { + @Override + public void request(long n) {} + + @Override + public void cancel() {} + }); + subscriber.onError( + new IllegalStateException( + "This publisher supports only a single subscriber (already subscribed)")); + return; + } + subscriber.onSubscribe(new Session(subscriber)); + } + + private final class Session implements Flow.Subscription { + private final Flow.Subscriber subscriber; + private final AtomicLong requested = new AtomicLong(); + private final AtomicBoolean draining = new AtomicBoolean(); + private final AtomicBoolean cancelled = new AtomicBoolean(); + private final AtomicBoolean terminated = new AtomicBoolean(); + + // Only touched from within the draining-guarded section; see AsyncPaginatedPublisher.Session. + private List buffer = List.of(); + private int pos; + private String nextCursor; + private long yielded; + private boolean started; + private boolean exhausted; + + Session(Flow.Subscriber subscriber) { + this.subscriber = subscriber; + } + + @Override + public void request(long n) { + if (n <= 0) { + if (terminated.compareAndSet(false, true)) { + subscriber.onError( + new IllegalArgumentException( + "Reactive Streams violation: requested amount must be positive, was " + n)); + } + return; } - if (totalLimit != null && yielded >= totalLimit) { - return false; + requested.updateAndGet(current -> current + n < 0 ? Long.MAX_VALUE : current + n); + drain(); + } + + @Override + public void cancel() { + cancelled.set(true); + } + + private void drain() { + if (draining.compareAndSet(false, true)) { + drainLoop(); } - fetchPage(); } - return true; - } - @Override - public RequestQueueRequest next() { - if (!hasNext()) { - throw new NoSuchElementException(); + private boolean isExhausted() { + return exhausted + || (started && (nextCursor == null || nextCursor.isEmpty())) + || (totalLimit != null && yielded >= totalLimit); } - yielded++; - return buffer.get(pos++); - } - private void fetchPage() { - QueryParams params = new QueryParams(); - Long capRemaining = totalLimit != null ? totalLimit - yielded : null; - Long pageLimit = - capRemaining == null - ? chunkSize - : (chunkSize == null ? capRemaining : Math.min(capRemaining, chunkSize)); - params.addLong("limit", pageLimit); - if (nextCursor != null && !nextCursor.isEmpty()) { - params.addString("cursor", nextCursor); + private void drainLoop() { + while (!cancelled.get() && requested.get() > 0 && pos < buffer.size()) { + RequestQueueRequest item = buffer.get(pos++); + yielded++; + requested.decrementAndGet(); + subscriber.onNext(item); + } + if (cancelled.get()) { + draining.set(false); + return; + } + if (pos < buffer.size()) { + draining.set(false); + if (requested.get() > 0 && pos < buffer.size()) { + drain(); + } + return; + } + if (isExhausted()) { + if (terminated.compareAndSet(false, true)) { + subscriber.onComplete(); + } + draining.set(false); + return; + } + fetchPage(); } - params.addCsv("filter", filter); - withClientKey(params); - RequestsList page = ctx.getResourceRequired("requests", params, RequestsList.class); - started = true; - buffer = page.getItems(); - pos = 0; - nextCursor = page.getNextCursor(); - // Defensively trim to the cap in case the server returned more than requested, matching - // PaginatedIterator's same guard, so a caller never sees more than `totalLimit` items even - // if the server ignores (or overshoots) the requested per-page `limit`. - if (capRemaining != null && buffer.size() > capRemaining) { - buffer = buffer.subList(0, capRemaining.intValue()); + + private void fetchPage() { + QueryParams params = new QueryParams(); + Long capRemaining = totalLimit != null ? totalLimit - yielded : null; + Long pageLimit = + capRemaining == null + ? chunkSize + : (chunkSize == null ? capRemaining : Math.min(capRemaining, chunkSize)); + params.addLong("limit", pageLimit); + if (nextCursor != null && !nextCursor.isEmpty()) { + params.addString("cursor", nextCursor); + } + params.addCsv("filter", filter); + withClientKey(params); + ctx.getResourceRequired("requests", params, RequestsList.class) + .whenComplete( + (page, error) -> { + if (cancelled.get()) { + draining.set(false); + return; + } + if (error != null) { + if (terminated.compareAndSet(false, true)) { + subscriber.onError(HttpClientCore.unwrapCompletion(error)); + } + draining.set(false); + return; + } + applyPage(page, capRemaining); + drainLoop(); + }); } - if (buffer.isEmpty() && (nextCursor == null || nextCursor.isEmpty())) { - exhausted = true; + + private void applyPage(RequestsList page, Long capRemaining) { + started = true; + buffer = page.getItems(); + pos = 0; + nextCursor = page.getNextCursor(); + // Defensively trim to the cap in case the server returned more than requested, matching + // AsyncPaginatedPublisher's same guard, so a subscriber never sees more than `totalLimit` + // items even if the server ignores (or overshoots) the requested per-page `limit`. + if (capRemaining != null && buffer.size() > capRemaining) { + buffer = buffer.subList(0, capRemaining.intValue()); + } + if (buffer.isEmpty() && (nextCursor == null || nextCursor.isEmpty())) { + exhausted = true; + } } } } + + /** Rethrows a classified cause as an unchecked exception, as-is (defensive fallback only). */ + private static RuntimeException rethrow(Throwable cause) { + if (cause instanceof RuntimeException runtimeException) { + return runtimeException; + } + return new ApifyTransportException(cause); + } } diff --git a/src/main/java/com/apify/client/requestqueue/RequestQueueCollectionClient.java b/src/main/java/com/apify/client/requestqueue/RequestQueueCollectionClient.java index 93fa6ab..e189b4d 100644 --- a/src/main/java/com/apify/client/requestqueue/RequestQueueCollectionClient.java +++ b/src/main/java/com/apify/client/requestqueue/RequestQueueCollectionClient.java @@ -5,6 +5,7 @@ import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.ResourceContext; +import java.util.concurrent.CompletableFuture; /** A client for the request queue collection ({@code GET/POST /v2/request-queues}). */ public final class RequestQueueCollectionClient @@ -21,7 +22,7 @@ public RequestQueueCollectionClient(HttpClientCore http, String baseUrl) { * Gets the queue with the given name, creating it if it does not exist. An empty/{@code null} * name creates a new unnamed queue. */ - public RequestQueue getOrCreate(String name) { + public CompletableFuture getOrCreate(String name) { return ctx.getOrCreateNamed(name, RequestQueue.class); } } diff --git a/src/main/java/com/apify/client/requestqueue/RequestQueueRequest.java b/src/main/java/com/apify/client/requestqueue/RequestQueueRequest.java index 511904e..245db39 100644 --- a/src/main/java/com/apify/client/requestqueue/RequestQueueRequest.java +++ b/src/main/java/com/apify/client/requestqueue/RequestQueueRequest.java @@ -1,12 +1,12 @@ package com.apify.client.requestqueue; import com.apify.client.ApifyResource; -import com.fasterxml.jackson.databind.JsonNode; import java.time.Instant; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import tools.jackson.databind.JsonNode; /** * A single request stored in a request queue. Fields left {@code null} are omitted when the request diff --git a/src/main/java/com/apify/client/run/ActorRun.java b/src/main/java/com/apify/client/run/ActorRun.java index a1dc609..059ddd7 100644 --- a/src/main/java/com/apify/client/run/ActorRun.java +++ b/src/main/java/com/apify/client/run/ActorRun.java @@ -2,10 +2,10 @@ import com.apify.client.ApifyResource; import com.apify.client.internal.Statuses; -import com.fasterxml.jackson.databind.JsonNode; import java.time.Instant; import java.util.Collections; import java.util.Map; +import tools.jackson.databind.JsonNode; /** A single execution of an Actor. */ public final class ActorRun extends ApifyResource { diff --git a/src/main/java/com/apify/client/run/RunClient.java b/src/main/java/com/apify/client/run/RunClient.java index 197ba8b..a8dd46b 100644 --- a/src/main/java/com/apify/client/run/RunClient.java +++ b/src/main/java/com/apify/client/run/RunClient.java @@ -1,7 +1,6 @@ package com.apify.client.run; import com.apify.client.ApifyClient; -import com.apify.client.actor.Actor; import com.apify.client.dataset.DatasetClient; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.Json; @@ -15,6 +14,7 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Consumer; import org.slf4j.Logger; @@ -75,7 +75,7 @@ public static RunClient lastRun( } /** Fetches the run object, or empty if it does not exist. */ - public Optional get() { + public CompletableFuture> get() { return getWithWait(null); } @@ -85,7 +85,7 @@ public Optional get() { * responds before the client's per-request timeout; the API itself caps server-side waiting at 60 * seconds. Pass {@code null} for an immediate fetch. */ - public Optional getWithWait(Long waitForFinishSecs) { + public CompletableFuture> getWithWait(Long waitForFinishSecs) { QueryParams params = new QueryParams(); // Clamp to the client's per-request timeout so a short custom timeout doesn't abort the call. params.addLong("waitForFinish", ctx.clampServerWait(waitForFinishSecs)); @@ -93,13 +93,13 @@ public Optional getWithWait(Long waitForFinishSecs) { } /** Updates the run with the given fields and returns the updated object. */ - public ActorRun update(Object newFields) { + public CompletableFuture update(Object newFields) { return ctx.updateResource("", newFields, ActorRun.class); } /** Deletes the run. */ - public void delete() { - ctx.deleteResource(""); + public CompletableFuture delete() { + return ctx.deleteResource(""); } /** @@ -107,7 +107,7 @@ public void delete() { * the current request before terminating; if {@code false} it is aborted immediately. Pass {@code * null} to omit the parameter and let the server apply its default (immediate abort). */ - public ActorRun abort(Boolean gracefully) { + public CompletableFuture abort(Boolean gracefully) { QueryParams params = new QueryParams(); params.addBool("gracefully", gracefully); return ctx.postWithBody("abort", params, null, "", ActorRun.class); @@ -119,7 +119,8 @@ public ActorRun abort(Boolean gracefully) { * URL-safe {@code username~actor-name} form before sending, matching the reference client's * {@code _toSafeId}); {@code input} is the new input ({@code null} for none). */ - public ActorRun metamorph(String targetActorId, Object input, MetamorphOptions options) { + public CompletableFuture metamorph( + String targetActorId, Object input, MetamorphOptions options) { if (targetActorId == null || targetActorId.isEmpty()) { throw new IllegalArgumentException("targetActorId is required and must not be empty"); } @@ -137,12 +138,12 @@ public ActorRun metamorph(String targetActorId, Object input, MetamorphOptions o } /** Reboots the run (restarts its container while keeping the same run). */ - public ActorRun reboot() { + public CompletableFuture reboot() { return ctx.postWithBody("reboot", new QueryParams(), null, "", ActorRun.class); } /** Resurrects a finished run, starting it again from the beginning. */ - public ActorRun resurrect(RunResurrectOptions options) { + public CompletableFuture resurrect(RunResurrectOptions options) { if (options == null) { throw new IllegalArgumentException("options is required and must not be null"); } @@ -158,7 +159,7 @@ public ActorRun resurrect(RunResurrectOptions options) { *

    An idempotency key is always sent (auto-generated if not provided), so a charge that is * retried by the transport is applied at most once, matching the reference client. */ - public void charge(RunChargeOptions options) { + public CompletableFuture charge(RunChargeOptions options) { String eventName = options.eventNameValue(); if (eventName == null || eventName.isEmpty()) { throw new IllegalArgumentException( @@ -175,13 +176,15 @@ public void charge(RunChargeOptions options) { // Route through mergedParams like the run's other actions, so a last-run-seeded context charges // the same run its read methods resolve to (its pinned status/origin filters are preserved). String url = ctx.mergedParams(new QueryParams()).applyToUrl(ctx.subUrl("charge")); - ctx.http.call( - "POST", - url, - Json.toBytes(body), - ResourceContext.CONTENT_TYPE_JSON, - headers, - ctx.http.baseRequestTimeout()); + return ctx.http + .call( + "POST", + url, + Json.toBytes(body), + ResourceContext.CONTENT_TYPE_JSON, + headers, + ctx.http.baseRequestTimeout()) + .thenApply(resp -> null); } /** @@ -201,12 +204,12 @@ private String generateIdempotencyKey(String eventName) { /** * Polls until the run reaches a terminal state or {@code waitSecs} elapses ({@code null} waits - * indefinitely). Returns the latest run fetched, whether or not it is terminal: if the wait - * budget runs out first, this returns the still-running run as of the last poll rather than - * throwing or blocking further — check {@link ActorRun#isTerminal()} on the result if the - * distinction matters to the caller. + * indefinitely). Completes with the latest run fetched, whether or not it is terminal: if the + * wait budget runs out first, this completes with the still-running run as of the last poll + * rather than failing or waiting further — check {@link ActorRun#isTerminal()} on the result if + * the distinction matters to the caller. */ - public ActorRun waitForFinish(Long waitSecs) { + public CompletableFuture waitForFinish(Long waitSecs) { return ctx.waitForFinish(waitSecs, "run", Json.type(ActorRun.class), ActorRun::isTerminal); } @@ -234,8 +237,8 @@ public LogClient log() { private static final String REDIRECT_LOGGER_NAME = "com.apify.client.ActorRunLog"; /** - * Returns a {@link StreamedLog} that redirects this run's live log to a default per-run SLF4J - * logger. + * Completes with a {@link StreamedLog} that redirects this run's live log to a default per-run + * SLF4J logger. * *

    The default destination is a {@link Logger} that receives each message at {@code INFO} * level, prefixed with the Actor name and run id (built by fetching the run and its Actor). Call @@ -246,27 +249,35 @@ public LogClient log() { * #getStreamedLog(StreamedLogOptions)}. For raw stream access without redirection, use {@link * #log()}{@code .stream(...)}. */ - public StreamedLog getStreamedLog() { + public CompletableFuture getStreamedLog() { return getStreamedLog(new StreamedLogOptions()); } /** - * Returns a {@link StreamedLog} that redirects this run's live log according to {@code options}. + * Completes with a {@link StreamedLog} that redirects this run's live log according to {@code + * options}. * *

    If {@link StreamedLogOptions#toLog(Consumer)} is set, each complete log message is passed to * that consumer; otherwise a default {@link Logger} destination is used with a per-run prefix (an - * explicit {@link StreamedLogOptions#prefix(String)} overrides the auto-built one). {@link - * StreamedLogOptions#fromStart(boolean)} controls whether log lines produced before redirection - * started are included. + * explicit {@link StreamedLogOptions#prefix(String)} overrides the auto-built one, in which case + * this completes without needing to fetch the run). {@link StreamedLogOptions#fromStart(boolean)} + * controls whether log lines produced before redirection started are included. */ - public StreamedLog getStreamedLog(StreamedLogOptions options) { + public CompletableFuture getStreamedLog(StreamedLogOptions options) { Consumer destination = options.destination(); - if (destination == null) { - String prefix = - options.prefixValue() != null ? options.prefixValue() : buildDefaultLogPrefix(); - destination = defaultLogDestination(prefix); + if (destination != null) { + return CompletableFuture.completedFuture( + new StreamedLog(log(), destination, options.fromStartValue())); } - return new StreamedLog(log(), destination, options.fromStartValue()); + if (options.prefixValue() != null) { + return CompletableFuture.completedFuture( + new StreamedLog( + log(), defaultLogDestination(options.prefixValue()), options.fromStartValue())); + } + return buildDefaultLogPrefix() + .thenApply( + prefix -> + new StreamedLog(log(), defaultLogDestination(prefix), options.fromStartValue())); } /** @@ -274,30 +285,40 @@ public StreamedLog getStreamedLog(StreamedLogOptions options) { * client: the Actor name (looked up from the run) and {@code runId:{id}}, joined with a space and * followed by {@code " -> "}. Falls back to just the run id when the Actor name is unavailable. */ - private String buildDefaultLogPrefix() { + private CompletableFuture buildDefaultLogPrefix() { // Fall back to the field `id`: correct for a direct run client, but it is the literal string // "last" for a `runs/last` accessor (see #lastRun) — replaced with the resolved run's real id // below as soon as the fetch succeeds. - String resolvedId = id; - String actorName = ""; - try { - Optional run = get(); - if (run.isPresent()) { - resolvedId = run.get().getId(); - if (run.get().getActId() != null && !run.get().getActId().isEmpty()) { - Optional actor = root.actor(run.get().getActId()).get(); - if (actor.isPresent() && actor.get().getName() != null) { - actorName = actor.get().getName(); - } - } - } - } catch (RuntimeException e) { - // The Actor-name lookup is cosmetic. The getters swallow 404, but an auth (401/403), - // transport, or 5xx-after-retries failure would otherwise throw out of getStreamedLog() and - // abort helper creation even though streaming itself might have worked. Fall back to the - // runId-only prefix (actorName stays "", resolvedId stays the unresolved field) so the helper - // is always created. - } + return get() + .thenCompose( + run -> { + if (run.isEmpty()) { + return CompletableFuture.completedFuture(formatPrefix(id, "")); + } + String resolvedId = run.get().getId(); + String actId = run.get().getActId(); + if (actId == null || actId.isEmpty()) { + return CompletableFuture.completedFuture(formatPrefix(resolvedId, "")); + } + return root.actor(actId) + .get() + .thenApply( + actor -> { + String actorName = + (actor.isPresent() && actor.get().getName() != null) + ? actor.get().getName() + : ""; + return formatPrefix(resolvedId, actorName); + }); + }) + // The Actor-name lookup is cosmetic. The getters treat 404 as empty, but an auth (401/403), + // transport, or 5xx-after-retries failure would otherwise fail getStreamedLog() and abort + // helper creation even though streaming itself might have worked. Fall back to the + // runId-only prefix so the helper is always created. + .exceptionally(error -> formatPrefix(id, "")); + } + + private static String formatPrefix(String resolvedId, String actorName) { String runPart = "runId:" + resolvedId; String name = actorName.isEmpty() ? runPart : actorName + " " + runPart; return name + " -> "; diff --git a/src/main/java/com/apify/client/run/RunCollectionClient.java b/src/main/java/com/apify/client/run/RunCollectionClient.java index 0c15984..021e0d2 100644 --- a/src/main/java/com/apify/client/run/RunCollectionClient.java +++ b/src/main/java/com/apify/client/run/RunCollectionClient.java @@ -6,7 +6,8 @@ import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; -import java.util.Iterator; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; /** * A client for a run collection: the account-wide collection ({@code GET /v2/actor-runs}), an @@ -29,7 +30,8 @@ public RunCollectionClient(HttpClientCore http, String baseUrl, String resourceP * Lists runs, applying the standard pagination and the run-specific filters. Both {@code options} * and {@code filter} may be {@code null}, which is treated as "no options"/"no filter". */ - public PaginationList list(ListOptions options, RunListOptions filter) { + public CompletableFuture> list( + ListOptions options, RunListOptions filter) { QueryParams params = new QueryParams(); if (options != null) { options.apply(params); @@ -41,12 +43,12 @@ public PaginationList list(ListOptions options, RunListOptions filter) } /** - * Returns a lazy iterator over the runs, applying the standard pagination and run-specific - * filters. The options' {@code limit} caps the total number yielded ({@code null} or non-positive - * = all); {@code chunkSize} is the per-request page size ({@code null} = server default). Both - * {@code options} and {@code filter} may be {@code null}. + * Returns a lazy, backpressure-aware publisher over the runs, applying the standard pagination + * and run-specific filters. The options' {@code limit} caps the total number yielded ({@code + * null} or non-positive = all); {@code chunkSize} is the per-request page size ({@code null} = + * server default). Both {@code options} and {@code filter} may be {@code null}. */ - public Iterator iterate(ListOptions options, RunListOptions filter) { + public Flow.Publisher iterate(ListOptions options, RunListOptions filter) { return iterate(options, filter, null); } @@ -54,7 +56,8 @@ public Iterator iterate(ListOptions options, RunListOptions filter) { * As {@link #iterate(ListOptions, RunListOptions)}, but {@code chunkSize} sets the per-request * page size. */ - public Iterator iterate(ListOptions options, RunListOptions filter, Long chunkSize) { + public Flow.Publisher iterate( + ListOptions options, RunListOptions filter, Long chunkSize) { ListOptions opts = options != null ? options : new ListOptions(); return iterateWithFilters( opts.limitValue(), diff --git a/src/main/java/com/apify/client/schedule/Schedule.java b/src/main/java/com/apify/client/schedule/Schedule.java index f2856ef..28ddb92 100644 --- a/src/main/java/com/apify/client/schedule/Schedule.java +++ b/src/main/java/com/apify/client/schedule/Schedule.java @@ -1,10 +1,10 @@ package com.apify.client.schedule; import com.apify.client.ApifyResource; -import com.fasterxml.jackson.databind.JsonNode; import java.time.Instant; import java.util.Collections; import java.util.List; +import tools.jackson.databind.JsonNode; /** A schedule automatically starts Actor or task runs at specified times. */ public final class Schedule extends ApifyResource { diff --git a/src/main/java/com/apify/client/schedule/ScheduleClient.java b/src/main/java/com/apify/client/schedule/ScheduleClient.java index 9604a74..81a81c8 100644 --- a/src/main/java/com/apify/client/schedule/ScheduleClient.java +++ b/src/main/java/com/apify/client/schedule/ScheduleClient.java @@ -1,12 +1,12 @@ package com.apify.client.schedule; -import com.apify.client.http.ApiResponse; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; import java.nio.charset.StandardCharsets; import java.util.Optional; +import java.util.concurrent.CompletableFuture; /** A client for a specific schedule ({@code /v2/schedules/{scheduleId}}). */ public final class ScheduleClient { @@ -17,26 +17,27 @@ public ScheduleClient(HttpClientCore http, String baseUrl, String id) { } /** Fetches the schedule, or empty if it does not exist. */ - public Optional get() { + public CompletableFuture> get() { return ctx.getResource("", new QueryParams(), Schedule.class); } /** Updates the schedule with the given fields and returns the updated object. */ - public Schedule update(Object newFields) { + public CompletableFuture update(Object newFields) { return ctx.updateResource("", newFields, Schedule.class); } /** Deletes the schedule. */ - public void delete() { - ctx.deleteResource(""); + public CompletableFuture delete() { + return ctx.deleteResource(""); } /** Fetches the schedule's invocation log as text, or empty if absent. */ - public Optional getLog() { - ApiResponse resp = ctx.getRaw("log", new QueryParams()); - if (resp == null) { - return Optional.empty(); - } - return Optional.of(new String(resp.body(), StandardCharsets.UTF_8)); + public CompletableFuture> getLog() { + return ctx.getRaw("log", new QueryParams()) + .thenApply( + resp -> + resp == null + ? Optional.empty() + : Optional.of(new String(resp.body(), StandardCharsets.UTF_8))); } } diff --git a/src/main/java/com/apify/client/schedule/ScheduleCollectionClient.java b/src/main/java/com/apify/client/schedule/ScheduleCollectionClient.java index a19904e..0f02e13 100644 --- a/src/main/java/com/apify/client/schedule/ScheduleCollectionClient.java +++ b/src/main/java/com/apify/client/schedule/ScheduleCollectionClient.java @@ -6,6 +6,7 @@ import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; +import java.util.concurrent.CompletableFuture; /** A client for the schedule collection ({@code GET/POST /v2/schedules}). */ public final class ScheduleCollectionClient @@ -19,7 +20,7 @@ public ScheduleCollectionClient(HttpClientCore http, String baseUrl) { } /** Creates a new schedule. {@code schedule} is any JSON-serializable schedule definition. */ - public Schedule create(Object schedule) { + public CompletableFuture create(Object schedule) { return ctx.createResource(new QueryParams(), schedule, Schedule.class); } } diff --git a/src/main/java/com/apify/client/task/Task.java b/src/main/java/com/apify/client/task/Task.java index 531af1f..53231cf 100644 --- a/src/main/java/com/apify/client/task/Task.java +++ b/src/main/java/com/apify/client/task/Task.java @@ -2,8 +2,8 @@ import com.apify.client.ApifyResource; import com.apify.client.actor.ActorStandby; -import com.fasterxml.jackson.databind.JsonNode; import java.time.Instant; +import tools.jackson.databind.JsonNode; /** A pre-configured Actor run (an Actor task). */ public final class Task extends ApifyResource { diff --git a/src/main/java/com/apify/client/task/TaskClient.java b/src/main/java/com/apify/client/task/TaskClient.java index 0802289..4404017 100644 --- a/src/main/java/com/apify/client/task/TaskClient.java +++ b/src/main/java/com/apify/client/task/TaskClient.java @@ -1,7 +1,6 @@ package com.apify.client.task; import com.apify.client.ApifyClient; -import com.apify.client.http.ApiResponse; import com.apify.client.internal.ApiPaths; import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.Json; @@ -13,8 +12,9 @@ import com.apify.client.run.RunClient; import com.apify.client.run.RunCollectionClient; import com.apify.client.webhook.NestedWebhookCollectionClient; -import com.fasterxml.jackson.databind.JsonNode; import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import tools.jackson.databind.JsonNode; /** * A client for a specific Actor task. @@ -34,42 +34,42 @@ public TaskClient(ApifyClient root, HttpClientCore http, String baseUrl, String } /** Fetches the task object, or empty if it does not exist. */ - public Optional get() { + public CompletableFuture> get() { return ctx.getResource("", new QueryParams(), Task.class); } /** Updates the task with the given fields and returns the updated object. */ - public Task update(Object newFields) { + public CompletableFuture update(Object newFields) { return ctx.updateResource("", newFields, Task.class); } /** Deletes the task. */ - public void delete() { - ctx.deleteResource(""); + public CompletableFuture delete() { + return ctx.deleteResource(""); } /** - * Starts the task and returns immediately with the created run. {@code input} optionally - * overrides the task's stored input ({@code null} to use the stored input). + * Starts the task and completes with the created run as soon as it exists (no waiting). {@code + * input} optionally overrides the task's stored input ({@code null} to use the stored input). */ - public ActorRun start(Object input, TaskStartOptions options) { + public CompletableFuture start(Object input, TaskStartOptions options) { return RunStartSupport.start(ctx, input, options::apply, options.contentTypeOrDefault()); } /** - * Starts the task and waits (client-side polling) for it to finish. {@code waitSecs} bounds the + * Starts the task and waits (non-blocking, polling) for it to finish. {@code waitSecs} bounds the * wait; {@code null} waits indefinitely. * *

    This overload does not stream the run's log; use {@link #call(Object, TaskCallOptions, * Long)} for that (matching the reference client's default {@code call} behavior). */ - public ActorRun call(Object input, TaskStartOptions options, Long waitSecs) { + public CompletableFuture call(Object input, TaskStartOptions options, Long waitSecs) { return RunStartSupport.call( root, ctx, input, options::apply, options.contentTypeOrDefault(), waitSecs); } /** - * Starts the task and waits (client-side polling) for it to finish, additionally streaming the + * Starts the task and waits (non-blocking, polling) for it to finish, additionally streaming the * run's log for the duration of the wait — matching the reference client's {@code call}, whose * {@code options.log} defaults to {@code 'default'}. {@code waitSecs} bounds the wait; {@code * null} waits indefinitely. @@ -79,7 +79,7 @@ public ActorRun call(Object input, TaskStartOptions options, Long waitSecs) { * TaskCallOptions#disableLogStreaming()} to opt out entirely, or {@link * TaskCallOptions#logOptions(com.apify.client.log.StreamedLogOptions)} for a custom destination. */ - public ActorRun call(Object input, TaskCallOptions options, Long waitSecs) { + public CompletableFuture call(Object input, TaskCallOptions options, Long waitSecs) { TaskCallOptions opts = options != null ? options : new TaskCallOptions(); TaskStartOptions startOptions = opts.toStartOptions(); return RunStartSupport.callWithLogStreaming( @@ -94,16 +94,17 @@ public ActorRun call(Object input, TaskCallOptions options, Long waitSecs) { } /** Fetches the task's stored input, or empty if none is set. */ - public Optional getInput() { - ApiResponse resp = ctx.getRaw("input", new QueryParams()); - if (resp == null) { - return Optional.empty(); - } - return Optional.of(Json.parse(resp.body(), JsonNode.class)); + public CompletableFuture> getInput() { + return ctx.getRaw("input", new QueryParams()) + .thenApply( + resp -> + resp == null + ? Optional.empty() + : Optional.of(Json.parse(resp.body(), JsonNode.class))); } /** Replaces the task's stored input and returns the updated input. */ - public JsonNode updateInput(Object input) { + public CompletableFuture updateInput(Object input) { return ctx.putWithBodyNoEnvelope( "input", new QueryParams(), diff --git a/src/main/java/com/apify/client/task/TaskCollectionClient.java b/src/main/java/com/apify/client/task/TaskCollectionClient.java index ef99d94..298920b 100644 --- a/src/main/java/com/apify/client/task/TaskCollectionClient.java +++ b/src/main/java/com/apify/client/task/TaskCollectionClient.java @@ -6,6 +6,7 @@ import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; +import java.util.concurrent.CompletableFuture; /** A client for the Actor task collection ({@code GET/POST /v2/actor-tasks}). */ public final class TaskCollectionClient extends AbstractCollectionClient { @@ -18,7 +19,7 @@ public TaskCollectionClient(HttpClientCore http, String baseUrl) { } /** Creates a new task. {@code task} is any JSON-serializable task definition. */ - public Task create(Object task) { + public CompletableFuture create(Object task) { return ctx.createResource(new QueryParams(), task, Task.class); } } diff --git a/src/main/java/com/apify/client/user/User.java b/src/main/java/com/apify/client/user/User.java index 2b73342..17494d7 100644 --- a/src/main/java/com/apify/client/user/User.java +++ b/src/main/java/com/apify/client/user/User.java @@ -1,8 +1,8 @@ package com.apify.client.user; import com.apify.client.ApifyResource; -import com.fasterxml.jackson.databind.JsonNode; import java.time.Instant; +import tools.jackson.databind.JsonNode; /** * An Apify user account. Private account details (available only for {@code me}) are typed below diff --git a/src/main/java/com/apify/client/user/UserClient.java b/src/main/java/com/apify/client/user/UserClient.java index f0042c8..b414c11 100644 --- a/src/main/java/com/apify/client/user/UserClient.java +++ b/src/main/java/com/apify/client/user/UserClient.java @@ -5,8 +5,9 @@ import com.apify.client.internal.Json; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; -import com.fasterxml.jackson.databind.JsonNode; import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import tools.jackson.databind.JsonNode; /** * A client for accessing user data ({@code /v2/users/{userId}} or {@code /v2/users/me}). @@ -33,15 +34,15 @@ public UserClient(HttpClientCore http, String baseUrl, String id) { * User#getExtra()}); for other users it returns the public profile. Returns empty if the user * does not exist. */ - public Optional get() { + public CompletableFuture> get() { return ctx.getResource("", new QueryParams(), User.class); } /** * Fetches the current account's monthly usage for the current month. Only available for {@code - * me}. Returns the raw JSON usage report. + * me}. Completes with the raw JSON usage report. */ - public JsonNode monthlyUsage() { + public CompletableFuture monthlyUsage() { return monthlyUsage(""); } @@ -49,7 +50,7 @@ public JsonNode monthlyUsage() { * Fetches the current account's monthly usage for the month containing the given date (formatted * as {@code YYYY-MM-DD}). An empty date reports the current month. Only available for {@code me}. */ - public JsonNode monthlyUsage(String date) { + public CompletableFuture monthlyUsage(String date) { requireMe(); QueryParams params = new QueryParams(); if (date != null && !date.isEmpty()) { @@ -59,18 +60,18 @@ public JsonNode monthlyUsage(String date) { } /** Fetches the current account's resource limits. Only available for {@code me}. */ - public JsonNode limits() { + public CompletableFuture limits() { requireMe(); return ctx.getResourceRequired("limits", new QueryParams(), JsonNode.class); } /** Updates the current account's resource limits. Only available for {@code me}. */ - public void updateLimits(Object newLimits) { + public CompletableFuture updateLimits(Object newLimits) { requireMe(); // Route through the shared PUT primitive for consistency with every other resource client's // update-style call; the response body (the updated limits) is intentionally not parsed, // matching the reference client's void-returning updateLimits. - ctx.putRaw( + return ctx.putRaw( "limits", new QueryParams(), Json.toBytes(newLimits), ResourceContext.CONTENT_TYPE_JSON); } diff --git a/src/main/java/com/apify/client/user/UserPlan.java b/src/main/java/com/apify/client/user/UserPlan.java index 3800c79..cfc069e 100644 --- a/src/main/java/com/apify/client/user/UserPlan.java +++ b/src/main/java/com/apify/client/user/UserPlan.java @@ -1,9 +1,9 @@ package com.apify.client.user; import com.apify.client.ApifyResource; -import com.fasterxml.jackson.databind.JsonNode; import java.util.Collections; import java.util.List; +import tools.jackson.databind.JsonNode; /** A {@link User}'s subscription plan and its associated limits. */ public final class UserPlan extends ApifyResource { diff --git a/src/main/java/com/apify/client/webhook/AbstractWebhookCollectionClient.java b/src/main/java/com/apify/client/webhook/AbstractWebhookCollectionClient.java index 7a7ea8c..6e46c56 100644 --- a/src/main/java/com/apify/client/webhook/AbstractWebhookCollectionClient.java +++ b/src/main/java/com/apify/client/webhook/AbstractWebhookCollectionClient.java @@ -6,7 +6,8 @@ import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; -import java.util.Iterator; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; /** * Shared read-only behavior for webhook collections. Both the account-wide collection ({@link @@ -22,23 +23,23 @@ abstract class AbstractWebhookCollectionClient { } /** Lists webhooks. */ - public PaginationList list(ListOptions options) { + public CompletableFuture> list(ListOptions options) { QueryParams params = new QueryParams(); options.apply(params); return ctx.listResource("", params, Webhook.class); } /** - * Returns a lazy iterator over the webhooks. The options' {@code limit} caps the total number - * yielded ({@code null} or non-positive = all); {@code chunkSize} is the per-request page size - * ({@code null} = server default). + * Returns a lazy, backpressure-aware publisher over the webhooks. The options' {@code limit} caps + * the total number yielded ({@code null} or non-positive = all); {@code chunkSize} is the + * per-request page size ({@code null} = server default). */ - public Iterator iterate(ListOptions options) { + public Flow.Publisher iterate(ListOptions options) { return iterate(options, null); } /** As {@link #iterate(ListOptions)}, but {@code chunkSize} sets the per-request page size. */ - public Iterator iterate(ListOptions options, Long chunkSize) { + public Flow.Publisher iterate(ListOptions options, Long chunkSize) { ListOptions opts = options != null ? options : new ListOptions(); return ctx.iterateResource( "", opts.limitValue(), chunkSize, opts.offsetValue(), opts::applyFilters, Webhook.class); diff --git a/src/main/java/com/apify/client/webhook/Webhook.java b/src/main/java/com/apify/client/webhook/Webhook.java index 3a9807a..bddf348 100644 --- a/src/main/java/com/apify/client/webhook/Webhook.java +++ b/src/main/java/com/apify/client/webhook/Webhook.java @@ -1,10 +1,10 @@ package com.apify.client.webhook; import com.apify.client.ApifyResource; -import com.fasterxml.jackson.databind.JsonNode; import java.time.Instant; import java.util.Collections; import java.util.List; +import tools.jackson.databind.JsonNode; /** A webhook notifies an external service when specific events occur. */ public final class Webhook extends ApifyResource { @@ -88,7 +88,10 @@ public boolean isDoNotRetry() { return doNotRetry; } - /** The URL the webhook posts to. */ + /** + * The URL the webhook posts to. {@code null} for a hook action other than the conventional HTTP + * case (e.g. a Slack or email notification). + */ public String getRequestUrl() { return requestUrl; } diff --git a/src/main/java/com/apify/client/webhook/WebhookClient.java b/src/main/java/com/apify/client/webhook/WebhookClient.java index d98be76..20f02c8 100644 --- a/src/main/java/com/apify/client/webhook/WebhookClient.java +++ b/src/main/java/com/apify/client/webhook/WebhookClient.java @@ -5,6 +5,7 @@ import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; import java.util.Optional; +import java.util.concurrent.CompletableFuture; /** A client for a specific webhook ({@code /v2/webhooks/{webhookId}}). */ public final class WebhookClient { @@ -17,22 +18,22 @@ public WebhookClient(HttpClientCore http, String baseUrl, String id) { } /** Fetches the webhook, or empty if it does not exist. */ - public Optional get() { + public CompletableFuture> get() { return ctx.getResource("", new QueryParams(), Webhook.class); } /** Updates the webhook with the given fields and returns the updated object. */ - public Webhook update(Object newFields) { + public CompletableFuture update(Object newFields) { return ctx.updateResource("", newFields, Webhook.class); } /** Deletes the webhook. */ - public void delete() { - ctx.deleteResource(""); + public CompletableFuture delete() { + return ctx.deleteResource(""); } - /** Dispatches the webhook immediately and returns the resulting dispatch. */ - public WebhookDispatch test() { + /** Dispatches the webhook immediately and completes with the resulting dispatch. */ + public CompletableFuture test() { return ctx.postWithBody("test", new QueryParams(), null, "", WebhookDispatch.class); } diff --git a/src/main/java/com/apify/client/webhook/WebhookCollectionClient.java b/src/main/java/com/apify/client/webhook/WebhookCollectionClient.java index 1d9f043..75b6a8d 100644 --- a/src/main/java/com/apify/client/webhook/WebhookCollectionClient.java +++ b/src/main/java/com/apify/client/webhook/WebhookCollectionClient.java @@ -2,6 +2,7 @@ import com.apify.client.internal.HttpClientCore; import com.apify.client.internal.QueryParams; +import java.util.concurrent.CompletableFuture; /** * A client for the account-wide webhook collection ({@code GET/POST /v2/webhooks}), supporting both @@ -14,7 +15,7 @@ public WebhookCollectionClient(HttpClientCore http, String baseUrl) { } /** Creates a new webhook. {@code webhook} is any JSON-serializable webhook definition. */ - public Webhook create(Object webhook) { + public CompletableFuture create(Object webhook) { return ctx.createResource(new QueryParams(), webhook, Webhook.class); } } diff --git a/src/main/java/com/apify/client/webhook/WebhookDispatchClient.java b/src/main/java/com/apify/client/webhook/WebhookDispatchClient.java index d922a5d..edf0208 100644 --- a/src/main/java/com/apify/client/webhook/WebhookDispatchClient.java +++ b/src/main/java/com/apify/client/webhook/WebhookDispatchClient.java @@ -5,6 +5,7 @@ import com.apify.client.internal.QueryParams; import com.apify.client.internal.ResourceContext; import java.util.Optional; +import java.util.concurrent.CompletableFuture; /** A client for a specific webhook dispatch ({@code /v2/webhook-dispatches/{dispatchId}}). */ public final class WebhookDispatchClient { @@ -15,7 +16,7 @@ public WebhookDispatchClient(HttpClientCore http, String baseUrl, String id) { } /** Fetches the dispatch, or empty if it does not exist. */ - public Optional get() { + public CompletableFuture> get() { return ctx.getResource("", new QueryParams(), WebhookDispatch.class); } } diff --git a/src/main/java/com/apify/client/webhook/WebhookDispatchWebhookInfo.java b/src/main/java/com/apify/client/webhook/WebhookDispatchWebhookInfo.java index bab429d..750ab6b 100644 --- a/src/main/java/com/apify/client/webhook/WebhookDispatchWebhookInfo.java +++ b/src/main/java/com/apify/client/webhook/WebhookDispatchWebhookInfo.java @@ -10,7 +10,10 @@ public final class WebhookDispatchWebhookInfo extends ApifyResource { private String requestUrl; private boolean isAdHoc; - /** The URL the webhook posts to. */ + /** + * The URL the webhook posts to. {@code null} for a hook action other than the conventional HTTP + * case (e.g. a Slack or email notification). + */ public String getRequestUrl() { return requestUrl; } diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index a2608c4..d3512e4 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -14,10 +14,10 @@ // resolves test sources on the module path when main sources declare one) uses the in-process // Java compiler API to compile documentation code snippets as part of the test suite. requires java.compiler; - requires com.fasterxml.jackson.annotation; - requires com.fasterxml.jackson.core; - requires com.fasterxml.jackson.databind; - requires com.fasterxml.jackson.datatype.jsr310; + // Jackson 3: tools.jackson.databind `requires transitive` both tools.jackson.core and + // com.fasterxml.jackson.annotation, so this one line also makes those two modules' packages + // (JacksonException, TypeReference, the Jackson annotations, ...) readable here. + requires tools.jackson.databind; requires org.slf4j; requires static com.aayushatharva.brotli4j; @@ -40,31 +40,31 @@ // which requires reflective access opened to jackson-databind for every package holding a model // or request/response DTO, including the non-exported internal package. opens com.apify.client to - com.fasterxml.jackson.databind; + tools.jackson.databind; opens com.apify.client.actor to - com.fasterxml.jackson.databind; + tools.jackson.databind; opens com.apify.client.build to - com.fasterxml.jackson.databind; + tools.jackson.databind; opens com.apify.client.run to - com.fasterxml.jackson.databind; + tools.jackson.databind; opens com.apify.client.dataset to - com.fasterxml.jackson.databind; + tools.jackson.databind; opens com.apify.client.keyvalue to - com.fasterxml.jackson.databind; + tools.jackson.databind; opens com.apify.client.requestqueue to - com.fasterxml.jackson.databind; + tools.jackson.databind; opens com.apify.client.task to - com.fasterxml.jackson.databind; + tools.jackson.databind; opens com.apify.client.schedule to - com.fasterxml.jackson.databind; + tools.jackson.databind; opens com.apify.client.webhook to - com.fasterxml.jackson.databind; + tools.jackson.databind; opens com.apify.client.user to - com.fasterxml.jackson.databind; + tools.jackson.databind; opens com.apify.client.store to - com.fasterxml.jackson.databind; + tools.jackson.databind; opens com.apify.client.http to - com.fasterxml.jackson.databind; + tools.jackson.databind; opens com.apify.client.internal to - com.fasterxml.jackson.databind; + tools.jackson.databind; } diff --git a/src/test/java/com/apify/client/AsyncPaginatedPublisherTest.java b/src/test/java/com/apify/client/AsyncPaginatedPublisherTest.java new file mode 100644 index 0000000..58a46f8 --- /dev/null +++ b/src/test/java/com/apify/client/AsyncPaginatedPublisherTest.java @@ -0,0 +1,210 @@ +package com.apify.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.apify.client.internal.AsyncPaginatedPublisher; +import com.apify.client.internal.ListPublisher; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +/** + * Hermetic (token-free) tests for the offset/limit async iteration engine. They drive {@link + * AsyncPaginatedPublisher} with a stub page fetcher over a synthetic collection, pinning the + * total-cap / chunk-size arithmetic, server-side page-size clamping, and termination — without any + * network or {@code APIFY_TOKEN}. + */ +class AsyncPaginatedPublisherTest { + + /** + * A fake paginated endpoint over integers {@code [0, available)}. Like the real API, it clamps a + * requested page size (or an unset one) to {@code maxPageSize} ({@code 0} = no clamp). + */ + private static final class StubFetcher implements AsyncPaginatedPublisher.PageFetcher { + final long available; + final long maxPageSize; + final List requests = new ArrayList<>(); + + StubFetcher(long available) { + this(available, 0); + } + + StubFetcher(long available, long maxPageSize) { + this.available = available; + this.maxPageSize = maxPageSize; + } + + @Override + public synchronized CompletableFuture> fetch(long offset, Long limit) { + requests.add(new long[] {offset, limit == null ? -1 : limit}); + long requested = limit == null ? Long.MAX_VALUE : limit; + long take = maxPageSize > 0 ? Math.min(requested, maxPageSize) : requested; + List items = new ArrayList<>(); + for (long i = offset; i < available && items.size() < take; i++) { + items.add((int) i); + } + PaginationList page = new PaginationList<>(); + page.setItems(items); + // Some endpoints (e.g. dataset items) report a total of 0 or a lagging value; the engine must + // not depend on it, so the stub deliberately reports an unhelpful total. + page.setTotal(0); + page.setOffset(offset); + page.setCount(items.size()); + return CompletableFuture.completedFuture(page); + } + } + + private static List drain(Flow.Publisher publisher) { + return Publishers.collect(publisher).join(); + } + + @Test + void totalCapTrimsLastPage() { + StubFetcher f = new StubFetcher(10); + List got = drain(new AsyncPaginatedPublisher<>(3L, 2L, null, f)); + assertEquals(List.of(0, 1, 2), got, "limit=3 caps the total yielded across pages"); + // First page requests min(cap,chunk)=2; second requests min(remaining=1,chunk=2)=1; the cap is + // reached exactly, so no trailing empty request is made. + assertEquals(2, f.requests.size()); + assertEquals(0L, f.requests.get(0)[0]); + assertEquals(2L, f.requests.get(0)[1]); + assertEquals(2L, f.requests.get(1)[0]); + assertEquals(1L, f.requests.get(1)[1]); + } + + @Test + void chunkSizePagesAllThenStopsOnEmptyPage() { + StubFetcher f = new StubFetcher(5); + List got = drain(new AsyncPaginatedPublisher<>(null, 2L, null, f)); + assertEquals(List.of(0, 1, 2, 3, 4), got); + // 5 items / page size 2 => pages of 2,2,1 then a trailing empty page confirms the end. + assertEquals(4, f.requests.size()); + assertEquals(5L, f.requests.get(3)[0], "trailing request pages past the last item"); + assertEquals(2L, f.requests.get(3)[1], "each page still requests the chunk size"); + } + + @Test + void serverClampsLargePageSizeSoShortPageIsNotTheEnd() { + // Server caps every page at 2 items; the caller asks for far more. A page shorter than the + // request must NOT be treated as end-of-collection. + StubFetcher f = new StubFetcher(5, 2); + List got = drain(new AsyncPaginatedPublisher<>(null, 100L, null, f)); + assertEquals(List.of(0, 1, 2, 3, 4), got, "clamped short pages must keep paging to the end"); + } + + @Test + void capIsHonoredEvenWhenServerClampsPages() { + StubFetcher f = new StubFetcher(100, 2); + List got = drain(new AsyncPaginatedPublisher<>(5L, 100L, null, f)); + assertEquals(List.of(0, 1, 2, 3, 4), got, "the total cap holds despite server page clamping"); + } + + @Test + void noChunkUsesServerDefaultAndPagesToEmpty() { + StubFetcher f = new StubFetcher(4); + List got = drain(new AsyncPaginatedPublisher<>(null, null, null, f)); + assertEquals(List.of(0, 1, 2, 3), got); + assertEquals(-1L, f.requests.get(0)[1], "no cap and no chunk => null limit (server default)"); + } + + @Test + void startOffsetIsHonored() { + StubFetcher f = new StubFetcher(10); + List got = drain(new AsyncPaginatedPublisher<>(null, 5L, 7L, f)); + assertEquals(List.of(7, 8, 9), got, "iteration starts at the requested offset"); + assertEquals(7L, f.requests.get(0)[0]); + } + + @Test + void trimsOverReturnedPageToCap() { + // A misbehaving server that returns more items than requested must not make the publisher + // overshoot the caller's total cap. + AsyncPaginatedPublisher.PageFetcher overReturning = + (offset, limit) -> { + PaginationList page = new PaginationList<>(); + page.setItems(List.of(1, 2, 3, 4, 5)); // ignores the requested limit + page.setTotal(5); + page.setCount(5); + return CompletableFuture.completedFuture(page); + }; + List got = drain(new AsyncPaginatedPublisher<>(3L, 2L, null, overReturning)); + assertEquals(List.of(1, 2, 3), got, "the last page is trimmed to the cap"); + } + + @Test + void emptyCollectionYieldsNothing() { + StubFetcher f = new StubFetcher(0); + List got = drain(new AsyncPaginatedPublisher<>(null, 5L, null, f)); + assertTrue(got.isEmpty()); + } + + /** + * Records onNext/onError deliveries for the reentrant-invalid-request regression tests below. A + * subscriber violating Reactive Streams §3.9 by issuing {@code request(n<=0)} from within {@code + * onNext} must see exactly the one item already being emitted when the violation is reported, + * then a single {@code onError} - never further {@code onNext} calls for the rest of an + * already-fetched/already-fully-in-hand page (RS §1.7: no signals after a terminal one). + */ + private static final class RecordingSubscriber implements Flow.Subscriber { + final List items = new ArrayList<>(); + final List errors = new ArrayList<>(); + final AtomicReference subscription = new AtomicReference<>(); + + @Override + public void onSubscribe(Flow.Subscription s) { + subscription.set(s); + s.request(5); + } + + @Override + public void onNext(Integer item) { + items.add(item); + subscription.get().request(-1); // reentrant RS §3.9 violation + } + + @Override + public void onError(Throwable t) { + errors.add(t); + } + + @Override + public void onComplete() {} + } + + @Test + void reentrantInvalidRequestStopsEmissionImmediately() { + // Regression test: the drain loop used to check only `cancelled`/demand/buffer-position, never + // `terminated`, so a reentrant request(n<=0) mid-emission still let the loop emit the rest of + // an + // already-fetched page (and even fetch and emit a further page) after onError had already + // fired. + StubFetcher f = new StubFetcher(5); + RecordingSubscriber subscriber = new RecordingSubscriber(); + new AsyncPaginatedPublisher<>(null, null, null, f).subscribe(subscriber); + assertEquals( + List.of(0), + subscriber.items, + "only the item already being emitted when the invalid request() fired is delivered"); + assertEquals(1, subscriber.errors.size()); + assertTrue(subscriber.errors.get(0) instanceof IllegalArgumentException); + } + + @Test + void listPublisherReentrantInvalidRequestStopsEmissionImmediately() { + // Same regression, for ListPublisher's independent (but structurally identical) drain loop. + CompletableFuture> source = + CompletableFuture.completedFuture(List.of(0, 1, 2, 3, 4)); + RecordingSubscriber subscriber = new RecordingSubscriber(); + new ListPublisher<>(source).subscribe(subscriber); + assertEquals( + List.of(0), + subscriber.items, + "only the item already being emitted when the invalid request() fired is delivered"); + assertEquals(1, subscriber.errors.size()); + assertTrue(subscriber.errors.get(0) instanceof IllegalArgumentException); + } +} diff --git a/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java b/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java index 2387563..ad6dde6 100644 --- a/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java +++ b/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java @@ -34,6 +34,9 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.Flow; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; /** @@ -59,7 +62,7 @@ private static ApifyClient client(MockTransport backend, int maxRetries) { @Test void chargeSendsIdempotencyKey() { MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{}}"); - client(backend).run("run123").charge(new RunChargeOptions("my-event")); + client(backend).run("run123").charge(new RunChargeOptions("my-event")).join(); String key = backend.lastHeaders.firstValue("idempotency-key").orElse(""); assertTrue( key.startsWith("run123-my-event-"), "idempotency key should embed run id + event: " + key); @@ -75,7 +78,8 @@ void lastRunForwardsStatusFilterToNestedStorages() { .actor("me/x") .lastRun("SUCCEEDED") .dataset() - .listItems(new DatasetListItemsOptions()); + .listItems(new DatasetListItemsOptions()) + .join(); assertTrue(ds.lastUrl.contains("runs/last/dataset/items"), ds.lastUrl); assertTrue(ds.lastUrl.contains("status=SUCCEEDED"), ds.lastUrl); @@ -84,7 +88,8 @@ void lastRunForwardsStatusFilterToNestedStorages() { .actor("me/x") .lastRun(new LastRunOptions().status("SUCCEEDED").origin("API")) .log() - .get(); + .get() + .join(); assertTrue(log.lastUrl.contains("runs/last/log"), log.lastUrl); assertTrue(log.lastUrl.contains("status=SUCCEEDED"), log.lastUrl); assertTrue(log.lastUrl.contains("origin=API"), log.lastUrl); @@ -94,7 +99,7 @@ void lastRunForwardsStatusFilterToNestedStorages() { void plainRunNestedStoragesCarryNoInheritedFilter() { // A non-last-run client has no pinned params, so nested accessors stay filter-free. MockTransport ds = MockTransport.ofConstant(200, "[]"); - client(ds).run("run123").dataset().listItems(new DatasetListItemsOptions()); + client(ds).run("run123").dataset().listItems(new DatasetListItemsOptions()).join(); assertTrue(ds.lastUrl.contains("actor-runs/run123/dataset/items"), ds.lastUrl); assertFalse(ds.lastUrl.contains("status="), ds.lastUrl); } @@ -102,7 +107,10 @@ void plainRunNestedStoragesCarryNoInheritedFilter() { @Test void chargeHonorsExplicitIdempotencyKey() { MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{}}"); - client(backend).run("run123").charge(new RunChargeOptions("e").idempotencyKey("fixed-key")); + client(backend) + .run("run123") + .charge(new RunChargeOptions("e").idempotencyKey("fixed-key")) + .join(); assertEquals("fixed-key", backend.lastHeaders.firstValue("idempotency-key").orElse("")); } @@ -143,8 +151,8 @@ void metamorphSendsTargetActorIdBuildAndInputBody() { MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{\"id\":\"run123\"}}"); client(backend) .run("run123") - .metamorph( - "apify/other-actor", Map.of("foo", "bar"), new MetamorphOptions().build("1.2.3")); + .metamorph("apify/other-actor", Map.of("foo", "bar"), new MetamorphOptions().build("1.2.3")) + .join(); assertTrue(backend.lastUrl.contains("actor-runs/run123/metamorph"), backend.lastUrl); // targetActorId is normalized to the URL-safe username~actor-name form before sending // (ResourceContext.toSafeId), matching the reference client's _toSafeId; '~' is then @@ -190,7 +198,7 @@ void requiredOptionsArgumentsRejectNullWithClearException() { @Test void rebootSendsPostToRebootWithNoBody() { MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{\"id\":\"run123\"}}"); - ActorRun run = client(backend).run("run123").reboot(); + ActorRun run = client(backend).run("run123").reboot().join(); assertEquals("run123", run.getId()); assertTrue(backend.lastUrl.contains("actor-runs/run123/reboot"), backend.lastUrl); assertEquals("POST", backend.lastMethod); @@ -199,7 +207,7 @@ void rebootSendsPostToRebootWithNoBody() { @Test void getRecordDefaultsAttachment() { MockTransport backend = MockTransport.ofConstant(200, "raw-bytes"); - client(backend).keyValueStore("store1").getRecord("OUTPUT"); + client(backend).keyValueStore("store1").getRecord("OUTPUT").join(); assertTrue(backend.lastUrl.contains("attachment=1"), backend.lastUrl); } @@ -207,7 +215,7 @@ void getRecordDefaultsAttachment() { void keyValueStoreRecordDefensivelyCopiesBytes() { MockTransport backend = MockTransport.ofConstant(200, "raw-bytes"); KeyValueStoreRecord record = - client(backend).keyValueStore("s").getRecord("OUTPUT").orElseThrow(); + client(backend).keyValueStore("s").getRecord("OUTPUT").join().orElseThrow(); byte[] first = record.getValue(); first[0] = 0; // mutate the returned array assertEquals('r', record.getValue()[0], "mutating a returned copy must not affect the record"); @@ -221,13 +229,14 @@ void setRecordDoesNotRetryTimeoutsWhenOptedOut() { assertThrows( ApifyTransportException.class, () -> - client(timeouts, 3) - .keyValueStore("s") - .setRecord( - "k", - new byte[] {1}, - "application/octet-stream", - new SetRecordOptions().doNotRetryTimeouts(true))); + TestAsync.await( + client(timeouts, 3) + .keyValueStore("s") + .setRecord( + "k", + new byte[] {1}, + "application/octet-stream", + new SetRecordOptions().doNotRetryTimeouts(true)))); assertEquals(1, timeouts.calls, "a timeout must not be retried when doNotRetryTimeouts is set"); } @@ -236,7 +245,11 @@ void setRecordRetriesTimeoutsByDefault() { MockTransport timeouts = new MockTransport(List.of(MockTransport.timeoutError())); assertThrows( ApifyTransportException.class, - () -> client(timeouts, 3).keyValueStore("s").setRecord("k", new byte[] {1}, "text/plain")); + () -> + TestAsync.await( + client(timeouts, 3) + .keyValueStore("s") + .setRecord("k", new byte[] {1}, "text/plain"))); assertEquals(4, timeouts.calls, "timeouts should be retried (maxRetries + 1 attempts)"); } @@ -246,7 +259,8 @@ void createKeysPublicUrlForwardsFilterOptions() { String url = client(backend) .keyValueStore("store1") - .createKeysPublicUrl(new ListKeysOptions().prefix("img-").limit(10L), null); + .createKeysPublicUrl(new ListKeysOptions().prefix("img-").limit(10L), null) + .join(); assertTrue(url.contains("prefix=img-"), url); assertTrue(url.contains("limit=10"), url); assertFalse(url.contains("signature="), "a public store should get no signature: " + url); @@ -261,7 +275,8 @@ void downloadItemsForwardsItemSelectionParams() { DownloadItemsFormat.CSV, new DatasetDownloadOptions() .items(new DatasetListItemsOptions().limit(5L).desc(true).fields(List.of("a", "b"))) - .bom(true)); + .bom(true)) + .join(); assertTrue(backend.lastUrl.contains("format=csv"), backend.lastUrl); assertTrue(backend.lastUrl.contains("limit=5"), backend.lastUrl); assertTrue(backend.lastUrl.contains("desc=1"), backend.lastUrl); @@ -283,7 +298,8 @@ void batchAddRequestsChunks() { .batchAddRequests( requests, false, - new BatchAddRequestsOptions().maxParallel(1).maxUnprocessedRequestsRetries(0)); + new BatchAddRequestsOptions().maxParallel(1).maxUnprocessedRequestsRetries(0)) + .join(); assertEquals(3, backend.calls); // 25 + 25 + 10 } @@ -305,7 +321,8 @@ void batchAddRequestsSplitsChunksByPayloadSize() { .batchAddRequests( requests, false, - new BatchAddRequestsOptions().maxParallel(1).maxUnprocessedRequestsRetries(0)); + new BatchAddRequestsOptions().maxParallel(1).maxUnprocessedRequestsRetries(0)) + .join(); assertTrue(backend.calls > 1, "large requests should be split across multiple batch calls"); } @@ -315,6 +332,8 @@ void batchAddRequestsThrowsWhenSingleRequestExceedsPayloadLimit() { String hugePayload = "x".repeat(10 * 1024 * 1024); // exceeds the ~9 MiB limit on its own List requests = List.of(new RequestQueueRequest("https://example.com", "k0").setPayload(hugePayload)); + // Chunk splitting (and this validation) runs synchronously before any async call is made, so + // the exception is thrown directly, not via the returned future. assertThrows( IllegalArgumentException.class, () -> client(backend).requestQueue("q1").batchAddRequests(requests, false)); @@ -323,8 +342,8 @@ void batchAddRequestsThrowsWhenSingleRequestExceedsPayloadLimit() { @Test void paginateRequestsTrimsPageToTotalLimitWhenServerOvershoots() { // If the server ignores (or overshoots) the requested per-page `limit` and returns more items - // than the caller's totalLimit cap allows, the iterator must still yield exactly totalLimit - // items, mirroring PaginatedIterator's own defensive trim. + // than the caller's totalLimit cap allows, the publisher must still yield exactly totalLimit + // items, mirroring AsyncPaginatedPublisher's own defensive trim. MockTransport backend = MockTransport.ofConstant( 200, @@ -335,12 +354,10 @@ void paginateRequestsTrimsPageToTotalLimitWhenServerOvershoots() { + "{\"id\":\"4\",\"url\":\"https://example.com/4\",\"uniqueKey\":\"k4\"}," + "{\"id\":\"5\",\"url\":\"https://example.com/5\",\"uniqueKey\":\"k5\"}" + "],\"limit\":3,\"nextCursor\":null}}"); - java.util.Iterator it = - client(backend).requestQueue("q1").paginateRequests(3L, null, null); - List ids = new ArrayList<>(); - while (it.hasNext()) { - ids.add(it.next().getId()); - } + List got = + Publishers.collect(client(backend).requestQueue("q1").paginateRequests(3L, null, null)) + .join(); + List ids = got.stream().map(RequestQueueRequest::getId).toList(); assertEquals(List.of("1", "2", "3"), ids); } @@ -367,7 +384,8 @@ void batchAddRequestsRetriesUnprocessed() { .batchAddRequests( requests, false, - new BatchAddRequestsOptions().minDelayBetweenUnprocessedRequestsRetriesMillis(1L)); + new BatchAddRequestsOptions().minDelayBetweenUnprocessedRequestsRetriesMillis(1L)) + .join(); assertEquals(2, backend.calls, "the unprocessed request should trigger one retry call"); assertEquals(2, result.getProcessedRequests().size()); assertTrue(result.getUnprocessedRequests().isEmpty(), "all requests should end up processed"); @@ -377,7 +395,7 @@ void batchAddRequestsRetriesUnprocessed() { void getWithWaitForwardsWaitForFinish() { MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{\"id\":\"r1\",\"status\":\"RUNNING\"}}"); - Optional run = client(backend).run("r1").getWithWait(30L); + Optional run = client(backend).run("r1").getWithWait(30L).join(); assertTrue(run.isPresent(), "run should be present"); assertEquals("r1", run.get().getId()); assertTrue(backend.lastUrl.contains("waitForFinish=30"), backend.lastUrl); @@ -386,8 +404,7 @@ void getWithWaitForwardsWaitForFinish() { @Test void getWithWaitClampsServerWaitToConfiguredTimeout() { // With a 10s per-request timeout, a caller asking for waitForFinish=60 must be clamped below - // the timeout (10 - 5s margin = 5) so the synchronous get can't abort itself on the socket - // timeout. + // the timeout (10 - 5s margin = 5) so the get can't abort itself on the socket timeout. MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{\"id\":\"r1\",\"status\":\"RUNNING\"}}"); ApifyClient client = @@ -397,7 +414,7 @@ void getWithWaitClampsServerWaitToConfiguredTimeout() { .maxRetries(0) .timeout(Duration.ofSeconds(10)) .build(); - client.run("r1").getWithWait(60L); + client.run("r1").getWithWait(60L).join(); assertTrue(backend.lastUrl.contains("waitForFinish=5"), backend.lastUrl); assertFalse(backend.lastUrl.contains("waitForFinish=60"), backend.lastUrl); } @@ -414,7 +431,7 @@ void waitForFinishClampsServerWaitToConfiguredTimeout() { .minDelayBetweenRetries(Duration.ofMillis(1)) .timeout(Duration.ofSeconds(20)) .build(); - client.run("r1").waitForFinish(120L); + client.run("r1").waitForFinish(120L).join(); // 20s timeout - 5s margin = 15s server wait cap (below the 60s API cap and the 120s budget). assertTrue(backend.lastUrl.contains("waitForFinish=15"), backend.lastUrl); } @@ -434,7 +451,7 @@ void builderRejectsZeroOrNegativeTimeout() { @Test void getResourceReturnsEmptyOnNullData() { MockTransport backend = MockTransport.ofConstant(200, "{\"data\":null}"); - Optional store = client(backend).keyValueStore("s").get(); + Optional store = client(backend).keyValueStore("s").get().join(); assertTrue(store.isEmpty(), "a 200 with null data must map to an empty Optional, not throw"); } @@ -444,7 +461,7 @@ void storeIterateDoesNotMutateCallerOptionsAndHonorsOffset() { MockTransport.ofConstant( 200, "{\"data\":{\"items\":[],\"total\":0,\"offset\":0,\"limit\":0,\"count\":0}}"); StoreListOptions options = new StoreListOptions().offset(100L).limit(50L); - client(backend).store().iterate(options, null).hasNext(); + Publishers.collect(client(backend).store().iterate(options, null)).join(); // The caller's initial offset must be honored for paging and left untouched afterwards. assertTrue(backend.lastUrl.contains("offset=100"), backend.lastUrl); assertEquals(100L, options.offsetValue(), "iteration must not mutate the caller's options"); @@ -461,21 +478,16 @@ void storeIterateWalksMultiplePages() { MockTransport.ok( 200, "{\"data\":{\"items\":[{}],\"total\":3,\"offset\":2,\"limit\":2,\"count\":1}}"), - // Trailing empty page: the iterator stops on an empty page (it does not trust the + // Trailing empty page: the publisher stops on an empty page (it does not trust the // reported total, which some endpoints under-report), so a final empty page is // required to terminate an uncapped walk. MockTransport.ok( 200, "{\"data\":{\"items\":[],\"total\":3,\"offset\":3,\"limit\":2,\"count\":0}}"))); // No total cap; page size 2 drives paging until the empty page. - java.util.Iterator it = - client(backend).store().iterate(new StoreListOptions(), 2L); - int count = 0; - while (it.hasNext()) { - it.next(); - count++; - } - assertEquals(3, count, "iteration should walk across both non-empty pages"); + List items = + Publishers.collect(client(backend).store().iterate(new StoreListOptions(), 2L)).join(); + assertEquals(3, items.size(), "iteration should walk across both non-empty pages"); assertEquals(3, backend.calls, "two data pages plus the terminating empty page"); } @@ -491,28 +503,49 @@ void collectionIterateSingleArgDelegatesToServerDefaultPageSize() { MockTransport.ok( 200, "{\"data\":{\"items\":[],\"total\":2,\"offset\":2,\"limit\":2,\"count\":0}}"))); - java.util.Iterator it = client(backend).actors().iterate(new ActorListOptions()); - int count = 0; - while (it.hasNext()) { - it.next(); - count++; - } - assertEquals(2, count, "single-arg iterate should yield every item"); + List items = + Publishers.collect(client(backend).actors().iterate(new ActorListOptions())).join(); + assertEquals(2, items.size(), "single-arg iterate should yield every item"); assertFalse( backend.lastUrl.contains("limit="), "no chunkSize => no limit param (server default)"); } @Test void iterateSnapshotsOptionsSoLaterMutationsDoNotLeak() { - // The iterator must capture the options (offset/limit AND filters) at call time; mutating the - // caller's options object afterwards must not change subsequent page requests. + // The publisher must capture the options (offset/limit AND filters) at call time; mutating the + // caller's options object afterwards must not change subsequent page requests. Note this + // backend + // always returns the same non-empty page (it never signals "empty page" / exhaustion - see + // AsyncPaginatedPublisher's class doc on why it never stops on a short page or reported total), + // so the test must not fully drain the publisher (that would fetch pages forever); it only + // needs + // to request a single item to trigger the one page fetch it is asserting on. MockTransport backend = MockTransport.ofConstant( 200, "{\"data\":{\"items\":[{}],\"total\":1,\"offset\":0,\"limit\":0,\"count\":1}}"); ActorListOptions options = new ActorListOptions().sortBy("createdAt"); - java.util.Iterator it = client(backend).actors().iterate(options); - options.sortBy("modifiedAt"); // mutate after obtaining the iterator - it.hasNext(); // triggers the first page fetch + var publisher = client(backend).actors().iterate(options); + options.sortBy("modifiedAt"); // mutate after obtaining the publisher + AtomicReference subscriptionRef = new AtomicReference<>(); + publisher.subscribe( + new Flow.Subscriber() { + @Override + public void onSubscribe(Flow.Subscription subscription) { + subscriptionRef.set(subscription); + } + + @Override + public void onNext(Actor item) {} + + @Override + public void onError(Throwable throwable) {} + + @Override + public void onComplete() {} + }); + // MockTransport completes synchronously, so this request(1) triggers (and finishes) the first + // page fetch reentrantly, before returning here. + subscriptionRef.get().request(1); assertTrue(backend.lastUrl.contains("sortBy=createdAt"), backend.lastUrl); assertFalse(backend.lastUrl.contains("modifiedAt"), backend.lastUrl); } @@ -522,7 +555,7 @@ void runCollectionListToleratesNullOptionsAndFilter() { MockTransport backend = MockTransport.ofConstant( 200, "{\"data\":{\"items\":[],\"total\":0,\"offset\":0,\"limit\":0,\"count\":0}}"); - PaginationList runs = client(backend).runs().list(null, null); + PaginationList runs = client(backend).runs().list(null, null).join(); assertEquals(0, runs.getItems().size()); } @@ -533,13 +566,13 @@ void nestedWebhookCollectionListsWithoutCreate() { 200, "{\"data\":{\"items\":[],\"total\":0,\"offset\":0,\"limit\":0,\"count\":0}}"); // Compile-time guarantee: the nested collection type has no create(); it only lists. NestedWebhookCollectionClient nested = client(backend).task("t1").webhooks(); - assertEquals(0, nested.list(new ListOptions()).getItems().size()); + assertEquals(0, nested.list(new ListOptions()).join().getItems().size()); } @Test void accountWebhookCollectionCanCreate() { MockTransport backend = MockTransport.ofConstant(200, "{\"data\":{\"id\":\"wh1\"}}"); - Webhook created = client(backend).webhooks().create(Map.of("eventTypes", List.of())); + Webhook created = client(backend).webhooks().create(Map.of("eventTypes", List.of())).join(); assertEquals("wh1", created.getId()); assertTrue(backend.lastUrl.endsWith("/webhooks"), backend.lastUrl); } @@ -547,7 +580,7 @@ void accountWebhookCollectionCanCreate() { @Test void updateLimitsSendsPutToMeLimits() { MockTransport backend = MockTransport.ofConstant(200, "{}"); - client(backend).me().updateLimits(Map.of("maxMonthlyUsageUsd", 100)); + client(backend).me().updateLimits(Map.of("maxMonthlyUsageUsd", 100)).join(); assertTrue(backend.lastUrl.endsWith("/users/me/limits"), backend.lastUrl); assertTrue(backend.lastBody.contains("\"maxMonthlyUsageUsd\":100"), backend.lastBody); assertEquals(1, backend.calls); @@ -568,7 +601,8 @@ void batchAddRequestsNeverThrowsOnNonRetryableClientError() { .batchAddRequests( List.of(request), false, - new BatchAddRequestsOptions().maxUnprocessedRequestsRetries(0)); + new BatchAddRequestsOptions().maxUnprocessedRequestsRetries(0)) + .join(); assertEquals(0, result.getProcessedRequests().size()); assertEquals(1, result.getUnprocessedRequests().size()); assertEquals("k0", result.getUnprocessedRequests().get(0).getUniqueKey()); @@ -588,7 +622,8 @@ void batchAddRequestsRunsChunksInParallel() { .batchAddRequests( requests, false, - new BatchAddRequestsOptions().maxParallel(3).maxUnprocessedRequestsRetries(0)); + new BatchAddRequestsOptions().maxParallel(3).maxUnprocessedRequestsRetries(0)) + .join(); assertEquals(3, backend.calls, "60 requests must be sent as 3 parallel chunks of 25/25/10"); } @@ -608,7 +643,8 @@ void batchAddRequestsNeverThrowsOnPersistentTransportFailure() { .batchAddRequests( List.of(request), false, - new BatchAddRequestsOptions().maxUnprocessedRequestsRetries(0)); + new BatchAddRequestsOptions().maxUnprocessedRequestsRetries(0)) + .join(); assertEquals(0, result.getProcessedRequests().size()); assertEquals(1, result.getUnprocessedRequests().size()); assertEquals("k0", result.getUnprocessedRequests().get(0).getUniqueKey()); @@ -616,9 +652,9 @@ void batchAddRequestsNeverThrowsOnPersistentTransportFailure() { @Test void batchAddRequestsNeverThrowsOnPersistentTransportFailureWithParallelChunks() { - // Same as above but exercising the `maxParallel > 1` path, where each chunk fails on its own - // executor thread — the never-throws contract must hold there too, and every request across - // every chunk must come back via getUnprocessedRequests(). + // Same as above but exercising the `maxParallel > 1` path, where each chunk fails independently + // — the never-throws contract must hold there too, and every request across every chunk must + // come back via getUnprocessedRequests(). MockTransport backend = new MockTransport(List.of(MockTransport.networkError())); List requests = new ArrayList<>(); for (int i = 0; i < 60; i++) { @@ -630,7 +666,8 @@ void batchAddRequestsNeverThrowsOnPersistentTransportFailureWithParallelChunks() .batchAddRequests( requests, false, - new BatchAddRequestsOptions().maxParallel(3).maxUnprocessedRequestsRetries(0)); + new BatchAddRequestsOptions().maxParallel(3).maxUnprocessedRequestsRetries(0)) + .join(); assertEquals(0, result.getProcessedRequests().size()); assertEquals(60, result.getUnprocessedRequests().size()); assertEquals( @@ -647,7 +684,7 @@ void waitForFinishReturnsWhenResourceAppearsAfter404() { MockTransport.ok( 404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"missing\"}}"), MockTransport.ok(200, "{\"data\":{\"id\":\"r1\",\"status\":\"SUCCEEDED\"}}"))); - ActorRun run = client(backend).run("r1").waitForFinish(5L); + ActorRun run = client(backend).run("r1").waitForFinish(5L).join(); assertEquals("r1", run.getId()); assertEquals("SUCCEEDED", run.getStatus()); assertEquals(2, backend.calls, "one 404 poll then the terminal poll"); @@ -660,7 +697,9 @@ void waitForFinishThrowsWhenResourceNeverAppears() { MockTransport backend = MockTransport.ofConstant( 404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"missing\"}}"); - assertThrows(IllegalStateException.class, () -> client(backend).run("r1").waitForFinish(0L)); + assertThrows( + IllegalStateException.class, + () -> TestAsync.await(client(backend).run("r1").waitForFinish(0L))); } @Test @@ -669,7 +708,8 @@ void logStreamThrowsOnNonSuccessStatus() { backend.scriptStream( 403, "{\"error\":{\"type\":\"insufficient-permissions\",\"message\":\"no\"}}"); ApifyApiException ex = - assertThrows(ApifyApiException.class, () -> client(backend).log("run1").stream()); + assertThrows( + ApifyApiException.class, () -> TestAsync.await(client(backend).log("run1").stream())); assertEquals(403, ex.getStatusCode()); } @@ -677,9 +717,11 @@ void logStreamThrowsOnNonSuccessStatus() { void versionsIterateSingleFetchTerminatesAndDoesNotDuplicate() { // GET /v2/actors/{actorId}/versions is NOT offset/limit paginated: the server ignores `offset` // and returns the full {total, items} list on every request. `ofConstant` reproduces exactly - // that (same non-empty page for every call). Draining the iterator must terminate and yield + // that (same non-empty page for every call). Draining the publisher must terminate and yield // each version once. Routing this endpoint through the offset/limit paging engine looped - // forever (empty-page termination never triggers), so this pins the single-fetch behaviour. + // forever (empty-page termination never triggers), so this pins the single-fetch behaviour. A + // bounded timeout converts a termination regression (infinite pagination) into a clean test + // failure instead of a hang. MockTransport backend = MockTransport.ofConstant( 200, @@ -687,16 +729,11 @@ void versionsIterateSingleFetchTerminatesAndDoesNotDuplicate() { + "{\"versionNumber\":\"0.1\"}," + "{\"versionNumber\":\"0.2\"}," + "{\"versionNumber\":\"0.3\"}]}}"); - java.util.Iterator it = - client(backend).actor("me/actor").versions().iterate(new ListOptions()); - List yielded = new ArrayList<>(); - int guard = 0; - while (it.hasNext()) { - // Guard converts a termination regression (infinite loop) into a clean failure, not a hang. - assertTrue( - ++guard <= 100, "versions iterator did not terminate (paged a non-paginated endpoint)"); - yielded.add(it.next().getVersionNumber()); - } + List items = + Publishers.collect(client(backend).actor("me/actor").versions().iterate(new ListOptions())) + .orTimeout(5, TimeUnit.SECONDS) + .join(); + List yielded = items.stream().map(ActorVersion::getVersionNumber).toList(); assertEquals(List.of("0.1", "0.2", "0.3"), yielded, "each version yielded once, in order"); assertEquals(1, backend.calls, "non-paginated versions endpoint must be fetched exactly once"); } @@ -710,12 +747,11 @@ void versionsIterateHonorsTotalLimitCap() { + "{\"versionNumber\":\"0.1\"}," + "{\"versionNumber\":\"0.2\"}," + "{\"versionNumber\":\"0.3\"}]}}"); - java.util.Iterator it = - client(backend).actor("me/actor").versions().iterate(new ListOptions().limit(2L)); - List yielded = new ArrayList<>(); - while (it.hasNext()) { - yielded.add(it.next().getVersionNumber()); - } + List items = + Publishers.collect( + client(backend).actor("me/actor").versions().iterate(new ListOptions().limit(2L))) + .join(); + List yielded = items.stream().map(ActorVersion::getVersionNumber).toList(); assertEquals(List.of("0.1", "0.2"), yielded, "limit caps the number yielded"); } } diff --git a/src/test/java/com/apify/client/CompressionTest.java b/src/test/java/com/apify/client/CompressionTest.java index 37cbe84..c074851 100644 --- a/src/test/java/com/apify/client/CompressionTest.java +++ b/src/test/java/com/apify/client/CompressionTest.java @@ -117,7 +117,7 @@ void largeBodyIsCompressedWithPreferredEncoding() throws IOException { MockTransport backend = MockTransport.ofConstant(201, ""); byte[] payload = payload(4096, (byte) 'a'); - client(backend).keyValueStore(STORE_ID).setRecord(RECORD_KEY, payload, CONTENT_TYPE); + client(backend).keyValueStore(STORE_ID).setRecord(RECORD_KEY, payload, CONTENT_TYPE).join(); boolean brotli = HttpClientCore.brotliAvailable(); String expected = brotli ? "br" : "gzip"; @@ -135,7 +135,7 @@ void smallBodyIsNotCompressed() { MockTransport backend = MockTransport.ofConstant(201, ""); byte[] payload = payload(16, (byte) 'b'); - client(backend).keyValueStore(STORE_ID).setRecord(RECORD_KEY, payload, CONTENT_TYPE); + client(backend).keyValueStore(STORE_ID).setRecord(RECORD_KEY, payload, CONTENT_TYPE).join(); assertFalse( backend.lastHeaders.firstValue("Content-Encoding").isPresent(), @@ -148,7 +148,7 @@ void bodyExactlyAtThresholdIsCompressed() throws IOException { MockTransport backend = MockTransport.ofConstant(201, ""); byte[] payload = payload(1024, (byte) 'c'); - client(backend).keyValueStore(STORE_ID).setRecord(RECORD_KEY, payload, CONTENT_TYPE); + client(backend).keyValueStore(STORE_ID).setRecord(RECORD_KEY, payload, CONTENT_TYPE).join(); boolean brotli = HttpClientCore.brotliAvailable(); assertEquals( diff --git a/src/test/java/com/apify/client/DatasetItemsIteratorTest.java b/src/test/java/com/apify/client/DatasetItemsIteratorTest.java index 5e775df..76ba1ac 100644 --- a/src/test/java/com/apify/client/DatasetItemsIteratorTest.java +++ b/src/test/java/com/apify/client/DatasetItemsIteratorTest.java @@ -5,12 +5,11 @@ import com.apify.client.dataset.DatasetClient; import com.apify.client.dataset.DatasetListItemsOptions; -import com.fasterxml.jackson.databind.JsonNode; import java.time.Duration; -import java.util.ArrayList; -import java.util.Iterator; import java.util.List; +import java.util.concurrent.Flow; import org.junit.jupiter.api.Test; +import tools.jackson.databind.JsonNode; /** * Hermetic (token-free) tests for {@link DatasetClient#iterateItems} and its {@code fetchItemsPage} @@ -29,6 +28,10 @@ private static ApifyClient client(MockTransport backend) { .build(); } + private static List collect(Flow.Publisher publisher) { + return Publishers.collect(publisher).join(); + } + @Test void pagesBareArrayBodyAndStopsOnEmptyPage() { MockTransport backend = @@ -37,13 +40,10 @@ void pagesBareArrayBodyAndStopsOnEmptyPage() { MockTransport.ok(200, "[{\"n\":1},{\"n\":2}]"), MockTransport.ok(200, "[{\"n\":3}]"), MockTransport.ok(200, "[]"))); - List seen = new ArrayList<>(); - Iterator it = - client(backend).dataset("d1").iterateItems(new DatasetListItemsOptions(), 2L); - while (it.hasNext()) { - seen.add(it.next().get("n").asInt()); - } - assertEquals(List.of(1, 2, 3), seen, "iterateItems should page the bare-array endpoint"); + List seen = + collect(client(backend).dataset("d1").iterateItems(new DatasetListItemsOptions(), 2L)); + List ns = seen.stream().map(n -> n.get("n").asInt()).toList(); + assertEquals(List.of(1, 2, 3), ns, "iterateItems should page the bare-array endpoint"); assertEquals(3, backend.calls, "two data pages plus the terminating empty page"); assertTrue(backend.lastUrl.contains("datasets/d1/items"), backend.lastUrl); assertTrue(backend.lastUrl.contains("limit=2"), "chunkSize drives the per-request page size"); @@ -58,13 +58,13 @@ void typedIterateItemsAtDefaultPageSize() { MockTransport backend = new MockTransport( List.of(MockTransport.ok(200, "[{\"n\":1},{\"n\":2}]"), MockTransport.ok(200, "[]"))); - List seen = new ArrayList<>(); - Iterator it = - client(backend).dataset("d1").iterateItems(new DatasetListItemsOptions(), null, Row.class); - while (it.hasNext()) { - seen.add(it.next().n); - } - assertEquals(List.of(1, 2), seen, "typed iteration at default page size yields decoded items"); + List seen = + collect( + client(backend) + .dataset("d1") + .iterateItems(new DatasetListItemsOptions(), null, Row.class)); + List ns = seen.stream().map(row -> row.n).toList(); + assertEquals(List.of(1, 2), ns, "typed iteration at default page size yields decoded items"); assertTrue(backend.lastUrl.contains("datasets/d1/items"), backend.lastUrl); } @@ -77,13 +77,13 @@ static final class Row { void totalCapTrimsDatasetItems() { // The cap wins even though the server would return more; only the first page is requested. MockTransport backend = MockTransport.ofConstant(200, "[{\"n\":1},{\"n\":2},{\"n\":3}]"); - List seen = new ArrayList<>(); - Iterator it = - client(backend).dataset("d1").iterateItems(new DatasetListItemsOptions().limit(2L), 5L); - while (it.hasNext()) { - seen.add(it.next().get("n").asInt()); - } - assertEquals(List.of(1, 2), seen, "limit caps the total items yielded"); + List seen = + collect( + client(backend) + .dataset("d1") + .iterateItems(new DatasetListItemsOptions().limit(2L), 5L)); + List ns = seen.stream().map(n -> n.get("n").asInt()).toList(); + assertEquals(List.of(1, 2), ns, "limit caps the total items yielded"); assertEquals(1, backend.calls); } @@ -95,12 +95,11 @@ void decodesIntoRequestedType() { MockTransport backend = new MockTransport( List.of(MockTransport.ok(200, "[{\"n\":7}]"), MockTransport.ok(200, "[]"))); - List seen = new ArrayList<>(); - Iterator it = - client(backend).dataset("d1").iterateItems(new DatasetListItemsOptions(), 2L, Item.class); - while (it.hasNext()) { - seen.add(it.next()); - } + List seen = + collect( + client(backend) + .dataset("d1") + .iterateItems(new DatasetListItemsOptions(), 2L, Item.class)); assertEquals(1, seen.size(), "typed iteration should decode and yield the item"); assertEquals(7, seen.get(0).n()); } diff --git a/src/test/java/com/apify/client/DocSnippetsTest.java b/src/test/java/com/apify/client/DocSnippetsTest.java index 7e4e525..d925a2a 100644 --- a/src/test/java/com/apify/client/DocSnippetsTest.java +++ b/src/test/java/com/apify/client/DocSnippetsTest.java @@ -103,8 +103,9 @@ private static String wrap(String className, String snippet) { + "import com.apify.client.task.*;\n" + "import com.apify.client.user.*;\n" + "import com.apify.client.webhook.*;\n" - + "import com.fasterxml.jackson.databind.*;\n" + + "import tools.jackson.databind.*;\n" + "import java.util.*;\n" + + "import java.util.concurrent.*;\n" + "import java.time.*;\n" + "import java.io.*;\n" + "@SuppressWarnings(\"all\")\n" diff --git a/src/test/java/com/apify/client/ExamplesTest.java b/src/test/java/com/apify/client/ExamplesTest.java index 5c6647e..8b36db9 100644 --- a/src/test/java/com/apify/client/ExamplesTest.java +++ b/src/test/java/com/apify/client/ExamplesTest.java @@ -47,7 +47,7 @@ void runAndLastRunStorages() { } @Test - void iterateStore() { + void iterateStore() throws Exception { requireToken(); IterateStore.main(new String[] {}); } diff --git a/src/test/java/com/apify/client/KeyIteratorTest.java b/src/test/java/com/apify/client/KeyIteratorTest.java index 570c5d2..ab64970 100644 --- a/src/test/java/com/apify/client/KeyIteratorTest.java +++ b/src/test/java/com/apify/client/KeyIteratorTest.java @@ -1,15 +1,12 @@ package com.apify.client; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import com.apify.client.keyvalue.KeyValueStoreClient; import com.apify.client.keyvalue.KeyValueStoreKey; import com.apify.client.keyvalue.ListKeysOptions; import java.time.Duration; -import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import org.junit.jupiter.api.Test; @@ -29,12 +26,17 @@ private static ApifyClient client(MockTransport backend) { .build(); } - private static List keys(Iterator it) { - List out = new ArrayList<>(); - while (it.hasNext()) { - out.add(it.next().getKey()); - } - return out; + private static List keys(KeyValueStoreClient store, ListKeysOptions options) { + return Publishers.collect(store.iterateKeys(options)).join().stream() + .map(KeyValueStoreKey::getKey) + .toList(); + } + + private static List keys( + KeyValueStoreClient store, ListKeysOptions options, Long chunkSize) { + return Publishers.collect(store.iterateKeys(options, chunkSize)).join().stream() + .map(KeyValueStoreKey::getKey) + .toList(); } @Test @@ -50,7 +52,7 @@ void chainsAcrossPagesUsingCursor() { 200, "{\"data\":{\"items\":[{\"key\":\"c\",\"size\":1}]," + "\"nextExclusiveStartKey\":null,\"isTruncated\":false}}"))); - List got = keys(client(backend).keyValueStore("s").iterateKeys(new ListKeysOptions())); + List got = keys(client(backend).keyValueStore("s"), new ListKeysOptions()); assertEquals(List.of("a", "b", "c"), got, "iterator should chain pages via the cursor"); assertEquals(2, backend.calls, "stops once the next cursor is null"); assertTrue(backend.lastUrl.contains("exclusiveStartKey=b"), backend.lastUrl); @@ -63,8 +65,7 @@ void limitCapsTotalKeysYielded() { 200, "{\"data\":{\"items\":[{\"key\":\"a\",\"size\":1},{\"key\":\"b\",\"size\":1}," + "{\"key\":\"c\",\"size\":1}],\"nextExclusiveStartKey\":\"c\",\"isTruncated\":true}}"); - List got = - keys(client(backend).keyValueStore("s").iterateKeys(new ListKeysOptions().limit(2L))); + List got = keys(client(backend).keyValueStore("s"), new ListKeysOptions().limit(2L)); assertEquals(List.of("a", "b"), got, "limit caps the total keys yielded"); assertEquals(1, backend.calls, "the cap is reached within the first page"); assertTrue(backend.lastUrl.contains("limit=2"), backend.lastUrl); @@ -87,8 +88,7 @@ void chunkSizeSetsPerRequestPageSize() { 200, "{\"data\":{\"items\":[{\"key\":\"e\",\"size\":1}]," + "\"nextExclusiveStartKey\":null,\"isTruncated\":false}}"))); - List got = - keys(client(backend).keyValueStore("s").iterateKeys(new ListKeysOptions(), 2L)); + List got = keys(client(backend).keyValueStore("s"), new ListKeysOptions(), 2L); assertEquals(List.of("a", "b", "c", "d", "e"), got, "chunkSize pages the full collection"); assertEquals(3, backend.calls); assertTrue(backend.lastUrl.contains("limit=2"), "chunkSize is sent as the page limit"); @@ -109,7 +109,7 @@ void chunkSizeAndLimitCombine() { "{\"data\":{\"items\":[{\"key\":\"c\",\"size\":1},{\"key\":\"d\",\"size\":1}]," + "\"nextExclusiveStartKey\":\"d\",\"isTruncated\":true}}"))); List got = - keys(client(backend).keyValueStore("s").iterateKeys(new ListKeysOptions().limit(3L), 2L)); + keys(client(backend).keyValueStore("s"), new ListKeysOptions().limit(3L), 2L); assertEquals(List.of("a", "b", "c"), got, "the total cap trims across chunked pages"); assertEquals(2, backend.calls); assertTrue( @@ -121,9 +121,8 @@ void stopsOnEmptyPage() { MockTransport backend = MockTransport.ofConstant( 200, "{\"data\":{\"items\":[],\"nextExclusiveStartKey\":\"z\",\"isTruncated\":true}}"); - Iterator it = - client(backend).keyValueStore("s").iterateKeys(new ListKeysOptions()); - assertFalse(it.hasNext(), "an empty page ends iteration even if a next cursor is reported"); + List got = keys(client(backend).keyValueStore("s"), new ListKeysOptions()); + assertTrue(got.isEmpty(), "an empty page ends iteration even if a next cursor is reported"); assertEquals(1, backend.calls); } } diff --git a/src/test/java/com/apify/client/MockTransport.java b/src/test/java/com/apify/client/MockTransport.java index 974c5fd..0c5b3d4 100644 --- a/src/test/java/com/apify/client/MockTransport.java +++ b/src/test/java/com/apify/client/MockTransport.java @@ -16,6 +16,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Flow; import javax.net.ssl.SSLSession; @@ -31,9 +32,9 @@ final class MockTransport implements HttpTransport { static final class Scripted { final int status; final byte[] body; - final IOException error; + final Exception error; - Scripted(int status, String body, IOException error) { + Scripted(int status, String body, Exception error) { this.status = status; this.body = body == null ? new byte[0] : body.getBytes(StandardCharsets.UTF_8); this.error = error; @@ -85,7 +86,7 @@ static Scripted timeoutError() { // synchronized: batchAddRequests may drive this transport from several threads at once. @Override - public synchronized HttpResponse send(HttpRequest request) throws IOException { + public synchronized CompletableFuture> sendAsync(HttpRequest request) { int idx = calls++; lastHeaders = request.headers(); lastUrl = request.uri().toString(); @@ -100,9 +101,11 @@ public synchronized HttpResponse send(HttpRequest request) throws IOExce } Scripted r = responses.get(idx); if (r.error != null) { - throw r.error; + CompletableFuture> failed = new CompletableFuture<>(); + failed.completeExceptionally(r.error); + return failed; } - return new FakeResponse(request.uri(), r.status, r.body); + return CompletableFuture.completedFuture(new FakeResponse(request.uri(), r.status, r.body)); } /** Scripts the next {@link #sendStreamingResponse} call to return the given status and body. */ @@ -124,16 +127,18 @@ void scriptStream(int status, InputStream body) { } @Override - public synchronized HttpResponse sendStreamingResponse(HttpRequest request) { + public synchronized CompletableFuture> sendStreamingAsync( + HttpRequest request) { calls++; lastUrl = request.uri().toString(); lastMethod = request.method(); if (streamBody != null) { - return new FakeStreamResponse(request.uri(), streamBodyStatus, streamBody); + return CompletableFuture.completedFuture( + new FakeStreamResponse(request.uri(), streamBodyStatus, streamBody)); } Scripted r = streamResponse != null ? streamResponse : responses.get(0); - return new FakeStreamResponse( - request.uri(), r.status, new java.io.ByteArrayInputStream(r.body)); + return CompletableFuture.completedFuture( + new FakeStreamResponse(request.uri(), r.status, new java.io.ByteArrayInputStream(r.body))); } private static byte[] readBody(HttpRequest request) { diff --git a/src/test/java/com/apify/client/PaginatedIteratorTest.java b/src/test/java/com/apify/client/PaginatedIteratorTest.java deleted file mode 100644 index e704b26..0000000 --- a/src/test/java/com/apify/client/PaginatedIteratorTest.java +++ /dev/null @@ -1,147 +0,0 @@ -package com.apify.client; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.apify.client.internal.PaginatedIterator; -import java.util.ArrayList; -import java.util.List; -import java.util.NoSuchElementException; -import org.junit.jupiter.api.Test; - -/** - * Hermetic (token-free) tests for the offset/limit iteration engine. They drive {@link - * PaginatedIterator} with a stub page fetcher over a synthetic collection, pinning the total-cap / - * chunk-size arithmetic, server-side page-size clamping, and termination — without any network or - * {@code APIFY_TOKEN}. - */ -class PaginatedIteratorTest { - - /** - * A fake paginated endpoint over integers {@code [0, available)}. Like the real API, it clamps a - * requested page size (or an unset one) to {@code maxPageSize} ({@code 0} = no clamp). - */ - private static final class StubFetcher implements PaginatedIterator.PageFetcher { - final long available; - final long maxPageSize; - final List requests = new ArrayList<>(); - - StubFetcher(long available) { - this(available, 0); - } - - StubFetcher(long available, long maxPageSize) { - this.available = available; - this.maxPageSize = maxPageSize; - } - - @Override - public PaginationList fetch(long offset, Long limit) { - requests.add(new long[] {offset, limit == null ? -1 : limit}); - long requested = limit == null ? Long.MAX_VALUE : limit; - long take = maxPageSize > 0 ? Math.min(requested, maxPageSize) : requested; - List items = new ArrayList<>(); - for (long i = offset; i < available && items.size() < take; i++) { - items.add((int) i); - } - PaginationList page = new PaginationList<>(); - page.setItems(items); - // Some endpoints (e.g. dataset items) report a total of 0 or a lagging value; the engine must - // not depend on it, so the stub deliberately reports an unhelpful total. - page.setTotal(0); - page.setOffset(offset); - page.setCount(items.size()); - return page; - } - } - - private static List drain(java.util.Iterator it) { - List out = new ArrayList<>(); - while (it.hasNext()) { - out.add(it.next()); - } - return out; - } - - @Test - void totalCapTrimsLastPage() { - StubFetcher f = new StubFetcher(10); - List got = drain(new PaginatedIterator<>(3L, 2L, null, f)); - assertEquals(List.of(0, 1, 2), got, "limit=3 caps the total yielded across pages"); - // First page requests min(cap,chunk)=2; second requests min(remaining=1,chunk=2)=1; the cap is - // reached exactly, so no trailing empty request is made. - assertEquals(2, f.requests.size()); - assertEquals(0L, f.requests.get(0)[0]); - assertEquals(2L, f.requests.get(0)[1]); - assertEquals(2L, f.requests.get(1)[0]); - assertEquals(1L, f.requests.get(1)[1]); - } - - @Test - void chunkSizePagesAllThenStopsOnEmptyPage() { - StubFetcher f = new StubFetcher(5); - List got = drain(new PaginatedIterator<>(null, 2L, null, f)); - assertEquals(List.of(0, 1, 2, 3, 4), got); - // 5 items / page size 2 => pages of 2,2,1 then a trailing empty page confirms the end. - assertEquals(4, f.requests.size()); - assertEquals(5L, f.requests.get(3)[0], "trailing request pages past the last item"); - assertEquals(2L, f.requests.get(3)[1], "each page still requests the chunk size"); - } - - @Test - void serverClampsLargePageSizeSoShortPageIsNotTheEnd() { - // Server caps every page at 2 items; the caller asks for far more. A page shorter than the - // request must NOT be treated as end-of-collection. - StubFetcher f = new StubFetcher(5, 2); - List got = drain(new PaginatedIterator<>(null, 100L, null, f)); - assertEquals(List.of(0, 1, 2, 3, 4), got, "clamped short pages must keep paging to the end"); - } - - @Test - void capIsHonoredEvenWhenServerClampsPages() { - StubFetcher f = new StubFetcher(100, 2); - List got = drain(new PaginatedIterator<>(5L, 100L, null, f)); - assertEquals(List.of(0, 1, 2, 3, 4), got, "the total cap holds despite server page clamping"); - } - - @Test - void noChunkUsesServerDefaultAndPagesToEmpty() { - StubFetcher f = new StubFetcher(4); - List got = drain(new PaginatedIterator<>(null, null, null, f)); - assertEquals(List.of(0, 1, 2, 3), got); - assertEquals(-1L, f.requests.get(0)[1], "no cap and no chunk => null limit (server default)"); - } - - @Test - void startOffsetIsHonored() { - StubFetcher f = new StubFetcher(10); - List got = drain(new PaginatedIterator<>(null, 5L, 7L, f)); - assertEquals(List.of(7, 8, 9), got, "iteration starts at the requested offset"); - assertEquals(7L, f.requests.get(0)[0]); - } - - @Test - void trimsOverReturnedPageToCap() { - // A misbehaving server that returns more items than requested must not make the iterator - // overshoot the caller's total cap. - PaginatedIterator.PageFetcher overReturning = - (offset, limit) -> { - PaginationList page = new PaginationList<>(); - page.setItems(List.of(1, 2, 3, 4, 5)); // ignores the requested limit - page.setTotal(5); - page.setCount(5); - return page; - }; - List got = drain(new PaginatedIterator<>(3L, 2L, null, overReturning)); - assertEquals(List.of(1, 2, 3), got, "the last page is trimmed to the cap"); - } - - @Test - void emptyCollectionYieldsNothing() { - StubFetcher f = new StubFetcher(0); - java.util.Iterator it = new PaginatedIterator<>(null, 5L, null, f); - assertFalse(it.hasNext()); - assertThrows(NoSuchElementException.class, it::next); - } -} diff --git a/src/test/java/com/apify/client/StreamedLogTest.java b/src/test/java/com/apify/client/StreamedLogTest.java index f469b5d..f41cbc8 100644 --- a/src/test/java/com/apify/client/StreamedLogTest.java +++ b/src/test/java/com/apify/client/StreamedLogTest.java @@ -50,8 +50,8 @@ private static List redirect(String streamBody, StreamedLogOptions optio backend.scriptStream(200, streamBody); List collected = new CopyOnWriteArrayList<>(); StreamedLog streamedLog = - client(backend).run("run123").getStreamedLog(options.toLog(collected::add)); - streamedLog.start(); + client(backend).run("run123").getStreamedLog(options.toLog(collected::add)).join(); + streamedLog.start().join(); // The scripted stream is finite, so redirection finishes on its own; stop() joins the thread. streamedLog.stop(); return collected; @@ -105,8 +105,8 @@ void defaultDestinationDoesNotThrow() throws InterruptedException { MockTransport.ofConstant( 200, "{\"data\":{\"id\":\"run123\",\"actId\":\"act1\",\"name\":\"a\"}}"); backend.scriptStream(200, "2999-01-01T00:00:00.000Z hello\n"); - StreamedLog streamedLog = client(backend).run("run123").getStreamedLog(); - streamedLog.start(); + StreamedLog streamedLog = client(backend).run("run123").getStreamedLog().join(); + streamedLog.start().join(); streamedLog.stop(); // No assertion on output (goes to the default SLF4J logger); the point is it runs without // throwing. @@ -143,8 +143,9 @@ void throwingConsumerDoesNotKillDaemonThreadUncaught() throws InterruptedExcepti message -> { calls.incrementAndGet(); throw new RuntimeException("consumer failed"); - })); - streamedLog.start(); + })) + .join(); + streamedLog.start().join(); streamedLog.stop(); // joins the reader; if it died uncaught, the handler above has fired assertNull( uncaught.get(), @@ -164,8 +165,11 @@ void startTwiceThrows() throws InterruptedException { MockTransport backend = MockTransport.ofConstant(200, ""); backend.scriptStream(200, "2999-01-01T00:00:00.000Z x\n"); StreamedLog streamedLog = - client(backend).run("run123").getStreamedLog(new StreamedLogOptions().toLog(m -> {})); - streamedLog.start(); + client(backend) + .run("run123") + .getStreamedLog(new StreamedLogOptions().toLog(m -> {})) + .join(); + streamedLog.start().join(); assertThrows(IllegalStateException.class, streamedLog::start); streamedLog.stop(); } @@ -174,7 +178,10 @@ void startTwiceThrows() throws InterruptedException { void stopWithoutStartThrows() { MockTransport backend = MockTransport.ofConstant(200, ""); StreamedLog streamedLog = - client(backend).run("run123").getStreamedLog(new StreamedLogOptions().toLog(m -> {})); + client(backend) + .run("run123") + .getStreamedLog(new StreamedLogOptions().toLog(m -> {})) + .join(); assertThrows(IllegalStateException.class, streamedLog::stop); } @@ -182,7 +189,10 @@ void stopWithoutStartThrows() { void closeWithoutStartIsNoOp() { MockTransport backend = MockTransport.ofConstant(200, ""); StreamedLog streamedLog = - client(backend).run("run123").getStreamedLog(new StreamedLogOptions().toLog(m -> {})); + client(backend) + .run("run123") + .getStreamedLog(new StreamedLogOptions().toLog(m -> {})) + .join(); // close() on a never-started helper must not throw (supports try-with-resources). streamedLog.close(); } @@ -206,8 +216,9 @@ void deliversLastMessageWhenStoppingLiveStream() { StreamedLog streamedLog = client(backend) .run("run123") - .getStreamedLog(new StreamedLogOptions().toLog(collected::add)); - streamedLog.start(); + .getStreamedLog(new StreamedLogOptions().toLog(collected::add)) + .join(); + streamedLog.start().join(); // Wait until a and b have been redirected, which proves the reader has consumed the body and is // now blocked with c held back as the pending last message. pollUntil(REDIRECT_POLL_ATTEMPTS, REDIRECT_POLL_MILLIS, () -> collected.size() >= 2); @@ -231,8 +242,11 @@ void closeAfterStopIsNoOp() throws InterruptedException { MockTransport backend = MockTransport.ofConstant(200, ""); backend.scriptStream(200, "2999-01-01T00:00:00.000Z x\n"); StreamedLog streamedLog = - client(backend).run("run123").getStreamedLog(new StreamedLogOptions().toLog(m -> {})); - streamedLog.start(); + client(backend) + .run("run123") + .getStreamedLog(new StreamedLogOptions().toLog(m -> {})) + .join(); + streamedLog.start().join(); streamedLog.stop(); streamedLog.close(); // must be a no-op, not an IllegalStateException streamedLog.close(); // idempotent on repeat @@ -250,8 +264,11 @@ void concurrentCloseNeverThrowsWhileStopRaces() throws InterruptedException { MockTransport backend = MockTransport.ofConstant(200, ""); backend.scriptStream(200, "2999-01-01T00:00:00.000Z x\n"); StreamedLog streamedLog = - client(backend).run("run123").getStreamedLog(new StreamedLogOptions().toLog(m -> {})); - streamedLog.start(); + client(backend) + .run("run123") + .getStreamedLog(new StreamedLogOptions().toLog(m -> {})) + .join(); + streamedLog.start().join(); CountDownLatch go = new CountDownLatch(1); AtomicReference closeError = new AtomicReference<>(); @@ -300,8 +317,9 @@ void closeStopsActiveRedirection() throws InterruptedException { try (StreamedLog streamedLog = client(backend) .run("run123") - .getStreamedLog(new StreamedLogOptions().toLog(collected::add))) { - streamedLog.start(); + .getStreamedLog(new StreamedLogOptions().toLog(collected::add)) + .join()) { + streamedLog.start().join(); // give the finite stream a moment to be consumed Thread.sleep(Duration.ofMillis(50).toMillis()); } @@ -337,9 +355,10 @@ void stopFromInsideConsumerDoesNotDeadlock() { if (stopRequested.compareAndSet(false, true)) { ref.get().stop(); } - })); + })) + .join(); ref.set(streamedLog); - streamedLog.start(); + streamedLog.start().join(); pollUntil(REDIRECT_POLL_ATTEMPTS, REDIRECT_POLL_MILLIS, () -> collected.size() >= 2); assertEquals( List.of("2999-01-01T00:00:00.000Z a", "2999-01-01T00:00:01.000Z b"), @@ -367,9 +386,10 @@ void closeFromInsideConsumerDoesNotDeadlock() { message -> { collected.add(message); ref.get().close(); // idempotent self-close on the reader thread - })); + })) + .join(); ref.set(streamedLog); - streamedLog.start(); + streamedLog.start().join(); pollUntil(REDIRECT_POLL_ATTEMPTS, REDIRECT_POLL_MILLIS, () -> collected.size() >= 2); assertEquals( List.of("2999-01-01T00:00:00.000Z a", "2999-01-01T00:00:01.000Z b"), @@ -380,12 +400,12 @@ void closeFromInsideConsumerDoesNotDeadlock() { @Test void defaultPrefixLookupFailureStillCreatesHelper() { // The no-arg getStreamedLog() builds a cosmetic per-run prefix by GETting the run (and its - // Actor). Those getters swallow only 404; a 401/403/5xx-after-retries would otherwise throw out - // of getStreamedLog() and abort helper creation even though streaming might work. The lookup is + // Actor). Those getters swallow only 404; a 401/403/5xx-after-retries would otherwise fail the + // returned future and abort helper creation even though streaming might work. The lookup is // wrapped so it falls back to the runId-only prefix instead of failing. MockTransport backend = new MockTransport(List.of(MockTransport.ok(401, "{\"error\":{}}"))); StreamedLog streamedLog = - assertDoesNotThrow(() -> client(backend).run("run123").getStreamedLog()); + assertDoesNotThrow(() -> client(backend).run("run123").getStreamedLog().join()); assertNotNull(streamedLog); } diff --git a/src/test/java/com/apify/client/TestAsync.java b/src/test/java/com/apify/client/TestAsync.java new file mode 100644 index 0000000..4dda8c4 --- /dev/null +++ b/src/test/java/com/apify/client/TestAsync.java @@ -0,0 +1,31 @@ +package com.apify.client; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +/** + * Test-only helper for consuming this now-async client's {@link CompletableFuture}-returning + * methods synchronously in JUnit assertions. + * + *

    {@link CompletableFuture#join()} wraps any exceptional completion in a {@link + * CompletionException}, which would otherwise defeat {@code assertThrows(ApifyApiException.class, + * ...)}-style assertions throughout this suite (they expect the client's own exception type, not a + * wrapper). {@link #await(CompletableFuture)} unwraps that one layer so the original exception + * (already unchecked - see {@code ApifyClientException}) propagates as-is. + */ +public final class TestAsync { + private TestAsync() {} + + /** Blocks for {@code future}'s result, unwrapping a {@link CompletionException} if it fails. */ + public static T await(CompletableFuture future) { + try { + return future.join(); + } catch (CompletionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException runtimeException) { + throw runtimeException; + } + throw e; + } + } +} diff --git a/src/test/java/com/apify/client/UnitHttpTest.java b/src/test/java/com/apify/client/UnitHttpTest.java index f132d29..f22be22 100644 --- a/src/test/java/com/apify/client/UnitHttpTest.java +++ b/src/test/java/com/apify/client/UnitHttpTest.java @@ -29,7 +29,7 @@ private static ApifyClient client(MockTransport transport, int maxRetries) { void successSingleCall() { MockTransport transport = MockTransport.ofConstant(200, "{\"data\":{\"id\":\"u1\",\"username\":\"bob\"}}"); - Optional user = client(transport, 8).me().get(); + Optional user = client(transport, 8).me().get().join(); assertTrue(user.isPresent()); assertEquals("u1", user.get().getId()); assertEquals("bob", user.get().getUsername()); @@ -42,7 +42,8 @@ void rateLimitIsRetried() { MockTransport.ofConstant( 429, "{\"error\":{\"type\":\"rate-limit-exceeded\",\"message\":\"slow down\"}}"); ApifyApiException ex = - assertThrows(ApifyApiException.class, () -> client(transport, 2).me().get()); + assertThrows( + ApifyApiException.class, () -> TestAsync.await(client(transport, 2).me().get())); assertEquals(429, ex.getStatusCode()); assertEquals(3, transport.calls); // 1 initial + 2 retries assertEquals(3, ex.getAttempt()); @@ -52,7 +53,7 @@ void rateLimitIsRetried() { void serverErrorIsRetried() { MockTransport transport = MockTransport.ofConstant(503, "{\"error\":{\"type\":\"internal\",\"message\":\"boom\"}}"); - assertThrows(ApifyApiException.class, () -> client(transport, 1).me().get()); + assertThrows(ApifyApiException.class, () -> TestAsync.await(client(transport, 1).me().get())); assertEquals(2, transport.calls); } @@ -61,14 +62,14 @@ void clientErrorNotRetried() { MockTransport transport = MockTransport.ofConstant( 400, "{\"error\":{\"type\":\"bad-request\",\"message\":\"nope\"}}"); - assertThrows(ApifyApiException.class, () -> client(transport, 5).me().get()); + assertThrows(ApifyApiException.class, () -> TestAsync.await(client(transport, 5).me().get())); assertEquals(1, transport.calls); } @Test void networkErrorIsRetried() { MockTransport transport = new MockTransport(List.of(MockTransport.networkError())); - assertThrows(RuntimeException.class, () -> client(transport, 3).me().get()); + assertThrows(RuntimeException.class, () -> TestAsync.await(client(transport, 3).me().get())); assertEquals(4, transport.calls); } @@ -80,7 +81,7 @@ void retryThenSuccess() { MockTransport.ok(500, "{\"error\":{\"type\":\"internal\",\"message\":\"x\"}}"), MockTransport.ok(500, "{\"error\":{\"type\":\"internal\",\"message\":\"x\"}}"), MockTransport.ok(200, "{\"data\":{\"id\":\"ok\"}}"))); - Optional user = client(transport, 5).me().get(); + Optional user = client(transport, 5).me().get().join(); assertTrue(user.isPresent()); assertEquals("ok", user.get().getId()); assertEquals(3, transport.calls); @@ -91,7 +92,7 @@ void notFoundMapsToEmpty() { MockTransport transport = MockTransport.ofConstant( 404, "{\"error\":{\"type\":\"record-not-found\",\"message\":\"missing\"}}"); - Optional actor = client(transport, 5).actor("nope").get(); + Optional actor = client(transport, 5).actor("nope").get().join(); assertFalse(actor.isPresent()); assertEquals(1, transport.calls); // no retry on 404 } @@ -101,7 +102,7 @@ void notFoundWithTypeButNoMessageMapsToEmpty() { // The `type` field alone must drive not-found detection; `message` may be absent. MockTransport transport = MockTransport.ofConstant(404, "{\"error\":{\"type\":\"record-not-found\"}}"); - Optional actor = client(transport, 5).actor("nope").get(); + Optional actor = client(transport, 5).actor("nope").get().join(); assertFalse(actor.isPresent()); assertEquals(1, transport.calls); // no retry on 404 } @@ -110,7 +111,7 @@ void notFoundWithTypeButNoMessageMapsToEmpty() { void deleteOnNotFoundWithTypeButNoMessageIsNoOp() { MockTransport transport = MockTransport.ofConstant(404, "{\"error\":{\"type\":\"record-or-token-not-found\"}}"); - client(transport, 5).actor("nope").delete(); // must not throw + client(transport, 5).actor("nope").delete().join(); // must not throw assertEquals(1, transport.calls); } @@ -119,7 +120,8 @@ void nonNotFoundErrorWithTypeButNoMessageStillThrowsWithType() { MockTransport transport = MockTransport.ofConstant(400, "{\"error\":{\"type\":\"bad-request\"}}"); ApifyApiException ex = - assertThrows(ApifyApiException.class, () -> client(transport, 0).me().get()); + assertThrows( + ApifyApiException.class, () -> TestAsync.await(client(transport, 0).me().get())); assertEquals(400, ex.getStatusCode()); assertEquals("bad-request", ex.getType()); } @@ -131,7 +133,8 @@ void errorBodyIsParsed() { 400, "{\"error\":{\"type\":\"bad-request\",\"message\":\"invalid input\",\"data\":{\"field\":\"name\"}}}"); ApifyApiException ex = - assertThrows(ApifyApiException.class, () -> client(transport, 0).me().get()); + assertThrows( + ApifyApiException.class, () -> TestAsync.await(client(transport, 0).me().get())); assertEquals(400, ex.getStatusCode()); assertEquals("bad-request", ex.getType()); assertTrue(ex.getMessage().contains("invalid input")); @@ -144,7 +147,7 @@ void errorBodyIsParsed() { void zeroRetriesSingleAttempt() { MockTransport transport = MockTransport.ofConstant(500, "{\"error\":{\"type\":\"internal\",\"message\":\"x\"}}"); - assertThrows(ApifyApiException.class, () -> client(transport, 0).me().get()); + assertThrows(ApifyApiException.class, () -> TestAsync.await(client(transport, 0).me().get())); assertEquals(1, transport.calls); } } diff --git a/src/test/java/com/apify/client/examples/CreateBuildRunActor.java b/src/test/java/com/apify/client/examples/CreateBuildRunActor.java index a568de0..099551e 100644 --- a/src/test/java/com/apify/client/examples/CreateBuildRunActor.java +++ b/src/test/java/com/apify/client/examples/CreateBuildRunActor.java @@ -50,22 +50,22 @@ public static void main(String[] args) { "content", "console.log('hello from the java client example');"))))); - Actor created = client.actors().create(actorDef); + Actor created = client.actors().create(actorDef).join(); try { System.out.println("Created actor " + created.getId()); ActorClient actor = client.actor(created.getId()); - Build build = actor.build("0.0", new ActorBuildOptions()); - client.build(build.getId()).waitForFinish(300L); + Build build = actor.build("0.0", new ActorBuildOptions()).join(); + client.build(build.getId()).waitForFinish(300L).join(); System.out.println("Built actor (build " + build.getId() + ")"); - ActorRun run = actor.call(null, new ActorStartOptions(), 120L); + ActorRun run = actor.call(null, new ActorStartOptions(), 120L).join(); System.out.println("Run " + run.getId() + " finished with status " + run.getStatus()); - Optional log = client.run(run.getId()).log().get(); + Optional log = client.run(run.getId()).log().get().join(); log.ifPresent(text -> System.out.println("--- run log ---\n" + text)); } finally { - client.actor(created.getId()).delete(); + client.actor(created.getId()).delete().join(); } } } diff --git a/src/test/java/com/apify/client/examples/GetAccount.java b/src/test/java/com/apify/client/examples/GetAccount.java index 69738b9..1d7dea5 100644 --- a/src/test/java/com/apify/client/examples/GetAccount.java +++ b/src/test/java/com/apify/client/examples/GetAccount.java @@ -14,7 +14,7 @@ private GetAccount() {} public static void main(String[] args) { ApifyClient client = ApifyClient.create(System.getenv("APIFY_TOKEN")); - Optional user = client.me().get(); + Optional user = client.me().get().join(); if (user.isEmpty()) { throw new IllegalStateException("current user not found"); } diff --git a/src/test/java/com/apify/client/examples/IterateStore.java b/src/test/java/com/apify/client/examples/IterateStore.java index 5179179..b0d687f 100644 --- a/src/test/java/com/apify/client/examples/IterateStore.java +++ b/src/test/java/com/apify/client/examples/IterateStore.java @@ -3,25 +3,57 @@ import com.apify.client.ApifyClient; import com.apify.client.store.ActorStoreListItem; import com.apify.client.store.StoreListOptions; -import java.util.Iterator; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Flow; /** * Lazily iterates over Actors in the Apify Store using the convenience iteration method, printing - * the first few. + * the first few without fetching (or even requesting) the rest of the collection. */ public final class IterateStore { private IterateStore() {} - public static void main(String[] args) { + public static void main(String[] args) throws InterruptedException { ApifyClient client = ApifyClient.create(System.getenv("APIFY_TOKEN")); - Iterator it = - client.store().iterate(new StoreListOptions().limit(10L), 10L); - int count = 0; - while (count < 5 && it.hasNext()) { - ActorStoreListItem item = it.next(); - System.out.println((count + 1) + ". " + item.getUsername() + "/" + item.getName()); - count++; - } + CountDownLatch done = new CountDownLatch(1); + client + .store() + .iterate(new StoreListOptions().limit(10L), 10L) + .subscribe( + new Flow.Subscriber<>() { + private Flow.Subscription subscription; + private int count; + + @Override + public void onSubscribe(Flow.Subscription subscription) { + this.subscription = subscription; + subscription.request(1); // pull one item at a time + } + + @Override + public void onNext(ActorStoreListItem item) { + count++; + System.out.println(count + ". " + item.getUsername() + "/" + item.getName()); + if (count >= 5) { + subscription.cancel(); // stop early; no further pages are fetched + done.countDown(); + } else { + subscription.request(1); + } + } + + @Override + public void onError(Throwable throwable) { + done.countDown(); + throw new RuntimeException(throwable); + } + + @Override + public void onComplete() { + done.countDown(); + } + }); + done.await(); } } diff --git a/src/test/java/com/apify/client/examples/LogRedirection.java b/src/test/java/com/apify/client/examples/LogRedirection.java index 825f9b1..8bb1929 100644 --- a/src/test/java/com/apify/client/examples/LogRedirection.java +++ b/src/test/java/com/apify/client/examples/LogRedirection.java @@ -16,14 +16,14 @@ private LogRedirection() {} public static void main(String[] args) { ApifyClient client = ApifyClient.create(System.getenv("APIFY_TOKEN")); - // Start the Actor and return immediately (do not wait for it to finish). - ActorRun run = client.actor("apify/hello-world").start(null, new ActorStartOptions()); + // Start the Actor and complete as soon as the run exists (do not wait for it to finish). + ActorRun run = client.actor("apify/hello-world").start(null, new ActorStartOptions()).join(); RunClient runClient = client.run(run.getId()); // Redirect the run's live log to the default per-run logger while we wait for it to finish. - try (StreamedLog streamedLog = runClient.getStreamedLog()) { - streamedLog.start(); - runClient.waitForFinish(120L); + try (StreamedLog streamedLog = runClient.getStreamedLog().join()) { + streamedLog.start().join(); + runClient.waitForFinish(120L).join(); } } } diff --git a/src/test/java/com/apify/client/examples/RunAndLastRunStorages.java b/src/test/java/com/apify/client/examples/RunAndLastRunStorages.java index 28e6500..ffbdbc9 100644 --- a/src/test/java/com/apify/client/examples/RunAndLastRunStorages.java +++ b/src/test/java/com/apify/client/examples/RunAndLastRunStorages.java @@ -16,17 +16,19 @@ private RunAndLastRunStorages() {} public static void main(String[] args) { ApifyClient client = ApifyClient.create(System.getenv("APIFY_TOKEN")); - client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L); + client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L).join(); - Optional lastRun = client.actor("apify/hello-world").lastRun("SUCCEEDED").get(); + Optional lastRun = + client.actor("apify/hello-world").lastRun("SUCCEEDED").get().join(); if (lastRun.isEmpty()) { throw new IllegalStateException("no succeeded run found"); } ActorRun run = lastRun.get(); System.out.println("Last run: " + run.getId()); - var items = client.dataset(run.getDefaultDatasetId()).listItems(new DatasetListItemsOptions()); + var items = + client.dataset(run.getDefaultDatasetId()).listItems(new DatasetListItemsOptions()).join(); System.out.println("Items in this page of the last run's dataset: " + items.getCount()); - client.keyValueStore(run.getDefaultKeyValueStoreId()).getRecord("OUTPUT"); + client.keyValueStore(run.getDefaultKeyValueStoreId()).getRecord("OUTPUT").join(); } } diff --git a/src/test/java/com/apify/client/examples/RunStoreActor.java b/src/test/java/com/apify/client/examples/RunStoreActor.java index 12ba9a2..8822941 100644 --- a/src/test/java/com/apify/client/examples/RunStoreActor.java +++ b/src/test/java/com/apify/client/examples/RunStoreActor.java @@ -15,10 +15,12 @@ private RunStoreActor() {} public static void main(String[] args) { ApifyClient client = ApifyClient.create(System.getenv("APIFY_TOKEN")); - ActorRun run = client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L); + ActorRun run = + client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L).join(); System.out.println("Run " + run.getId() + " finished with status " + run.getStatus()); - var items = client.dataset(run.getDefaultDatasetId()).listItems(new DatasetListItemsOptions()); + var items = + client.dataset(run.getDefaultDatasetId()).listItems(new DatasetListItemsOptions()).join(); System.out.println("Items in this page of the default dataset: " + items.getCount()); } } diff --git a/src/test/java/com/apify/client/examples/Storages.java b/src/test/java/com/apify/client/examples/Storages.java index 93733ad..7ad5012 100644 --- a/src/test/java/com/apify/client/examples/Storages.java +++ b/src/test/java/com/apify/client/examples/Storages.java @@ -28,36 +28,37 @@ public static void main(String[] args) { String suffix = UUID.randomUUID().toString().substring(0, 8); // Dataset: create, push items, read them back. - Dataset dataset = client.datasets().getOrCreate("java-example-ds-" + suffix); + Dataset dataset = client.datasets().getOrCreate("java-example-ds-" + suffix).join(); try { - client.dataset(dataset.getId()).pushItems(List.of(Map.of("hello", "world"))); - var items = client.dataset(dataset.getId()).listItems(new DatasetListItemsOptions()); + client.dataset(dataset.getId()).pushItems(List.of(Map.of("hello", "world"))).join(); + var items = client.dataset(dataset.getId()).listItems(new DatasetListItemsOptions()).join(); System.out.println("Dataset items: " + items.getItems()); } finally { - client.dataset(dataset.getId()).delete(); + client.dataset(dataset.getId()).delete().join(); } // Key-value store: create, set a record, read it back. - KeyValueStore store = client.keyValueStores().getOrCreate("java-example-kvs-" + suffix); + KeyValueStore store = client.keyValueStores().getOrCreate("java-example-kvs-" + suffix).join(); try { - client.keyValueStore(store.getId()).setRecordJson("OUTPUT", Map.of("answer", 42)); + client.keyValueStore(store.getId()).setRecordJson("OUTPUT", Map.of("answer", 42)).join(); Optional record = - client.keyValueStore(store.getId()).getRecord("OUTPUT"); + client.keyValueStore(store.getId()).getRecord("OUTPUT").join(); record.ifPresent(r -> System.out.println("KVS record bytes: " + r.getValue().length)); } finally { - client.keyValueStore(store.getId()).delete(); + client.keyValueStore(store.getId()).delete().join(); } // Request queue: create, add a request, read the head. - RequestQueue queue = client.requestQueues().getOrCreate("java-example-rq-" + suffix); + RequestQueue queue = client.requestQueues().getOrCreate("java-example-rq-" + suffix).join(); try { client .requestQueue(queue.getId()) - .addRequest(new RequestQueueRequest("https://example.com", "example"), false); - var head = client.requestQueue(queue.getId()).listHead(10L); + .addRequest(new RequestQueueRequest("https://example.com", "example"), false) + .join(); + var head = client.requestQueue(queue.getId()).listHead(10L).join(); System.out.println("Request queue head size: " + head.getItems().size()); } finally { - client.requestQueue(queue.getId()).delete(); + client.requestQueue(queue.getId()).delete().join(); } } } diff --git a/src/test/java/com/apify/client/integration/ActorIntegrationTest.java b/src/test/java/com/apify/client/integration/ActorIntegrationTest.java index bb96f61..79f626c 100644 --- a/src/test/java/com/apify/client/integration/ActorIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/ActorIntegrationTest.java @@ -6,6 +6,7 @@ import com.apify.client.ApifyClient; import com.apify.client.ListOptions; import com.apify.client.PaginationList; +import com.apify.client.Publishers; import com.apify.client.actor.Actor; import com.apify.client.actor.ActorBuildOptions; import com.apify.client.actor.ActorClient; @@ -50,34 +51,35 @@ static Map minimalActor(String name) { @Test void listActors() { ApifyClient client = requireClient(); - PaginationList page = client.actors().list(new ActorListOptions().my(true).limit(5L)); + PaginationList page = + client.actors().list(new ActorListOptions().my(true).limit(5L)).join(); assertTrue(page.getTotal() >= 0); } @Test void getActor() { ApifyClient client = requireClient(); - Actor created = client.actors().create(minimalActor(uniqueName("get"))); + Actor created = client.actors().create(minimalActor(uniqueName("get"))).join(); try { - var got = client.actor(created.getId()).get(); + var got = client.actor(created.getId()).get().join(); assertTrue(got.isPresent()); assertEquals(created.getId(), got.get().getId()); } finally { - client.actor(created.getId()).delete(); + client.actor(created.getId()).delete().join(); } } @Test void actorCrudFlow() { ApifyClient client = requireClient(); - Actor created = client.actors().create(minimalActor(uniqueName("crud"))); + Actor created = client.actors().create(minimalActor(uniqueName("crud"))).join(); try { ActorClient actor = client.actor(created.getId()); - assertTrue(actor.get().isPresent()); - Actor updated = actor.update(Map.of("title", "Updated Title")); + assertTrue(actor.get().join().isPresent()); + Actor updated = actor.update(Map.of("title", "Updated Title")).join(); assertEquals("Updated Title", updated.getTitle()); - actor.builds().list(new ListOptions()); - actor.versions().list(new ListOptions()); + actor.builds().list(new ListOptions()).join(); + actor.versions().list(new ListOptions()).join(); // list() step of the create/get/modify/list/delete flow: verify the just-created Actor // appears in the top-level collection listing. @@ -89,19 +91,20 @@ void actorCrudFlow() { client .actors() .list(new ActorListOptions().my(true).desc(true).limit(10L)) + .join() .getItems() .stream() .anyMatch(a -> created.getId().equals(a.getId()))); assertTrue(foundInList, "expected the just-created Actor to appear in the top-level list"); } finally { - client.actor(created.getId()).delete(); + client.actor(created.getId()).delete().join(); } } @Test void actorVersionCrudFlow() { ApifyClient client = requireClient(); - Actor created = client.actors().create(minimalActor(uniqueName("ver"))); + Actor created = client.actors().create(minimalActor(uniqueName("ver"))).join(); try { ActorClient actor = client.actor(created.getId()); ActorVersion version = @@ -112,17 +115,19 @@ void actorVersionCrudFlow() { "versionNumber", "0.1", "sourceType", "SOURCE_FILES", "buildTag", "latest", - "sourceFiles", List.of())); + "sourceFiles", List.of())) + .join(); assertEquals("0.1", version.getVersionNumber()); - assertTrue(actor.version("0.1").get().isPresent()); - actor.versions().list(new ListOptions()); + assertTrue(actor.version("0.1").get().join().isPresent()); + actor.versions().list(new ListOptions()).join(); actor .version("0.1") .update( - Map.of("buildTag", "beta", "sourceType", "SOURCE_FILES", "sourceFiles", List.of())); - actor.version("0.1").delete(); + Map.of("buildTag", "beta", "sourceType", "SOURCE_FILES", "sourceFiles", List.of())) + .join(); + actor.version("0.1").delete().join(); } finally { - client.actor(created.getId()).delete(); + client.actor(created.getId()).delete().join(); } } @@ -131,21 +136,22 @@ void validateInput() { ApifyClient client = requireClient(); // apify/hello-world is a public store Actor; validate-input is read-only and returns // {"valid": }. A well-formed input validates true. - boolean valid = client.actor("apify/hello-world").validateInput(Map.of("firstNumber", 1)); + boolean valid = + client.actor("apify/hello-world").validateInput(Map.of("firstNumber", 1)).join(); assertTrue(valid); } @Test void actorDefaultBuildAndWebhooks() { ApifyClient client = requireClient(); - Actor created = client.actors().create(minimalActor(uniqueName("default-build"))); + Actor created = client.actors().create(minimalActor(uniqueName("default-build"))).join(); try { ActorClient actor = client.actor(created.getId()); - Build build = actor.build("0.0", new ActorBuildOptions()); - client.build(build.getId()).waitForFinish(300L); + Build build = actor.build("0.0", new ActorBuildOptions()).join(); + client.build(build.getId()).waitForFinish(300L).join(); - BuildClient defaultBuild = actor.defaultBuild(60L); - assertTrue(defaultBuild.get().isPresent()); + BuildClient defaultBuild = actor.defaultBuild(60L).join(); + assertTrue(defaultBuild.get().join().isPresent()); // Read-only nested webhook collection (GET + iterate); the account has none registered for // this fresh Actor, so this just exercises both calls succeeding against an empty result — @@ -153,27 +159,27 @@ void actorDefaultBuildAndWebhooks() { // implementation with `AbstractWebhookCollectionClient`, whose paging/iteration behavior is // already exercised with real multi-item data (2 created webhooks, paged one at a time) by // `IterationIntegrationTest#iterateWebhooks` through the sibling `WebhookCollectionClient`. - assertTrue(actor.webhooks().list(new ListOptions()).getTotal() >= 0); - assertTrue(!actor.webhooks().iterate(new ListOptions()).hasNext()); + assertTrue(actor.webhooks().list(new ListOptions()).join().getTotal() >= 0); + assertTrue(Publishers.collect(actor.webhooks().iterate(new ListOptions())).join().isEmpty()); } finally { - client.actor(created.getId()).delete(); + client.actor(created.getId()).delete().join(); } } @Test void actorEnvVarCrudFlow() { ApifyClient client = requireClient(); - Actor created = client.actors().create(minimalActor(uniqueName("env"))); + Actor created = client.actors().create(minimalActor(uniqueName("env"))).join(); try { ActorClient actor = client.actor(created.getId()); var envVars = actor.version("0.0").envVars(); - envVars.create(new ActorEnvVar("MY_VAR", "value1")); - assertTrue(actor.version("0.0").envVar("MY_VAR").get().isPresent()); - envVars.list(); - actor.version("0.0").envVar("MY_VAR").update(new ActorEnvVar("MY_VAR", "value2")); - actor.version("0.0").envVar("MY_VAR").delete(); + envVars.create(new ActorEnvVar("MY_VAR", "value1")).join(); + assertTrue(actor.version("0.0").envVar("MY_VAR").get().join().isPresent()); + envVars.list().join(); + actor.version("0.0").envVar("MY_VAR").update(new ActorEnvVar("MY_VAR", "value2")).join(); + actor.version("0.0").envVar("MY_VAR").delete().join(); } finally { - client.actor(created.getId()).delete(); + client.actor(created.getId()).delete().join(); } } } diff --git a/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java b/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java index db9e6f3..c4cf1a2 100644 --- a/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java @@ -5,6 +5,7 @@ import com.apify.client.ApifyClient; import com.apify.client.ListOptions; +import com.apify.client.Publishers; import com.apify.client.actor.ActorCallOptions; import com.apify.client.actor.ActorClient; import com.apify.client.actor.ActorStartOptions; @@ -41,7 +42,7 @@ class ActorRunIntegrationTest extends IntegrationBase { @Test void listRuns() { ApifyClient client = requireClient(); - var page = client.runs().list(new ListOptions().limit(5L), new RunListOptions()); + var page = client.runs().list(new ListOptions().limit(5L), new RunListOptions()).join(); assertTrue(page.getTotal() >= 0); } @@ -49,35 +50,39 @@ void listRuns() { void runActorAndReadOutputs() { ApifyClient client = requireClient(); ActorRun run = - client.actor("apify/hello-world").call(null, new ActorStartOptions(), TEST_ACTOR_WAIT_SECS); + client + .actor("apify/hello-world") + .call(null, new ActorStartOptions(), TEST_ACTOR_WAIT_SECS) + .join(); assertEquals("SUCCEEDED", run.getStatus()); - assertTrue(client.run(run.getId()).get().isPresent()); + assertTrue(client.run(run.getId()).get().join().isPresent()); - Optional log = client.run(run.getId()).log().get(); + Optional log = client.run(run.getId()).log().get().join(); assertTrue(log.isPresent() && !log.get().isEmpty()); - client.run(run.getId()).dataset().listItems(new DatasetListItemsOptions()); - client.run(run.getId()).keyValueStore().getRecord("OUTPUT"); - client.run(run.getId()).requestQueue().listHead(null); + client.run(run.getId()).dataset().listItems(new DatasetListItemsOptions()).join(); + client.run(run.getId()).keyValueStore().getRecord("OUTPUT").join(); + client.run(run.getId()).requestQueue().listHead(null).join(); // Nested run-storage metadata GETs (previously untested): confirm the run's default storages // are reachable as full metadata resources, not just via the item/record/head-level calls // above. - assertTrue(client.run(run.getId()).dataset().get().isPresent()); - assertTrue(client.run(run.getId()).keyValueStore().get().isPresent()); - assertTrue(client.run(run.getId()).requestQueue().get().isPresent()); + assertTrue(client.run(run.getId()).dataset().get().join().isPresent()); + assertTrue(client.run(run.getId()).keyValueStore().get().join().isPresent()); + assertTrue(client.run(run.getId()).requestQueue().get().join().isPresent()); // A few more run-scoped GETs not otherwise exercised on this particular (run-nested) storage // path; the same operations are already covered end-to-end against standalone storages // elsewhere (DatasetIntegrationTest/KeyValueStoreIntegrationTest/RequestQueueIntegrationTest), // this only confirms they also work when reached through `run().()`. - client.run(run.getId()).dataset().getStatistics(); - client.run(run.getId()).keyValueStore().listKeys(new ListKeysOptions()); - client.run(run.getId()).requestQueue().listRequests(new ListRequestsOptions()); + client.run(run.getId()).dataset().getStatistics().join(); + client.run(run.getId()).keyValueStore().listKeys(new ListKeysOptions()).join(); + client.run(run.getId()).requestQueue().listRequests(new ListRequestsOptions()).join(); // hello-world's default request queue is empty, so a made-up id is expected to resolve to // nothing; this still exercises the live GET-by-id code path through the run-nested client. - assertTrue(client.run(run.getId()).requestQueue().getRequest("does-not-exist").isEmpty()); + assertTrue( + client.run(run.getId()).requestQueue().getRequest("does-not-exist").join().isEmpty()); // Typed getters (previously only reachable via getExtra()): verify the API's response // actually deserializes into them, not just that the code compiles. @@ -102,7 +107,7 @@ void runActorAndReadOutputs() { void actorRunsNestedCollection() { ApifyClient client = requireClient(); ActorClient actor = client.actor("apify/hello-world"); - ActorRun run = actor.call(null, new ActorStartOptions(), TEST_ACTOR_WAIT_SECS); + ActorRun run = actor.call(null, new ActorStartOptions(), TEST_ACTOR_WAIT_SECS).join(); // Runs are sorted ascending by startedAt by default, and "apify/hello-world" is a heavily // used public Actor, so the just-started run would never surface within a small default-order @@ -115,7 +120,10 @@ void actorRunsNestedCollection() { RUN_LIST_FIND_BACKOFF_MILLIS, () -> { var page = - actor.runs().list(new ListOptions().limit(5L).desc(true), new RunListOptions()); + actor + .runs() + .list(new ListOptions().limit(5L).desc(true), new RunListOptions()) + .join(); assertTrue(page.getTotal() >= 0); return page.getItems().stream().anyMatch(r -> run.getId().equals(r.getId())); }); @@ -123,7 +131,7 @@ void actorRunsNestedCollection() { var iterated = actor.runs().iterate(new ListOptions().limit(5L).desc(true), new RunListOptions()); - assertTrue(iterated.hasNext()); + assertTrue(!Publishers.collect(iterated).join().isEmpty()); } @Test @@ -136,10 +144,12 @@ void callWithActorCallOptionsStreamsLogByDefault() { // the wait without any explicit opt-in. runActorAndReadOutputs above exercises the plain // (non-streaming) ActorStartOptions overload. ActorRun run = - actor.call( - null, - new ActorCallOptions().logOptions(new StreamedLogOptions().toLog(collected::add)), - TEST_ACTOR_WAIT_SECS); + actor + .call( + null, + new ActorCallOptions().logOptions(new StreamedLogOptions().toLog(collected::add)), + TEST_ACTOR_WAIT_SECS) + .join(); assertEquals("SUCCEEDED", run.getStatus()); // call()'s internal log-streaming lifecycle closes the stream as soon as the run is finished // (RunStartSupport#callWithLogStreaming), which can race the background reader the same way @@ -156,7 +166,7 @@ void callWithActorCallOptionsCanDisableLogStreaming() { ApifyClient client = requireClient(); ActorClient actor = client.actor("apify/hello-world"); ActorRun run = - actor.call(null, new ActorCallOptions().disableLogStreaming(), TEST_ACTOR_WAIT_SECS); + actor.call(null, new ActorCallOptions().disableLogStreaming(), TEST_ACTOR_WAIT_SECS).join(); assertEquals("SUCCEEDED", run.getStatus()); } @@ -167,34 +177,37 @@ void callWithActorCallOptionsCanDisableLogStreaming() { @Test void runAbortUpdateResurrectDelete() { ApifyClient client = requireClient(); - ActorRun run = client.actor("apify/hello-world").start(null, new ActorStartOptions()); + ActorRun run = client.actor("apify/hello-world").start(null, new ActorStartOptions()).join(); RunClient runClient = client.run(run.getId()); - ActorRun aborted = runClient.abort(false); + ActorRun aborted = runClient.abort(false).join(); assertTrue(!"READY".equals(aborted.getStatus()), aborted.getStatus()); - ActorRun finished = runClient.waitForFinish(60L); + ActorRun finished = runClient.waitForFinish(60L).join(); assertTrue( finished.isTerminal(), "run did not reach a terminal state: " + finished.getStatus()); - ActorRun updated = runClient.update(Map.of("statusMessage", "integration-test-update")); + ActorRun updated = runClient.update(Map.of("statusMessage", "integration-test-update")).join(); assertEquals("integration-test-update", updated.getStatusMessage()); - ActorRun resurrected = runClient.resurrect(new RunResurrectOptions()); + ActorRun resurrected = runClient.resurrect(new RunResurrectOptions()).join(); assertTrue( !resurrected.isTerminal(), "resurrected run should not already be terminal: " + resurrected.getStatus()); // Clean up the resurrected run so it doesn't linger on the shared test account. RunClient resurrectedClient = client.run(resurrected.getId()); - resurrectedClient.abort(false); - resurrectedClient.delete(); + resurrectedClient.abort(false).join(); + resurrectedClient.delete().join(); } @Test void lastRunAccess() { ApifyClient client = requireClient(); ActorRun run = - client.actor("apify/hello-world").call(null, new ActorStartOptions(), TEST_ACTOR_WAIT_SECS); + client + .actor("apify/hello-world") + .call(null, new ActorStartOptions(), TEST_ACTOR_WAIT_SECS) + .join(); assertEquals("SUCCEEDED", run.getStatus()); // `apify/hello-world` is a shared public store Actor: under the account's concurrent-execution @@ -202,7 +215,7 @@ void lastRunAccess() { // running at the same time against the same test user) can legitimately be "last" between the // `call` above and the `lastRun` lookup below. Assert presence/status/origin, not identity // with the run just started. - var lastRun = client.actor("apify/hello-world").lastRun("SUCCEEDED").get(); + var lastRun = client.actor("apify/hello-world").lastRun("SUCCEEDED").get().join(); assertTrue(lastRun.isPresent()); assertEquals("SUCCEEDED", lastRun.get().getStatus()); @@ -210,7 +223,8 @@ void lastRunAccess() { client .actor("apify/hello-world") .lastRun(new LastRunOptions().status("SUCCEEDED").origin("API")) - .get(); + .get() + .join(); assertTrue(byOrigin.isPresent()); assertEquals("SUCCEEDED", byOrigin.get().getStatus()); assertEquals("API", byOrigin.get().getMeta().getOrigin()); @@ -221,22 +235,22 @@ void lastRunAccess() { // the concurrency note above) is a hello-world run, so its default storages are guaranteed to // exist. RunClient lastRunClient = client.actor("apify/hello-world").lastRun("SUCCEEDED"); - lastRunClient.dataset().listItems(new DatasetListItemsOptions()); - lastRunClient.keyValueStore().getRecord("OUTPUT"); - assertTrue(lastRunClient.log().get().isPresent()); + lastRunClient.dataset().listItems(new DatasetListItemsOptions()).join(); + lastRunClient.keyValueStore().getRecord("OUTPUT").join(); + assertTrue(lastRunClient.log().get().join().isPresent()); } @Test void streamedLogRedirection() { ApifyClient client = requireClient(); - ActorRun run = client.actor("apify/hello-world").start(null, new ActorStartOptions()); + ActorRun run = client.actor("apify/hello-world").start(null, new ActorStartOptions()).join(); RunClient runClient = client.run(run.getId()); List collected = new CopyOnWriteArrayList<>(); try (StreamedLog streamedLog = - runClient.getStreamedLog(new StreamedLogOptions().toLog(collected::add))) { - streamedLog.start(); - runClient.waitForFinish(TEST_ACTOR_WAIT_SECS); + runClient.getStreamedLog(new StreamedLogOptions().toLog(collected::add)).join()) { + streamedLog.start().join(); + runClient.waitForFinish(TEST_ACTOR_WAIT_SECS).join(); } // The *first* stream above was opened before the run finished, following the live log as the diff --git a/src/test/java/com/apify/client/integration/BuildIntegrationTest.java b/src/test/java/com/apify/client/integration/BuildIntegrationTest.java index 4a0aa58..88502c9 100644 --- a/src/test/java/com/apify/client/integration/BuildIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/BuildIntegrationTest.java @@ -16,27 +16,28 @@ class BuildIntegrationTest extends IntegrationBase { @Test void listBuilds() { ApifyClient client = requireClient(); - var page = client.builds().list(new ListOptions().limit(5L)); + var page = client.builds().list(new ListOptions().limit(5L)).join(); assertTrue(page.getTotal() >= 0); } @Test void buildActorFlow() { ApifyClient client = requireClient(); - Actor created = client.actors().create(ActorIntegrationTest.minimalActor(uniqueName("build"))); + Actor created = + client.actors().create(ActorIntegrationTest.minimalActor(uniqueName("build"))).join(); try { - Build build = client.actor(created.getId()).build("0.0", new ActorBuildOptions()); - Build finished = client.build(build.getId()).waitForFinish(300L); + Build build = client.actor(created.getId()).build("0.0", new ActorBuildOptions()).join(); + Build finished = client.build(build.getId()).waitForFinish(300L).join(); assertTrue(finished.isTerminal(), "build did not finish: " + finished.getStatus()); - assertTrue(client.build(build.getId()).get().isPresent()); - client.build(build.getId()).log().get(); + assertTrue(client.build(build.getId()).get().join().isPresent()); + client.build(build.getId()).log().get().join(); // Exercise the standalone log endpoint (GET /v2/logs/{buildOrRunId}) directly, not only via // the build-nested .../log accessor, so the "simple GET per endpoint" rule is met for it. - assertNotNull(client.log(build.getId()).get()); - client.build(build.getId()).getOpenApiDefinition(); + assertNotNull(client.log(build.getId()).get().join()); + client.build(build.getId()).getOpenApiDefinition().join(); } finally { - client.actor(created.getId()).delete(); + client.actor(created.getId()).delete().join(); } } @@ -44,16 +45,16 @@ void buildActorFlow() { void buildAbortAndDelete() { ApifyClient client = requireClient(); Actor created = - client.actors().create(ActorIntegrationTest.minimalActor(uniqueName("build-abort"))); + client.actors().create(ActorIntegrationTest.minimalActor(uniqueName("build-abort"))).join(); try { - Build build = client.actor(created.getId()).build("0.0", new ActorBuildOptions()); + Build build = client.actor(created.getId()).build("0.0", new ActorBuildOptions()).join(); BuildClient buildClient = client.build(build.getId()); - Build aborted = buildClient.abort(); + Build aborted = buildClient.abort().join(); assertNotNull(aborted.getStatus()); - buildClient.waitForFinish(60L); - buildClient.delete(); + buildClient.waitForFinish(60L).join(); + buildClient.delete().join(); } finally { - client.actor(created.getId()).delete(); + client.actor(created.getId()).delete().join(); } } } diff --git a/src/test/java/com/apify/client/integration/DatasetIntegrationTest.java b/src/test/java/com/apify/client/integration/DatasetIntegrationTest.java index 45cf5df..2414ffe 100644 --- a/src/test/java/com/apify/client/integration/DatasetIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/DatasetIntegrationTest.java @@ -11,62 +11,66 @@ import com.apify.client.dataset.DatasetDownloadOptions; import com.apify.client.dataset.DatasetListItemsOptions; import com.apify.client.dataset.DownloadItemsFormat; -import com.fasterxml.jackson.databind.JsonNode; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; +import tools.jackson.databind.JsonNode; class DatasetIntegrationTest extends IntegrationBase { @Test void listDatasets() { ApifyClient client = requireClient(); - assertTrue(client.datasets().list(new StorageListOptions().limit(5L)).getTotal() >= 0); + assertTrue(client.datasets().list(new StorageListOptions().limit(5L)).join().getTotal() >= 0); } @Test void getDataset() { ApifyClient client = requireClient(); - Dataset ds = client.datasets().getOrCreate(uniqueName("ds-get")); + Dataset ds = client.datasets().getOrCreate(uniqueName("ds-get")).join(); try { - var got = client.dataset(ds.getId()).get(); + var got = client.dataset(ds.getId()).get().join(); assertTrue(got.isPresent()); assertEquals(ds.getId(), got.get().getId()); } finally { - client.dataset(ds.getId()).delete(); + client.dataset(ds.getId()).delete().join(); } } @Test void datasetCrudFlow() { ApifyClient client = requireClient(); - Dataset ds = client.datasets().getOrCreate(uniqueName("ds-crud")); + Dataset ds = client.datasets().getOrCreate(uniqueName("ds-crud")).join(); try { DatasetClient dataset = client.dataset(ds.getId()); - assertTrue(dataset.get().isPresent()); + assertTrue(dataset.get().join().isPresent()); - dataset.pushItems( - List.of( - Map.of("url", "https://a.com", "n", 1), - Map.of("url", "https://b.com", "n", 2), - Map.of("url", "https://c.com", "n", 3))); + dataset + .pushItems( + List.of( + Map.of("url", "https://a.com", "n", 1), + Map.of("url", "https://b.com", "n", 2), + Map.of("url", "https://c.com", "n", 3))) + .join(); - PaginationList page = dataset.listItems(new DatasetListItemsOptions()); + PaginationList page = dataset.listItems(new DatasetListItemsOptions()).join(); assertEquals(3, page.getCount()); assertEquals(3, page.getItems().size()); assertEquals(1, page.getItems().get(0).get("n").asInt()); byte[] csv = - dataset.downloadItems(DownloadItemsFormat.CSV, new DatasetDownloadOptions().bom(true)); + dataset + .downloadItems(DownloadItemsFormat.CSV, new DatasetDownloadOptions().bom(true)) + .join(); assertTrue(new String(csv, StandardCharsets.UTF_8).contains("url")); - String url = dataset.createItemsPublicUrl(new DatasetListItemsOptions(), null); + String url = dataset.createItemsPublicUrl(new DatasetListItemsOptions(), null).join(); assertTrue(url != null && !url.isEmpty()); - dataset.getStatistics(); + dataset.getStatistics().join(); - Dataset updated = dataset.update(Map.of("name", uniqueName("ds-renamed"))); + Dataset updated = dataset.update(Map.of("name", uniqueName("ds-renamed"))).join(); assertTrue(updated.getName() != null && !updated.getName().isEmpty()); // list() step of the create/get/modify/list/delete flow: verify the just-created dataset @@ -79,12 +83,13 @@ void datasetCrudFlow() { client .datasets() .list(new StorageListOptions().desc(true).limit(10L)) + .join() .getItems() .stream() .anyMatch(d -> ds.getId().equals(d.getId()))); assertTrue(foundInList, "expected the just-created dataset to appear in the top-level list"); } finally { - client.dataset(ds.getId()).delete(); + client.dataset(ds.getId()).delete().join(); } } } diff --git a/src/test/java/com/apify/client/integration/IntegrationBase.java b/src/test/java/com/apify/client/integration/IntegrationBase.java index e9cf237..f2c50af 100644 --- a/src/test/java/com/apify/client/integration/IntegrationBase.java +++ b/src/test/java/com/apify/client/integration/IntegrationBase.java @@ -150,9 +150,10 @@ public static boolean pollUntil(int maxAttempts, long backoffMillis, BooleanSupp static List collectFinishedRunLog(RunClient runClient) { List retryCollected = new CopyOnWriteArrayList<>(); try (StreamedLog retryStream = - runClient.getStreamedLog( - new StreamedLogOptions().toLog(retryCollected::add).fromStart(true))) { - retryStream.start(); + runClient + .getStreamedLog(new StreamedLogOptions().toLog(retryCollected::add).fromStart(true)) + .join()) { + retryStream.start().join(); pollUntil( STREAM_CATCH_UP_ATTEMPTS, STREAM_CATCH_UP_POLL_MILLIS, () -> !retryCollected.isEmpty()); } @@ -187,7 +188,7 @@ static List collectFinishedRunLog(RunClient runClient) { static void assertStreamedLogNonEmptyIfProduced(RunClient runClient, List collected) { pollUntil(STREAM_CATCH_UP_ATTEMPTS, STREAM_CATCH_UP_POLL_MILLIS, () -> !collected.isEmpty()); - Optional authoritativeLog = runClient.log().get(); + Optional authoritativeLog = runClient.log().get().join(); if (runProducedLog(authoritativeLog) && collected.isEmpty()) { collected.addAll(collectFinishedRunLog(runClient)); } @@ -213,7 +214,7 @@ static void assertStreamedLogNonEmptyIfProduced(RunClient runClient, List collected) { pollUntil(STREAM_CATCH_UP_ATTEMPTS, STREAM_CATCH_UP_POLL_MILLIS, () -> !collected.isEmpty()); - assertNonEmptyIfLogProduced(runClient.log().get(), collected); + assertNonEmptyIfLogProduced(runClient.log().get().join(), collected); } /** Whether the authoritative, statically-persisted log shows the run produced any output. */ diff --git a/src/test/java/com/apify/client/integration/IterationIntegrationTest.java b/src/test/java/com/apify/client/integration/IterationIntegrationTest.java index dc9009c..1d472aa 100644 --- a/src/test/java/com/apify/client/integration/IterationIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/IterationIntegrationTest.java @@ -5,6 +5,7 @@ import com.apify.client.ApifyClient; import com.apify.client.ListOptions; +import com.apify.client.Publishers; import com.apify.client.StorageListOptions; import com.apify.client.actor.Actor; import com.apify.client.actor.ActorEnvVar; @@ -25,23 +26,24 @@ import com.apify.client.task.Task; import com.apify.client.webhook.Webhook; import com.apify.client.webhook.WebhookDispatch; -import com.fasterxml.jackson.databind.JsonNode; -import java.util.ArrayList; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.function.Supplier; import org.junit.jupiter.api.Test; +import tools.jackson.databind.JsonNode; /** * Iteration coverage for every paginated collection client that exposes an {@code iterate} helper, * mirroring the reference client's async iterators. Where creation is cheap, tests create a few * uniquely-named resources and assert iteration finds them (so paging is exercised across pages); * for resources whose setup is expensive (builds, runs, dispatches) they iterate a bounded slice - * and assert the iterator behaves. All resources are uniquely named for parallel isolation. + * and assert the publisher behaves. All resources are uniquely named for parallel isolation. */ class IterationIntegrationTest extends IntegrationBase { @@ -55,19 +57,54 @@ class IterationIntegrationTest extends IntegrationBase { /** * Upper bound on items scanned by {@link #findsAll}, purely as a safety net against an infinite - * loop if a collection endpoint's iterator ever misbehaves (e.g. never terminates); no assertion - * in this suite iterates anywhere near this many items. + * scan if a collection endpoint's publisher ever misbehaves (e.g. never completes); no assertion + * in this suite scans anywhere near this many items. Enforced by requesting one item at a time + * and cancelling the subscription once the limit is hit, rather than draining unbounded. */ private static final int FIND_ALL_SAFETY_LIMIT = 10_000; - /** Iterates until every target id is seen (lazily; stops early) or the iterator is exhausted. */ + /** + * Subscribes to {@code publisher} with one-at-a-time demand, removing each seen id from a copy of + * {@code targets}, and stops (cancelling the subscription) as soon as every target is found, the + * publisher completes, or {@link #FIND_ALL_SAFETY_LIMIT} items have been scanned. + */ private static boolean findsAll( - Iterator it, Function idOf, Set targets) { + Flow.Publisher publisher, Function idOf, Set targets) { Set remaining = new HashSet<>(targets); - int safety = 0; - while (it.hasNext() && !remaining.isEmpty() && safety++ < FIND_ALL_SAFETY_LIMIT) { - remaining.remove(idOf.apply(it.next())); - } + CompletableFuture done = new CompletableFuture<>(); + AtomicInteger scanned = new AtomicInteger(); + publisher.subscribe( + new Flow.Subscriber<>() { + private Flow.Subscription subscription; + + @Override + public void onSubscribe(Flow.Subscription subscription) { + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(T item) { + remaining.remove(idOf.apply(item)); + if (remaining.isEmpty() || scanned.incrementAndGet() >= FIND_ALL_SAFETY_LIMIT) { + subscription.cancel(); + done.complete(null); + } else { + subscription.request(1); + } + } + + @Override + public void onError(Throwable throwable) { + done.completeExceptionally(throwable); + } + + @Override + public void onComplete() { + done.complete(null); + } + }); + done.join(); return remaining.isEmpty(); } @@ -77,81 +114,74 @@ private static boolean findsAll( * collection's LIST response — the write and the list index converge asynchronously on the * server, so a create-then-iterate assertion that scans exactly once races that convergence and * flakes when the just-created entity has not yet propagated. Delegates the retry/backoff shape - * to {@link IntegrationBase#pollUntil}, rebuilding a fresh iterator via {@code newIterator} on + * to {@link IntegrationBase#pollUntil}, rebuilding a fresh publisher via {@code newPublisher} on * every attempt (up to {@link #ITER_FIND_ATTEMPTS} times, sleeping {@link * #ITER_FIND_BACKOFF_MILLIS} between attempts) and returning as soon as every target id is seen. * An already-consistent account matches on the first pass with no sleeping. */ private static boolean findsAllEventually( - Supplier> newIterator, Function idOf, Set targets) { + Supplier> newPublisher, Function idOf, Set targets) { return pollUntil( ITER_FIND_ATTEMPTS, ITER_FIND_BACKOFF_MILLIS, - () -> findsAll(newIterator.get(), idOf, targets)); + () -> findsAll(newPublisher.get(), idOf, targets)); } @Test void iterateDatasetItems() { ApifyClient client = requireClient(); - Dataset ds = client.datasets().getOrCreate(uniqueName("it-ds-items")); + Dataset ds = client.datasets().getOrCreate(uniqueName("it-ds-items")).join(); try { DatasetClient dataset = client.dataset(ds.getId()); - List> items = new ArrayList<>(); + List> items = new java.util.ArrayList<>(); for (int i = 1; i <= 5; i++) { items.add(Map.of("n", i)); } - dataset.pushItems(items); + dataset.pushItems(items).join(); - List seen = new ArrayList<>(); - Iterator it = dataset.iterateItems(new DatasetListItemsOptions(), 2L); - while (it.hasNext()) { - seen.add(it.next().get("n").asInt()); - } + List all = + Publishers.collect(dataset.iterateItems(new DatasetListItemsOptions(), 2L)).join(); + List seen = all.stream().map(n -> n.get("n").asInt()).toList(); assertEquals(List.of(1, 2, 3, 4, 5), seen, "iterateItems should yield all items in order"); // The total cap trims the tail: limit=3 over 5 items with page size 2 yields exactly 3. - List capped = new ArrayList<>(); - Iterator cappedIt = - dataset.iterateItems(new DatasetListItemsOptions().limit(3L), 2L); - while (cappedIt.hasNext()) { - capped.add(cappedIt.next().get("n").asInt()); - } + List cappedAll = + Publishers.collect(dataset.iterateItems(new DatasetListItemsOptions().limit(3L), 2L)) + .join(); + List capped = cappedAll.stream().map(n -> n.get("n").asInt()).toList(); assertEquals(List.of(1, 2, 3), capped); } finally { - client.dataset(ds.getId()).delete(); + client.dataset(ds.getId()).delete().join(); } } @Test void iterateKeyValueStoreKeys() { ApifyClient client = requireClient(); - KeyValueStore store = client.keyValueStores().getOrCreate(uniqueName("it-kvs-keys")); + KeyValueStore store = client.keyValueStores().getOrCreate(uniqueName("it-kvs-keys")).join(); try { KeyValueStoreClient kvs = client.keyValueStore(store.getId()); Set keys = new HashSet<>(); for (int i = 0; i < 5; i++) { String key = "key-" + i; - kvs.setRecordJson(key, Map.of("i", i)); + kvs.setRecordJson(key, Map.of("i", i)).join(); keys.add(key); } + List all = + Publishers.collect(kvs.iterateKeys(new ListKeysOptions())).join(); Set seen = new HashSet<>(); - Iterator it = kvs.iterateKeys(new ListKeysOptions()); - while (it.hasNext()) { - seen.add(it.next().getKey()); + for (KeyValueStoreKey k : all) { + seen.add(k.getKey()); } assertTrue(seen.containsAll(keys), "iterateKeys should yield every key; saw " + seen); // The limit caps the total number of keys yielded. - int count = 0; - Iterator capped = kvs.iterateKeys(new ListKeysOptions().limit(3L)); - while (capped.hasNext()) { - capped.next(); - count++; - } - assertEquals(3, count, "limit should cap the keys yielded"); + List capped = + Publishers.collect(kvs.iterateKeys(new ListKeysOptions().limit(3L))).join(); + assertEquals(3, capped.size(), "limit should cap the keys yielded"); } finally { - client.keyValueStore(store.getId()).delete(); + client.keyValueStore(store.getId()).delete().join(); } } @@ -161,9 +191,9 @@ void iterateDatasets() { Set ids = new HashSet<>(); try { for (int i = 0; i < 3; i++) { - ids.add(client.datasets().getOrCreate(uniqueName("it-ds-" + i)).getId()); + ids.add(client.datasets().getOrCreate(uniqueName("it-ds-" + i)).join().getId()); } - // Page size 1 forces the offset iterator across multiple pages; desc puts the fresh ones + // Page size 1 forces the offset publisher across multiple pages; desc puts the fresh ones // first. assertTrue( findsAllEventually( @@ -173,7 +203,7 @@ void iterateDatasets() { "iterate should find every created dataset"); } finally { for (String id : ids) { - client.dataset(id).delete(); + client.dataset(id).delete().join(); } } } @@ -184,7 +214,7 @@ void iterateKeyValueStores() { Set ids = new HashSet<>(); try { for (int i = 0; i < 2; i++) { - ids.add(client.keyValueStores().getOrCreate(uniqueName("it-kvs-" + i)).getId()); + ids.add(client.keyValueStores().getOrCreate(uniqueName("it-kvs-" + i)).join().getId()); } assertTrue( findsAllEventually( @@ -193,7 +223,7 @@ void iterateKeyValueStores() { ids)); } finally { for (String id : ids) { - client.keyValueStore(id).delete(); + client.keyValueStore(id).delete().join(); } } } @@ -204,7 +234,7 @@ void iterateRequestQueues() { Set ids = new HashSet<>(); try { for (int i = 0; i < 2; i++) { - ids.add(client.requestQueues().getOrCreate(uniqueName("it-rq-" + i)).getId()); + ids.add(client.requestQueues().getOrCreate(uniqueName("it-rq-" + i)).join().getId()); } assertTrue( findsAllEventually( @@ -213,7 +243,7 @@ void iterateRequestQueues() { ids)); } finally { for (String id : ids) { - client.requestQueue(id).delete(); + client.requestQueue(id).delete().join(); } } } @@ -225,14 +255,18 @@ void iterateTasks() { try { for (int i = 0; i < 2; i++) { ids.add( - client.tasks().create(TaskIntegrationTest.taskDef(uniqueName("it-task-" + i))).getId()); + client + .tasks() + .create(TaskIntegrationTest.taskDef(uniqueName("it-task-" + i))) + .join() + .getId()); } assertTrue( findsAllEventually( () -> client.tasks().iterate(new ListOptions().desc(true), 1L), Task::getId, ids)); } finally { for (String id : ids) { - client.task(id).delete(); + client.task(id).delete().join(); } } } @@ -247,6 +281,7 @@ void iterateSchedules() { client .schedules() .create(ScheduleIntegrationTest.scheduleDef(uniqueName("it-sch-" + i))) + .join() .getId()); } assertTrue( @@ -256,7 +291,7 @@ void iterateSchedules() { ids)); } finally { for (String id : ids) { - client.schedule(id).delete(); + client.schedule(id).delete().join(); } } } @@ -271,6 +306,7 @@ void iterateWebhooks() { client .webhooks() .create(WebhookIntegrationTest.webhookDef("https://example.com/it-wh-" + i)) + .join() .getId()); } assertTrue( @@ -280,7 +316,7 @@ void iterateWebhooks() { ids)); } finally { for (String id : ids) { - client.webhook(id).delete(); + client.webhook(id).delete().join(); } } } @@ -295,6 +331,7 @@ void iterateActors() { client .actors() .create(ActorIntegrationTest.minimalActor(uniqueName("it-act-" + i))) + .join() .getId()); } // Restrict to the current user's Actors so iteration finds the freshly-created ones quickly. @@ -305,7 +342,7 @@ void iterateActors() { ids)); } finally { for (String id : ids) { - client.actor(id).delete(); + client.actor(id).delete().join(); } } } @@ -313,35 +350,36 @@ void iterateActors() { @Test void iterateActorVersionsAndEnvVars() { ApifyClient client = requireClient(); - Actor actor = client.actors().create(ActorIntegrationTest.minimalActor(uniqueName("it-ver"))); + Actor actor = + client.actors().create(ActorIntegrationTest.minimalActor(uniqueName("it-ver"))).join(); try { var actorClient = client.actor(actor.getId()); // The versions endpoint is not paginated (one fetch returns every version); fully draining - // the iterator must terminate and must not re-yield a version. The minimal Actor ships with + // the publisher must terminate and must not re-yield a version. The minimal Actor ships with // version 0.0, so iteration yields at least one version, each exactly once. + List versions = + Publishers.collect(actorClient.versions().iterate(new ListOptions())).join(); Set versionNumbers = new HashSet<>(); - int versionCount = 0; - Iterator versions = actorClient.versions().iterate(new ListOptions()); - while (versions.hasNext()) { - versionCount++; - versionNumbers.add(versions.next().getVersionNumber()); + for (ActorVersion v : versions) { + versionNumbers.add(v.getVersionNumber()); } - assertTrue(versionCount >= 1, "expected at least the initial version"); - assertEquals(versionCount, versionNumbers.size(), "versions iterator re-yielded a version"); + assertTrue(versions.size() >= 1, "expected at least the initial version"); + assertEquals( + versions.size(), versionNumbers.size(), "versions publisher re-yielded a version"); assertTrue( versionNumbers.contains("0.0"), "expected initial version 0.0, saw " + versionNumbers); var envVars = actorClient.version("0.0").envVars(); - envVars.create(new ActorEnvVar("IT_VAR_A", "a")); - envVars.create(new ActorEnvVar("IT_VAR_B", "b")); + envVars.create(new ActorEnvVar("IT_VAR_A", "a")).join(); + envVars.create(new ActorEnvVar("IT_VAR_B", "b")).join(); + List envVarItems = Publishers.collect(envVars.iterate()).join(); Set seen = new HashSet<>(); - Iterator it = envVars.iterate(); - while (it.hasNext()) { - seen.add(it.next().getName()); + for (ActorEnvVar v : envVarItems) { + seen.add(v.getName()); } assertTrue(seen.contains("IT_VAR_A") && seen.contains("IT_VAR_B"), "saw " + seen); } finally { - client.actor(actor.getId()).delete(); + client.actor(actor.getId()).delete().join(); } } @@ -349,38 +387,34 @@ void iterateActorVersionsAndEnvVars() { void iterateBuildsBounded() { ApifyClient client = requireClient(); // Builds require building an Actor (expensive); assert a bounded slice iterates cleanly. - Iterator it = client.builds().iterate(new ListOptions().limit(5L), 2L); - int count = 0; - while (it.hasNext()) { - assertTrue(it.next().getId() != null); - count++; + List builds = + Publishers.collect(client.builds().iterate(new ListOptions().limit(5L), 2L)).join(); + for (Build build : builds) { + assertTrue(build.getId() != null); } - assertTrue(count <= 5, "the total-cap limit must bound iteration; got " + count); + assertTrue( + builds.size() <= 5, "the total-cap limit must bound iteration; got " + builds.size()); } @Test void iterateRunsBounded() { ApifyClient client = requireClient(); - Iterator it = - client.runs().iterate(new ListOptions().limit(5L), new RunListOptions(), 2L); - int count = 0; - while (it.hasNext()) { - assertTrue(it.next().getId() != null); - count++; + List runs = + Publishers.collect( + client.runs().iterate(new ListOptions().limit(5L), new RunListOptions(), 2L)) + .join(); + for (ActorRun run : runs) { + assertTrue(run.getId() != null); } - assertTrue(count <= 5); + assertTrue(runs.size() <= 5); } @Test void iterateWebhookDispatchesBounded() { ApifyClient client = requireClient(); - Iterator it = - client.webhookDispatches().iterate(new ListOptions().limit(5L), 2L); - int count = 0; - while (it.hasNext()) { - it.next(); - count++; - } - assertTrue(count <= 5); + List dispatches = + Publishers.collect(client.webhookDispatches().iterate(new ListOptions().limit(5L), 2L)) + .join(); + assertTrue(dispatches.size() <= 5); } } diff --git a/src/test/java/com/apify/client/integration/KeyValueStoreIntegrationTest.java b/src/test/java/com/apify/client/integration/KeyValueStoreIntegrationTest.java index cf37a31..b4417b4 100644 --- a/src/test/java/com/apify/client/integration/KeyValueStoreIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/KeyValueStoreIntegrationTest.java @@ -25,57 +25,58 @@ class KeyValueStoreIntegrationTest extends IntegrationBase { @Test void listKeyValueStores() { ApifyClient client = requireClient(); - assertTrue(client.keyValueStores().list(new StorageListOptions().limit(5L)).getTotal() >= 0); + assertTrue( + client.keyValueStores().list(new StorageListOptions().limit(5L)).join().getTotal() >= 0); } @Test void getKeyValueStore() { ApifyClient client = requireClient(); - KeyValueStore store = client.keyValueStores().getOrCreate(uniqueName("kvs-get")); + KeyValueStore store = client.keyValueStores().getOrCreate(uniqueName("kvs-get")).join(); try { - var got = client.keyValueStore(store.getId()).get(); + var got = client.keyValueStore(store.getId()).get().join(); assertTrue(got.isPresent()); assertEquals(store.getId(), got.get().getId()); } finally { - client.keyValueStore(store.getId()).delete(); + client.keyValueStore(store.getId()).delete().join(); } } @Test void recordKeyWithSpecialChars() { ApifyClient client = requireClient(); - KeyValueStore store = client.keyValueStores().getOrCreate(uniqueName("kvs-special")); + KeyValueStore store = client.keyValueStores().getOrCreate(uniqueName("kvs-special")).join(); try { KeyValueStoreClient kvs = client.keyValueStore(store.getId()); String key = "weird-key!'()"; - kvs.setRecordJson(key, Map.of("ok", true)); - assertTrue(kvs.recordExists(key)); - Optional rec = kvs.getRecord(key); + kvs.setRecordJson(key, Map.of("ok", true)).join(); + assertTrue(kvs.recordExists(key).join()); + Optional rec = kvs.getRecord(key).join(); assertTrue(rec.isPresent()); - kvs.deleteRecord(key); + kvs.deleteRecord(key).join(); } finally { - client.keyValueStore(store.getId()).delete(); + client.keyValueStore(store.getId()).delete().join(); } } @Test void keyValueStoreCrudFlow() throws Exception { ApifyClient client = requireClient(); - KeyValueStore store = client.keyValueStores().getOrCreate(uniqueName("kvs-crud")); + KeyValueStore store = client.keyValueStores().getOrCreate(uniqueName("kvs-crud")).join(); try { KeyValueStoreClient kvs = client.keyValueStore(store.getId()); - assertTrue(kvs.get().isPresent()); - kvs.setRecordJson("OUTPUT", Map.of("hello", "world")); - assertTrue(kvs.recordExists("OUTPUT")); - Optional rec = kvs.getRecord("OUTPUT"); + assertTrue(kvs.get().join().isPresent()); + kvs.setRecordJson("OUTPUT", Map.of("hello", "world")).join(); + assertTrue(kvs.recordExists("OUTPUT").join()); + Optional rec = kvs.getRecord("OUTPUT").join(); assertTrue(rec.isPresent()); String value = new String(rec.get().getValue(), StandardCharsets.UTF_8); assertTrue(value.contains("world"), value); - kvs.getRecord("OUTPUT", new GetRecordOptions().attachment(false)); - var keys = kvs.listKeys(new ListKeysOptions()); + kvs.getRecord("OUTPUT", new GetRecordOptions().attachment(false)).join(); + var keys = kvs.listKeys(new ListKeysOptions()).join(); assertTrue(!keys.getItems().isEmpty()); - kvs.update(Map.of("name", uniqueName("kvs-renamed"))); - kvs.deleteRecord("OUTPUT"); + kvs.update(Map.of("name", uniqueName("kvs-renamed"))).join(); + kvs.deleteRecord("OUTPUT").join(); // list() step of the create/get/modify/list/delete flow: verify the just-created key-value // store appears in the top-level collection listing. @@ -87,24 +88,25 @@ void keyValueStoreCrudFlow() throws Exception { client .keyValueStores() .list(new StorageListOptions().desc(true).limit(10L)) + .join() .getItems() .stream() .anyMatch(s -> store.getId().equals(s.getId()))); assertTrue( foundInList, "expected the just-created key-value store to appear in the top-level list"); } finally { - client.keyValueStore(store.getId()).delete(); + client.keyValueStore(store.getId()).delete().join(); } } @Test void recordPublicUrlIsFetchable() throws IOException, InterruptedException { ApifyClient client = requireClient(); - KeyValueStore store = client.keyValueStores().getOrCreate(uniqueName("kvs-pub")); + KeyValueStore store = client.keyValueStores().getOrCreate(uniqueName("kvs-pub")).join(); try { KeyValueStoreClient kvs = client.keyValueStore(store.getId()); - kvs.setRecordJson("OUTPUT", Map.of("pub", true)); - String url = kvs.getRecordPublicUrl("OUTPUT"); + kvs.setRecordJson("OUTPUT", Map.of("pub", true)).join(); + String url = kvs.getRecordPublicUrl("OUTPUT").join(); assertTrue(url != null && !url.isEmpty()); HttpResponse resp = @@ -116,7 +118,7 @@ void recordPublicUrlIsFetchable() throws IOException, InterruptedException { resp.statusCode() < 300, "expected success fetching public url, got " + resp.statusCode()); } finally { - client.keyValueStore(store.getId()).delete(); + client.keyValueStore(store.getId()).delete().join(); } } } diff --git a/src/test/java/com/apify/client/integration/RequestQueueIntegrationTest.java b/src/test/java/com/apify/client/integration/RequestQueueIntegrationTest.java index c4b85a6..e161edc 100644 --- a/src/test/java/com/apify/client/integration/RequestQueueIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/RequestQueueIntegrationTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.apify.client.ApifyClient; +import com.apify.client.Publishers; import com.apify.client.StorageListOptions; import com.apify.client.requestqueue.BatchAddResult; import com.apify.client.requestqueue.BatchDeleteResult; @@ -18,7 +19,6 @@ import com.apify.client.requestqueue.UnlockRequestsResult; import java.util.ArrayList; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -29,42 +29,45 @@ class RequestQueueIntegrationTest extends IntegrationBase { @Test void listRequestQueues() { ApifyClient client = requireClient(); - assertTrue(client.requestQueues().list(new StorageListOptions().limit(5L)).getTotal() >= 0); + assertTrue( + client.requestQueues().list(new StorageListOptions().limit(5L)).join().getTotal() >= 0); } @Test void getRequestQueue() { ApifyClient client = requireClient(); - RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-get")); + RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-get")).join(); try { - var got = client.requestQueue(rq.getId()).get(); + var got = client.requestQueue(rq.getId()).get().join(); assertTrue(got.isPresent()); assertEquals(rq.getId(), got.get().getId()); } finally { - client.requestQueue(rq.getId()).delete(); + client.requestQueue(rq.getId()).delete().join(); } } @Test void requestQueueCrudFlow() { ApifyClient client = requireClient(); - RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-crud")); + RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-crud")).join(); try { RequestQueueClient queue = client.requestQueue(rq.getId()); - assertTrue(queue.get().isPresent()); + assertTrue(queue.get().join().isPresent()); RequestQueueOperationInfo info = - queue.addRequest( - new RequestQueueRequest("https://example.com", "example").setMethod("GET"), false); + queue + .addRequest( + new RequestQueueRequest("https://example.com", "example").setMethod("GET"), false) + .join(); assertTrue(info.getRequestId() != null && !info.getRequestId().isEmpty()); - var got = queue.getRequest(info.getRequestId()); + var got = queue.getRequest(info.getRequestId()).join(); assertTrue(got.isPresent()); assertEquals("https://example.com", got.get().getUrl()); - assertTrue(!queue.listHead(10L).getItems().isEmpty()); - queue.update(Map.of("name", uniqueName("rq-renamed"))); - queue.deleteRequest(info.getRequestId()); + assertTrue(!queue.listHead(10L).join().getItems().isEmpty()); + queue.update(Map.of("name", uniqueName("rq-renamed"))).join(); + queue.deleteRequest(info.getRequestId()).join(); // list() step of the create/get/modify/list/delete flow: verify the just-created request // queue appears in the top-level collection listing. @@ -76,66 +79,63 @@ void requestQueueCrudFlow() { client .requestQueues() .list(new StorageListOptions().desc(true).limit(10L)) + .join() .getItems() .stream() .anyMatch(q -> rq.getId().equals(q.getId()))); assertTrue( foundInList, "expected the just-created request queue to appear in the top-level list"); } finally { - client.requestQueue(rq.getId()).delete(); + client.requestQueue(rq.getId()).delete().join(); } } @Test void requestQueuePaginateMultiplePages() { ApifyClient client = requireClient(); - RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-page")); + RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-page")).join(); try { RequestQueueClient queue = client.requestQueue(rq.getId()); int total = 5; for (int i = 0; i < total; i++) { String url = "https://example.com/" + i; - queue.addRequest(new RequestQueueRequest(url, url), false); + queue.addRequest(new RequestQueueRequest(url, url), false).join(); } Set seen = new HashSet<>(); - Iterator it = queue.paginateRequests(2L); - while (it.hasNext()) { - seen.add(it.next().getUrl()); + List items = Publishers.collect(queue.paginateRequests(2L)).join(); + for (RequestQueueRequest r : items) { + seen.add(r.getUrl()); } assertEquals(total, seen.size()); } finally { - client.requestQueue(rq.getId()).delete(); + client.requestQueue(rq.getId()).delete().join(); } } @Test void requestQueuePaginateWithTotalLimit() { ApifyClient client = requireClient(); - RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-page-limit")); + RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-page-limit")).join(); try { RequestQueueClient queue = client.requestQueue(rq.getId()); for (int i = 0; i < 5; i++) { String url = "https://example.com/" + i; - queue.addRequest(new RequestQueueRequest(url, url), false); + queue.addRequest(new RequestQueueRequest(url, url), false).join(); } // totalLimit caps the number yielded across all pages, independent of the per-page chunk // size (chunkSize=2 forces at least two page fetches to satisfy a totalLimit of 3). - Iterator it = queue.paginateRequests(3L, 2L, null); - int count = 0; - while (it.hasNext()) { - it.next(); - count++; - } - assertEquals(3, count); + List items = + Publishers.collect(queue.paginateRequests(3L, 2L, null)).join(); + assertEquals(3, items.size()); } finally { - client.requestQueue(rq.getId()).delete(); + client.requestQueue(rq.getId()).delete().join(); } } @Test void requestQueueBatchAddRequests() { ApifyClient client = requireClient(); - RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-batch")); + RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-batch")).join(); try { RequestQueueClient queue = client.requestQueue(rq.getId()); int total = 30; // > 25, so the client must split into multiple chunks @@ -144,67 +144,78 @@ void requestQueueBatchAddRequests() { String url = "https://batch.example.com/" + i; requests.add(new RequestQueueRequest(url, url)); } - BatchAddResult result = queue.batchAddRequests(requests, false); + BatchAddResult result = queue.batchAddRequests(requests, false).join(); assertEquals(total, result.getProcessedRequests().size()); } finally { - client.requestQueue(rq.getId()).delete(); + client.requestQueue(rq.getId()).delete().join(); } } @Test void requestQueueLockLifecycle() { ApifyClient client = requireClient(); - RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-lock")); + RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-lock")).join(); try { RequestQueueClient queue = client.requestQueue(rq.getId()).withClientKey("java-test-client-key"); RequestQueueOperationInfo info = - queue.addRequest(new RequestQueueRequest("https://lock.example.com", "lock"), false); + queue + .addRequest(new RequestQueueRequest("https://lock.example.com", "lock"), false) + .join(); - RequestsList listed = queue.listRequests(new ListRequestsOptions()); + RequestsList listed = queue.listRequests(new ListRequestsOptions()).join(); assertTrue(listed.getLimit() > 0); assertTrue(!listed.getItems().isEmpty()); - queue.listRequests( - new ListRequestsOptions() - .filter( - List.of(ListRequestsOptions.FILTER_LOCKED, ListRequestsOptions.FILTER_PENDING))); - - LockedRequestQueueHead locked = queue.listAndLockHead(60, 10L); + queue + .listRequests( + new ListRequestsOptions() + .filter( + List.of( + ListRequestsOptions.FILTER_LOCKED, ListRequestsOptions.FILTER_PENDING))) + .join(); + + LockedRequestQueueHead locked = queue.listAndLockHead(60, 10L).join(); assertEquals(60, locked.getLockSecs()); assertTrue(!locked.getItems().isEmpty()); assertTrue(locked.getItems().get(0).getLockExpiresAt() != null); - RequestLockInfo prolonged = queue.prolongRequestLock(info.getRequestId(), 30, false); + RequestLockInfo prolonged = queue.prolongRequestLock(info.getRequestId(), 30, false).join(); assertTrue(prolonged.getLockExpiresAt() != null); - queue.deleteRequestLock(info.getRequestId(), false); - UnlockRequestsResult unlocked = queue.unlockRequests(); + queue.deleteRequestLock(info.getRequestId(), false).join(); + UnlockRequestsResult unlocked = queue.unlockRequests().join(); assertTrue(unlocked.getUnlockedCount() >= 0); } finally { - client.requestQueue(rq.getId()).delete(); + client.requestQueue(rq.getId()).delete().join(); } } @Test void requestQueueBatchDeleteRequests() { ApifyClient client = requireClient(); - RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-batch-delete")); + RequestQueue rq = client.requestQueues().getOrCreate(uniqueName("rq-batch-delete")).join(); try { RequestQueueClient queue = client.requestQueue(rq.getId()); RequestQueueOperationInfo first = - queue.addRequest( - new RequestQueueRequest("https://batch-delete.example.com/1", "bd1"), false); + queue + .addRequest( + new RequestQueueRequest("https://batch-delete.example.com/1", "bd1"), false) + .join(); RequestQueueOperationInfo second = - queue.addRequest( - new RequestQueueRequest("https://batch-delete.example.com/2", "bd2"), false); + queue + .addRequest( + new RequestQueueRequest("https://batch-delete.example.com/2", "bd2"), false) + .join(); BatchDeleteResult result = - queue.batchDeleteRequests( - List.of(Map.of("id", first.getRequestId()), Map.of("uniqueKey", "bd2"))); + queue + .batchDeleteRequests( + List.of(Map.of("id", first.getRequestId()), Map.of("uniqueKey", "bd2"))) + .join(); assertEquals(2, result.getProcessedRequests().size()); assertTrue(result.getUnprocessedRequests().isEmpty()); } finally { - client.requestQueue(rq.getId()).delete(); + client.requestQueue(rq.getId()).delete().join(); } } } diff --git a/src/test/java/com/apify/client/integration/ScheduleIntegrationTest.java b/src/test/java/com/apify/client/integration/ScheduleIntegrationTest.java index 46a8d5b..90d00df 100644 --- a/src/test/java/com/apify/client/integration/ScheduleIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/ScheduleIntegrationTest.java @@ -30,47 +30,47 @@ static Map scheduleDef(String name) { @Test void listSchedules() { ApifyClient client = requireClient(); - assertTrue(client.schedules().list(new ListOptions().limit(5L)).getTotal() >= 0); + assertTrue(client.schedules().list(new ListOptions().limit(5L)).join().getTotal() >= 0); } @Test void getSchedule() { ApifyClient client = requireClient(); - Schedule sch = client.schedules().create(scheduleDef(uniqueName("sch-get"))); + Schedule sch = client.schedules().create(scheduleDef(uniqueName("sch-get"))).join(); try { - var got = client.schedule(sch.getId()).get(); + var got = client.schedule(sch.getId()).get().join(); assertTrue(got.isPresent()); assertEquals(sch.getId(), got.get().getId()); } finally { - client.schedule(sch.getId()).delete(); + client.schedule(sch.getId()).delete().join(); } } @Test void getScheduleLog() { ApifyClient client = requireClient(); - Schedule sch = client.schedules().create(scheduleDef(uniqueName("sch-log"))); + Schedule sch = client.schedules().create(scheduleDef(uniqueName("sch-log"))).join(); try { // Simple GET on the schedule-log endpoint; a fresh schedule may have no log yet (empty // Optional), which is a valid result — we only assert the call itself succeeds. - client.schedule(sch.getId()).getLog(); + client.schedule(sch.getId()).getLog().join(); } finally { - client.schedule(sch.getId()).delete(); + client.schedule(sch.getId()).delete().join(); } } @Test void scheduleCrudFlow() { ApifyClient client = requireClient(); - Schedule sch = client.schedules().create(scheduleDef(uniqueName("sch-crud"))); + Schedule sch = client.schedules().create(scheduleDef(uniqueName("sch-crud"))).join(); try { ScheduleClient schedule = client.schedule(sch.getId()); - assertTrue(schedule.get().isPresent()); - Schedule updated = schedule.update(Map.of("cronExpression", "0 12 * * *")); + assertTrue(schedule.get().join().isPresent()); + Schedule updated = schedule.update(Map.of("cronExpression", "0 12 * * *")).join(); assertEquals("0 12 * * *", updated.getCronExpression()); - schedule.getLog(); + schedule.getLog().join(); // list() step of the create/get/modify/list/delete flow. - assertTrue(client.schedules().list(new ListOptions().limit(5L)).getTotal() >= 0); + assertTrue(client.schedules().list(new ListOptions().limit(5L)).join().getTotal() >= 0); // Typed getters (previously only reachable via getExtra()): verify the API's response // actually deserializes into them, not just that the code compiles. @@ -81,7 +81,7 @@ void scheduleCrudFlow() { assertTrue(sch.getNotifications() != null); assertTrue(sch.getActions() != null && sch.getActions().isEmpty()); } finally { - client.schedule(sch.getId()).delete(); + client.schedule(sch.getId()).delete().join(); } } } diff --git a/src/test/java/com/apify/client/integration/StoreIntegrationTest.java b/src/test/java/com/apify/client/integration/StoreIntegrationTest.java index ea0863a..9b2f7d2 100644 --- a/src/test/java/com/apify/client/integration/StoreIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/StoreIntegrationTest.java @@ -5,7 +5,9 @@ import com.apify.client.ApifyClient; import com.apify.client.store.ActorStoreListItem; import com.apify.client.store.StoreListOptions; -import java.util.Iterator; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Flow; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; class StoreIntegrationTest extends IntegrationBase { @@ -13,20 +15,53 @@ class StoreIntegrationTest extends IntegrationBase { @Test void listStore() { ApifyClient client = requireClient(); - var page = client.store().list(new StoreListOptions().limit(5L)); + var page = client.store().list(new StoreListOptions().limit(5L)).join(); assertTrue(page.getItems().size() <= 5); } @Test void iterateStore() { ApifyClient client = requireClient(); - Iterator it = client.store().iterate(new StoreListOptions().limit(20L), 5L); - int count = 0; - while (count < 12 && it.hasNext()) { - ActorStoreListItem item = it.next(); - assertTrue(item.getId() != null && !item.getId().isEmpty()); - count++; - } - assertTrue(count >= 12, "expected to iterate at least 12 store actors, got " + count); + // Requests items one at a time (real backpressure) and stops once at least 12 have been seen, + // without ever asking the publisher for more than that - unlike Publishers.collect(...), which + // would drain the whole (large, shared) Store collection. + AtomicInteger count = new AtomicInteger(); + Flow.Publisher publisher = + client.store().iterate(new StoreListOptions().limit(20L), 5L); + CompletableFuture done = new CompletableFuture<>(); + publisher.subscribe( + new Flow.Subscriber<>() { + private Flow.Subscription subscription; + + @Override + public void onSubscribe(Flow.Subscription subscription) { + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(ActorStoreListItem item) { + assertTrue(item.getId() != null && !item.getId().isEmpty()); + if (count.incrementAndGet() >= 12) { + subscription.cancel(); + done.complete(null); + } else { + subscription.request(1); + } + } + + @Override + public void onError(Throwable throwable) { + done.completeExceptionally(throwable); + } + + @Override + public void onComplete() { + done.complete(null); + } + }); + done.join(); + assertTrue( + count.get() >= 12, "expected to iterate at least 12 store actors, got " + count.get()); } } diff --git a/src/test/java/com/apify/client/integration/TaskIntegrationTest.java b/src/test/java/com/apify/client/integration/TaskIntegrationTest.java index 3340f2b..b47c2fa 100644 --- a/src/test/java/com/apify/client/integration/TaskIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/TaskIntegrationTest.java @@ -5,6 +5,7 @@ import com.apify.client.ApifyClient; import com.apify.client.ListOptions; +import com.apify.client.Publishers; import com.apify.client.dataset.DatasetListItemsOptions; import com.apify.client.log.StreamedLogOptions; import com.apify.client.run.ActorRun; @@ -37,33 +38,33 @@ static Map taskDef(String name) { @Test void listTasks() { ApifyClient client = requireClient(); - assertTrue(client.tasks().list(new ListOptions().limit(5L)).getTotal() >= 0); + assertTrue(client.tasks().list(new ListOptions().limit(5L)).join().getTotal() >= 0); } @Test void getTask() { ApifyClient client = requireClient(); - Task task = client.tasks().create(taskDef(uniqueName("task-get"))); + Task task = client.tasks().create(taskDef(uniqueName("task-get"))).join(); try { - var got = client.task(task.getId()).get(); + var got = client.task(task.getId()).get().join(); assertTrue(got.isPresent()); assertEquals(task.getId(), got.get().getId()); } finally { - client.task(task.getId()).delete(); + client.task(task.getId()).delete().join(); } } @Test void taskCrudFlow() { ApifyClient client = requireClient(); - Task task = client.tasks().create(taskDef(uniqueName("task-crud"))); + Task task = client.tasks().create(taskDef(uniqueName("task-crud"))).join(); try { TaskClient tc = client.task(task.getId()); - assertTrue(tc.get().isPresent()); - tc.updateInput(Map.of("message", "updated")); - assertTrue(tc.getInput().isPresent()); - tc.update(Map.of("name", uniqueName("task-renamed"))); - tc.runs().list(new ListOptions(), new RunListOptions()); + assertTrue(tc.get().join().isPresent()); + tc.updateInput(Map.of("message", "updated")).join(); + assertTrue(tc.getInput().join().isPresent()); + tc.update(Map.of("name", uniqueName("task-renamed"))).join(); + tc.runs().list(new ListOptions(), new RunListOptions()).join(); // list() step of the create/get/modify/list/delete flow: verify the just-created task // appears in the top-level collection listing. @@ -72,7 +73,12 @@ void taskCrudFlow() { LIST_FIND_ATTEMPTS, LIST_FIND_BACKOFF_MILLIS, () -> - client.tasks().list(new ListOptions().desc(true).limit(10L)).getItems().stream() + client + .tasks() + .list(new ListOptions().desc(true).limit(10L)) + .join() + .getItems() + .stream() .anyMatch(t -> task.getId().equals(t.getId()))); assertTrue(foundInList, "expected the just-created task to appear in the top-level list"); @@ -83,46 +89,46 @@ void taskCrudFlow() { assertTrue(task.getOptions() != null && "latest".equals(task.getOptions().getBuild())); assertTrue(task.getInput() != null); } finally { - client.task(task.getId()).delete(); + client.task(task.getId()).delete().join(); } } @Test void taskLastRunAndWebhooks() { ApifyClient client = requireClient(); - Task task = client.tasks().create(taskDef(uniqueName("task-lastrun"))); + Task task = client.tasks().create(taskDef(uniqueName("task-lastrun"))).join(); try { TaskClient tc = client.task(task.getId()); - ActorRun run = tc.call(null, new TaskStartOptions(), TEST_ACTOR_WAIT_SECS); + ActorRun run = tc.call(null, new TaskStartOptions(), TEST_ACTOR_WAIT_SECS).join(); assertEquals("SUCCEEDED", run.getStatus()); assertTrue(run.getStats() != null); assertTrue(run.getOptions() != null); assertTrue(run.getMeta() != null && run.getMeta().getOrigin() != null); - var lastRun = tc.lastRun("SUCCEEDED").get(); + var lastRun = tc.lastRun("SUCCEEDED").get().join(); assertTrue(lastRun.isPresent()); assertEquals(run.getId(), lastRun.get().getId()); // Last-run-scoped nested storage GETs (previously untested — only `lastRun().get()` itself // was). This task is exclusively owned by this test, so `lastRun` is deterministically the // run started above and its default storages are guaranteed to exist. - tc.lastRun("SUCCEEDED").dataset().listItems(new DatasetListItemsOptions()); - tc.lastRun("SUCCEEDED").keyValueStore().getRecord("OUTPUT"); - assertTrue(tc.lastRun("SUCCEEDED").log().get().isPresent()); + tc.lastRun("SUCCEEDED").dataset().listItems(new DatasetListItemsOptions()).join(); + tc.lastRun("SUCCEEDED").keyValueStore().getRecord("OUTPUT").join(); + assertTrue(tc.lastRun("SUCCEEDED").log().get().join().isPresent()); // Read-only nested webhook collection (GET + iterate); no webhooks are registered for this // fresh task, so this just exercises both calls succeeding against an empty result. - assertTrue(tc.webhooks().list(new ListOptions()).getTotal() >= 0); - assertTrue(!tc.webhooks().iterate(new ListOptions()).hasNext()); + assertTrue(tc.webhooks().list(new ListOptions()).join().getTotal() >= 0); + assertTrue(Publishers.collect(tc.webhooks().iterate(new ListOptions())).join().isEmpty()); } finally { - client.task(task.getId()).delete(); + client.task(task.getId()).delete().join(); } } @Test void taskCallStreamsLogByDefault() { ApifyClient client = requireClient(); - Task task = client.tasks().create(taskDef(uniqueName("task-call-log"))); + Task task = client.tasks().create(taskDef(uniqueName("task-call-log"))).join(); try { TaskClient tc = client.task(task.getId()); List collected = new CopyOnWriteArrayList<>(); @@ -131,9 +137,10 @@ void taskCallStreamsLogByDefault() { // of the wait without any explicit opt-in. ActorRun run = tc.call( - null, - new TaskCallOptions().logOptions(new StreamedLogOptions().toLog(collected::add)), - TEST_ACTOR_WAIT_SECS); + null, + new TaskCallOptions().logOptions(new StreamedLogOptions().toLog(collected::add)), + TEST_ACTOR_WAIT_SECS) + .join(); assertEquals("SUCCEEDED", run.getStatus()); // As in ActorRunIntegrationTest#callWithActorCallOptionsStreamsLogByDefault: call()'s // internal log-streaming lifecycle closes the stream as soon as the run finishes, which can @@ -146,7 +153,7 @@ void taskCallStreamsLogByDefault() { // fails here rather than being masked. assertCallDefaultStreamedLogNonEmptyIfProduced(client.run(run.getId()), collected); } finally { - client.task(task.getId()).delete(); + client.task(task.getId()).delete().join(); } } } diff --git a/src/test/java/com/apify/client/integration/UserIntegrationTest.java b/src/test/java/com/apify/client/integration/UserIntegrationTest.java index 6065aaa..5f2993f 100644 --- a/src/test/java/com/apify/client/integration/UserIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/UserIntegrationTest.java @@ -3,15 +3,15 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.apify.client.ApifyClient; -import com.fasterxml.jackson.databind.JsonNode; import org.junit.jupiter.api.Test; +import tools.jackson.databind.JsonNode; class UserIntegrationTest extends IntegrationBase { @Test void getOwnAccount() { ApifyClient client = requireClient(); - var user = client.me().get(); + var user = client.me().get().join(); assertTrue(user.isPresent()); assertTrue(user.get().getId() != null && !user.get().getId().isEmpty()); } @@ -19,33 +19,33 @@ void getOwnAccount() { @Test void getUserById() { ApifyClient client = requireClient(); - var me = client.me().get(); + var me = client.me().get().join(); assertTrue(me.isPresent()); // The account's own id is always a valid target for the by-id accessor (public profile view), // covering `ApifyClient.user(id)` in addition to the `me()` convenience already exercised // above. - var byId = client.user(me.get().getId()).get(); + var byId = client.user(me.get().getId()).get().join(); assertTrue(byId.isPresent()); } @Test void getMonthlyUsage() { ApifyClient client = requireClient(); - JsonNode usage = client.me().monthlyUsage(); + JsonNode usage = client.me().monthlyUsage().join(); assertTrue(usage != null && !usage.isEmpty()); } @Test void getMonthlyUsageForDate() { ApifyClient client = requireClient(); - JsonNode usage = client.me().monthlyUsage("2026-06-01"); + JsonNode usage = client.me().monthlyUsage("2026-06-01").join(); assertTrue(usage != null && !usage.isEmpty()); } @Test void getLimits() { ApifyClient client = requireClient(); - JsonNode limits = client.me().limits(); + JsonNode limits = client.me().limits().join(); assertTrue(limits != null && !limits.isEmpty()); } diff --git a/src/test/java/com/apify/client/integration/WebhookIntegrationTest.java b/src/test/java/com/apify/client/integration/WebhookIntegrationTest.java index 86a3f4b..a8ca400 100644 --- a/src/test/java/com/apify/client/integration/WebhookIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/WebhookIntegrationTest.java @@ -29,53 +29,53 @@ static Map webhookDef(String url) { @Test void listWebhooks() { ApifyClient client = requireClient(); - assertTrue(client.webhooks().list(new ListOptions().limit(5L)).getTotal() >= 0); + assertTrue(client.webhooks().list(new ListOptions().limit(5L)).join().getTotal() >= 0); } @Test void listWebhookDispatches() { ApifyClient client = requireClient(); - assertTrue(client.webhookDispatches().list(new ListOptions().limit(5L)).getTotal() >= 0); + assertTrue(client.webhookDispatches().list(new ListOptions().limit(5L)).join().getTotal() >= 0); } @Test void getWebhook() { ApifyClient client = requireClient(); - Webhook wh = client.webhooks().create(webhookDef("https://example.com/webhook")); + Webhook wh = client.webhooks().create(webhookDef("https://example.com/webhook")).join(); try { - var got = client.webhook(wh.getId()).get(); + var got = client.webhook(wh.getId()).get().join(); assertTrue(got.isPresent()); assertEquals(wh.getId(), got.get().getId()); } finally { - client.webhook(wh.getId()).delete(); + client.webhook(wh.getId()).delete().join(); } } @Test void getWebhookDispatch() { ApifyClient client = requireClient(); - Webhook wh = client.webhooks().create(webhookDef("https://example.com/webhook")); + Webhook wh = client.webhooks().create(webhookDef("https://example.com/webhook")).join(); try { - WebhookDispatch dispatch = client.webhook(wh.getId()).test(); - var got = client.webhookDispatch(dispatch.getId()).get(); + WebhookDispatch dispatch = client.webhook(wh.getId()).test().join(); + var got = client.webhookDispatch(dispatch.getId()).get().join(); assertTrue(got.isPresent()); assertEquals(dispatch.getId(), got.get().getId()); } finally { - client.webhook(wh.getId()).delete(); + client.webhook(wh.getId()).delete().join(); } } @Test void webhookCrudFlow() { ApifyClient client = requireClient(); - Webhook wh = client.webhooks().create(webhookDef("https://example.com/webhook")); + Webhook wh = client.webhooks().create(webhookDef("https://example.com/webhook")).join(); try { WebhookClient webhook = client.webhook(wh.getId()); - assertTrue(webhook.get().isPresent()); - Webhook updated = webhook.update(Map.of("requestUrl", "https://example.com/updated")); + assertTrue(webhook.get().join().isPresent()); + Webhook updated = webhook.update(Map.of("requestUrl", "https://example.com/updated")).join(); assertEquals("https://example.com/updated", updated.getRequestUrl()); - webhook.dispatches().list(new ListOptions()); - webhook.test(); + webhook.dispatches().list(new ListOptions()).join(); + webhook.test().join(); // list() step of the create/get/modify/list/delete flow: verify the just-created webhook // appears in the top-level collection listing. @@ -87,6 +87,7 @@ void webhookCrudFlow() { client .webhooks() .list(new ListOptions().desc(true).limit(10L)) + .join() .getItems() .stream() .anyMatch(w -> wh.getId().equals(w.getId()))); @@ -100,7 +101,7 @@ void webhookCrudFlow() { assertTrue(wh.getModifiedAt() != null); assertTrue(wh.getStats() != null); } finally { - client.webhook(wh.getId()).delete(); + client.webhook(wh.getId()).delete().join(); } } }