From 473825a2bbe0acbbd72f6d0b024cb97cbea26919 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 01/38] refactor: message collector releases messages as BatchMessage per channel --- .../Ecotone/src/Messaging/BatchMessage.php | 48 +++++++++++++++ .../Collector/CollectorSenderInterceptor.php | 26 +++++++-- .../Channel/SendingInterceptorAdapter.php | 19 ++++++ .../tests/Messaging/Unit/BatchMessageTest.php | 39 +++++++++++++ .../Unit/Channel/BatchMessageSendingTest.php | 58 +++++++++++++++++++ 5 files changed, 185 insertions(+), 5 deletions(-) create mode 100644 packages/Ecotone/src/Messaging/BatchMessage.php create mode 100644 packages/Ecotone/tests/Messaging/Unit/BatchMessageTest.php create mode 100644 packages/Ecotone/tests/Messaging/Unit/Channel/BatchMessageSendingTest.php diff --git a/packages/Ecotone/src/Messaging/BatchMessage.php b/packages/Ecotone/src/Messaging/BatchMessage.php new file mode 100644 index 000000000..6e9f44ecf --- /dev/null +++ b/packages/Ecotone/src/Messaging/BatchMessage.php @@ -0,0 +1,48 @@ +}> */ + private array $entries = []; + + private function __construct() + { + } + + public static function constructEmpty(): self + { + return new self(); + } + + /** + * @param array $headers + */ + public function append(mixed $payload, array $headers = []): self + { + $this->entries[] = ['payload' => $payload, 'headers' => $headers]; + + return $this; + } + + /** + * @return array}> + */ + public function getEntries(): array + { + return $this->entries; + } + + public function count(): int + { + return count($this->entries); + } +} diff --git a/packages/Ecotone/src/Messaging/Channel/Collector/CollectorSenderInterceptor.php b/packages/Ecotone/src/Messaging/Channel/Collector/CollectorSenderInterceptor.php index d58bf3969..c4eae5a19 100644 --- a/packages/Ecotone/src/Messaging/Channel/Collector/CollectorSenderInterceptor.php +++ b/packages/Ecotone/src/Messaging/Channel/Collector/CollectorSenderInterceptor.php @@ -6,11 +6,13 @@ use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\Attribute\WithoutMessageCollector; +use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Config\ConfiguredMessagingSystem; use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Ecotone\Messaging\Handler\Processor\MethodInvoker\MethodInvocation; use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageChannel; +use Ecotone\Messaging\Support\MessageBuilder; /** * licence Apache-2.0 @@ -41,11 +43,9 @@ public function send( $result = $methodInvocation->proceed(); $collectedMessages = $this->collectorStorage->releaseMessages($logger, $message); if ($collectedMessages !== []) { - $messageChannel = $this->getTargetChannel($configuredMessagingSystem); - - foreach ($collectedMessages as $collectedMessage) { - $messageChannel->send($collectedMessage); - } + $this->getTargetChannel($configuredMessagingSystem)->send( + MessageBuilder::withPayload($this->combineIntoBatch($collectedMessages))->build() + ); } } finally { $this->collectorStorage->disable(); @@ -58,4 +58,20 @@ private function getTargetChannel(ConfiguredMessagingSystem $configuredMessaging { return $configuredMessagingSystem->getMessageChannelByName($this->targetChannel); } + + /** + * @param Message[] $collectedMessages + */ + private function combineIntoBatch(array $collectedMessages): BatchMessage + { + $batchMessage = BatchMessage::constructEmpty(); + foreach ($collectedMessages as $collectedMessage) { + $batchMessage = $batchMessage->append( + $collectedMessage->getPayload(), + $collectedMessage->getHeaders()->headers() + ); + } + + return $batchMessage; + } } diff --git a/packages/Ecotone/src/Messaging/Channel/SendingInterceptorAdapter.php b/packages/Ecotone/src/Messaging/Channel/SendingInterceptorAdapter.php index 91a887bd4..78baf2786 100644 --- a/packages/Ecotone/src/Messaging/Channel/SendingInterceptorAdapter.php +++ b/packages/Ecotone/src/Messaging/Channel/SendingInterceptorAdapter.php @@ -4,9 +4,11 @@ namespace Ecotone\Messaging\Channel; +use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageChannel; use Ecotone\Messaging\Support\Assert; +use Ecotone\Messaging\Support\MessageBuilder; use Throwable; /** @@ -48,6 +50,12 @@ public function __construct(MessageChannel $messageChannel, array $sortedChannel */ public function send(Message $message): void { + if ($message->getPayload() instanceof BatchMessage) { + $this->sendEachMessageFromBatch($message->getPayload()); + + return; + } + $messageToSend = $message; $executedInterceptors = []; $isMessageDropped = false; @@ -102,6 +110,17 @@ public function send(Message $message): void $this->executePostSend($messageToSend, $executedInterceptors, $firstCleanupFailure); } + private function sendEachMessageFromBatch(BatchMessage $batchMessage): void + { + foreach ($batchMessage->getEntries() as $entry) { + $this->send( + MessageBuilder::withPayload($entry['payload']) + ->setMultipleHeaders($entry['headers']) + ->build() + ); + } + } + /** * @param ChannelInterceptor[] $executedInterceptors */ diff --git a/packages/Ecotone/tests/Messaging/Unit/BatchMessageTest.php b/packages/Ecotone/tests/Messaging/Unit/BatchMessageTest.php new file mode 100644 index 000000000..d6748e263 --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Unit/BatchMessageTest.php @@ -0,0 +1,39 @@ +append('first payload') + ->append($orderPlaced, ['priority' => 5]); + + $this->assertSame( + [ + ['payload' => 'first payload', 'headers' => []], + ['payload' => $orderPlaced, 'headers' => ['priority' => 5]], + ], + $batch->getEntries(), + ); + } + + public function test_counting_appended_messages(): void + { + $this->assertCount(0, BatchMessage::constructEmpty()); + $this->assertCount(2, BatchMessage::constructEmpty()->append('one')->append('two')); + } +} diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/BatchMessageSendingTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/BatchMessageSendingTest.php new file mode 100644 index 000000000..cc20a9eee --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/BatchMessageSendingTest.php @@ -0,0 +1,58 @@ +append('first order') + ->append('second order', ['priority' => 5]); + + $ecotoneLite->getMessageChannel('orders')->send( + MessageBuilder::withPayload($batch)->build() + ); + + $firstMessage = $ecotoneLite->receiveMessageFrom('orders'); + $secondMessage = $ecotoneLite->receiveMessageFrom('orders'); + + $this->assertSame('first order', $firstMessage->getPayload()); + $this->assertSame('second order', $secondMessage->getPayload()); + $this->assertSame(5, $secondMessage->getHeaders()->get('priority')); + $this->assertNull($ecotoneLite->receiveMessageFrom('orders')); + } + + public function test_empty_batch_message_delivers_nothing(): void + { + $ecotoneLite = EcotoneLite::bootstrapFlowTesting( + enableAsynchronousProcessing: [ + SimpleMessageChannelBuilder::createQueueChannel('orders'), + ], + ); + + $ecotoneLite->getMessageChannel('orders')->send( + MessageBuilder::withPayload(BatchMessage::constructEmpty())->build() + ); + + $this->assertNull($ecotoneLite->receiveMessageFrom('orders')); + } +} From e1b7331f3d0ee969c6944253583042c7dadb4f84 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 02/38] feat: MessagePublisher asyncPublish returning Future with pending delivery tracking --- .../AsyncPublishingFailedException.php | 52 ++++++++ .../AsyncPublishingGateway.php | 43 +++++++ .../AsyncPublishingRegistry.php | 68 +++++++++++ .../AsyncPublishing/DeliveryFuture.php | 60 ++++++++++ .../AsyncPublishing/DeliveryResult.php | 44 +++++++ .../AsyncPublishing/FailedDelivery.php | 29 +++++ .../AsyncPublishing/PendingDelivery.php | 15 +++ .../RegisterSingletonMessagingServices.php | 2 + .../Handler/Gateway/FutureReplyReceiver.php | 20 +++- .../src/Messaging/MessagePublisher.php | 2 + .../InMemoryAsyncOutboundAdapter.php | 74 ++++++++++++ .../InMemoryAsyncPublisherModule.php | 86 ++++++++++++++ .../InMemoryPendingDelivery.php | 47 ++++++++ .../MessagePublisherAsyncPublishTest.php | 112 ++++++++++++++++++ 14 files changed, 653 insertions(+), 1 deletion(-) create mode 100644 packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingFailedException.php create mode 100644 packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php create mode 100644 packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php create mode 100644 packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php create mode 100644 packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryResult.php create mode 100644 packages/Ecotone/src/Messaging/Channel/AsyncPublishing/FailedDelivery.php create mode 100644 packages/Ecotone/src/Messaging/Channel/AsyncPublishing/PendingDelivery.php create mode 100644 packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncOutboundAdapter.php create mode 100644 packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublisherModule.php create mode 100644 packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php create mode 100644 packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/MessagePublisherAsyncPublishTest.php diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingFailedException.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingFailedException.php new file mode 100644 index 000000000..9e4b03163 --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingFailedException.php @@ -0,0 +1,52 @@ + $failedDelivery->getFailureReason(), + $failedDeliveries, + ); + + $exception = new self(sprintf( + 'Failed to deliver %d asynchronously published message(s): %s', + count($failedDeliveries), + implode('; ', array_unique($failureReasons)), + )); + $exception->failedDeliveries = $failedDeliveries; + + return $exception; + } + + /** @var FailedDelivery[] */ + private array $failedDeliveries = []; + + public static function publisherNotConfiguredForAsyncPublishing(string $publisherReference): self + { + return new self(sprintf( + 'Message Publisher `%s` is not configured for asynchronous publishing. Enable async publishing on the publisher configuration to make use of asyncPublish.', + $publisherReference, + )); + } + + /** + * @return FailedDelivery[] + */ + public function getFailedDeliveries(): array + { + return $this->failedDeliveries; + } +} diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php new file mode 100644 index 000000000..ad0c4873d --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php @@ -0,0 +1,43 @@ +asyncPublishingRegistry->collectionPoint(); + + $this->configuredMessagingSystem->getMessageChannelByName($this->publisherReference)->send( + MessageBuilder::fromMessage($message) + ->removeHeader(MessageHeaders::REPLY_CHANNEL) + ->removeHeader(MessageHeaders::ROUTING_SLIP) + ->build() + ); + + $pendingDeliveries = $this->asyncPublishingRegistry->registeredSince($collectionPoint); + if ($pendingDeliveries === []) { + throw AsyncPublishingFailedException::publisherNotConfiguredForAsyncPublishing($this->publisherReference); + } + + return DeliveryFuture::forPendingDeliveries($pendingDeliveries); + } +} diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php new file mode 100644 index 000000000..913a89043 --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php @@ -0,0 +1,68 @@ + */ + private array $pendingDeliveries = []; + + private int $nextRegistrationIndex = 0; + + private bool $shutdownFlushRegistered = false; + + public function register(string $channelName, PendingDelivery $pendingDelivery): void + { + $this->pruneAwaitedDeliveries(); + $this->pendingDeliveries[$this->nextRegistrationIndex++] = ['channelName' => $channelName, 'pendingDelivery' => $pendingDelivery]; + + if (! $this->shutdownFlushRegistered) { + register_shutdown_function(fn () => $this->flushUnawaitedDeliveries()); + $this->shutdownFlushRegistered = true; + } + } + + public function collectionPoint(): int + { + return $this->nextRegistrationIndex; + } + + /** + * @return PendingDelivery[] + */ + public function registeredSince(int $collectionPoint): array + { + $registered = []; + foreach ($this->pendingDeliveries as $index => $registration) { + if ($index >= $collectionPoint) { + $registered[] = $registration['pendingDelivery']; + } + } + + return $registered; + } + + public function flushUnawaitedDeliveries(): void + { + foreach ($this->pendingDeliveries as $registration) { + if (! $registration['pendingDelivery']->isAwaited()) { + $registration['pendingDelivery']->awaitDelivery(); + } + } + $this->pendingDeliveries = []; + } + + private function pruneAwaitedDeliveries(): void + { + foreach ($this->pendingDeliveries as $index => $registration) { + if ($registration['pendingDelivery']->isAwaited()) { + unset($this->pendingDeliveries[$index]); + } + } + } +} diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php new file mode 100644 index 000000000..9fee0a5c5 --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php @@ -0,0 +1,60 @@ +resolved) { + if ($this->failure !== null) { + throw $this->failure; + } + + return; + } + + $this->resolved = true; + $failedDeliveries = []; + foreach ($this->pendingDeliveries as $pendingDelivery) { + $deliveryResult = $pendingDelivery->awaitDelivery(); + if (! $deliveryResult->isSuccessful()) { + $failedDeliveries = array_merge($failedDeliveries, $deliveryResult->getFailedDeliveries()); + } + } + + if ($failedDeliveries !== []) { + $this->failure = AsyncPublishingFailedException::withFailedDeliveries($failedDeliveries); + + throw $this->failure; + } + + + } +} diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryResult.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryResult.php new file mode 100644 index 000000000..42542df25 --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryResult.php @@ -0,0 +1,44 @@ +failedDeliveries === []; + } + + /** + * @return FailedDelivery[] + */ + public function getFailedDeliveries(): array + { + return $this->failedDeliveries; + } +} diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/FailedDelivery.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/FailedDelivery.php new file mode 100644 index 000000000..831a6fb42 --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/FailedDelivery.php @@ -0,0 +1,29 @@ +message; + } + + public function getFailureReason(): string + { + return $this->failureReason; + } +} diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/PendingDelivery.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/PendingDelivery.php new file mode 100644 index 000000000..444704b82 --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/PendingDelivery.php @@ -0,0 +1,15 @@ +registerDefault($builder, ConfiguredMessagingSystem::class, new Definition(MessagingSystemContainer::class, [new Reference(ContainerInterface::class), [], []])); $this->registerDefault($builder, EventMapper::class, new Definition(EventMapper::class, factory: 'createEmpty')); $this->registerDefault($builder, LicenceDecider::class, new Definition(LicenceDecider::class, [$this->serviceConfiguration->isRunningForEnterprise()])); + $this->registerDefault($builder, AsyncPublishingRegistry::class, new Definition(AsyncPublishingRegistry::class)); } private function registerDefault(ContainerBuilder $builder, string $id, Definition|Reference $definition): void diff --git a/packages/Ecotone/src/Messaging/Handler/Gateway/FutureReplyReceiver.php b/packages/Ecotone/src/Messaging/Handler/Gateway/FutureReplyReceiver.php index cf4b437d2..41e3075b2 100644 --- a/packages/Ecotone/src/Messaging/Handler/Gateway/FutureReplyReceiver.php +++ b/packages/Ecotone/src/Messaging/Handler/Gateway/FutureReplyReceiver.php @@ -21,6 +21,10 @@ class FutureReplyReceiver implements Future */ private $replyCallable; + private bool $resolved = false; + + private mixed $resolvedValue = null; + /** * FutureReplySender constructor. * @param callable $replyCallable @@ -44,10 +48,24 @@ public static function create(callable $replyCallable): self */ public function resolve() { + if ($this->resolved) { + if ($this->resolvedValue instanceof Future) { + return $this->resolvedValue->resolve(); + } + + return $this->resolvedValue; + } + $replyCallable = $this->replyCallable; /** @var Message $message */ $message = $replyCallable(); + $this->resolvedValue = $message ? $message->getPayload() : null; + $this->resolved = true; + + if ($this->resolvedValue instanceof Future) { + return $this->resolvedValue->resolve(); + } - return $message ? $message->getPayload() : null; + return $this->resolvedValue; } } diff --git a/packages/Ecotone/src/Messaging/MessagePublisher.php b/packages/Ecotone/src/Messaging/MessagePublisher.php index 92146b3c7..2ffec2626 100644 --- a/packages/Ecotone/src/Messaging/MessagePublisher.php +++ b/packages/Ecotone/src/Messaging/MessagePublisher.php @@ -18,4 +18,6 @@ public function sendWithMetadata(string $data, string $sourceMediaType = MediaTy public function convertAndSend(object|array $data): void; public function convertAndSendWithMetadata(object|array $data, array $metadata): void; + + public function asyncPublish(mixed $data, string $sourceMediaType = MediaType::APPLICATION_X_PHP, array $metadata = []): Future; } diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncOutboundAdapter.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncOutboundAdapter.php new file mode 100644 index 000000000..3927324fe --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncOutboundAdapter.php @@ -0,0 +1,74 @@ +sentMessages[] = $message; + + if (! $this->registersPendingDeliveries) { + return; + } + + $pendingDelivery = new InMemoryPendingDelivery($message, $this->deliveryFailureReason); + $this->pendingDeliveries[] = $pendingDelivery; + $asyncPublishingRegistry->register(InMemoryAsyncPublisherModule::PUBLISHER_REFERENCE, $pendingDelivery); + } + + /** + * @return Message[] + */ + public function getSentMessages(): array + { + return $this->sentMessages; + } + + /** + * @return mixed[] + */ + public function getSentPayloads(): array + { + return array_map(fn (Message $message) => $message->getPayload(), $this->sentMessages); + } + + public function awaitedDeliveriesCount(): int + { + return count(array_filter($this->pendingDeliveries, fn (InMemoryPendingDelivery $pendingDelivery) => $pendingDelivery->isAwaited())); + } + + public function totalAwaitCalls(): int + { + return array_sum(array_map(fn (InMemoryPendingDelivery $pendingDelivery) => $pendingDelivery->awaitCalls(), $this->pendingDeliveries)); + } + + public function failDeliveriesWith(string $failureReason): void + { + $this->deliveryFailureReason = $failureReason; + } + + public function actAsSynchronousPublisher(): void + { + $this->registersPendingDeliveries = false; + } +} diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublisherModule.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublisherModule.php new file mode 100644 index 000000000..314f91237 --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublisherModule.php @@ -0,0 +1,86 @@ +registerGatewayBuilder( + GatewayProxyBuilder::create($publisherReference, MessagePublisher::class, 'asyncPublish', $asyncPublishRequestChannel) + ->withParameterConverters([ + GatewayPayloadBuilder::create('data'), + GatewayHeaderBuilder::create('sourceMediaType', MessageHeaders::CONTENT_TYPE), + GatewayHeadersBuilder::create('metadata'), + ]) + ) + ->registerMessageChannel(SimpleMessageChannelBuilder::createDirectMessageChannel($asyncPublishRequestChannel)) + ->registerMessageHandler( + ServiceActivatorBuilder::createWithDefinition( + new Definition(AsyncPublishingGateway::class, [ + $publisherReference, + new Reference(ConfiguredMessagingSystem::class), + new Reference(AsyncPublishingRegistry::class), + ]), + 'publish' + ) + ->withInputChannelName($asyncPublishRequestChannel) + ->withEndpointId($asyncPublishRequestChannel . '.endpoint') + ) + ->registerMessageChannel(SimpleMessageChannelBuilder::createDirectMessageChannel($publisherReference)) + ->registerMessageHandler( + ServiceActivatorBuilder::create(InMemoryAsyncOutboundAdapter::class, 'handle') + ->withInputChannelName($publisherReference) + ->withEndpointId($publisherReference . '.handler') + ); + } + + public function canHandle($extensionObject): bool + { + return false; + } + + public function getModulePackageName(): string + { + return ModulePackageList::CORE_PACKAGE; + } +} diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php new file mode 100644 index 000000000..31ebb72bb --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php @@ -0,0 +1,47 @@ +awaitCalls++; + + if ($this->failureReason !== null) { + return DeliveryResult::withFailedDeliveries([ + new FailedDelivery($this->message, $this->failureReason), + ]); + } + + return DeliveryResult::successful(); + } + + public function isAwaited(): bool + { + return $this->awaitCalls > 0; + } + + public function awaitCalls(): int + { + return $this->awaitCalls; + } +} diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/MessagePublisherAsyncPublishTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/MessagePublisherAsyncPublishTest.php new file mode 100644 index 000000000..5f96dfa38 --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/MessagePublisherAsyncPublishTest.php @@ -0,0 +1,112 @@ +bootstrapPublisher($outboundAdapter); + + $future = $publisher->asyncPublish('order was placed'); + + $this->assertSame(['order was placed'], $outboundAdapter->getSentPayloads()); + $this->assertSame(0, $outboundAdapter->awaitedDeliveriesCount()); + + $future->resolve(); + + $this->assertSame(1, $outboundAdapter->awaitedDeliveriesCount()); + } + + public function test_resolving_future_twice_awaits_delivery_only_once(): void + { + $outboundAdapter = new InMemoryAsyncOutboundAdapter(); + $publisher = $this->bootstrapPublisher($outboundAdapter); + + $future = $publisher->asyncPublish('order was placed'); + $future->resolve(); + $future->resolve(); + + $this->assertSame(1, $outboundAdapter->totalAwaitCalls()); + } + + public function test_resolving_future_throws_when_delivery_failed(): void + { + $outboundAdapter = new InMemoryAsyncOutboundAdapter(); + $outboundAdapter->failDeliveriesWith('broker rejected message'); + $publisher = $this->bootstrapPublisher($outboundAdapter); + + $future = $publisher->asyncPublish('order was placed'); + + $this->expectException(AsyncPublishingFailedException::class); + + $future->resolve(); + } + + public function test_async_publish_on_synchronous_publisher_throws_clear_exception(): void + { + $outboundAdapter = new InMemoryAsyncOutboundAdapter(); + $outboundAdapter->actAsSynchronousPublisher(); + $publisher = $this->bootstrapPublisher($outboundAdapter); + + $this->expectException(AsyncPublishingFailedException::class); + $this->expectExceptionMessageMatches('/not configured for asynchronous publishing/'); + + $publisher->asyncPublish('order was placed'); + } + + public function test_metadata_passed_to_async_publish_lands_on_published_message(): void + { + $outboundAdapter = new InMemoryAsyncOutboundAdapter(); + $publisher = $this->bootstrapPublisher($outboundAdapter); + + $publisher->asyncPublish('order was placed', metadata: ['orderId' => '123']); + + $this->assertSame('123', $outboundAdapter->getSentMessages()[0]->getHeaders()->get('orderId')); + } + + public function test_flushing_unawaited_deliveries_awaits_only_unresolved_futures(): void + { + $outboundAdapter = new InMemoryAsyncOutboundAdapter(); + $ecotoneLite = $this->bootstrapEcotone($outboundAdapter); + $publisher = $ecotoneLite->getGateway(InMemoryAsyncPublisherModule::PUBLISHER_REFERENCE); + + $resolvedFuture = $publisher->asyncPublish('first order'); + $resolvedFuture->resolve(); + $publisher->asyncPublish('second order'); + + $ecotoneLite->getServiceFromContainer(AsyncPublishingRegistry::class)->flushUnawaitedDeliveries(); + + $this->assertSame(2, $outboundAdapter->awaitedDeliveriesCount()); + $this->assertSame(2, $outboundAdapter->totalAwaitCalls()); + } + + private function bootstrapPublisher(InMemoryAsyncOutboundAdapter $outboundAdapter): MessagePublisher + { + return $this->bootstrapEcotone($outboundAdapter)->getGateway(InMemoryAsyncPublisherModule::PUBLISHER_REFERENCE); + } + + private function bootstrapEcotone(InMemoryAsyncOutboundAdapter $outboundAdapter): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + [InMemoryAsyncPublisherModule::class, InMemoryAsyncOutboundAdapter::class], + [$outboundAdapter], + ); + } +} From 69f94885087bb6ad9e31c30cc2f332dc6a42b116 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 03/38] feat: asyncPublish accepts BatchMessage for whole-batch delivery --- .../AsyncPublishingGateway.php | 6 ++ .../MessagePublisherAsyncPublishBatchTest.php | 68 +++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/MessagePublisherAsyncPublishBatchTest.php diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php index ad0c4873d..5a6c6c1a2 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php @@ -4,6 +4,7 @@ namespace Ecotone\Messaging\Channel\AsyncPublishing; +use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Config\ConfiguredMessagingSystem; use Ecotone\Messaging\Future; use Ecotone\Messaging\Message; @@ -24,6 +25,11 @@ public function __construct( public function publish(Message $message): Future { + $payload = $message->getPayload(); + if ($payload instanceof BatchMessage && count($payload) === 0) { + return DeliveryFuture::forPendingDeliveries([]); + } + $collectionPoint = $this->asyncPublishingRegistry->collectionPoint(); $this->configuredMessagingSystem->getMessageChannelByName($this->publisherReference)->send( diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/MessagePublisherAsyncPublishBatchTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/MessagePublisherAsyncPublishBatchTest.php new file mode 100644 index 000000000..6a815d99c --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/MessagePublisherAsyncPublishBatchTest.php @@ -0,0 +1,68 @@ +bootstrapPublisher($outboundAdapter); + + $future = $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('first order') + ->append('second order', ['priority' => 5]) + ); + + $this->assertCount(1, $outboundAdapter->getSentMessages()); + $batchPayload = $outboundAdapter->getSentMessages()[0]->getPayload(); + $this->assertInstanceOf(BatchMessage::class, $batchPayload); + $this->assertSame( + [ + ['payload' => 'first order', 'headers' => []], + ['payload' => 'second order', 'headers' => ['priority' => 5]], + ], + $batchPayload->getEntries(), + ); + $this->assertSame(0, $outboundAdapter->awaitedDeliveriesCount()); + + $future->resolve(); + + $this->assertSame(1, $outboundAdapter->awaitedDeliveriesCount()); + } + + public function test_publishing_empty_batch_resolves_without_sending_anything(): void + { + $outboundAdapter = new InMemoryAsyncOutboundAdapter(); + $publisher = $this->bootstrapPublisher($outboundAdapter); + + $future = $publisher->asyncPublish(BatchMessage::constructEmpty()); + $future->resolve(); + + $this->assertCount(0, $outboundAdapter->getSentMessages()); + } + + private function bootstrapPublisher(InMemoryAsyncOutboundAdapter $outboundAdapter): MessagePublisher + { + $ecotoneLite = EcotoneLite::bootstrapFlowTesting( + [InMemoryAsyncPublisherModule::class, InMemoryAsyncOutboundAdapter::class], + [$outboundAdapter], + ); + + return $ecotoneLite->getGateway(InMemoryAsyncPublisherModule::PUBLISHER_REFERENCE); + } +} From 9af6f7312af07db6791e39d043391a2cad6ad35e Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 04/38] feat: async publishing channels awaited via interceptor before transaction commits --- .../AsyncPublishingRegistry.php | 35 ++++++++ .../AsyncPublishingWaiterInterceptor.php | 38 +++++++++ .../Config/AsyncPublishingModule.php | 60 ++++++++++++++ .../Channel/BatchSupportingMessageChannel.php | 13 +++ .../Channel/SendingInterceptorAdapter.php | 9 +- .../src/Messaging/Config/ModuleClassList.php | 2 + packages/Ecotone/src/Messaging/Precedence.php | 7 +- .../AsyncPublishing/AsyncOrderSubscriber.php | 32 ++++++++ .../FakeTransactionInterceptor.php | 33 ++++++++ .../AsyncPublishing/FakeTransactionModule.php | 58 +++++++++++++ .../InMemoryAsyncPublishingChannel.php | 77 +++++++++++++++++ .../InMemoryAsyncPublishingChannelBuilder.php | 50 +++++++++++ .../InMemoryPendingDelivery.php | 2 + .../Fixture/AsyncPublishing/OperationsLog.php | 27 ++++++ .../Fixture/AsyncPublishing/OrderService.php | 26 ++++++ .../AsyncPublishing/OrderWasPlaced.php | 15 ++++ .../AsyncPublishingChannelTest.php | 82 +++++++++++++++++++ .../tests/Messaging/Unit/PrecedenceTest.php | 23 ++++++ 18 files changed, 587 insertions(+), 2 deletions(-) create mode 100644 packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php create mode 100644 packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishingModule.php create mode 100644 packages/Ecotone/src/Messaging/Channel/BatchSupportingMessageChannel.php create mode 100644 packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/AsyncOrderSubscriber.php create mode 100644 packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/FakeTransactionInterceptor.php create mode 100644 packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/FakeTransactionModule.php create mode 100644 packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php create mode 100644 packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannelBuilder.php create mode 100644 packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OperationsLog.php create mode 100644 packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OrderService.php create mode 100644 packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OrderWasPlaced.php create mode 100644 packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingChannelTest.php create mode 100644 packages/Ecotone/tests/Messaging/Unit/PrecedenceTest.php diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php index 913a89043..05952df8d 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php @@ -14,8 +14,43 @@ final class AsyncPublishingRegistry private int $nextRegistrationIndex = 0; + private bool $scopeActive = false; + private bool $shutdownFlushRegistered = false; + public function openScope(): void + { + $this->scopeActive = true; + } + + public function isScopeActive(): bool + { + return $this->scopeActive; + } + + public function closeScope(): void + { + $this->scopeActive = false; + $this->pendingDeliveries = []; + } + + public function awaitAll(): DeliveryResult + { + $failedDeliveries = []; + foreach ($this->pendingDeliveries as $registration) { + if ($registration['pendingDelivery']->isAwaited()) { + continue; + } + + $deliveryResult = $registration['pendingDelivery']->awaitDelivery(); + if (! $deliveryResult->isSuccessful()) { + $failedDeliveries = array_merge($failedDeliveries, $deliveryResult->getFailedDeliveries()); + } + } + + return $failedDeliveries === [] ? DeliveryResult::successful() : DeliveryResult::withFailedDeliveries($failedDeliveries); + } + public function register(string $channelName, PendingDelivery $pendingDelivery): void { $this->pruneAwaitedDeliveries(); diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php new file mode 100644 index 000000000..c01013e6f --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php @@ -0,0 +1,38 @@ +asyncPublishingRegistry->isScopeActive()) { + return $methodInvocation->proceed(); + } + + $this->asyncPublishingRegistry->openScope(); + try { + $result = $methodInvocation->proceed(); + + $deliveryResult = $this->asyncPublishingRegistry->awaitAll(); + if (! $deliveryResult->isSuccessful()) { + throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + } + } finally { + $this->asyncPublishingRegistry->closeScope(); + } + + return $result; + } +} diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishingModule.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishingModule.php new file mode 100644 index 000000000..d0a8c1b19 --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishingModule.php @@ -0,0 +1,60 @@ +registerServiceDefinition( + AsyncPublishingWaiterInterceptor::class, + new Definition(AsyncPublishingWaiterInterceptor::class, [new Reference(AsyncPublishingRegistry::class)]) + ); + $messagingConfiguration->registerAroundMethodInterceptor( + AroundInterceptorBuilder::create( + AsyncPublishingWaiterInterceptor::class, + $interfaceToCallRegistry->getFor(AsyncPublishingWaiterInterceptor::class, 'await'), + Precedence::ASYNC_PUBLISHING_AWAIT_PRECEDENCE, + CommandBus::class . '||' . AsynchronousRunningEndpoint::class, + ) + ); + } + + public function canHandle($extensionObject): bool + { + return false; + } + + public function getModulePackageName(): string + { + return ModulePackageList::CORE_PACKAGE; + } +} diff --git a/packages/Ecotone/src/Messaging/Channel/BatchSupportingMessageChannel.php b/packages/Ecotone/src/Messaging/Channel/BatchSupportingMessageChannel.php new file mode 100644 index 000000000..269632151 --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/BatchSupportingMessageChannel.php @@ -0,0 +1,13 @@ +getPayload() instanceof BatchMessage) { + if ($message->getPayload() instanceof BatchMessage && ! $this->targetChannelSupportsBatchMessages()) { $this->sendEachMessageFromBatch($message->getPayload()); return; @@ -110,6 +110,13 @@ public function send(Message $message): void $this->executePostSend($messageToSend, $executedInterceptors, $firstCleanupFailure); } + private function targetChannelSupportsBatchMessages(): bool + { + $targetChannel = $this->getInternalMessageChannel(); + + return $targetChannel instanceof BatchSupportingMessageChannel && $targetChannel->supportsBatchMessages(); + } + private function sendEachMessageFromBatch(BatchMessage $batchMessage): void { foreach ($batchMessage->getEntries() as $entry) { diff --git a/packages/Ecotone/src/Messaging/Config/ModuleClassList.php b/packages/Ecotone/src/Messaging/Config/ModuleClassList.php index 713232361..1046e033c 100644 --- a/packages/Ecotone/src/Messaging/Config/ModuleClassList.php +++ b/packages/Ecotone/src/Messaging/Config/ModuleClassList.php @@ -26,6 +26,7 @@ use Ecotone\Kafka\Configuration\KafkaModule; use Ecotone\Laravel\Config\LaravelConnectionModule; use Ecotone\Lite\Test\Configuration\EcotoneTestSupportModule; +use Ecotone\Messaging\Channel\AsyncPublishing\Config\AsyncPublishingModule; use Ecotone\Messaging\Channel\Collector\Config\CollectorModule; use Ecotone\Messaging\Channel\DynamicChannel\Config\DynamicMessageChannelModule; use Ecotone\Messaging\Channel\Manager\ChannelSetupModule; @@ -108,6 +109,7 @@ class ModuleClassList RouterModule::class, ScheduledModule::class, CollectorModule::class, + AsyncPublishingModule::class, ChannelSetupModule::class, SerializerModule::class, ServiceActivatorModule::class, diff --git a/packages/Ecotone/src/Messaging/Precedence.php b/packages/Ecotone/src/Messaging/Precedence.php index 21f301284..2744a9945 100644 --- a/packages/Ecotone/src/Messaging/Precedence.php +++ b/packages/Ecotone/src/Messaging/Precedence.php @@ -58,10 +58,15 @@ interface Precedence */ public const DATABASE_TRANSACTION_PRECEDENCE = -2000; + /** + * Awaits delivery confirmations of asynchronously published messages before transaction commits + */ + public const ASYNC_PUBLISHING_AWAIT_PRECEDENCE = self::DATABASE_TRANSACTION_PRECEDENCE + 1; + /** * Collects messages to be sent to asynchronous channels. */ - public const COLLECTOR_SENDER_PRECEDENCE = self::DATABASE_TRANSACTION_PRECEDENCE + 1; + public const COLLECTOR_SENDER_PRECEDENCE = self::ASYNC_PUBLISHING_AWAIT_PRECEDENCE + 1; public const DATABASE_OBJECT_MANAGER_PRECEDENCE = self::COLLECTOR_SENDER_PRECEDENCE + 1; diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/AsyncOrderSubscriber.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/AsyncOrderSubscriber.php new file mode 100644 index 000000000..8926ce032 --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/AsyncOrderSubscriber.php @@ -0,0 +1,32 @@ +receivedEvents[] = $event; + } + + /** + * @return string[] + */ + public function getReceivedOrderIds(): array + { + return array_map(fn (OrderWasPlaced $event) => $event->orderId, $this->receivedEvents); + } +} diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/FakeTransactionInterceptor.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/FakeTransactionInterceptor.php new file mode 100644 index 000000000..bce9f9cd0 --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/FakeTransactionInterceptor.php @@ -0,0 +1,33 @@ +operationsLog->log('transaction started'); + try { + $result = $methodInvocation->proceed(); + $this->operationsLog->log('transaction committed'); + + return $result; + } catch (Throwable $exception) { + $this->operationsLog->log('transaction rolled back'); + + throw $exception; + } + } +} diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/FakeTransactionModule.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/FakeTransactionModule.php new file mode 100644 index 000000000..b241c01dd --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/FakeTransactionModule.php @@ -0,0 +1,58 @@ +registerServiceDefinition( + FakeTransactionInterceptor::class, + new Definition(FakeTransactionInterceptor::class, [new Reference(OperationsLog::class)]) + ); + $messagingConfiguration->registerAroundMethodInterceptor( + AroundInterceptorBuilder::create( + FakeTransactionInterceptor::class, + $interfaceToCallRegistry->getFor(FakeTransactionInterceptor::class, 'transactional'), + Precedence::DATABASE_TRANSACTION_PRECEDENCE, + CommandBus::class . '||' . AsynchronousRunningEndpoint::class, + ) + ); + } + + public function canHandle($extensionObject): bool + { + return false; + } + + public function getModulePackageName(): string + { + return ModulePackageList::CORE_PACKAGE; + } +} diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php new file mode 100644 index 000000000..99c8e7248 --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php @@ -0,0 +1,77 @@ +getPayload(); + + if ($payload instanceof BatchMessage) { + $this->operationsLog->log(sprintf('published batch of %d messages to broker', count($payload))); + foreach ($payload->getEntries() as $entry) { + $this->queue[] = MessageBuilder::withPayload($entry['payload']) + ->setMultipleHeaders($entry['headers']) + ->build(); + } + } else { + $this->operationsLog->log('published message to broker'); + $this->queue[] = $message; + } + + $this->asyncPublishingRegistry->register( + $this->channelName, + new InMemoryPendingDelivery($message, $this->deliveryFailureReason, $this->operationsLog), + ); + } + + public function receive(): ?Message + { + return array_shift($this->queue) ?: null; + } + + public function receiveWithTimeout(PollingMetadata $pollingMetadata): ?Message + { + return $this->receive(); + } + + public function onConsumerStop(): void + { + } + + public function supportsBatchMessages(): bool + { + return true; + } + + public function failDeliveriesWith(string $failureReason): void + { + $this->deliveryFailureReason = $failureReason; + } +} diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannelBuilder.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannelBuilder.php new file mode 100644 index 000000000..edf9fbbf0 --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannelBuilder.php @@ -0,0 +1,50 @@ +channelName; + } + + public function isPollable(): bool + { + return true; + } + + public function isStreamingChannel(): bool + { + return false; + } + + public function compile(MessagingContainerBuilder $builder): Definition|Reference + { + return new Definition(InMemoryAsyncPublishingChannel::class, [ + $this->channelName, + new Reference(AsyncPublishingRegistry::class), + new Reference(OperationsLog::class), + ]); + } +} diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php index 31ebb72bb..510e7c8f6 100644 --- a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php @@ -19,12 +19,14 @@ final class InMemoryPendingDelivery implements PendingDelivery public function __construct( private Message $message, private ?string $failureReason = null, + private ?OperationsLog $operationsLog = null, ) { } public function awaitDelivery(): DeliveryResult { $this->awaitCalls++; + $this->operationsLog?->log('delivery confirmations awaited'); if ($this->failureReason !== null) { return DeliveryResult::withFailedDeliveries([ diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OperationsLog.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OperationsLog.php new file mode 100644 index 000000000..2813100db --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OperationsLog.php @@ -0,0 +1,27 @@ +operations[] = $operation; + } + + /** + * @return string[] + */ + public function getOperations(): array + { + return $this->operations; + } +} diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OrderService.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OrderService.php new file mode 100644 index 000000000..99f1ed2bf --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OrderService.php @@ -0,0 +1,26 @@ +operationsLog->log('command handler executed'); + $eventBus->publish(new OrderWasPlaced($order . '-1')); + $eventBus->publish(new OrderWasPlaced($order . '-2')); + } +} diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OrderWasPlaced.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OrderWasPlaced.php new file mode 100644 index 000000000..c10c29e5c --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OrderWasPlaced.php @@ -0,0 +1,15 @@ +bootstrapEcotone($operationsLog); + + $ecotoneLite->sendCommandWithRoutingKey('order.place', 'espresso'); + + $this->assertSame( + [ + 'transaction started', + 'command handler executed', + 'published batch of 2 messages to broker', + 'delivery confirmations awaited', + 'transaction committed', + ], + $operationsLog->getOperations(), + ); + } + + public function test_failed_delivery_confirmation_fails_command_execution_and_rolls_back_transaction(): void + { + $operationsLog = new OperationsLog(); + $ecotoneLite = $this->bootstrapEcotone($operationsLog); + $channel = $ecotoneLite->getMessageChannel('async_orders'); + assert($channel instanceof MessageChannelInterceptorAdapter); + $channel->getInternalMessageChannel()->failDeliveriesWith('broker not available'); + + $commandException = null; + try { + $ecotoneLite->sendCommandWithRoutingKey('order.place', 'espresso'); + } catch (AsyncPublishingFailedException $exception) { + $commandException = $exception; + } + + $this->assertInstanceOf(AsyncPublishingFailedException::class, $commandException); + $this->assertStringContainsString('broker not available', $commandException->getMessage()); + $this->assertSame( + [ + 'transaction started', + 'command handler executed', + 'published batch of 2 messages to broker', + 'delivery confirmations awaited', + 'transaction rolled back', + ], + $operationsLog->getOperations(), + ); + } + + private function bootstrapEcotone(OperationsLog $operationsLog): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + [OrderService::class, AsyncOrderSubscriber::class, FakeTransactionModule::class], + [new OrderService($operationsLog), new AsyncOrderSubscriber(), OperationsLog::class => $operationsLog], + enableAsynchronousProcessing: [ + InMemoryAsyncPublishingChannelBuilder::create('async_orders'), + ], + ); + } +} diff --git a/packages/Ecotone/tests/Messaging/Unit/PrecedenceTest.php b/packages/Ecotone/tests/Messaging/Unit/PrecedenceTest.php new file mode 100644 index 000000000..87ebd5e9a --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Unit/PrecedenceTest.php @@ -0,0 +1,23 @@ +assertGreaterThan(Precedence::DATABASE_TRANSACTION_PRECEDENCE, Precedence::ASYNC_PUBLISHING_AWAIT_PRECEDENCE); + $this->assertGreaterThan(Precedence::ASYNC_PUBLISHING_AWAIT_PRECEDENCE, Precedence::COLLECTOR_SENDER_PRECEDENCE); + $this->assertGreaterThan(Precedence::COLLECTOR_SENDER_PRECEDENCE, Precedence::DATABASE_OBJECT_MANAGER_PRECEDENCE); + $this->assertGreaterThan(Precedence::DATABASE_OBJECT_MANAGER_PRECEDENCE, Precedence::LAZY_EVENT_PUBLICATION_PRECEDENCE); + } +} From f15f738b79577a022706a632165ea3678bcc2ccd Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 05/38] test: async publishing collector on/off matrix coverage --- .../AsyncPublishingCollectorMatrixTest.php | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingCollectorMatrixTest.php diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingCollectorMatrixTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingCollectorMatrixTest.php new file mode 100644 index 000000000..8841b2a1f --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingCollectorMatrixTest.php @@ -0,0 +1,77 @@ +bootstrapEcotone($operationsLog, collectorEnabled: false); + + $ecotoneLite->sendCommandWithRoutingKey('order.place', 'espresso'); + + $this->assertSame( + [ + 'transaction started', + 'command handler executed', + 'published message to broker', + 'published message to broker', + 'delivery confirmations awaited', + 'delivery confirmations awaited', + 'transaction committed', + ], + $operationsLog->getOperations(), + ); + } + + public function test_messages_are_consumable_from_channel_with_and_without_collector(): void + { + foreach ([true, false] as $collectorEnabled) { + $ecotoneLite = $this->bootstrapEcotone(new OperationsLog(), $collectorEnabled); + + $ecotoneLite->sendCommandWithRoutingKey('order.place', 'espresso'); + + $this->assertEquals( + [new OrderWasPlaced('espresso-1'), new OrderWasPlaced('espresso-2')], + [ + $ecotoneLite->receiveMessageFrom('async_orders')->getPayload(), + $ecotoneLite->receiveMessageFrom('async_orders')->getPayload(), + ], + ); + $this->assertNull($ecotoneLite->receiveMessageFrom('async_orders')); + } + } + + private function bootstrapEcotone(OperationsLog $operationsLog, bool $collectorEnabled): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + [OrderService::class, AsyncOrderSubscriber::class, FakeTransactionModule::class], + [new OrderService($operationsLog), new AsyncOrderSubscriber(), OperationsLog::class => $operationsLog], + ServiceConfiguration::createWithDefaults()->withExtensionObjects([ + PollableChannelConfiguration::neverRetry('async_orders')->withCollector($collectorEnabled), + ]), + enableAsynchronousProcessing: [ + InMemoryAsyncPublishingChannelBuilder::create('async_orders'), + ], + ); + } +} From e3da996116dfef9abb1dd041e771ee3a51d05037 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 06/38] feat: error channel routing and transaction semantics for async publishing --- .../AsyncPublishingTestChannel.php | 77 +++++++++ .../AsyncPublishingTestChannelBuilder.php | 50 ++++++ .../AsyncPublishing/TestPendingDelivery.php | 43 +++++ .../AsyncPublishingTransactionTest.php | 104 +++++++++++++ .../AsyncPublishingGateway.php | 31 +++- .../AsyncPublishingRegistry.php | 24 ++- .../AsyncPublishingWaiterInterceptor.php | 69 +++++++- .../Config/AsyncPublishingModule.php | 22 ++- .../AsyncPublishing/FailedDelivery.php | 6 + .../AsyncPublishing/AsyncOrderForwarder.php | 27 ++++ .../FakeTransactionInterceptor.php | 9 ++ .../InMemoryAsyncPublishingChannel.php | 17 +- .../InMemoryPendingDelivery.php | 3 +- .../AsyncPublishing/OrderRequestReceived.php | 15 ++ .../Fixture/AsyncPublishing/OrderService.php | 8 + .../AsyncPublishingScenariosTest.php | 147 ++++++++++++++++++ 16 files changed, 630 insertions(+), 22 deletions(-) create mode 100644 packages/Dbal/tests/Fixture/AsyncPublishing/AsyncPublishingTestChannel.php create mode 100644 packages/Dbal/tests/Fixture/AsyncPublishing/AsyncPublishingTestChannelBuilder.php create mode 100644 packages/Dbal/tests/Fixture/AsyncPublishing/TestPendingDelivery.php create mode 100644 packages/Dbal/tests/Integration/AsyncPublishingTransactionTest.php create mode 100644 packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/AsyncOrderForwarder.php create mode 100644 packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OrderRequestReceived.php create mode 100644 packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingScenariosTest.php diff --git a/packages/Dbal/tests/Fixture/AsyncPublishing/AsyncPublishingTestChannel.php b/packages/Dbal/tests/Fixture/AsyncPublishing/AsyncPublishingTestChannel.php new file mode 100644 index 000000000..cbe5a9632 --- /dev/null +++ b/packages/Dbal/tests/Fixture/AsyncPublishing/AsyncPublishingTestChannel.php @@ -0,0 +1,77 @@ +getPayload(); + + if ($payload instanceof BatchMessage) { + foreach ($payload->getEntries() as $entry) { + $this->queue[] = MessageBuilder::withPayload($entry['payload']) + ->setMultipleHeaders($entry['headers']) + ->build(); + } + } else { + $this->queue[] = $message; + } + + $pendingDelivery = new TestPendingDelivery($message, $this->channelName, $this->deliveryFailureReason); + + if (! $this->asyncPublishingRegistry->isScopeActive()) { + $deliveryResult = $pendingDelivery->awaitDelivery(); + if (! $deliveryResult->isSuccessful()) { + throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + } + + return; + } + + $this->asyncPublishingRegistry->register($this->channelName, $pendingDelivery); + } + + public function receive(): ?Message + { + return array_shift($this->queue) ?: null; + } + + public function receiveWithTimeout(PollingMetadata $pollingMetadata): ?Message + { + return $this->receive(); + } + + public function onConsumerStop(): void + { + } + + public function supportsBatchMessages(): bool + { + return true; + } +} diff --git a/packages/Dbal/tests/Fixture/AsyncPublishing/AsyncPublishingTestChannelBuilder.php b/packages/Dbal/tests/Fixture/AsyncPublishing/AsyncPublishingTestChannelBuilder.php new file mode 100644 index 000000000..56d0d2877 --- /dev/null +++ b/packages/Dbal/tests/Fixture/AsyncPublishing/AsyncPublishingTestChannelBuilder.php @@ -0,0 +1,50 @@ +channelName; + } + + public function isPollable(): bool + { + return true; + } + + public function isStreamingChannel(): bool + { + return false; + } + + public function compile(MessagingContainerBuilder $builder): Definition|Reference + { + return new Definition(AsyncPublishingTestChannel::class, [ + $this->channelName, + new Reference(AsyncPublishingRegistry::class), + $this->deliveryFailureReason, + ]); + } +} diff --git a/packages/Dbal/tests/Fixture/AsyncPublishing/TestPendingDelivery.php b/packages/Dbal/tests/Fixture/AsyncPublishing/TestPendingDelivery.php new file mode 100644 index 000000000..388a3102e --- /dev/null +++ b/packages/Dbal/tests/Fixture/AsyncPublishing/TestPendingDelivery.php @@ -0,0 +1,43 @@ +awaited = true; + + if ($this->failureReason !== null) { + return DeliveryResult::withFailedDeliveries([ + new FailedDelivery($this->message, $this->failureReason, $this->channelName), + ]); + } + + return DeliveryResult::successful(); + } + + public function isAwaited(): bool + { + return $this->awaited; + } +} diff --git a/packages/Dbal/tests/Integration/AsyncPublishingTransactionTest.php b/packages/Dbal/tests/Integration/AsyncPublishingTransactionTest.php new file mode 100644 index 000000000..7843e3d7b --- /dev/null +++ b/packages/Dbal/tests/Integration/AsyncPublishingTransactionTest.php @@ -0,0 +1,104 @@ +bootstrapEcotone( + [AsyncPublishingTestChannelBuilder::create('notifications')], + [] + ); + + $ecotoneLite->sendCommand(new RegisterPerson(100, 'Johny')); + + $this->assertNotNull($ecotoneLite->sendQueryWithRouting('person.getName', metadata: ['aggregate.id' => 100])); + $this->assertNotNull($ecotoneLite->getMessageChannel('notifications')->receive()); + } + + public function test_failed_delivery_confirmation_rolls_back_database_transaction(): void + { + $ecotoneLite = $this->bootstrapEcotone( + [AsyncPublishingTestChannelBuilder::create('notifications', deliveryFailureReason: 'broker not available')], + [] + ); + + $deliveryFailed = false; + try { + $ecotoneLite->sendCommand(new RegisterPerson(100, 'Johny')); + } catch (AsyncPublishingFailedException) { + $deliveryFailed = true; + } + $this->assertTrue($deliveryFailed); + + $this->expectException(AggregateNotFoundException::class); + + $ecotoneLite->sendQueryWithRouting('person.getName', metadata: ['aggregate.id' => 100]); + } + + public function test_failed_delivery_routed_to_error_channel_commits_database_transaction(): void + { + $ecotoneLite = $this->bootstrapEcotone( + [ + AsyncPublishingTestChannelBuilder::create('notifications', deliveryFailureReason: 'broker not available'), + SimpleMessageChannelBuilder::createQueueChannel('failure_channel'), + ], + [GlobalPollableChannelConfiguration::createWithDefaults()->withErrorChannel('failure_channel')] + ); + + $ecotoneLite->sendCommand(new RegisterPerson(100, 'Johny')); + + $this->assertNotNull($ecotoneLite->sendQueryWithRouting('person.getName', metadata: ['aggregate.id' => 100])); + + $failedMessage = $ecotoneLite->receiveMessageFrom('failure_channel'); + $this->assertNotNull($failedMessage); + $this->assertStringContainsString('broker not available', $failedMessage->getHeaders()->get(ErrorContext::EXCEPTION_MESSAGE)); + } + + private function bootstrapEcotone(array $channelBuilders, array $extensionObjects): FlowTestSupport + { + $this->setupUserTable(); + + return EcotoneLite::bootstrapFlowTesting( + [Person::class, NotificationService::class], + [new NotificationService(), DbalConnectionFactory::class => $this->getORMConnectionFactory([__DIR__ . '/../Fixture/ORM/Person'])], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::DBAL_PACKAGE, ModulePackageList::ASYNCHRONOUS_PACKAGE])) + ->withExtensionObjects(array_merge( + $extensionObjects, + $channelBuilders, + [ + DbalConfiguration::createWithDefaults() + ->withTransactionOnCommandBus(true) + ->withTransactionOnAsynchronousEndpoints(true) + ->withDoctrineORMRepositories(true), + ] + )), + addInMemoryStateStoredRepository: false + ); + } +} diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php index 5a6c6c1a2..a84ceebf1 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php @@ -30,16 +30,31 @@ public function publish(Message $message): Future return DeliveryFuture::forPendingDeliveries([]); } - $collectionPoint = $this->asyncPublishingRegistry->collectionPoint(); + $scopeWasActive = $this->asyncPublishingRegistry->isScopeActive(); + if (! $scopeWasActive) { + $this->asyncPublishingRegistry->openScope(); + } + + try { + $collectionPoint = $this->asyncPublishingRegistry->collectionPoint(); - $this->configuredMessagingSystem->getMessageChannelByName($this->publisherReference)->send( - MessageBuilder::fromMessage($message) - ->removeHeader(MessageHeaders::REPLY_CHANNEL) - ->removeHeader(MessageHeaders::ROUTING_SLIP) - ->build() - ); + $this->configuredMessagingSystem->getMessageChannelByName($this->publisherReference)->send( + MessageBuilder::fromMessage($message) + ->removeHeader(MessageHeaders::REPLY_CHANNEL) + ->removeHeader(MessageHeaders::ROUTING_SLIP) + ->build() + ); + + $pendingDeliveries = $this->asyncPublishingRegistry->registeredSince($collectionPoint); + if (! $scopeWasActive) { + $this->asyncPublishingRegistry->markAsPublisherOwned($pendingDeliveries); + } + } finally { + if (! $scopeWasActive) { + $this->asyncPublishingRegistry->closeScope(); + } + } - $pendingDeliveries = $this->asyncPublishingRegistry->registeredSince($collectionPoint); if ($pendingDeliveries === []) { throw AsyncPublishingFailedException::publisherNotConfiguredForAsyncPublishing($this->publisherReference); } diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php index 05952df8d..aa9d926e0 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php @@ -9,7 +9,7 @@ */ final class AsyncPublishingRegistry { - /** @var array */ + /** @var array */ private array $pendingDeliveries = []; private int $nextRegistrationIndex = 0; @@ -31,14 +31,30 @@ public function isScopeActive(): bool public function closeScope(): void { $this->scopeActive = false; - $this->pendingDeliveries = []; + foreach ($this->pendingDeliveries as $index => $registration) { + if ($registration['scopeOwned']) { + unset($this->pendingDeliveries[$index]); + } + } + } + + /** + * @param PendingDelivery[] $pendingDeliveries + */ + public function markAsPublisherOwned(array $pendingDeliveries): void + { + foreach ($this->pendingDeliveries as $index => $registration) { + if (in_array($registration['pendingDelivery'], $pendingDeliveries, true)) { + $this->pendingDeliveries[$index]['scopeOwned'] = false; + } + } } public function awaitAll(): DeliveryResult { $failedDeliveries = []; foreach ($this->pendingDeliveries as $registration) { - if ($registration['pendingDelivery']->isAwaited()) { + if (! $registration['scopeOwned'] || $registration['pendingDelivery']->isAwaited()) { continue; } @@ -54,7 +70,7 @@ public function awaitAll(): DeliveryResult public function register(string $channelName, PendingDelivery $pendingDelivery): void { $this->pruneAwaitedDeliveries(); - $this->pendingDeliveries[$this->nextRegistrationIndex++] = ['channelName' => $channelName, 'pendingDelivery' => $pendingDelivery]; + $this->pendingDeliveries[$this->nextRegistrationIndex++] = ['channelName' => $channelName, 'pendingDelivery' => $pendingDelivery, 'scopeOwned' => $this->scopeActive]; if (! $this->shutdownFlushRegistered) { register_shutdown_function(fn () => $this->flushUnawaitedDeliveries()); diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php index c01013e6f..5a74e9cdb 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php @@ -4,15 +4,28 @@ namespace Ecotone\Messaging\Channel\AsyncPublishing; +use Ecotone\Messaging\BatchMessage; +use Ecotone\Messaging\Config\ConfiguredMessagingSystem; +use Ecotone\Messaging\Handler\Gateway\ErrorChannelService; use Ecotone\Messaging\Handler\Processor\MethodInvoker\MethodInvocation; +use Ecotone\Messaging\Message; +use Ecotone\Messaging\Support\MessageBuilder; /** * licence Enterprise */ final class AsyncPublishingWaiterInterceptor { - public function __construct(private AsyncPublishingRegistry $asyncPublishingRegistry) - { + /** + * @param array $errorChannels + */ + public function __construct( + private AsyncPublishingRegistry $asyncPublishingRegistry, + private array $errorChannels, + private ?string $globalErrorChannelName, + private ErrorChannelService $errorChannelService, + private ConfiguredMessagingSystem $configuredMessagingSystem, + ) { } public function await(MethodInvocation $methodInvocation): mixed @@ -27,7 +40,7 @@ public function await(MethodInvocation $methodInvocation): mixed $deliveryResult = $this->asyncPublishingRegistry->awaitAll(); if (! $deliveryResult->isSuccessful()) { - throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + $this->handleFailedDeliveries($deliveryResult->getFailedDeliveries()); } } finally { $this->asyncPublishingRegistry->closeScope(); @@ -35,4 +48,54 @@ public function await(MethodInvocation $methodInvocation): mixed return $result; } + + /** + * @param FailedDelivery[] $failedDeliveries + */ + private function handleFailedDeliveries(array $failedDeliveries): void + { + $unroutedFailedDeliveries = []; + foreach ($failedDeliveries as $failedDelivery) { + $errorChannelName = array_key_exists($failedDelivery->getChannelName(), $this->errorChannels) + ? $this->errorChannels[$failedDelivery->getChannelName()] + : $this->globalErrorChannelName; + + if ($errorChannelName === null) { + $unroutedFailedDeliveries[] = $failedDelivery; + + continue; + } + + foreach ($this->unpackFailedMessages($failedDelivery->getMessage()) as $failedMessage) { + $this->errorChannelService->handle( + $failedMessage, + AsyncPublishingFailedException::withFailedDeliveries([$failedDelivery]), + $this->configuredMessagingSystem->getMessageChannelByName($errorChannelName), + $failedDelivery->getChannelName(), + ); + } + } + + if ($unroutedFailedDeliveries !== []) { + throw AsyncPublishingFailedException::withFailedDeliveries($unroutedFailedDeliveries); + } + } + + /** + * @return Message[] + */ + private function unpackFailedMessages(Message $message): array + { + $payload = $message->getPayload(); + if (! $payload instanceof BatchMessage) { + return [$message]; + } + + return array_map( + fn (array $entry): Message => MessageBuilder::withPayload($entry['payload']) + ->setMultipleHeaders($entry['headers']) + ->build(), + $payload->getEntries(), + ); + } } diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishingModule.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishingModule.php index d0a8c1b19..55e089750 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishingModule.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishingModule.php @@ -9,13 +9,18 @@ use Ecotone\Messaging\Attribute\ModuleAnnotation; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingWaiterInterceptor; +use Ecotone\Messaging\Channel\PollableChannel\GlobalPollableChannelConfiguration; +use Ecotone\Messaging\Channel\PollableChannel\PollableChannelConfiguration; use Ecotone\Messaging\Config\Annotation\AnnotationModule; +use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\ExtensionObjectResolver; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\NoExternalConfigurationModule; use Ecotone\Messaging\Config\Configuration; +use Ecotone\Messaging\Config\ConfiguredMessagingSystem; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\Reference; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ModuleReferenceSearchService; +use Ecotone\Messaging\Handler\Gateway\ErrorChannelService; use Ecotone\Messaging\Handler\InterfaceToCallRegistry; use Ecotone\Messaging\Handler\Processor\MethodInvoker\AroundInterceptorBuilder; use Ecotone\Messaging\Precedence; @@ -34,9 +39,21 @@ public static function create(AnnotationFinder $annotationRegistrationService, I public function prepare(Configuration $messagingConfiguration, array $extensionObjects, ModuleReferenceSearchService $moduleReferenceSearchService, InterfaceToCallRegistry $interfaceToCallRegistry): void { + $globalPollableChannelConfiguration = ExtensionObjectResolver::resolveUnique(GlobalPollableChannelConfiguration::class, $extensionObjects, GlobalPollableChannelConfiguration::createWithDefaults()); + $errorChannels = []; + foreach (ExtensionObjectResolver::resolve(PollableChannelConfiguration::class, $extensionObjects) as $pollableChannelConfiguration) { + $errorChannels[$pollableChannelConfiguration->getChannelName()] = $pollableChannelConfiguration->getErrorChannelName(); + } + $messagingConfiguration->registerServiceDefinition( AsyncPublishingWaiterInterceptor::class, - new Definition(AsyncPublishingWaiterInterceptor::class, [new Reference(AsyncPublishingRegistry::class)]) + new Definition(AsyncPublishingWaiterInterceptor::class, [ + new Reference(AsyncPublishingRegistry::class), + $errorChannels, + $globalPollableChannelConfiguration->getErrorChannelName(), + new Reference(ErrorChannelService::class), + new Reference(ConfiguredMessagingSystem::class), + ]) ); $messagingConfiguration->registerAroundMethodInterceptor( AroundInterceptorBuilder::create( @@ -50,7 +67,8 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO public function canHandle($extensionObject): bool { - return false; + return $extensionObject instanceof PollableChannelConfiguration + || $extensionObject instanceof GlobalPollableChannelConfiguration; } public function getModulePackageName(): string diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/FailedDelivery.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/FailedDelivery.php index 831a6fb42..d62fdbbb0 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/FailedDelivery.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/FailedDelivery.php @@ -14,9 +14,15 @@ final class FailedDelivery public function __construct( private Message $message, private string $failureReason, + private string $channelName, ) { } + public function getChannelName(): string + { + return $this->channelName; + } + public function getMessage(): Message { return $this->message; diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/AsyncOrderForwarder.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/AsyncOrderForwarder.php new file mode 100644 index 000000000..6dadbde4e --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/AsyncOrderForwarder.php @@ -0,0 +1,27 @@ +operationsLog->log('consumer handler executed'); + $eventBus->publish(new OrderWasPlaced($event->order . '-forwarded')); + } +} diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/FakeTransactionInterceptor.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/FakeTransactionInterceptor.php index bce9f9cd0..01edb2d77 100644 --- a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/FakeTransactionInterceptor.php +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/FakeTransactionInterceptor.php @@ -12,12 +12,19 @@ */ final class FakeTransactionInterceptor { + private bool $transactionActive = false; + public function __construct(private OperationsLog $operationsLog) { } public function transactional(MethodInvocation $methodInvocation): mixed { + if ($this->transactionActive) { + return $methodInvocation->proceed(); + } + + $this->transactionActive = true; $this->operationsLog->log('transaction started'); try { $result = $methodInvocation->proceed(); @@ -28,6 +35,8 @@ public function transactional(MethodInvocation $methodInvocation): mixed $this->operationsLog->log('transaction rolled back'); throw $exception; + } finally { + $this->transactionActive = false; } } } diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php index 99c8e7248..fd853b35c 100644 --- a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php @@ -5,6 +5,7 @@ namespace Test\Ecotone\Messaging\Fixture\AsyncPublishing; use Ecotone\Messaging\BatchMessage; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\BatchSupportingMessageChannel; use Ecotone\Messaging\Endpoint\PollingMetadata; @@ -45,10 +46,18 @@ public function send(Message $message): void $this->queue[] = $message; } - $this->asyncPublishingRegistry->register( - $this->channelName, - new InMemoryPendingDelivery($message, $this->deliveryFailureReason, $this->operationsLog), - ); + $pendingDelivery = new InMemoryPendingDelivery($message, $this->deliveryFailureReason, $this->operationsLog, $this->channelName); + + if (! $this->asyncPublishingRegistry->isScopeActive()) { + $deliveryResult = $pendingDelivery->awaitDelivery(); + if (! $deliveryResult->isSuccessful()) { + throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + } + + return; + } + + $this->asyncPublishingRegistry->register($this->channelName, $pendingDelivery); } public function receive(): ?Message diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php index 510e7c8f6..7cbb8de4e 100644 --- a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php @@ -20,6 +20,7 @@ public function __construct( private Message $message, private ?string $failureReason = null, private ?OperationsLog $operationsLog = null, + private string $channelName = 'in_memory_channel', ) { } @@ -30,7 +31,7 @@ public function awaitDelivery(): DeliveryResult if ($this->failureReason !== null) { return DeliveryResult::withFailedDeliveries([ - new FailedDelivery($this->message, $this->failureReason), + new FailedDelivery($this->message, $this->failureReason, $this->channelName), ]); } diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OrderRequestReceived.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OrderRequestReceived.php new file mode 100644 index 000000000..8d576a417 --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/OrderRequestReceived.php @@ -0,0 +1,15 @@ +publish(new OrderWasPlaced($order . '-1')); $eventBus->publish(new OrderWasPlaced($order . '-2')); } + + #[CommandHandler('order.forward')] + public function forwardOrder(string $order, CommandBus $commandBus): void + { + $this->operationsLog->log('forwarding command handler executed'); + $commandBus->sendWithRouting('order.place', $order); + } } diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingScenariosTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingScenariosTest.php new file mode 100644 index 000000000..ad7299621 --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingScenariosTest.php @@ -0,0 +1,147 @@ +bootstrapEcotone($operationsLog); + + $ecotoneLite->sendCommandWithRoutingKey('order.forward', 'espresso'); + + $this->assertSame( + [ + 'transaction started', + 'forwarding command handler executed', + 'command handler executed', + 'published batch of 2 messages to broker', + 'delivery confirmations awaited', + 'transaction committed', + ], + $operationsLog->getOperations(), + ); + } + + public function test_polling_consumer_awaits_deliveries_before_acknowledging_inbound_message(): void + { + $operationsLog = new OperationsLog(); + $ecotoneLite = $this->bootstrapEcotone($operationsLog); + + $ecotoneLite->publishEvent(new OrderRequestReceived('espresso')); + $this->assertSame([], $operationsLog->getOperations()); + + $ecotoneLite->run('incoming_orders', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertSame( + [ + 'transaction started', + 'consumer handler executed', + 'published batch of 1 messages to broker', + 'delivery confirmations awaited', + 'transaction committed', + ], + $operationsLog->getOperations(), + ); + $this->assertEquals( + new OrderWasPlaced('espresso-forwarded'), + $ecotoneLite->receiveMessageFrom('async_orders')->getPayload(), + ); + } + + public function test_bus_driven_sends_outside_any_scope_fall_back_to_synchronous_awaiting(): void + { + $operationsLog = new OperationsLog(); + $ecotoneLite = $this->bootstrapEcotone($operationsLog); + + $ecotoneLite->publishEvent(new OrderWasPlaced('espresso-1')); + + $this->assertSame( + [ + 'published message to broker', + 'delivery confirmations awaited', + ], + $operationsLog->getOperations(), + ); + $this->assertEquals( + new OrderWasPlaced('espresso-1'), + $ecotoneLite->receiveMessageFrom('async_orders')->getPayload(), + ); + } + + public function test_failed_deliveries_are_routed_to_error_channel_and_transaction_commits(): void + { + $operationsLog = new OperationsLog(); + $ecotoneLite = EcotoneLite::bootstrapFlowTesting( + [OrderService::class, AsyncOrderSubscriber::class, FakeTransactionModule::class], + [new OrderService($operationsLog), new AsyncOrderSubscriber(), OperationsLog::class => $operationsLog], + ServiceConfiguration::createWithDefaults()->withExtensionObjects([ + GlobalPollableChannelConfiguration::createWithDefaults()->withErrorChannel('failure_channel'), + PollableChannelConfiguration::neverRetry('async_orders')->withCollector(false)->withErrorChannel('failure_channel'), + ]), + enableAsynchronousProcessing: [ + InMemoryAsyncPublishingChannelBuilder::create('async_orders'), + SimpleMessageChannelBuilder::createQueueChannel('failure_channel'), + ], + ); + $channel = $ecotoneLite->getMessageChannel('async_orders'); + assert($channel instanceof MessageChannelInterceptorAdapter); + $channel->getInternalMessageChannel()->failDeliveriesWith('broker not available'); + + $ecotoneLite->sendCommandWithRoutingKey('order.place', 'espresso'); + + $this->assertSame('transaction committed', $operationsLog->getOperations()[count($operationsLog->getOperations()) - 1]); + + $firstFailedMessage = $ecotoneLite->receiveMessageFrom('failure_channel'); + $secondFailedMessage = $ecotoneLite->receiveMessageFrom('failure_channel'); + $this->assertStringContainsString('espresso-1', $firstFailedMessage->getPayload()); + $this->assertStringContainsString('espresso-2', $secondFailedMessage->getPayload()); + $this->assertStringContainsString(OrderWasPlaced::class, $firstFailedMessage->getHeaders()->get(MessageHeaders::TYPE_ID)); + $this->assertStringContainsString('broker not available', $firstFailedMessage->getHeaders()->get(ErrorContext::EXCEPTION_MESSAGE)); + $this->assertNull($ecotoneLite->receiveMessageFrom('failure_channel')); + } + + private function bootstrapEcotone(OperationsLog $operationsLog): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + [OrderService::class, AsyncOrderSubscriber::class, AsyncOrderForwarder::class, FakeTransactionModule::class], + [ + new OrderService($operationsLog), + new AsyncOrderSubscriber(), + new AsyncOrderForwarder($operationsLog), + OperationsLog::class => $operationsLog, + ], + enableAsynchronousProcessing: [ + InMemoryAsyncPublishingChannelBuilder::create('async_orders'), + SimpleMessageChannelBuilder::createQueueChannel('incoming_orders'), + ], + ); + } +} From 3459b7af6bd43d52a29f0fe23825ef33459fe49a Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 07/38] feat: Kafka async publishing with delivery reports and deferred flush --- ...utboundSerializationChannelInterceptor.php | 5 + .../Kafka/src/Channel/KafkaMessageChannel.php | 8 +- .../Channel/KafkaMessageChannelBuilder.php | 22 ++++ .../Kafka/src/Configuration/KafkaAdmin.php | 19 +++ .../Kafka/src/Configuration/KafkaModule.php | 3 +- .../KafkaPublisherConfiguration.php | 26 ++++ .../src/Outbound/KafkaDeliveryTracker.php | 67 ++++++++++ .../Outbound/KafkaOutboundChannelAdapter.php | 92 +++++++++++++- .../KafkaOutboundChannelAdapterBuilder.php | 2 + .../src/Outbound/KafkaPendingDelivery.php | 42 +++++++ .../tests/Integration/AsyncPublishingTest.php | 119 ++++++++++++++++++ 11 files changed, 401 insertions(+), 4 deletions(-) create mode 100644 packages/Kafka/src/Outbound/KafkaDeliveryTracker.php create mode 100644 packages/Kafka/src/Outbound/KafkaPendingDelivery.php create mode 100644 packages/Kafka/tests/Integration/AsyncPublishingTest.php diff --git a/packages/Ecotone/src/Messaging/Channel/PollableChannel/Serialization/OutboundSerializationChannelInterceptor.php b/packages/Ecotone/src/Messaging/Channel/PollableChannel/Serialization/OutboundSerializationChannelInterceptor.php index d2be53928..b43612507 100644 --- a/packages/Ecotone/src/Messaging/Channel/PollableChannel/Serialization/OutboundSerializationChannelInterceptor.php +++ b/packages/Ecotone/src/Messaging/Channel/PollableChannel/Serialization/OutboundSerializationChannelInterceptor.php @@ -4,6 +4,7 @@ namespace Ecotone\Messaging\Channel\PollableChannel\Serialization; +use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AbstractChannelInterceptor; use Ecotone\Messaging\Conversion\ConversionService; use Ecotone\Messaging\Message; @@ -32,6 +33,10 @@ public function preSend(Message $message, MessageChannel $messageChannel): ?Mess return $message; } + if ($message->getPayload() instanceof BatchMessage) { + return $message; + } + $outboundMessage = $this->outboundMessageConverter->prepare($message, $this->conversionService); $preparedMessage = MessageBuilder::withPayload($outboundMessage->getPayload()) ->setMultipleHeaders($outboundMessage->getHeaders()); diff --git a/packages/Kafka/src/Channel/KafkaMessageChannel.php b/packages/Kafka/src/Channel/KafkaMessageChannel.php index f94213407..99a5bca78 100644 --- a/packages/Kafka/src/Channel/KafkaMessageChannel.php +++ b/packages/Kafka/src/Channel/KafkaMessageChannel.php @@ -7,6 +7,7 @@ use Ecotone\Kafka\Configuration\KafkaConsumerConfiguration; use Ecotone\Kafka\Inbound\KafkaInboundChannelAdapter; use Ecotone\Kafka\Outbound\KafkaOutboundChannelAdapter; +use Ecotone\Messaging\Channel\BatchSupportingMessageChannel; use Ecotone\Messaging\Endpoint\PollingMetadata; use Ecotone\Messaging\Message; use Ecotone\Messaging\PollableChannel; @@ -14,7 +15,7 @@ /** * licence Enterprise */ -final class KafkaMessageChannel implements PollableChannel +final class KafkaMessageChannel implements PollableChannel, BatchSupportingMessageChannel { public function __construct( private KafkaInboundChannelAdapter $inboundChannelAdapter, @@ -23,6 +24,11 @@ public function __construct( } + public function supportsBatchMessages(): bool + { + return $this->outboundChannelAdapter->isAsyncPublishingEnabled(); + } + public function send(Message $message): void { $this->outboundChannelAdapter->handle($message); diff --git a/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php b/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php index 2e74a3a8d..838fa9a97 100644 --- a/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php +++ b/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php @@ -25,6 +25,8 @@ final class KafkaMessageChannelBuilder implements MessageChannelWithSerializatio private KafkaOutboundChannelAdapterBuilder $outboundChannelAdapterBuilder; private string $headerMapper; private ?MediaType $conversionMediaType = null; + private bool $asyncPublishing = false; + private ?int $asyncPublishingTimeout = null; private function __construct( private string $channelName, @@ -116,6 +118,26 @@ public function withDefaultConversionMediaType(string $mediaType): self return $this; } + public function withAsyncPublishing(bool $enabled = true, ?int $timeoutInMilliseconds = null): self + { + $this->asyncPublishing = $enabled; + if ($timeoutInMilliseconds !== null) { + $this->asyncPublishingTimeout = $timeoutInMilliseconds; + } + + return $this; + } + + public function isAsyncPublishingEnabled(): bool + { + return $this->asyncPublishing; + } + + public function getAsyncPublishingTimeout(): ?int + { + return $this->asyncPublishingTimeout; + } + /** * Set the commit interval in messages. Offsets will be committed every X messages. * diff --git a/packages/Kafka/src/Configuration/KafkaAdmin.php b/packages/Kafka/src/Configuration/KafkaAdmin.php index d2a8537ae..61d8502ad 100644 --- a/packages/Kafka/src/Configuration/KafkaAdmin.php +++ b/packages/Kafka/src/Configuration/KafkaAdmin.php @@ -5,6 +5,7 @@ namespace Ecotone\Kafka\Configuration; use Ecotone\Kafka\Attribute\KafkaConsumer as KafkaConsumerAttribute; +use Ecotone\Kafka\Outbound\KafkaDeliveryTracker; use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Exception; @@ -28,6 +29,11 @@ final class KafkaAdmin */ private array $initializedConsumers = []; + /** + * @var KafkaDeliveryTracker[] + */ + private array $deliveryTrackers = []; + /** * @param KafkaConsumerAttribute[] $consumerConfigurations * @param KafkaConsumerConfiguration[] $rdKafkaConsumerConfigurations @@ -121,6 +127,14 @@ public function getProducer(string $referenceName): Producer $conf = $configuration->getAsKafkaConfig(); $conf->set('metadata.broker.list', implode(',', $this->kafkaBrokerConfigurations[$configuration->getBrokerConfigurationReference()]->getBootstrapServers())); $this->setLoggerCallbacks($conf, $referenceName); + if ($configuration->isAsyncPublishingEnabled()) { + $deliveryTracker = $this->getDeliveryTracker($referenceName); + $conf->setDrMsgCb( + function ($producer, $kafkaMessage) use ($deliveryTracker): void { + $deliveryTracker->recordDeliveryReport($kafkaMessage); + } + ); + } $producer = new Producer($conf); $producer->addBrokers(implode(',', $this->kafkaBrokerConfigurations[$configuration->getBrokerConfigurationReference()]->getBootstrapServers())); @@ -130,6 +144,11 @@ public function getProducer(string $referenceName): Producer return $this->initializedProducers[$referenceName]; } + public function getDeliveryTracker(string $referenceName): KafkaDeliveryTracker + { + return $this->deliveryTrackers[$referenceName] ??= new KafkaDeliveryTracker(); + } + public function getTopicForProducer(string $referenceName): ProducerTopic { $producer = $this->getProducer($referenceName); diff --git a/packages/Kafka/src/Configuration/KafkaModule.php b/packages/Kafka/src/Configuration/KafkaModule.php index 5bfd31f5d..b95e9ad3c 100644 --- a/packages/Kafka/src/Configuration/KafkaModule.php +++ b/packages/Kafka/src/Configuration/KafkaModule.php @@ -111,7 +111,8 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO $extensionObject->topicName, MessagePublisher::class . '::' . $extensionObject->getMessageChannelName(), ) - ->withHeaderMapper($extensionObject->getHeaderMapper()); + ->withHeaderMapper($extensionObject->getHeaderMapper()) + ->withAsyncPublishing($extensionObject->isAsyncPublishingEnabled(), $extensionObject->getAsyncPublishingTimeout()); } } diff --git a/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php b/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php index f0edc688a..9e164fe47 100644 --- a/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php +++ b/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php @@ -20,6 +20,8 @@ final class KafkaPublisherConfiguration implements DefinedObject { public const ACKNOWLEDGE_TIMEOUT = '8000'; + public const DEFAULT_ASYNC_PUBLISHING_TIMEOUT = 12000; + /** * @param array $configuration */ @@ -30,6 +32,8 @@ public function __construct( private string $brokerConfigurationReference, private HeaderMapper $headerMapper, private ?string $outputDefaultConversionMediaType = null, + private bool $asyncPublishing = false, + private int $asyncPublishingTimeout = self::DEFAULT_ASYNC_PUBLISHING_TIMEOUT, ) { } @@ -99,6 +103,26 @@ public function getHeaderMapper(): HeaderMapper return $this->headerMapper; } + public function withAsyncPublishing(bool $asyncPublishing = true, ?int $timeoutInMilliseconds = null): self + { + $this->asyncPublishing = $asyncPublishing; + if ($timeoutInMilliseconds !== null) { + $this->asyncPublishingTimeout = $timeoutInMilliseconds; + } + + return $this; + } + + public function isAsyncPublishingEnabled(): bool + { + return $this->asyncPublishing; + } + + public function getAsyncPublishingTimeout(): int + { + return $this->asyncPublishingTimeout; + } + public function getOutputDefaultConversionMediaType(): ?string { return $this->outputDefaultConversionMediaType; @@ -138,6 +162,8 @@ public function getDefinition(): Definition $this->brokerConfigurationReference, $this->headerMapper->getDefinition(), $this->outputDefaultConversionMediaType, + $this->asyncPublishing, + $this->asyncPublishingTimeout, ]); } } diff --git a/packages/Kafka/src/Outbound/KafkaDeliveryTracker.php b/packages/Kafka/src/Outbound/KafkaDeliveryTracker.php new file mode 100644 index 000000000..90ac2e7c7 --- /dev/null +++ b/packages/Kafka/src/Outbound/KafkaDeliveryTracker.php @@ -0,0 +1,67 @@ + */ + private array $inFlightMessages = []; + + /** @var array */ + private array $deliveryFailures = []; + + private int $nextDeliveryId = 0; + + public function trackInFlight(Message $message): string + { + $deliveryId = (string) $this->nextDeliveryId++; + $this->inFlightMessages[$deliveryId] = $message; + + return $deliveryId; + } + + public function recordDeliveryReport(KafkaMessage $kafkaMessage): void + { + $deliveryId = $kafkaMessage->opaque; + if (! is_string($deliveryId) || ! array_key_exists($deliveryId, $this->inFlightMessages)) { + return; + } + + if ($kafkaMessage->err !== RD_KAFKA_RESP_ERR_NO_ERROR) { + $this->deliveryFailures[$deliveryId] = rd_kafka_err2str($kafkaMessage->err); + + return; + } + + unset($this->inFlightMessages[$deliveryId]); + } + + /** + * @param string[] $deliveryIds + */ + public function collectResult(array $deliveryIds, string $channelName): DeliveryResult + { + $failedDeliveries = []; + foreach ($deliveryIds as $deliveryId) { + if (array_key_exists($deliveryId, $this->deliveryFailures)) { + $failedDeliveries[] = new FailedDelivery($this->inFlightMessages[$deliveryId], $this->deliveryFailures[$deliveryId], $channelName); + } elseif (array_key_exists($deliveryId, $this->inFlightMessages)) { + $failedDeliveries[] = new FailedDelivery($this->inFlightMessages[$deliveryId], 'Timed out awaiting delivery confirmation from Kafka broker', $channelName); + } + + unset($this->inFlightMessages[$deliveryId], $this->deliveryFailures[$deliveryId]); + } + + return $failedDeliveries === [] ? DeliveryResult::successful() : DeliveryResult::withFailedDeliveries($failedDeliveries); + } +} diff --git a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php index 2630a3ed8..f90e71c1f 100644 --- a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php +++ b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php @@ -7,13 +7,18 @@ use Ecotone\Kafka\Api\KafkaHeader; use Ecotone\Kafka\Configuration\KafkaAdmin; use Ecotone\Kafka\Configuration\KafkaPublisherConfiguration; +use Ecotone\Messaging\BatchMessage; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Conversion\ConversionService; use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageHandler; use Ecotone\Messaging\MessageHeaders; +use Ecotone\Messaging\Support\MessageBuilder; use Ecotone\Modelling\AggregateFlow\AggregateIdMetadata; use Ecotone\Modelling\AggregateMessage; +use RdKafka\Producer; +use RdKafka\ProducerTopic; /** * licence Enterprise @@ -24,7 +29,8 @@ public function __construct( private string $referenceName, private KafkaAdmin $kafkaAdmin, private ConversionService $conversionService, - private OutboundMessageConverter $outboundMessageConverter + private OutboundMessageConverter $outboundMessageConverter, + private AsyncPublishingRegistry $asyncPublishingRegistry, ) { } @@ -35,6 +41,55 @@ public function handle(Message $message): void { $producer = $this->kafkaAdmin->getProducer($this->referenceName); $topic = $this->kafkaAdmin->getTopicForProducer($this->referenceName); + + if ($message->getPayload() instanceof BatchMessage) { + $this->handleBatch($message->getPayload(), $producer, $topic); + + return; + } + + if ($this->canPublishAsynchronously()) { + $deliveryId = $this->produce($message, $topic, trackDelivery: true); + $producer->poll(0); + $this->registerPendingDelivery($producer, [$deliveryId]); + + return; + } + + $this->produce($message, $topic, trackDelivery: false); + $this->flushSynchronously($producer); + } + + public function isAsyncPublishingEnabled(): bool + { + return $this->kafkaAdmin->getConfigurationForPublisher($this->referenceName)->isAsyncPublishingEnabled(); + } + + private function handleBatch(BatchMessage $batchMessage, Producer $producer, ProducerTopic $topic): void + { + $publishAsynchronously = $this->canPublishAsynchronously(); + + $deliveryIds = []; + foreach ($batchMessage->getEntries() as $entry) { + $entryMessage = MessageBuilder::withPayload($entry['payload']) + ->setMultipleHeaders($entry['headers']) + ->build(); + + $deliveryIds[] = $this->produce($entryMessage, $topic, trackDelivery: $publishAsynchronously); + $producer->poll(0); + } + + if ($publishAsynchronously) { + $this->registerPendingDelivery($producer, $deliveryIds); + + return; + } + + $this->flushSynchronously($producer); + } + + private function produce(Message $message, ProducerTopic $topic, bool $trackDelivery): ?string + { $outboundMessage = $this->outboundMessageConverter->prepare($message, $this->conversionService); if ($message->getHeaders()->containsKey(KafkaHeader::KAFKA_TARGET_PARTITION_KEY_HEADER_NAME)) { @@ -50,6 +105,10 @@ public function handle(Message $message): void $headers = $outboundMessage->getHeaders(); unset($headers[KafkaHeader::KAFKA_TARGET_PARTITION_KEY_HEADER_NAME]); + $deliveryId = $trackDelivery + ? $this->kafkaAdmin->getDeliveryTracker($this->referenceName)->trackInFlight($message) + : null; + $topic->producev( RD_KAFKA_PARTITION_UA, 0, @@ -60,9 +119,38 @@ public function handle(Message $message): void [ KafkaHeader::KAFKA_SOURCE_PARTITION_KEY_HEADER_NAME => $partitionKey, ] - ) + ), + null, + $deliveryId, + ); + + return $deliveryId; + } + + private function canPublishAsynchronously(): bool + { + return $this->isAsyncPublishingEnabled() && $this->asyncPublishingRegistry->isScopeActive(); + } + + /** + * @param string[] $deliveryIds + */ + private function registerPendingDelivery(Producer $producer, array $deliveryIds): void + { + $this->asyncPublishingRegistry->register( + $this->referenceName, + new KafkaPendingDelivery( + $producer, + $this->kafkaAdmin->getDeliveryTracker($this->referenceName), + $deliveryIds, + $this->kafkaAdmin->getConfigurationForPublisher($this->referenceName)->getAsyncPublishingTimeout(), + $this->referenceName, + ), ); + } + private function flushSynchronously(Producer $producer): void + { /** * Producer won't produce the message to the broker immediately it will wait until the producer queue (queue.buffering.max.messages)gets full or size of the queue(queue.buffering.max.kbytes). * calling flush immediately after produce will publish all messages to the broker irrespective of these two config values. diff --git a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapterBuilder.php b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapterBuilder.php index 3e1a6dd62..1c0497bae 100644 --- a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapterBuilder.php +++ b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapterBuilder.php @@ -5,6 +5,7 @@ namespace Ecotone\Kafka\Outbound; use Ecotone\Kafka\Configuration\KafkaAdmin; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\MessagingContainerBuilder; @@ -99,6 +100,7 @@ public function compile(MessagingContainerBuilder $builder): Definition new Reference(KafkaAdmin::class), new Reference(ConversionService::REFERENCE_NAME), $outboundMessageConverter, + new Reference(AsyncPublishingRegistry::class), ]); } diff --git a/packages/Kafka/src/Outbound/KafkaPendingDelivery.php b/packages/Kafka/src/Outbound/KafkaPendingDelivery.php new file mode 100644 index 000000000..d274114eb --- /dev/null +++ b/packages/Kafka/src/Outbound/KafkaPendingDelivery.php @@ -0,0 +1,42 @@ +awaited = true; + $this->producer->flush($this->timeoutInMilliseconds); + + return $this->deliveryTracker->collectResult($this->deliveryIds, $this->channelName); + } + + public function isAwaited(): bool + { + return $this->awaited; + } +} diff --git a/packages/Kafka/tests/Integration/AsyncPublishingTest.php b/packages/Kafka/tests/Integration/AsyncPublishingTest.php new file mode 100644 index 000000000..7459da730 --- /dev/null +++ b/packages/Kafka/tests/Integration/AsyncPublishingTest.php @@ -0,0 +1,119 @@ +createOrderService($channelName); + $messaging = $this->bootstrapEcotone($channelName, $orderService, ConnectionTestCase::getConnection()); + + $messaging->sendCommandWithRoutingKey('order.place', 'espresso'); + + $this->assertSame([], $messaging->sendQueryWithRouting('order.getReceived')); + + $messaging->run($channelName, ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 3, maxExecutionTimeInMilliseconds: 10000)); + + $this->assertCount(3, $messaging->sendQueryWithRouting('order.getReceived')); + } + + public function test_failing_to_deliver_asynchronously_published_messages_throws(): void + { + $channelName = 'async_orders'; + $orderService = $this->createOrderService($channelName); + $messaging = $this->bootstrapEcotone( + $channelName, + $orderService, + KafkaBrokerConfiguration::createWithDefaults(['wronghost:9092']), + asyncPublishingTimeout: 500, + ); + + $this->expectException(AsyncPublishingFailedException::class); + + $messaging->sendCommandWithRoutingKey('order.place', 'espresso'); + } + + private function createOrderService(string $channelName): object + { + return new class ($channelName) { + /** @var string[] */ + private array $receivedEvents = []; + + public function __construct(private string $channelName) + { + } + + #[CommandHandler('order.place')] + public function placeOrder(string $order, EventBus $eventBus): void + { + $eventBus->publish(new ExampleEvent($order . '-1')); + $eventBus->publish(new ExampleEvent($order . '-2')); + $eventBus->publish(new ExampleEvent($order . '-3')); + } + + #[Asynchronous('async_orders')] + #[EventHandler(endpointId: 'async_order_collector')] + public function collect(ExampleEvent $event): void + { + $this->receivedEvents[] = $event->id; + } + + #[QueryHandler('order.getReceived')] + public function getReceived(): array + { + return $this->receivedEvents; + } + }; + } + + private function bootstrapEcotone(string $channelName, object $orderService, KafkaBrokerConfiguration $brokerConfiguration, ?int $asyncPublishingTimeout = null): FlowTestSupport + { + $channelBuilder = KafkaMessageChannelBuilder::create( + $channelName, + topicName: $uniqueId = Uuid::v7()->toRfc4122(), + messageGroupId: $uniqueId, + )->withAsyncPublishing(); + + if ($asyncPublishingTimeout !== null) { + $channelBuilder = $channelBuilder->withAsyncPublishing(timeoutInMilliseconds: $asyncPublishingTimeout); + } + + return EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [KafkaBrokerConfiguration::class => $brokerConfiguration, $orderService], + ServiceConfiguration::createWithAsynchronicityOnly() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::KAFKA_PACKAGE])) + ->withExtensionObjects([$channelBuilder]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } +} From edcf893325e8796cceaa3d7323d95d7cf93214f1 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 08/38] feat: AMQP async publishing with batched publisher confirms --- .../src/AmqpBackedMessageChannelBuilder.php | 12 +++ .../Amqp/src/AmqpOutboundChannelAdapter.php | 98 +++++++++++++++-- .../src/AmqpOutboundChannelAdapterBuilder.php | 33 ++++++ packages/Amqp/src/AmqpPendingDelivery.php | 59 ++++++++++ .../AsyncPublishing/OrderWasPlaced.php | 15 +++ .../tests/Integration/AsyncPublishingTest.php | 102 ++++++++++++++++++ .../Enqueue/src/EnqueueMessageChannel.php | 10 +- .../src/EnqueueMessageChannelBuilder.php | 6 ++ 8 files changed, 325 insertions(+), 10 deletions(-) create mode 100644 packages/Amqp/src/AmqpPendingDelivery.php create mode 100644 packages/Amqp/tests/Fixture/AsyncPublishing/OrderWasPlaced.php create mode 100644 packages/Amqp/tests/Integration/AsyncPublishingTest.php diff --git a/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php b/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php index ec7a392e7..710b81f5b 100644 --- a/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php +++ b/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php @@ -71,6 +71,13 @@ public function withPublisherConfirms(bool $enabled): self return $this; } + public function withAsyncPublishing(bool $enabled = true, ?int $timeoutInMilliseconds = null): self + { + $this->getAmqpOutboundChannelAdapter()->withAsyncPublishing($enabled, $timeoutInMilliseconds); + + return $this; + } + public function withDelayStrategy(string $delayStrategyReferenceName): self { $this->getAmqpOutboundChannelAdapter()->withDelayStrategy($delayStrategyReferenceName); @@ -83,6 +90,11 @@ public function getMessageChannelName(): string return $this->channelName; } + protected function supportsBatchMessages(): bool + { + return $this->getAmqpOutboundChannelAdapter()->isAsyncPublishingEnabled(); + } + public function getQueueName() { return $this->getInboundChannelAdapter()->getMessageChannelName(); diff --git a/packages/Amqp/src/AmqpOutboundChannelAdapter.php b/packages/Amqp/src/AmqpOutboundChannelAdapter.php index 0639fa791..e2b68dff8 100644 --- a/packages/Amqp/src/AmqpOutboundChannelAdapter.php +++ b/packages/Amqp/src/AmqpOutboundChannelAdapter.php @@ -6,11 +6,14 @@ use Ecotone\Amqp\Transaction\AmqpTransactionInterceptor; use Ecotone\Enqueue\CachedConnectionFactory; +use Ecotone\Messaging\BatchMessage; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Conversion\ConversionService; use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageHandler; use Ecotone\Messaging\Support\Assert; +use Ecotone\Messaging\Support\MessageBuilder; use Enqueue\AmqpExt\AmqpContext as AmqpExtContext; use Enqueue\AmqpLib\AmqpContext as AmqpLibContext; use Enqueue\AmqpTools\DelayStrategy; @@ -44,7 +47,11 @@ public function __construct( private OutboundMessageConverter $outboundMessageConverter, private ConversionService $conversionService, private AmqpTransactionInterceptor $amqpTransactionInterceptor, - private ?DelayStrategy $delayStrategy = null + private ?DelayStrategy $delayStrategy = null, + private ?AsyncPublishingRegistry $asyncPublishingRegistry = null, + private bool $asyncPublishing = false, + private int $asyncPublishingTimeout = AmqpOutboundChannelAdapterBuilder::DEFAULT_ASYNC_PUBLISHING_TIMEOUT, + private string $channelName = '', ) { } @@ -52,6 +59,51 @@ public function __construct( * @inheritDoc */ public function handle(Message $message): void + { + if ($message->getPayload() instanceof BatchMessage) { + $this->handleBatch($message->getPayload(), $message); + + return; + } + + $this->publish($message); + + if ($this->canPublishAsynchronously()) { + $this->registerPendingDelivery([$message]); + + return; + } + + $this->awaitPublisherConfirmsSynchronously(); + } + + public function isAsyncPublishingEnabled(): bool + { + return $this->asyncPublishing; + } + + private function handleBatch(BatchMessage $batchMessage, Message $carrierMessage): void + { + $publishedMessages = []; + foreach ($batchMessage->getEntries() as $entry) { + $entryMessage = MessageBuilder::withPayload($entry['payload']) + ->setMultipleHeaders($entry['headers']) + ->build(); + + $this->publish($entryMessage); + $publishedMessages[] = $entryMessage; + } + + if ($this->canPublishAsynchronously()) { + $this->registerPendingDelivery($publishedMessages); + + return; + } + + $this->awaitPublisherConfirmsSynchronously(); + } + + private function publish(Message $message): void { $exchangeName = $this->exchangeName; if ($this->exchangeFromHeaderName) { @@ -86,20 +138,50 @@ public function handle(Message $message): void Assert::isFalse($this->amqpTransactionInterceptor->isRunningInTransaction(), 'Cannot use publisher acknowledgments together with transactions. Please disable one of them.'); } - $context = $this->connectionFactory->createContext(); + $this->connectionFactory->createContext(); $this->connectionFactory->getProducer() ->setTimeToLive($outboundMessage->getTimeToLive()) ->setDelayStrategy($this->delayStrategy ?? new HeadersExchangeDelayStrategy()) ->setDeliveryDelay($outboundMessage->getDeliveryDelay()) // this allow for having queue per delay instead of queue per delay + exchangeName ->send(new AmqpTopic($exchangeName), $messageToSend); + } + + private function canPublishAsynchronously(): bool + { + return $this->asyncPublishing + && $this->publisherConfirms + && $this->asyncPublishingRegistry !== null + && $this->asyncPublishingRegistry->isScopeActive(); + } - if ($this->publisherConfirms && ! $this->amqpTransactionInterceptor->isRunningInTransaction()) { - if ($context instanceof AmqpLibContext) { - $context->getLibChannel()->wait_for_pending_acks(5); - } elseif ($context instanceof AmqpExtContext) { - $context->getExtChannel()->waitForConfirm(5); - } + /** + * @param Message[] $publishedMessages + */ + private function registerPendingDelivery(array $publishedMessages): void + { + $this->asyncPublishingRegistry->register( + $this->channelName, + new AmqpPendingDelivery( + $this->connectionFactory->createContext(), + $publishedMessages, + $this->asyncPublishingTimeout, + $this->channelName, + ), + ); + } + + private function awaitPublisherConfirmsSynchronously(): void + { + if (! $this->publisherConfirms || $this->amqpTransactionInterceptor->isRunningInTransaction()) { + return; + } + + $context = $this->connectionFactory->createContext(); + if ($context instanceof AmqpLibContext) { + $context->getLibChannel()->wait_for_pending_acks(5); + } elseif ($context instanceof AmqpExtContext) { + $context->getExtChannel()->waitForConfirm(5); } } } diff --git a/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php b/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php index fe7020749..f50064c87 100644 --- a/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php +++ b/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php @@ -7,11 +7,14 @@ use Ecotone\Amqp\Transaction\AmqpTransactionInterceptor; use Ecotone\Enqueue\CachedConnectionFactory; use Ecotone\Enqueue\EnqueueOutboundChannelAdapterBuilder; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\MessagingContainerBuilder; use Ecotone\Messaging\Config\Container\Reference; use Ecotone\Messaging\Conversion\ConversionService; +use Ecotone\Messaging\Support\Assert; +use Ecotone\Messaging\Support\LicensingException; /** * licence Apache-2.0 @@ -20,6 +23,8 @@ class AmqpOutboundChannelAdapterBuilder extends EnqueueOutboundChannelAdapterBui { private const DEFAULT_PERSISTENT_MODE = true; + public const DEFAULT_ASYNC_PUBLISHING_TIMEOUT = 5000; + private string $amqpConnectionFactoryReferenceName; private string $defaultRoutingKey = ''; private ?string $routingKeyFromHeader = null; @@ -29,6 +34,8 @@ class AmqpOutboundChannelAdapterBuilder extends EnqueueOutboundChannelAdapterBui private array $staticHeadersToAdd = []; private bool $publisherConfirms = true; private ?string $delayStrategyReferenceName = null; + private bool $asyncPublishing = false; + private int $asyncPublishingTimeout = self::DEFAULT_ASYNC_PUBLISHING_TIMEOUT; private function __construct(string $exchangeName, string $amqpConnectionFactoryReferenceName) { @@ -66,6 +73,21 @@ public function withPublisherConfirms(bool $publisherConfirms): self return $this; } + public function withAsyncPublishing(bool $enabled = true, ?int $timeoutInMilliseconds = null): self + { + $this->asyncPublishing = $enabled; + if ($timeoutInMilliseconds !== null) { + $this->asyncPublishingTimeout = $timeoutInMilliseconds; + } + + return $this; + } + + public function isAsyncPublishingEnabled(): bool + { + return $this->asyncPublishing; + } + public function withDelayStrategy(string $delayStrategyReferenceName): self { $this->delayStrategyReferenceName = $delayStrategyReferenceName; @@ -118,6 +140,13 @@ public function withDefaultPersistentMode(bool $isPersistent): self public function compile(MessagingContainerBuilder $builder): Definition { + if ($this->asyncPublishing) { + if (! $builder->getServiceConfiguration()->isRunningForEnterprise()) { + throw LicensingException::create('Asynchronous publishing is available only with Ecotone Enterprise licence.'); + } + Assert::isTrue($this->publisherConfirms, 'Asynchronous publishing requires publisher confirms to be enabled.'); + } + $connectionFactory = new Definition(CachedConnectionFactory::class, [ new Definition(AmqpReconnectableConnectionFactory::class, [ new Reference($this->amqpConnectionFactoryReferenceName), @@ -149,6 +178,10 @@ public function compile(MessagingContainerBuilder $builder): Definition new Reference(ConversionService::REFERENCE_NAME), Reference::to(AmqpTransactionInterceptor::class), $this->delayStrategyReferenceName ? new Reference($this->delayStrategyReferenceName) : null, + new Reference(AsyncPublishingRegistry::class), + $this->asyncPublishing, + $this->asyncPublishingTimeout, + $this->exchangeName, ]); } } diff --git a/packages/Amqp/src/AmqpPendingDelivery.php b/packages/Amqp/src/AmqpPendingDelivery.php new file mode 100644 index 000000000..025e333fc --- /dev/null +++ b/packages/Amqp/src/AmqpPendingDelivery.php @@ -0,0 +1,59 @@ +awaited = true; + $timeoutInSeconds = $this->timeoutInMilliseconds / 1000; + + try { + if ($this->context instanceof AmqpLibContext) { + $this->context->getLibChannel()->wait_for_pending_acks($timeoutInSeconds); + } elseif ($this->context instanceof AmqpExtContext) { + $this->context->getExtChannel()->waitForConfirm($timeoutInSeconds); + } + } catch (Throwable $exception) { + return DeliveryResult::withFailedDeliveries(array_map( + fn (Message $message) => new FailedDelivery($message, $exception->getMessage(), $this->channelName), + $this->trackedMessages, + )); + } + + return DeliveryResult::successful(); + } + + public function isAwaited(): bool + { + return $this->awaited; + } +} diff --git a/packages/Amqp/tests/Fixture/AsyncPublishing/OrderWasPlaced.php b/packages/Amqp/tests/Fixture/AsyncPublishing/OrderWasPlaced.php new file mode 100644 index 000000000..1787fab3b --- /dev/null +++ b/packages/Amqp/tests/Fixture/AsyncPublishing/OrderWasPlaced.php @@ -0,0 +1,15 @@ +toRfc4122(); + $orderService = $this->createOrderService($channelName); + $messaging = $this->bootstrapEcotone($channelName, $orderService, LicenceTesting::VALID_LICENCE); + + $messaging->sendCommandWithRoutingKey('order.place', 'espresso'); + + $this->assertSame([], $messaging->sendQueryWithRouting('order.getReceived')); + + $messaging->run('asyncOrdersChannel', ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 3, maxExecutionTimeInMilliseconds: 10000)); + + $this->assertCount(3, $messaging->sendQueryWithRouting('order.getReceived')); + } + + public function test_async_publishing_requires_enterprise_licence(): void + { + $channelName = Uuid::v7()->toRfc4122(); + $orderService = $this->createOrderService($channelName); + + $this->expectException(LicensingException::class); + + $this->bootstrapEcotone($channelName, $orderService, licenceKey: null); + } + + private function createOrderService(string $channelName): object + { + return new class ($channelName) { + /** @var string[] */ + private array $receivedEvents = []; + + public function __construct(private string $channelName) + { + } + + #[CommandHandler('order.place')] + public function placeOrder(string $order, EventBus $eventBus): void + { + $eventBus->publish(new OrderWasPlaced($order . '-1')); + $eventBus->publish(new OrderWasPlaced($order . '-2')); + $eventBus->publish(new OrderWasPlaced($order . '-3')); + } + + #[Asynchronous('asyncOrdersChannel')] + #[EventHandler(endpointId: 'async_amqp_order_collector')] + public function collect(OrderWasPlaced $event): void + { + $this->receivedEvents[] = $event->order; + } + + #[QueryHandler('order.getReceived')] + public function getReceived(): array + { + return $this->receivedEvents; + } + }; + } + + private function bootstrapEcotone(string $channelName, object $orderService, ?string $licenceKey): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [...$this->getConnectionFactoryReferences(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::AMQP_PACKAGE])) + ->withExtensionObjects([ + AmqpBackedMessageChannelBuilder::create('asyncOrdersChannel', queueName: $channelName) + ->withAsyncPublishing(), + ]), + licenceKey: $licenceKey, + ); + } +} diff --git a/packages/Enqueue/src/EnqueueMessageChannel.php b/packages/Enqueue/src/EnqueueMessageChannel.php index fc9cbf415..990fbab9a 100644 --- a/packages/Enqueue/src/EnqueueMessageChannel.php +++ b/packages/Enqueue/src/EnqueueMessageChannel.php @@ -4,6 +4,7 @@ namespace Ecotone\Enqueue; +use Ecotone\Messaging\Channel\BatchSupportingMessageChannel; use Ecotone\Messaging\Endpoint\PollingMetadata; use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageHandler; @@ -12,12 +13,17 @@ /** * licence Apache-2.0 */ -final class EnqueueMessageChannel implements PollableChannel +final class EnqueueMessageChannel implements PollableChannel, BatchSupportingMessageChannel { - public function __construct(private EnqueueInboundChannelAdapter $inboundChannelAdapter, private MessageHandler $outboundChannelAdapter) + public function __construct(private EnqueueInboundChannelAdapter $inboundChannelAdapter, private MessageHandler $outboundChannelAdapter, private bool $supportsBatchMessages = false) { } + public function supportsBatchMessages(): bool + { + return $this->supportsBatchMessages; + } + public function send(Message $message): void { $this->outboundChannelAdapter->handle($message); diff --git a/packages/Enqueue/src/EnqueueMessageChannelBuilder.php b/packages/Enqueue/src/EnqueueMessageChannelBuilder.php index 6996677ef..1b962fe81 100644 --- a/packages/Enqueue/src/EnqueueMessageChannelBuilder.php +++ b/packages/Enqueue/src/EnqueueMessageChannelBuilder.php @@ -121,6 +121,12 @@ public function compile(MessagingContainerBuilder $builder): Definition return new Definition(EnqueueMessageChannel::class, [ $this->inboundChannelAdapter->compile($builder), $this->outboundChannelAdapter->compile($builder), + $this->supportsBatchMessages(), ]); } + + protected function supportsBatchMessages(): bool + { + return false; + } } From aa10185639026bfd28e379d8b01960fe72ae3643 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 09/38] feat: CI benchmark comparing synchronous and async publishing for AMQP and Kafka --- .github/workflows/benchmark-pr.yml | 18 + .../AsyncPublishing/BenchmarkOrderPlaced.php | 12 + .../AsyncPublishing/OrderPublisherService.php | 27 + .../Benchmark/AsyncPublishingBenchmark.php | 128 +++ .../ExampleApp/Symfony/config/reference.php | 801 +++++++++--------- .../Symfony/config/reference.php | 801 +++++++++--------- 6 files changed, 991 insertions(+), 796 deletions(-) create mode 100644 Monorepo/Benchmark/AsyncPublishing/BenchmarkOrderPlaced.php create mode 100644 Monorepo/Benchmark/AsyncPublishing/OrderPublisherService.php create mode 100644 Monorepo/Benchmark/AsyncPublishingBenchmark.php diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index fa7a831b5..74529d0f8 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -19,6 +19,23 @@ jobs: php-versions: [ 8.5 ] stability: [prefer-stable] services: + kafka: + image: apache/kafka:3.9.0 + options: >- + --env KAFKA_NODE_ID=0 + --env KAFKA_PROCESS_ROLES=broker,controller + --env KAFKA_CONTROLLER_QUORUM_VOTERS=0@127.0.0.1:9093 + --env KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER + --env KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092,CONTROLLER://:9093 + --env KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092 + --env KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + --env KAFKA_AUTO_CREATE_TOPICS_ENABLE=true + --env KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + --env KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 + --env KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 + --env KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + ports: + - 9092:9092 rabbitmq: image: rabbitmq:4.1.4-management-alpine env: @@ -66,6 +83,7 @@ jobs: - '6379:6379' env: RABBIT_HOST: amqp://127.0.0.1:5672 + KAFKA_DSN: 127.0.0.1:9092 SQS_DSN: sqs:?key=key&secret=secret®ion=us-east-1&endpoint=http://127.0.0.1:4566&version=latest REDIS_DSN: redis://127.0.0.1:6379 DATABASE_DSN: pgsql://ecotone:secret@127.0.0.1:5432/ecotone diff --git a/Monorepo/Benchmark/AsyncPublishing/BenchmarkOrderPlaced.php b/Monorepo/Benchmark/AsyncPublishing/BenchmarkOrderPlaced.php new file mode 100644 index 000000000..1155c2b1b --- /dev/null +++ b/Monorepo/Benchmark/AsyncPublishing/BenchmarkOrderPlaced.php @@ -0,0 +1,12 @@ +publish(new BenchmarkOrderPlaced((string) $orderNumber)); + } + } + + #[Asynchronous('benchmark_orders')] + #[EventHandler(endpointId: 'benchmark_order_consumer')] + public function consume(BenchmarkOrderPlaced $event): void + { + } +} diff --git a/Monorepo/Benchmark/AsyncPublishingBenchmark.php b/Monorepo/Benchmark/AsyncPublishingBenchmark.php new file mode 100644 index 000000000..7221f5260 --- /dev/null +++ b/Monorepo/Benchmark/AsyncPublishingBenchmark.php @@ -0,0 +1,128 @@ +messaging = $this->bootstrapWithAmqp(asyncPublishing: false); + } + + public function setUpAmqpAsyncPublishing(): void + { + $this->messaging = $this->bootstrapWithAmqp(asyncPublishing: true); + } + + public function setUpKafkaSynchronousPublishing(): void + { + $this->messaging = $this->bootstrapWithKafka(asyncPublishing: false); + } + + public function setUpKafkaAsyncPublishing(): void + { + $this->messaging = $this->bootstrapWithKafka(asyncPublishing: true); + } + + #[BeforeMethods('setUpAmqpSynchronousPublishing')] + public function bench_amqp_synchronous_publishing(): void + { + $this->publishMessagesThroughCommandHandler(); + } + + #[BeforeMethods('setUpAmqpAsyncPublishing')] + public function bench_amqp_async_publishing(): void + { + $this->publishMessagesThroughCommandHandler(); + } + + #[BeforeMethods('setUpKafkaSynchronousPublishing')] + public function bench_kafka_synchronous_publishing(): void + { + $this->publishMessagesThroughCommandHandler(); + } + + #[BeforeMethods('setUpKafkaAsyncPublishing')] + public function bench_kafka_async_publishing(): void + { + $this->publishMessagesThroughCommandHandler(); + } + + private function publishMessagesThroughCommandHandler(): void + { + $this->messaging->sendCommandWithRoutingKey('benchmark.publishOrders', self::AMOUNT_OF_PUBLISHED_MESSAGES); + } + + private function bootstrapWithAmqp(bool $asyncPublishing): FlowTestSupport + { + $channelBuilder = AmqpBackedMessageChannelBuilder::create('benchmark_orders', queueName: uniqid('benchmark_orders_')); + if ($asyncPublishing) { + $channelBuilder = $channelBuilder->withAsyncPublishing(); + } + + return EcotoneLite::bootstrapFlowTesting( + [OrderPublisherService::class], + [ + new OrderPublisherService(), + AmqpConnectionFactory::class => new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']), + DbalConnectionFactory::class => new DbalConnectionFactory(getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone'), + ], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::AMQP_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + $channelBuilder, + DbalConfiguration::createWithDefaults()->withTransactionOnCommandBus(true), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + + private function bootstrapWithKafka(bool $asyncPublishing): FlowTestSupport + { + $topicName = uniqid('benchmark_orders_'); + $channelBuilder = KafkaMessageChannelBuilder::create('benchmark_orders', topicName: $topicName, messageGroupId: $topicName); + if ($asyncPublishing) { + $channelBuilder = $channelBuilder->withAsyncPublishing(); + } + + return EcotoneLite::bootstrapFlowTesting( + [OrderPublisherService::class], + [ + new OrderPublisherService(), + KafkaBrokerConfiguration::class => KafkaBrokerConfiguration::createWithDefaults([getenv('KAFKA_DSN') ?: 'localhost:9094']), + DbalConnectionFactory::class => new DbalConnectionFactory(getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone'), + ], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::KAFKA_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + $channelBuilder, + DbalConfiguration::createWithDefaults()->withTransactionOnCommandBus(true), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } +} diff --git a/Monorepo/ExampleApp/Symfony/config/reference.php b/Monorepo/ExampleApp/Symfony/config/reference.php index fe5ba5be8..d0b355025 100644 --- a/Monorepo/ExampleApp/Symfony/config/reference.php +++ b/Monorepo/ExampleApp/Symfony/config/reference.php @@ -4,6 +4,8 @@ namespace Symfony\Component\DependencyInjection\Loader\Configurator; +use Symfony\Component\Config\Loader\ParamConfigurator as Param; + /** * This class provides array-shapes for configuring the services and bundles of an application. * @@ -31,7 +33,7 @@ * type?: string|null, * ignore_errors?: bool, * }> - * @psalm-type ParametersConfig = array|null>|null> + * @psalm-type ParametersConfig = array|Param|null>|Param|null> * @psalm-type ArgumentsType = list|array * @psalm-type CallType = array|array{0:string, 1?:ArgumentsType, 2?:bool}|array{method:string, arguments?:ArgumentsType, returns_clone?:bool} * @psalm-type TagsType = list>> // arrays inside the list must have only one element, with the tag name as the key @@ -119,592 +121,592 @@ * } * @psalm-type ServicesConfig = array{ * _defaults?: DefaultsType, - * _instanceof?: InstanceofType, + * _instanceof?: array, * ... * } * @psalm-type ExtensionType = array * @psalm-type FrameworkConfig = array{ - * secret?: scalar|null, - * http_method_override?: bool, // Set true to enable support for the '_method' request parameter to determine the intended HTTP method on POST requests. // Default: false - * allowed_http_method_override?: list|null, - * trust_x_sendfile_type_header?: scalar|null, // Set true to enable support for xsendfile in binary file responses. // Default: "%env(bool:default::SYMFONY_TRUST_X_SENDFILE_TYPE_HEADER)%" - * ide?: scalar|null, // Default: "%env(default::SYMFONY_IDE)%" - * test?: bool, - * default_locale?: scalar|null, // Default: "en" - * set_locale_from_accept_language?: bool, // Whether to use the Accept-Language HTTP header to set the Request locale (only when the "_locale" request attribute is not passed). // Default: false - * set_content_language_from_locale?: bool, // Whether to set the Content-Language HTTP header on the Response using the Request locale. // Default: false - * enabled_locales?: list, - * trusted_hosts?: list, + * secret?: scalar|Param|null, + * http_method_override?: bool|Param, // Set true to enable support for the '_method' request parameter to determine the intended HTTP method on POST requests. // Default: false + * allowed_http_method_override?: null|list, + * trust_x_sendfile_type_header?: scalar|Param|null, // Set true to enable support for xsendfile in binary file responses. // Default: "%env(bool:default::SYMFONY_TRUST_X_SENDFILE_TYPE_HEADER)%" + * ide?: scalar|Param|null, // Default: "%env(default::SYMFONY_IDE)%" + * test?: bool|Param, + * default_locale?: scalar|Param|null, // Default: "en" + * set_locale_from_accept_language?: bool|Param, // Whether to use the Accept-Language HTTP header to set the Request locale (only when the "_locale" request attribute is not passed). // Default: false + * set_content_language_from_locale?: bool|Param, // Whether to set the Content-Language HTTP header on the Response using the Request locale. // Default: false + * enabled_locales?: list, + * trusted_hosts?: string|list, * trusted_proxies?: mixed, // Default: ["%env(default::SYMFONY_TRUSTED_PROXIES)%"] - * trusted_headers?: list, - * error_controller?: scalar|null, // Default: "error_controller" - * handle_all_throwables?: bool, // HttpKernel will handle all kinds of \Throwable. // Default: true + * trusted_headers?: string|list, + * error_controller?: scalar|Param|null, // Default: "error_controller" + * handle_all_throwables?: bool|Param, // HttpKernel will handle all kinds of \Throwable. // Default: true * csrf_protection?: bool|array{ - * enabled?: scalar|null, // Default: null - * stateless_token_ids?: list, - * check_header?: scalar|null, // Whether to check the CSRF token in a header in addition to a cookie when using stateless protection. // Default: false - * cookie_name?: scalar|null, // The name of the cookie to use when using stateless protection. // Default: "csrf-token" + * enabled?: scalar|Param|null, // Default: null + * stateless_token_ids?: list, + * check_header?: scalar|Param|null, // Whether to check the CSRF token in a header in addition to a cookie when using stateless protection. // Default: false + * cookie_name?: scalar|Param|null, // The name of the cookie to use when using stateless protection. // Default: "csrf-token" * }, * form?: bool|array{ // Form configuration - * enabled?: bool, // Default: false - * csrf_protection?: array{ - * enabled?: scalar|null, // Default: null - * token_id?: scalar|null, // Default: null - * field_name?: scalar|null, // Default: "_token" - * field_attr?: array, + * enabled?: bool|Param, // Default: false + * csrf_protection?: bool|array{ + * enabled?: scalar|Param|null, // Default: null + * token_id?: scalar|Param|null, // Default: null + * field_name?: scalar|Param|null, // Default: "_token" + * field_attr?: array, * }, * }, * http_cache?: bool|array{ // HTTP cache configuration - * enabled?: bool, // Default: false - * debug?: bool, // Default: "%kernel.debug%" - * trace_level?: "none"|"short"|"full", - * trace_header?: scalar|null, - * default_ttl?: int, - * private_headers?: list, - * skip_response_headers?: list, - * allow_reload?: bool, - * allow_revalidate?: bool, - * stale_while_revalidate?: int, - * stale_if_error?: int, - * terminate_on_cache_hit?: bool, + * enabled?: bool|Param, // Default: false + * debug?: bool|Param, // Default: "%kernel.debug%" + * trace_level?: "none"|"short"|"full"|Param, + * trace_header?: scalar|Param|null, + * default_ttl?: int|Param, + * private_headers?: list, + * skip_response_headers?: list, + * allow_reload?: bool|Param, + * allow_revalidate?: bool|Param, + * stale_while_revalidate?: int|Param, + * stale_if_error?: int|Param, + * terminate_on_cache_hit?: bool|Param, * }, * esi?: bool|array{ // ESI configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, * ssi?: bool|array{ // SSI configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, * fragments?: bool|array{ // Fragments configuration - * enabled?: bool, // Default: false - * hinclude_default_template?: scalar|null, // Default: null - * path?: scalar|null, // Default: "/_fragment" + * enabled?: bool|Param, // Default: false + * hinclude_default_template?: scalar|Param|null, // Default: null + * path?: scalar|Param|null, // Default: "/_fragment" * }, * profiler?: bool|array{ // Profiler configuration - * enabled?: bool, // Default: false - * collect?: bool, // Default: true - * collect_parameter?: scalar|null, // The name of the parameter to use to enable or disable collection on a per request basis. // Default: null - * only_exceptions?: bool, // Default: false - * only_main_requests?: bool, // Default: false - * dsn?: scalar|null, // Default: "file:%kernel.cache_dir%/profiler" - * collect_serializer_data?: bool, // Enables the serializer data collector and profiler panel. // Default: false + * enabled?: bool|Param, // Default: false + * collect?: bool|Param, // Default: true + * collect_parameter?: scalar|Param|null, // The name of the parameter to use to enable or disable collection on a per request basis. // Default: null + * only_exceptions?: bool|Param, // Default: false + * only_main_requests?: bool|Param, // Default: false + * dsn?: scalar|Param|null, // Default: "file:%kernel.cache_dir%/profiler" + * collect_serializer_data?: bool|Param, // Enables the serializer data collector and profiler panel. // Default: false * }, * workflows?: bool|array{ - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * workflows?: array, - * definition_validators?: list, - * support_strategy?: scalar|null, - * initial_marking?: list, - * events_to_dispatch?: list|null, - * places?: list, + * supports?: string|list, + * definition_validators?: list, + * support_strategy?: scalar|Param|null, + * initial_marking?: \BackedEnum|string|list, + * events_to_dispatch?: null|list, + * places?: string|list, * }>, - * transitions: list, - * to?: list, - * weight?: int, // Default: 1 - * metadata?: list, + * weight?: int|Param, // Default: 1 + * metadata?: array, * }>, - * metadata?: list, + * metadata?: array, * }>, * }, * router?: bool|array{ // Router configuration - * enabled?: bool, // Default: false - * resource: scalar|null, - * type?: scalar|null, - * cache_dir?: scalar|null, // Deprecated: Setting the "framework.router.cache_dir.cache_dir" configuration option is deprecated. It will be removed in version 8.0. // Default: "%kernel.build_dir%" - * default_uri?: scalar|null, // The default URI used to generate URLs in a non-HTTP context. // Default: null - * http_port?: scalar|null, // Default: 80 - * https_port?: scalar|null, // Default: 443 - * strict_requirements?: scalar|null, // set to true to throw an exception when a parameter does not match the requirements set to false to disable exceptions when a parameter does not match the requirements (and return null instead) set to null to disable parameter checks against requirements 'true' is the preferred configuration in development mode, while 'false' or 'null' might be preferred in production // Default: true - * utf8?: bool, // Default: true + * enabled?: bool|Param, // Default: false + * resource?: scalar|Param|null, + * type?: scalar|Param|null, + * cache_dir?: scalar|Param|null, // Deprecated: Setting the "framework.router.cache_dir.cache_dir" configuration option is deprecated. It will be removed in version 8.0. // Default: "%kernel.build_dir%" + * default_uri?: scalar|Param|null, // The default URI used to generate URLs in a non-HTTP context. // Default: null + * http_port?: scalar|Param|null, // Default: 80 + * https_port?: scalar|Param|null, // Default: 443 + * strict_requirements?: scalar|Param|null, // set to true to throw an exception when a parameter does not match the requirements set to false to disable exceptions when a parameter does not match the requirements (and return null instead) set to null to disable parameter checks against requirements 'true' is the preferred configuration in development mode, while 'false' or 'null' might be preferred in production // Default: true + * utf8?: bool|Param, // Default: true * }, * session?: bool|array{ // Session configuration - * enabled?: bool, // Default: false - * storage_factory_id?: scalar|null, // Default: "session.storage.factory.native" - * handler_id?: scalar|null, // Defaults to using the native session handler, or to the native *file* session handler if "save_path" is not null. - * name?: scalar|null, - * cookie_lifetime?: scalar|null, - * cookie_path?: scalar|null, - * cookie_domain?: scalar|null, - * cookie_secure?: true|false|"auto", // Default: "auto" - * cookie_httponly?: bool, // Default: true - * cookie_samesite?: null|"lax"|"strict"|"none", // Default: "lax" - * use_cookies?: bool, - * gc_divisor?: scalar|null, - * gc_probability?: scalar|null, - * gc_maxlifetime?: scalar|null, - * save_path?: scalar|null, // Defaults to "%kernel.cache_dir%/sessions" if the "handler_id" option is not null. - * metadata_update_threshold?: int, // Seconds to wait between 2 session metadata updates. // Default: 0 - * sid_length?: int, // Deprecated: Setting the "framework.session.sid_length.sid_length" configuration option is deprecated. It will be removed in version 8.0. No alternative is provided as PHP 8.4 has deprecated the related option. - * sid_bits_per_character?: int, // Deprecated: Setting the "framework.session.sid_bits_per_character.sid_bits_per_character" configuration option is deprecated. It will be removed in version 8.0. No alternative is provided as PHP 8.4 has deprecated the related option. + * enabled?: bool|Param, // Default: false + * storage_factory_id?: scalar|Param|null, // Default: "session.storage.factory.native" + * handler_id?: scalar|Param|null, // Defaults to using the native session handler, or to the native *file* session handler if "save_path" is not null. + * name?: scalar|Param|null, + * cookie_lifetime?: scalar|Param|null, + * cookie_path?: scalar|Param|null, + * cookie_domain?: scalar|Param|null, + * cookie_secure?: true|false|"auto"|Param, // Default: "auto" + * cookie_httponly?: bool|Param, // Default: true + * cookie_samesite?: null|"lax"|"strict"|"none"|Param, // Default: "lax" + * use_cookies?: bool|Param, + * gc_divisor?: scalar|Param|null, + * gc_probability?: scalar|Param|null, + * gc_maxlifetime?: scalar|Param|null, + * save_path?: scalar|Param|null, // Defaults to "%kernel.cache_dir%/sessions" if the "handler_id" option is not null. + * metadata_update_threshold?: int|Param, // Seconds to wait between 2 session metadata updates. // Default: 0 + * sid_length?: int|Param, // Deprecated: Setting the "framework.session.sid_length.sid_length" configuration option is deprecated. It will be removed in version 8.0. No alternative is provided as PHP 8.4 has deprecated the related option. + * sid_bits_per_character?: int|Param, // Deprecated: Setting the "framework.session.sid_bits_per_character.sid_bits_per_character" configuration option is deprecated. It will be removed in version 8.0. No alternative is provided as PHP 8.4 has deprecated the related option. * }, * request?: bool|array{ // Request configuration - * enabled?: bool, // Default: false - * formats?: array>, + * enabled?: bool|Param, // Default: false + * formats?: array>, * }, * assets?: bool|array{ // Assets configuration - * enabled?: bool, // Default: false - * strict_mode?: bool, // Throw an exception if an entry is missing from the manifest.json. // Default: false - * version_strategy?: scalar|null, // Default: null - * version?: scalar|null, // Default: null - * version_format?: scalar|null, // Default: "%%s?%%s" - * json_manifest_path?: scalar|null, // Default: null - * base_path?: scalar|null, // Default: "" - * base_urls?: list, + * enabled?: bool|Param, // Default: false + * strict_mode?: bool|Param, // Throw an exception if an entry is missing from the manifest.json. // Default: false + * version_strategy?: scalar|Param|null, // Default: null + * version?: scalar|Param|null, // Default: null + * version_format?: scalar|Param|null, // Default: "%%s?%%s" + * json_manifest_path?: scalar|Param|null, // Default: null + * base_path?: scalar|Param|null, // Default: "" + * base_urls?: string|list, * packages?: array, + * strict_mode?: bool|Param, // Throw an exception if an entry is missing from the manifest.json. // Default: false + * version_strategy?: scalar|Param|null, // Default: null + * version?: scalar|Param|null, + * version_format?: scalar|Param|null, // Default: null + * json_manifest_path?: scalar|Param|null, // Default: null + * base_path?: scalar|Param|null, // Default: "" + * base_urls?: string|list, * }>, * }, * asset_mapper?: bool|array{ // Asset Mapper configuration - * enabled?: bool, // Default: false - * paths?: array, - * excluded_patterns?: list, - * exclude_dotfiles?: bool, // If true, any files starting with "." will be excluded from the asset mapper. // Default: true - * server?: bool, // If true, a "dev server" will return the assets from the public directory (true in "debug" mode only by default). // Default: true - * public_prefix?: scalar|null, // The public path where the assets will be written to (and served from when "server" is true). // Default: "/assets/" - * missing_import_mode?: "strict"|"warn"|"ignore", // Behavior if an asset cannot be found when imported from JavaScript or CSS files - e.g. "import './non-existent.js'". "strict" means an exception is thrown, "warn" means a warning is logged, "ignore" means the import is left as-is. // Default: "warn" - * extensions?: array, - * importmap_path?: scalar|null, // The path of the importmap.php file. // Default: "%kernel.project_dir%/importmap.php" - * importmap_polyfill?: scalar|null, // The importmap name that will be used to load the polyfill. Set to false to disable. // Default: "es-module-shims" - * importmap_script_attributes?: array, - * vendor_dir?: scalar|null, // The directory to store JavaScript vendors. // Default: "%kernel.project_dir%/assets/vendor" + * enabled?: bool|Param, // Default: false + * paths?: string|array, + * excluded_patterns?: list, + * exclude_dotfiles?: bool|Param, // If true, any files starting with "." will be excluded from the asset mapper. // Default: true + * server?: bool|Param, // If true, a "dev server" will return the assets from the public directory (true in "debug" mode only by default). // Default: true + * public_prefix?: scalar|Param|null, // The public path where the assets will be written to (and served from when "server" is true). // Default: "/assets/" + * missing_import_mode?: "strict"|"warn"|"ignore"|Param, // Behavior if an asset cannot be found when imported from JavaScript or CSS files - e.g. "import './non-existent.js'". "strict" means an exception is thrown, "warn" means a warning is logged, "ignore" means the import is left as-is. // Default: "warn" + * extensions?: array, + * importmap_path?: scalar|Param|null, // The path of the importmap.php file. // Default: "%kernel.project_dir%/importmap.php" + * importmap_polyfill?: scalar|Param|null, // The importmap name that will be used to load the polyfill. Set to false to disable. // Default: "es-module-shims" + * importmap_script_attributes?: array, + * vendor_dir?: scalar|Param|null, // The directory to store JavaScript vendors. // Default: "%kernel.project_dir%/assets/vendor" * precompress?: bool|array{ // Precompress assets with Brotli, Zstandard and gzip. - * enabled?: bool, // Default: false - * formats?: list, - * extensions?: list, + * enabled?: bool|Param, // Default: false + * formats?: list, + * extensions?: list, * }, * }, * translator?: bool|array{ // Translator configuration - * enabled?: bool, // Default: true - * fallbacks?: list, - * logging?: bool, // Default: false - * formatter?: scalar|null, // Default: "translator.formatter.default" - * cache_dir?: scalar|null, // Default: "%kernel.cache_dir%/translations" - * default_path?: scalar|null, // The default path used to load translations. // Default: "%kernel.project_dir%/translations" - * paths?: list, + * enabled?: bool|Param, // Default: true + * fallbacks?: string|list, + * logging?: bool|Param, // Default: false + * formatter?: scalar|Param|null, // Default: "translator.formatter.default" + * cache_dir?: scalar|Param|null, // Default: "%kernel.cache_dir%/translations" + * default_path?: scalar|Param|null, // The default path used to load translations. // Default: "%kernel.project_dir%/translations" + * paths?: list, * pseudo_localization?: bool|array{ - * enabled?: bool, // Default: false - * accents?: bool, // Default: true - * expansion_factor?: float, // Default: 1.0 - * brackets?: bool, // Default: true - * parse_html?: bool, // Default: false - * localizable_html_attributes?: list, + * enabled?: bool|Param, // Default: false + * accents?: bool|Param, // Default: true + * expansion_factor?: float|Param, // Default: 1.0 + * brackets?: bool|Param, // Default: true + * parse_html?: bool|Param, // Default: false + * localizable_html_attributes?: list, * }, * providers?: array, - * locales?: list, + * dsn?: scalar|Param|null, + * domains?: list, + * locales?: list, * }>, * globals?: array, - * domain?: string, + * message?: string|Param, + * parameters?: array, + * domain?: string|Param, * }>, * }, * validation?: bool|array{ // Validation configuration - * enabled?: bool, // Default: false - * cache?: scalar|null, // Deprecated: Setting the "framework.validation.cache.cache" configuration option is deprecated. It will be removed in version 8.0. - * enable_attributes?: bool, // Default: true - * static_method?: list, - * translation_domain?: scalar|null, // Default: "validators" - * email_validation_mode?: "html5"|"html5-allow-no-tld"|"strict"|"loose", // Default: "html5" + * enabled?: bool|Param, // Default: false + * cache?: scalar|Param|null, // Deprecated: Setting the "framework.validation.cache.cache" configuration option is deprecated. It will be removed in version 8.0. + * enable_attributes?: bool|Param, // Default: true + * static_method?: string|list, + * translation_domain?: scalar|Param|null, // Default: "validators" + * email_validation_mode?: "html5"|"html5-allow-no-tld"|"strict"|"loose"|Param, // Default: "html5" * mapping?: array{ - * paths?: list, + * paths?: list, * }, * not_compromised_password?: bool|array{ - * enabled?: bool, // When disabled, compromised passwords will be accepted as valid. // Default: true - * endpoint?: scalar|null, // API endpoint for the NotCompromisedPassword Validator. // Default: null + * enabled?: bool|Param, // When disabled, compromised passwords will be accepted as valid. // Default: true + * endpoint?: scalar|Param|null, // API endpoint for the NotCompromisedPassword Validator. // Default: null * }, - * disable_translation?: bool, // Default: false + * disable_translation?: bool|Param, // Default: false * auto_mapping?: array, + * services?: list, * }>, * }, * annotations?: bool|array{ - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, * serializer?: bool|array{ // Serializer configuration - * enabled?: bool, // Default: false - * enable_attributes?: bool, // Default: true - * name_converter?: scalar|null, - * circular_reference_handler?: scalar|null, - * max_depth_handler?: scalar|null, + * enabled?: bool|Param, // Default: false + * enable_attributes?: bool|Param, // Default: true + * name_converter?: scalar|Param|null, + * circular_reference_handler?: scalar|Param|null, + * max_depth_handler?: scalar|Param|null, * mapping?: array{ - * paths?: list, + * paths?: list, * }, - * default_context?: list, + * default_context?: array, * named_serializers?: array, - * include_built_in_normalizers?: bool, // Whether to include the built-in normalizers // Default: true - * include_built_in_encoders?: bool, // Whether to include the built-in encoders // Default: true + * name_converter?: scalar|Param|null, + * default_context?: array, + * include_built_in_normalizers?: bool|Param, // Whether to include the built-in normalizers // Default: true + * include_built_in_encoders?: bool|Param, // Whether to include the built-in encoders // Default: true * }>, * }, * property_access?: bool|array{ // Property access configuration - * enabled?: bool, // Default: false - * magic_call?: bool, // Default: false - * magic_get?: bool, // Default: true - * magic_set?: bool, // Default: true - * throw_exception_on_invalid_index?: bool, // Default: false - * throw_exception_on_invalid_property_path?: bool, // Default: true + * enabled?: bool|Param, // Default: false + * magic_call?: bool|Param, // Default: false + * magic_get?: bool|Param, // Default: true + * magic_set?: bool|Param, // Default: true + * throw_exception_on_invalid_index?: bool|Param, // Default: false + * throw_exception_on_invalid_property_path?: bool|Param, // Default: true * }, * type_info?: bool|array{ // Type info configuration - * enabled?: bool, // Default: false - * aliases?: array, + * enabled?: bool|Param, // Default: false + * aliases?: array, * }, * property_info?: bool|array{ // Property info configuration - * enabled?: bool, // Default: false - * with_constructor_extractor?: bool, // Registers the constructor extractor. + * enabled?: bool|Param, // Default: false + * with_constructor_extractor?: bool|Param, // Registers the constructor extractor. * }, * cache?: array{ // Cache configuration - * prefix_seed?: scalar|null, // Used to namespace cache keys when using several apps with the same shared backend. // Default: "_%kernel.project_dir%.%kernel.container_class%" - * app?: scalar|null, // App related cache pools configuration. // Default: "cache.adapter.filesystem" - * system?: scalar|null, // System related cache pools configuration. // Default: "cache.adapter.system" - * directory?: scalar|null, // Default: "%kernel.share_dir%/pools/app" - * default_psr6_provider?: scalar|null, - * default_redis_provider?: scalar|null, // Default: "redis://localhost" - * default_valkey_provider?: scalar|null, // Default: "valkey://localhost" - * default_memcached_provider?: scalar|null, // Default: "memcached://localhost" - * default_doctrine_dbal_provider?: scalar|null, // Default: "database_connection" - * default_pdo_provider?: scalar|null, // Default: null + * prefix_seed?: scalar|Param|null, // Used to namespace cache keys when using several apps with the same shared backend. // Default: "_%kernel.project_dir%.%kernel.container_class%" + * app?: scalar|Param|null, // App related cache pools configuration. // Default: "cache.adapter.filesystem" + * system?: scalar|Param|null, // System related cache pools configuration. // Default: "cache.adapter.system" + * directory?: scalar|Param|null, // Default: "%kernel.share_dir%/pools/app" + * default_psr6_provider?: scalar|Param|null, + * default_redis_provider?: scalar|Param|null, // Default: "redis://localhost" + * default_valkey_provider?: scalar|Param|null, // Default: "valkey://localhost" + * default_memcached_provider?: scalar|Param|null, // Default: "memcached://localhost" + * default_doctrine_dbal_provider?: scalar|Param|null, // Default: "database_connection" + * default_pdo_provider?: scalar|Param|null, // Default: null * pools?: array, - * tags?: scalar|null, // Default: null - * public?: bool, // Default: false - * default_lifetime?: scalar|null, // Default lifetime of the pool. - * provider?: scalar|null, // Overwrite the setting from the default provider for this adapter. - * early_expiration_message_bus?: scalar|null, - * clearer?: scalar|null, + * adapters?: string|list, + * tags?: scalar|Param|null, // Default: null + * public?: bool|Param, // Default: false + * default_lifetime?: scalar|Param|null, // Default lifetime of the pool. + * provider?: scalar|Param|null, // Overwrite the setting from the default provider for this adapter. + * early_expiration_message_bus?: scalar|Param|null, + * clearer?: scalar|Param|null, * }>, * }, * php_errors?: array{ // PHP errors handling configuration * log?: mixed, // Use the application logger instead of the PHP logger for logging PHP errors. // Default: true - * throw?: bool, // Throw PHP errors as \ErrorException instances. // Default: true + * throw?: bool|Param, // Throw PHP errors as \ErrorException instances. // Default: true * }, * exceptions?: array, * web_link?: bool|array{ // Web links configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, * lock?: bool|string|array{ // Lock configuration - * enabled?: bool, // Default: false - * resources?: array>, + * enabled?: bool|Param, // Default: false + * resources?: string|array>, * }, * semaphore?: bool|string|array{ // Semaphore configuration - * enabled?: bool, // Default: false - * resources?: array, + * enabled?: bool|Param, // Default: false + * resources?: string|array, * }, * messenger?: bool|array{ // Messenger configuration - * enabled?: bool, // Default: false - * routing?: array, + * enabled?: bool|Param, // Default: false + * routing?: array, * }>, * serializer?: array{ - * default_serializer?: scalar|null, // Service id to use as the default serializer for the transports. // Default: "messenger.transport.native_php_serializer" + * default_serializer?: scalar|Param|null, // Service id to use as the default serializer for the transports. // Default: "messenger.transport.native_php_serializer" * symfony_serializer?: array{ - * format?: scalar|null, // Serialization format for the messenger.transport.symfony_serializer service (which is not the serializer used by default). // Default: "json" + * format?: scalar|Param|null, // Serialization format for the messenger.transport.symfony_serializer service (which is not the serializer used by default). // Default: "json" * context?: array, * }, * }, * transports?: array, - * failure_transport?: scalar|null, // Transport name to send failed messages to (after all retries have failed). // Default: null + * dsn?: scalar|Param|null, + * serializer?: scalar|Param|null, // Service id of a custom serializer to use. // Default: null + * options?: array, + * failure_transport?: scalar|Param|null, // Transport name to send failed messages to (after all retries have failed). // Default: null * retry_strategy?: string|array{ - * service?: scalar|null, // Service id to override the retry strategy entirely. // Default: null - * max_retries?: int, // Default: 3 - * delay?: int, // Time in ms to delay (or the initial value when multiplier is used). // Default: 1000 - * multiplier?: float, // If greater than 1, delay will grow exponentially for each retry: this delay = (delay * (multiple ^ retries)). // Default: 2 - * max_delay?: int, // Max time in ms that a retry should ever be delayed (0 = infinite). // Default: 0 - * jitter?: float, // Randomness to apply to the delay (between 0 and 1). // Default: 0.1 + * service?: scalar|Param|null, // Service id to override the retry strategy entirely. // Default: null + * max_retries?: int|Param, // Default: 3 + * delay?: int|Param, // Time in ms to delay (or the initial value when multiplier is used). // Default: 1000 + * multiplier?: float|Param, // If greater than 1, delay will grow exponentially for each retry: this delay = (delay * (multiple ^ retries)). // Default: 2 + * max_delay?: int|Param, // Max time in ms that a retry should ever be delayed (0 = infinite). // Default: 0 + * jitter?: float|Param, // Randomness to apply to the delay (between 0 and 1). // Default: 0.1 * }, - * rate_limiter?: scalar|null, // Rate limiter name to use when processing messages. // Default: null + * rate_limiter?: scalar|Param|null, // Rate limiter name to use when processing messages. // Default: null * }>, - * failure_transport?: scalar|null, // Transport name to send failed messages to (after all retries have failed). // Default: null - * stop_worker_on_signals?: list, - * default_bus?: scalar|null, // Default: null + * failure_transport?: scalar|Param|null, // Transport name to send failed messages to (after all retries have failed). // Default: null + * stop_worker_on_signals?: int|string|list, + * default_bus?: scalar|Param|null, // Default: null * buses?: array, * }>, * }>, * }, * scheduler?: bool|array{ // Scheduler configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, - * disallow_search_engine_index?: bool, // Enabled by default when debug is enabled. // Default: true + * disallow_search_engine_index?: bool|Param, // Enabled by default when debug is enabled. // Default: true * http_client?: bool|array{ // HTTP Client configuration - * enabled?: bool, // Default: false - * max_host_connections?: int, // The maximum number of connections to a single host. + * enabled?: bool|Param, // Default: false + * max_host_connections?: int|Param, // The maximum number of connections to a single host. * default_options?: array{ * headers?: array, * vars?: array, - * max_redirects?: int, // The maximum number of redirects to follow. - * http_version?: scalar|null, // The default HTTP version, typically 1.1 or 2.0, leave to null for the best version. - * resolve?: array, - * proxy?: scalar|null, // The URL of the proxy to pass requests through or null for automatic detection. - * no_proxy?: scalar|null, // A comma separated list of hosts that do not require a proxy to be reached. - * timeout?: float, // The idle timeout, defaults to the "default_socket_timeout" ini parameter. - * max_duration?: float, // The maximum execution time for the request+response as a whole. - * bindto?: scalar|null, // A network interface name, IP address, a host name or a UNIX socket to bind to. - * verify_peer?: bool, // Indicates if the peer should be verified in a TLS context. - * verify_host?: bool, // Indicates if the host should exist as a certificate common name. - * cafile?: scalar|null, // A certificate authority file. - * capath?: scalar|null, // A directory that contains multiple certificate authority files. - * local_cert?: scalar|null, // A PEM formatted certificate file. - * local_pk?: scalar|null, // A private key file. - * passphrase?: scalar|null, // The passphrase used to encrypt the "local_pk" file. - * ciphers?: scalar|null, // A list of TLS ciphers separated by colons, commas or spaces (e.g. "RC3-SHA:TLS13-AES-128-GCM-SHA256"...) + * max_redirects?: int|Param, // The maximum number of redirects to follow. + * http_version?: scalar|Param|null, // The default HTTP version, typically 1.1 or 2.0, leave to null for the best version. + * resolve?: array, + * proxy?: scalar|Param|null, // The URL of the proxy to pass requests through or null for automatic detection. + * no_proxy?: scalar|Param|null, // A comma separated list of hosts that do not require a proxy to be reached. + * timeout?: float|Param, // The idle timeout, defaults to the "default_socket_timeout" ini parameter. + * max_duration?: float|Param, // The maximum execution time for the request+response as a whole. + * bindto?: scalar|Param|null, // A network interface name, IP address, a host name or a UNIX socket to bind to. + * verify_peer?: bool|Param, // Indicates if the peer should be verified in a TLS context. + * verify_host?: bool|Param, // Indicates if the host should exist as a certificate common name. + * cafile?: scalar|Param|null, // A certificate authority file. + * capath?: scalar|Param|null, // A directory that contains multiple certificate authority files. + * local_cert?: scalar|Param|null, // A PEM formatted certificate file. + * local_pk?: scalar|Param|null, // A private key file. + * passphrase?: scalar|Param|null, // The passphrase used to encrypt the "local_pk" file. + * ciphers?: scalar|Param|null, // A list of TLS ciphers separated by colons, commas or spaces (e.g. "RC3-SHA:TLS13-AES-128-GCM-SHA256"...) * peer_fingerprint?: array{ // Associative array: hashing algorithm => hash(es). * sha1?: mixed, * pin-sha256?: mixed, * md5?: mixed, * }, - * crypto_method?: scalar|null, // The minimum version of TLS to accept; must be one of STREAM_CRYPTO_METHOD_TLSv*_CLIENT constants. + * crypto_method?: scalar|Param|null, // The minimum version of TLS to accept; must be one of STREAM_CRYPTO_METHOD_TLSv*_CLIENT constants. * extra?: array, - * rate_limiter?: scalar|null, // Rate limiter name to use for throttling requests. // Default: null + * rate_limiter?: scalar|Param|null, // Rate limiter name to use for throttling requests. // Default: null * caching?: bool|array{ // Caching configuration. - * enabled?: bool, // Default: false - * cache_pool?: string, // The taggable cache pool to use for storing the responses. // Default: "cache.http_client" - * shared?: bool, // Indicates whether the cache is shared (public) or private. // Default: true - * max_ttl?: int, // The maximum TTL (in seconds) allowed for cached responses. Null means no cap. // Default: null + * enabled?: bool|Param, // Default: false + * cache_pool?: string|Param, // The taggable cache pool to use for storing the responses. // Default: "cache.http_client" + * shared?: bool|Param, // Indicates whether the cache is shared (public) or private. // Default: true + * max_ttl?: int|Param, // The maximum TTL (in seconds) allowed for cached responses. Null means no cap. // Default: null * }, * retry_failed?: bool|array{ - * enabled?: bool, // Default: false - * retry_strategy?: scalar|null, // service id to override the retry strategy. // Default: null - * http_codes?: array, + * enabled?: bool|Param, // Default: false + * retry_strategy?: scalar|Param|null, // service id to override the retry strategy. // Default: null + * http_codes?: int|string|array, * }>, - * max_retries?: int, // Default: 3 - * delay?: int, // Time in ms to delay (or the initial value when multiplier is used). // Default: 1000 - * multiplier?: float, // If greater than 1, delay will grow exponentially for each retry: delay * (multiple ^ retries). // Default: 2 - * max_delay?: int, // Max time in ms that a retry should ever be delayed (0 = infinite). // Default: 0 - * jitter?: float, // Randomness in percent (between 0 and 1) to apply to the delay. // Default: 0.1 + * max_retries?: int|Param, // Default: 3 + * delay?: int|Param, // Time in ms to delay (or the initial value when multiplier is used). // Default: 1000 + * multiplier?: float|Param, // If greater than 1, delay will grow exponentially for each retry: delay * (multiple ^ retries). // Default: 2 + * max_delay?: int|Param, // Max time in ms that a retry should ever be delayed (0 = infinite). // Default: 0 + * jitter?: float|Param, // Randomness in percent (between 0 and 1) to apply to the delay. // Default: 0.1 * }, * }, - * mock_response_factory?: scalar|null, // The id of the service that should generate mock responses. It should be either an invokable or an iterable. + * mock_response_factory?: scalar|Param|null, // The id of the service that should generate mock responses. It should be either an invokable or an iterable. * scoped_clients?: array, + * scope?: scalar|Param|null, // The regular expression that the request URL must match before adding the other options. When none is provided, the base URI is used instead. + * base_uri?: scalar|Param|null, // The URI to resolve relative URLs, following rules in RFC 3985, section 2. + * auth_basic?: scalar|Param|null, // An HTTP Basic authentication "username:password". + * auth_bearer?: scalar|Param|null, // A token enabling HTTP Bearer authorization. + * auth_ntlm?: scalar|Param|null, // A "username:password" pair to use Microsoft NTLM authentication (requires the cURL extension). + * query?: array, * headers?: array, - * max_redirects?: int, // The maximum number of redirects to follow. - * http_version?: scalar|null, // The default HTTP version, typically 1.1 or 2.0, leave to null for the best version. - * resolve?: array, - * proxy?: scalar|null, // The URL of the proxy to pass requests through or null for automatic detection. - * no_proxy?: scalar|null, // A comma separated list of hosts that do not require a proxy to be reached. - * timeout?: float, // The idle timeout, defaults to the "default_socket_timeout" ini parameter. - * max_duration?: float, // The maximum execution time for the request+response as a whole. - * bindto?: scalar|null, // A network interface name, IP address, a host name or a UNIX socket to bind to. - * verify_peer?: bool, // Indicates if the peer should be verified in a TLS context. - * verify_host?: bool, // Indicates if the host should exist as a certificate common name. - * cafile?: scalar|null, // A certificate authority file. - * capath?: scalar|null, // A directory that contains multiple certificate authority files. - * local_cert?: scalar|null, // A PEM formatted certificate file. - * local_pk?: scalar|null, // A private key file. - * passphrase?: scalar|null, // The passphrase used to encrypt the "local_pk" file. - * ciphers?: scalar|null, // A list of TLS ciphers separated by colons, commas or spaces (e.g. "RC3-SHA:TLS13-AES-128-GCM-SHA256"...). + * max_redirects?: int|Param, // The maximum number of redirects to follow. + * http_version?: scalar|Param|null, // The default HTTP version, typically 1.1 or 2.0, leave to null for the best version. + * resolve?: array, + * proxy?: scalar|Param|null, // The URL of the proxy to pass requests through or null for automatic detection. + * no_proxy?: scalar|Param|null, // A comma separated list of hosts that do not require a proxy to be reached. + * timeout?: float|Param, // The idle timeout, defaults to the "default_socket_timeout" ini parameter. + * max_duration?: float|Param, // The maximum execution time for the request+response as a whole. + * bindto?: scalar|Param|null, // A network interface name, IP address, a host name or a UNIX socket to bind to. + * verify_peer?: bool|Param, // Indicates if the peer should be verified in a TLS context. + * verify_host?: bool|Param, // Indicates if the host should exist as a certificate common name. + * cafile?: scalar|Param|null, // A certificate authority file. + * capath?: scalar|Param|null, // A directory that contains multiple certificate authority files. + * local_cert?: scalar|Param|null, // A PEM formatted certificate file. + * local_pk?: scalar|Param|null, // A private key file. + * passphrase?: scalar|Param|null, // The passphrase used to encrypt the "local_pk" file. + * ciphers?: scalar|Param|null, // A list of TLS ciphers separated by colons, commas or spaces (e.g. "RC3-SHA:TLS13-AES-128-GCM-SHA256"...). * peer_fingerprint?: array{ // Associative array: hashing algorithm => hash(es). * sha1?: mixed, * pin-sha256?: mixed, * md5?: mixed, * }, - * crypto_method?: scalar|null, // The minimum version of TLS to accept; must be one of STREAM_CRYPTO_METHOD_TLSv*_CLIENT constants. + * crypto_method?: scalar|Param|null, // The minimum version of TLS to accept; must be one of STREAM_CRYPTO_METHOD_TLSv*_CLIENT constants. * extra?: array, - * rate_limiter?: scalar|null, // Rate limiter name to use for throttling requests. // Default: null + * rate_limiter?: scalar|Param|null, // Rate limiter name to use for throttling requests. // Default: null * caching?: bool|array{ // Caching configuration. - * enabled?: bool, // Default: false - * cache_pool?: string, // The taggable cache pool to use for storing the responses. // Default: "cache.http_client" - * shared?: bool, // Indicates whether the cache is shared (public) or private. // Default: true - * max_ttl?: int, // The maximum TTL (in seconds) allowed for cached responses. Null means no cap. // Default: null + * enabled?: bool|Param, // Default: false + * cache_pool?: string|Param, // The taggable cache pool to use for storing the responses. // Default: "cache.http_client" + * shared?: bool|Param, // Indicates whether the cache is shared (public) or private. // Default: true + * max_ttl?: int|Param, // The maximum TTL (in seconds) allowed for cached responses. Null means no cap. // Default: null * }, * retry_failed?: bool|array{ - * enabled?: bool, // Default: false - * retry_strategy?: scalar|null, // service id to override the retry strategy. // Default: null - * http_codes?: array, + * enabled?: bool|Param, // Default: false + * retry_strategy?: scalar|Param|null, // service id to override the retry strategy. // Default: null + * http_codes?: int|string|array, * }>, - * max_retries?: int, // Default: 3 - * delay?: int, // Time in ms to delay (or the initial value when multiplier is used). // Default: 1000 - * multiplier?: float, // If greater than 1, delay will grow exponentially for each retry: delay * (multiple ^ retries). // Default: 2 - * max_delay?: int, // Max time in ms that a retry should ever be delayed (0 = infinite). // Default: 0 - * jitter?: float, // Randomness in percent (between 0 and 1) to apply to the delay. // Default: 0.1 + * max_retries?: int|Param, // Default: 3 + * delay?: int|Param, // Time in ms to delay (or the initial value when multiplier is used). // Default: 1000 + * multiplier?: float|Param, // If greater than 1, delay will grow exponentially for each retry: delay * (multiple ^ retries). // Default: 2 + * max_delay?: int|Param, // Max time in ms that a retry should ever be delayed (0 = infinite). // Default: 0 + * jitter?: float|Param, // Randomness in percent (between 0 and 1) to apply to the delay. // Default: 0.1 * }, * }>, * }, * mailer?: bool|array{ // Mailer configuration - * enabled?: bool, // Default: true - * message_bus?: scalar|null, // The message bus to use. Defaults to the default bus if the Messenger component is installed. // Default: null - * dsn?: scalar|null, // Default: null - * transports?: array, + * enabled?: bool|Param, // Default: true + * message_bus?: scalar|Param|null, // The message bus to use. Defaults to the default bus if the Messenger component is installed. // Default: null + * dsn?: scalar|Param|null, // Default: null + * transports?: array, * envelope?: array{ // Mailer Envelope configuration - * sender?: scalar|null, - * recipients?: list, - * allowed_recipients?: list, + * sender?: scalar|Param|null, + * recipients?: string|list, + * allowed_recipients?: string|list, * }, * headers?: array, * dkim_signer?: bool|array{ // DKIM signer configuration - * enabled?: bool, // Default: false - * key?: scalar|null, // Key content, or path to key (in PEM format with the `file://` prefix) // Default: "" - * domain?: scalar|null, // Default: "" - * select?: scalar|null, // Default: "" - * passphrase?: scalar|null, // The private key passphrase // Default: "" + * enabled?: bool|Param, // Default: false + * key?: scalar|Param|null, // Key content, or path to key (in PEM format with the `file://` prefix) // Default: "" + * domain?: scalar|Param|null, // Default: "" + * select?: scalar|Param|null, // Default: "" + * passphrase?: scalar|Param|null, // The private key passphrase // Default: "" * options?: array, * }, * smime_signer?: bool|array{ // S/MIME signer configuration - * enabled?: bool, // Default: false - * key?: scalar|null, // Path to key (in PEM format) // Default: "" - * certificate?: scalar|null, // Path to certificate (in PEM format without the `file://` prefix) // Default: "" - * passphrase?: scalar|null, // The private key passphrase // Default: null - * extra_certificates?: scalar|null, // Default: null - * sign_options?: int, // Default: null + * enabled?: bool|Param, // Default: false + * key?: scalar|Param|null, // Path to key (in PEM format) // Default: "" + * certificate?: scalar|Param|null, // Path to certificate (in PEM format without the `file://` prefix) // Default: "" + * passphrase?: scalar|Param|null, // The private key passphrase // Default: null + * extra_certificates?: scalar|Param|null, // Default: null + * sign_options?: int|Param, // Default: null * }, * smime_encrypter?: bool|array{ // S/MIME encrypter configuration - * enabled?: bool, // Default: false - * repository?: scalar|null, // S/MIME certificate repository service. This service shall implement the `Symfony\Component\Mailer\EventListener\SmimeCertificateRepositoryInterface`. // Default: "" - * cipher?: int, // A set of algorithms used to encrypt the message // Default: null + * enabled?: bool|Param, // Default: false + * repository?: scalar|Param|null, // S/MIME certificate repository service. This service shall implement the `Symfony\Component\Mailer\EventListener\SmimeCertificateRepositoryInterface`. // Default: "" + * cipher?: int|Param, // A set of algorithms used to encrypt the message // Default: null * }, * }, * secrets?: bool|array{ - * enabled?: bool, // Default: true - * vault_directory?: scalar|null, // Default: "%kernel.project_dir%/config/secrets/%kernel.runtime_environment%" - * local_dotenv_file?: scalar|null, // Default: "%kernel.project_dir%/.env.%kernel.runtime_environment%.local" - * decryption_env_var?: scalar|null, // Default: "base64:default::SYMFONY_DECRYPTION_SECRET" + * enabled?: bool|Param, // Default: true + * vault_directory?: scalar|Param|null, // Default: "%kernel.project_dir%/config/secrets/%kernel.runtime_environment%" + * local_dotenv_file?: scalar|Param|null, // Default: "%kernel.project_dir%/.env.%kernel.environment%.local" + * decryption_env_var?: scalar|Param|null, // Default: "base64:default::SYMFONY_DECRYPTION_SECRET" * }, * notifier?: bool|array{ // Notifier configuration - * enabled?: bool, // Default: false - * message_bus?: scalar|null, // The message bus to use. Defaults to the default bus if the Messenger component is installed. // Default: null - * chatter_transports?: array, - * texter_transports?: array, - * notification_on_failed_messages?: bool, // Default: false - * channel_policy?: array>, + * enabled?: bool|Param, // Default: false + * message_bus?: scalar|Param|null, // The message bus to use. Defaults to the default bus if the Messenger component is installed. // Default: null + * chatter_transports?: array, + * texter_transports?: array, + * notification_on_failed_messages?: bool|Param, // Default: false + * channel_policy?: array>, * admin_recipients?: list, * }, * rate_limiter?: bool|array{ // Rate limiter configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * limiters?: array, - * limit?: int, // The maximum allowed hits in a fixed interval or burst. - * interval?: scalar|null, // Configures the fixed interval if "policy" is set to "fixed_window" or "sliding_window". The value must be a number followed by "second", "minute", "hour", "day", "week" or "month" (or their plural equivalent). + * lock_factory?: scalar|Param|null, // The service ID of the lock factory used by this limiter (or null to disable locking). // Default: "auto" + * cache_pool?: scalar|Param|null, // The cache pool to use for storing the current limiter state. // Default: "cache.rate_limiter" + * storage_service?: scalar|Param|null, // The service ID of a custom storage implementation, this precedes any configured "cache_pool". // Default: null + * policy?: "fixed_window"|"token_bucket"|"sliding_window"|"compound"|"no_limit"|Param, // The algorithm to be used by this limiter. + * limiters?: string|list, + * limit?: int|Param, // The maximum allowed hits in a fixed interval or burst. + * interval?: scalar|Param|null, // Configures the fixed interval if "policy" is set to "fixed_window" or "sliding_window". The value must be a number followed by "second", "minute", "hour", "day", "week" or "month" (or their plural equivalent). * rate?: array{ // Configures the fill rate if "policy" is set to "token_bucket". - * interval?: scalar|null, // Configures the rate interval. The value must be a number followed by "second", "minute", "hour", "day", "week" or "month" (or their plural equivalent). - * amount?: int, // Amount of tokens to add each interval. // Default: 1 + * interval?: scalar|Param|null, // Configures the rate interval. The value must be a number followed by "second", "minute", "hour", "day", "week" or "month" (or their plural equivalent). + * amount?: int|Param, // Amount of tokens to add each interval. // Default: 1 * }, * }>, * }, * uid?: bool|array{ // Uid configuration - * enabled?: bool, // Default: true - * default_uuid_version?: 7|6|4|1, // Default: 7 - * name_based_uuid_version?: 5|3, // Default: 5 - * name_based_uuid_namespace?: scalar|null, - * time_based_uuid_version?: 7|6|1, // Default: 7 - * time_based_uuid_node?: scalar|null, + * enabled?: bool|Param, // Default: true + * default_uuid_version?: 7|6|4|1|Param, // Default: 7 + * name_based_uuid_version?: 5|3|Param, // Default: 5 + * name_based_uuid_namespace?: scalar|Param|null, + * time_based_uuid_version?: 7|6|1|Param, // Default: 7 + * time_based_uuid_node?: scalar|Param|null, * }, * html_sanitizer?: bool|array{ // HtmlSanitizer configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * sanitizers?: array, - * block_elements?: list, - * drop_elements?: list, + * block_elements?: string|list, + * drop_elements?: string|list, * allow_attributes?: array, * drop_attributes?: array, - * force_attributes?: array>, - * force_https_urls?: bool, // Transforms URLs using the HTTP scheme to use the HTTPS scheme instead. // Default: false - * allowed_link_schemes?: list, - * allowed_link_hosts?: list|null, - * allow_relative_links?: bool, // Allows relative URLs to be used in links href attributes. // Default: false - * allowed_media_schemes?: list, - * allowed_media_hosts?: list|null, - * allow_relative_medias?: bool, // Allows relative URLs to be used in media source attributes (img, audio, video, ...). // Default: false - * with_attribute_sanitizers?: list, - * without_attribute_sanitizers?: list, - * max_input_length?: int, // The maximum length allowed for the sanitized input. // Default: 0 + * force_attributes?: array>, + * force_https_urls?: bool|Param, // Transforms URLs using the HTTP scheme to use the HTTPS scheme instead. // Default: false + * allowed_link_schemes?: string|list, + * allowed_link_hosts?: null|string|list, + * allow_relative_links?: bool|Param, // Allows relative URLs to be used in links href attributes. // Default: false + * allowed_media_schemes?: string|list, + * allowed_media_hosts?: null|string|list, + * allow_relative_medias?: bool|Param, // Allows relative URLs to be used in media source attributes (img, audio, video, ...). // Default: false + * with_attribute_sanitizers?: string|list, + * without_attribute_sanitizers?: string|list, + * max_input_length?: int|Param, // The maximum length allowed for the sanitized input. // Default: 0 * }>, * }, * webhook?: bool|array{ // Webhook configuration - * enabled?: bool, // Default: false - * message_bus?: scalar|null, // The message bus to use. // Default: "messenger.default_bus" + * enabled?: bool|Param, // Default: false + * message_bus?: scalar|Param|null, // The message bus to use. // Default: "messenger.default_bus" * routing?: array, * }, * remote-event?: bool|array{ // RemoteEvent configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, * json_streamer?: bool|array{ // JSON streamer configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, * } * @psalm-type EcotoneConfig = array{ - * serviceName?: scalar|null, // Default: null - * cacheConfiguration?: bool, // Default: false - * failFast?: bool, // Default: false - * test?: bool, // Default: false - * loadSrcNamespaces?: bool, // Default: true - * defaultSerializationMediaType?: scalar|null, // Default: null - * defaultErrorChannel?: scalar|null, // Default: null - * namespaces?: list, - * defaultMemoryLimit?: int, // Default: null + * serviceName?: scalar|Param|null, // Default: null + * cacheConfiguration?: bool|Param, // Default: false + * failFast?: bool|Param, // Default: false + * test?: bool|Param, // Default: false + * loadSrcNamespaces?: bool|Param, // Default: true + * defaultSerializationMediaType?: scalar|Param|null, // Default: null + * defaultErrorChannel?: scalar|Param|null, // Default: null + * namespaces?: list, + * defaultMemoryLimit?: int|Param, // Default: null * defaultConnectionExceptionRetry?: array{ - * initialDelay: int, - * maxAttempts: int, - * multiplier: int, + * initialDelay?: int|Param, + * maxAttempts?: int|Param, + * multiplier?: int|Param, * }, - * licenceKey?: scalar|null, // Default: null - * skippedModulePackageNames?: list, + * licenceKey?: scalar|Param|null, // Default: null + * skippedModulePackageNames?: list, * } * @psalm-type ConfigType = array{ * imports?: ImportsConfig, @@ -736,7 +738,10 @@ final class App */ public static function config(array $config): array { - return AppReference::config($config); + /** @var ConfigType $config */ + $config = AppReference::config($config); + + return $config; } } diff --git a/Monorepo/ExampleAppEventSourcing/Symfony/config/reference.php b/Monorepo/ExampleAppEventSourcing/Symfony/config/reference.php index fe5ba5be8..d0b355025 100644 --- a/Monorepo/ExampleAppEventSourcing/Symfony/config/reference.php +++ b/Monorepo/ExampleAppEventSourcing/Symfony/config/reference.php @@ -4,6 +4,8 @@ namespace Symfony\Component\DependencyInjection\Loader\Configurator; +use Symfony\Component\Config\Loader\ParamConfigurator as Param; + /** * This class provides array-shapes for configuring the services and bundles of an application. * @@ -31,7 +33,7 @@ * type?: string|null, * ignore_errors?: bool, * }> - * @psalm-type ParametersConfig = array|null>|null> + * @psalm-type ParametersConfig = array|Param|null>|Param|null> * @psalm-type ArgumentsType = list|array * @psalm-type CallType = array|array{0:string, 1?:ArgumentsType, 2?:bool}|array{method:string, arguments?:ArgumentsType, returns_clone?:bool} * @psalm-type TagsType = list>> // arrays inside the list must have only one element, with the tag name as the key @@ -119,592 +121,592 @@ * } * @psalm-type ServicesConfig = array{ * _defaults?: DefaultsType, - * _instanceof?: InstanceofType, + * _instanceof?: array, * ... * } * @psalm-type ExtensionType = array * @psalm-type FrameworkConfig = array{ - * secret?: scalar|null, - * http_method_override?: bool, // Set true to enable support for the '_method' request parameter to determine the intended HTTP method on POST requests. // Default: false - * allowed_http_method_override?: list|null, - * trust_x_sendfile_type_header?: scalar|null, // Set true to enable support for xsendfile in binary file responses. // Default: "%env(bool:default::SYMFONY_TRUST_X_SENDFILE_TYPE_HEADER)%" - * ide?: scalar|null, // Default: "%env(default::SYMFONY_IDE)%" - * test?: bool, - * default_locale?: scalar|null, // Default: "en" - * set_locale_from_accept_language?: bool, // Whether to use the Accept-Language HTTP header to set the Request locale (only when the "_locale" request attribute is not passed). // Default: false - * set_content_language_from_locale?: bool, // Whether to set the Content-Language HTTP header on the Response using the Request locale. // Default: false - * enabled_locales?: list, - * trusted_hosts?: list, + * secret?: scalar|Param|null, + * http_method_override?: bool|Param, // Set true to enable support for the '_method' request parameter to determine the intended HTTP method on POST requests. // Default: false + * allowed_http_method_override?: null|list, + * trust_x_sendfile_type_header?: scalar|Param|null, // Set true to enable support for xsendfile in binary file responses. // Default: "%env(bool:default::SYMFONY_TRUST_X_SENDFILE_TYPE_HEADER)%" + * ide?: scalar|Param|null, // Default: "%env(default::SYMFONY_IDE)%" + * test?: bool|Param, + * default_locale?: scalar|Param|null, // Default: "en" + * set_locale_from_accept_language?: bool|Param, // Whether to use the Accept-Language HTTP header to set the Request locale (only when the "_locale" request attribute is not passed). // Default: false + * set_content_language_from_locale?: bool|Param, // Whether to set the Content-Language HTTP header on the Response using the Request locale. // Default: false + * enabled_locales?: list, + * trusted_hosts?: string|list, * trusted_proxies?: mixed, // Default: ["%env(default::SYMFONY_TRUSTED_PROXIES)%"] - * trusted_headers?: list, - * error_controller?: scalar|null, // Default: "error_controller" - * handle_all_throwables?: bool, // HttpKernel will handle all kinds of \Throwable. // Default: true + * trusted_headers?: string|list, + * error_controller?: scalar|Param|null, // Default: "error_controller" + * handle_all_throwables?: bool|Param, // HttpKernel will handle all kinds of \Throwable. // Default: true * csrf_protection?: bool|array{ - * enabled?: scalar|null, // Default: null - * stateless_token_ids?: list, - * check_header?: scalar|null, // Whether to check the CSRF token in a header in addition to a cookie when using stateless protection. // Default: false - * cookie_name?: scalar|null, // The name of the cookie to use when using stateless protection. // Default: "csrf-token" + * enabled?: scalar|Param|null, // Default: null + * stateless_token_ids?: list, + * check_header?: scalar|Param|null, // Whether to check the CSRF token in a header in addition to a cookie when using stateless protection. // Default: false + * cookie_name?: scalar|Param|null, // The name of the cookie to use when using stateless protection. // Default: "csrf-token" * }, * form?: bool|array{ // Form configuration - * enabled?: bool, // Default: false - * csrf_protection?: array{ - * enabled?: scalar|null, // Default: null - * token_id?: scalar|null, // Default: null - * field_name?: scalar|null, // Default: "_token" - * field_attr?: array, + * enabled?: bool|Param, // Default: false + * csrf_protection?: bool|array{ + * enabled?: scalar|Param|null, // Default: null + * token_id?: scalar|Param|null, // Default: null + * field_name?: scalar|Param|null, // Default: "_token" + * field_attr?: array, * }, * }, * http_cache?: bool|array{ // HTTP cache configuration - * enabled?: bool, // Default: false - * debug?: bool, // Default: "%kernel.debug%" - * trace_level?: "none"|"short"|"full", - * trace_header?: scalar|null, - * default_ttl?: int, - * private_headers?: list, - * skip_response_headers?: list, - * allow_reload?: bool, - * allow_revalidate?: bool, - * stale_while_revalidate?: int, - * stale_if_error?: int, - * terminate_on_cache_hit?: bool, + * enabled?: bool|Param, // Default: false + * debug?: bool|Param, // Default: "%kernel.debug%" + * trace_level?: "none"|"short"|"full"|Param, + * trace_header?: scalar|Param|null, + * default_ttl?: int|Param, + * private_headers?: list, + * skip_response_headers?: list, + * allow_reload?: bool|Param, + * allow_revalidate?: bool|Param, + * stale_while_revalidate?: int|Param, + * stale_if_error?: int|Param, + * terminate_on_cache_hit?: bool|Param, * }, * esi?: bool|array{ // ESI configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, * ssi?: bool|array{ // SSI configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, * fragments?: bool|array{ // Fragments configuration - * enabled?: bool, // Default: false - * hinclude_default_template?: scalar|null, // Default: null - * path?: scalar|null, // Default: "/_fragment" + * enabled?: bool|Param, // Default: false + * hinclude_default_template?: scalar|Param|null, // Default: null + * path?: scalar|Param|null, // Default: "/_fragment" * }, * profiler?: bool|array{ // Profiler configuration - * enabled?: bool, // Default: false - * collect?: bool, // Default: true - * collect_parameter?: scalar|null, // The name of the parameter to use to enable or disable collection on a per request basis. // Default: null - * only_exceptions?: bool, // Default: false - * only_main_requests?: bool, // Default: false - * dsn?: scalar|null, // Default: "file:%kernel.cache_dir%/profiler" - * collect_serializer_data?: bool, // Enables the serializer data collector and profiler panel. // Default: false + * enabled?: bool|Param, // Default: false + * collect?: bool|Param, // Default: true + * collect_parameter?: scalar|Param|null, // The name of the parameter to use to enable or disable collection on a per request basis. // Default: null + * only_exceptions?: bool|Param, // Default: false + * only_main_requests?: bool|Param, // Default: false + * dsn?: scalar|Param|null, // Default: "file:%kernel.cache_dir%/profiler" + * collect_serializer_data?: bool|Param, // Enables the serializer data collector and profiler panel. // Default: false * }, * workflows?: bool|array{ - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * workflows?: array, - * definition_validators?: list, - * support_strategy?: scalar|null, - * initial_marking?: list, - * events_to_dispatch?: list|null, - * places?: list, + * supports?: string|list, + * definition_validators?: list, + * support_strategy?: scalar|Param|null, + * initial_marking?: \BackedEnum|string|list, + * events_to_dispatch?: null|list, + * places?: string|list, * }>, - * transitions: list, - * to?: list, - * weight?: int, // Default: 1 - * metadata?: list, + * weight?: int|Param, // Default: 1 + * metadata?: array, * }>, - * metadata?: list, + * metadata?: array, * }>, * }, * router?: bool|array{ // Router configuration - * enabled?: bool, // Default: false - * resource: scalar|null, - * type?: scalar|null, - * cache_dir?: scalar|null, // Deprecated: Setting the "framework.router.cache_dir.cache_dir" configuration option is deprecated. It will be removed in version 8.0. // Default: "%kernel.build_dir%" - * default_uri?: scalar|null, // The default URI used to generate URLs in a non-HTTP context. // Default: null - * http_port?: scalar|null, // Default: 80 - * https_port?: scalar|null, // Default: 443 - * strict_requirements?: scalar|null, // set to true to throw an exception when a parameter does not match the requirements set to false to disable exceptions when a parameter does not match the requirements (and return null instead) set to null to disable parameter checks against requirements 'true' is the preferred configuration in development mode, while 'false' or 'null' might be preferred in production // Default: true - * utf8?: bool, // Default: true + * enabled?: bool|Param, // Default: false + * resource?: scalar|Param|null, + * type?: scalar|Param|null, + * cache_dir?: scalar|Param|null, // Deprecated: Setting the "framework.router.cache_dir.cache_dir" configuration option is deprecated. It will be removed in version 8.0. // Default: "%kernel.build_dir%" + * default_uri?: scalar|Param|null, // The default URI used to generate URLs in a non-HTTP context. // Default: null + * http_port?: scalar|Param|null, // Default: 80 + * https_port?: scalar|Param|null, // Default: 443 + * strict_requirements?: scalar|Param|null, // set to true to throw an exception when a parameter does not match the requirements set to false to disable exceptions when a parameter does not match the requirements (and return null instead) set to null to disable parameter checks against requirements 'true' is the preferred configuration in development mode, while 'false' or 'null' might be preferred in production // Default: true + * utf8?: bool|Param, // Default: true * }, * session?: bool|array{ // Session configuration - * enabled?: bool, // Default: false - * storage_factory_id?: scalar|null, // Default: "session.storage.factory.native" - * handler_id?: scalar|null, // Defaults to using the native session handler, or to the native *file* session handler if "save_path" is not null. - * name?: scalar|null, - * cookie_lifetime?: scalar|null, - * cookie_path?: scalar|null, - * cookie_domain?: scalar|null, - * cookie_secure?: true|false|"auto", // Default: "auto" - * cookie_httponly?: bool, // Default: true - * cookie_samesite?: null|"lax"|"strict"|"none", // Default: "lax" - * use_cookies?: bool, - * gc_divisor?: scalar|null, - * gc_probability?: scalar|null, - * gc_maxlifetime?: scalar|null, - * save_path?: scalar|null, // Defaults to "%kernel.cache_dir%/sessions" if the "handler_id" option is not null. - * metadata_update_threshold?: int, // Seconds to wait between 2 session metadata updates. // Default: 0 - * sid_length?: int, // Deprecated: Setting the "framework.session.sid_length.sid_length" configuration option is deprecated. It will be removed in version 8.0. No alternative is provided as PHP 8.4 has deprecated the related option. - * sid_bits_per_character?: int, // Deprecated: Setting the "framework.session.sid_bits_per_character.sid_bits_per_character" configuration option is deprecated. It will be removed in version 8.0. No alternative is provided as PHP 8.4 has deprecated the related option. + * enabled?: bool|Param, // Default: false + * storage_factory_id?: scalar|Param|null, // Default: "session.storage.factory.native" + * handler_id?: scalar|Param|null, // Defaults to using the native session handler, or to the native *file* session handler if "save_path" is not null. + * name?: scalar|Param|null, + * cookie_lifetime?: scalar|Param|null, + * cookie_path?: scalar|Param|null, + * cookie_domain?: scalar|Param|null, + * cookie_secure?: true|false|"auto"|Param, // Default: "auto" + * cookie_httponly?: bool|Param, // Default: true + * cookie_samesite?: null|"lax"|"strict"|"none"|Param, // Default: "lax" + * use_cookies?: bool|Param, + * gc_divisor?: scalar|Param|null, + * gc_probability?: scalar|Param|null, + * gc_maxlifetime?: scalar|Param|null, + * save_path?: scalar|Param|null, // Defaults to "%kernel.cache_dir%/sessions" if the "handler_id" option is not null. + * metadata_update_threshold?: int|Param, // Seconds to wait between 2 session metadata updates. // Default: 0 + * sid_length?: int|Param, // Deprecated: Setting the "framework.session.sid_length.sid_length" configuration option is deprecated. It will be removed in version 8.0. No alternative is provided as PHP 8.4 has deprecated the related option. + * sid_bits_per_character?: int|Param, // Deprecated: Setting the "framework.session.sid_bits_per_character.sid_bits_per_character" configuration option is deprecated. It will be removed in version 8.0. No alternative is provided as PHP 8.4 has deprecated the related option. * }, * request?: bool|array{ // Request configuration - * enabled?: bool, // Default: false - * formats?: array>, + * enabled?: bool|Param, // Default: false + * formats?: array>, * }, * assets?: bool|array{ // Assets configuration - * enabled?: bool, // Default: false - * strict_mode?: bool, // Throw an exception if an entry is missing from the manifest.json. // Default: false - * version_strategy?: scalar|null, // Default: null - * version?: scalar|null, // Default: null - * version_format?: scalar|null, // Default: "%%s?%%s" - * json_manifest_path?: scalar|null, // Default: null - * base_path?: scalar|null, // Default: "" - * base_urls?: list, + * enabled?: bool|Param, // Default: false + * strict_mode?: bool|Param, // Throw an exception if an entry is missing from the manifest.json. // Default: false + * version_strategy?: scalar|Param|null, // Default: null + * version?: scalar|Param|null, // Default: null + * version_format?: scalar|Param|null, // Default: "%%s?%%s" + * json_manifest_path?: scalar|Param|null, // Default: null + * base_path?: scalar|Param|null, // Default: "" + * base_urls?: string|list, * packages?: array, + * strict_mode?: bool|Param, // Throw an exception if an entry is missing from the manifest.json. // Default: false + * version_strategy?: scalar|Param|null, // Default: null + * version?: scalar|Param|null, + * version_format?: scalar|Param|null, // Default: null + * json_manifest_path?: scalar|Param|null, // Default: null + * base_path?: scalar|Param|null, // Default: "" + * base_urls?: string|list, * }>, * }, * asset_mapper?: bool|array{ // Asset Mapper configuration - * enabled?: bool, // Default: false - * paths?: array, - * excluded_patterns?: list, - * exclude_dotfiles?: bool, // If true, any files starting with "." will be excluded from the asset mapper. // Default: true - * server?: bool, // If true, a "dev server" will return the assets from the public directory (true in "debug" mode only by default). // Default: true - * public_prefix?: scalar|null, // The public path where the assets will be written to (and served from when "server" is true). // Default: "/assets/" - * missing_import_mode?: "strict"|"warn"|"ignore", // Behavior if an asset cannot be found when imported from JavaScript or CSS files - e.g. "import './non-existent.js'". "strict" means an exception is thrown, "warn" means a warning is logged, "ignore" means the import is left as-is. // Default: "warn" - * extensions?: array, - * importmap_path?: scalar|null, // The path of the importmap.php file. // Default: "%kernel.project_dir%/importmap.php" - * importmap_polyfill?: scalar|null, // The importmap name that will be used to load the polyfill. Set to false to disable. // Default: "es-module-shims" - * importmap_script_attributes?: array, - * vendor_dir?: scalar|null, // The directory to store JavaScript vendors. // Default: "%kernel.project_dir%/assets/vendor" + * enabled?: bool|Param, // Default: false + * paths?: string|array, + * excluded_patterns?: list, + * exclude_dotfiles?: bool|Param, // If true, any files starting with "." will be excluded from the asset mapper. // Default: true + * server?: bool|Param, // If true, a "dev server" will return the assets from the public directory (true in "debug" mode only by default). // Default: true + * public_prefix?: scalar|Param|null, // The public path where the assets will be written to (and served from when "server" is true). // Default: "/assets/" + * missing_import_mode?: "strict"|"warn"|"ignore"|Param, // Behavior if an asset cannot be found when imported from JavaScript or CSS files - e.g. "import './non-existent.js'". "strict" means an exception is thrown, "warn" means a warning is logged, "ignore" means the import is left as-is. // Default: "warn" + * extensions?: array, + * importmap_path?: scalar|Param|null, // The path of the importmap.php file. // Default: "%kernel.project_dir%/importmap.php" + * importmap_polyfill?: scalar|Param|null, // The importmap name that will be used to load the polyfill. Set to false to disable. // Default: "es-module-shims" + * importmap_script_attributes?: array, + * vendor_dir?: scalar|Param|null, // The directory to store JavaScript vendors. // Default: "%kernel.project_dir%/assets/vendor" * precompress?: bool|array{ // Precompress assets with Brotli, Zstandard and gzip. - * enabled?: bool, // Default: false - * formats?: list, - * extensions?: list, + * enabled?: bool|Param, // Default: false + * formats?: list, + * extensions?: list, * }, * }, * translator?: bool|array{ // Translator configuration - * enabled?: bool, // Default: true - * fallbacks?: list, - * logging?: bool, // Default: false - * formatter?: scalar|null, // Default: "translator.formatter.default" - * cache_dir?: scalar|null, // Default: "%kernel.cache_dir%/translations" - * default_path?: scalar|null, // The default path used to load translations. // Default: "%kernel.project_dir%/translations" - * paths?: list, + * enabled?: bool|Param, // Default: true + * fallbacks?: string|list, + * logging?: bool|Param, // Default: false + * formatter?: scalar|Param|null, // Default: "translator.formatter.default" + * cache_dir?: scalar|Param|null, // Default: "%kernel.cache_dir%/translations" + * default_path?: scalar|Param|null, // The default path used to load translations. // Default: "%kernel.project_dir%/translations" + * paths?: list, * pseudo_localization?: bool|array{ - * enabled?: bool, // Default: false - * accents?: bool, // Default: true - * expansion_factor?: float, // Default: 1.0 - * brackets?: bool, // Default: true - * parse_html?: bool, // Default: false - * localizable_html_attributes?: list, + * enabled?: bool|Param, // Default: false + * accents?: bool|Param, // Default: true + * expansion_factor?: float|Param, // Default: 1.0 + * brackets?: bool|Param, // Default: true + * parse_html?: bool|Param, // Default: false + * localizable_html_attributes?: list, * }, * providers?: array, - * locales?: list, + * dsn?: scalar|Param|null, + * domains?: list, + * locales?: list, * }>, * globals?: array, - * domain?: string, + * message?: string|Param, + * parameters?: array, + * domain?: string|Param, * }>, * }, * validation?: bool|array{ // Validation configuration - * enabled?: bool, // Default: false - * cache?: scalar|null, // Deprecated: Setting the "framework.validation.cache.cache" configuration option is deprecated. It will be removed in version 8.0. - * enable_attributes?: bool, // Default: true - * static_method?: list, - * translation_domain?: scalar|null, // Default: "validators" - * email_validation_mode?: "html5"|"html5-allow-no-tld"|"strict"|"loose", // Default: "html5" + * enabled?: bool|Param, // Default: false + * cache?: scalar|Param|null, // Deprecated: Setting the "framework.validation.cache.cache" configuration option is deprecated. It will be removed in version 8.0. + * enable_attributes?: bool|Param, // Default: true + * static_method?: string|list, + * translation_domain?: scalar|Param|null, // Default: "validators" + * email_validation_mode?: "html5"|"html5-allow-no-tld"|"strict"|"loose"|Param, // Default: "html5" * mapping?: array{ - * paths?: list, + * paths?: list, * }, * not_compromised_password?: bool|array{ - * enabled?: bool, // When disabled, compromised passwords will be accepted as valid. // Default: true - * endpoint?: scalar|null, // API endpoint for the NotCompromisedPassword Validator. // Default: null + * enabled?: bool|Param, // When disabled, compromised passwords will be accepted as valid. // Default: true + * endpoint?: scalar|Param|null, // API endpoint for the NotCompromisedPassword Validator. // Default: null * }, - * disable_translation?: bool, // Default: false + * disable_translation?: bool|Param, // Default: false * auto_mapping?: array, + * services?: list, * }>, * }, * annotations?: bool|array{ - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, * serializer?: bool|array{ // Serializer configuration - * enabled?: bool, // Default: false - * enable_attributes?: bool, // Default: true - * name_converter?: scalar|null, - * circular_reference_handler?: scalar|null, - * max_depth_handler?: scalar|null, + * enabled?: bool|Param, // Default: false + * enable_attributes?: bool|Param, // Default: true + * name_converter?: scalar|Param|null, + * circular_reference_handler?: scalar|Param|null, + * max_depth_handler?: scalar|Param|null, * mapping?: array{ - * paths?: list, + * paths?: list, * }, - * default_context?: list, + * default_context?: array, * named_serializers?: array, - * include_built_in_normalizers?: bool, // Whether to include the built-in normalizers // Default: true - * include_built_in_encoders?: bool, // Whether to include the built-in encoders // Default: true + * name_converter?: scalar|Param|null, + * default_context?: array, + * include_built_in_normalizers?: bool|Param, // Whether to include the built-in normalizers // Default: true + * include_built_in_encoders?: bool|Param, // Whether to include the built-in encoders // Default: true * }>, * }, * property_access?: bool|array{ // Property access configuration - * enabled?: bool, // Default: false - * magic_call?: bool, // Default: false - * magic_get?: bool, // Default: true - * magic_set?: bool, // Default: true - * throw_exception_on_invalid_index?: bool, // Default: false - * throw_exception_on_invalid_property_path?: bool, // Default: true + * enabled?: bool|Param, // Default: false + * magic_call?: bool|Param, // Default: false + * magic_get?: bool|Param, // Default: true + * magic_set?: bool|Param, // Default: true + * throw_exception_on_invalid_index?: bool|Param, // Default: false + * throw_exception_on_invalid_property_path?: bool|Param, // Default: true * }, * type_info?: bool|array{ // Type info configuration - * enabled?: bool, // Default: false - * aliases?: array, + * enabled?: bool|Param, // Default: false + * aliases?: array, * }, * property_info?: bool|array{ // Property info configuration - * enabled?: bool, // Default: false - * with_constructor_extractor?: bool, // Registers the constructor extractor. + * enabled?: bool|Param, // Default: false + * with_constructor_extractor?: bool|Param, // Registers the constructor extractor. * }, * cache?: array{ // Cache configuration - * prefix_seed?: scalar|null, // Used to namespace cache keys when using several apps with the same shared backend. // Default: "_%kernel.project_dir%.%kernel.container_class%" - * app?: scalar|null, // App related cache pools configuration. // Default: "cache.adapter.filesystem" - * system?: scalar|null, // System related cache pools configuration. // Default: "cache.adapter.system" - * directory?: scalar|null, // Default: "%kernel.share_dir%/pools/app" - * default_psr6_provider?: scalar|null, - * default_redis_provider?: scalar|null, // Default: "redis://localhost" - * default_valkey_provider?: scalar|null, // Default: "valkey://localhost" - * default_memcached_provider?: scalar|null, // Default: "memcached://localhost" - * default_doctrine_dbal_provider?: scalar|null, // Default: "database_connection" - * default_pdo_provider?: scalar|null, // Default: null + * prefix_seed?: scalar|Param|null, // Used to namespace cache keys when using several apps with the same shared backend. // Default: "_%kernel.project_dir%.%kernel.container_class%" + * app?: scalar|Param|null, // App related cache pools configuration. // Default: "cache.adapter.filesystem" + * system?: scalar|Param|null, // System related cache pools configuration. // Default: "cache.adapter.system" + * directory?: scalar|Param|null, // Default: "%kernel.share_dir%/pools/app" + * default_psr6_provider?: scalar|Param|null, + * default_redis_provider?: scalar|Param|null, // Default: "redis://localhost" + * default_valkey_provider?: scalar|Param|null, // Default: "valkey://localhost" + * default_memcached_provider?: scalar|Param|null, // Default: "memcached://localhost" + * default_doctrine_dbal_provider?: scalar|Param|null, // Default: "database_connection" + * default_pdo_provider?: scalar|Param|null, // Default: null * pools?: array, - * tags?: scalar|null, // Default: null - * public?: bool, // Default: false - * default_lifetime?: scalar|null, // Default lifetime of the pool. - * provider?: scalar|null, // Overwrite the setting from the default provider for this adapter. - * early_expiration_message_bus?: scalar|null, - * clearer?: scalar|null, + * adapters?: string|list, + * tags?: scalar|Param|null, // Default: null + * public?: bool|Param, // Default: false + * default_lifetime?: scalar|Param|null, // Default lifetime of the pool. + * provider?: scalar|Param|null, // Overwrite the setting from the default provider for this adapter. + * early_expiration_message_bus?: scalar|Param|null, + * clearer?: scalar|Param|null, * }>, * }, * php_errors?: array{ // PHP errors handling configuration * log?: mixed, // Use the application logger instead of the PHP logger for logging PHP errors. // Default: true - * throw?: bool, // Throw PHP errors as \ErrorException instances. // Default: true + * throw?: bool|Param, // Throw PHP errors as \ErrorException instances. // Default: true * }, * exceptions?: array, * web_link?: bool|array{ // Web links configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, * lock?: bool|string|array{ // Lock configuration - * enabled?: bool, // Default: false - * resources?: array>, + * enabled?: bool|Param, // Default: false + * resources?: string|array>, * }, * semaphore?: bool|string|array{ // Semaphore configuration - * enabled?: bool, // Default: false - * resources?: array, + * enabled?: bool|Param, // Default: false + * resources?: string|array, * }, * messenger?: bool|array{ // Messenger configuration - * enabled?: bool, // Default: false - * routing?: array, + * enabled?: bool|Param, // Default: false + * routing?: array, * }>, * serializer?: array{ - * default_serializer?: scalar|null, // Service id to use as the default serializer for the transports. // Default: "messenger.transport.native_php_serializer" + * default_serializer?: scalar|Param|null, // Service id to use as the default serializer for the transports. // Default: "messenger.transport.native_php_serializer" * symfony_serializer?: array{ - * format?: scalar|null, // Serialization format for the messenger.transport.symfony_serializer service (which is not the serializer used by default). // Default: "json" + * format?: scalar|Param|null, // Serialization format for the messenger.transport.symfony_serializer service (which is not the serializer used by default). // Default: "json" * context?: array, * }, * }, * transports?: array, - * failure_transport?: scalar|null, // Transport name to send failed messages to (after all retries have failed). // Default: null + * dsn?: scalar|Param|null, + * serializer?: scalar|Param|null, // Service id of a custom serializer to use. // Default: null + * options?: array, + * failure_transport?: scalar|Param|null, // Transport name to send failed messages to (after all retries have failed). // Default: null * retry_strategy?: string|array{ - * service?: scalar|null, // Service id to override the retry strategy entirely. // Default: null - * max_retries?: int, // Default: 3 - * delay?: int, // Time in ms to delay (or the initial value when multiplier is used). // Default: 1000 - * multiplier?: float, // If greater than 1, delay will grow exponentially for each retry: this delay = (delay * (multiple ^ retries)). // Default: 2 - * max_delay?: int, // Max time in ms that a retry should ever be delayed (0 = infinite). // Default: 0 - * jitter?: float, // Randomness to apply to the delay (between 0 and 1). // Default: 0.1 + * service?: scalar|Param|null, // Service id to override the retry strategy entirely. // Default: null + * max_retries?: int|Param, // Default: 3 + * delay?: int|Param, // Time in ms to delay (or the initial value when multiplier is used). // Default: 1000 + * multiplier?: float|Param, // If greater than 1, delay will grow exponentially for each retry: this delay = (delay * (multiple ^ retries)). // Default: 2 + * max_delay?: int|Param, // Max time in ms that a retry should ever be delayed (0 = infinite). // Default: 0 + * jitter?: float|Param, // Randomness to apply to the delay (between 0 and 1). // Default: 0.1 * }, - * rate_limiter?: scalar|null, // Rate limiter name to use when processing messages. // Default: null + * rate_limiter?: scalar|Param|null, // Rate limiter name to use when processing messages. // Default: null * }>, - * failure_transport?: scalar|null, // Transport name to send failed messages to (after all retries have failed). // Default: null - * stop_worker_on_signals?: list, - * default_bus?: scalar|null, // Default: null + * failure_transport?: scalar|Param|null, // Transport name to send failed messages to (after all retries have failed). // Default: null + * stop_worker_on_signals?: int|string|list, + * default_bus?: scalar|Param|null, // Default: null * buses?: array, * }>, * }>, * }, * scheduler?: bool|array{ // Scheduler configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, - * disallow_search_engine_index?: bool, // Enabled by default when debug is enabled. // Default: true + * disallow_search_engine_index?: bool|Param, // Enabled by default when debug is enabled. // Default: true * http_client?: bool|array{ // HTTP Client configuration - * enabled?: bool, // Default: false - * max_host_connections?: int, // The maximum number of connections to a single host. + * enabled?: bool|Param, // Default: false + * max_host_connections?: int|Param, // The maximum number of connections to a single host. * default_options?: array{ * headers?: array, * vars?: array, - * max_redirects?: int, // The maximum number of redirects to follow. - * http_version?: scalar|null, // The default HTTP version, typically 1.1 or 2.0, leave to null for the best version. - * resolve?: array, - * proxy?: scalar|null, // The URL of the proxy to pass requests through or null for automatic detection. - * no_proxy?: scalar|null, // A comma separated list of hosts that do not require a proxy to be reached. - * timeout?: float, // The idle timeout, defaults to the "default_socket_timeout" ini parameter. - * max_duration?: float, // The maximum execution time for the request+response as a whole. - * bindto?: scalar|null, // A network interface name, IP address, a host name or a UNIX socket to bind to. - * verify_peer?: bool, // Indicates if the peer should be verified in a TLS context. - * verify_host?: bool, // Indicates if the host should exist as a certificate common name. - * cafile?: scalar|null, // A certificate authority file. - * capath?: scalar|null, // A directory that contains multiple certificate authority files. - * local_cert?: scalar|null, // A PEM formatted certificate file. - * local_pk?: scalar|null, // A private key file. - * passphrase?: scalar|null, // The passphrase used to encrypt the "local_pk" file. - * ciphers?: scalar|null, // A list of TLS ciphers separated by colons, commas or spaces (e.g. "RC3-SHA:TLS13-AES-128-GCM-SHA256"...) + * max_redirects?: int|Param, // The maximum number of redirects to follow. + * http_version?: scalar|Param|null, // The default HTTP version, typically 1.1 or 2.0, leave to null for the best version. + * resolve?: array, + * proxy?: scalar|Param|null, // The URL of the proxy to pass requests through or null for automatic detection. + * no_proxy?: scalar|Param|null, // A comma separated list of hosts that do not require a proxy to be reached. + * timeout?: float|Param, // The idle timeout, defaults to the "default_socket_timeout" ini parameter. + * max_duration?: float|Param, // The maximum execution time for the request+response as a whole. + * bindto?: scalar|Param|null, // A network interface name, IP address, a host name or a UNIX socket to bind to. + * verify_peer?: bool|Param, // Indicates if the peer should be verified in a TLS context. + * verify_host?: bool|Param, // Indicates if the host should exist as a certificate common name. + * cafile?: scalar|Param|null, // A certificate authority file. + * capath?: scalar|Param|null, // A directory that contains multiple certificate authority files. + * local_cert?: scalar|Param|null, // A PEM formatted certificate file. + * local_pk?: scalar|Param|null, // A private key file. + * passphrase?: scalar|Param|null, // The passphrase used to encrypt the "local_pk" file. + * ciphers?: scalar|Param|null, // A list of TLS ciphers separated by colons, commas or spaces (e.g. "RC3-SHA:TLS13-AES-128-GCM-SHA256"...) * peer_fingerprint?: array{ // Associative array: hashing algorithm => hash(es). * sha1?: mixed, * pin-sha256?: mixed, * md5?: mixed, * }, - * crypto_method?: scalar|null, // The minimum version of TLS to accept; must be one of STREAM_CRYPTO_METHOD_TLSv*_CLIENT constants. + * crypto_method?: scalar|Param|null, // The minimum version of TLS to accept; must be one of STREAM_CRYPTO_METHOD_TLSv*_CLIENT constants. * extra?: array, - * rate_limiter?: scalar|null, // Rate limiter name to use for throttling requests. // Default: null + * rate_limiter?: scalar|Param|null, // Rate limiter name to use for throttling requests. // Default: null * caching?: bool|array{ // Caching configuration. - * enabled?: bool, // Default: false - * cache_pool?: string, // The taggable cache pool to use for storing the responses. // Default: "cache.http_client" - * shared?: bool, // Indicates whether the cache is shared (public) or private. // Default: true - * max_ttl?: int, // The maximum TTL (in seconds) allowed for cached responses. Null means no cap. // Default: null + * enabled?: bool|Param, // Default: false + * cache_pool?: string|Param, // The taggable cache pool to use for storing the responses. // Default: "cache.http_client" + * shared?: bool|Param, // Indicates whether the cache is shared (public) or private. // Default: true + * max_ttl?: int|Param, // The maximum TTL (in seconds) allowed for cached responses. Null means no cap. // Default: null * }, * retry_failed?: bool|array{ - * enabled?: bool, // Default: false - * retry_strategy?: scalar|null, // service id to override the retry strategy. // Default: null - * http_codes?: array, + * enabled?: bool|Param, // Default: false + * retry_strategy?: scalar|Param|null, // service id to override the retry strategy. // Default: null + * http_codes?: int|string|array, * }>, - * max_retries?: int, // Default: 3 - * delay?: int, // Time in ms to delay (or the initial value when multiplier is used). // Default: 1000 - * multiplier?: float, // If greater than 1, delay will grow exponentially for each retry: delay * (multiple ^ retries). // Default: 2 - * max_delay?: int, // Max time in ms that a retry should ever be delayed (0 = infinite). // Default: 0 - * jitter?: float, // Randomness in percent (between 0 and 1) to apply to the delay. // Default: 0.1 + * max_retries?: int|Param, // Default: 3 + * delay?: int|Param, // Time in ms to delay (or the initial value when multiplier is used). // Default: 1000 + * multiplier?: float|Param, // If greater than 1, delay will grow exponentially for each retry: delay * (multiple ^ retries). // Default: 2 + * max_delay?: int|Param, // Max time in ms that a retry should ever be delayed (0 = infinite). // Default: 0 + * jitter?: float|Param, // Randomness in percent (between 0 and 1) to apply to the delay. // Default: 0.1 * }, * }, - * mock_response_factory?: scalar|null, // The id of the service that should generate mock responses. It should be either an invokable or an iterable. + * mock_response_factory?: scalar|Param|null, // The id of the service that should generate mock responses. It should be either an invokable or an iterable. * scoped_clients?: array, + * scope?: scalar|Param|null, // The regular expression that the request URL must match before adding the other options. When none is provided, the base URI is used instead. + * base_uri?: scalar|Param|null, // The URI to resolve relative URLs, following rules in RFC 3985, section 2. + * auth_basic?: scalar|Param|null, // An HTTP Basic authentication "username:password". + * auth_bearer?: scalar|Param|null, // A token enabling HTTP Bearer authorization. + * auth_ntlm?: scalar|Param|null, // A "username:password" pair to use Microsoft NTLM authentication (requires the cURL extension). + * query?: array, * headers?: array, - * max_redirects?: int, // The maximum number of redirects to follow. - * http_version?: scalar|null, // The default HTTP version, typically 1.1 or 2.0, leave to null for the best version. - * resolve?: array, - * proxy?: scalar|null, // The URL of the proxy to pass requests through or null for automatic detection. - * no_proxy?: scalar|null, // A comma separated list of hosts that do not require a proxy to be reached. - * timeout?: float, // The idle timeout, defaults to the "default_socket_timeout" ini parameter. - * max_duration?: float, // The maximum execution time for the request+response as a whole. - * bindto?: scalar|null, // A network interface name, IP address, a host name or a UNIX socket to bind to. - * verify_peer?: bool, // Indicates if the peer should be verified in a TLS context. - * verify_host?: bool, // Indicates if the host should exist as a certificate common name. - * cafile?: scalar|null, // A certificate authority file. - * capath?: scalar|null, // A directory that contains multiple certificate authority files. - * local_cert?: scalar|null, // A PEM formatted certificate file. - * local_pk?: scalar|null, // A private key file. - * passphrase?: scalar|null, // The passphrase used to encrypt the "local_pk" file. - * ciphers?: scalar|null, // A list of TLS ciphers separated by colons, commas or spaces (e.g. "RC3-SHA:TLS13-AES-128-GCM-SHA256"...). + * max_redirects?: int|Param, // The maximum number of redirects to follow. + * http_version?: scalar|Param|null, // The default HTTP version, typically 1.1 or 2.0, leave to null for the best version. + * resolve?: array, + * proxy?: scalar|Param|null, // The URL of the proxy to pass requests through or null for automatic detection. + * no_proxy?: scalar|Param|null, // A comma separated list of hosts that do not require a proxy to be reached. + * timeout?: float|Param, // The idle timeout, defaults to the "default_socket_timeout" ini parameter. + * max_duration?: float|Param, // The maximum execution time for the request+response as a whole. + * bindto?: scalar|Param|null, // A network interface name, IP address, a host name or a UNIX socket to bind to. + * verify_peer?: bool|Param, // Indicates if the peer should be verified in a TLS context. + * verify_host?: bool|Param, // Indicates if the host should exist as a certificate common name. + * cafile?: scalar|Param|null, // A certificate authority file. + * capath?: scalar|Param|null, // A directory that contains multiple certificate authority files. + * local_cert?: scalar|Param|null, // A PEM formatted certificate file. + * local_pk?: scalar|Param|null, // A private key file. + * passphrase?: scalar|Param|null, // The passphrase used to encrypt the "local_pk" file. + * ciphers?: scalar|Param|null, // A list of TLS ciphers separated by colons, commas or spaces (e.g. "RC3-SHA:TLS13-AES-128-GCM-SHA256"...). * peer_fingerprint?: array{ // Associative array: hashing algorithm => hash(es). * sha1?: mixed, * pin-sha256?: mixed, * md5?: mixed, * }, - * crypto_method?: scalar|null, // The minimum version of TLS to accept; must be one of STREAM_CRYPTO_METHOD_TLSv*_CLIENT constants. + * crypto_method?: scalar|Param|null, // The minimum version of TLS to accept; must be one of STREAM_CRYPTO_METHOD_TLSv*_CLIENT constants. * extra?: array, - * rate_limiter?: scalar|null, // Rate limiter name to use for throttling requests. // Default: null + * rate_limiter?: scalar|Param|null, // Rate limiter name to use for throttling requests. // Default: null * caching?: bool|array{ // Caching configuration. - * enabled?: bool, // Default: false - * cache_pool?: string, // The taggable cache pool to use for storing the responses. // Default: "cache.http_client" - * shared?: bool, // Indicates whether the cache is shared (public) or private. // Default: true - * max_ttl?: int, // The maximum TTL (in seconds) allowed for cached responses. Null means no cap. // Default: null + * enabled?: bool|Param, // Default: false + * cache_pool?: string|Param, // The taggable cache pool to use for storing the responses. // Default: "cache.http_client" + * shared?: bool|Param, // Indicates whether the cache is shared (public) or private. // Default: true + * max_ttl?: int|Param, // The maximum TTL (in seconds) allowed for cached responses. Null means no cap. // Default: null * }, * retry_failed?: bool|array{ - * enabled?: bool, // Default: false - * retry_strategy?: scalar|null, // service id to override the retry strategy. // Default: null - * http_codes?: array, + * enabled?: bool|Param, // Default: false + * retry_strategy?: scalar|Param|null, // service id to override the retry strategy. // Default: null + * http_codes?: int|string|array, * }>, - * max_retries?: int, // Default: 3 - * delay?: int, // Time in ms to delay (or the initial value when multiplier is used). // Default: 1000 - * multiplier?: float, // If greater than 1, delay will grow exponentially for each retry: delay * (multiple ^ retries). // Default: 2 - * max_delay?: int, // Max time in ms that a retry should ever be delayed (0 = infinite). // Default: 0 - * jitter?: float, // Randomness in percent (between 0 and 1) to apply to the delay. // Default: 0.1 + * max_retries?: int|Param, // Default: 3 + * delay?: int|Param, // Time in ms to delay (or the initial value when multiplier is used). // Default: 1000 + * multiplier?: float|Param, // If greater than 1, delay will grow exponentially for each retry: delay * (multiple ^ retries). // Default: 2 + * max_delay?: int|Param, // Max time in ms that a retry should ever be delayed (0 = infinite). // Default: 0 + * jitter?: float|Param, // Randomness in percent (between 0 and 1) to apply to the delay. // Default: 0.1 * }, * }>, * }, * mailer?: bool|array{ // Mailer configuration - * enabled?: bool, // Default: true - * message_bus?: scalar|null, // The message bus to use. Defaults to the default bus if the Messenger component is installed. // Default: null - * dsn?: scalar|null, // Default: null - * transports?: array, + * enabled?: bool|Param, // Default: true + * message_bus?: scalar|Param|null, // The message bus to use. Defaults to the default bus if the Messenger component is installed. // Default: null + * dsn?: scalar|Param|null, // Default: null + * transports?: array, * envelope?: array{ // Mailer Envelope configuration - * sender?: scalar|null, - * recipients?: list, - * allowed_recipients?: list, + * sender?: scalar|Param|null, + * recipients?: string|list, + * allowed_recipients?: string|list, * }, * headers?: array, * dkim_signer?: bool|array{ // DKIM signer configuration - * enabled?: bool, // Default: false - * key?: scalar|null, // Key content, or path to key (in PEM format with the `file://` prefix) // Default: "" - * domain?: scalar|null, // Default: "" - * select?: scalar|null, // Default: "" - * passphrase?: scalar|null, // The private key passphrase // Default: "" + * enabled?: bool|Param, // Default: false + * key?: scalar|Param|null, // Key content, or path to key (in PEM format with the `file://` prefix) // Default: "" + * domain?: scalar|Param|null, // Default: "" + * select?: scalar|Param|null, // Default: "" + * passphrase?: scalar|Param|null, // The private key passphrase // Default: "" * options?: array, * }, * smime_signer?: bool|array{ // S/MIME signer configuration - * enabled?: bool, // Default: false - * key?: scalar|null, // Path to key (in PEM format) // Default: "" - * certificate?: scalar|null, // Path to certificate (in PEM format without the `file://` prefix) // Default: "" - * passphrase?: scalar|null, // The private key passphrase // Default: null - * extra_certificates?: scalar|null, // Default: null - * sign_options?: int, // Default: null + * enabled?: bool|Param, // Default: false + * key?: scalar|Param|null, // Path to key (in PEM format) // Default: "" + * certificate?: scalar|Param|null, // Path to certificate (in PEM format without the `file://` prefix) // Default: "" + * passphrase?: scalar|Param|null, // The private key passphrase // Default: null + * extra_certificates?: scalar|Param|null, // Default: null + * sign_options?: int|Param, // Default: null * }, * smime_encrypter?: bool|array{ // S/MIME encrypter configuration - * enabled?: bool, // Default: false - * repository?: scalar|null, // S/MIME certificate repository service. This service shall implement the `Symfony\Component\Mailer\EventListener\SmimeCertificateRepositoryInterface`. // Default: "" - * cipher?: int, // A set of algorithms used to encrypt the message // Default: null + * enabled?: bool|Param, // Default: false + * repository?: scalar|Param|null, // S/MIME certificate repository service. This service shall implement the `Symfony\Component\Mailer\EventListener\SmimeCertificateRepositoryInterface`. // Default: "" + * cipher?: int|Param, // A set of algorithms used to encrypt the message // Default: null * }, * }, * secrets?: bool|array{ - * enabled?: bool, // Default: true - * vault_directory?: scalar|null, // Default: "%kernel.project_dir%/config/secrets/%kernel.runtime_environment%" - * local_dotenv_file?: scalar|null, // Default: "%kernel.project_dir%/.env.%kernel.runtime_environment%.local" - * decryption_env_var?: scalar|null, // Default: "base64:default::SYMFONY_DECRYPTION_SECRET" + * enabled?: bool|Param, // Default: true + * vault_directory?: scalar|Param|null, // Default: "%kernel.project_dir%/config/secrets/%kernel.runtime_environment%" + * local_dotenv_file?: scalar|Param|null, // Default: "%kernel.project_dir%/.env.%kernel.environment%.local" + * decryption_env_var?: scalar|Param|null, // Default: "base64:default::SYMFONY_DECRYPTION_SECRET" * }, * notifier?: bool|array{ // Notifier configuration - * enabled?: bool, // Default: false - * message_bus?: scalar|null, // The message bus to use. Defaults to the default bus if the Messenger component is installed. // Default: null - * chatter_transports?: array, - * texter_transports?: array, - * notification_on_failed_messages?: bool, // Default: false - * channel_policy?: array>, + * enabled?: bool|Param, // Default: false + * message_bus?: scalar|Param|null, // The message bus to use. Defaults to the default bus if the Messenger component is installed. // Default: null + * chatter_transports?: array, + * texter_transports?: array, + * notification_on_failed_messages?: bool|Param, // Default: false + * channel_policy?: array>, * admin_recipients?: list, * }, * rate_limiter?: bool|array{ // Rate limiter configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * limiters?: array, - * limit?: int, // The maximum allowed hits in a fixed interval or burst. - * interval?: scalar|null, // Configures the fixed interval if "policy" is set to "fixed_window" or "sliding_window". The value must be a number followed by "second", "minute", "hour", "day", "week" or "month" (or their plural equivalent). + * lock_factory?: scalar|Param|null, // The service ID of the lock factory used by this limiter (or null to disable locking). // Default: "auto" + * cache_pool?: scalar|Param|null, // The cache pool to use for storing the current limiter state. // Default: "cache.rate_limiter" + * storage_service?: scalar|Param|null, // The service ID of a custom storage implementation, this precedes any configured "cache_pool". // Default: null + * policy?: "fixed_window"|"token_bucket"|"sliding_window"|"compound"|"no_limit"|Param, // The algorithm to be used by this limiter. + * limiters?: string|list, + * limit?: int|Param, // The maximum allowed hits in a fixed interval or burst. + * interval?: scalar|Param|null, // Configures the fixed interval if "policy" is set to "fixed_window" or "sliding_window". The value must be a number followed by "second", "minute", "hour", "day", "week" or "month" (or their plural equivalent). * rate?: array{ // Configures the fill rate if "policy" is set to "token_bucket". - * interval?: scalar|null, // Configures the rate interval. The value must be a number followed by "second", "minute", "hour", "day", "week" or "month" (or their plural equivalent). - * amount?: int, // Amount of tokens to add each interval. // Default: 1 + * interval?: scalar|Param|null, // Configures the rate interval. The value must be a number followed by "second", "minute", "hour", "day", "week" or "month" (or their plural equivalent). + * amount?: int|Param, // Amount of tokens to add each interval. // Default: 1 * }, * }>, * }, * uid?: bool|array{ // Uid configuration - * enabled?: bool, // Default: true - * default_uuid_version?: 7|6|4|1, // Default: 7 - * name_based_uuid_version?: 5|3, // Default: 5 - * name_based_uuid_namespace?: scalar|null, - * time_based_uuid_version?: 7|6|1, // Default: 7 - * time_based_uuid_node?: scalar|null, + * enabled?: bool|Param, // Default: true + * default_uuid_version?: 7|6|4|1|Param, // Default: 7 + * name_based_uuid_version?: 5|3|Param, // Default: 5 + * name_based_uuid_namespace?: scalar|Param|null, + * time_based_uuid_version?: 7|6|1|Param, // Default: 7 + * time_based_uuid_node?: scalar|Param|null, * }, * html_sanitizer?: bool|array{ // HtmlSanitizer configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * sanitizers?: array, - * block_elements?: list, - * drop_elements?: list, + * block_elements?: string|list, + * drop_elements?: string|list, * allow_attributes?: array, * drop_attributes?: array, - * force_attributes?: array>, - * force_https_urls?: bool, // Transforms URLs using the HTTP scheme to use the HTTPS scheme instead. // Default: false - * allowed_link_schemes?: list, - * allowed_link_hosts?: list|null, - * allow_relative_links?: bool, // Allows relative URLs to be used in links href attributes. // Default: false - * allowed_media_schemes?: list, - * allowed_media_hosts?: list|null, - * allow_relative_medias?: bool, // Allows relative URLs to be used in media source attributes (img, audio, video, ...). // Default: false - * with_attribute_sanitizers?: list, - * without_attribute_sanitizers?: list, - * max_input_length?: int, // The maximum length allowed for the sanitized input. // Default: 0 + * force_attributes?: array>, + * force_https_urls?: bool|Param, // Transforms URLs using the HTTP scheme to use the HTTPS scheme instead. // Default: false + * allowed_link_schemes?: string|list, + * allowed_link_hosts?: null|string|list, + * allow_relative_links?: bool|Param, // Allows relative URLs to be used in links href attributes. // Default: false + * allowed_media_schemes?: string|list, + * allowed_media_hosts?: null|string|list, + * allow_relative_medias?: bool|Param, // Allows relative URLs to be used in media source attributes (img, audio, video, ...). // Default: false + * with_attribute_sanitizers?: string|list, + * without_attribute_sanitizers?: string|list, + * max_input_length?: int|Param, // The maximum length allowed for the sanitized input. // Default: 0 * }>, * }, * webhook?: bool|array{ // Webhook configuration - * enabled?: bool, // Default: false - * message_bus?: scalar|null, // The message bus to use. // Default: "messenger.default_bus" + * enabled?: bool|Param, // Default: false + * message_bus?: scalar|Param|null, // The message bus to use. // Default: "messenger.default_bus" * routing?: array, * }, * remote-event?: bool|array{ // RemoteEvent configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, * json_streamer?: bool|array{ // JSON streamer configuration - * enabled?: bool, // Default: false + * enabled?: bool|Param, // Default: false * }, * } * @psalm-type EcotoneConfig = array{ - * serviceName?: scalar|null, // Default: null - * cacheConfiguration?: bool, // Default: false - * failFast?: bool, // Default: false - * test?: bool, // Default: false - * loadSrcNamespaces?: bool, // Default: true - * defaultSerializationMediaType?: scalar|null, // Default: null - * defaultErrorChannel?: scalar|null, // Default: null - * namespaces?: list, - * defaultMemoryLimit?: int, // Default: null + * serviceName?: scalar|Param|null, // Default: null + * cacheConfiguration?: bool|Param, // Default: false + * failFast?: bool|Param, // Default: false + * test?: bool|Param, // Default: false + * loadSrcNamespaces?: bool|Param, // Default: true + * defaultSerializationMediaType?: scalar|Param|null, // Default: null + * defaultErrorChannel?: scalar|Param|null, // Default: null + * namespaces?: list, + * defaultMemoryLimit?: int|Param, // Default: null * defaultConnectionExceptionRetry?: array{ - * initialDelay: int, - * maxAttempts: int, - * multiplier: int, + * initialDelay?: int|Param, + * maxAttempts?: int|Param, + * multiplier?: int|Param, * }, - * licenceKey?: scalar|null, // Default: null - * skippedModulePackageNames?: list, + * licenceKey?: scalar|Param|null, // Default: null + * skippedModulePackageNames?: list, * } * @psalm-type ConfigType = array{ * imports?: ImportsConfig, @@ -736,7 +738,10 @@ final class App */ public static function config(array $config): array { - return AppReference::config($config); + /** @var ConfigType $config */ + $config = AppReference::config($config); + + return $config; } } From 6689f2c16a5dc60472c74cb3f0022ac564c6189b Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 10/38] feat: asyncPublish gateway for Kafka and AMQP MessagePublishers, Kafka TCP_NODELAY default --- .../AmqpMessagePublisherConfiguration.php | 24 +++++++ .../Publisher/AmqpMessagePublisherModule.php | 4 ++ .../tests/Integration/AsyncPublishingTest.php | 64 +++++++++++++++++++ .../AsyncPublishGatewayRegistration.php | 54 ++++++++++++++++ .../InMemoryAsyncPublisherModule.php | 36 +---------- .../Kafka/src/Configuration/KafkaModule.php | 3 + .../KafkaPublisherConfiguration.php | 2 + .../tests/Integration/AsyncPublishingTest.php | 29 +++++++++ 8 files changed, 183 insertions(+), 33 deletions(-) create mode 100644 packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishGatewayRegistration.php diff --git a/packages/Amqp/src/Publisher/AmqpMessagePublisherConfiguration.php b/packages/Amqp/src/Publisher/AmqpMessagePublisherConfiguration.php index a846e6e9b..695604fe3 100644 --- a/packages/Amqp/src/Publisher/AmqpMessagePublisherConfiguration.php +++ b/packages/Amqp/src/Publisher/AmqpMessagePublisherConfiguration.php @@ -52,6 +52,10 @@ class AmqpMessagePublisherConfiguration */ private $defaultPersistentDelivery = true; + private bool $asyncPublishing = false; + + private ?int $asyncPublishingTimeout = null; + private function __construct(string $connectionReference, string $exchangeName, ?string $outputDefaultConversionMediaType, string $referenceName) { $this->connectionReference = $connectionReference; @@ -150,6 +154,26 @@ public function getDefaultPersistentDelivery(): bool return $this->defaultPersistentDelivery; } + public function withAsyncPublishing(bool $enabled = true, ?int $timeoutInMilliseconds = null): AmqpMessagePublisherConfiguration + { + $this->asyncPublishing = $enabled; + if ($timeoutInMilliseconds !== null) { + $this->asyncPublishingTimeout = $timeoutInMilliseconds; + } + + return $this; + } + + public function isAsyncPublishingEnabled(): bool + { + return $this->asyncPublishing; + } + + public function getAsyncPublishingTimeout(): ?int + { + return $this->asyncPublishingTimeout; + } + /** * @return bool */ diff --git a/packages/Amqp/src/Publisher/AmqpMessagePublisherModule.php b/packages/Amqp/src/Publisher/AmqpMessagePublisherModule.php index 6bf12efb2..9798c70c1 100644 --- a/packages/Amqp/src/Publisher/AmqpMessagePublisherModule.php +++ b/packages/Amqp/src/Publisher/AmqpMessagePublisherModule.php @@ -5,6 +5,7 @@ use Ecotone\Amqp\AmqpOutboundChannelAdapterBuilder; use Ecotone\AnnotationFinder\AnnotationFinder; use Ecotone\Messaging\Attribute\ModuleAnnotation; +use Ecotone\Messaging\Channel\AsyncPublishing\Config\AsyncPublishGatewayRegistration; use Ecotone\Messaging\Config\Annotation\AnnotationModule; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\ExtensionObjectResolver; use Ecotone\Messaging\Config\Configuration; @@ -88,7 +89,10 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ->withDefaultRoutingKey($amqpPublisher->getDefaultRoutingKey()) ->withRoutingKeyFromHeader($amqpPublisher->getRoutingKeyFromHeader()) ->withDefaultConversionMediaType($mediaType) + ->withAsyncPublishing($amqpPublisher->isAsyncPublishingEnabled(), $amqpPublisher->getAsyncPublishingTimeout()) ); + + AsyncPublishGatewayRegistration::registerFor($messagingConfiguration, $amqpPublisher->getReferenceName()); } } diff --git a/packages/Amqp/tests/Integration/AsyncPublishingTest.php b/packages/Amqp/tests/Integration/AsyncPublishingTest.php index ae3b1a03b..c0562d2d3 100644 --- a/packages/Amqp/tests/Integration/AsyncPublishingTest.php +++ b/packages/Amqp/tests/Integration/AsyncPublishingTest.php @@ -5,18 +5,23 @@ namespace Test\Ecotone\Amqp\Integration; use Ecotone\Amqp\AmqpBackedMessageChannelBuilder; +use Ecotone\Amqp\Publisher\AmqpMessagePublisherConfiguration; use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; +use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; +use Ecotone\Messaging\MessagePublisher; use Ecotone\Messaging\Support\LicensingException; use Ecotone\Modelling\Attribute\CommandHandler; use Ecotone\Modelling\Attribute\EventHandler; use Ecotone\Modelling\Attribute\QueryHandler; use Ecotone\Modelling\EventBus; use Ecotone\Test\LicenceTesting; +use Enqueue\AmqpExt\AmqpConnectionFactory; +use Enqueue\AmqpLib\AmqpConnectionFactory as AmqpLibConnection; use Symfony\Component\Uid\Uuid; use Test\Ecotone\Amqp\AmqpMessagingTestCase; use Test\Ecotone\Amqp\Fixture\AsyncPublishing\OrderWasPlaced; @@ -52,6 +57,65 @@ public function test_async_publishing_requires_enterprise_licence(): void $this->bootstrapEcotone($channelName, $orderService, licenceKey: null); } + public function test_message_publisher_async_publish_confirms_delivery_on_future_resolve(): void + { + $messaging = EcotoneLite::bootstrapFlowTesting( + [], + [...$this->getConnectionFactoryReferences()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::AMQP_PACKAGE])) + ->withExtensionObjects([ + AmqpMessagePublisherConfiguration::create() + ->withAutoDeclareQueueOnSend(true) + ->withDefaultRoutingKey(Uuid::v7()->toRfc4122()) + ->withAsyncPublishing(), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $singleFuture = $publisher->asyncPublish('single order'); + $batchFuture = $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('first order') + ->append('second order', ['priority' => '5']) + ); + + $this->assertNull($singleFuture->resolve()); + $this->assertNull($batchFuture->resolve()); + } + + public function test_batch_published_over_amqp_lib_connection_is_delivered(): void + { + $channelName = Uuid::v7()->toRfc4122(); + $orderService = $this->createOrderService($channelName); + $libConnectionFactory = new AmqpLibConnection(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']); + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [ + AmqpConnectionFactory::class => $libConnectionFactory, + AmqpLibConnection::class => $libConnectionFactory, + $orderService, + ], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::AMQP_PACKAGE])) + ->withExtensionObjects([ + AmqpBackedMessageChannelBuilder::create('asyncOrdersChannel', queueName: $channelName) + ->withAsyncPublishing(), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.place', 'espresso'); + + $messaging->run('asyncOrdersChannel', ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 3, maxExecutionTimeInMilliseconds: 10000)); + + $this->assertSame( + ['espresso-1', 'espresso-2', 'espresso-3'], + $messaging->sendQueryWithRouting('order.getReceived'), + ); + } + private function createOrderService(string $channelName): object { return new class ($channelName) { diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishGatewayRegistration.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishGatewayRegistration.php new file mode 100644 index 000000000..64a062085 --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishGatewayRegistration.php @@ -0,0 +1,54 @@ +registerGatewayBuilder( + GatewayProxyBuilder::create($publisherReferenceName, MessagePublisher::class, 'asyncPublish', $asyncPublishRequestChannel) + ->withParameterConverters([ + GatewayPayloadBuilder::create('data'), + GatewayHeaderBuilder::create('sourceMediaType', MessageHeaders::CONTENT_TYPE), + GatewayHeadersBuilder::create('metadata'), + ]) + ) + ->registerMessageChannel(SimpleMessageChannelBuilder::createDirectMessageChannel($asyncPublishRequestChannel)) + ->registerMessageHandler( + ServiceActivatorBuilder::createWithDefinition( + new Definition(AsyncPublishingGateway::class, [ + $publisherReferenceName, + new Reference(ConfiguredMessagingSystem::class), + new Reference(AsyncPublishingRegistry::class), + ]), + 'publish' + ) + ->withInputChannelName($asyncPublishRequestChannel) + ->withEndpointId($asyncPublishRequestChannel . '.endpoint') + ); + } +} diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublisherModule.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublisherModule.php index 314f91237..4e3a3ebb0 100644 --- a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublisherModule.php +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublisherModule.php @@ -6,25 +6,15 @@ use Ecotone\AnnotationFinder\AnnotationFinder; use Ecotone\Messaging\Attribute\ModuleAnnotation; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingGateway; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; +use Ecotone\Messaging\Channel\AsyncPublishing\Config\AsyncPublishGatewayRegistration; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; use Ecotone\Messaging\Config\Annotation\AnnotationModule; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\NoExternalConfigurationModule; use Ecotone\Messaging\Config\Configuration; -use Ecotone\Messaging\Config\ConfiguredMessagingSystem; -use Ecotone\Messaging\Config\Container\Definition; -use Ecotone\Messaging\Config\Container\Reference; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ModuleReferenceSearchService; -use Ecotone\Messaging\Handler\Gateway\GatewayProxyBuilder; -use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayHeaderBuilder; -use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayHeadersBuilder; -use Ecotone\Messaging\Handler\Gateway\ParameterToMessageConverter\GatewayPayloadBuilder; use Ecotone\Messaging\Handler\InterfaceToCallRegistry; use Ecotone\Messaging\Handler\ServiceActivator\ServiceActivatorBuilder; -use Ecotone\Messaging\MessageHeaders; -use Ecotone\Messaging\MessagePublisher; #[ModuleAnnotation] /** @@ -42,30 +32,10 @@ public static function create(AnnotationFinder $annotationRegistrationService, I public function prepare(Configuration $messagingConfiguration, array $extensionObjects, ModuleReferenceSearchService $moduleReferenceSearchService, InterfaceToCallRegistry $interfaceToCallRegistry): void { $publisherReference = self::PUBLISHER_REFERENCE; - $asyncPublishRequestChannel = $publisherReference . '.asyncPublish'; + + AsyncPublishGatewayRegistration::registerFor($messagingConfiguration, $publisherReference); $messagingConfiguration - ->registerGatewayBuilder( - GatewayProxyBuilder::create($publisherReference, MessagePublisher::class, 'asyncPublish', $asyncPublishRequestChannel) - ->withParameterConverters([ - GatewayPayloadBuilder::create('data'), - GatewayHeaderBuilder::create('sourceMediaType', MessageHeaders::CONTENT_TYPE), - GatewayHeadersBuilder::create('metadata'), - ]) - ) - ->registerMessageChannel(SimpleMessageChannelBuilder::createDirectMessageChannel($asyncPublishRequestChannel)) - ->registerMessageHandler( - ServiceActivatorBuilder::createWithDefinition( - new Definition(AsyncPublishingGateway::class, [ - $publisherReference, - new Reference(ConfiguredMessagingSystem::class), - new Reference(AsyncPublishingRegistry::class), - ]), - 'publish' - ) - ->withInputChannelName($asyncPublishRequestChannel) - ->withEndpointId($asyncPublishRequestChannel . '.endpoint') - ) ->registerMessageChannel(SimpleMessageChannelBuilder::createDirectMessageChannel($publisherReference)) ->registerMessageHandler( ServiceActivatorBuilder::create(InMemoryAsyncOutboundAdapter::class, 'handle') diff --git a/packages/Kafka/src/Configuration/KafkaModule.php b/packages/Kafka/src/Configuration/KafkaModule.php index b95e9ad3c..05f42f574 100644 --- a/packages/Kafka/src/Configuration/KafkaModule.php +++ b/packages/Kafka/src/Configuration/KafkaModule.php @@ -11,6 +11,7 @@ use Ecotone\Kafka\Inbound\KafkaInboundChannelAdapterBuilder; use Ecotone\Kafka\Outbound\KafkaOutboundChannelAdapterBuilder; use Ecotone\Messaging\Attribute\ModuleAnnotation; +use Ecotone\Messaging\Channel\AsyncPublishing\Config\AsyncPublishGatewayRegistration; use Ecotone\Messaging\Config\Annotation\AnnotationModule; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\ExtensionObjectResolver; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\NoExternalConfigurationModule; @@ -225,6 +226,8 @@ private function registerMessagePublisher(Configuration $messagingConfiguration, ->withHeaderMapper($extensionObject->getHeaderMapper()) ->withDefaultConversionMediaType($mediaType) ); + + AsyncPublishGatewayRegistration::registerFor($messagingConfiguration, $extensionObject->getReferenceName()); } private function getPublisherEndpointId(string $referenceName): string diff --git a/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php b/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php index 9e164fe47..104c84a9b 100644 --- a/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php +++ b/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php @@ -62,6 +62,8 @@ public static function createWithDefaults(string $topicName = '', string $refere 'retries' => '5', // Backoff time between retries in milliseconds 'retry.backoff.ms' => '300', + // Disables Nagle algorithm (TCP_NODELAY) so small produce requests are not delayed. Default in librdkafka only since v2.1 + 'socket.nagle.disable' => 'true', ], $brokerConfigurationReference, DefaultHeaderMapper::createAllHeadersMapping(), diff --git a/packages/Kafka/tests/Integration/AsyncPublishingTest.php b/packages/Kafka/tests/Integration/AsyncPublishingTest.php index 7459da730..4a89888c5 100644 --- a/packages/Kafka/tests/Integration/AsyncPublishingTest.php +++ b/packages/Kafka/tests/Integration/AsyncPublishingTest.php @@ -6,13 +6,16 @@ use Ecotone\Kafka\Channel\KafkaMessageChannelBuilder; use Ecotone\Kafka\Configuration\KafkaBrokerConfiguration; +use Ecotone\Kafka\Configuration\KafkaPublisherConfiguration; use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; +use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; +use Ecotone\Messaging\MessagePublisher; use Ecotone\Modelling\Attribute\CommandHandler; use Ecotone\Modelling\Attribute\EventHandler; use Ecotone\Modelling\Attribute\QueryHandler; @@ -62,6 +65,32 @@ public function test_failing_to_deliver_asynchronously_published_messages_throws $messaging->sendCommandWithRoutingKey('order.place', 'espresso'); } + public function test_message_publisher_async_publish_confirms_delivery_on_future_resolve(): void + { + $messaging = EcotoneLite::bootstrapFlowTesting( + [], + [KafkaBrokerConfiguration::class => ConnectionTestCase::getConnection()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::KAFKA_PACKAGE])) + ->withExtensionObjects([ + KafkaPublisherConfiguration::createWithDefaults(topicName: Uuid::v7()->toRfc4122()) + ->withAsyncPublishing(), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $singleFuture = $publisher->asyncPublish('single order'); + $batchFuture = $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('first order') + ->append('second order', ['priority' => '5']) + ); + + $this->assertNull($singleFuture->resolve()); + $this->assertNull($batchFuture->resolve()); + } + private function createOrderService(string $channelName): object { return new class ($channelName) { From f60e678f017d9f1516031a48965f4071eb848659 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 11/38] feat: AMQP frame batching publishes BatchMessage in single socket write --- .../Amqp/src/AmqpOutboundChannelAdapter.php | 81 ++++++++++++++++--- 1 file changed, 70 insertions(+), 11 deletions(-) diff --git a/packages/Amqp/src/AmqpOutboundChannelAdapter.php b/packages/Amqp/src/AmqpOutboundChannelAdapter.php index e2b68dff8..663f2f765 100644 --- a/packages/Amqp/src/AmqpOutboundChannelAdapter.php +++ b/packages/Amqp/src/AmqpOutboundChannelAdapter.php @@ -19,6 +19,8 @@ use Enqueue\AmqpTools\DelayStrategy; use Interop\Amqp\AmqpMessage; use Interop\Amqp\Impl\AmqpTopic; +use PhpAmqpLib\Message\AMQPMessage as LibAMQPMessage; +use PhpAmqpLib\Wire\AMQPTable; /** * @author Dariusz Gafka @@ -84,18 +86,24 @@ public function isAsyncPublishingEnabled(): bool private function handleBatch(BatchMessage $batchMessage, Message $carrierMessage): void { - $publishedMessages = []; + $entryMessages = []; foreach ($batchMessage->getEntries() as $entry) { - $entryMessage = MessageBuilder::withPayload($entry['payload']) + $entryMessages[] = MessageBuilder::withPayload($entry['payload']) ->setMultipleHeaders($entry['headers']) ->build(); + } - $this->publish($entryMessage); - $publishedMessages[] = $entryMessage; + $context = $this->connectionFactory->createContext(); + if ($context instanceof AmqpLibContext) { + $this->publishBatchThroughSingleWrite($entryMessages, $context); + } else { + foreach ($entryMessages as $entryMessage) { + $this->publish($entryMessage); + } } if ($this->canPublishAsynchronously()) { - $this->registerPendingDelivery($publishedMessages); + $this->registerPendingDelivery($entryMessages); return; } @@ -103,7 +111,57 @@ private function handleBatch(BatchMessage $batchMessage, Message $carrierMessage $this->awaitPublisherConfirmsSynchronously(); } + /** + * @param Message[] $messages + */ + private function publishBatchThroughSingleWrite(array $messages, AmqpLibContext $context): void + { + $libChannel = $context->getLibChannel(); + $anyMessageBatched = false; + + foreach ($messages as $message) { + [$interopMessage, $exchangeName, $deliveryDelay] = $this->prepareInteropMessage($message); + + if ($deliveryDelay) { + $this->publish($message); + + continue; + } + + $amqpProperties = $interopMessage->getHeaders(); + if ($applicationProperties = $interopMessage->getProperties()) { + $amqpProperties['application_headers'] = new AMQPTable($applicationProperties); + } + + $libChannel->batch_basic_publish( + new LibAMQPMessage($interopMessage->getBody(), $amqpProperties), + $exchangeName, + $interopMessage->getRoutingKey() ?? '', + ); + $anyMessageBatched = true; + } + + if ($anyMessageBatched) { + $libChannel->publish_batch(); + } + } + private function publish(Message $message): void + { + [$messageToSend, $exchangeName, $deliveryDelay, $timeToLive] = $this->prepareInteropMessage($message); + + $this->connectionFactory->getProducer() + ->setTimeToLive($timeToLive) + ->setDelayStrategy($this->delayStrategy ?? new HeadersExchangeDelayStrategy()) + ->setDeliveryDelay($deliveryDelay) +// this allow for having queue per delay instead of queue per delay + exchangeName + ->send(new AmqpTopic($exchangeName), $messageToSend); + } + + /** + * @return array{0: \Interop\Amqp\Impl\AmqpMessage, 1: string, 2: int|null, 3: int|null} + */ + private function prepareInteropMessage(Message $message): array { $exchangeName = $this->exchangeName; if ($this->exchangeFromHeaderName) { @@ -131,6 +189,11 @@ private function publish(Message $message): void $messageToSend->setRoutingKey($routingKey); } + $timeToLive = $outboundMessage->getTimeToLive(); + if ($timeToLive !== null && $messageToSend->getExpiration() === null) { + $messageToSend->setExpiration($timeToLive); + } + $messageToSend ->setDeliveryMode($this->defaultPersistentDelivery ? AmqpMessage::DELIVERY_MODE_PERSISTENT : AmqpMessage::DELIVERY_MODE_NON_PERSISTENT); @@ -139,12 +202,8 @@ private function publish(Message $message): void } $this->connectionFactory->createContext(); - $this->connectionFactory->getProducer() - ->setTimeToLive($outboundMessage->getTimeToLive()) - ->setDelayStrategy($this->delayStrategy ?? new HeadersExchangeDelayStrategy()) - ->setDeliveryDelay($outboundMessage->getDeliveryDelay()) -// this allow for having queue per delay instead of queue per delay + exchangeName - ->send(new AmqpTopic($exchangeName), $messageToSend); + + return [$messageToSend, $exchangeName, $outboundMessage->getDeliveryDelay(), $timeToLive]; } private function canPublishAsynchronously(): bool From b02e2b395d17cab7997da01b78b9f2f47531c955 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 12/38] perf: amortized registry pruning and MessagePublisher-based publishing benchmark --- .../AsyncPublishing/BenchmarkOrderPlaced.php | 12 -- .../AsyncPublishing/OrderPublisherService.php | 27 ---- .../Benchmark/AsyncPublishingBenchmark.php | 125 ++++++++++++------ .../AsyncPublishingGateway.php | 2 +- .../AsyncPublishingRegistry.php | 18 ++- 5 files changed, 93 insertions(+), 91 deletions(-) delete mode 100644 Monorepo/Benchmark/AsyncPublishing/BenchmarkOrderPlaced.php delete mode 100644 Monorepo/Benchmark/AsyncPublishing/OrderPublisherService.php diff --git a/Monorepo/Benchmark/AsyncPublishing/BenchmarkOrderPlaced.php b/Monorepo/Benchmark/AsyncPublishing/BenchmarkOrderPlaced.php deleted file mode 100644 index 1155c2b1b..000000000 --- a/Monorepo/Benchmark/AsyncPublishing/BenchmarkOrderPlaced.php +++ /dev/null @@ -1,12 +0,0 @@ -publish(new BenchmarkOrderPlaced((string) $orderNumber)); - } - } - - #[Asynchronous('benchmark_orders')] - #[EventHandler(endpointId: 'benchmark_order_consumer')] - public function consume(BenchmarkOrderPlaced $event): void - { - } -} diff --git a/Monorepo/Benchmark/AsyncPublishingBenchmark.php b/Monorepo/Benchmark/AsyncPublishingBenchmark.php index 7221f5260..5e1127c04 100644 --- a/Monorepo/Benchmark/AsyncPublishingBenchmark.php +++ b/Monorepo/Benchmark/AsyncPublishingBenchmark.php @@ -4,125 +4,162 @@ namespace Monorepo\Benchmark; -use Ecotone\Amqp\AmqpBackedMessageChannelBuilder; -use Ecotone\Dbal\Configuration\DbalConfiguration; -use Ecotone\Kafka\Channel\KafkaMessageChannelBuilder; +use Ecotone\Amqp\Publisher\AmqpMessagePublisherConfiguration; use Ecotone\Kafka\Configuration\KafkaBrokerConfiguration; +use Ecotone\Kafka\Configuration\KafkaPublisherConfiguration; use Ecotone\Lite\EcotoneLite; -use Ecotone\Lite\Test\FlowTestSupport; +use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; +use Ecotone\Messaging\Conversion\MediaType; +use Ecotone\Messaging\MessagePublisher; use Ecotone\Test\LicenceTesting; use Enqueue\AmqpExt\AmqpConnectionFactory; -use Enqueue\Dbal\DbalConnectionFactory; -use Monorepo\Benchmark\AsyncPublishing\OrderPublisherService; use PhpBench\Attributes\BeforeMethods; use PhpBench\Attributes\Iterations; use PhpBench\Attributes\Revs; use PhpBench\Attributes\Warmup; -#[Warmup(1), Revs(5), Iterations(3)] +#[Warmup(0), Revs(1), Iterations(10)] class AsyncPublishingBenchmark { - private const AMOUNT_OF_PUBLISHED_MESSAGES = 100; + private const AMOUNT_OF_PUBLISHED_MESSAGES = 1000; - private FlowTestSupport $messaging; + private const MESSAGE_PAYLOAD = 'benchmark order payload for async publishing comparison'; + + private MessagePublisher $publisher; public function setUpAmqpSynchronousPublishing(): void { - $this->messaging = $this->bootstrapWithAmqp(asyncPublishing: false); + $this->publisher = $this->bootstrapAmqpPublisher(asyncPublishing: false); + $this->warmUpPublisher(); } public function setUpAmqpAsyncPublishing(): void { - $this->messaging = $this->bootstrapWithAmqp(asyncPublishing: true); + $this->publisher = $this->bootstrapAmqpPublisher(asyncPublishing: true); + $this->warmUpPublisher(); } public function setUpKafkaSynchronousPublishing(): void { - $this->messaging = $this->bootstrapWithKafka(asyncPublishing: false); + $this->publisher = $this->bootstrapKafkaPublisher(asyncPublishing: false); + $this->warmUpPublisher(); } public function setUpKafkaAsyncPublishing(): void { - $this->messaging = $this->bootstrapWithKafka(asyncPublishing: true); + $this->publisher = $this->bootstrapKafkaPublisher(asyncPublishing: true); + $this->warmUpPublisher(); } #[BeforeMethods('setUpAmqpSynchronousPublishing')] public function bench_amqp_synchronous_publishing(): void { - $this->publishMessagesThroughCommandHandler(); + for ($messageNumber = 0; $messageNumber < self::AMOUNT_OF_PUBLISHED_MESSAGES; $messageNumber++) { + $this->publisher->send(self::MESSAGE_PAYLOAD); + } } #[BeforeMethods('setUpAmqpAsyncPublishing')] public function bench_amqp_async_publishing(): void { - $this->publishMessagesThroughCommandHandler(); + $this->publishAsynchronouslyOneByOne(); + } + + #[BeforeMethods('setUpAmqpAsyncPublishing')] + public function bench_amqp_async_batch_publishing(): void + { + $this->publishAsynchronouslyAsBatch(); } #[BeforeMethods('setUpKafkaSynchronousPublishing')] public function bench_kafka_synchronous_publishing(): void { - $this->publishMessagesThroughCommandHandler(); + for ($messageNumber = 0; $messageNumber < self::AMOUNT_OF_PUBLISHED_MESSAGES; $messageNumber++) { + $this->publisher->send(self::MESSAGE_PAYLOAD); + } } #[BeforeMethods('setUpKafkaAsyncPublishing')] public function bench_kafka_async_publishing(): void { - $this->publishMessagesThroughCommandHandler(); + $this->publishAsynchronouslyOneByOne(); } - private function publishMessagesThroughCommandHandler(): void + #[BeforeMethods('setUpKafkaAsyncPublishing')] + public function bench_kafka_async_batch_publishing(): void { - $this->messaging->sendCommandWithRoutingKey('benchmark.publishOrders', self::AMOUNT_OF_PUBLISHED_MESSAGES); + $this->publishAsynchronouslyAsBatch(); } - private function bootstrapWithAmqp(bool $asyncPublishing): FlowTestSupport + private function publishAsynchronouslyOneByOne(): void { - $channelBuilder = AmqpBackedMessageChannelBuilder::create('benchmark_orders', queueName: uniqid('benchmark_orders_')); + $futures = []; + for ($messageNumber = 0; $messageNumber < self::AMOUNT_OF_PUBLISHED_MESSAGES; $messageNumber++) { + $futures[] = $this->publisher->asyncPublish(self::MESSAGE_PAYLOAD, MediaType::TEXT_PLAIN); + } + foreach ($futures as $future) { + $future->resolve(); + } + } + + private function publishAsynchronouslyAsBatch(): void + { + $batch = BatchMessage::constructEmpty(); + for ($messageNumber = 0; $messageNumber < self::AMOUNT_OF_PUBLISHED_MESSAGES; $messageNumber++) { + $batch = $batch->append(self::MESSAGE_PAYLOAD, ['contentType' => MediaType::TEXT_PLAIN]); + } + + $this->publisher->asyncPublish($batch, MediaType::TEXT_PLAIN)->resolve(); + } + + private function warmUpPublisher(): void + { + $this->publisher->send(self::MESSAGE_PAYLOAD); + } + + private function bootstrapAmqpPublisher(bool $asyncPublishing): MessagePublisher + { + $publisherConfiguration = AmqpMessagePublisherConfiguration::create() + ->withAutoDeclareQueueOnSend(true) + ->withDefaultRoutingKey(uniqid('benchmark_orders_')); if ($asyncPublishing) { - $channelBuilder = $channelBuilder->withAsyncPublishing(); + $publisherConfiguration = $publisherConfiguration->withAsyncPublishing(); } - return EcotoneLite::bootstrapFlowTesting( - [OrderPublisherService::class], + $messaging = EcotoneLite::bootstrapFlowTesting( + [], [ - new OrderPublisherService(), AmqpConnectionFactory::class => new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']), - DbalConnectionFactory::class => new DbalConnectionFactory(getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone'), ], ServiceConfiguration::createWithDefaults() - ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::AMQP_PACKAGE, ModulePackageList::DBAL_PACKAGE])) - ->withExtensionObjects([ - $channelBuilder, - DbalConfiguration::createWithDefaults()->withTransactionOnCommandBus(true), - ]), + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::AMQP_PACKAGE])) + ->withExtensionObjects([$publisherConfiguration]), licenceKey: LicenceTesting::VALID_LICENCE, ); + + return $messaging->getGateway(MessagePublisher::class); } - private function bootstrapWithKafka(bool $asyncPublishing): FlowTestSupport + private function bootstrapKafkaPublisher(bool $asyncPublishing): MessagePublisher { - $topicName = uniqid('benchmark_orders_'); - $channelBuilder = KafkaMessageChannelBuilder::create('benchmark_orders', topicName: $topicName, messageGroupId: $topicName); + $publisherConfiguration = KafkaPublisherConfiguration::createWithDefaults(topicName: uniqid('benchmark_orders_')); if ($asyncPublishing) { - $channelBuilder = $channelBuilder->withAsyncPublishing(); + $publisherConfiguration = $publisherConfiguration->withAsyncPublishing(); } - return EcotoneLite::bootstrapFlowTesting( - [OrderPublisherService::class], + $messaging = EcotoneLite::bootstrapFlowTesting( + [], [ - new OrderPublisherService(), KafkaBrokerConfiguration::class => KafkaBrokerConfiguration::createWithDefaults([getenv('KAFKA_DSN') ?: 'localhost:9094']), - DbalConnectionFactory::class => new DbalConnectionFactory(getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone'), ], ServiceConfiguration::createWithDefaults() - ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::KAFKA_PACKAGE, ModulePackageList::DBAL_PACKAGE])) - ->withExtensionObjects([ - $channelBuilder, - DbalConfiguration::createWithDefaults()->withTransactionOnCommandBus(true), - ]), + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::KAFKA_PACKAGE])) + ->withExtensionObjects([$publisherConfiguration]), licenceKey: LicenceTesting::VALID_LICENCE, ); + + return $messaging->getGateway(MessagePublisher::class); } } diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php index a84ceebf1..9d1c91131 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php @@ -47,7 +47,7 @@ public function publish(Message $message): Future $pendingDeliveries = $this->asyncPublishingRegistry->registeredSince($collectionPoint); if (! $scopeWasActive) { - $this->asyncPublishingRegistry->markAsPublisherOwned($pendingDeliveries); + $this->asyncPublishingRegistry->markRegisteredSinceAsPublisherOwned($collectionPoint); } } finally { if (! $scopeWasActive) { diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php index aa9d926e0..916e41078 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php @@ -12,8 +12,12 @@ final class AsyncPublishingRegistry /** @var array */ private array $pendingDeliveries = []; + private const PRUNE_INTERVAL = 256; + private int $nextRegistrationIndex = 0; + private int $registrationsSinceLastPrune = 0; + private bool $scopeActive = false; private bool $shutdownFlushRegistered = false; @@ -38,13 +42,10 @@ public function closeScope(): void } } - /** - * @param PendingDelivery[] $pendingDeliveries - */ - public function markAsPublisherOwned(array $pendingDeliveries): void + public function markRegisteredSinceAsPublisherOwned(int $collectionPoint): void { - foreach ($this->pendingDeliveries as $index => $registration) { - if (in_array($registration['pendingDelivery'], $pendingDeliveries, true)) { + for ($index = $collectionPoint; $index < $this->nextRegistrationIndex; $index++) { + if (isset($this->pendingDeliveries[$index])) { $this->pendingDeliveries[$index]['scopeOwned'] = false; } } @@ -69,7 +70,10 @@ public function awaitAll(): DeliveryResult public function register(string $channelName, PendingDelivery $pendingDelivery): void { - $this->pruneAwaitedDeliveries(); + if (++$this->registrationsSinceLastPrune >= self::PRUNE_INTERVAL) { + $this->pruneAwaitedDeliveries(); + $this->registrationsSinceLastPrune = 0; + } $this->pendingDeliveries[$this->nextRegistrationIndex++] = ['channelName' => $channelName, 'pendingDelivery' => $pendingDelivery, 'scopeOwned' => $this->scopeActive]; if (! $this->shutdownFlushRegistered) { From e139dce80c25d973452b212d9f0eac8b41bb9bd0 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 13/38] test: async publishing without enterprise licence fails for channels and publishers --- .../tests/Integration/AsyncPublishingTest.php | 17 +++++++++++++++++ .../tests/Integration/AsyncPublishingTest.php | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/packages/Amqp/tests/Integration/AsyncPublishingTest.php b/packages/Amqp/tests/Integration/AsyncPublishingTest.php index c0562d2d3..fdb43c20d 100644 --- a/packages/Amqp/tests/Integration/AsyncPublishingTest.php +++ b/packages/Amqp/tests/Integration/AsyncPublishingTest.php @@ -57,6 +57,23 @@ public function test_async_publishing_requires_enterprise_licence(): void $this->bootstrapEcotone($channelName, $orderService, licenceKey: null); } + public function test_async_publishing_via_message_publisher_requires_enterprise_licence(): void + { + $this->expectException(LicensingException::class); + + EcotoneLite::bootstrapFlowTesting( + [], + [...$this->getConnectionFactoryReferences()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::AMQP_PACKAGE])) + ->withExtensionObjects([ + AmqpMessagePublisherConfiguration::create() + ->withDefaultRoutingKey(Uuid::v7()->toRfc4122()) + ->withAsyncPublishing(), + ]), + ); + } + public function test_message_publisher_async_publish_confirms_delivery_on_future_resolve(): void { $messaging = EcotoneLite::bootstrapFlowTesting( diff --git a/packages/Kafka/tests/Integration/AsyncPublishingTest.php b/packages/Kafka/tests/Integration/AsyncPublishingTest.php index 4a89888c5..9ae788750 100644 --- a/packages/Kafka/tests/Integration/AsyncPublishingTest.php +++ b/packages/Kafka/tests/Integration/AsyncPublishingTest.php @@ -16,6 +16,7 @@ use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; use Ecotone\Messaging\MessagePublisher; +use Ecotone\Messaging\Support\LicensingException; use Ecotone\Modelling\Attribute\CommandHandler; use Ecotone\Modelling\Attribute\EventHandler; use Ecotone\Modelling\Attribute\QueryHandler; @@ -65,6 +66,22 @@ public function test_failing_to_deliver_asynchronously_published_messages_throws $messaging->sendCommandWithRoutingKey('order.place', 'espresso'); } + public function test_async_publishing_requires_enterprise_licence(): void + { + $this->expectException(LicensingException::class); + + EcotoneLite::bootstrapFlowTesting( + [], + [KafkaBrokerConfiguration::class => ConnectionTestCase::getConnection()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::KAFKA_PACKAGE])) + ->withExtensionObjects([ + KafkaPublisherConfiguration::createWithDefaults(topicName: Uuid::v7()->toRfc4122()) + ->withAsyncPublishing(), + ]), + ); + } + public function test_message_publisher_async_publish_confirms_delivery_on_future_resolve(): void { $messaging = EcotoneLite::bootstrapFlowTesting( From 800ab62067ad11ef65101328d7dee2341bdd82fd Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 14/38] fix: reliability hardening for async publishing based on review findings --- .../src/AmqpBackedMessageChannelBuilder.php | 1 + .../src/AmqpExtPublisherConfirmations.php | 53 +++++++++++++++++++ .../Amqp/src/AmqpOutboundChannelAdapter.php | 41 +++++++++++++- .../src/AmqpOutboundChannelAdapterBuilder.php | 10 +++- packages/Amqp/src/AmqpPendingDelivery.php | 23 +++++++- .../AmqpReconnectableConnectionFactory.php | 18 ++++++- .../Publisher/AmqpMessagePublisherModule.php | 3 +- .../tests/Integration/AsyncPublishingTest.php | 30 +++++++++++ .../AsyncPublishingGateway.php | 5 ++ .../AsyncPublishingRegistry.php | 39 +++++++++++++- .../AsyncPublishingWaiterInterceptor.php | 5 ++ .../AsyncPublishGatewayRegistration.php | 3 +- .../AsyncPublishing/DeliveryFuture.php | 21 ++++++-- .../Kafka/src/Configuration/KafkaModule.php | 2 +- .../Outbound/KafkaOutboundChannelAdapter.php | 30 +++++++---- 15 files changed, 263 insertions(+), 21 deletions(-) create mode 100644 packages/Amqp/src/AmqpExtPublisherConfirmations.php diff --git a/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php b/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php index 710b81f5b..3aa4a822b 100644 --- a/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php +++ b/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php @@ -30,6 +30,7 @@ private function __construct( ->withDefaultRoutingKey($queueName) ->withAutoDeclareOnSend(true) ->withDefaultPersistentMode(true) + ->withAsyncPublishingChannelName($channelName) ); } diff --git a/packages/Amqp/src/AmqpExtPublisherConfirmations.php b/packages/Amqp/src/AmqpExtPublisherConfirmations.php new file mode 100644 index 000000000..723817eb7 --- /dev/null +++ b/packages/Amqp/src/AmqpExtPublisherConfirmations.php @@ -0,0 +1,53 @@ + */ + private array $individuallyConfirmedTags = []; + + public function recordPublishedMessage(): void + { + $this->publishedCount++; + } + + public function recordConfirmation(int $deliveryTag, bool $multiple): void + { + if ($multiple) { + $this->highestConfirmedTag = max($this->highestConfirmedTag, $deliveryTag); + foreach ($this->individuallyConfirmedTags as $tag => $confirmed) { + if ($tag <= $this->highestConfirmedTag) { + unset($this->individuallyConfirmedTags[$tag]); + } + } + + return; + } + + if ($deliveryTag > $this->highestConfirmedTag) { + $this->individuallyConfirmedTags[$deliveryTag] = true; + } + } + + public function hasOutstandingConfirmations(): bool + { + return $this->publishedCount > ($this->highestConfirmedTag + count($this->individuallyConfirmedTags)); + } + + public function reset(): void + { + $this->publishedCount = 0; + $this->highestConfirmedTag = 0; + $this->individuallyConfirmedTags = []; + } +} diff --git a/packages/Amqp/src/AmqpOutboundChannelAdapter.php b/packages/Amqp/src/AmqpOutboundChannelAdapter.php index 663f2f765..0f3aac4c1 100644 --- a/packages/Amqp/src/AmqpOutboundChannelAdapter.php +++ b/packages/Amqp/src/AmqpOutboundChannelAdapter.php @@ -21,6 +21,7 @@ use Interop\Amqp\Impl\AmqpTopic; use PhpAmqpLib\Message\AMQPMessage as LibAMQPMessage; use PhpAmqpLib\Wire\AMQPTable; +use RuntimeException; /** * @author Dariusz Gafka @@ -156,6 +157,28 @@ private function publish(Message $message): void ->setDeliveryDelay($deliveryDelay) // this allow for having queue per delay instead of queue per delay + exchangeName ->send(new AmqpTopic($exchangeName), $messageToSend); + + $this->recordExtPublishedMessage(); + } + + private function recordExtPublishedMessage(): void + { + if (! $this->publisherConfirms) { + return; + } + + if ($this->connectionFactory->createContext() instanceof AmqpExtContext) { + $this->getExtPublisherConfirmations()?->recordPublishedMessage(); + } + } + + private function getExtPublisherConfirmations(): ?AmqpExtPublisherConfirmations + { + $innerConnectionFactory = $this->connectionFactory->getInnerConnectionFactory(); + + return $innerConnectionFactory instanceof AmqpReconnectableConnectionFactory + ? $innerConnectionFactory->getExtPublisherConfirmations() + : null; } /** @@ -226,6 +249,7 @@ private function registerPendingDelivery(array $publishedMessages): void $publishedMessages, $this->asyncPublishingTimeout, $this->channelName, + $this->getExtPublisherConfirmations(), ), ); } @@ -240,7 +264,22 @@ private function awaitPublisherConfirmsSynchronously(): void if ($context instanceof AmqpLibContext) { $context->getLibChannel()->wait_for_pending_acks(5); } elseif ($context instanceof AmqpExtContext) { - $context->getExtChannel()->waitForConfirm(5); + $extPublisherConfirmations = $this->getExtPublisherConfirmations(); + if ($extPublisherConfirmations === null) { + $context->getExtChannel()->waitForConfirm(5); + + return; + } + + $deadline = microtime(true) + 5; + while ($extPublisherConfirmations->hasOutstandingConfirmations()) { + $remainingSeconds = $deadline - microtime(true); + if ($remainingSeconds <= 0) { + throw new RuntimeException('Timed out awaiting publisher confirms from RabbitMQ instance.'); + } + + $context->getExtChannel()->waitForConfirm($remainingSeconds); + } } } } diff --git a/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php b/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php index f50064c87..eb82b79c4 100644 --- a/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php +++ b/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php @@ -36,6 +36,7 @@ class AmqpOutboundChannelAdapterBuilder extends EnqueueOutboundChannelAdapterBui private ?string $delayStrategyReferenceName = null; private bool $asyncPublishing = false; private int $asyncPublishingTimeout = self::DEFAULT_ASYNC_PUBLISHING_TIMEOUT; + private ?string $asyncPublishingChannelName = null; private function __construct(string $exchangeName, string $amqpConnectionFactoryReferenceName) { @@ -88,6 +89,13 @@ public function isAsyncPublishingEnabled(): bool return $this->asyncPublishing; } + public function withAsyncPublishingChannelName(string $channelName): self + { + $this->asyncPublishingChannelName = $channelName; + + return $this; + } + public function withDelayStrategy(string $delayStrategyReferenceName): self { $this->delayStrategyReferenceName = $delayStrategyReferenceName; @@ -181,7 +189,7 @@ public function compile(MessagingContainerBuilder $builder): Definition new Reference(AsyncPublishingRegistry::class), $this->asyncPublishing, $this->asyncPublishingTimeout, - $this->exchangeName, + $this->asyncPublishingChannelName ?? $this->exchangeName, ]); } } diff --git a/packages/Amqp/src/AmqpPendingDelivery.php b/packages/Amqp/src/AmqpPendingDelivery.php index 025e333fc..c6e968329 100644 --- a/packages/Amqp/src/AmqpPendingDelivery.php +++ b/packages/Amqp/src/AmqpPendingDelivery.php @@ -11,6 +11,7 @@ use Enqueue\AmqpExt\AmqpContext as AmqpExtContext; use Enqueue\AmqpLib\AmqpContext as AmqpLibContext; use Interop\Amqp\AmqpContext; +use RuntimeException; use Throwable; /** @@ -28,6 +29,7 @@ public function __construct( private array $trackedMessages, private int $timeoutInMilliseconds, private string $channelName, + private ?AmqpExtPublisherConfirmations $extPublisherConfirmations = null, ) { } @@ -40,7 +42,7 @@ public function awaitDelivery(): DeliveryResult if ($this->context instanceof AmqpLibContext) { $this->context->getLibChannel()->wait_for_pending_acks($timeoutInSeconds); } elseif ($this->context instanceof AmqpExtContext) { - $this->context->getExtChannel()->waitForConfirm($timeoutInSeconds); + $this->awaitAllExtConfirmations($timeoutInSeconds); } } catch (Throwable $exception) { return DeliveryResult::withFailedDeliveries(array_map( @@ -52,6 +54,25 @@ public function awaitDelivery(): DeliveryResult return DeliveryResult::successful(); } + private function awaitAllExtConfirmations(float $timeoutInSeconds): void + { + if ($this->extPublisherConfirmations === null) { + $this->context->getExtChannel()->waitForConfirm($timeoutInSeconds); + + return; + } + + $deadline = microtime(true) + $timeoutInSeconds; + while ($this->extPublisherConfirmations->hasOutstandingConfirmations()) { + $remainingSeconds = $deadline - microtime(true); + if ($remainingSeconds <= 0) { + throw new RuntimeException('Timed out awaiting publisher confirms from RabbitMQ instance.'); + } + + $this->context->getExtChannel()->waitForConfirm($remainingSeconds); + } + } + public function isAwaited(): bool { return $this->awaited; diff --git a/packages/Amqp/src/AmqpReconnectableConnectionFactory.php b/packages/Amqp/src/AmqpReconnectableConnectionFactory.php index 205a4c934..f680328da 100644 --- a/packages/Amqp/src/AmqpReconnectableConnectionFactory.php +++ b/packages/Amqp/src/AmqpReconnectableConnectionFactory.php @@ -28,6 +28,7 @@ class AmqpReconnectableConnectionFactory implements ReconnectableConnectionFacto private string $connectionInstanceId; private AmqpConnectionFactory $connectionFactory; private ?SubscriptionConsumer $subscriptionConsumer = null; + private ?AmqpExtPublisherConfirmations $extPublisherConfirmations = null; public function __construct(AmqpExtConnectionFactory|AmqpLibConnectionFactory $connectionFactory, ?string $connectionInstanceId = null, private bool $publisherConfirms = false) { @@ -51,15 +52,30 @@ public function createContext(): Context if ($this->publisherConfirms) { if ($context instanceof AmqpLibContext) { $context->getLibChannel()->confirm_select(); + $context->getLibChannel()->set_nack_handler(fn () => throw new RuntimeException('Message was rejected (nack) by RabbitMQ instance. Check RabbitMQ server logs.')); } elseif ($context instanceof AmqpExtContext) { + $confirmations = $this->getExtPublisherConfirmations(); + $confirmations->reset(); $context->getExtChannel()->confirmSelect(); - $context->getExtChannel()->setConfirmCallback(fn () => false, fn () => throw new RuntimeException('Message was failed to be persisted in RabbitMQ instance. Check RabbitMQ server logs.')); + $context->getExtChannel()->setConfirmCallback( + function (int $deliveryTag, bool $multiple) use ($confirmations): bool { + $confirmations->recordConfirmation($deliveryTag, $multiple); + + return $confirmations->hasOutstandingConfirmations(); + }, + fn () => throw new RuntimeException('Message was failed to be persisted in RabbitMQ instance. Check RabbitMQ server logs.') + ); } } return $context; } + public function getExtPublisherConfirmations(): AmqpExtPublisherConfirmations + { + return $this->extPublisherConfirmations ??= new AmqpExtPublisherConfirmations(); + } + public function getConnectionInstanceId(): string { return get_class($this->connectionFactory) . $this->connectionInstanceId; diff --git a/packages/Amqp/src/Publisher/AmqpMessagePublisherModule.php b/packages/Amqp/src/Publisher/AmqpMessagePublisherModule.php index 9798c70c1..b08a4bd6d 100644 --- a/packages/Amqp/src/Publisher/AmqpMessagePublisherModule.php +++ b/packages/Amqp/src/Publisher/AmqpMessagePublisherModule.php @@ -90,9 +90,10 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ->withRoutingKeyFromHeader($amqpPublisher->getRoutingKeyFromHeader()) ->withDefaultConversionMediaType($mediaType) ->withAsyncPublishing($amqpPublisher->isAsyncPublishingEnabled(), $amqpPublisher->getAsyncPublishingTimeout()) + ->withAsyncPublishingChannelName($amqpPublisher->getReferenceName()) ); - AsyncPublishGatewayRegistration::registerFor($messagingConfiguration, $amqpPublisher->getReferenceName()); + AsyncPublishGatewayRegistration::registerFor($messagingConfiguration, $amqpPublisher->getReferenceName(), $amqpPublisher->isAsyncPublishingEnabled()); } } diff --git a/packages/Amqp/tests/Integration/AsyncPublishingTest.php b/packages/Amqp/tests/Integration/AsyncPublishingTest.php index fdb43c20d..dfdd4f2e7 100644 --- a/packages/Amqp/tests/Integration/AsyncPublishingTest.php +++ b/packages/Amqp/tests/Integration/AsyncPublishingTest.php @@ -10,9 +10,11 @@ use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\BatchMessage; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; +use Ecotone\Messaging\Endpoint\PollingMetadata; use Ecotone\Messaging\MessagePublisher; use Ecotone\Messaging\Support\LicensingException; use Ecotone\Modelling\Attribute\CommandHandler; @@ -74,6 +76,34 @@ public function test_async_publishing_via_message_publisher_requires_enterprise_ ); } + public function test_async_publish_on_publisher_without_async_configuration_throws_before_publishing(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = EcotoneLite::bootstrapFlowTesting( + [], + [...$this->getConnectionFactoryReferences()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::AMQP_PACKAGE])) + ->withExtensionObjects([ + AmqpMessagePublisherConfiguration::create() + ->withDefaultRoutingKey($queueName), + AmqpBackedMessageChannelBuilder::create('verificationChannel', queueName: $queueName), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $publishFailed = false; + try { + $publisher->asyncPublish('order that must not be published'); + } catch (AsyncPublishingFailedException) { + $publishFailed = true; + } + + $this->assertTrue($publishFailed); + $this->assertNull($messaging->getMessageChannel('verificationChannel')->receiveWithTimeout(PollingMetadata::create('verification')->setFixedRateInMilliseconds(200))); + } + public function test_message_publisher_async_publish_confirms_delivery_on_future_resolve(): void { $messaging = EcotoneLite::bootstrapFlowTesting( diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php index 9d1c91131..f2c8c8515 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php @@ -18,6 +18,7 @@ final class AsyncPublishingGateway { public function __construct( private string $publisherReference, + private bool $asyncPublishingEnabled, private ConfiguredMessagingSystem $configuredMessagingSystem, private AsyncPublishingRegistry $asyncPublishingRegistry, ) { @@ -25,6 +26,10 @@ public function __construct( public function publish(Message $message): Future { + if (! $this->asyncPublishingEnabled) { + throw AsyncPublishingFailedException::publisherNotConfiguredForAsyncPublishing($this->publisherReference); + } + $payload = $message->getPayload(); if ($payload instanceof BatchMessage && count($payload) === 0) { return DeliveryFuture::forPendingDeliveries([]); diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php index 916e41078..21a89f79f 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php @@ -14,6 +14,8 @@ final class AsyncPublishingRegistry private const PRUNE_INTERVAL = 256; + private const MAX_UNAWAITED_BACKLOG = 1024; + private int $nextRegistrationIndex = 0; private int $registrationsSinceLastPrune = 0; @@ -106,7 +108,7 @@ public function flushUnawaitedDeliveries(): void { foreach ($this->pendingDeliveries as $registration) { if (! $registration['pendingDelivery']->isAwaited()) { - $registration['pendingDelivery']->awaitDelivery(); + $this->awaitAndLogFailures($registration['pendingDelivery']); } } $this->pendingDeliveries = []; @@ -119,5 +121,40 @@ private function pruneAwaitedDeliveries(): void unset($this->pendingDeliveries[$index]); } } + + $this->flushOldestPublisherOwnedDeliveriesAboveBacklogLimit(); + } + + private function flushOldestPublisherOwnedDeliveriesAboveBacklogLimit(): void + { + $exceedingBacklogLimit = count($this->pendingDeliveries) - self::MAX_UNAWAITED_BACKLOG; + if ($exceedingBacklogLimit <= 0) { + return; + } + + foreach ($this->pendingDeliveries as $index => $registration) { + if ($registration['scopeOwned']) { + continue; + } + + $this->awaitAndLogFailures($registration['pendingDelivery']); + unset($this->pendingDeliveries[$index]); + + if (--$exceedingBacklogLimit <= 0) { + return; + } + } + } + + private function awaitAndLogFailures(PendingDelivery $pendingDelivery): void + { + $deliveryResult = $pendingDelivery->awaitDelivery(); + foreach ($deliveryResult->getFailedDeliveries() as $failedDelivery) { + error_log(sprintf( + 'Ecotone async publishing: unresolved delivery for channel `%s` failed confirmation: %s', + $failedDelivery->getChannelName(), + $failedDelivery->getFailureReason(), + )); + } } } diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php index 5a74e9cdb..c1787a030 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php @@ -41,6 +41,11 @@ public function await(MethodInvocation $methodInvocation): mixed $deliveryResult = $this->asyncPublishingRegistry->awaitAll(); if (! $deliveryResult->isSuccessful()) { $this->handleFailedDeliveries($deliveryResult->getFailedDeliveries()); + + $errorChannelDeliveryResult = $this->asyncPublishingRegistry->awaitAll(); + if (! $errorChannelDeliveryResult->isSuccessful()) { + throw AsyncPublishingFailedException::withFailedDeliveries($errorChannelDeliveryResult->getFailedDeliveries()); + } } } finally { $this->asyncPublishingRegistry->closeScope(); diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishGatewayRegistration.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishGatewayRegistration.php index 64a062085..bebede3e5 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishGatewayRegistration.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/Config/AsyncPublishGatewayRegistration.php @@ -24,7 +24,7 @@ */ final class AsyncPublishGatewayRegistration { - public static function registerFor(Configuration $messagingConfiguration, string $publisherReferenceName): void + public static function registerFor(Configuration $messagingConfiguration, string $publisherReferenceName, bool $asyncPublishingEnabled = true): void { $asyncPublishRequestChannel = $publisherReferenceName . '.asyncPublish'; @@ -42,6 +42,7 @@ public static function registerFor(Configuration $messagingConfiguration, string ServiceActivatorBuilder::createWithDefinition( new Definition(AsyncPublishingGateway::class, [ $publisherReferenceName, + $asyncPublishingEnabled, new Reference(ConfiguredMessagingSystem::class), new Reference(AsyncPublishingRegistry::class), ]), diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php index 9fee0a5c5..c30631f39 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php @@ -5,6 +5,7 @@ namespace Ecotone\Messaging\Channel\AsyncPublishing; use Ecotone\Messaging\Future; +use Throwable; /** * licence Enterprise @@ -42,11 +43,23 @@ public function resolve() $this->resolved = true; $failedDeliveries = []; - foreach ($this->pendingDeliveries as $pendingDelivery) { - $deliveryResult = $pendingDelivery->awaitDelivery(); - if (! $deliveryResult->isSuccessful()) { - $failedDeliveries = array_merge($failedDeliveries, $deliveryResult->getFailedDeliveries()); + try { + foreach ($this->pendingDeliveries as $pendingDelivery) { + if ($pendingDelivery->isAwaited()) { + continue; + } + + $deliveryResult = $pendingDelivery->awaitDelivery(); + if (! $deliveryResult->isSuccessful()) { + $failedDeliveries = array_merge($failedDeliveries, $deliveryResult->getFailedDeliveries()); + } } + } catch (Throwable $exception) { + $this->failure = $exception instanceof AsyncPublishingFailedException + ? $exception + : new AsyncPublishingFailedException(sprintf('Awaiting delivery confirmation failed: %s', $exception->getMessage()), 0, $exception); + + throw $this->failure; } if ($failedDeliveries !== []) { diff --git a/packages/Kafka/src/Configuration/KafkaModule.php b/packages/Kafka/src/Configuration/KafkaModule.php index 05f42f574..f4bd3a87d 100644 --- a/packages/Kafka/src/Configuration/KafkaModule.php +++ b/packages/Kafka/src/Configuration/KafkaModule.php @@ -227,7 +227,7 @@ private function registerMessagePublisher(Configuration $messagingConfiguration, ->withDefaultConversionMediaType($mediaType) ); - AsyncPublishGatewayRegistration::registerFor($messagingConfiguration, $extensionObject->getReferenceName()); + AsyncPublishGatewayRegistration::registerFor($messagingConfiguration, $extensionObject->getReferenceName(), $extensionObject->isAsyncPublishingEnabled()); } private function getPublisherEndpointId(string $referenceName): string diff --git a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php index f90e71c1f..2d90f46d1 100644 --- a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php +++ b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php @@ -8,6 +8,7 @@ use Ecotone\Kafka\Configuration\KafkaAdmin; use Ecotone\Kafka\Configuration\KafkaPublisherConfiguration; use Ecotone\Messaging\BatchMessage; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Conversion\ConversionService; @@ -48,16 +49,17 @@ public function handle(Message $message): void return; } + $trackDelivery = $this->isAsyncPublishingEnabled(); + $deliveryId = $this->produce($message, $topic, trackDelivery: $trackDelivery); + $producer->poll(0); + if ($this->canPublishAsynchronously()) { - $deliveryId = $this->produce($message, $topic, trackDelivery: true); - $producer->poll(0); $this->registerPendingDelivery($producer, [$deliveryId]); return; } - $this->produce($message, $topic, trackDelivery: false); - $this->flushSynchronously($producer); + $this->flushSynchronously($producer, $trackDelivery ? [$deliveryId] : []); } public function isAsyncPublishingEnabled(): bool @@ -67,7 +69,7 @@ public function isAsyncPublishingEnabled(): bool private function handleBatch(BatchMessage $batchMessage, Producer $producer, ProducerTopic $topic): void { - $publishAsynchronously = $this->canPublishAsynchronously(); + $trackDelivery = $this->isAsyncPublishingEnabled(); $deliveryIds = []; foreach ($batchMessage->getEntries() as $entry) { @@ -75,17 +77,17 @@ private function handleBatch(BatchMessage $batchMessage, Producer $producer, Pro ->setMultipleHeaders($entry['headers']) ->build(); - $deliveryIds[] = $this->produce($entryMessage, $topic, trackDelivery: $publishAsynchronously); + $deliveryIds[] = $this->produce($entryMessage, $topic, trackDelivery: $trackDelivery); $producer->poll(0); } - if ($publishAsynchronously) { + if ($this->canPublishAsynchronously()) { $this->registerPendingDelivery($producer, $deliveryIds); return; } - $this->flushSynchronously($producer); + $this->flushSynchronously($producer, $trackDelivery ? $deliveryIds : []); } private function produce(Message $message, ProducerTopic $topic, bool $trackDelivery): ?string @@ -149,7 +151,10 @@ private function registerPendingDelivery(Producer $producer, array $deliveryIds) ); } - private function flushSynchronously(Producer $producer): void + /** + * @param string[] $deliveryIds + */ + private function flushSynchronously(Producer $producer, array $deliveryIds = []): void { /** * Producer won't produce the message to the broker immediately it will wait until the producer queue (queue.buffering.max.messages)gets full or size of the queue(queue.buffering.max.kbytes). @@ -159,5 +164,12 @@ private function flushSynchronously(Producer $producer): void if ($result !== 0) { throw MessagePublishingException::create('Failed to send message to Kafka'); } + + if ($deliveryIds !== []) { + $deliveryResult = $this->kafkaAdmin->getDeliveryTracker($this->referenceName)->collectResult($deliveryIds, $this->referenceName); + if (! $deliveryResult->isSuccessful()) { + throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + } + } } } From d7756236a380026811a85825564081fd1f886fdc Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 07:00:00 +0200 Subject: [PATCH 15/38] test: failure-proving reliability tests for async publishing --- .../AsyncPublishingReliabilityTest.php | 120 ++++++++++++++++++ .../InMemoryPendingDelivery.php | 6 + .../AsyncPublishingReliabilityTest.php | 102 +++++++++++++++ .../AsyncPublishingReliabilityTest.php | 64 ++++++++++ 4 files changed, 292 insertions(+) create mode 100644 packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php create mode 100644 packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php create mode 100644 packages/Kafka/tests/Integration/AsyncPublishingReliabilityTest.php diff --git a/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php new file mode 100644 index 000000000..d74ba0543 --- /dev/null +++ b/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php @@ -0,0 +1,120 @@ + getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']); + $queueName = $this->declareQueueRejectingOverflow($libConnectionFactory); + $publisher = $this->bootstrapPublisher($libConnectionFactory, $queueName); + + $this->expectException(AsyncPublishingFailedException::class); + + $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('first message fills the queue') + ->append('second message overflows and gets nacked') + )->resolve(); + } + + public function test_nacked_message_fails_delivery_confirmation_over_amqp_ext(): void + { + $extConnectionFactory = new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']); + $queueName = $this->declareQueueRejectingOverflow($extConnectionFactory); + $publisher = $this->bootstrapPublisher($extConnectionFactory, $queueName); + + $this->expectException(AsyncPublishingFailedException::class); + + $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('first message fills the queue') + ->append('second message overflows and gets nacked') + )->resolve(); + } + + public function test_ext_publisher_confirmations_track_outstanding_until_all_confirmed(): void + { + $confirmations = new AmqpExtPublisherConfirmations(); + + $confirmations->recordPublishedMessage(); + $confirmations->recordPublishedMessage(); + $confirmations->recordPublishedMessage(); + $this->assertTrue($confirmations->hasOutstandingConfirmations()); + + $confirmations->recordConfirmation(1, multiple: false); + $this->assertTrue($confirmations->hasOutstandingConfirmations()); + + $confirmations->recordConfirmation(3, multiple: true); + $this->assertFalse($confirmations->hasOutstandingConfirmations()); + } + + public function test_ext_publisher_confirmations_handle_multiple_flag_covering_individual_confirmations(): void + { + $confirmations = new AmqpExtPublisherConfirmations(); + + $confirmations->recordPublishedMessage(); + $confirmations->recordPublishedMessage(); + + $confirmations->recordConfirmation(2, multiple: false); + $this->assertTrue($confirmations->hasOutstandingConfirmations()); + + $confirmations->recordConfirmation(2, multiple: true); + $this->assertFalse($confirmations->hasOutstandingConfirmations()); + } + + private function declareQueueRejectingOverflow(AmqpLibConnection|AmqpConnectionFactory $connectionFactory): string + { + $queueName = Uuid::v7()->toRfc4122(); + $context = $connectionFactory->createContext(); + $queue = $context->createQueue($queueName); + $queue->addFlag(AmqpQueue::FLAG_DURABLE); + $queue->setArguments(['x-max-length' => 1, 'x-overflow' => 'reject-publish']); + $context->declareQueue($queue); + + return $queueName; + } + + private function bootstrapPublisher(AmqpLibConnection|AmqpConnectionFactory $connectionFactory, string $queueName): MessagePublisher + { + $messaging = EcotoneLite::bootstrapFlowTesting( + [], + [ + AmqpConnectionFactory::class => $connectionFactory, + ], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::AMQP_PACKAGE])) + ->withExtensionObjects([ + AmqpMessagePublisherConfiguration::create() + ->withAutoDeclareQueueOnSend(false) + ->withDefaultRoutingKey($queueName) + ->withAsyncPublishing(timeoutInMilliseconds: 3000), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + return $messaging->getGateway(MessagePublisher::class); + } +} diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php index 7cbb8de4e..52a7280a8 100644 --- a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php @@ -8,6 +8,7 @@ use Ecotone\Messaging\Channel\AsyncPublishing\FailedDelivery; use Ecotone\Messaging\Channel\AsyncPublishing\PendingDelivery; use Ecotone\Messaging\Message; +use RuntimeException; /** * licence Apache-2.0 @@ -21,6 +22,7 @@ public function __construct( private ?string $failureReason = null, private ?OperationsLog $operationsLog = null, private string $channelName = 'in_memory_channel', + private bool $throwOnAwait = false, ) { } @@ -29,6 +31,10 @@ public function awaitDelivery(): DeliveryResult $this->awaitCalls++; $this->operationsLog?->log('delivery confirmations awaited'); + if ($this->throwOnAwait) { + throw new RuntimeException('broker connection lost while awaiting confirmation'); + } + if ($this->failureReason !== null) { return DeliveryResult::withFailedDeliveries([ new FailedDelivery($this->message, $this->failureReason, $this->channelName), diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php new file mode 100644 index 000000000..219479792 --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php @@ -0,0 +1,102 @@ +build(), + throwOnAwait: true, + ); + $future = DeliveryFuture::forPendingDeliveries([$throwingDelivery]); + + $firstResolveException = null; + try { + $future->resolve(); + } catch (AsyncPublishingFailedException $exception) { + $firstResolveException = $exception; + } + $this->assertNotNull($firstResolveException); + + $this->expectException(AsyncPublishingFailedException::class); + + $future->resolve(); + } + + public function test_future_does_not_reawait_deliveries_already_awaited_by_interceptor_scope(): void + { + $alreadyAwaitedDelivery = new InMemoryPendingDelivery(MessageBuilder::withPayload('order')->build()); + $alreadyAwaitedDelivery->awaitDelivery(); + + DeliveryFuture::forPendingDeliveries([$alreadyAwaitedDelivery])->resolve(); + + $this->assertSame(1, $alreadyAwaitedDelivery->awaitCalls()); + } + + public function test_failed_deliveries_routed_to_async_error_channel_are_awaited_before_commit(): void + { + $operationsLog = new OperationsLog(); + $ecotoneLite = EcotoneLite::bootstrapFlowTesting( + [OrderService::class, AsyncOrderSubscriber::class, FakeTransactionModule::class], + [new OrderService($operationsLog), new AsyncOrderSubscriber(), OperationsLog::class => $operationsLog], + ServiceConfiguration::createWithDefaults()->withExtensionObjects([ + \Ecotone\Messaging\Channel\PollableChannel\GlobalPollableChannelConfiguration::createWithDefaults()->withErrorChannel('failure_channel'), + ]), + enableAsynchronousProcessing: [ + InMemoryAsyncPublishingChannelBuilder::create('async_orders'), + InMemoryAsyncPublishingChannelBuilder::create('failure_channel'), + ], + ); + $ordersChannel = $ecotoneLite->getMessageChannel('async_orders'); + assert($ordersChannel instanceof MessageChannelInterceptorAdapter); + $ordersChannel->getInternalMessageChannel()->failDeliveriesWith('broker not available'); + + $ecotoneLite->sendCommandWithRoutingKey('order.place', 'espresso'); + + $operations = $operationsLog->getOperations(); + $this->assertSame('transaction committed', $operations[count($operations) - 1]); + $awaitedConfirmationsForFailedBatchAndEachErrorChannelMessage = count(array_filter($operations, fn (string $operation) => $operation === 'delivery confirmations awaited')); + $this->assertSame(3, $awaitedConfirmationsForFailedBatchAndEachErrorChannelMessage); + } + + public function test_unresolved_publisher_futures_above_backlog_limit_are_flushed(): void + { + $outboundAdapter = new InMemoryAsyncOutboundAdapter(); + $ecotoneLite = EcotoneLite::bootstrapFlowTesting( + [InMemoryAsyncPublisherModule::class, InMemoryAsyncOutboundAdapter::class], + [$outboundAdapter], + ); + $publisher = $ecotoneLite->getGateway(InMemoryAsyncPublisherModule::PUBLISHER_REFERENCE); + + for ($messageNumber = 0; $messageNumber < 1300; $messageNumber++) { + $publisher->asyncPublish('unresolved order ' . $messageNumber); + } + + $this->assertGreaterThan(0, $outboundAdapter->awaitedDeliveriesCount()); + $this->assertLessThan(1300, $outboundAdapter->awaitedDeliveriesCount()); + } +} diff --git a/packages/Kafka/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Kafka/tests/Integration/AsyncPublishingReliabilityTest.php new file mode 100644 index 000000000..8c8c8ecb2 --- /dev/null +++ b/packages/Kafka/tests/Integration/AsyncPublishingReliabilityTest.php @@ -0,0 +1,64 @@ +bootstrapPublisher(); + + $this->expectException(AsyncPublishingFailedException::class); + + $publisher->send(str_repeat('x', 2_000_000)); + } + + public function test_broker_rejected_message_fails_async_publish_on_future_resolve(): void + { + $publisher = $this->bootstrapPublisher(); + + $future = $publisher->asyncPublish(str_repeat('x', 2_000_000)); + + $this->expectException(AsyncPublishingFailedException::class); + + $future->resolve(); + } + + private function bootstrapPublisher(): MessagePublisher + { + $messaging = EcotoneLite::bootstrapFlowTesting( + [], + [KafkaBrokerConfiguration::class => ConnectionTestCase::getConnection()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::KAFKA_PACKAGE])) + ->withExtensionObjects([ + KafkaPublisherConfiguration::createWithDefaults(topicName: Uuid::v7()->toRfc4122()) + ->withAsyncPublishing(timeoutInMilliseconds: 10000) + ->setConfiguration('message.max.bytes', '4000000'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + return $messaging->getGateway(MessagePublisher::class); + } +} From 36fd62922001d6e32f4b0ebf175da01491e43a88 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 08:00:00 +0200 Subject: [PATCH 16/38] feat: batch and asyncPublish support for DBAL backed channels and publishers --- .../DbalMessagePublisherConfiguration.php | 14 ++ .../src/Configuration/DbalPublisherModule.php | 4 + .../src/DbalBackedMessageChannelBuilder.php | 17 ++ .../Dbal/src/DbalOutboundChannelAdapter.php | 32 +++- .../src/DbalOutboundChannelAdapterBuilder.php | 22 +++ .../Dbal/src/EnqueueDbal/DbalProducer.php | 127 +++++++++--- .../AsyncPublishing/OrderWasPlaced.php | 15 ++ .../tests/Integration/AsyncPublishingTest.php | 181 ++++++++++++++++++ .../AsyncPublishing/ConfirmedDelivery.php | 21 ++ .../src/EnqueueOutboundChannelAdapter.php | 52 ++++- 10 files changed, 454 insertions(+), 31 deletions(-) create mode 100644 packages/Dbal/tests/Fixture/AsyncPublishing/OrderWasPlaced.php create mode 100644 packages/Dbal/tests/Integration/AsyncPublishingTest.php create mode 100644 packages/Ecotone/src/Messaging/Channel/AsyncPublishing/ConfirmedDelivery.php diff --git a/packages/Dbal/src/Configuration/DbalMessagePublisherConfiguration.php b/packages/Dbal/src/Configuration/DbalMessagePublisherConfiguration.php index 59bab6d8b..16711c084 100644 --- a/packages/Dbal/src/Configuration/DbalMessagePublisherConfiguration.php +++ b/packages/Dbal/src/Configuration/DbalMessagePublisherConfiguration.php @@ -35,6 +35,8 @@ class DbalMessagePublisherConfiguration */ private $queueName; + private bool $asyncPublishing = false; + private function __construct(string $connectionReference, string $queueName, ?string $outputDefaultConversionMediaType, string $referenceName) { $this->connectionReference = $connectionReference; @@ -119,4 +121,16 @@ public function getReferenceName(): string { return $this->referenceName; } + + public function withAsyncPublishing(bool $asyncPublishing = true): self + { + $this->asyncPublishing = $asyncPublishing; + + return $this; + } + + public function isAsyncPublishingEnabled(): bool + { + return $this->asyncPublishing; + } } diff --git a/packages/Dbal/src/Configuration/DbalPublisherModule.php b/packages/Dbal/src/Configuration/DbalPublisherModule.php index 76317eb55..0446c6123 100644 --- a/packages/Dbal/src/Configuration/DbalPublisherModule.php +++ b/packages/Dbal/src/Configuration/DbalPublisherModule.php @@ -8,6 +8,7 @@ use Ecotone\Dbal\DbalBackedMessageChannelBuilder; use Ecotone\Dbal\DbalOutboundChannelAdapterBuilder; use Ecotone\Messaging\Attribute\ModuleAnnotation; +use Ecotone\Messaging\Channel\AsyncPublishing\Config\AsyncPublishGatewayRegistration; use Ecotone\Messaging\Config\Annotation\AnnotationModule; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\ExtensionObjectResolver; use Ecotone\Messaging\Config\Configuration; @@ -116,7 +117,10 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ->withAutoDeclareOnSend($dbalPublisher->isAutoDeclareQueueOnSend()) ->withHeaderMapper($dbalPublisher->getHeaderMapper()) ->withDefaultConversionMediaType($mediaType) + ->withAsyncPublishing($dbalPublisher->isAsyncPublishingEnabled()) ); + + AsyncPublishGatewayRegistration::registerFor($messagingConfiguration, $dbalPublisher->getReferenceName(), $dbalPublisher->isAsyncPublishingEnabled()); } } diff --git a/packages/Dbal/src/DbalBackedMessageChannelBuilder.php b/packages/Dbal/src/DbalBackedMessageChannelBuilder.php index 8d7ef7d33..29adc9994 100644 --- a/packages/Dbal/src/DbalBackedMessageChannelBuilder.php +++ b/packages/Dbal/src/DbalBackedMessageChannelBuilder.php @@ -30,4 +30,21 @@ public static function create(string $channelName, string $connectionReferenceNa { return new self($channelName, $connectionReferenceName); } + + public function withAsyncPublishing(bool $asyncPublishing = true): self + { + $this->getDbalOutboundChannelAdapter()->withAsyncPublishing($asyncPublishing); + + return $this; + } + + protected function supportsBatchMessages(): bool + { + return $this->getDbalOutboundChannelAdapter()->isAsyncPublishingEnabled(); + } + + private function getDbalOutboundChannelAdapter(): DbalOutboundChannelAdapterBuilder + { + return $this->outboundChannelAdapter; + } } diff --git a/packages/Dbal/src/DbalOutboundChannelAdapter.php b/packages/Dbal/src/DbalOutboundChannelAdapter.php index e16dd58ea..7e93721cd 100644 --- a/packages/Dbal/src/DbalOutboundChannelAdapter.php +++ b/packages/Dbal/src/DbalOutboundChannelAdapter.php @@ -7,10 +7,15 @@ use Ecotone\Dbal\Database\EnqueueTableManager; use Ecotone\Enqueue\CachedConnectionFactory; use Ecotone\Enqueue\EnqueueOutboundChannelAdapter; +use Ecotone\Messaging\BatchMessage; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Conversion\ConversionService; +use Ecotone\Messaging\MessageHeaders; use Enqueue\Dbal\DbalContext; use Enqueue\Dbal\DbalDestination; +use Enqueue\Dbal\DbalProducer; +use Interop\Queue\Context; /** * licence Apache-2.0 @@ -24,13 +29,18 @@ public function __construct( OutboundMessageConverter $outboundMessageConverter, ConversionService $conversionService, private EnqueueTableManager $tableManager, + AsyncPublishingRegistry $asyncPublishingRegistry, + bool $asyncPublishing = false, ) { parent::__construct( $connectionFactory, new DbalDestination($this->queueName), $autoDeclare, $outboundMessageConverter, - $conversionService + $conversionService, + $asyncPublishingRegistry, + $asyncPublishing, + $this->queueName, ); } @@ -46,4 +56,24 @@ public function initialize(): void $this->tableManager->createTable($context->getDbalConnection()); $context->createQueue($this->queueName); } + + protected function handleBatch(BatchMessage $batchMessage, Context $context): void + { + $messagesToSend = []; + foreach ($batchMessage->getEntries() as $entry) { + $outboundMessage = $this->prepareOutboundMessage($this->convertBatchEntryToMessage($entry)); + $headers = $outboundMessage->getHeaders(); + $headers[MessageHeaders::CONTENT_TYPE] = $outboundMessage->getContentType(); + + $messageToSend = $context->createMessage($outboundMessage->getPayload(), $headers, []); + $messageToSend->setDeliveryDelay($outboundMessage->getDeliveryDelay()); + $messageToSend->setTimeToLive($outboundMessage->getTimeToLive()); + + $messagesToSend[] = $messageToSend; + } + + /** @var DbalProducer $producer */ + $producer = $context->createProducer(); + $producer->sendBatch($this->destination, $messagesToSend); + } } diff --git a/packages/Dbal/src/DbalOutboundChannelAdapterBuilder.php b/packages/Dbal/src/DbalOutboundChannelAdapterBuilder.php index 165dd7ef6..7d01a237b 100644 --- a/packages/Dbal/src/DbalOutboundChannelAdapterBuilder.php +++ b/packages/Dbal/src/DbalOutboundChannelAdapterBuilder.php @@ -5,11 +5,13 @@ use Ecotone\Dbal\Database\EnqueueTableManager; use Ecotone\Enqueue\CachedConnectionFactory; use Ecotone\Enqueue\EnqueueOutboundChannelAdapterBuilder; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\MessagingContainerBuilder; use Ecotone\Messaging\Config\Container\Reference; use Ecotone\Messaging\Conversion\ConversionService; +use Ecotone\Messaging\Support\LicensingException; use Enqueue\Dbal\DbalConnectionFactory; /** @@ -26,6 +28,8 @@ class DbalOutboundChannelAdapterBuilder extends EnqueueOutboundChannelAdapterBui */ private $connectionFactoryReferenceName; + private bool $asyncPublishing = false; + private function __construct(string $queueName, string $connectionFactoryReferenceName) { $this->initialize($connectionFactoryReferenceName); @@ -38,8 +42,24 @@ public static function create(string $queueName, string $connectionFactoryRefere return new self($queueName, $connectionFactoryReferenceName); } + public function withAsyncPublishing(bool $asyncPublishing = true): self + { + $this->asyncPublishing = $asyncPublishing; + + return $this; + } + + public function isAsyncPublishingEnabled(): bool + { + return $this->asyncPublishing; + } + public function compile(MessagingContainerBuilder $builder): Definition { + if ($this->asyncPublishing && ! $builder->getServiceConfiguration()->isRunningForEnterprise()) { + throw LicensingException::create('Asynchronous publishing is available only with Ecotone Enterprise licence.'); + } + $connectionFactory = new Definition(CachedConnectionFactory::class, [ new Definition(DbalReconnectableConnectionFactory::class, [ new Reference($this->connectionFactoryReferenceName), @@ -62,6 +82,8 @@ public function compile(MessagingContainerBuilder $builder): Definition $outboundMessageConverter, new Reference(ConversionService::REFERENCE_NAME), new Reference(EnqueueTableManager::class), + new Reference(AsyncPublishingRegistry::class), + $this->asyncPublishing, ]); } } diff --git a/packages/Dbal/src/EnqueueDbal/DbalProducer.php b/packages/Dbal/src/EnqueueDbal/DbalProducer.php index 8b4b2b258..e8bbad458 100644 --- a/packages/Dbal/src/EnqueueDbal/DbalProducer.php +++ b/packages/Dbal/src/EnqueueDbal/DbalProducer.php @@ -20,6 +20,23 @@ */ class DbalProducer implements Producer { + private const BATCH_INSERT_CHUNK_SIZE = 250; + + private const COLUMN_TYPES = [ + 'id' => DbalType::GUID, + 'published_at' => DbalType::INTEGER, + 'body' => DbalType::TEXT, + 'headers' => DbalType::TEXT, + 'properties' => DbalType::TEXT, + 'priority' => DbalType::SMALLINT, + 'queue' => DbalType::STRING, + 'redelivered' => DbalType::SMALLINT, + 'delivery_id' => DbalType::STRING, + 'redeliver_after' => DbalType::BIGINT, + 'delayed_until' => DbalType::INTEGER, + 'time_to_live' => DbalType::INTEGER, + ]; + /** * @var int|null */ @@ -54,6 +71,78 @@ public function send(Destination $destination, Message $message): void InvalidDestinationException::assertDestinationInstanceOf($destination, DbalDestination::class); InvalidMessageException::assertMessageInstanceOf($message, DbalMessage::class); + $this->applyProducerDefaults($message); + $record = $this->createRecord($destination, $message); + + try { + $rowsAffected = $this->context->getDbalConnection()->insert($this->context->getTableName(), $record, self::COLUMN_TYPES); + + if (1 !== $rowsAffected) { + throw new Exception('The message was not enqueued. Dbal did not confirm that the record is inserted.'); + } + } catch (\Exception $e) { + throw new Exception('The transport fails to send the message due to some internal error.', 0, $e); + } + } + + /** + * @param DbalMessage[] $messages + */ + public function sendBatch(Destination $destination, array $messages): void + { + InvalidDestinationException::assertDestinationInstanceOf($destination, DbalDestination::class); + + if ([] === $messages) { + return; + } + + $records = []; + foreach ($messages as $message) { + InvalidMessageException::assertMessageInstanceOf($message, DbalMessage::class); + $this->applyProducerDefaults($message); + $records[] = $this->createRecord($destination, $message); + } + + try { + $rowsAffected = 0; + foreach (array_chunk($records, self::BATCH_INSERT_CHUNK_SIZE) as $recordsChunk) { + $rowsAffected += $this->insertRecords($recordsChunk); + } + + if (count($records) !== $rowsAffected) { + throw new Exception('The batch was not enqueued. Dbal did not confirm that all records are inserted.'); + } + } catch (\Exception $e) { + throw new Exception('The transport fails to send the message due to some internal error.', 0, $e); + } + } + + private function insertRecords(array $records): int + { + $columns = array_keys(self::COLUMN_TYPES); + $rowPlaceholders = '(' . implode(', ', array_fill(0, count($columns), '?')) . ')'; + + $sql = sprintf( + 'INSERT INTO %s (%s) VALUES %s', + $this->context->getTableName(), + implode(', ', $columns), + implode(', ', array_fill(0, count($records), $rowPlaceholders)), + ); + + $parameters = []; + $types = []; + foreach ($records as $record) { + foreach ($columns as $column) { + $parameters[] = $record[$column]; + $types[] = self::COLUMN_TYPES[$column]; + } + } + + return (int) $this->context->getDbalConnection()->executeStatement($sql, $parameters, $types); + } + + private function applyProducerDefaults(DbalMessage $message): void + { if (null !== $this->priority && null === $message->getPriority()) { $message->setPriority($this->priority); } @@ -63,16 +152,17 @@ public function send(Destination $destination, Message $message): void if (null !== $this->timeToLive && null === $message->getTimeToLive()) { $message->setTimeToLive($this->timeToLive); } + } - $body = $message->getBody(); - + private function createRecord(DbalDestination $destination, DbalMessage $message): array + { $publishedAt = $message->getPublishedAt() ?? (int) ($this->context->getClock()->now()->unixTime()->toFloat() * 10_000); // x 10_000 ?!?!!?? - $dbalMessage = [ + $record = [ 'id' => Uuid::v7()->toRfc4122(), 'published_at' => $publishedAt, - 'body' => $body, + 'body' => $message->getBody(), 'headers' => JSON::encode($message->getHeaders()), 'properties' => JSON::encode($message->getProperties()), 'priority' => -1 * $message->getPriority(), @@ -80,6 +170,8 @@ public function send(Destination $destination, Message $message): void 'redelivered' => false, 'delivery_id' => null, 'redeliver_after' => null, + 'delayed_until' => null, + 'time_to_live' => null, ]; $delay = $message->getDeliveryDelay(); @@ -92,7 +184,7 @@ public function send(Destination $destination, Message $message): void throw new LogicException(sprintf('Delay must be positive integer but got: "%s"', $delay)); } - $dbalMessage['delayed_until'] = $this->context->getClock()->now()->add(Duration::milliseconds($delay))->unixTime()->inSeconds(); + $record['delayed_until'] = $this->context->getClock()->now()->add(Duration::milliseconds($delay))->unixTime()->inSeconds(); } $timeToLive = $message->getTimeToLive(); @@ -105,31 +197,10 @@ public function send(Destination $destination, Message $message): void throw new LogicException(sprintf('TimeToLive must be positive integer but got: "%s"', $timeToLive)); } - $dbalMessage['time_to_live'] = $this->context->getClock()->now()->add(Duration::milliseconds($timeToLive))->unixTime()->inSeconds(); + $record['time_to_live'] = $this->context->getClock()->now()->add(Duration::milliseconds($timeToLive))->unixTime()->inSeconds(); } - try { - $rowsAffected = $this->context->getDbalConnection()->insert($this->context->getTableName(), $dbalMessage, [ - 'id' => DbalType::GUID, - 'published_at' => DbalType::INTEGER, - 'body' => DbalType::TEXT, - 'headers' => DbalType::TEXT, - 'properties' => DbalType::TEXT, - 'priority' => DbalType::SMALLINT, - 'queue' => DbalType::STRING, - 'time_to_live' => DbalType::INTEGER, - 'delayed_until' => DbalType::INTEGER, - 'redelivered' => DbalType::SMALLINT, - 'delivery_id' => DbalType::STRING, - 'redeliver_after' => DbalType::BIGINT, - ]); - - if (1 !== $rowsAffected) { - throw new Exception('The message was not enqueued. Dbal did not confirm that the record is inserted.'); - } - } catch (\Exception $e) { - throw new Exception('The transport fails to send the message due to some internal error.', 0, $e); - } + return $record; } public function setDeliveryDelay(?int $deliveryDelay = null): Producer diff --git a/packages/Dbal/tests/Fixture/AsyncPublishing/OrderWasPlaced.php b/packages/Dbal/tests/Fixture/AsyncPublishing/OrderWasPlaced.php new file mode 100644 index 000000000..b1b0edf39 --- /dev/null +++ b/packages/Dbal/tests/Fixture/AsyncPublishing/OrderWasPlaced.php @@ -0,0 +1,15 @@ +createOrderService(); + $messaging = $this->bootstrapEcotoneWithChannel($orderService, LicenceTesting::VALID_LICENCE); + + $messaging->sendCommandWithRoutingKey('order.place', 'espresso'); + + $this->assertSame([], $messaging->sendQueryWithRouting('order.getReceived')); + + $messaging->run('asyncOrdersChannel', ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 3, maxExecutionTimeInMilliseconds: 10000)); + + $this->assertSame( + ['espresso-1', 'espresso-2', 'espresso-3'], + $messaging->sendQueryWithRouting('order.getReceived'), + ); + } + + public function test_async_publishing_requires_enterprise_licence(): void + { + $orderService = $this->createOrderService(); + + $this->expectException(LicensingException::class); + + $this->bootstrapEcotoneWithChannel($orderService, licenceKey: null); + } + + public function test_async_publishing_via_message_publisher_requires_enterprise_licence(): void + { + $this->expectException(LicensingException::class); + + EcotoneLite::bootstrapFlowTesting( + [], + [DbalConnectionFactory::class => $this->getConnectionFactory()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + DbalMessagePublisherConfiguration::create(MessagePublisher::class, Uuid::v7()->toRfc4122()) + ->withAsyncPublishing(), + ]), + ); + } + + public function test_async_publish_on_publisher_without_async_configuration_throws_before_publishing(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: false); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $publishFailed = false; + try { + $publisher->asyncPublish('order that must not be published'); + } catch (AsyncPublishingFailedException) { + $publishFailed = true; + } + + $this->assertTrue($publishFailed); + $this->assertNull($messaging->getMessageChannel($queueName)->receive()); + } + + public function test_message_publisher_async_publish_confirms_delivery_on_future_resolve(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: true); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $singleFuture = $publisher->asyncPublish('single order'); + $batchFuture = $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('first order') + ->append('second order', ['priority' => '5']) + ); + + $this->assertNull($singleFuture->resolve()); + $this->assertNull($batchFuture->resolve()); + + $receivedPayloads = []; + while ($message = $messaging->getMessageChannel($queueName)->receive()) { + $receivedPayloads[] = $message->getPayload(); + } + sort($receivedPayloads); + $this->assertSame(['first order', 'second order', 'single order'], $receivedPayloads); + } + + private function createOrderService(): object + { + return new class () { + /** @var string[] */ + private array $receivedEvents = []; + + #[CommandHandler('order.place')] + public function placeOrder(string $order, EventBus $eventBus): void + { + $eventBus->publish(new OrderWasPlaced($order . '-1')); + $eventBus->publish(new OrderWasPlaced($order . '-2')); + $eventBus->publish(new OrderWasPlaced($order . '-3')); + } + + #[Asynchronous('asyncOrdersChannel')] + #[EventHandler(endpointId: 'async_dbal_order_collector')] + public function collect(OrderWasPlaced $event): void + { + $this->receivedEvents[] = $event->order; + } + + #[QueryHandler('order.getReceived')] + public function getReceived(): array + { + return $this->receivedEvents; + } + }; + } + + private function bootstrapEcotoneWithChannel(object $orderService, ?string $licenceKey): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + DbalBackedMessageChannelBuilder::create('asyncOrdersChannel') + ->withAsyncPublishing(), + ]), + licenceKey: $licenceKey, + ); + } + + private function bootstrapPublisher(string $queueName, bool $asyncPublishing): FlowTestSupport + { + $publisherConfiguration = DbalMessagePublisherConfiguration::create(MessagePublisher::class, $queueName); + if ($asyncPublishing) { + $publisherConfiguration = $publisherConfiguration->withAsyncPublishing(); + } + + return EcotoneLite::bootstrapFlowTesting( + [], + [DbalConnectionFactory::class => $this->getConnectionFactory()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + $publisherConfiguration, + DbalBackedMessageChannelBuilder::create($queueName), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } +} diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/ConfirmedDelivery.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/ConfirmedDelivery.php new file mode 100644 index 000000000..f354fb9aa --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/ConfirmedDelivery.php @@ -0,0 +1,21 @@ +outboundMessageConverter->prepare($message, $this->conversionService); + if ($message->getPayload() instanceof BatchMessage) { + $this->handleBatch($message->getPayload(), $context); + } else { + $this->sendSingleMessage($message, $context); + } + + $this->registerSynchronouslyConfirmedDelivery(); + } + + protected function registerSynchronouslyConfirmedDelivery(): void + { + if (! $this->asyncPublishing || $this->asyncPublishingRegistry === null || ! $this->asyncPublishingRegistry->isScopeActive()) { + return; + } + + $this->asyncPublishingRegistry->register($this->asyncPublishingChannelName, new ConfirmedDelivery()); + } + + protected function handleBatch(BatchMessage $batchMessage, Context $context): void + { + foreach ($batchMessage->getEntries() as $entry) { + $this->sendSingleMessage($this->convertBatchEntryToMessage($entry), $context); + } + } + + protected function convertBatchEntryToMessage(array $entry): Message + { + return MessageBuilder::withPayload($entry['payload']) + ->setMultipleHeaders($entry['headers']) + ->build(); + } + + protected function prepareOutboundMessage(Message $message): OutboundMessage + { + return $this->outboundMessageConverter->prepare($message, $this->conversionService); + } + + private function sendSingleMessage(Message $message, Context $context): void + { + $outboundMessage = $this->prepareOutboundMessage($message); $headers = $outboundMessage->getHeaders(); $headers[MessageHeaders::CONTENT_TYPE] = $outboundMessage->getContentType(); From cd9d88f4695d14b80da8d8b5b29ec9fad033d0f4 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 08:00:00 +0200 Subject: [PATCH 17/38] feat: batch and asyncPublish support for Redis backed channels and publishers --- .../RedisMessagePublisherConfiguration.php | 13 ++ .../RedisMessagePublisherModule.php | 4 + .../src/RedisBackedMessageChannelBuilder.php | 17 ++ .../Redis/src/RedisOutboundChannelAdapter.php | 94 +++++++- .../RedisOutboundChannelAdapterBuilder.php | 22 ++ .../AsyncPublishing/OrderWasPlaced.php | 15 ++ .../tests/Integration/AsyncPublishingTest.php | 214 ++++++++++++++++++ 7 files changed, 376 insertions(+), 3 deletions(-) create mode 100644 packages/Redis/tests/Fixture/AsyncPublishing/OrderWasPlaced.php create mode 100644 packages/Redis/tests/Integration/AsyncPublishingTest.php diff --git a/packages/Redis/src/Configuration/RedisMessagePublisherConfiguration.php b/packages/Redis/src/Configuration/RedisMessagePublisherConfiguration.php index b3df241ff..4aaafbc3c 100644 --- a/packages/Redis/src/Configuration/RedisMessagePublisherConfiguration.php +++ b/packages/Redis/src/Configuration/RedisMessagePublisherConfiguration.php @@ -14,6 +14,7 @@ final class RedisMessagePublisherConfiguration { private bool $autoDeclareOnSend = true; private string $headerMapper = ''; + private bool $asyncPublishing = false; private function __construct(private string $connectionReference, private string $queueName, private ?string $outputDefaultConversionMediaType, private string $referenceName) { @@ -71,4 +72,16 @@ public function getReferenceName(): string { return $this->referenceName; } + + public function withAsyncPublishing(bool $asyncPublishing = true): self + { + $this->asyncPublishing = $asyncPublishing; + + return $this; + } + + public function isAsyncPublishingEnabled(): bool + { + return $this->asyncPublishing; + } } diff --git a/packages/Redis/src/Configuration/RedisMessagePublisherModule.php b/packages/Redis/src/Configuration/RedisMessagePublisherModule.php index c74edeca9..82435d4af 100644 --- a/packages/Redis/src/Configuration/RedisMessagePublisherModule.php +++ b/packages/Redis/src/Configuration/RedisMessagePublisherModule.php @@ -6,6 +6,7 @@ use Ecotone\AnnotationFinder\AnnotationFinder; use Ecotone\Messaging\Attribute\ModuleAnnotation; +use Ecotone\Messaging\Channel\AsyncPublishing\Config\AsyncPublishGatewayRegistration; use Ecotone\Messaging\Config\Annotation\AnnotationModule; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\ExtensionObjectResolver; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\NoExternalConfigurationModule; @@ -81,7 +82,10 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ->withAutoDeclareOnSend($messagePublisher->isAutoDeclareOnSend()) ->withHeaderMapper($messagePublisher->getHeaderMapper()) ->withDefaultConversionMediaType($mediaType) + ->withAsyncPublishing($messagePublisher->isAsyncPublishingEnabled()) ); + + AsyncPublishGatewayRegistration::registerFor($messagingConfiguration, $messagePublisher->getReferenceName(), $messagePublisher->isAsyncPublishingEnabled()); } } diff --git a/packages/Redis/src/RedisBackedMessageChannelBuilder.php b/packages/Redis/src/RedisBackedMessageChannelBuilder.php index 210e7e854..df716069f 100644 --- a/packages/Redis/src/RedisBackedMessageChannelBuilder.php +++ b/packages/Redis/src/RedisBackedMessageChannelBuilder.php @@ -32,4 +32,21 @@ public static function create(string $channelName, string $connectionReferenceNa { return new self($channelName, $connectionReferenceName); } + + public function withAsyncPublishing(bool $asyncPublishing = true): self + { + $this->getRedisOutboundChannelAdapter()->withAsyncPublishing($asyncPublishing); + + return $this; + } + + protected function supportsBatchMessages(): bool + { + return $this->getRedisOutboundChannelAdapter()->isAsyncPublishingEnabled(); + } + + private function getRedisOutboundChannelAdapter(): RedisOutboundChannelAdapterBuilder + { + return $this->outboundChannelAdapter; + } } diff --git a/packages/Redis/src/RedisOutboundChannelAdapter.php b/packages/Redis/src/RedisOutboundChannelAdapter.php index 850b49c74..aa8dddcfd 100644 --- a/packages/Redis/src/RedisOutboundChannelAdapter.php +++ b/packages/Redis/src/RedisOutboundChannelAdapter.php @@ -6,24 +6,57 @@ use Ecotone\Enqueue\CachedConnectionFactory; use Ecotone\Enqueue\EnqueueOutboundChannelAdapter; +use Ecotone\Messaging\BatchMessage; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Conversion\ConversionService; +use Ecotone\Messaging\MessageHeaders; use Enqueue\Redis\RedisContext; use Enqueue\Redis\RedisDestination; +use Enqueue\Redis\RedisMessage; +use Interop\Queue\Context; +use Ramsey\Uuid\Uuid; +use RuntimeException; /** * licence Apache-2.0 */ final class RedisOutboundChannelAdapter extends EnqueueOutboundChannelAdapter { - public function __construct(CachedConnectionFactory $connectionFactory, private string $queueName, bool $autoDeclare, OutboundMessageConverter $outboundMessageConverter, ConversionService $conversionService) - { + private const BATCH_PUBLISH_SCRIPT = <<<'LUA' + local pushed = 0 + local immediateAmount = tonumber(ARGV[1]) + for argumentIndex = 2, immediateAmount + 1 do + redis.call("lpush", KEYS[1], ARGV[argumentIndex]) + pushed = pushed + 1 + end + local argumentIndex = immediateAmount + 2 + while argumentIndex <= #ARGV do + redis.call("zadd", KEYS[2], ARGV[argumentIndex], ARGV[argumentIndex + 1]) + pushed = pushed + 1 + argumentIndex = argumentIndex + 2 + end + return pushed + LUA; + + public function __construct( + CachedConnectionFactory $connectionFactory, + private string $queueName, + bool $autoDeclare, + OutboundMessageConverter $outboundMessageConverter, + ConversionService $conversionService, + AsyncPublishingRegistry $asyncPublishingRegistry, + bool $asyncPublishing = false, + ) { parent::__construct( $connectionFactory, new RedisDestination($queueName), $autoDeclare, $outboundMessageConverter, - $conversionService + $conversionService, + $asyncPublishingRegistry, + $asyncPublishing, + $queueName, ); } @@ -33,4 +66,59 @@ public function initialize(): void $context = $this->connectionFactory->createContext(); $context->createQueue($this->queueName); } + + protected function handleBatch(BatchMessage $batchMessage, Context $context): void + { + if (count($batchMessage) === 0) { + return; + } + + /** @var RedisContext $context */ + $immediatePayloads = []; + $delayedEntries = []; + foreach ($batchMessage->getEntries() as $entry) { + $outboundMessage = $this->prepareOutboundMessage($this->convertBatchEntryToMessage($entry)); + $headers = $outboundMessage->getHeaders(); + $headers[MessageHeaders::CONTENT_TYPE] = $outboundMessage->getContentType(); + + /** @var RedisMessage $messageToSend */ + $messageToSend = $context->createMessage($outboundMessage->getPayload(), $headers, []); + $messageToSend->setMessageId(Uuid::uuid4()->toString()); + $messageToSend->setHeader('attempts', 0); + + if ($outboundMessage->getTimeToLive()) { + $messageToSend->setTimeToLive($outboundMessage->getTimeToLive()); + $messageToSend->setHeader('expires_at', time() + $messageToSend->getTimeToLive()); + } + + $payload = $context->getSerializer()->toString($messageToSend); + + if ($outboundMessage->getDeliveryDelay()) { + $delayedEntries[] = ['score' => time() + $outboundMessage->getDeliveryDelay() / 1000, 'payload' => $payload]; + } else { + $immediatePayloads[] = $payload; + } + } + + $arguments = [count($immediatePayloads), ...$immediatePayloads]; + foreach ($delayedEntries as $delayedEntry) { + $arguments[] = $delayedEntry['score']; + $arguments[] = $delayedEntry['payload']; + } + + $pushedMessages = $context->getRedis()->eval( + self::BATCH_PUBLISH_SCRIPT, + [$this->queueName, $this->queueName . ':delayed'], + $arguments, + ); + + if ($pushedMessages !== count($batchMessage)) { + throw new RuntimeException(sprintf( + 'Redis did not confirm publishing whole batch to queue %s. Expected %d published messages, got %s.', + $this->queueName, + count($batchMessage), + var_export($pushedMessages, true), + )); + } + } } diff --git a/packages/Redis/src/RedisOutboundChannelAdapterBuilder.php b/packages/Redis/src/RedisOutboundChannelAdapterBuilder.php index b3205a17b..22311888f 100644 --- a/packages/Redis/src/RedisOutboundChannelAdapterBuilder.php +++ b/packages/Redis/src/RedisOutboundChannelAdapterBuilder.php @@ -7,11 +7,13 @@ use Ecotone\Enqueue\CachedConnectionFactory; use Ecotone\Enqueue\EnqueueOutboundChannelAdapterBuilder; use Ecotone\Enqueue\HttpReconnectableConnectionFactory; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\MessagingContainerBuilder; use Ecotone\Messaging\Config\Container\Reference; use Ecotone\Messaging\Conversion\ConversionService; +use Ecotone\Messaging\Support\LicensingException; use Enqueue\Redis\RedisConnectionFactory; /** @@ -19,6 +21,8 @@ */ final class RedisOutboundChannelAdapterBuilder extends EnqueueOutboundChannelAdapterBuilder { + private bool $asyncPublishing = false; + private function __construct(private string $queueName, private string $connectionFactoryReferenceName) { $this->initialize($connectionFactoryReferenceName); @@ -32,8 +36,24 @@ public static function createWith(string $queueName, string $connectionFactoryRe ); } + public function withAsyncPublishing(bool $asyncPublishing = true): self + { + $this->asyncPublishing = $asyncPublishing; + + return $this; + } + + public function isAsyncPublishingEnabled(): bool + { + return $this->asyncPublishing; + } + public function compile(MessagingContainerBuilder $builder): Definition { + if ($this->asyncPublishing && ! $builder->getServiceConfiguration()->isRunningForEnterprise()) { + throw LicensingException::create('Asynchronous publishing is available only with Ecotone Enterprise licence.'); + } + $connectionFactory = new Definition(CachedConnectionFactory::class, [ new Definition(HttpReconnectableConnectionFactory::class, [ new Reference($this->connectionFactoryReferenceName), @@ -55,6 +75,8 @@ public function compile(MessagingContainerBuilder $builder): Definition $this->autoDeclare, $outboundMessageConverter, new Reference(ConversionService::REFERENCE_NAME), + new Reference(AsyncPublishingRegistry::class), + $this->asyncPublishing, ]); } } diff --git a/packages/Redis/tests/Fixture/AsyncPublishing/OrderWasPlaced.php b/packages/Redis/tests/Fixture/AsyncPublishing/OrderWasPlaced.php new file mode 100644 index 000000000..1bb32fa51 --- /dev/null +++ b/packages/Redis/tests/Fixture/AsyncPublishing/OrderWasPlaced.php @@ -0,0 +1,15 @@ +getConnectionFactory()->createContext(); + $context->getRedis()->del('asyncOrdersChannel'); + $context->getRedis()->del('asyncOrdersChannel:delayed'); + } + + public function test_multiple_messages_published_asynchronously_from_command_handler_are_delivered(): void + { + $orderService = $this->createOrderService(); + $messaging = $this->bootstrapEcotoneWithChannel($orderService, LicenceTesting::VALID_LICENCE); + + $messaging->sendCommandWithRoutingKey('order.place', 'espresso'); + + $this->assertSame([], $messaging->sendQueryWithRouting('order.getReceived')); + + $messaging->run('asyncOrdersChannel', ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 3, maxExecutionTimeInMilliseconds: 10000)); + + $this->assertSame( + ['espresso-1', 'espresso-2', 'espresso-3'], + $messaging->sendQueryWithRouting('order.getReceived'), + ); + } + + public function test_async_publishing_requires_enterprise_licence(): void + { + $orderService = $this->createOrderService(); + + $this->expectException(LicensingException::class); + + $this->bootstrapEcotoneWithChannel($orderService, licenceKey: null); + } + + public function test_async_publishing_via_message_publisher_requires_enterprise_licence(): void + { + $this->expectException(LicensingException::class); + + EcotoneLite::bootstrapFlowTesting( + [], + [RedisConnectionFactory::class => $this->getConnectionFactory()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::REDIS_PACKAGE])) + ->withExtensionObjects([ + RedisMessagePublisherConfiguration::create(queueName: Uuid::v7()->toRfc4122()) + ->withAsyncPublishing(), + ]), + ); + } + + public function test_async_publish_on_publisher_without_async_configuration_throws_before_publishing(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: false); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $publishFailed = false; + try { + $publisher->asyncPublish('order that must not be published'); + } catch (AsyncPublishingFailedException) { + $publishFailed = true; + } + + $this->assertTrue($publishFailed); + $this->assertNull($messaging->getMessageChannel($queueName)->receive()); + } + + public function test_message_publisher_async_publish_confirms_delivery_on_future_resolve(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: true); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $singleFuture = $publisher->asyncPublish('single order'); + $batchFuture = $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('first order') + ->append('second order', ['priority' => '5']) + ); + + $this->assertNull($singleFuture->resolve()); + $this->assertNull($batchFuture->resolve()); + + $receivedPayloads = []; + while ($message = $messaging->getMessageChannel($queueName)->receive()) { + $receivedPayloads[] = $message->getPayload(); + } + sort($receivedPayloads); + $this->assertSame(['first order', 'second order', 'single order'], $receivedPayloads); + } + + public function test_delayed_entry_of_published_batch_lands_in_delayed_set(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: true); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('immediate order') + ->append('delayed order', [MessageHeaders::DELIVERY_DELAY => 60000]) + )->resolve(); + + $receivedPayloads = []; + while ($message = $messaging->getMessageChannel($queueName)->receive()) { + $receivedPayloads[] = $message->getPayload(); + } + $this->assertSame(['immediate order'], $receivedPayloads); + + /** @var RedisContext $context */ + $context = $this->getConnectionFactory()->createContext(); + $this->assertSame(1, $context->getRedis()->eval('return redis.call("zcard", KEYS[1])', [$queueName . ':delayed'])); + } + + private function createOrderService(): object + { + return new class () { + /** @var string[] */ + private array $receivedEvents = []; + + #[CommandHandler('order.place')] + public function placeOrder(string $order, EventBus $eventBus): void + { + $eventBus->publish(new OrderWasPlaced($order . '-1')); + $eventBus->publish(new OrderWasPlaced($order . '-2')); + $eventBus->publish(new OrderWasPlaced($order . '-3')); + } + + #[Asynchronous('asyncOrdersChannel')] + #[EventHandler(endpointId: 'async_redis_order_collector')] + public function collect(OrderWasPlaced $event): void + { + $this->receivedEvents[] = $event->order; + } + + #[QueryHandler('order.getReceived')] + public function getReceived(): array + { + return $this->receivedEvents; + } + }; + } + + private function bootstrapEcotoneWithChannel(object $orderService, ?string $licenceKey): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [RedisConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::REDIS_PACKAGE])) + ->withExtensionObjects([ + RedisBackedMessageChannelBuilder::create('asyncOrdersChannel') + ->withAsyncPublishing(), + ]), + licenceKey: $licenceKey, + ); + } + + private function bootstrapPublisher(string $queueName, bool $asyncPublishing): FlowTestSupport + { + $publisherConfiguration = RedisMessagePublisherConfiguration::create(queueName: $queueName); + if ($asyncPublishing) { + $publisherConfiguration = $publisherConfiguration->withAsyncPublishing(); + } + + return EcotoneLite::bootstrapFlowTesting( + [], + [RedisConnectionFactory::class => $this->getConnectionFactory()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::REDIS_PACKAGE])) + ->withExtensionObjects([ + $publisherConfiguration, + RedisBackedMessageChannelBuilder::create($queueName), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } +} From 9c45c22e4ee3be94611e18aac5cddb182764e2f1 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 08:00:00 +0200 Subject: [PATCH 18/38] feat: batch and promise-based async publishing for SQS backed channels and publishers --- .../src/EnqueueOutboundChannelAdapter.php | 21 +- .../SqsMessagePublisherConfiguration.php | 22 ++ .../SqsMessagePublisherModule.php | 4 + .../src/SqsBackedMessageChannelBuilder.php | 17 ++ .../Sqs/src/SqsOutboundChannelAdapter.php | 155 +++++++++++++- .../src/SqsOutboundChannelAdapterBuilder.php | 27 +++ packages/Sqs/src/SqsPendingDelivery.php | 95 +++++++++ .../AsyncPublishing/OrderWasPlaced.php | 15 ++ .../AsyncPublishingReliabilityTest.php | 87 ++++++++ .../tests/Integration/AsyncPublishingTest.php | 200 ++++++++++++++++++ .../Sqs/tests/Unit/SqsPendingDeliveryTest.php | 97 +++++++++ 11 files changed, 730 insertions(+), 10 deletions(-) create mode 100644 packages/Sqs/src/SqsPendingDelivery.php create mode 100644 packages/Sqs/tests/Fixture/AsyncPublishing/OrderWasPlaced.php create mode 100644 packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php create mode 100644 packages/Sqs/tests/Integration/AsyncPublishingTest.php create mode 100644 packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php diff --git a/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php b/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php index fa6f40341..270305135 100644 --- a/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php +++ b/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php @@ -41,6 +41,19 @@ public function __construct( abstract public function initialize(): void; public function handle(Message $message): void + { + $context = $this->createOutboundContext(); + + if ($message->getPayload() instanceof BatchMessage) { + $this->handleBatch($message->getPayload(), $context); + } else { + $this->sendSingleMessage($message, $context); + } + + $this->registerSynchronouslyConfirmedDelivery(); + } + + protected function createOutboundContext(): Context { $context = $this->connectionFactory->createContext(); if ($this->autoDeclare) { @@ -52,13 +65,7 @@ public function handle(Message $message): void } } - if ($message->getPayload() instanceof BatchMessage) { - $this->handleBatch($message->getPayload(), $context); - } else { - $this->sendSingleMessage($message, $context); - } - - $this->registerSynchronouslyConfirmedDelivery(); + return $context; } protected function registerSynchronouslyConfirmedDelivery(): void diff --git a/packages/Sqs/src/Configuration/SqsMessagePublisherConfiguration.php b/packages/Sqs/src/Configuration/SqsMessagePublisherConfiguration.php index 34df22145..d809efee7 100644 --- a/packages/Sqs/src/Configuration/SqsMessagePublisherConfiguration.php +++ b/packages/Sqs/src/Configuration/SqsMessagePublisherConfiguration.php @@ -14,6 +14,8 @@ final class SqsMessagePublisherConfiguration { private bool $autoDeclareOnSend = true; private string $headerMapper = ''; + private bool $asyncPublishing = false; + private ?int $asyncPublishingTimeout = null; private function __construct(private string $connectionReference, private string $queueName, private ?string $outputDefaultConversionMediaType, private string $referenceName) { @@ -71,4 +73,24 @@ public function getReferenceName(): string { return $this->referenceName; } + + public function withAsyncPublishing(bool $asyncPublishing = true, ?int $timeoutInMilliseconds = null): self + { + $this->asyncPublishing = $asyncPublishing; + if ($timeoutInMilliseconds !== null) { + $this->asyncPublishingTimeout = $timeoutInMilliseconds; + } + + return $this; + } + + public function isAsyncPublishingEnabled(): bool + { + return $this->asyncPublishing; + } + + public function getAsyncPublishingTimeout(): ?int + { + return $this->asyncPublishingTimeout; + } } diff --git a/packages/Sqs/src/Configuration/SqsMessagePublisherModule.php b/packages/Sqs/src/Configuration/SqsMessagePublisherModule.php index db11f707f..b65b234ad 100644 --- a/packages/Sqs/src/Configuration/SqsMessagePublisherModule.php +++ b/packages/Sqs/src/Configuration/SqsMessagePublisherModule.php @@ -6,6 +6,7 @@ use Ecotone\AnnotationFinder\AnnotationFinder; use Ecotone\Messaging\Attribute\ModuleAnnotation; +use Ecotone\Messaging\Channel\AsyncPublishing\Config\AsyncPublishGatewayRegistration; use Ecotone\Messaging\Config\Annotation\AnnotationModule; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\ExtensionObjectResolver; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\NoExternalConfigurationModule; @@ -81,7 +82,10 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ->withAutoDeclareOnSend($messagePublisher->isAutoDeclareOnSend()) ->withHeaderMapper($messagePublisher->getHeaderMapper()) ->withDefaultConversionMediaType($mediaType) + ->withAsyncPublishing($messagePublisher->isAsyncPublishingEnabled(), $messagePublisher->getAsyncPublishingTimeout()) ); + + AsyncPublishGatewayRegistration::registerFor($messagingConfiguration, $messagePublisher->getReferenceName(), $messagePublisher->isAsyncPublishingEnabled()); } } diff --git a/packages/Sqs/src/SqsBackedMessageChannelBuilder.php b/packages/Sqs/src/SqsBackedMessageChannelBuilder.php index 74efabaf8..c12326cba 100644 --- a/packages/Sqs/src/SqsBackedMessageChannelBuilder.php +++ b/packages/Sqs/src/SqsBackedMessageChannelBuilder.php @@ -32,4 +32,21 @@ public static function create(string $channelName, string $connectionReferenceNa { return new self($channelName, $connectionReferenceName); } + + public function withAsyncPublishing(bool $asyncPublishing = true, ?int $timeoutInMilliseconds = null): self + { + $this->getSqsOutboundChannelAdapter()->withAsyncPublishing($asyncPublishing, $timeoutInMilliseconds); + + return $this; + } + + protected function supportsBatchMessages(): bool + { + return $this->getSqsOutboundChannelAdapter()->isAsyncPublishingEnabled(); + } + + private function getSqsOutboundChannelAdapter(): SqsOutboundChannelAdapterBuilder + { + return $this->outboundChannelAdapter; + } } diff --git a/packages/Sqs/src/SqsOutboundChannelAdapter.php b/packages/Sqs/src/SqsOutboundChannelAdapter.php index 151d233be..62b6ed14d 100644 --- a/packages/Sqs/src/SqsOutboundChannelAdapter.php +++ b/packages/Sqs/src/SqsOutboundChannelAdapter.php @@ -6,24 +6,44 @@ use Ecotone\Enqueue\CachedConnectionFactory; use Ecotone\Enqueue\EnqueueOutboundChannelAdapter; +use Ecotone\Messaging\BatchMessage; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Conversion\ConversionService; +use Ecotone\Messaging\Message; +use Ecotone\Messaging\MessageHeaders; use Enqueue\Sqs\SqsContext; use Enqueue\Sqs\SqsDestination; +use Interop\Queue\Exception\InvalidMessageException; /** * licence Apache-2.0 */ final class SqsOutboundChannelAdapter extends EnqueueOutboundChannelAdapter { - public function __construct(CachedConnectionFactory $connectionFactory, private string $queueName, bool $autoDeclare, OutboundMessageConverter $outboundMessageConverter, ConversionService $conversionService) - { + private const MAX_ENTRIES_PER_BATCH_REQUEST = 10; + private const BATCH_REQUEST_PAYLOAD_BUDGET_IN_BYTES = 204800; + + public function __construct( + CachedConnectionFactory $connectionFactory, + private string $queueName, + bool $autoDeclare, + OutboundMessageConverter $outboundMessageConverter, + ConversionService $conversionService, + private ?AsyncPublishingRegistry $asyncPublishingRegistry = null, + private bool $asyncPublishing = false, + private ?int $asyncPublishingTimeout = null, + ) { parent::__construct( $connectionFactory, new SqsDestination($queueName), $autoDeclare, $outboundMessageConverter, - $conversionService + $conversionService, + $asyncPublishingRegistry, + $asyncPublishing, + $queueName, ); } @@ -34,4 +54,133 @@ public function initialize(): void $context->declareQueue($context->createQueue($this->queueName)); } + + public function handle(Message $message): void + { + if (! $this->asyncPublishing) { + parent::handle($message); + + return; + } + + /** @var SqsContext $context */ + $context = $this->createOutboundContext(); + + $payload = $message->getPayload(); + $messagesToPublish = $payload instanceof BatchMessage + ? array_map(fn (array $entry): Message => $this->convertBatchEntryToMessage($entry), $payload->getEntries()) + : [$message]; + + if ($messagesToPublish === []) { + return; + } + + $sendRequestPromises = []; + $trackedMessagesPerRequest = []; + foreach ($this->buildBatchRequests($messagesToPublish, $context) as $batchRequest) { + $sendRequestPromises[] = $context->getAwsSqsClient()->sendMessageBatchAsync($batchRequest['arguments']); + $trackedMessagesPerRequest[] = $batchRequest['trackedMessages']; + } + + $pendingDelivery = new SqsPendingDelivery($sendRequestPromises, $trackedMessagesPerRequest, $this->queueName); + + if ($this->asyncPublishingRegistry->isScopeActive()) { + $this->asyncPublishingRegistry->register($this->queueName, $pendingDelivery); + + return; + } + + $deliveryResult = $pendingDelivery->awaitDelivery(); + if (! $deliveryResult->isSuccessful()) { + throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + } + } + + /** + * @param Message[] $messagesToPublish + * @return array}> + */ + private function buildBatchRequests(array $messagesToPublish, SqsContext $context): array + { + /** @var SqsDestination $destination */ + $destination = $this->destination; + $queueUrl = $context->getQueueUrl($destination); + + $batchRequests = []; + $entries = []; + $trackedMessages = []; + $payloadSizeInBytes = 0; + + foreach ($messagesToPublish as $messageIndex => $messageToPublish) { + $entry = $this->buildBatchEntry((string) $messageIndex, $messageToPublish, $context); + $entrySizeInBytes = strlen($entry['MessageBody']) + strlen($entry['MessageAttributes']['Headers']['StringValue']); + + $currentBatchIsFull = count($entries) >= self::MAX_ENTRIES_PER_BATCH_REQUEST + || ($entries !== [] && $payloadSizeInBytes + $entrySizeInBytes > self::BATCH_REQUEST_PAYLOAD_BUDGET_IN_BYTES); + if ($currentBatchIsFull) { + $batchRequests[] = $this->buildBatchRequest($destination, $queueUrl, $entries, $trackedMessages); + $entries = []; + $trackedMessages = []; + $payloadSizeInBytes = 0; + } + + $entries[] = $entry; + $trackedMessages[$entry['Id']] = $messageToPublish; + $payloadSizeInBytes += $entrySizeInBytes; + } + + if ($entries !== []) { + $batchRequests[] = $this->buildBatchRequest($destination, $queueUrl, $entries, $trackedMessages); + } + + return $batchRequests; + } + + private function buildBatchEntry(string $entryId, Message $messageToPublish, SqsContext $context): array + { + $outboundMessage = $this->prepareOutboundMessage($messageToPublish); + $headers = $outboundMessage->getHeaders(); + $headers[MessageHeaders::CONTENT_TYPE] = $outboundMessage->getContentType(); + + $sqsMessage = $context->createMessage($outboundMessage->getPayload(), $headers, []); + if (empty($sqsMessage->getBody())) { + throw new InvalidMessageException('The message body must be a non-empty string.'); + } + + $entry = [ + 'Id' => $entryId, + 'MessageBody' => $sqsMessage->getBody(), + 'MessageAttributes' => [ + 'Headers' => [ + 'DataType' => 'String', + 'StringValue' => json_encode([$sqsMessage->getHeaders(), $sqsMessage->getProperties()]), + ], + ], + ]; + + if ($outboundMessage->getDeliveryDelay()) { + $entry['DelaySeconds'] = (int) ceil($outboundMessage->getDeliveryDelay() / 1000); + } + + return $entry; + } + + /** + * @param array $trackedMessages + * @return array{arguments: array, trackedMessages: array} + */ + private function buildBatchRequest(SqsDestination $destination, string $queueUrl, array $entries, array $trackedMessages): array + { + $arguments = [ + '@region' => $destination->getRegion(), + 'QueueUrl' => $queueUrl, + 'Entries' => $entries, + ]; + + if ($this->asyncPublishingTimeout !== null) { + $arguments['@http'] = ['timeout' => $this->asyncPublishingTimeout / 1000]; + } + + return ['arguments' => $arguments, 'trackedMessages' => $trackedMessages]; + } } diff --git a/packages/Sqs/src/SqsOutboundChannelAdapterBuilder.php b/packages/Sqs/src/SqsOutboundChannelAdapterBuilder.php index 20d773753..0d900282c 100644 --- a/packages/Sqs/src/SqsOutboundChannelAdapterBuilder.php +++ b/packages/Sqs/src/SqsOutboundChannelAdapterBuilder.php @@ -7,11 +7,13 @@ use Ecotone\Enqueue\CachedConnectionFactory; use Ecotone\Enqueue\EnqueueOutboundChannelAdapterBuilder; use Ecotone\Enqueue\HttpReconnectableConnectionFactory; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\MessagingContainerBuilder; use Ecotone\Messaging\Config\Container\Reference; use Ecotone\Messaging\Conversion\ConversionService; +use Ecotone\Messaging\Support\LicensingException; use Enqueue\Sqs\SqsConnectionFactory; /** @@ -19,6 +21,9 @@ */ final class SqsOutboundChannelAdapterBuilder extends EnqueueOutboundChannelAdapterBuilder { + private bool $asyncPublishing = false; + private ?int $asyncPublishingTimeout = null; + private function __construct(private string $queueName, private string $connectionFactoryReferenceName) { $this->initialize($connectionFactoryReferenceName); @@ -29,8 +34,27 @@ public static function create(string $queueName, string $connectionFactoryRefere return new self($queueName, $connectionFactoryReferenceName); } + public function withAsyncPublishing(bool $asyncPublishing = true, ?int $timeoutInMilliseconds = null): self + { + $this->asyncPublishing = $asyncPublishing; + if ($timeoutInMilliseconds !== null) { + $this->asyncPublishingTimeout = $timeoutInMilliseconds; + } + + return $this; + } + + public function isAsyncPublishingEnabled(): bool + { + return $this->asyncPublishing; + } + public function compile(MessagingContainerBuilder $builder): Definition { + if ($this->asyncPublishing && ! $builder->getServiceConfiguration()->isRunningForEnterprise()) { + throw LicensingException::create('Asynchronous publishing is available only with Ecotone Enterprise licence.'); + } + $connectionFactory = new Definition(CachedConnectionFactory::class, [ new Definition(HttpReconnectableConnectionFactory::class, [ new Reference($this->connectionFactoryReferenceName), @@ -52,6 +76,9 @@ public function compile(MessagingContainerBuilder $builder): Definition $this->autoDeclare, $outboundMessageConverter, new Reference(ConversionService::REFERENCE_NAME), + new Reference(AsyncPublishingRegistry::class), + $this->asyncPublishing, + $this->asyncPublishingTimeout, ]); } } diff --git a/packages/Sqs/src/SqsPendingDelivery.php b/packages/Sqs/src/SqsPendingDelivery.php new file mode 100644 index 000000000..ba1644980 --- /dev/null +++ b/packages/Sqs/src/SqsPendingDelivery.php @@ -0,0 +1,95 @@ +> $trackedMessagesPerRequest keyed by request index, then by batch entry id + */ + public function __construct( + private array $sendRequestPromises, + private array $trackedMessagesPerRequest, + private string $channelName, + ) { + } + + public function awaitDelivery(): DeliveryResult + { + $this->awaited = true; + + $failedDeliveries = []; + $settledResults = Utils::settle($this->sendRequestPromises)->wait(); + + foreach ($settledResults as $requestIndex => $settledResult) { + $trackedMessages = $this->trackedMessagesPerRequest[$requestIndex] ?? []; + + if ($settledResult['state'] !== PromiseInterface::FULFILLED) { + $failureReason = $settledResult['reason'] instanceof Throwable + ? $settledResult['reason']->getMessage() + : (string) $settledResult['reason']; + + foreach ($trackedMessages as $trackedMessage) { + $failedDeliveries[] = new FailedDelivery($trackedMessage, $failureReason, $this->channelName); + } + + continue; + } + + /** @var Result $awsResult */ + $awsResult = $settledResult['value']; + $unaccountedEntryIds = array_map('strval', array_keys($trackedMessages)); + + foreach ($awsResult->get('Successful') ?? [] as $successfulEntry) { + $unaccountedEntryIds = array_diff($unaccountedEntryIds, [(string) $successfulEntry['Id']]); + } + + foreach ($awsResult->get('Failed') ?? [] as $failedEntry) { + $entryId = (string) $failedEntry['Id']; + $unaccountedEntryIds = array_diff($unaccountedEntryIds, [$entryId]); + + if (isset($trackedMessages[$entryId])) { + $failedDeliveries[] = new FailedDelivery( + $trackedMessages[$entryId], + sprintf('%s: %s', $failedEntry['Code'] ?? 'Unknown', $failedEntry['Message'] ?? 'SQS rejected batch entry'), + $this->channelName, + ); + } + } + + foreach ($unaccountedEntryIds as $unaccountedEntryId) { + $failedDeliveries[] = new FailedDelivery( + $trackedMessages[$unaccountedEntryId], + 'SQS did not confirm delivery of the batch entry', + $this->channelName, + ); + } + } + + return $failedDeliveries === [] + ? DeliveryResult::successful() + : DeliveryResult::withFailedDeliveries($failedDeliveries); + } + + public function isAwaited(): bool + { + return $this->awaited; + } +} diff --git a/packages/Sqs/tests/Fixture/AsyncPublishing/OrderWasPlaced.php b/packages/Sqs/tests/Fixture/AsyncPublishing/OrderWasPlaced.php new file mode 100644 index 000000000..1cf3396e4 --- /dev/null +++ b/packages/Sqs/tests/Fixture/AsyncPublishing/OrderWasPlaced.php @@ -0,0 +1,15 @@ +bootstrapPublisher(Uuid::v7()->toRfc4122()); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $future = $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('valid order') + ->append(str_repeat('x', 300_000)) + ); + + $this->expectException(AsyncPublishingFailedException::class); + + $future->resolve(); + } + + public function test_broker_rejected_batch_sent_without_active_scope_throws_immediately(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapChannel($queueName); + + $this->expectException(AsyncPublishingFailedException::class); + + $messaging->getMessageChannel($queueName)->send( + MessageBuilder::withPayload( + BatchMessage::constructEmpty()->append(str_repeat('x', 300_000)) + )->build() + ); + } + + private function bootstrapPublisher(string $queueName): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + [], + [SqsConnectionFactory::class => $this->getConnectionFactory()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::SQS_PACKAGE])) + ->withExtensionObjects([ + SqsMessagePublisherConfiguration::create(queueName: $queueName) + ->withAsyncPublishing(timeoutInMilliseconds: 10000), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + + private function bootstrapChannel(string $channelName): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + [], + [SqsConnectionFactory::class => $this->getConnectionFactory()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::SQS_PACKAGE])) + ->withExtensionObjects([ + SqsBackedMessageChannelBuilder::create($channelName) + ->withAsyncPublishing(), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } +} diff --git a/packages/Sqs/tests/Integration/AsyncPublishingTest.php b/packages/Sqs/tests/Integration/AsyncPublishingTest.php new file mode 100644 index 000000000..e3bc1bcff --- /dev/null +++ b/packages/Sqs/tests/Integration/AsyncPublishingTest.php @@ -0,0 +1,200 @@ +createOrderService(); + $messaging = $this->bootstrapEcotoneWithChannel($orderService, LicenceTesting::VALID_LICENCE); + + $messaging->sendCommandWithRoutingKey('order.place', 'espresso'); + + $this->assertSame([], $messaging->sendQueryWithRouting('order.getReceived')); + + $messaging->run('asyncOrdersChannel', ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 3, maxExecutionTimeInMilliseconds: 20000)); + + $receivedEvents = $messaging->sendQueryWithRouting('order.getReceived'); + sort($receivedEvents); + $this->assertSame(['espresso-1', 'espresso-2', 'espresso-3'], $receivedEvents); + } + + public function test_async_publishing_requires_enterprise_licence(): void + { + $orderService = $this->createOrderService(); + + $this->expectException(LicensingException::class); + + $this->bootstrapEcotoneWithChannel($orderService, licenceKey: null); + } + + public function test_async_publishing_via_message_publisher_requires_enterprise_licence(): void + { + $this->expectException(LicensingException::class); + + EcotoneLite::bootstrapFlowTesting( + [], + [SqsConnectionFactory::class => $this->getConnectionFactory()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::SQS_PACKAGE])) + ->withExtensionObjects([ + SqsMessagePublisherConfiguration::create(queueName: Uuid::v7()->toRfc4122()) + ->withAsyncPublishing(), + ]), + ); + } + + public function test_async_publish_on_publisher_without_async_configuration_throws_before_publishing(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: false); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $publishFailed = false; + try { + $publisher->asyncPublish('order that must not be published'); + } catch (AsyncPublishingFailedException) { + $publishFailed = true; + } + + $this->assertTrue($publishFailed); + $this->assertNull($messaging->getMessageChannel($queueName)->receive()); + } + + public function test_message_publisher_async_publish_confirms_delivery_on_future_resolve(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: true); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $singleFuture = $publisher->asyncPublish('single order'); + $batchFuture = $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('first order') + ->append('second order', ['priority' => '5']) + ); + + $this->assertNull($singleFuture->resolve()); + $this->assertNull($batchFuture->resolve()); + + $receivedPayloads = []; + while ($message = $messaging->getMessageChannel($queueName)->receive()) { + $receivedPayloads[] = $message->getPayload(); + } + sort($receivedPayloads); + $this->assertSame(['first order', 'second order', 'single order'], $receivedPayloads); + } + + public function test_batch_larger_than_ten_messages_is_chunked_and_delivered(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: true); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $batch = BatchMessage::constructEmpty(); + for ($orderNumber = 1; $orderNumber <= 25; $orderNumber++) { + $batch = $batch->append('order ' . $orderNumber); + } + + $this->assertNull($publisher->asyncPublish($batch)->resolve()); + + $receivedPayloads = []; + while ($message = $messaging->getMessageChannel($queueName)->receive()) { + $receivedPayloads[] = $message->getPayload(); + } + $this->assertCount(25, $receivedPayloads); + } + + private function createOrderService(): object + { + return new class () { + /** @var string[] */ + private array $receivedEvents = []; + + #[CommandHandler('order.place')] + public function placeOrder(string $order, EventBus $eventBus): void + { + $eventBus->publish(new OrderWasPlaced($order . '-1')); + $eventBus->publish(new OrderWasPlaced($order . '-2')); + $eventBus->publish(new OrderWasPlaced($order . '-3')); + } + + #[Asynchronous('asyncOrdersChannel')] + #[EventHandler(endpointId: 'async_sqs_order_collector')] + public function collect(OrderWasPlaced $event): void + { + $this->receivedEvents[] = $event->order; + } + + #[QueryHandler('order.getReceived')] + public function getReceived(): array + { + return $this->receivedEvents; + } + }; + } + + private function bootstrapEcotoneWithChannel(object $orderService, ?string $licenceKey): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [SqsConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::SQS_PACKAGE])) + ->withExtensionObjects([ + SqsBackedMessageChannelBuilder::create('asyncOrdersChannel') + ->withAsyncPublishing(), + ]), + licenceKey: $licenceKey, + ); + } + + private function bootstrapPublisher(string $queueName, bool $asyncPublishing): FlowTestSupport + { + $publisherConfiguration = SqsMessagePublisherConfiguration::create(queueName: $queueName); + if ($asyncPublishing) { + $publisherConfiguration = $publisherConfiguration->withAsyncPublishing(); + } + + return EcotoneLite::bootstrapFlowTesting( + [], + [SqsConnectionFactory::class => $this->getConnectionFactory()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::SQS_PACKAGE])) + ->withExtensionObjects([ + $publisherConfiguration, + SqsBackedMessageChannelBuilder::create($queueName), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } +} diff --git a/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php b/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php new file mode 100644 index 000000000..df867d6a5 --- /dev/null +++ b/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php @@ -0,0 +1,97 @@ +build(); + $rejectedMessage = MessageBuilder::withPayload('rejected order')->build(); + $pendingDelivery = new SqsPendingDelivery( + [new FulfilledPromise(new Result([ + 'Successful' => [['Id' => '0', 'MessageId' => 'aws-message-id']], + 'Failed' => [['Id' => '1', 'Code' => 'InternalError', 'Message' => 'server hiccup', 'SenderFault' => false]], + ]))], + [['0' => $deliveredMessage, '1' => $rejectedMessage]], + 'orders', + ); + + $deliveryResult = $pendingDelivery->awaitDelivery(); + + $this->assertFalse($deliveryResult->isSuccessful()); + $failedDeliveries = $deliveryResult->getFailedDeliveries(); + $this->assertCount(1, $failedDeliveries); + $this->assertSame($rejectedMessage, $failedDeliveries[0]->getMessage()); + $this->assertStringContainsString('InternalError', $failedDeliveries[0]->getFailureReason()); + $this->assertSame('orders', $failedDeliveries[0]->getChannelName()); + } + + public function test_rejected_request_reports_all_messages_of_that_request_as_failed(): void + { + $firstMessage = MessageBuilder::withPayload('first order')->build(); + $secondMessage = MessageBuilder::withPayload('second order')->build(); + $pendingDelivery = new SqsPendingDelivery( + [new RejectedPromise(new RuntimeException('connection refused'))], + [['0' => $firstMessage, '1' => $secondMessage]], + 'orders', + ); + + $deliveryResult = $pendingDelivery->awaitDelivery(); + + $this->assertFalse($deliveryResult->isSuccessful()); + $this->assertCount(2, $deliveryResult->getFailedDeliveries()); + $this->assertStringContainsString('connection refused', $deliveryResult->getFailedDeliveries()[0]->getFailureReason()); + } + + public function test_entries_missing_from_successful_and_failed_lists_are_reported_as_failed(): void + { + $confirmedMessage = MessageBuilder::withPayload('confirmed order')->build(); + $unaccountedMessage = MessageBuilder::withPayload('unaccounted order')->build(); + $pendingDelivery = new SqsPendingDelivery( + [new FulfilledPromise(new Result([ + 'Successful' => [['Id' => '0', 'MessageId' => 'aws-message-id']], + ]))], + [['0' => $confirmedMessage, '1' => $unaccountedMessage]], + 'orders', + ); + + $deliveryResult = $pendingDelivery->awaitDelivery(); + + $this->assertFalse($deliveryResult->isSuccessful()); + $this->assertCount(1, $deliveryResult->getFailedDeliveries()); + $this->assertSame($unaccountedMessage, $deliveryResult->getFailedDeliveries()[0]->getMessage()); + } + + public function test_fully_confirmed_batch_reports_success_and_is_marked_as_awaited(): void + { + $pendingDelivery = new SqsPendingDelivery( + [new FulfilledPromise(new Result([ + 'Successful' => [['Id' => '0', 'MessageId' => 'first-id'], ['Id' => '1', 'MessageId' => 'second-id']], + ]))], + [['0' => MessageBuilder::withPayload('first order')->build(), '1' => MessageBuilder::withPayload('second order')->build()]], + 'orders', + ); + + $this->assertFalse($pendingDelivery->isAwaited()); + + $deliveryResult = $pendingDelivery->awaitDelivery(); + + $this->assertTrue($deliveryResult->isSuccessful()); + $this->assertTrue($pendingDelivery->isAwaited()); + } +} From 7e59e0f0aeddbabafa5797813c6b65f6b0e992e9 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 08:00:00 +0200 Subject: [PATCH 19/38] refactor: share single and batch send flow in enqueue provider adapters --- packages/Dbal/src/EnqueueDbal/DbalProducer.php | 16 +--------------- .../src/EnqueueOutboundChannelAdapter.php | 2 +- .../Redis/src/RedisOutboundChannelAdapter.php | 9 +++++++++ packages/Sqs/src/SqsOutboundChannelAdapter.php | 8 +------- 4 files changed, 12 insertions(+), 23 deletions(-) diff --git a/packages/Dbal/src/EnqueueDbal/DbalProducer.php b/packages/Dbal/src/EnqueueDbal/DbalProducer.php index e8bbad458..025b9b032 100644 --- a/packages/Dbal/src/EnqueueDbal/DbalProducer.php +++ b/packages/Dbal/src/EnqueueDbal/DbalProducer.php @@ -68,21 +68,7 @@ public function __construct(DbalContext $context) */ public function send(Destination $destination, Message $message): void { - InvalidDestinationException::assertDestinationInstanceOf($destination, DbalDestination::class); - InvalidMessageException::assertMessageInstanceOf($message, DbalMessage::class); - - $this->applyProducerDefaults($message); - $record = $this->createRecord($destination, $message); - - try { - $rowsAffected = $this->context->getDbalConnection()->insert($this->context->getTableName(), $record, self::COLUMN_TYPES); - - if (1 !== $rowsAffected) { - throw new Exception('The message was not enqueued. Dbal did not confirm that the record is inserted.'); - } - } catch (\Exception $e) { - throw new Exception('The transport fails to send the message due to some internal error.', 0, $e); - } + $this->sendBatch($destination, [$message]); } /** diff --git a/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php b/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php index 270305135..a0da428c5 100644 --- a/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php +++ b/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php @@ -96,7 +96,7 @@ protected function prepareOutboundMessage(Message $message): OutboundMessage return $this->outboundMessageConverter->prepare($message, $this->conversionService); } - private function sendSingleMessage(Message $message, Context $context): void + protected function sendSingleMessage(Message $message, Context $context): void { $outboundMessage = $this->prepareOutboundMessage($message); $headers = $outboundMessage->getHeaders(); diff --git a/packages/Redis/src/RedisOutboundChannelAdapter.php b/packages/Redis/src/RedisOutboundChannelAdapter.php index aa8dddcfd..dc221d4d5 100644 --- a/packages/Redis/src/RedisOutboundChannelAdapter.php +++ b/packages/Redis/src/RedisOutboundChannelAdapter.php @@ -10,6 +10,7 @@ use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Conversion\ConversionService; +use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageHeaders; use Enqueue\Redis\RedisContext; use Enqueue\Redis\RedisDestination; @@ -67,6 +68,14 @@ public function initialize(): void $context->createQueue($this->queueName); } + protected function sendSingleMessage(Message $message, Context $context): void + { + $this->handleBatch( + BatchMessage::constructEmpty()->append($message->getPayload(), $message->getHeaders()->headers()), + $context, + ); + } + protected function handleBatch(BatchMessage $batchMessage, Context $context): void { if (count($batchMessage) === 0) { diff --git a/packages/Sqs/src/SqsOutboundChannelAdapter.php b/packages/Sqs/src/SqsOutboundChannelAdapter.php index 62b6ed14d..3747aa4e4 100644 --- a/packages/Sqs/src/SqsOutboundChannelAdapter.php +++ b/packages/Sqs/src/SqsOutboundChannelAdapter.php @@ -57,12 +57,6 @@ public function initialize(): void public function handle(Message $message): void { - if (! $this->asyncPublishing) { - parent::handle($message); - - return; - } - /** @var SqsContext $context */ $context = $this->createOutboundContext(); @@ -84,7 +78,7 @@ public function handle(Message $message): void $pendingDelivery = new SqsPendingDelivery($sendRequestPromises, $trackedMessagesPerRequest, $this->queueName); - if ($this->asyncPublishingRegistry->isScopeActive()) { + if ($this->asyncPublishing && $this->asyncPublishingRegistry->isScopeActive()) { $this->asyncPublishingRegistry->register($this->queueName, $pendingDelivery); return; From 198c55095c257e213b054eb2404c329b837d5ede Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 08:00:00 +0200 Subject: [PATCH 20/38] feat: benchmark single/batch sync/async publishing matrix for all providers --- .../Benchmark/AsyncPublishingBenchmark.php | 280 +++++++++++++++++- .../Benchmark/AsynchronousStackBenchmark.php | 2 +- .../Benchmark/BootingEcotoneBenchmark.php | 4 +- Monorepo/Benchmark/DbConnectBenchmark.php | 2 +- Monorepo/Benchmark/EventSourcingBenchmark.php | 4 +- Monorepo/Benchmark/FullAppBenchmarkCase.php | 32 +- Monorepo/Benchmark/HttpStackBenchmark.php | 12 +- Monorepo/Benchmark/KernelBootBenchmark.php | 3 +- Monorepo/Benchmark/LiteContainerAccessor.php | 5 +- Monorepo/Benchmark/ProjectingBenchmark.php | 51 ++-- phpbench.json | 27 +- 11 files changed, 348 insertions(+), 74 deletions(-) diff --git a/Monorepo/Benchmark/AsyncPublishingBenchmark.php b/Monorepo/Benchmark/AsyncPublishingBenchmark.php index 5e1127c04..1d8b31601 100644 --- a/Monorepo/Benchmark/AsyncPublishingBenchmark.php +++ b/Monorepo/Benchmark/AsyncPublishingBenchmark.php @@ -4,7 +4,11 @@ namespace Monorepo\Benchmark; +use Ecotone\Amqp\AmqpBackedMessageChannelBuilder; use Ecotone\Amqp\Publisher\AmqpMessagePublisherConfiguration; +use Ecotone\Dbal\Configuration\DbalMessagePublisherConfiguration; +use Ecotone\Dbal\DbalBackedMessageChannelBuilder; +use Ecotone\Kafka\Channel\KafkaMessageChannelBuilder; use Ecotone\Kafka\Configuration\KafkaBrokerConfiguration; use Ecotone\Kafka\Configuration\KafkaPublisherConfiguration; use Ecotone\Lite\EcotoneLite; @@ -12,14 +16,30 @@ use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Conversion\MediaType; +use Ecotone\Messaging\MessageChannel; use Ecotone\Messaging\MessagePublisher; +use Ecotone\Messaging\Support\MessageBuilder; +use Ecotone\Redis\Configuration\RedisMessagePublisherConfiguration; +use Ecotone\Redis\RedisBackedMessageChannelBuilder; +use Ecotone\Sqs\Configuration\SqsMessagePublisherConfiguration; +use Ecotone\Sqs\SqsBackedMessageChannelBuilder; use Ecotone\Test\LicenceTesting; use Enqueue\AmqpExt\AmqpConnectionFactory; +use Enqueue\Dbal\DbalConnectionFactory; +use Enqueue\Redis\RedisConnectionFactory; +use Enqueue\Sqs\SqsConnectionFactory; use PhpBench\Attributes\BeforeMethods; use PhpBench\Attributes\Iterations; use PhpBench\Attributes\Revs; use PhpBench\Attributes\Warmup; +/** + * Compares publishing scenarios per provider: + * single message synchronous | single message asynchronous | batch message synchronous | batch message asynchronous. + * + * DBAL and Redis confirm deliveries synchronously as part of the send call itself (database statement result + * and command reply respectively), so the asynchronous scenarios are not supported for them and are not benchmarked. + */ #[Warmup(0), Revs(1), Iterations(10)] class AsyncPublishingBenchmark { @@ -29,6 +49,8 @@ class AsyncPublishingBenchmark private MessagePublisher $publisher; + private MessageChannel $batchChannel; + public function setUpAmqpSynchronousPublishing(): void { $this->publisher = $this->bootstrapAmqpPublisher(asyncPublishing: false); @@ -41,6 +63,16 @@ public function setUpAmqpAsyncPublishing(): void $this->warmUpPublisher(); } + public function setUpAmqpBatchChannel(): void + { + $this->batchChannel = $this->bootstrapBatchChannel( + ModulePackageList::AMQP_PACKAGE, + AmqpBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withAsyncPublishing(), + [AmqpConnectionFactory::class => new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f'])], + ); + $this->warmUpBatchChannel(); + } + public function setUpKafkaSynchronousPublishing(): void { $this->publisher = $this->bootstrapKafkaPublisher(asyncPublishing: false); @@ -53,44 +85,172 @@ public function setUpKafkaAsyncPublishing(): void $this->warmUpPublisher(); } + public function setUpKafkaBatchChannel(): void + { + $uniqueId = uniqid('benchmark_orders_'); + $this->batchChannel = $this->bootstrapBatchChannel( + ModulePackageList::KAFKA_PACKAGE, + KafkaMessageChannelBuilder::create($uniqueId, topicName: $uniqueId, messageGroupId: $uniqueId)->withAsyncPublishing(), + [KafkaBrokerConfiguration::class => KafkaBrokerConfiguration::createWithDefaults([getenv('KAFKA_DSN') ?: 'localhost:9094'])], + ); + $this->warmUpBatchChannel(); + } + + public function setUpDbalSynchronousPublishing(): void + { + $this->publisher = $this->bootstrapDbalPublisher(); + $this->warmUpPublisher(); + } + + public function setUpDbalBatchChannel(): void + { + $this->batchChannel = $this->bootstrapBatchChannel( + ModulePackageList::DBAL_PACKAGE, + DbalBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withAsyncPublishing(), + [DbalConnectionFactory::class => new DbalConnectionFactory(getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone')], + ); + $this->warmUpBatchChannel(); + } + + public function setUpRedisSynchronousPublishing(): void + { + $this->publisher = $this->bootstrapRedisPublisher(); + $this->warmUpPublisher(); + } + + public function setUpRedisBatchChannel(): void + { + $this->batchChannel = $this->bootstrapBatchChannel( + ModulePackageList::REDIS_PACKAGE, + RedisBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withAsyncPublishing(), + [RedisConnectionFactory::class => new RedisConnectionFactory(getenv('REDIS_DSN') ?: 'redis://localhost:6379')], + ); + $this->warmUpBatchChannel(); + } + + public function setUpSqsSynchronousPublishing(): void + { + $this->publisher = $this->bootstrapSqsPublisher(asyncPublishing: false); + $this->warmUpPublisher(); + } + + public function setUpSqsAsyncPublishing(): void + { + $this->publisher = $this->bootstrapSqsPublisher(asyncPublishing: true); + $this->warmUpPublisher(); + } + + public function setUpSqsBatchChannel(): void + { + $this->batchChannel = $this->bootstrapBatchChannel( + ModulePackageList::SQS_PACKAGE, + SqsBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withAsyncPublishing(), + [SqsConnectionFactory::class => new SqsConnectionFactory(getenv('SQS_DSN') ?: 'sqs:?key=key&secret=secret®ion=us-east-1&endpoint=http://localhost:4566&version=latest')], + ); + $this->warmUpBatchChannel(); + } + #[BeforeMethods('setUpAmqpSynchronousPublishing')] - public function bench_amqp_synchronous_publishing(): void + public function bench_amqp_single_message_synchronous(): void { - for ($messageNumber = 0; $messageNumber < self::AMOUNT_OF_PUBLISHED_MESSAGES; $messageNumber++) { - $this->publisher->send(self::MESSAGE_PAYLOAD); - } + $this->publishSynchronouslyOneByOne(); } #[BeforeMethods('setUpAmqpAsyncPublishing')] - public function bench_amqp_async_publishing(): void + public function bench_amqp_single_message_asynchronous(): void { $this->publishAsynchronouslyOneByOne(); } + #[BeforeMethods('setUpAmqpBatchChannel')] + public function bench_amqp_batch_message_synchronous(): void + { + $this->publishBatchSynchronously(); + } + #[BeforeMethods('setUpAmqpAsyncPublishing')] - public function bench_amqp_async_batch_publishing(): void + public function bench_amqp_batch_message_asynchronous(): void { - $this->publishAsynchronouslyAsBatch(); + $this->publishBatchAsynchronously(); } #[BeforeMethods('setUpKafkaSynchronousPublishing')] - public function bench_kafka_synchronous_publishing(): void + public function bench_kafka_single_message_synchronous(): void { - for ($messageNumber = 0; $messageNumber < self::AMOUNT_OF_PUBLISHED_MESSAGES; $messageNumber++) { - $this->publisher->send(self::MESSAGE_PAYLOAD); - } + $this->publishSynchronouslyOneByOne(); } #[BeforeMethods('setUpKafkaAsyncPublishing')] - public function bench_kafka_async_publishing(): void + public function bench_kafka_single_message_asynchronous(): void { $this->publishAsynchronouslyOneByOne(); } + #[BeforeMethods('setUpKafkaBatchChannel')] + public function bench_kafka_batch_message_synchronous(): void + { + $this->publishBatchSynchronously(); + } + #[BeforeMethods('setUpKafkaAsyncPublishing')] - public function bench_kafka_async_batch_publishing(): void + public function bench_kafka_batch_message_asynchronous(): void + { + $this->publishBatchAsynchronously(); + } + + #[BeforeMethods('setUpDbalSynchronousPublishing')] + public function bench_dbal_single_message_synchronous(): void + { + $this->publishSynchronouslyOneByOne(); + } + + #[BeforeMethods('setUpDbalBatchChannel')] + public function bench_dbal_batch_message_synchronous(): void + { + $this->publishBatchSynchronously(); + } + + #[BeforeMethods('setUpRedisSynchronousPublishing')] + public function bench_redis_single_message_synchronous(): void + { + $this->publishSynchronouslyOneByOne(); + } + + #[BeforeMethods('setUpRedisBatchChannel')] + public function bench_redis_batch_message_synchronous(): void + { + $this->publishBatchSynchronously(); + } + + #[BeforeMethods('setUpSqsSynchronousPublishing')] + public function bench_sqs_single_message_synchronous(): void + { + $this->publishSynchronouslyOneByOne(); + } + + #[BeforeMethods('setUpSqsAsyncPublishing')] + public function bench_sqs_single_message_asynchronous(): void + { + $this->publishAsynchronouslyOneByOne(); + } + + #[BeforeMethods('setUpSqsBatchChannel')] + public function bench_sqs_batch_message_synchronous(): void { - $this->publishAsynchronouslyAsBatch(); + $this->publishBatchSynchronously(); + } + + #[BeforeMethods('setUpSqsAsyncPublishing')] + public function bench_sqs_batch_message_asynchronous(): void + { + $this->publishBatchAsynchronously(); + } + + private function publishSynchronouslyOneByOne(): void + { + for ($messageNumber = 0; $messageNumber < self::AMOUNT_OF_PUBLISHED_MESSAGES; $messageNumber++) { + $this->publisher->send(self::MESSAGE_PAYLOAD); + } } private function publishAsynchronouslyOneByOne(): void @@ -104,14 +264,26 @@ private function publishAsynchronouslyOneByOne(): void } } - private function publishAsynchronouslyAsBatch(): void + private function publishBatchSynchronously(): void + { + $this->batchChannel->send( + MessageBuilder::withPayload($this->buildBatch())->build() + ); + } + + private function publishBatchAsynchronously(): void + { + $this->publisher->asyncPublish($this->buildBatch(), MediaType::TEXT_PLAIN)->resolve(); + } + + private function buildBatch(): BatchMessage { $batch = BatchMessage::constructEmpty(); for ($messageNumber = 0; $messageNumber < self::AMOUNT_OF_PUBLISHED_MESSAGES; $messageNumber++) { $batch = $batch->append(self::MESSAGE_PAYLOAD, ['contentType' => MediaType::TEXT_PLAIN]); } - $this->publisher->asyncPublish($batch, MediaType::TEXT_PLAIN)->resolve(); + return $batch; } private function warmUpPublisher(): void @@ -119,6 +291,29 @@ private function warmUpPublisher(): void $this->publisher->send(self::MESSAGE_PAYLOAD); } + private function warmUpBatchChannel(): void + { + $this->batchChannel->send( + MessageBuilder::withPayload(self::MESSAGE_PAYLOAD) + ->setContentType(MediaType::createTextPlain()) + ->build() + ); + } + + private function bootstrapBatchChannel(string $modulePackage, object $channelBuilder, array $services): MessageChannel + { + $messaging = EcotoneLite::bootstrapFlowTesting( + [], + $services, + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, $modulePackage])) + ->withExtensionObjects([$channelBuilder]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + return $messaging->getMessageChannel($channelBuilder->getMessageChannelName()); + } + private function bootstrapAmqpPublisher(bool $asyncPublishing): MessagePublisher { $publisherConfiguration = AmqpMessagePublisherConfiguration::create() @@ -142,6 +337,59 @@ private function bootstrapAmqpPublisher(bool $asyncPublishing): MessagePublisher return $messaging->getGateway(MessagePublisher::class); } + private function bootstrapDbalPublisher(): MessagePublisher + { + $messaging = EcotoneLite::bootstrapFlowTesting( + [], + [ + DbalConnectionFactory::class => new DbalConnectionFactory(getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone'), + ], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([DbalMessagePublisherConfiguration::create(MessagePublisher::class, uniqid('benchmark_orders_'))]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + return $messaging->getGateway(MessagePublisher::class); + } + + private function bootstrapRedisPublisher(): MessagePublisher + { + $messaging = EcotoneLite::bootstrapFlowTesting( + [], + [ + RedisConnectionFactory::class => new RedisConnectionFactory(getenv('REDIS_DSN') ?: 'redis://localhost:6379'), + ], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::REDIS_PACKAGE])) + ->withExtensionObjects([RedisMessagePublisherConfiguration::create(queueName: uniqid('benchmark_orders_'))]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + return $messaging->getGateway(MessagePublisher::class); + } + + private function bootstrapSqsPublisher(bool $asyncPublishing): MessagePublisher + { + $publisherConfiguration = SqsMessagePublisherConfiguration::create(queueName: uniqid('benchmark_orders_')); + if ($asyncPublishing) { + $publisherConfiguration = $publisherConfiguration->withAsyncPublishing(); + } + + $messaging = EcotoneLite::bootstrapFlowTesting( + [], + [ + SqsConnectionFactory::class => new SqsConnectionFactory(getenv('SQS_DSN') ?: 'sqs:?key=key&secret=secret®ion=us-east-1&endpoint=http://localhost:4566&version=latest'), + ], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::SQS_PACKAGE])) + ->withExtensionObjects([$publisherConfiguration]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + return $messaging->getGateway(MessagePublisher::class); + } + private function bootstrapKafkaPublisher(bool $asyncPublishing): MessagePublisher { $publisherConfiguration = KafkaPublisherConfiguration::createWithDefaults(topicName: uniqid('benchmark_orders_')); diff --git a/Monorepo/Benchmark/AsynchronousStackBenchmark.php b/Monorepo/Benchmark/AsynchronousStackBenchmark.php index a27c17257..e33203412 100644 --- a/Monorepo/Benchmark/AsynchronousStackBenchmark.php +++ b/Monorepo/Benchmark/AsynchronousStackBenchmark.php @@ -100,4 +100,4 @@ private function placeOrder(mixed $commandBus, mixed $configuration): void ) ); } -} \ No newline at end of file +} diff --git a/Monorepo/Benchmark/BootingEcotoneBenchmark.php b/Monorepo/Benchmark/BootingEcotoneBenchmark.php index b56db67ac..f5a6e2a9b 100644 --- a/Monorepo/Benchmark/BootingEcotoneBenchmark.php +++ b/Monorepo/Benchmark/BootingEcotoneBenchmark.php @@ -2,11 +2,9 @@ namespace Monorepo\Benchmark; -use Ecotone\Lite\EcotoneLiteApplication; use Ecotone\Messaging\Config\ConfiguredMessagingSystem; use Illuminate\Foundation\Http\Kernel as LaravelKernel; use Monorepo\ExampleApp\ExampleAppCaseTrait; -use Monorepo\ExampleApp\Symfony\Kernel; use PhpBench\Attributes\Iterations; use PhpBench\Attributes\Revs; use PhpBench\Attributes\Warmup; @@ -36,4 +34,4 @@ public function executeForLite(ConfiguredMessagingSystem $messagingSystem): void { $messagingSystem->list(); } -} \ No newline at end of file +} diff --git a/Monorepo/Benchmark/DbConnectBenchmark.php b/Monorepo/Benchmark/DbConnectBenchmark.php index 027841cf4..0146ec874 100644 --- a/Monorepo/Benchmark/DbConnectBenchmark.php +++ b/Monorepo/Benchmark/DbConnectBenchmark.php @@ -15,4 +15,4 @@ public function bench_db_connect(): void $connection->executeQuery('SELECT 1'); } -} \ No newline at end of file +} diff --git a/Monorepo/Benchmark/EventSourcingBenchmark.php b/Monorepo/Benchmark/EventSourcingBenchmark.php index 4269f3a84..16d1b3b03 100644 --- a/Monorepo/Benchmark/EventSourcingBenchmark.php +++ b/Monorepo/Benchmark/EventSourcingBenchmark.php @@ -27,7 +27,7 @@ public static function skippedPackages(): array { return ModulePackageList::allPackagesExcept([ ModulePackageList::EVENT_SOURCING_PACKAGE, - ModulePackageList::JMS_CONVERTER_PACKAGE + ModulePackageList::JMS_CONVERTER_PACKAGE, ]); } @@ -74,4 +74,4 @@ public function executeForLite(ConfiguredMessagingSystem $messagingSystem): void $messagingSystem->getQueryBus() ); } -} \ No newline at end of file +} diff --git a/Monorepo/Benchmark/FullAppBenchmarkCase.php b/Monorepo/Benchmark/FullAppBenchmarkCase.php index d426c51eb..afa6b2598 100644 --- a/Monorepo/Benchmark/FullAppBenchmarkCase.php +++ b/Monorepo/Benchmark/FullAppBenchmarkCase.php @@ -8,9 +8,15 @@ use Illuminate\Foundation\Application; use Illuminate\Foundation\Http\Kernel as LaravelKernel; use Illuminate\Support\Facades\Artisan; + +use function json_encode; + use PHPUnit\Framework\Assert; use PHPUnit\Framework\TestCase; use Psr\Container\ContainerInterface; + +use function putenv; + use Symfony\Bundle\FrameworkBundle\Console\Application as SymfonyConsoleApplication; use Symfony\Component\Console\Input\ArrayInput; use Symfony\Component\Console\Output\NullOutput; @@ -168,39 +174,39 @@ private static function deleteFiles(string $target, bool $deleteDirectory): void } } - public abstract function executeForSymfony( + abstract public function executeForSymfony( ContainerInterface $container, SymfonyKernel $kernel ): void; - public abstract function executeForLaravel( + abstract public function executeForLaravel( ContainerInterface $container, LaravelKernel $kernel ): void; - public abstract function executeForLiteApplication( + abstract public function executeForLiteApplication( ContainerInterface $container ): void; - public abstract function executeForLite( + abstract public function executeForLite( ConfiguredMessagingSystem $messagingSystem ): void; - protected abstract static function getSymfonyKernelClass(): string; - protected abstract static function getProjectDir(): string; + abstract protected static function getSymfonyKernelClass(): string; + abstract protected static function getProjectDir(): string; private static function productionEnvironments(): void { - \putenv('APP_ENV=prod'); - \putenv('APP_DEBUG=false'); - \putenv(sprintf('APP_SKIPPED_PACKAGES=%s', \json_encode(static::skippedPackages(), JSON_THROW_ON_ERROR))); + putenv('APP_ENV=prod'); + putenv('APP_DEBUG=false'); + putenv(sprintf('APP_SKIPPED_PACKAGES=%s', json_encode(static::skippedPackages(), JSON_THROW_ON_ERROR))); } private static function developmentEnvironments(): void { - \putenv('APP_ENV=dev'); - \putenv('APP_DEBUG=true'); - \putenv(sprintf("APP_SKIPPED_PACKAGES=%s", \json_encode(static::skippedPackages(), JSON_THROW_ON_ERROR))); + putenv('APP_ENV=dev'); + putenv('APP_DEBUG=true'); + putenv(sprintf('APP_SKIPPED_PACKAGES=%s', json_encode(static::skippedPackages(), JSON_THROW_ON_ERROR))); } public static function skippedPackages(): array @@ -261,4 +267,4 @@ public static function runConsumerForMessaging(string $consumerName, ConfiguredM ->withExecutionTimeLimitInMilliseconds(2000) ); } -} \ No newline at end of file +} diff --git a/Monorepo/Benchmark/HttpStackBenchmark.php b/Monorepo/Benchmark/HttpStackBenchmark.php index b94cca685..f83689fc6 100644 --- a/Monorepo/Benchmark/HttpStackBenchmark.php +++ b/Monorepo/Benchmark/HttpStackBenchmark.php @@ -10,7 +10,6 @@ use Monorepo\ExampleApp\Common\Infrastructure\Configuration; use Monorepo\ExampleApp\Common\UI\OrderController; use Monorepo\ExampleApp\ExampleAppCaseTrait; -use Monorepo\ExampleApp\Symfony\Kernel as SymfonyKernel; use PhpBench\Attributes\Iterations; use PhpBench\Attributes\Revs; use PhpBench\Attributes\Warmup; @@ -27,7 +26,8 @@ public function executeForSymfony(ContainerInterface $container, \Symfony\Compon { $configuration = $container->get(Configuration::class); $response = $kernel->handle( - SymfonyRequest::create('/place-order', + SymfonyRequest::create( + '/place-order', 'POST', content: json_encode([ 'orderId' => Uuid::uuid4()->toString(), @@ -35,7 +35,7 @@ public function executeForSymfony(ContainerInterface $container, \Symfony\Compon 'street' => 'Washington', 'houseNumber' => '15', 'postCode' => '81-221', - 'country' => 'Netherlands' + 'country' => 'Netherlands', ], 'productId' => $configuration->productId(), ]) @@ -58,7 +58,7 @@ public function executeForLaravel(ContainerInterface $container, LaravelKernel $ 'street' => 'Washington', 'houseNumber' => '15', 'postCode' => '81-221', - 'country' => 'Netherlands' + 'country' => 'Netherlands', ], 'productId' => $configuration->productId(), ]) @@ -79,7 +79,7 @@ public function executeForLiteApplication(ContainerInterface $container): void 'street' => 'Washington', 'houseNumber' => '15', 'postCode' => '81-221', - 'country' => 'Netherlands' + 'country' => 'Netherlands', ], 'productId' => $configuration->productId(), ]))); @@ -103,4 +103,4 @@ public function executeForLite(ConfiguredMessagingSystem $messagingSystem): void ) ); } -} \ No newline at end of file +} diff --git a/Monorepo/Benchmark/KernelBootBenchmark.php b/Monorepo/Benchmark/KernelBootBenchmark.php index 088c3bc29..81953069a 100644 --- a/Monorepo/Benchmark/KernelBootBenchmark.php +++ b/Monorepo/Benchmark/KernelBootBenchmark.php @@ -5,7 +5,6 @@ use Ecotone\Messaging\Config\ConfiguredMessagingSystem; use Illuminate\Foundation\Http\Kernel as LaravelKernel; use Monorepo\ExampleApp\ExampleAppCaseTrait; -use Monorepo\ExampleApp\Symfony\Kernel; use PhpBench\Attributes\Iterations; use PhpBench\Attributes\Revs; use PhpBench\Attributes\Warmup; @@ -35,4 +34,4 @@ public function executeForLite(ConfiguredMessagingSystem $messagingSystem): void { // do nothing } -} \ No newline at end of file +} diff --git a/Monorepo/Benchmark/LiteContainerAccessor.php b/Monorepo/Benchmark/LiteContainerAccessor.php index 5c37dd2cc..909586ecd 100644 --- a/Monorepo/Benchmark/LiteContainerAccessor.php +++ b/Monorepo/Benchmark/LiteContainerAccessor.php @@ -3,6 +3,7 @@ namespace Monorepo\Benchmark; use Ecotone\Messaging\Config\ConfiguredMessagingSystem; +use Exception; use Psr\Container\ContainerInterface; class LiteContainerAccessor implements ContainerInterface @@ -18,6 +19,6 @@ public function get(string $id) public function has(string $id): bool { - throw new \Exception("Not implemented"); + throw new Exception('Not implemented'); } -} \ No newline at end of file +} diff --git a/Monorepo/Benchmark/ProjectingBenchmark.php b/Monorepo/Benchmark/ProjectingBenchmark.php index 46137c926..fb1be9630 100644 --- a/Monorepo/Benchmark/ProjectingBenchmark.php +++ b/Monorepo/Benchmark/ProjectingBenchmark.php @@ -2,13 +2,13 @@ namespace Monorepo\Benchmark; +use Closure; use Ecotone\EventSourcing\EventStore; use Ecotone\EventSourcing\ProjectionManager; use Ecotone\Lite\EcotoneLite; use Ecotone\Messaging\Config\ConfiguredMessagingSystem; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; -use Ecotone\Modelling\CommandBus; use Ecotone\Projecting\ProjectionRegistry; use Ecotone\Test\LicenceTesting; use Enqueue\Dbal\DbalConnectionFactory; @@ -81,25 +81,25 @@ public function setUp(): void self::deleteProophProjection(); } - #[BeforeMethods("setUp")] + #[BeforeMethods('setUp')] public function bench_ecotone_projection(): void { self::execute(self::$ecotone); } - #[BeforeMethods("setUp")] + #[BeforeMethods('setUp')] public function bench_prooph_projection(): void { self::execute(self::$prooph); } - #[BeforeMethods("setUp")] + #[BeforeMethods('setUp')] public function bench_ecotone_projection_with_deletion(): void { self::executeWithDeletion(self::$ecotone, self::deleteEcotoneProjection(...)); } - #[BeforeMethods("setUp")] + #[BeforeMethods('setUp')] public function bench_prooph_projection_with_deletion(): void { self::executeWithDeletion(self::$prooph, self::deleteProophProjection(...)); @@ -132,7 +132,7 @@ public static function execute(ConfiguredMessagingSystem $messagingSystem): void Assert::assertEquals([new PriceChange(100, 0), new PriceChange(120, 20)], $queryBus->sendWithRouting('product.getPriceChange', $productId), 'Price change should equal to 0 after registration'); } - private static function executeWithDeletion(ConfiguredMessagingSystem $messagingSystem, \Closure $deleteProjection): void + private static function executeWithDeletion(ConfiguredMessagingSystem $messagingSystem, Closure $deleteProjection): void { $commandBus = $messagingSystem->getCommandBus(); $queryBus = $messagingSystem->getQueryBus(); @@ -150,11 +150,12 @@ private static function executeWithDeletion(ConfiguredMessagingSystem $messaging Assert::assertEquals([ new PriceChange(100, 0), new PriceChange(120, 20), - new PriceChange(130, 10) + new PriceChange(130, 10), ], $queryBus->sendWithRouting('product.getPriceChange', $productId), 'Price changes should be projected again after deletion'); } - public function fill(): void { + public function fill(): void + { $commandBus = self::$ecotone->getCommandBus(); self::$expectedProductIds = []; for ($i = 0; $i < 100; $i++) { @@ -165,41 +166,45 @@ public function fill(): void { } } - #[BeforeMethods(["setUp", "fill"])] + #[BeforeMethods(['setUp', 'fill'])] #[Iterations(1), Warmup(0)] public function bench_ecotone_projection_backfill(): void { $projectionManager = self::$ecotone->getServiceFromContainer(ProjectionRegistry::class)->get(PriceChangeOverTimeProjectionWithEcotoneProjection::NAME); $projectionManager->delete(); - Assert::assertEquals([], + Assert::assertEquals( + [], self::$ecotone->getQueryBus()->sendWithRouting('product.getPriceChange', self::$expectedProductIds[0]) ); $projectionManager->prepareBackfill(); - Assert::assertEquals([ - new PriceChange(100, 0), - new PriceChange(120, 20), - new PriceChange(130, 10), - ], + Assert::assertEquals( + [ + new PriceChange(100, 0), + new PriceChange(120, 20), + new PriceChange(130, 10), + ], self::$ecotone->getQueryBus()->sendWithRouting('product.getPriceChange', self::$expectedProductIds[0]) ); } - #[BeforeMethods(["setUp", "fill"])] + #[BeforeMethods(['setUp', 'fill'])] #[Iterations(1), Warmup(0)] public function bench_prooph_projection_backfill(): void { $projectionManager = self::$prooph->getServiceFromContainer(ProjectionManager::class); $projectionManager->deleteProjection(PriceChangeOverTimeProjection::NAME); - Assert::assertEquals([], + Assert::assertEquals( + [], self::$prooph->getQueryBus()->sendWithRouting('product.getPriceChange', self::$expectedProductIds[0]) ); $projectionManager->triggerProjection(PriceChangeOverTimeProjection::NAME); - Assert::assertEquals([ - new PriceChange(100, 0), - new PriceChange(120, 20), - new PriceChange(130, 10), - ], + Assert::assertEquals( + [ + new PriceChange(100, 0), + new PriceChange(120, 20), + new PriceChange(130, 10), + ], self::$prooph->getQueryBus()->sendWithRouting('product.getPriceChange', self::$expectedProductIds[0]) ); } -} \ No newline at end of file +} diff --git a/phpbench.json b/phpbench.json index ae44447e0..7d08e3e21 100644 --- a/phpbench.json +++ b/phpbench.json @@ -1,5 +1,5 @@ { - "$schema":"./vendor/phpbench/phpbench/phpbench.schema.json", + "$schema": "./vendor/phpbench/phpbench/phpbench.schema.json", "runner.bootstrap": "./vendor/autoload.php", "runner.file_pattern": "*Benchmark.php", "runner.path": "Monorepo/Benchmark", @@ -9,7 +9,8 @@ "opcache_disabled": { "runner.php_config": { "opcache.enable": 0, - "opcache.enable_cli": 0 + "opcache.enable_cli": 0, + "display_errors": "0" } }, "opcache_enabled": { @@ -20,15 +21,31 @@ "opcache.validate_timestamps": 0, "opcache.max_accelerated_files": 20000, "opcache.memory_consumption": 256, - "opcache.jit_buffer_size": "0" + "opcache.jit_buffer_size": "0", + "display_errors": "0" } } }, "report.generators": { "github-report": { "generator": "expression", - "aggregate": ["benchmark_class", "subject_name", "variant_name"], - "cols": ["benchmark", "subject", "revs", "its", "mem_peak", "mode", "rstdev"] + "aggregate": [ + "benchmark_class", + "subject_name", + "variant_name" + ], + "cols": [ + "benchmark", + "subject", + "revs", + "its", + "mem_peak", + "mode", + "rstdev" + ] } + }, + "runner.php_config": { + "display_errors": "0" } } \ No newline at end of file From f5c29239dd883d296bc817ea30c158e96ed89952 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 08:00:00 +0200 Subject: [PATCH 21/38] perf: bound SQS async publishing concurrency via lazy request dispatching --- .github/workflows/benchmark-pr.yml | 4 +- .github/workflows/quickstart-examples.yml | 4 +- .github/workflows/split-testing.yml | 4 +- .github/workflows/test-monorepo.yml | 4 +- docker-compose.yml | 4 +- .../Sqs/src/SqsOutboundChannelAdapter.php | 7 ++- packages/Sqs/src/SqsPendingDelivery.php | 42 ++++++++++--- .../Sqs/tests/Unit/SqsPendingDeliveryTest.php | 60 +++++++++++++++++-- 8 files changed, 110 insertions(+), 19 deletions(-) diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 74529d0f8..68f4a334c 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -71,9 +71,11 @@ jobs: ports: - 5432:5432 localstack: - image: localstack/localstack:3.0.0 + image: localstack/localstack:3.8 env: SERVICES: 'sqs,sns' + SQS_DISABLE_CLOUDWATCH_METRICS: '1' + EAGER_SERVICE_LOADING: '1' ports: - "4566:4566" - "4510-4559:4510-4559" diff --git a/.github/workflows/quickstart-examples.yml b/.github/workflows/quickstart-examples.yml index e7a21a18a..573774c38 100644 --- a/.github/workflows/quickstart-examples.yml +++ b/.github/workflows/quickstart-examples.yml @@ -75,9 +75,11 @@ jobs: ports: - 5432:5432 localstack: - image: localstack/localstack:3.0.0 + image: localstack/localstack:3.8 env: SERVICES: 'sqs,sns' + SQS_DISABLE_CLOUDWATCH_METRICS: '1' + EAGER_SERVICE_LOADING: '1' ports: - "4566:4566" - "4510-4559:4510-4559" diff --git a/.github/workflows/split-testing.yml b/.github/workflows/split-testing.yml index 856127d0f..92cc954ea 100644 --- a/.github/workflows/split-testing.yml +++ b/.github/workflows/split-testing.yml @@ -78,9 +78,11 @@ jobs: ports: - 5432:5432 localstack: - image: localstack/localstack:3.0.0 + image: localstack/localstack:3.8 env: SERVICES: 'sqs,sns' + SQS_DISABLE_CLOUDWATCH_METRICS: '1' + EAGER_SERVICE_LOADING: '1' ports: - "4566:4566" - "4510-4559:4510-4559" diff --git a/.github/workflows/test-monorepo.yml b/.github/workflows/test-monorepo.yml index f7e0f243d..765c3615d 100644 --- a/.github/workflows/test-monorepo.yml +++ b/.github/workflows/test-monorepo.yml @@ -71,9 +71,11 @@ jobs: ports: - 5432:5432 localstack: - image: localstack/localstack:3.0.0 + image: localstack/localstack:3.8 env: SERVICES: 'sqs,sns' + SQS_DISABLE_CLOUDWATCH_METRICS: '1' + EAGER_SERVICE_LOADING: '1' ports: - "4566:4566" - "4510-4559:4510-4559" diff --git a/docker-compose.yml b/docker-compose.yml index 1b0132b73..0347271df 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,10 +81,12 @@ services: - '${RABBITMQ_PORT:-0}:5672' - '${RABBITMQ_MGMT_PORT:-0}:15672' localstack: - image: localstack/localstack:3.0.0 + image: localstack/localstack:3.8 environment: LOCALSTACK_HOST: 'localstack' SERVICES: 'sqs,sns' + SQS_DISABLE_CLOUDWATCH_METRICS: '1' + EAGER_SERVICE_LOADING: '1' ports: - "${LOCALSTACK_PORT:-0}:4566" # LocalStack Gateway # - "4510-4559:4510-4559" # external services port range diff --git a/packages/Sqs/src/SqsOutboundChannelAdapter.php b/packages/Sqs/src/SqsOutboundChannelAdapter.php index 3747aa4e4..1aba33eb9 100644 --- a/packages/Sqs/src/SqsOutboundChannelAdapter.php +++ b/packages/Sqs/src/SqsOutboundChannelAdapter.php @@ -69,14 +69,15 @@ public function handle(Message $message): void return; } - $sendRequestPromises = []; + $sendRequestDispatchers = []; $trackedMessagesPerRequest = []; foreach ($this->buildBatchRequests($messagesToPublish, $context) as $batchRequest) { - $sendRequestPromises[] = $context->getAwsSqsClient()->sendMessageBatchAsync($batchRequest['arguments']); + $requestArguments = $batchRequest['arguments']; + $sendRequestDispatchers[] = fn () => $context->getAwsSqsClient()->sendMessageBatchAsync($requestArguments); $trackedMessagesPerRequest[] = $batchRequest['trackedMessages']; } - $pendingDelivery = new SqsPendingDelivery($sendRequestPromises, $trackedMessagesPerRequest, $this->queueName); + $pendingDelivery = new SqsPendingDelivery($sendRequestDispatchers, $trackedMessagesPerRequest, $this->queueName); if ($this->asyncPublishing && $this->asyncPublishingRegistry->isScopeActive()) { $this->asyncPublishingRegistry->register($this->queueName, $pendingDelivery); diff --git a/packages/Sqs/src/SqsPendingDelivery.php b/packages/Sqs/src/SqsPendingDelivery.php index ba1644980..c4beab5c0 100644 --- a/packages/Sqs/src/SqsPendingDelivery.php +++ b/packages/Sqs/src/SqsPendingDelivery.php @@ -5,12 +5,13 @@ namespace Ecotone\Sqs; use Aws\Result; +use Closure; use Ecotone\Messaging\Channel\AsyncPublishing\DeliveryResult; use Ecotone\Messaging\Channel\AsyncPublishing\FailedDelivery; use Ecotone\Messaging\Channel\AsyncPublishing\PendingDelivery; use Ecotone\Messaging\Message; +use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; -use GuzzleHttp\Promise\Utils; use Throwable; /** @@ -18,16 +19,19 @@ */ final class SqsPendingDelivery implements PendingDelivery { + public const DEFAULT_MAX_CONCURRENT_REQUESTS = 25; + private bool $awaited = false; /** - * @param PromiseInterface[] $sendRequestPromises + * @param Closure[] $sendRequestDispatchers each returns a PromiseInterface when invoked, so requests are dispatched lazily with bounded concurrency * @param array> $trackedMessagesPerRequest keyed by request index, then by batch entry id */ public function __construct( - private array $sendRequestPromises, + private array $sendRequestDispatchers, private array $trackedMessagesPerRequest, private string $channelName, + private int $maxConcurrentRequests = self::DEFAULT_MAX_CONCURRENT_REQUESTS, ) { } @@ -35,11 +39,11 @@ public function awaitDelivery(): DeliveryResult { $this->awaited = true; - $failedDeliveries = []; - $settledResults = Utils::settle($this->sendRequestPromises)->wait(); + $settledResults = $this->dispatchWithBoundedConcurrency(); - foreach ($settledResults as $requestIndex => $settledResult) { - $trackedMessages = $this->trackedMessagesPerRequest[$requestIndex] ?? []; + $failedDeliveries = []; + foreach ($this->trackedMessagesPerRequest as $requestIndex => $trackedMessages) { + $settledResult = $settledResults[$requestIndex] ?? ['state' => PromiseInterface::REJECTED, 'reason' => 'SQS send request was never dispatched']; if ($settledResult['state'] !== PromiseInterface::FULFILLED) { $failureReason = $settledResult['reason'] instanceof Throwable @@ -92,4 +96,28 @@ public function isAwaited(): bool { return $this->awaited; } + + /** + * @return array + */ + private function dispatchWithBoundedConcurrency(): array + { + $settledResults = []; + $sendRequestPromises = (function () use (&$settledResults) { + foreach ($this->sendRequestDispatchers as $requestIndex => $dispatchSendRequest) { + yield $dispatchSendRequest()->then( + function (mixed $value) use (&$settledResults, $requestIndex): void { + $settledResults[$requestIndex] = ['state' => PromiseInterface::FULFILLED, 'value' => $value]; + }, + function (mixed $reason) use (&$settledResults, $requestIndex): void { + $settledResults[$requestIndex] = ['state' => PromiseInterface::REJECTED, 'reason' => $reason]; + }, + ); + } + })(); + + Each::ofLimit($sendRequestPromises, $this->maxConcurrentRequests)->wait(); + + return $settledResults; + } } diff --git a/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php b/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php index df867d6a5..d994f0195 100644 --- a/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php +++ b/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php @@ -23,7 +23,7 @@ public function test_partially_failed_batch_reports_failed_entries_mapped_to_ori $deliveredMessage = MessageBuilder::withPayload('delivered order')->build(); $rejectedMessage = MessageBuilder::withPayload('rejected order')->build(); $pendingDelivery = new SqsPendingDelivery( - [new FulfilledPromise(new Result([ + [fn () => new FulfilledPromise(new Result([ 'Successful' => [['Id' => '0', 'MessageId' => 'aws-message-id']], 'Failed' => [['Id' => '1', 'Code' => 'InternalError', 'Message' => 'server hiccup', 'SenderFault' => false]], ]))], @@ -46,7 +46,7 @@ public function test_rejected_request_reports_all_messages_of_that_request_as_fa $firstMessage = MessageBuilder::withPayload('first order')->build(); $secondMessage = MessageBuilder::withPayload('second order')->build(); $pendingDelivery = new SqsPendingDelivery( - [new RejectedPromise(new RuntimeException('connection refused'))], + [fn () => new RejectedPromise(new RuntimeException('connection refused'))], [['0' => $firstMessage, '1' => $secondMessage]], 'orders', ); @@ -63,7 +63,7 @@ public function test_entries_missing_from_successful_and_failed_lists_are_report $confirmedMessage = MessageBuilder::withPayload('confirmed order')->build(); $unaccountedMessage = MessageBuilder::withPayload('unaccounted order')->build(); $pendingDelivery = new SqsPendingDelivery( - [new FulfilledPromise(new Result([ + [fn () => new FulfilledPromise(new Result([ 'Successful' => [['Id' => '0', 'MessageId' => 'aws-message-id']], ]))], [['0' => $confirmedMessage, '1' => $unaccountedMessage]], @@ -80,7 +80,7 @@ public function test_entries_missing_from_successful_and_failed_lists_are_report public function test_fully_confirmed_batch_reports_success_and_is_marked_as_awaited(): void { $pendingDelivery = new SqsPendingDelivery( - [new FulfilledPromise(new Result([ + [fn () => new FulfilledPromise(new Result([ 'Successful' => [['Id' => '0', 'MessageId' => 'first-id'], ['Id' => '1', 'MessageId' => 'second-id']], ]))], [['0' => MessageBuilder::withPayload('first order')->build(), '1' => MessageBuilder::withPayload('second order')->build()]], @@ -94,4 +94,56 @@ public function test_fully_confirmed_batch_reports_success_and_is_marked_as_awai $this->assertTrue($deliveryResult->isSuccessful()); $this->assertTrue($pendingDelivery->isAwaited()); } + + public function test_send_requests_are_dispatched_lazily_on_await_not_on_creation(): void + { + $dispatchedRequests = 0; + $pendingDelivery = new SqsPendingDelivery( + [function () use (&$dispatchedRequests) { + $dispatchedRequests++; + + return new FulfilledPromise(new Result(['Successful' => [['Id' => '0', 'MessageId' => 'aws-message-id']]])); + }], + [['0' => MessageBuilder::withPayload('order')->build()]], + 'orders', + ); + + $this->assertSame(0, $dispatchedRequests); + + $pendingDelivery->awaitDelivery(); + + $this->assertSame(1, $dispatchedRequests); + } + + public function test_all_requests_are_dispatched_even_when_earlier_request_is_rejected(): void + { + $dispatchedRequests = 0; + $countingDispatcher = function ($promise) use (&$dispatchedRequests) { + return function () use ($promise, &$dispatchedRequests) { + $dispatchedRequests++; + + return $promise; + }; + }; + $pendingDelivery = new SqsPendingDelivery( + [ + $countingDispatcher(new RejectedPromise(new RuntimeException('connection refused'))), + $countingDispatcher(new FulfilledPromise(new Result(['Successful' => [['Id' => '0', 'MessageId' => 'second-id']]]))), + $countingDispatcher(new FulfilledPromise(new Result(['Successful' => [['Id' => '0', 'MessageId' => 'third-id']]]))), + ], + [ + ['0' => MessageBuilder::withPayload('first order')->build()], + ['0' => MessageBuilder::withPayload('second order')->build()], + ['0' => MessageBuilder::withPayload('third order')->build()], + ], + 'orders', + maxConcurrentRequests: 1, + ); + + $deliveryResult = $pendingDelivery->awaitDelivery(); + + $this->assertSame(3, $dispatchedRequests); + $this->assertCount(1, $deliveryResult->getFailedDeliveries()); + $this->assertSame('first order', $deliveryResult->getFailedDeliveries()[0]->getMessage()->getPayload()); + } } From 4e88568ed2b908d0a7bd9aecec5f4155bafa76ba Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Fri, 3 Jul 2026 08:00:00 +0200 Subject: [PATCH 22/38] fix: close silent-loss paths across async publishing providers --- .../src/AmqpExtPublisherConfirmations.php | 8 +++ .../Amqp/src/AmqpOutboundChannelAdapter.php | 37 ++++++----- .../src/AmqpOutboundChannelAdapterBuilder.php | 1 + packages/Amqp/src/AmqpPendingDelivery.php | 23 ++++++- .../AmqpReconnectableConnectionFactory.php | 4 +- .../Integration/AmqpMessageChannelTest.php | 8 ++- .../AsyncPublishingReliabilityTest.php | 32 ++++++++++ .../tests/Integration/AsyncPublishingTest.php | 5 +- .../Dbal/src/EnqueueDbal/DbalProducer.php | 8 +-- .../tests/Integration/AsyncPublishingTest.php | 15 +++++ .../AsyncPublishingRegistry.php | 19 ++++-- .../AsyncPublishingWaiterInterceptor.php | 14 ++--- .../AsyncPublishing/DeliveryFuture.php | 32 +++++----- .../InMemoryPendingDelivery.php | 10 ++- .../AsyncPublishingReliabilityTest.php | 61 ++++++++++++++++++ .../Kafka/src/Configuration/KafkaAdmin.php | 14 ++--- .../src/Outbound/KafkaDeliveryTracker.php | 5 ++ .../Outbound/KafkaOutboundChannelAdapter.php | 63 +++++++++++++------ .../src/Outbound/KafkaPendingDelivery.php | 8 ++- .../AsyncPublishingReliabilityTest.php | 21 +++++++ .../Redis/src/RedisOutboundChannelAdapter.php | 18 ++++++ .../tests/Integration/AsyncPublishingTest.php | 50 +++++++++++++++ packages/Sqs/src/SqsPendingDelivery.php | 20 ++++-- .../Sqs/tests/Unit/SqsPendingDeliveryTest.php | 47 ++++++++++++++ 24 files changed, 437 insertions(+), 86 deletions(-) diff --git a/packages/Amqp/src/AmqpExtPublisherConfirmations.php b/packages/Amqp/src/AmqpExtPublisherConfirmations.php index 723817eb7..40f1bce29 100644 --- a/packages/Amqp/src/AmqpExtPublisherConfirmations.php +++ b/packages/Amqp/src/AmqpExtPublisherConfirmations.php @@ -11,6 +11,8 @@ final class AmqpExtPublisherConfirmations { private int $publishedCount = 0; + private int $epoch = 0; + private int $highestConfirmedTag = 0; /** @var array */ @@ -46,8 +48,14 @@ public function hasOutstandingConfirmations(): bool public function reset(): void { + $this->epoch++; $this->publishedCount = 0; $this->highestConfirmedTag = 0; $this->individuallyConfirmedTags = []; } + + public function getEpoch(): int + { + return $this->epoch; + } } diff --git a/packages/Amqp/src/AmqpOutboundChannelAdapter.php b/packages/Amqp/src/AmqpOutboundChannelAdapter.php index 0f3aac4c1..9d6c0edf5 100644 --- a/packages/Amqp/src/AmqpOutboundChannelAdapter.php +++ b/packages/Amqp/src/AmqpOutboundChannelAdapter.php @@ -117,14 +117,13 @@ private function handleBatch(BatchMessage $batchMessage, Message $carrierMessage */ private function publishBatchThroughSingleWrite(array $messages, AmqpLibContext $context): void { - $libChannel = $context->getLibChannel(); - $anyMessageBatched = false; - + $preparedEntries = []; + $delayedMessages = []; foreach ($messages as $message) { [$interopMessage, $exchangeName, $deliveryDelay] = $this->prepareInteropMessage($message); if ($deliveryDelay) { - $this->publish($message); + $delayedMessages[] = $message; continue; } @@ -134,15 +133,19 @@ private function publishBatchThroughSingleWrite(array $messages, AmqpLibContext $amqpProperties['application_headers'] = new AMQPTable($applicationProperties); } - $libChannel->batch_basic_publish( - new LibAMQPMessage($interopMessage->getBody(), $amqpProperties), - $exchangeName, - $interopMessage->getRoutingKey() ?? '', - ); - $anyMessageBatched = true; + $preparedEntries[] = [new LibAMQPMessage($interopMessage->getBody(), $amqpProperties), $exchangeName, $interopMessage->getRoutingKey() ?? '']; + } + + foreach ($delayedMessages as $delayedMessage) { + $this->publish($delayedMessage); } - if ($anyMessageBatched) { + $libChannel = $context->getLibChannel(); + foreach ($preparedEntries as [$libMessage, $exchangeName, $routingKey]) { + $libChannel->batch_basic_publish($libMessage, $exchangeName, $routingKey, mandatory: $this->publisherConfirms); + } + + if ($preparedEntries !== []) { $libChannel->publish_batch(); } } @@ -153,7 +156,7 @@ private function publish(Message $message): void $this->connectionFactory->getProducer() ->setTimeToLive($timeToLive) - ->setDelayStrategy($this->delayStrategy ?? new HeadersExchangeDelayStrategy()) + ->setDelayStrategy($this->delayStrategy ??= new HeadersExchangeDelayStrategy()) ->setDeliveryDelay($deliveryDelay) // this allow for having queue per delay instead of queue per delay + exchangeName ->send(new AmqpTopic($exchangeName), $messageToSend); @@ -222,10 +225,9 @@ private function prepareInteropMessage(Message $message): array if ($this->publisherConfirms) { Assert::isFalse($this->amqpTransactionInterceptor->isRunningInTransaction(), 'Cannot use publisher acknowledgments together with transactions. Please disable one of them.'); + $messageToSend->addFlag(AmqpMessage::FLAG_MANDATORY); } - $this->connectionFactory->createContext(); - return [$messageToSend, $exchangeName, $outboundMessage->getDeliveryDelay(), $timeToLive]; } @@ -260,18 +262,19 @@ private function awaitPublisherConfirmsSynchronously(): void return; } + $timeoutInSeconds = $this->asyncPublishingTimeout / 1000; $context = $this->connectionFactory->createContext(); if ($context instanceof AmqpLibContext) { - $context->getLibChannel()->wait_for_pending_acks(5); + $context->getLibChannel()->wait_for_pending_acks_returns($timeoutInSeconds); } elseif ($context instanceof AmqpExtContext) { $extPublisherConfirmations = $this->getExtPublisherConfirmations(); if ($extPublisherConfirmations === null) { - $context->getExtChannel()->waitForConfirm(5); + $context->getExtChannel()->waitForConfirm($timeoutInSeconds); return; } - $deadline = microtime(true) + 5; + $deadline = microtime(true) + $timeoutInSeconds; while ($extPublisherConfirmations->hasOutstandingConfirmations()) { $remainingSeconds = $deadline - microtime(true); if ($remainingSeconds <= 0) { diff --git a/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php b/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php index eb82b79c4..d1adac772 100644 --- a/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php +++ b/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php @@ -76,6 +76,7 @@ public function withPublisherConfirms(bool $publisherConfirms): self public function withAsyncPublishing(bool $enabled = true, ?int $timeoutInMilliseconds = null): self { + Assert::isTrue($timeoutInMilliseconds === null || $timeoutInMilliseconds > 0, 'Async publishing timeout must be a positive amount of milliseconds.'); $this->asyncPublishing = $enabled; if ($timeoutInMilliseconds !== null) { $this->asyncPublishingTimeout = $timeoutInMilliseconds; diff --git a/packages/Amqp/src/AmqpPendingDelivery.php b/packages/Amqp/src/AmqpPendingDelivery.php index c6e968329..be7b3b48a 100644 --- a/packages/Amqp/src/AmqpPendingDelivery.php +++ b/packages/Amqp/src/AmqpPendingDelivery.php @@ -21,6 +21,10 @@ final class AmqpPendingDelivery implements PendingDelivery { private bool $awaited = false; + private ?DeliveryResult $deliveryResult = null; + + private int $confirmationsEpoch; + /** * @param Message[] $trackedMessages */ @@ -31,27 +35,36 @@ public function __construct( private string $channelName, private ?AmqpExtPublisherConfirmations $extPublisherConfirmations = null, ) { + $this->confirmationsEpoch = $this->extPublisherConfirmations?->getEpoch() ?? 0; } public function awaitDelivery(): DeliveryResult { + if ($this->deliveryResult !== null) { + return $this->deliveryResult; + } + $this->awaited = true; $timeoutInSeconds = $this->timeoutInMilliseconds / 1000; try { + if ($this->extPublisherConfirmations !== null && $this->extPublisherConfirmations->getEpoch() !== $this->confirmationsEpoch) { + throw new RuntimeException('AMQP connection was reset while awaiting publisher confirms. Delivery confirmation is unknown.'); + } + if ($this->context instanceof AmqpLibContext) { - $this->context->getLibChannel()->wait_for_pending_acks($timeoutInSeconds); + $this->context->getLibChannel()->wait_for_pending_acks_returns($timeoutInSeconds); } elseif ($this->context instanceof AmqpExtContext) { $this->awaitAllExtConfirmations($timeoutInSeconds); } } catch (Throwable $exception) { - return DeliveryResult::withFailedDeliveries(array_map( + return $this->deliveryResult = DeliveryResult::withFailedDeliveries(array_map( fn (Message $message) => new FailedDelivery($message, $exception->getMessage(), $this->channelName), $this->trackedMessages, )); } - return DeliveryResult::successful(); + return $this->deliveryResult = DeliveryResult::successful(); } private function awaitAllExtConfirmations(float $timeoutInSeconds): void @@ -64,6 +77,10 @@ private function awaitAllExtConfirmations(float $timeoutInSeconds): void $deadline = microtime(true) + $timeoutInSeconds; while ($this->extPublisherConfirmations->hasOutstandingConfirmations()) { + if ($this->extPublisherConfirmations->getEpoch() !== $this->confirmationsEpoch) { + throw new RuntimeException('AMQP connection was reset while awaiting publisher confirms. Delivery confirmation is unknown.'); + } + $remainingSeconds = $deadline - microtime(true); if ($remainingSeconds <= 0) { throw new RuntimeException('Timed out awaiting publisher confirms from RabbitMQ instance.'); diff --git a/packages/Amqp/src/AmqpReconnectableConnectionFactory.php b/packages/Amqp/src/AmqpReconnectableConnectionFactory.php index f680328da..47671ed2a 100644 --- a/packages/Amqp/src/AmqpReconnectableConnectionFactory.php +++ b/packages/Amqp/src/AmqpReconnectableConnectionFactory.php @@ -53,10 +53,12 @@ public function createContext(): Context if ($context instanceof AmqpLibContext) { $context->getLibChannel()->confirm_select(); $context->getLibChannel()->set_nack_handler(fn () => throw new RuntimeException('Message was rejected (nack) by RabbitMQ instance. Check RabbitMQ server logs.')); + $context->getLibChannel()->set_return_listener(fn (int $replyCode, string $replyText, string $exchange, string $routingKey) => throw new RuntimeException(sprintf('Message was returned as unroutable by RabbitMQ instance (%d %s) for exchange `%s` and routing key `%s`.', $replyCode, $replyText, $exchange, $routingKey))); } elseif ($context instanceof AmqpExtContext) { $confirmations = $this->getExtPublisherConfirmations(); $confirmations->reset(); $context->getExtChannel()->confirmSelect(); + $context->getExtChannel()->setReturnCallback(fn (int $replyCode, string $replyText, string $exchange, string $routingKey) => throw new RuntimeException(sprintf('Message was returned as unroutable by RabbitMQ instance (%d %s) for exchange `%s` and routing key `%s`.', $replyCode, $replyText, $exchange, $routingKey))); $context->getExtChannel()->setConfirmCallback( function (int $deliveryTag, bool $multiple) use ($confirmations): bool { $confirmations->recordConfirmation($deliveryTag, $multiple); @@ -78,7 +80,7 @@ public function getExtPublisherConfirmations(): AmqpExtPublisherConfirmations public function getConnectionInstanceId(): string { - return get_class($this->connectionFactory) . $this->connectionInstanceId; + return get_class($this->connectionFactory) . $this->connectionInstanceId . ($this->publisherConfirms ? '.confirms' : ''); } /** diff --git a/packages/Amqp/tests/Integration/AmqpMessageChannelTest.php b/packages/Amqp/tests/Integration/AmqpMessageChannelTest.php index 870f8d228..579eba4de 100644 --- a/packages/Amqp/tests/Integration/AmqpMessageChannelTest.php +++ b/packages/Amqp/tests/Integration/AmqpMessageChannelTest.php @@ -229,7 +229,13 @@ public function test_failing_to_receive_message_when_not_declared() /** @var PollableChannel $messageChannel */ $messageChannel = $ecotoneLite->getMessageChannelByName($queueName); - $messageChannel->send(MessageBuilder::withPayload($messagePayload)->build()); + $sendFailed = false; + try { + $messageChannel->send(MessageBuilder::withPayload($messagePayload)->build()); + } catch (Throwable) { + $sendFailed = true; + } + $this->assertTrue($sendFailed); // AMQP Ext throws AMQPException, AMQP Lib throws AMQPProtocolChannelException $this->expectException(Throwable::class); diff --git a/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php index d74ba0543..f9639ca10 100644 --- a/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php +++ b/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php @@ -85,6 +85,38 @@ public function test_ext_publisher_confirmations_handle_multiple_flag_covering_i $this->assertFalse($confirmations->hasOutstandingConfirmations()); } + public function test_unroutable_message_fails_delivery_confirmation_over_amqp_lib(): void + { + $libConnectionFactory = new AmqpLibConnection(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']); + $publisher = $this->bootstrapPublisher($libConnectionFactory, Uuid::v7()->toRfc4122()); + + $this->expectException(AsyncPublishingFailedException::class); + + $publisher->asyncPublish('order that routes nowhere')->resolve(); + } + + public function test_unroutable_message_fails_delivery_confirmation_over_amqp_ext(): void + { + $extConnectionFactory = new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']); + $publisher = $this->bootstrapPublisher($extConnectionFactory, Uuid::v7()->toRfc4122()); + + $this->expectException(AsyncPublishingFailedException::class); + + $publisher->asyncPublish('order that routes nowhere')->resolve(); + } + + public function test_ext_confirmations_reset_while_awaiting_is_detectable_through_epoch(): void + { + $confirmations = new AmqpExtPublisherConfirmations(); + $epochBeforeReset = $confirmations->getEpoch(); + $confirmations->recordPublishedMessage(); + + $confirmations->reset(); + + $this->assertNotSame($epochBeforeReset, $confirmations->getEpoch()); + $this->assertFalse($confirmations->hasOutstandingConfirmations()); + } + private function declareQueueRejectingOverflow(AmqpLibConnection|AmqpConnectionFactory $connectionFactory): string { $queueName = Uuid::v7()->toRfc4122(); diff --git a/packages/Amqp/tests/Integration/AsyncPublishingTest.php b/packages/Amqp/tests/Integration/AsyncPublishingTest.php index dfdd4f2e7..18ce35592 100644 --- a/packages/Amqp/tests/Integration/AsyncPublishingTest.php +++ b/packages/Amqp/tests/Integration/AsyncPublishingTest.php @@ -106,6 +106,9 @@ public function test_async_publish_on_publisher_without_async_configuration_thro public function test_message_publisher_async_publish_confirms_delivery_on_future_resolve(): void { + $queueName = Uuid::v7()->toRfc4122(); + $context = self::getRabbitConnectionFactory()->createContext(); + $context->declareQueue($context->createQueue($queueName)); $messaging = EcotoneLite::bootstrapFlowTesting( [], [...$this->getConnectionFactoryReferences()], @@ -114,7 +117,7 @@ public function test_message_publisher_async_publish_confirms_delivery_on_future ->withExtensionObjects([ AmqpMessagePublisherConfiguration::create() ->withAutoDeclareQueueOnSend(true) - ->withDefaultRoutingKey(Uuid::v7()->toRfc4122()) + ->withDefaultRoutingKey($queueName) ->withAsyncPublishing(), ]), licenceKey: LicenceTesting::VALID_LICENCE, diff --git a/packages/Dbal/src/EnqueueDbal/DbalProducer.php b/packages/Dbal/src/EnqueueDbal/DbalProducer.php index 025b9b032..ad3121d48 100644 --- a/packages/Dbal/src/EnqueueDbal/DbalProducer.php +++ b/packages/Dbal/src/EnqueueDbal/DbalProducer.php @@ -94,13 +94,13 @@ public function sendBatch(Destination $destination, array $messages): void foreach (array_chunk($records, self::BATCH_INSERT_CHUNK_SIZE) as $recordsChunk) { $rowsAffected += $this->insertRecords($recordsChunk); } - - if (count($records) !== $rowsAffected) { - throw new Exception('The batch was not enqueued. Dbal did not confirm that all records are inserted.'); - } } catch (\Exception $e) { throw new Exception('The transport fails to send the message due to some internal error.', 0, $e); } + + if (count($records) !== $rowsAffected) { + throw new Exception('The batch was not enqueued. Dbal did not confirm that all records are inserted.'); + } } private function insertRecords(array $records): int diff --git a/packages/Dbal/tests/Integration/AsyncPublishingTest.php b/packages/Dbal/tests/Integration/AsyncPublishingTest.php index 2f0190655..29a733982 100644 --- a/packages/Dbal/tests/Integration/AsyncPublishingTest.php +++ b/packages/Dbal/tests/Integration/AsyncPublishingTest.php @@ -22,6 +22,7 @@ use Ecotone\Modelling\EventBus; use Ecotone\Test\LicenceTesting; use Enqueue\Dbal\DbalConnectionFactory; +use Interop\Queue\Exception\Exception; use Symfony\Component\Uid\Uuid; use Test\Ecotone\Dbal\DbalMessagingTestCase; use Test\Ecotone\Dbal\Fixture\AsyncPublishing\OrderWasPlaced; @@ -115,6 +116,20 @@ public function test_message_publisher_async_publish_confirms_delivery_on_future $this->assertSame(['first order', 'second order', 'single order'], $receivedPayloads); } + public function test_publishing_after_queue_table_is_dropped_throws(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: true); + $publisher = $messaging->getGateway(MessagePublisher::class); + $publisher->asyncPublish('first order')->resolve(); + + $this->getConnection()->executeStatement('DROP TABLE enqueue'); + + $this->expectException(Exception::class); + + $publisher->asyncPublish('order published into missing table'); + } + private function createOrderService(): object { return new class () { diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php index 21a89f79f..741a880bf 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php @@ -4,6 +4,8 @@ namespace Ecotone\Messaging\Channel\AsyncPublishing; +use Throwable; + /** * licence Enterprise */ @@ -39,6 +41,9 @@ public function closeScope(): void $this->scopeActive = false; foreach ($this->pendingDeliveries as $index => $registration) { if ($registration['scopeOwned']) { + if (! $registration['pendingDelivery']->isAwaited()) { + $this->awaitAndLogFailures($registration['pendingDelivery']); + } unset($this->pendingDeliveries[$index]); } } @@ -95,9 +100,9 @@ public function collectionPoint(): int public function registeredSince(int $collectionPoint): array { $registered = []; - foreach ($this->pendingDeliveries as $index => $registration) { - if ($index >= $collectionPoint) { - $registered[] = $registration['pendingDelivery']; + for ($index = $collectionPoint; $index < $this->nextRegistrationIndex; $index++) { + if (isset($this->pendingDeliveries[$index])) { + $registered[] = $this->pendingDeliveries[$index]['pendingDelivery']; } } @@ -148,7 +153,13 @@ private function flushOldestPublisherOwnedDeliveriesAboveBacklogLimit(): void private function awaitAndLogFailures(PendingDelivery $pendingDelivery): void { - $deliveryResult = $pendingDelivery->awaitDelivery(); + try { + $deliveryResult = $pendingDelivery->awaitDelivery(); + } catch (Throwable $exception) { + error_log(sprintf('Ecotone async publishing: awaiting unresolved delivery failed: %s', $exception->getMessage())); + + return; + } foreach ($deliveryResult->getFailedDeliveries() as $failedDelivery) { error_log(sprintf( 'Ecotone async publishing: unresolved delivery for channel `%s` failed confirmation: %s', diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php index c1787a030..8021ade5a 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php @@ -40,11 +40,12 @@ public function await(MethodInvocation $methodInvocation): mixed $deliveryResult = $this->asyncPublishingRegistry->awaitAll(); if (! $deliveryResult->isSuccessful()) { - $this->handleFailedDeliveries($deliveryResult->getFailedDeliveries()); + $unroutedFailedDeliveries = $this->handleFailedDeliveries($deliveryResult->getFailedDeliveries()); $errorChannelDeliveryResult = $this->asyncPublishingRegistry->awaitAll(); - if (! $errorChannelDeliveryResult->isSuccessful()) { - throw AsyncPublishingFailedException::withFailedDeliveries($errorChannelDeliveryResult->getFailedDeliveries()); + $remainingFailedDeliveries = array_merge($unroutedFailedDeliveries, $errorChannelDeliveryResult->getFailedDeliveries()); + if ($remainingFailedDeliveries !== []) { + throw AsyncPublishingFailedException::withFailedDeliveries($remainingFailedDeliveries); } } } finally { @@ -56,8 +57,9 @@ public function await(MethodInvocation $methodInvocation): mixed /** * @param FailedDelivery[] $failedDeliveries + * @return FailedDelivery[] */ - private function handleFailedDeliveries(array $failedDeliveries): void + private function handleFailedDeliveries(array $failedDeliveries): array { $unroutedFailedDeliveries = []; foreach ($failedDeliveries as $failedDelivery) { @@ -81,9 +83,7 @@ private function handleFailedDeliveries(array $failedDeliveries): void } } - if ($unroutedFailedDeliveries !== []) { - throw AsyncPublishingFailedException::withFailedDeliveries($unroutedFailedDeliveries); - } + return $unroutedFailedDeliveries; } /** diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php index c30631f39..549672992 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php @@ -43,21 +43,25 @@ public function resolve() $this->resolved = true; $failedDeliveries = []; - try { - foreach ($this->pendingDeliveries as $pendingDelivery) { - if ($pendingDelivery->isAwaited()) { - continue; - } - + $awaitFailure = null; + foreach ($this->pendingDeliveries as $pendingDelivery) { + try { $deliveryResult = $pendingDelivery->awaitDelivery(); - if (! $deliveryResult->isSuccessful()) { - $failedDeliveries = array_merge($failedDeliveries, $deliveryResult->getFailedDeliveries()); - } + } catch (Throwable $exception) { + $awaitFailure ??= $exception; + + continue; + } + + if (! $deliveryResult->isSuccessful()) { + $failedDeliveries = array_merge($failedDeliveries, $deliveryResult->getFailedDeliveries()); } - } catch (Throwable $exception) { - $this->failure = $exception instanceof AsyncPublishingFailedException - ? $exception - : new AsyncPublishingFailedException(sprintf('Awaiting delivery confirmation failed: %s', $exception->getMessage()), 0, $exception); + } + + if ($awaitFailure !== null) { + $this->failure = $awaitFailure instanceof AsyncPublishingFailedException && $failedDeliveries === [] + ? $awaitFailure + : new AsyncPublishingFailedException(sprintf('Awaiting delivery confirmation failed: %s', $awaitFailure->getMessage()), 0, $awaitFailure); throw $this->failure; } @@ -67,7 +71,5 @@ public function resolve() throw $this->failure; } - - } } diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php index 52a7280a8..2902accb1 100644 --- a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php @@ -17,6 +17,8 @@ final class InMemoryPendingDelivery implements PendingDelivery { private int $awaitCalls = 0; + private ?DeliveryResult $deliveryResult = null; + public function __construct( private Message $message, private ?string $failureReason = null, @@ -28,6 +30,10 @@ public function __construct( public function awaitDelivery(): DeliveryResult { + if ($this->deliveryResult !== null) { + return $this->deliveryResult; + } + $this->awaitCalls++; $this->operationsLog?->log('delivery confirmations awaited'); @@ -36,12 +42,12 @@ public function awaitDelivery(): DeliveryResult } if ($this->failureReason !== null) { - return DeliveryResult::withFailedDeliveries([ + return $this->deliveryResult = DeliveryResult::withFailedDeliveries([ new FailedDelivery($this->message, $this->failureReason, $this->channelName), ]); } - return DeliveryResult::successful(); + return $this->deliveryResult = DeliveryResult::successful(); } public function isAwaited(): bool diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php index 219479792..ceb53a7af 100644 --- a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php @@ -6,6 +6,7 @@ use Ecotone\Lite\EcotoneLite; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\AsyncPublishing\DeliveryFuture; use Ecotone\Messaging\Channel\MessageChannelInterceptorAdapter; use Ecotone\Messaging\Config\ServiceConfiguration; @@ -83,6 +84,66 @@ public function test_failed_deliveries_routed_to_async_error_channel_are_awaited $this->assertSame(3, $awaitedConfirmationsForFailedBatchAndEachErrorChannelMessage); } + public function test_future_of_delivery_flushed_by_backlog_limit_still_reports_failure(): void + { + $outboundAdapter = new InMemoryAsyncOutboundAdapter(); + $ecotoneLite = EcotoneLite::bootstrapFlowTesting( + [InMemoryAsyncPublisherModule::class, InMemoryAsyncOutboundAdapter::class], + [$outboundAdapter], + ); + $publisher = $ecotoneLite->getGateway(InMemoryAsyncPublisherModule::PUBLISHER_REFERENCE); + $outboundAdapter->failDeliveriesWith('broker down'); + + $firstFuture = $publisher->asyncPublish('first order'); + for ($messageNumber = 0; $messageNumber < 1300; $messageNumber++) { + $publisher->asyncPublish('unresolved order ' . $messageNumber); + } + $this->assertGreaterThan(0, $outboundAdapter->awaitedDeliveriesCount()); + + $this->expectException(AsyncPublishingFailedException::class); + + $firstFuture->resolve(); + } + + public function test_shutdown_flush_continues_when_one_delivery_throws(): void + { + $registry = new AsyncPublishingRegistry(); + $throwingDelivery = new InMemoryPendingDelivery(MessageBuilder::withPayload('first order')->build(), throwOnAwait: true); + $followingDelivery = new InMemoryPendingDelivery(MessageBuilder::withPayload('second order')->build()); + $registry->register('orders', $throwingDelivery); + $registry->register('orders', $followingDelivery); + + $registry->flushUnawaitedDeliveries(); + + $this->assertTrue($followingDelivery->isAwaited()); + } + + public function test_closing_scope_awaits_deliveries_left_unawaited_when_execution_fails_before_await(): void + { + $registry = new AsyncPublishingRegistry(); + $registry->openScope(); + $delivery = new InMemoryPendingDelivery(MessageBuilder::withPayload('order')->build()); + $registry->register('orders', $delivery); + + $registry->closeScope(); + + $this->assertTrue($delivery->isAwaited()); + } + + public function test_future_awaits_remaining_deliveries_when_earlier_delivery_throws(): void + { + $throwingDelivery = new InMemoryPendingDelivery(MessageBuilder::withPayload('first order')->build(), throwOnAwait: true); + $followingDelivery = new InMemoryPendingDelivery(MessageBuilder::withPayload('second order')->build()); + $future = DeliveryFuture::forPendingDeliveries([$throwingDelivery, $followingDelivery]); + + try { + $future->resolve(); + } catch (AsyncPublishingFailedException) { + } + + $this->assertTrue($followingDelivery->isAwaited()); + } + public function test_unresolved_publisher_futures_above_backlog_limit_are_flushed(): void { $outboundAdapter = new InMemoryAsyncOutboundAdapter(); diff --git a/packages/Kafka/src/Configuration/KafkaAdmin.php b/packages/Kafka/src/Configuration/KafkaAdmin.php index 61d8502ad..427379210 100644 --- a/packages/Kafka/src/Configuration/KafkaAdmin.php +++ b/packages/Kafka/src/Configuration/KafkaAdmin.php @@ -127,14 +127,12 @@ public function getProducer(string $referenceName): Producer $conf = $configuration->getAsKafkaConfig(); $conf->set('metadata.broker.list', implode(',', $this->kafkaBrokerConfigurations[$configuration->getBrokerConfigurationReference()]->getBootstrapServers())); $this->setLoggerCallbacks($conf, $referenceName); - if ($configuration->isAsyncPublishingEnabled()) { - $deliveryTracker = $this->getDeliveryTracker($referenceName); - $conf->setDrMsgCb( - function ($producer, $kafkaMessage) use ($deliveryTracker): void { - $deliveryTracker->recordDeliveryReport($kafkaMessage); - } - ); - } + $deliveryTracker = $this->getDeliveryTracker($referenceName); + $conf->setDrMsgCb( + function ($producer, $kafkaMessage) use ($deliveryTracker): void { + $deliveryTracker->recordDeliveryReport($kafkaMessage); + } + ); $producer = new Producer($conf); $producer->addBrokers(implode(',', $this->kafkaBrokerConfigurations[$configuration->getBrokerConfigurationReference()]->getBootstrapServers())); diff --git a/packages/Kafka/src/Outbound/KafkaDeliveryTracker.php b/packages/Kafka/src/Outbound/KafkaDeliveryTracker.php index 90ac2e7c7..a6fc25814 100644 --- a/packages/Kafka/src/Outbound/KafkaDeliveryTracker.php +++ b/packages/Kafka/src/Outbound/KafkaDeliveryTracker.php @@ -49,6 +49,11 @@ public function recordDeliveryReport(KafkaMessage $kafkaMessage): void /** * @param string[] $deliveryIds */ + public function discard(string $deliveryId): void + { + unset($this->inFlightMessages[$deliveryId], $this->deliveryFailures[$deliveryId]); + } + public function collectResult(array $deliveryIds, string $channelName): DeliveryResult { $failedDeliveries = []; diff --git a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php index 2d90f46d1..3752b4849 100644 --- a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php +++ b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php @@ -20,6 +20,7 @@ use Ecotone\Modelling\AggregateMessage; use RdKafka\Producer; use RdKafka\ProducerTopic; +use Throwable; /** * licence Enterprise @@ -49,8 +50,7 @@ public function handle(Message $message): void return; } - $trackDelivery = $this->isAsyncPublishingEnabled(); - $deliveryId = $this->produce($message, $topic, trackDelivery: $trackDelivery); + $deliveryId = $this->produce($message, $topic, trackDelivery: true); $producer->poll(0); if ($this->canPublishAsynchronously()) { @@ -59,7 +59,7 @@ public function handle(Message $message): void return; } - $this->flushSynchronously($producer, $trackDelivery ? [$deliveryId] : []); + $this->flushSynchronously($producer, [$deliveryId]); } public function isAsyncPublishingEnabled(): bool @@ -69,16 +69,22 @@ public function isAsyncPublishingEnabled(): bool private function handleBatch(BatchMessage $batchMessage, Producer $producer, ProducerTopic $topic): void { - $trackDelivery = $this->isAsyncPublishingEnabled(); - $deliveryIds = []; - foreach ($batchMessage->getEntries() as $entry) { - $entryMessage = MessageBuilder::withPayload($entry['payload']) - ->setMultipleHeaders($entry['headers']) - ->build(); + try { + foreach ($batchMessage->getEntries() as $entry) { + $entryMessage = MessageBuilder::withPayload($entry['payload']) + ->setMultipleHeaders($entry['headers']) + ->build(); + + $deliveryIds[] = $this->produce($entryMessage, $topic, trackDelivery: true); + $producer->poll(0); + } + } catch (Throwable $exception) { + if ($deliveryIds !== []) { + $this->registerPendingDelivery($producer, $deliveryIds); + } - $deliveryIds[] = $this->produce($entryMessage, $topic, trackDelivery: $trackDelivery); - $producer->poll(0); + throw $exception; } if ($this->canPublishAsynchronously()) { @@ -87,7 +93,7 @@ private function handleBatch(BatchMessage $batchMessage, Producer $producer, Pro return; } - $this->flushSynchronously($producer, $trackDelivery ? $deliveryIds : []); + $this->flushSynchronously($producer, $deliveryIds); } private function produce(Message $message, ProducerTopic $topic, bool $trackDelivery): ?string @@ -111,6 +117,21 @@ private function produce(Message $message, ProducerTopic $topic, bool $trackDeli ? $this->kafkaAdmin->getDeliveryTracker($this->referenceName)->trackInFlight($message) : null; + try { + $this->produceTracked($topic, $outboundMessage, $partitionKey, $headers, $deliveryId); + } catch (Throwable $exception) { + if ($deliveryId !== null) { + $this->kafkaAdmin->getDeliveryTracker($this->referenceName)->discard($deliveryId); + } + + throw $exception; + } + + return $deliveryId; + } + + private function produceTracked(ProducerTopic $topic, \Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessage $outboundMessage, mixed $partitionKey, array $headers, ?string $deliveryId): void + { $topic->producev( RD_KAFKA_PARTITION_UA, 0, @@ -125,8 +146,6 @@ private function produce(Message $message, ProducerTopic $topic, bool $trackDeli null, $deliveryId, ); - - return $deliveryId; } private function canPublishAsynchronously(): bool @@ -161,15 +180,23 @@ private function flushSynchronously(Producer $producer, array $deliveryIds = []) * calling flush immediately after produce will publish all messages to the broker irrespective of these two config values. */ $result = $producer->flush((int)(KafkaPublisherConfiguration::ACKNOWLEDGE_TIMEOUT * 1.5)); - if ($result !== 0) { - throw MessagePublishingException::create('Failed to send message to Kafka'); - } if ($deliveryIds !== []) { $deliveryResult = $this->kafkaAdmin->getDeliveryTracker($this->referenceName)->collectResult($deliveryIds, $this->referenceName); if (! $deliveryResult->isSuccessful()) { - throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + if ($this->isAsyncPublishingEnabled()) { + throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + } + + throw MessagePublishingException::create(sprintf( + 'Failed to send message to Kafka: %s', + $deliveryResult->getFailedDeliveries()[0]->getFailureReason(), + )); } } + + if ($result !== 0) { + throw MessagePublishingException::create('Failed to send message to Kafka'); + } } } diff --git a/packages/Kafka/src/Outbound/KafkaPendingDelivery.php b/packages/Kafka/src/Outbound/KafkaPendingDelivery.php index d274114eb..3811e2da7 100644 --- a/packages/Kafka/src/Outbound/KafkaPendingDelivery.php +++ b/packages/Kafka/src/Outbound/KafkaPendingDelivery.php @@ -15,6 +15,8 @@ final class KafkaPendingDelivery implements PendingDelivery { private bool $awaited = false; + private ?DeliveryResult $deliveryResult = null; + /** * @param string[] $deliveryIds */ @@ -29,10 +31,14 @@ public function __construct( public function awaitDelivery(): DeliveryResult { + if ($this->deliveryResult !== null) { + return $this->deliveryResult; + } + $this->awaited = true; $this->producer->flush($this->timeoutInMilliseconds); - return $this->deliveryTracker->collectResult($this->deliveryIds, $this->channelName); + return $this->deliveryResult = $this->deliveryTracker->collectResult($this->deliveryIds, $this->channelName); } public function isAwaited(): bool diff --git a/packages/Kafka/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Kafka/tests/Integration/AsyncPublishingReliabilityTest.php index 8c8c8ecb2..b0c8d94d1 100644 --- a/packages/Kafka/tests/Integration/AsyncPublishingReliabilityTest.php +++ b/packages/Kafka/tests/Integration/AsyncPublishingReliabilityTest.php @@ -6,6 +6,7 @@ use Ecotone\Kafka\Configuration\KafkaBrokerConfiguration; use Ecotone\Kafka\Configuration\KafkaPublisherConfiguration; +use Ecotone\Kafka\Outbound\MessagePublishingException; use Ecotone\Lite\EcotoneLite; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; @@ -44,6 +45,26 @@ public function test_broker_rejected_message_fails_async_publish_on_future_resol $future->resolve(); } + public function test_broker_rejected_message_fails_plain_synchronous_publisher(): void + { + $messaging = EcotoneLite::bootstrapFlowTesting( + [], + [KafkaBrokerConfiguration::class => ConnectionTestCase::getConnection()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::KAFKA_PACKAGE])) + ->withExtensionObjects([ + KafkaPublisherConfiguration::createWithDefaults(topicName: Uuid::v7()->toRfc4122()) + ->setConfiguration('message.max.bytes', '4000000'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $this->expectException(MessagePublishingException::class); + + $publisher->send(str_repeat('x', 2_000_000)); + } + private function bootstrapPublisher(): MessagePublisher { $messaging = EcotoneLite::bootstrapFlowTesting( diff --git a/packages/Redis/src/RedisOutboundChannelAdapter.php b/packages/Redis/src/RedisOutboundChannelAdapter.php index dc221d4d5..71dcb48de 100644 --- a/packages/Redis/src/RedisOutboundChannelAdapter.php +++ b/packages/Redis/src/RedisOutboundChannelAdapter.php @@ -109,6 +109,24 @@ protected function handleBatch(BatchMessage $batchMessage, Context $context): vo } } + if (count($immediatePayloads) === 1 && $delayedEntries === []) { + $queueLength = $context->getRedis()->lpush($this->queueName, $immediatePayloads[0]); + if ($queueLength < 1) { + throw new RuntimeException(sprintf('Redis did not confirm publishing message to queue %s.', $this->queueName)); + } + + return; + } + + if ($immediatePayloads === [] && count($delayedEntries) === 1) { + $addedMessages = $context->getRedis()->zadd($this->queueName . ':delayed', $delayedEntries[0]['payload'], $delayedEntries[0]['score']); + if ($addedMessages !== 1) { + throw new RuntimeException(sprintf('Redis did not confirm publishing delayed message to queue %s.', $this->queueName)); + } + + return; + } + $arguments = [count($immediatePayloads), ...$immediatePayloads]; foreach ($delayedEntries as $delayedEntry) { $arguments[] = $delayedEntry['score']; diff --git a/packages/Redis/tests/Integration/AsyncPublishingTest.php b/packages/Redis/tests/Integration/AsyncPublishingTest.php index 280ef6a4b..840480156 100644 --- a/packages/Redis/tests/Integration/AsyncPublishingTest.php +++ b/packages/Redis/tests/Integration/AsyncPublishingTest.php @@ -27,6 +27,7 @@ use Symfony\Component\Uid\Uuid; use Test\Ecotone\Redis\ConnectionTestCase; use Test\Ecotone\Redis\Fixture\AsyncPublishing\OrderWasPlaced; +use Throwable; /** * licence Apache-2.0 @@ -148,6 +149,55 @@ public function test_delayed_entry_of_published_batch_lands_in_delayed_set(): vo $this->assertSame(1, $context->getRedis()->eval('return redis.call("zcard", KEYS[1])', [$queueName . ':delayed'])); } + public function test_publishing_to_wrong_type_key_throws(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: true); + /** @var RedisContext $context */ + $context = $this->getConnectionFactory()->createContext(); + $context->getRedis()->eval('redis.call("set", KEYS[1], "blocked") return 1', [$queueName]); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $publishFailed = false; + try { + $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('first order') + ->append('second order') + ); + } catch (Throwable) { + $publishFailed = true; + } + + $this->assertTrue($publishFailed); + } + + public function test_partially_applied_mixed_batch_throws_while_immediate_entries_remain(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: true); + /** @var RedisContext $context */ + $context = $this->getConnectionFactory()->createContext(); + $context->getRedis()->lpush($queueName . ':delayed', 'poison entry forcing wrong type on the delayed set'); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $publishFailed = false; + try { + $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('immediate order') + ->append('delayed order', [MessageHeaders::DELIVERY_DELAY => 60000]) + ); + } catch (Throwable) { + $publishFailed = true; + } + + $this->assertTrue($publishFailed); + + $context->getRedis()->del($queueName . ':delayed'); + $this->assertSame('immediate order', $messaging->getMessageChannel($queueName)->receive()->getPayload()); + } + private function createOrderService(): object { return new class () { diff --git a/packages/Sqs/src/SqsPendingDelivery.php b/packages/Sqs/src/SqsPendingDelivery.php index c4beab5c0..39edf65a5 100644 --- a/packages/Sqs/src/SqsPendingDelivery.php +++ b/packages/Sqs/src/SqsPendingDelivery.php @@ -12,6 +12,7 @@ use Ecotone\Messaging\Message; use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; +use GuzzleHttp\Promise\RejectedPromise; use Throwable; /** @@ -23,6 +24,8 @@ final class SqsPendingDelivery implements PendingDelivery private bool $awaited = false; + private ?DeliveryResult $deliveryResult = null; + /** * @param Closure[] $sendRequestDispatchers each returns a PromiseInterface when invoked, so requests are dispatched lazily with bounded concurrency * @param array> $trackedMessagesPerRequest keyed by request index, then by batch entry id @@ -37,9 +40,12 @@ public function __construct( public function awaitDelivery(): DeliveryResult { - $this->awaited = true; + if ($this->deliveryResult !== null) { + return $this->deliveryResult; + } $settledResults = $this->dispatchWithBoundedConcurrency(); + $this->awaited = true; $failedDeliveries = []; foreach ($this->trackedMessagesPerRequest as $requestIndex => $trackedMessages) { @@ -87,9 +93,9 @@ public function awaitDelivery(): DeliveryResult } } - return $failedDeliveries === [] + return $this->deliveryResult = ($failedDeliveries === [] ? DeliveryResult::successful() - : DeliveryResult::withFailedDeliveries($failedDeliveries); + : DeliveryResult::withFailedDeliveries($failedDeliveries)); } public function isAwaited(): bool @@ -105,7 +111,13 @@ private function dispatchWithBoundedConcurrency(): array $settledResults = []; $sendRequestPromises = (function () use (&$settledResults) { foreach ($this->sendRequestDispatchers as $requestIndex => $dispatchSendRequest) { - yield $dispatchSendRequest()->then( + try { + $sendRequestPromise = $dispatchSendRequest(); + } catch (Throwable $dispatchFailure) { + $sendRequestPromise = new RejectedPromise($dispatchFailure); + } + + yield $sendRequestPromise->then( function (mixed $value) use (&$settledResults, $requestIndex): void { $settledResults[$requestIndex] = ['state' => PromiseInterface::FULFILLED, 'value' => $value]; }, diff --git a/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php b/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php index d994f0195..5fecd18b6 100644 --- a/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php +++ b/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php @@ -115,6 +115,53 @@ public function test_send_requests_are_dispatched_lazily_on_await_not_on_creatio $this->assertSame(1, $dispatchedRequests); } + public function test_synchronously_throwing_dispatcher_marks_only_its_messages_failed_and_dispatches_remaining_requests(): void + { + $secondDispatcherInvocations = 0; + $pendingDelivery = new SqsPendingDelivery( + [ + fn () => throw new RuntimeException('curl init failed'), + function () use (&$secondDispatcherInvocations) { + $secondDispatcherInvocations++; + + return new FulfilledPromise(new Result(['Successful' => [['Id' => '0', 'MessageId' => 'second-id']]])); + }, + ], + [ + ['0' => MessageBuilder::withPayload('first order')->build()], + ['0' => MessageBuilder::withPayload('second order')->build()], + ], + 'orders', + ); + + $deliveryResult = $pendingDelivery->awaitDelivery(); + + $this->assertSame(1, $secondDispatcherInvocations); + $this->assertCount(1, $deliveryResult->getFailedDeliveries()); + $this->assertSame('first order', $deliveryResult->getFailedDeliveries()[0]->getMessage()->getPayload()); + $this->assertStringContainsString('curl init failed', $deliveryResult->getFailedDeliveries()[0]->getFailureReason()); + } + + public function test_second_await_returns_memoized_result_without_redispatching_requests(): void + { + $dispatchedRequests = 0; + $pendingDelivery = new SqsPendingDelivery( + [function () use (&$dispatchedRequests) { + $dispatchedRequests++; + + return new FulfilledPromise(new Result(['Successful' => [['Id' => '0', 'MessageId' => 'aws-message-id']]])); + }], + [['0' => MessageBuilder::withPayload('order')->build()]], + 'orders', + ); + + $firstResult = $pendingDelivery->awaitDelivery(); + $secondResult = $pendingDelivery->awaitDelivery(); + + $this->assertSame(1, $dispatchedRequests); + $this->assertSame($firstResult, $secondResult); + } + public function test_all_requests_are_dispatched_even_when_earlier_request_is_rejected(): void { $dispatchedRequests = 0; From 83cb096eefbfef9bcce22780f37bd8791b524f06 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Mon, 13 Jul 2026 08:00:00 +0200 Subject: [PATCH 23/38] fix: declare benchmark queue so AMQP publisher subjects measure real delivery --- Monorepo/Benchmark/AsyncPublishingBenchmark.php | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Monorepo/Benchmark/AsyncPublishingBenchmark.php b/Monorepo/Benchmark/AsyncPublishingBenchmark.php index 1d8b31601..63b30f538 100644 --- a/Monorepo/Benchmark/AsyncPublishingBenchmark.php +++ b/Monorepo/Benchmark/AsyncPublishingBenchmark.php @@ -316,9 +316,14 @@ private function bootstrapBatchChannel(string $modulePackage, object $channelBui private function bootstrapAmqpPublisher(bool $asyncPublishing): MessagePublisher { + $queueName = uniqid('benchmark_orders_'); + $connectionFactory = new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']); + $context = $connectionFactory->createContext(); + $context->declareQueue($context->createQueue($queueName)); + $publisherConfiguration = AmqpMessagePublisherConfiguration::create() ->withAutoDeclareQueueOnSend(true) - ->withDefaultRoutingKey(uniqid('benchmark_orders_')); + ->withDefaultRoutingKey($queueName); if ($asyncPublishing) { $publisherConfiguration = $publisherConfiguration->withAsyncPublishing(); } @@ -326,7 +331,7 @@ private function bootstrapAmqpPublisher(bool $asyncPublishing): MessagePublisher $messaging = EcotoneLite::bootstrapFlowTesting( [], [ - AmqpConnectionFactory::class => new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']), + AmqpConnectionFactory::class => $connectionFactory, ], ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::AMQP_PACKAGE])) From e475ea8e06bf96948ed3d21af6169f8a6814f39f Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Mon, 13 Jul 2026 08:00:00 +0200 Subject: [PATCH 24/38] feat: benchmark firing multiple batches before awaiting confirmations --- .../Benchmark/AsyncPublishingBenchmark.php | 71 +++++++++++++++++-- 1 file changed, 66 insertions(+), 5 deletions(-) diff --git a/Monorepo/Benchmark/AsyncPublishingBenchmark.php b/Monorepo/Benchmark/AsyncPublishingBenchmark.php index 63b30f538..63a00ca13 100644 --- a/Monorepo/Benchmark/AsyncPublishingBenchmark.php +++ b/Monorepo/Benchmark/AsyncPublishingBenchmark.php @@ -35,7 +35,8 @@ /** * Compares publishing scenarios per provider: - * single message synchronous | single message asynchronous | batch message synchronous | batch message asynchronous. + * single message synchronous | single message asynchronous | batch message synchronous | batch message asynchronous + * | multiple batches synchronous | multiple batches asynchronous (fire all batches first, await all confirmations once). * * DBAL and Redis confirm deliveries synchronously as part of the send call itself (database statement result * and command reply respectively), so the asynchronous scenarios are not supported for them and are not benchmarked. @@ -45,6 +46,10 @@ class AsyncPublishingBenchmark { private const AMOUNT_OF_PUBLISHED_MESSAGES = 1000; + private const AMOUNT_OF_BATCHES = 10; + + private const MESSAGES_PER_BATCH = 100; + private const MESSAGE_PAYLOAD = 'benchmark order payload for async publishing comparison'; private MessagePublisher $publisher; @@ -174,6 +179,18 @@ public function bench_amqp_batch_message_asynchronous(): void $this->publishBatchAsynchronously(); } + #[BeforeMethods('setUpAmqpBatchChannel')] + public function bench_amqp_multiple_batches_synchronous(): void + { + $this->publishMultipleBatchesSynchronously(); + } + + #[BeforeMethods('setUpAmqpAsyncPublishing')] + public function bench_amqp_multiple_batches_asynchronous(): void + { + $this->publishMultipleBatchesAsynchronously(); + } + #[BeforeMethods('setUpKafkaSynchronousPublishing')] public function bench_kafka_single_message_synchronous(): void { @@ -198,6 +215,18 @@ public function bench_kafka_batch_message_asynchronous(): void $this->publishBatchAsynchronously(); } + #[BeforeMethods('setUpKafkaBatchChannel')] + public function bench_kafka_multiple_batches_synchronous(): void + { + $this->publishMultipleBatchesSynchronously(); + } + + #[BeforeMethods('setUpKafkaAsyncPublishing')] + public function bench_kafka_multiple_batches_asynchronous(): void + { + $this->publishMultipleBatchesAsynchronously(); + } + #[BeforeMethods('setUpDbalSynchronousPublishing')] public function bench_dbal_single_message_synchronous(): void { @@ -246,6 +275,18 @@ public function bench_sqs_batch_message_asynchronous(): void $this->publishBatchAsynchronously(); } + #[BeforeMethods('setUpSqsBatchChannel')] + public function bench_sqs_multiple_batches_synchronous(): void + { + $this->publishMultipleBatchesSynchronously(); + } + + #[BeforeMethods('setUpSqsAsyncPublishing')] + public function bench_sqs_multiple_batches_asynchronous(): void + { + $this->publishMultipleBatchesAsynchronously(); + } + private function publishSynchronouslyOneByOne(): void { for ($messageNumber = 0; $messageNumber < self::AMOUNT_OF_PUBLISHED_MESSAGES; $messageNumber++) { @@ -267,19 +308,39 @@ private function publishAsynchronouslyOneByOne(): void private function publishBatchSynchronously(): void { $this->batchChannel->send( - MessageBuilder::withPayload($this->buildBatch())->build() + MessageBuilder::withPayload($this->buildBatch(self::AMOUNT_OF_PUBLISHED_MESSAGES))->build() ); } private function publishBatchAsynchronously(): void { - $this->publisher->asyncPublish($this->buildBatch(), MediaType::TEXT_PLAIN)->resolve(); + $this->publisher->asyncPublish($this->buildBatch(self::AMOUNT_OF_PUBLISHED_MESSAGES), MediaType::TEXT_PLAIN)->resolve(); + } + + private function publishMultipleBatchesSynchronously(): void + { + for ($batchNumber = 0; $batchNumber < self::AMOUNT_OF_BATCHES; $batchNumber++) { + $this->batchChannel->send( + MessageBuilder::withPayload($this->buildBatch(self::MESSAGES_PER_BATCH))->build() + ); + } } - private function buildBatch(): BatchMessage + private function publishMultipleBatchesAsynchronously(): void + { + $futures = []; + for ($batchNumber = 0; $batchNumber < self::AMOUNT_OF_BATCHES; $batchNumber++) { + $futures[] = $this->publisher->asyncPublish($this->buildBatch(self::MESSAGES_PER_BATCH), MediaType::TEXT_PLAIN); + } + foreach ($futures as $future) { + $future->resolve(); + } + } + + private function buildBatch(int $amountOfMessages): BatchMessage { $batch = BatchMessage::constructEmpty(); - for ($messageNumber = 0; $messageNumber < self::AMOUNT_OF_PUBLISHED_MESSAGES; $messageNumber++) { + for ($messageNumber = 0; $messageNumber < $amountOfMessages; $messageNumber++) { $batch = $batch->append(self::MESSAGE_PAYLOAD, ['contentType' => MediaType::TEXT_PLAIN]); } From 63657b1d6166c182d7e9e45e4fc6a35561e5e528 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Mon, 13 Jul 2026 08:00:00 +0200 Subject: [PATCH 25/38] perf: shared SQS dispatch pool for bounded cross-delivery request concurrency --- .../Sqs/src/SqsOutboundChannelAdapter.php | 10 +- packages/Sqs/src/SqsPendingDelivery.php | 43 +------ packages/Sqs/src/SqsRequestDispatchPool.php | 105 +++++++++++++++++ .../Sqs/tests/Unit/SqsPendingDeliveryTest.php | 97 +--------------- .../tests/Unit/SqsRequestDispatchPoolTest.php | 107 ++++++++++++++++++ 5 files changed, 229 insertions(+), 133 deletions(-) create mode 100644 packages/Sqs/src/SqsRequestDispatchPool.php create mode 100644 packages/Sqs/tests/Unit/SqsRequestDispatchPoolTest.php diff --git a/packages/Sqs/src/SqsOutboundChannelAdapter.php b/packages/Sqs/src/SqsOutboundChannelAdapter.php index 1aba33eb9..232835dc4 100644 --- a/packages/Sqs/src/SqsOutboundChannelAdapter.php +++ b/packages/Sqs/src/SqsOutboundChannelAdapter.php @@ -25,6 +25,8 @@ final class SqsOutboundChannelAdapter extends EnqueueOutboundChannelAdapter private const MAX_ENTRIES_PER_BATCH_REQUEST = 10; private const BATCH_REQUEST_PAYLOAD_BUDGET_IN_BYTES = 204800; + private SqsRequestDispatchPool $requestDispatchPool; + public function __construct( CachedConnectionFactory $connectionFactory, private string $queueName, @@ -35,6 +37,7 @@ public function __construct( private bool $asyncPublishing = false, private ?int $asyncPublishingTimeout = null, ) { + $this->requestDispatchPool = new SqsRequestDispatchPool(); parent::__construct( $connectionFactory, new SqsDestination($queueName), @@ -69,15 +72,16 @@ public function handle(Message $message): void return; } - $sendRequestDispatchers = []; + $awsSqsClient = $context->getAwsSqsClient(); + $sendRequestPromises = []; $trackedMessagesPerRequest = []; foreach ($this->buildBatchRequests($messagesToPublish, $context) as $batchRequest) { $requestArguments = $batchRequest['arguments']; - $sendRequestDispatchers[] = fn () => $context->getAwsSqsClient()->sendMessageBatchAsync($requestArguments); + $sendRequestPromises[] = $this->requestDispatchPool->dispatch(fn () => $awsSqsClient->sendMessageBatchAsync($requestArguments)); $trackedMessagesPerRequest[] = $batchRequest['trackedMessages']; } - $pendingDelivery = new SqsPendingDelivery($sendRequestDispatchers, $trackedMessagesPerRequest, $this->queueName); + $pendingDelivery = new SqsPendingDelivery($sendRequestPromises, $trackedMessagesPerRequest, $this->queueName); if ($this->asyncPublishing && $this->asyncPublishingRegistry->isScopeActive()) { $this->asyncPublishingRegistry->register($this->queueName, $pendingDelivery); diff --git a/packages/Sqs/src/SqsPendingDelivery.php b/packages/Sqs/src/SqsPendingDelivery.php index 39edf65a5..5d1e3ca6e 100644 --- a/packages/Sqs/src/SqsPendingDelivery.php +++ b/packages/Sqs/src/SqsPendingDelivery.php @@ -5,14 +5,12 @@ namespace Ecotone\Sqs; use Aws\Result; -use Closure; use Ecotone\Messaging\Channel\AsyncPublishing\DeliveryResult; use Ecotone\Messaging\Channel\AsyncPublishing\FailedDelivery; use Ecotone\Messaging\Channel\AsyncPublishing\PendingDelivery; use Ecotone\Messaging\Message; -use GuzzleHttp\Promise\Each; use GuzzleHttp\Promise\PromiseInterface; -use GuzzleHttp\Promise\RejectedPromise; +use GuzzleHttp\Promise\Utils; use Throwable; /** @@ -20,21 +18,18 @@ */ final class SqsPendingDelivery implements PendingDelivery { - public const DEFAULT_MAX_CONCURRENT_REQUESTS = 25; - private bool $awaited = false; private ?DeliveryResult $deliveryResult = null; /** - * @param Closure[] $sendRequestDispatchers each returns a PromiseInterface when invoked, so requests are dispatched lazily with bounded concurrency + * @param PromiseInterface[] $sendRequestPromises keyed by request index, dispatched with bounded concurrency through SqsRequestDispatchPool * @param array> $trackedMessagesPerRequest keyed by request index, then by batch entry id */ public function __construct( - private array $sendRequestDispatchers, + private array $sendRequestPromises, private array $trackedMessagesPerRequest, private string $channelName, - private int $maxConcurrentRequests = self::DEFAULT_MAX_CONCURRENT_REQUESTS, ) { } @@ -44,7 +39,7 @@ public function awaitDelivery(): DeliveryResult return $this->deliveryResult; } - $settledResults = $this->dispatchWithBoundedConcurrency(); + $settledResults = Utils::settle($this->sendRequestPromises)->wait(); $this->awaited = true; $failedDeliveries = []; @@ -102,34 +97,4 @@ public function isAwaited(): bool { return $this->awaited; } - - /** - * @return array - */ - private function dispatchWithBoundedConcurrency(): array - { - $settledResults = []; - $sendRequestPromises = (function () use (&$settledResults) { - foreach ($this->sendRequestDispatchers as $requestIndex => $dispatchSendRequest) { - try { - $sendRequestPromise = $dispatchSendRequest(); - } catch (Throwable $dispatchFailure) { - $sendRequestPromise = new RejectedPromise($dispatchFailure); - } - - yield $sendRequestPromise->then( - function (mixed $value) use (&$settledResults, $requestIndex): void { - $settledResults[$requestIndex] = ['state' => PromiseInterface::FULFILLED, 'value' => $value]; - }, - function (mixed $reason) use (&$settledResults, $requestIndex): void { - $settledResults[$requestIndex] = ['state' => PromiseInterface::REJECTED, 'reason' => $reason]; - }, - ); - } - })(); - - Each::ofLimit($sendRequestPromises, $this->maxConcurrentRequests)->wait(); - - return $settledResults; - } } diff --git a/packages/Sqs/src/SqsRequestDispatchPool.php b/packages/Sqs/src/SqsRequestDispatchPool.php new file mode 100644 index 000000000..aa0c78299 --- /dev/null +++ b/packages/Sqs/src/SqsRequestDispatchPool.php @@ -0,0 +1,105 @@ + */ + private array $trackedRequests = []; + + /** @var array */ + private array $awaitingDispatch = []; + + /** @var array */ + private array $inFlightRequests = []; + + private int $nextRequestIndex = 0; + + public function __construct(private int $maxConcurrentRequests = self::DEFAULT_MAX_CONCURRENT_REQUESTS) + { + } + + public function dispatch(Closure $dispatchSendRequest): PromiseInterface + { + $requestIndex = $this->nextRequestIndex++; + $proxy = new Promise(fn () => $this->driveUntilSettled($requestIndex)); + $this->trackedRequests[$requestIndex] = ['proxy' => $proxy, 'underlying' => null]; + $this->awaitingDispatch[$requestIndex] = $dispatchSendRequest; + $this->dispatchWithinBudget(); + + return $proxy; + } + + private function dispatchWithinBudget(): void + { + while (count($this->inFlightRequests) < $this->maxConcurrentRequests && $this->awaitingDispatch !== []) { + $requestIndex = array_key_first($this->awaitingDispatch); + $dispatchSendRequest = $this->awaitingDispatch[$requestIndex]; + unset($this->awaitingDispatch[$requestIndex]); + + try { + $underlying = $dispatchSendRequest(); + } catch (Throwable $dispatchFailure) { + $proxy = $this->trackedRequests[$requestIndex]['proxy']; + unset($this->trackedRequests[$requestIndex]); + $proxy->reject($dispatchFailure); + + continue; + } + + $this->trackedRequests[$requestIndex]['underlying'] = $underlying; + $this->inFlightRequests[$requestIndex] = $underlying; + + $underlying->then( + function (mixed $value) use ($requestIndex): void { + $this->settleProxy($requestIndex, fn (Promise $proxy) => $proxy->resolve($value)); + }, + function (mixed $reason) use ($requestIndex): void { + $this->settleProxy($requestIndex, fn (Promise $proxy) => $proxy->reject($reason)); + }, + ); + } + } + + private function settleProxy(int $requestIndex, Closure $settle): void + { + unset($this->inFlightRequests[$requestIndex]); + $this->dispatchWithinBudget(); + + $proxy = $this->trackedRequests[$requestIndex]['proxy']; + unset($this->trackedRequests[$requestIndex]); + $settle($proxy); + } + + private function driveUntilSettled(int $requestIndex): void + { + while (isset($this->trackedRequests[$requestIndex]) && $this->trackedRequests[$requestIndex]['proxy']->getState() === PromiseInterface::PENDING) { + $underlying = $this->trackedRequests[$requestIndex]['underlying']; + if ($underlying !== null) { + $underlying->wait(false); + + continue; + } + + if ($this->inFlightRequests !== []) { + $this->inFlightRequests[array_key_first($this->inFlightRequests)]->wait(false); + + continue; + } + + $this->dispatchWithinBudget(); + } + } +} diff --git a/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php b/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php index 5fecd18b6..2b2a9f095 100644 --- a/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php +++ b/packages/Sqs/tests/Unit/SqsPendingDeliveryTest.php @@ -23,7 +23,7 @@ public function test_partially_failed_batch_reports_failed_entries_mapped_to_ori $deliveredMessage = MessageBuilder::withPayload('delivered order')->build(); $rejectedMessage = MessageBuilder::withPayload('rejected order')->build(); $pendingDelivery = new SqsPendingDelivery( - [fn () => new FulfilledPromise(new Result([ + [new FulfilledPromise(new Result([ 'Successful' => [['Id' => '0', 'MessageId' => 'aws-message-id']], 'Failed' => [['Id' => '1', 'Code' => 'InternalError', 'Message' => 'server hiccup', 'SenderFault' => false]], ]))], @@ -46,7 +46,7 @@ public function test_rejected_request_reports_all_messages_of_that_request_as_fa $firstMessage = MessageBuilder::withPayload('first order')->build(); $secondMessage = MessageBuilder::withPayload('second order')->build(); $pendingDelivery = new SqsPendingDelivery( - [fn () => new RejectedPromise(new RuntimeException('connection refused'))], + [new RejectedPromise(new RuntimeException('connection refused'))], [['0' => $firstMessage, '1' => $secondMessage]], 'orders', ); @@ -63,7 +63,7 @@ public function test_entries_missing_from_successful_and_failed_lists_are_report $confirmedMessage = MessageBuilder::withPayload('confirmed order')->build(); $unaccountedMessage = MessageBuilder::withPayload('unaccounted order')->build(); $pendingDelivery = new SqsPendingDelivery( - [fn () => new FulfilledPromise(new Result([ + [new FulfilledPromise(new Result([ 'Successful' => [['Id' => '0', 'MessageId' => 'aws-message-id']], ]))], [['0' => $confirmedMessage, '1' => $unaccountedMessage]], @@ -80,7 +80,7 @@ public function test_entries_missing_from_successful_and_failed_lists_are_report public function test_fully_confirmed_batch_reports_success_and_is_marked_as_awaited(): void { $pendingDelivery = new SqsPendingDelivery( - [fn () => new FulfilledPromise(new Result([ + [new FulfilledPromise(new Result([ 'Successful' => [['Id' => '0', 'MessageId' => 'first-id'], ['Id' => '1', 'MessageId' => 'second-id']], ]))], [['0' => MessageBuilder::withPayload('first order')->build(), '1' => MessageBuilder::withPayload('second order')->build()]], @@ -95,62 +95,10 @@ public function test_fully_confirmed_batch_reports_success_and_is_marked_as_awai $this->assertTrue($pendingDelivery->isAwaited()); } - public function test_send_requests_are_dispatched_lazily_on_await_not_on_creation(): void + public function test_second_await_returns_memoized_result(): void { - $dispatchedRequests = 0; $pendingDelivery = new SqsPendingDelivery( - [function () use (&$dispatchedRequests) { - $dispatchedRequests++; - - return new FulfilledPromise(new Result(['Successful' => [['Id' => '0', 'MessageId' => 'aws-message-id']]])); - }], - [['0' => MessageBuilder::withPayload('order')->build()]], - 'orders', - ); - - $this->assertSame(0, $dispatchedRequests); - - $pendingDelivery->awaitDelivery(); - - $this->assertSame(1, $dispatchedRequests); - } - - public function test_synchronously_throwing_dispatcher_marks_only_its_messages_failed_and_dispatches_remaining_requests(): void - { - $secondDispatcherInvocations = 0; - $pendingDelivery = new SqsPendingDelivery( - [ - fn () => throw new RuntimeException('curl init failed'), - function () use (&$secondDispatcherInvocations) { - $secondDispatcherInvocations++; - - return new FulfilledPromise(new Result(['Successful' => [['Id' => '0', 'MessageId' => 'second-id']]])); - }, - ], - [ - ['0' => MessageBuilder::withPayload('first order')->build()], - ['0' => MessageBuilder::withPayload('second order')->build()], - ], - 'orders', - ); - - $deliveryResult = $pendingDelivery->awaitDelivery(); - - $this->assertSame(1, $secondDispatcherInvocations); - $this->assertCount(1, $deliveryResult->getFailedDeliveries()); - $this->assertSame('first order', $deliveryResult->getFailedDeliveries()[0]->getMessage()->getPayload()); - $this->assertStringContainsString('curl init failed', $deliveryResult->getFailedDeliveries()[0]->getFailureReason()); - } - - public function test_second_await_returns_memoized_result_without_redispatching_requests(): void - { - $dispatchedRequests = 0; - $pendingDelivery = new SqsPendingDelivery( - [function () use (&$dispatchedRequests) { - $dispatchedRequests++; - - return new FulfilledPromise(new Result(['Successful' => [['Id' => '0', 'MessageId' => 'aws-message-id']]])); - }], + [new FulfilledPromise(new Result(['Successful' => [['Id' => '0', 'MessageId' => 'aws-message-id']]]))], [['0' => MessageBuilder::withPayload('order')->build()]], 'orders', ); @@ -158,39 +106,6 @@ public function test_second_await_returns_memoized_result_without_redispatching_ $firstResult = $pendingDelivery->awaitDelivery(); $secondResult = $pendingDelivery->awaitDelivery(); - $this->assertSame(1, $dispatchedRequests); $this->assertSame($firstResult, $secondResult); } - - public function test_all_requests_are_dispatched_even_when_earlier_request_is_rejected(): void - { - $dispatchedRequests = 0; - $countingDispatcher = function ($promise) use (&$dispatchedRequests) { - return function () use ($promise, &$dispatchedRequests) { - $dispatchedRequests++; - - return $promise; - }; - }; - $pendingDelivery = new SqsPendingDelivery( - [ - $countingDispatcher(new RejectedPromise(new RuntimeException('connection refused'))), - $countingDispatcher(new FulfilledPromise(new Result(['Successful' => [['Id' => '0', 'MessageId' => 'second-id']]]))), - $countingDispatcher(new FulfilledPromise(new Result(['Successful' => [['Id' => '0', 'MessageId' => 'third-id']]]))), - ], - [ - ['0' => MessageBuilder::withPayload('first order')->build()], - ['0' => MessageBuilder::withPayload('second order')->build()], - ['0' => MessageBuilder::withPayload('third order')->build()], - ], - 'orders', - maxConcurrentRequests: 1, - ); - - $deliveryResult = $pendingDelivery->awaitDelivery(); - - $this->assertSame(3, $dispatchedRequests); - $this->assertCount(1, $deliveryResult->getFailedDeliveries()); - $this->assertSame('first order', $deliveryResult->getFailedDeliveries()[0]->getMessage()->getPayload()); - } } diff --git a/packages/Sqs/tests/Unit/SqsRequestDispatchPoolTest.php b/packages/Sqs/tests/Unit/SqsRequestDispatchPoolTest.php new file mode 100644 index 000000000..39b57f90b --- /dev/null +++ b/packages/Sqs/tests/Unit/SqsRequestDispatchPoolTest.php @@ -0,0 +1,107 @@ +dispatch($dispatcher); + $pool->dispatch($dispatcher); + $pool->dispatch($dispatcher); + + $this->assertSame(2, $dispatchedRequests); + } + + public function test_queued_request_is_dispatched_when_earlier_request_settles(): void + { + $pool = new SqsRequestDispatchPool(maxConcurrentRequests: 1); + $firstUnderlying = new Promise(); + $dispatchedSecond = false; + + $pool->dispatch(fn () => $firstUnderlying); + $pool->dispatch(function () use (&$dispatchedSecond) { + $dispatchedSecond = true; + + return new Promise(); + }); + + $this->assertFalse($dispatchedSecond); + + $firstUnderlying->resolve('confirmed'); + \GuzzleHttp\Promise\Utils::queue()->run(); + + $this->assertTrue($dispatchedSecond); + } + + public function test_proxy_resolves_with_underlying_value_and_rejects_with_underlying_reason(): void + { + $pool = new SqsRequestDispatchPool(maxConcurrentRequests: 2); + $fulfilledUnderlying = new Promise(); + $rejectedUnderlying = new Promise(); + + $fulfilledProxy = $pool->dispatch(fn () => $fulfilledUnderlying); + $rejectedProxy = $pool->dispatch(fn () => $rejectedUnderlying); + + $fulfilledUnderlying->resolve('confirmed'); + $rejectedUnderlying->reject(new RuntimeException('connection refused')); + \GuzzleHttp\Promise\Utils::queue()->run(); + + $this->assertSame(PromiseInterface::FULFILLED, $fulfilledProxy->getState()); + $this->assertSame('confirmed', $fulfilledProxy->wait()); + $this->assertSame(PromiseInterface::REJECTED, $rejectedProxy->getState()); + } + + public function test_synchronously_throwing_dispatcher_rejects_only_its_proxy_and_frees_the_slot(): void + { + $pool = new SqsRequestDispatchPool(maxConcurrentRequests: 1); + $dispatchedSecond = false; + + $throwingProxy = $pool->dispatch(fn () => throw new RuntimeException('curl init failed')); + $pool->dispatch(function () use (&$dispatchedSecond) { + $dispatchedSecond = true; + + return new Promise(); + }); + + $this->assertSame(PromiseInterface::REJECTED, $throwingProxy->getState()); + $this->assertTrue($dispatchedSecond); + } + + public function test_waiting_on_queued_proxy_drives_earlier_requests_until_slot_frees(): void + { + $pool = new SqsRequestDispatchPool(maxConcurrentRequests: 1); + $firstUnderlying = new Promise(function () use (&$firstUnderlying) { + $firstUnderlying->resolve('first confirmed'); + }); + $secondUnderlying = new Promise(function () use (&$secondUnderlying) { + $secondUnderlying->resolve('second confirmed'); + }); + + $pool->dispatch(fn () => $firstUnderlying); + $queuedProxy = $pool->dispatch(fn () => $secondUnderlying); + + $this->assertSame('second confirmed', $queuedProxy->wait()); + $this->assertSame(PromiseInterface::FULFILLED, $firstUnderlying->getState()); + } +} From 0c6b40c194a09336bf4cb962cae898050644d36c Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Mon, 13 Jul 2026 08:00:00 +0200 Subject: [PATCH 26/38] perf: kafka async publishing batching via linger, poll cadence and queue-full backpressure --- .../KafkaPublisherConfiguration.php | 7 +++++- .../Outbound/KafkaOutboundChannelAdapter.php | 23 +++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php b/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php index 104c84a9b..016bfe080 100644 --- a/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php +++ b/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php @@ -137,8 +137,13 @@ public function getBrokerConfigurationReference(): string public function getAsKafkaConfig(): Conf { + $configuration = $this->configuration; + if ($this->asyncPublishing && ! isset($configuration['linger.ms']) && ! isset($configuration['queue.buffering.max.ms'])) { + $configuration['linger.ms'] = '20'; + } + $conf = new Conf(); - foreach ($this->configuration as $key => $value) { + foreach ($configuration as $key => $value) { $conf->set($key, $value); } diff --git a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php index 3752b4849..20db2f89c 100644 --- a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php +++ b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php @@ -27,6 +27,8 @@ */ final class KafkaOutboundChannelAdapter implements MessageHandler { + private const POLL_EVERY_PRODUCED_MESSAGES = 100; + public function __construct( private string $referenceName, private KafkaAdmin $kafkaAdmin, @@ -71,14 +73,18 @@ private function handleBatch(BatchMessage $batchMessage, Producer $producer, Pro { $deliveryIds = []; try { + $producedMessages = 0; foreach ($batchMessage->getEntries() as $entry) { $entryMessage = MessageBuilder::withPayload($entry['payload']) ->setMultipleHeaders($entry['headers']) ->build(); $deliveryIds[] = $this->produce($entryMessage, $topic, trackDelivery: true); - $producer->poll(0); + if (++$producedMessages % self::POLL_EVERY_PRODUCED_MESSAGES === 0) { + $producer->poll(0); + } } + $producer->poll(0); } catch (Throwable $exception) { if ($deliveryIds !== []) { $this->registerPendingDelivery($producer, $deliveryIds); @@ -118,7 +124,20 @@ private function produce(Message $message, ProducerTopic $topic, bool $trackDeli : null; try { - $this->produceTracked($topic, $outboundMessage, $partitionKey, $headers, $deliveryId); + $retryDeadline = microtime(true) + ($this->kafkaAdmin->getConfigurationForPublisher($this->referenceName)->getAsyncPublishingTimeout() / 1000); + while (true) { + try { + $this->produceTracked($topic, $outboundMessage, $partitionKey, $headers, $deliveryId); + + break; + } catch (KafkaException $exception) { + if ($exception->getCode() !== RD_KAFKA_RESP_ERR__QUEUE_FULL || microtime(true) >= $retryDeadline) { + throw $exception; + } + + $this->kafkaAdmin->getProducer($this->referenceName)->poll(100); + } + } } catch (Throwable $exception) { if ($deliveryId !== null) { $this->kafkaAdmin->getDeliveryTracker($this->referenceName)->discard($deliveryId); From 15610e09d226d7cac21425ac8489db8278daf7d2 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Mon, 13 Jul 2026 08:00:00 +0200 Subject: [PATCH 27/38] test: failing batch among many unresolved asyncPublish calls fails command before commit --- .../InMemoryAsyncOutboundAdapter.php | 31 +++++++++++++++- .../AsyncPublishingReliabilityTest.php | 36 +++++++++++++++++++ .../AsyncPublishingReliabilityTest.php | 30 ++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncOutboundAdapter.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncOutboundAdapter.php index 3927324fe..67d9caef1 100644 --- a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncOutboundAdapter.php +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncOutboundAdapter.php @@ -5,6 +5,7 @@ namespace Test\Ecotone\Messaging\Fixture\AsyncPublishing; use Ecotone\Messaging\Attribute\Parameter\Reference; +use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Message; @@ -21,6 +22,8 @@ final class InMemoryAsyncOutboundAdapter private ?string $deliveryFailureReason = null; + private ?string $failingPayloadFragment = null; + private bool $registersPendingDeliveries = true; public function handle(Message $message, #[Reference] AsyncPublishingRegistry $asyncPublishingRegistry): void @@ -31,7 +34,7 @@ public function handle(Message $message, #[Reference] AsyncPublishingRegistry $a return; } - $pendingDelivery = new InMemoryPendingDelivery($message, $this->deliveryFailureReason); + $pendingDelivery = new InMemoryPendingDelivery($message, $this->resolveFailureReason($message)); $this->pendingDeliveries[] = $pendingDelivery; $asyncPublishingRegistry->register(InMemoryAsyncPublisherModule::PUBLISHER_REFERENCE, $pendingDelivery); } @@ -67,6 +70,32 @@ public function failDeliveriesWith(string $failureReason): void $this->deliveryFailureReason = $failureReason; } + public function failDeliveriesContaining(string $payloadFragment, string $failureReason): void + { + $this->failingPayloadFragment = $payloadFragment; + $this->deliveryFailureReason = $failureReason; + } + + private function resolveFailureReason(Message $message): ?string + { + if ($this->failingPayloadFragment === null) { + return $this->deliveryFailureReason; + } + + $payload = $message->getPayload(); + $payloadsToInspect = $payload instanceof BatchMessage + ? array_column($payload->getEntries(), 'payload') + : [$payload]; + + foreach ($payloadsToInspect as $payloadToInspect) { + if (is_string($payloadToInspect) && str_contains($payloadToInspect, $this->failingPayloadFragment)) { + return $this->deliveryFailureReason; + } + } + + return null; + } + public function actAsSynchronousPublisher(): void { $this->registersPendingDeliveries = false; diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php index ceb53a7af..d4451b3d8 100644 --- a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php @@ -5,12 +5,16 @@ namespace Test\Ecotone\Messaging\Unit\Channel\AsyncPublishing; use Ecotone\Lite\EcotoneLite; +use Ecotone\Messaging\Attribute\Parameter\Reference; +use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\AsyncPublishing\DeliveryFuture; use Ecotone\Messaging\Channel\MessageChannelInterceptorAdapter; use Ecotone\Messaging\Config\ServiceConfiguration; +use Ecotone\Messaging\MessagePublisher; use Ecotone\Messaging\Support\MessageBuilder; +use Ecotone\Modelling\Attribute\CommandHandler; use PHPUnit\Framework\TestCase; use Test\Ecotone\Messaging\Fixture\AsyncPublishing\AsyncOrderSubscriber; use Test\Ecotone\Messaging\Fixture\AsyncPublishing\FakeTransactionModule; @@ -144,6 +148,38 @@ public function test_future_awaits_remaining_deliveries_when_earlier_delivery_th $this->assertTrue($followingDelivery->isAwaited()); } + public function test_one_failing_batch_among_many_published_in_command_handler_rolls_back_transaction(): void + { + $operationsLog = new OperationsLog(); + $outboundAdapter = new InMemoryAsyncOutboundAdapter(); + $commandHandler = new class () { + #[CommandHandler('order.placeAllBatches')] + public function handle(string $order, #[Reference(InMemoryAsyncPublisherModule::PUBLISHER_REFERENCE)] MessagePublisher $publisher): void + { + $publisher->asyncPublish(BatchMessage::constructEmpty()->append($order . ' first batch')); + $publisher->asyncPublish(BatchMessage::constructEmpty()->append($order . ' poisoned batch')); + $publisher->asyncPublish(BatchMessage::constructEmpty()->append($order . ' third batch')); + } + }; + $ecotoneLite = EcotoneLite::bootstrapFlowTesting( + [$commandHandler::class, InMemoryAsyncPublisherModule::class, InMemoryAsyncOutboundAdapter::class, FakeTransactionModule::class], + [$commandHandler, $outboundAdapter, OperationsLog::class => $operationsLog], + ); + $outboundAdapter->failDeliveriesContaining('poisoned', 'broker rejected the batch'); + + $commandFailed = false; + try { + $ecotoneLite->sendCommandWithRoutingKey('order.placeAllBatches', 'espresso'); + } catch (AsyncPublishingFailedException) { + $commandFailed = true; + } + + $this->assertTrue($commandFailed); + $operations = $operationsLog->getOperations(); + $this->assertSame('transaction rolled back', $operations[count($operations) - 1]); + $this->assertSame(3, $outboundAdapter->awaitedDeliveriesCount()); + } + public function test_unresolved_publisher_futures_above_backlog_limit_are_flushed(): void { $outboundAdapter = new InMemoryAsyncOutboundAdapter(); diff --git a/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php index 0db30f0dc..4c3cce4f6 100644 --- a/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php +++ b/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php @@ -6,12 +6,14 @@ use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; +use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\MessagePublisher; use Ecotone\Messaging\Support\MessageBuilder; +use Ecotone\Modelling\Attribute\CommandHandler; use Ecotone\Sqs\Configuration\SqsMessagePublisherConfiguration; use Ecotone\Sqs\SqsBackedMessageChannelBuilder; use Ecotone\Test\LicenceTesting; @@ -55,6 +57,34 @@ public function test_broker_rejected_batch_sent_without_active_scope_throws_imme ); } + public function test_one_failing_batch_among_many_published_in_command_handler_fails_before_commit(): void + { + $commandHandler = new class () { + #[CommandHandler('order.placeAllBatches')] + public function handle(string $order, #[Reference(MessagePublisher::class)] MessagePublisher $publisher): void + { + $publisher->asyncPublish(BatchMessage::constructEmpty()->append($order . ' first valid order')); + $publisher->asyncPublish(BatchMessage::constructEmpty()->append(str_repeat('x', 300_000))); + $publisher->asyncPublish(BatchMessage::constructEmpty()->append($order . ' third valid order')); + } + }; + $messaging = EcotoneLite::bootstrapFlowTesting( + [$commandHandler::class], + [SqsConnectionFactory::class => $this->getConnectionFactory(), $commandHandler], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::SQS_PACKAGE])) + ->withExtensionObjects([ + SqsMessagePublisherConfiguration::create(queueName: Uuid::v7()->toRfc4122()) + ->withAsyncPublishing(timeoutInMilliseconds: 10000), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $this->expectException(AsyncPublishingFailedException::class); + + $messaging->sendCommandWithRoutingKey('order.placeAllBatches', 'espresso'); + } + private function bootstrapPublisher(string $queueName): FlowTestSupport { return EcotoneLite::bootstrapFlowTesting( From 2746f48bb7876d9d6ff203f91cf935f05d25ed68 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 14 Jul 2026 08:00:00 +0200 Subject: [PATCH 28/38] fix: honor millisecond ttl on redis and drop null kafka headers, proven by command handler batch publishing tests with delay and ttl across providers --- .../tests/Integration/AsyncPublishingTest.php | 101 ++++++++++++++++++ .../tests/Integration/AsyncPublishingTest.php | 93 ++++++++++++++++ .../Outbound/KafkaOutboundChannelAdapter.php | 2 +- .../tests/Integration/AsyncPublishingTest.php | 54 ++++++++++ .../Redis/src/RedisOutboundChannelAdapter.php | 2 +- .../tests/Integration/AsyncPublishingTest.php | 57 ++++++++++ .../tests/Integration/AsyncPublishingTest.php | 68 ++++++++++++ 7 files changed, 375 insertions(+), 2 deletions(-) diff --git a/packages/Amqp/tests/Integration/AsyncPublishingTest.php b/packages/Amqp/tests/Integration/AsyncPublishingTest.php index 18ce35592..2489e6cd5 100644 --- a/packages/Amqp/tests/Integration/AsyncPublishingTest.php +++ b/packages/Amqp/tests/Integration/AsyncPublishingTest.php @@ -9,13 +9,17 @@ use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; +use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; use Ecotone\Messaging\Endpoint\PollingMetadata; +use Ecotone\Messaging\Message; +use Ecotone\Messaging\MessageHeaders; use Ecotone\Messaging\MessagePublisher; +use Ecotone\Messaging\PollableChannel; use Ecotone\Messaging\Support\LicensingException; use Ecotone\Modelling\Attribute\CommandHandler; use Ecotone\Modelling\Attribute\EventHandler; @@ -135,6 +139,71 @@ public function test_message_publisher_async_publish_confirms_delivery_on_future $this->assertNull($batchFuture->resolve()); } + public function test_batch_message_published_synchronously_from_command_handler_is_delivered(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $commandHandler = new class () { + #[CommandHandler('order.placeBatch')] + public function handle(string $order, #[Reference(MessagePublisher::class)] MessagePublisher $publisher): void + { + $publisher->convertAndSend( + BatchMessage::constructEmpty() + ->append($order . ' first order') + ->append($order . ' second order') + ); + } + }; + $messaging = $this->bootstrapPublisherWithVerificationChannel($queueName, $commandHandler); + + $messaging->sendCommandWithRoutingKey('order.placeBatch', 'espresso'); + + $verificationChannel = $messaging->getMessageChannel('verificationChannel'); + $receivedPayloads = [ + $verificationChannel->receive()->getPayload(), + $verificationChannel->receive()->getPayload(), + ]; + sort($receivedPayloads); + $this->assertSame(['espresso first order', 'espresso second order'], $receivedPayloads); + } + + public function test_delayed_entry_of_published_batch_is_delivered_after_delay(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisherWithVerificationChannel($queueName); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('immediate order') + ->append('delayed order', [MessageHeaders::DELIVERY_DELAY => 2000]) + )->resolve(); + + $verificationChannel = $messaging->getMessageChannel('verificationChannel'); + $this->assertSame('immediate order', $verificationChannel->receive()->getPayload()); + $this->assertNull($verificationChannel->receiveWithTimeout(PollingMetadata::create('assertNotYetDelivered')->setExecutionTimeLimitInMilliseconds(500))); + + $this->assertSame('delayed order', $this->receiveWithDeadline($verificationChannel, 10)?->getPayload()); + } + + public function test_expired_entry_of_published_batch_is_not_delivered(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisherWithVerificationChannel($queueName); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('expiring order', [MessageHeaders::TIME_TO_LIVE => 100]) + ->append('kept order') + )->resolve(); + + usleep(300000); + + $verificationChannel = $messaging->getMessageChannel('verificationChannel'); + $this->assertSame('kept order', $verificationChannel->receive()->getPayload()); + $this->assertNull($verificationChannel->receiveWithTimeout(PollingMetadata::create('assertExpired')->setExecutionTimeLimitInMilliseconds(500))); + } + public function test_batch_published_over_amqp_lib_connection_is_delivered(): void { $channelName = Uuid::v7()->toRfc4122(); @@ -199,6 +268,38 @@ public function getReceived(): array }; } + private function bootstrapPublisherWithVerificationChannel(string $queueName, ?object $commandHandler = null): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + $commandHandler === null ? [] : [$commandHandler::class], + $commandHandler === null + ? [...$this->getConnectionFactoryReferences()] + : [...$this->getConnectionFactoryReferences(), $commandHandler], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::AMQP_PACKAGE])) + ->withExtensionObjects([ + AmqpMessagePublisherConfiguration::create() + ->withDefaultRoutingKey($queueName) + ->withAsyncPublishing(), + AmqpBackedMessageChannelBuilder::create('verificationChannel', queueName: $queueName), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + + private function receiveWithDeadline(PollableChannel $channel, int $deadlineInSeconds): ?Message + { + $deadline = microtime(true) + $deadlineInSeconds; + while (microtime(true) < $deadline) { + if ($message = $channel->receive()) { + return $message; + } + usleep(100000); + } + + return null; + } + private function bootstrapEcotone(string $channelName, object $orderService, ?string $licenceKey): FlowTestSupport { return EcotoneLite::bootstrapFlowTesting( diff --git a/packages/Dbal/tests/Integration/AsyncPublishingTest.php b/packages/Dbal/tests/Integration/AsyncPublishingTest.php index 29a733982..7ebbb0890 100644 --- a/packages/Dbal/tests/Integration/AsyncPublishingTest.php +++ b/packages/Dbal/tests/Integration/AsyncPublishingTest.php @@ -9,12 +9,17 @@ use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; +use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; +use Ecotone\Messaging\Endpoint\PollingMetadata; +use Ecotone\Messaging\Message; +use Ecotone\Messaging\MessageHeaders; use Ecotone\Messaging\MessagePublisher; +use Ecotone\Messaging\PollableChannel; use Ecotone\Messaging\Support\LicensingException; use Ecotone\Modelling\Attribute\CommandHandler; use Ecotone\Modelling\Attribute\EventHandler; @@ -116,6 +121,81 @@ public function test_message_publisher_async_publish_confirms_delivery_on_future $this->assertSame(['first order', 'second order', 'single order'], $receivedPayloads); } + public function test_batch_message_published_synchronously_from_command_handler_is_delivered(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $commandHandler = new class () { + #[CommandHandler('order.placeBatch')] + public function handle(string $order, #[Reference(MessagePublisher::class)] MessagePublisher $publisher): void + { + $publisher->convertAndSend( + BatchMessage::constructEmpty() + ->append($order . ' first order') + ->append($order . ' second order') + ); + } + }; + $messaging = EcotoneLite::bootstrapFlowTesting( + [$commandHandler::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $commandHandler], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + DbalMessagePublisherConfiguration::create(MessagePublisher::class, $queueName) + ->withAsyncPublishing(), + DbalBackedMessageChannelBuilder::create($queueName), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.placeBatch', 'espresso'); + + $receivedPayloads = []; + while ($message = $messaging->getMessageChannel($queueName)->receive()) { + $receivedPayloads[] = $message->getPayload(); + } + sort($receivedPayloads); + $this->assertSame(['espresso first order', 'espresso second order'], $receivedPayloads); + } + + public function test_delayed_entry_of_published_batch_is_delivered_after_delay(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: true); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('immediate order') + ->append('delayed order', [MessageHeaders::DELIVERY_DELAY => 3000]) + )->resolve(); + + $channel = $messaging->getMessageChannel($queueName); + $this->assertSame('immediate order', $channel->receive()->getPayload()); + $this->assertNull($channel->receiveWithTimeout(PollingMetadata::create('assertNotYetDelivered')->setExecutionTimeLimitInMilliseconds(500))); + + $this->assertSame('delayed order', $this->receiveWithDeadline($channel, 10)?->getPayload()); + } + + public function test_expired_entry_of_published_batch_is_not_delivered(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: true); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('expiring order', [MessageHeaders::TIME_TO_LIVE => 1000]) + ->append('kept order') + )->resolve(); + + sleep(2); + + $channel = $messaging->getMessageChannel($queueName); + $this->assertSame('kept order', $channel->receive()->getPayload()); + $this->assertNull($channel->receive()); + } + public function test_publishing_after_queue_table_is_dropped_throws(): void { $queueName = Uuid::v7()->toRfc4122(); @@ -130,6 +210,19 @@ public function test_publishing_after_queue_table_is_dropped_throws(): void $publisher->asyncPublish('order published into missing table'); } + private function receiveWithDeadline(PollableChannel $channel, int $deadlineInSeconds): ?Message + { + $deadline = microtime(true) + $deadlineInSeconds; + while (microtime(true) < $deadline) { + if ($message = $channel->receive()) { + return $message; + } + usleep(100000); + } + + return null; + } + private function createOrderService(): object { return new class () { diff --git a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php index 20db2f89c..2d6b6008e 100644 --- a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php +++ b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php @@ -116,7 +116,7 @@ private function produce(Message $message, ProducerTopic $topic, bool $trackDeli $partitionKey = $message->getHeaders()->getMessageId(); } - $headers = $outboundMessage->getHeaders(); + $headers = array_filter($outboundMessage->getHeaders(), fn (mixed $headerValue) => $headerValue !== null); unset($headers[KafkaHeader::KAFKA_TARGET_PARTITION_KEY_HEADER_NAME]); $deliveryId = $trackDelivery diff --git a/packages/Kafka/tests/Integration/AsyncPublishingTest.php b/packages/Kafka/tests/Integration/AsyncPublishingTest.php index 9ae788750..329ac18ef 100644 --- a/packages/Kafka/tests/Integration/AsyncPublishingTest.php +++ b/packages/Kafka/tests/Integration/AsyncPublishingTest.php @@ -4,12 +4,15 @@ namespace Test\Ecotone\Kafka\Integration; +use Ecotone\Kafka\Attribute\KafkaConsumer; use Ecotone\Kafka\Channel\KafkaMessageChannelBuilder; use Ecotone\Kafka\Configuration\KafkaBrokerConfiguration; use Ecotone\Kafka\Configuration\KafkaPublisherConfiguration; +use Ecotone\Kafka\Configuration\TopicConfiguration; use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; +use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; @@ -108,6 +111,57 @@ public function test_message_publisher_async_publish_confirms_delivery_on_future $this->assertNull($batchFuture->resolve()); } + public function test_batch_message_published_synchronously_from_command_handler_is_delivered(): void + { + $topicName = Uuid::v7()->toRfc4122(); + $commandHandler = new class () { + /** @var string[] */ + private array $receivedPayloads = []; + + #[CommandHandler('order.placeBatch')] + public function handle(string $order, #[Reference(MessagePublisher::class)] MessagePublisher $publisher): void + { + $publisher->convertAndSend( + BatchMessage::constructEmpty() + ->append($order . ' first order') + ->append($order . ' second order') + ); + } + + #[KafkaConsumer('batchOrdersConsumer', 'batchOrdersTopic')] + public function collect(string $payload): void + { + $this->receivedPayloads[] = $payload; + } + + #[QueryHandler('order.getReceivedBatchOrders')] + public function getReceived(): array + { + return $this->receivedPayloads; + } + }; + $messaging = EcotoneLite::bootstrapFlowTesting( + [$commandHandler::class], + [KafkaBrokerConfiguration::class => ConnectionTestCase::getConnection(), $commandHandler], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::KAFKA_PACKAGE])) + ->withExtensionObjects([ + KafkaPublisherConfiguration::createWithDefaults(topicName: $topicName) + ->withAsyncPublishing(), + TopicConfiguration::createWithReferenceName('batchOrdersTopic', $topicName), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.placeBatch', 'espresso'); + + $messaging->run('batchOrdersConsumer', ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 2, maxExecutionTimeInMilliseconds: 30000)); + + $receivedPayloads = $messaging->sendQueryWithRouting('order.getReceivedBatchOrders'); + sort($receivedPayloads); + $this->assertSame(['espresso first order', 'espresso second order'], $receivedPayloads); + } + private function createOrderService(string $channelName): object { return new class ($channelName) { diff --git a/packages/Redis/src/RedisOutboundChannelAdapter.php b/packages/Redis/src/RedisOutboundChannelAdapter.php index 71dcb48de..131931709 100644 --- a/packages/Redis/src/RedisOutboundChannelAdapter.php +++ b/packages/Redis/src/RedisOutboundChannelAdapter.php @@ -97,7 +97,7 @@ protected function handleBatch(BatchMessage $batchMessage, Context $context): vo if ($outboundMessage->getTimeToLive()) { $messageToSend->setTimeToLive($outboundMessage->getTimeToLive()); - $messageToSend->setHeader('expires_at', time() + $messageToSend->getTimeToLive()); + $messageToSend->setHeader('expires_at', time() + (int) ceil($outboundMessage->getTimeToLive() / 1000)); } $payload = $context->getSerializer()->toString($messageToSend); diff --git a/packages/Redis/tests/Integration/AsyncPublishingTest.php b/packages/Redis/tests/Integration/AsyncPublishingTest.php index 840480156..42f9437e0 100644 --- a/packages/Redis/tests/Integration/AsyncPublishingTest.php +++ b/packages/Redis/tests/Integration/AsyncPublishingTest.php @@ -7,6 +7,7 @@ use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; +use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; @@ -126,6 +127,43 @@ public function test_message_publisher_async_publish_confirms_delivery_on_future $this->assertSame(['first order', 'second order', 'single order'], $receivedPayloads); } + public function test_batch_message_published_synchronously_from_command_handler_is_delivered(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $commandHandler = new class () { + #[CommandHandler('order.placeBatch')] + public function handle(string $order, #[Reference(MessagePublisher::class)] MessagePublisher $publisher): void + { + $publisher->convertAndSend( + BatchMessage::constructEmpty() + ->append($order . ' first order') + ->append($order . ' second order') + ); + } + }; + $messaging = EcotoneLite::bootstrapFlowTesting( + [$commandHandler::class], + [RedisConnectionFactory::class => $this->getConnectionFactory(), $commandHandler], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::REDIS_PACKAGE])) + ->withExtensionObjects([ + RedisMessagePublisherConfiguration::create(queueName: $queueName) + ->withAsyncPublishing(), + RedisBackedMessageChannelBuilder::create($queueName), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.placeBatch', 'espresso'); + + $receivedPayloads = []; + while ($message = $messaging->getMessageChannel($queueName)->receive()) { + $receivedPayloads[] = $message->getPayload(); + } + sort($receivedPayloads); + $this->assertSame(['espresso first order', 'espresso second order'], $receivedPayloads); + } + public function test_delayed_entry_of_published_batch_lands_in_delayed_set(): void { $queueName = Uuid::v7()->toRfc4122(); @@ -149,6 +187,25 @@ public function test_delayed_entry_of_published_batch_lands_in_delayed_set(): vo $this->assertSame(1, $context->getRedis()->eval('return redis.call("zcard", KEYS[1])', [$queueName . ':delayed'])); } + public function test_expired_entry_of_published_batch_is_not_delivered(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: true); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('expiring order', [MessageHeaders::TIME_TO_LIVE => 1000]) + ->append('kept order') + )->resolve(); + + sleep(2); + + $channel = $messaging->getMessageChannel($queueName); + $this->assertSame('kept order', $channel->receive()->getPayload()); + $this->assertNull($channel->receive()); + } + public function test_publishing_to_wrong_type_key_throws(): void { $queueName = Uuid::v7()->toRfc4122(); diff --git a/packages/Sqs/tests/Integration/AsyncPublishingTest.php b/packages/Sqs/tests/Integration/AsyncPublishingTest.php index e3bc1bcff..cd5cba5ea 100644 --- a/packages/Sqs/tests/Integration/AsyncPublishingTest.php +++ b/packages/Sqs/tests/Integration/AsyncPublishingTest.php @@ -7,11 +7,13 @@ use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; +use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; +use Ecotone\Messaging\MessageHeaders; use Ecotone\Messaging\MessagePublisher; use Ecotone\Messaging\Support\LicensingException; use Ecotone\Modelling\Attribute\CommandHandler; @@ -114,6 +116,43 @@ public function test_message_publisher_async_publish_confirms_delivery_on_future $this->assertSame(['first order', 'second order', 'single order'], $receivedPayloads); } + public function test_batch_message_published_synchronously_from_command_handler_is_delivered(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $commandHandler = new class () { + #[CommandHandler('order.placeBatch')] + public function handle(string $order, #[Reference(MessagePublisher::class)] MessagePublisher $publisher): void + { + $publisher->convertAndSend( + BatchMessage::constructEmpty() + ->append($order . ' first order') + ->append($order . ' second order') + ); + } + }; + $messaging = EcotoneLite::bootstrapFlowTesting( + [$commandHandler::class], + [SqsConnectionFactory::class => $this->getConnectionFactory(), $commandHandler], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::SQS_PACKAGE])) + ->withExtensionObjects([ + SqsMessagePublisherConfiguration::create(queueName: $queueName) + ->withAsyncPublishing(timeoutInMilliseconds: 10000), + SqsBackedMessageChannelBuilder::create($queueName), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.placeBatch', 'espresso'); + + $receivedPayloads = []; + while ($message = $messaging->getMessageChannel($queueName)->receive()) { + $receivedPayloads[] = $message->getPayload(); + } + sort($receivedPayloads); + $this->assertSame(['espresso first order', 'espresso second order'], $receivedPayloads); + } + public function test_batch_larger_than_ten_messages_is_chunked_and_delivered(): void { $queueName = Uuid::v7()->toRfc4122(); @@ -134,6 +173,35 @@ public function test_batch_larger_than_ten_messages_is_chunked_and_delivered(): $this->assertCount(25, $receivedPayloads); } + public function test_delayed_entry_of_published_batch_is_delivered_after_delay(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: true); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $publishedAt = microtime(true); + $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('immediate order') + ->append('delayed order', [MessageHeaders::DELIVERY_DELAY => 1000]) + )->resolve(); + + $channel = $messaging->getMessageChannel($queueName); + $receivedAt = []; + $deadline = microtime(true) + 15; + while (count($receivedAt) < 2 && microtime(true) < $deadline) { + if ($message = $channel->receive()) { + $receivedAt[$message->getPayload()] = microtime(true); + } else { + usleep(100000); + } + } + + $this->assertArrayHasKey('immediate order', $receivedAt); + $this->assertArrayHasKey('delayed order', $receivedAt); + $this->assertGreaterThanOrEqual(1.0, $receivedAt['delayed order'] - $publishedAt); + } + private function createOrderService(): object { return new class () { From a88dc053b8a61a47dcd198beb3c6c1150c08badc Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 14 Jul 2026 08:00:00 +0200 Subject: [PATCH 29/38] refactor: publish single messages through shared batch flow in amqp and dbal adapters --- .../Amqp/src/AmqpOutboundChannelAdapter.php | 65 ++++++++++--------- .../Dbal/src/DbalOutboundChannelAdapter.php | 9 +++ 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/packages/Amqp/src/AmqpOutboundChannelAdapter.php b/packages/Amqp/src/AmqpOutboundChannelAdapter.php index 9d6c0edf5..16473b878 100644 --- a/packages/Amqp/src/AmqpOutboundChannelAdapter.php +++ b/packages/Amqp/src/AmqpOutboundChannelAdapter.php @@ -63,16 +63,22 @@ public function __construct( */ public function handle(Message $message): void { - if ($message->getPayload() instanceof BatchMessage) { - $this->handleBatch($message->getPayload(), $message); - + $payload = $message->getPayload(); + $messagesToPublish = $payload instanceof BatchMessage + ? array_map( + fn (array $entry): Message => MessageBuilder::withPayload($entry['payload'])->setMultipleHeaders($entry['headers'])->build(), + $payload->getEntries(), + ) + : [$message]; + + if ($messagesToPublish === []) { return; } - $this->publish($message); + $this->publishMessages($messagesToPublish); if ($this->canPublishAsynchronously()) { - $this->registerPendingDelivery([$message]); + $this->registerPendingDelivery($messagesToPublish); return; } @@ -85,37 +91,27 @@ public function isAsyncPublishingEnabled(): bool return $this->asyncPublishing; } - private function handleBatch(BatchMessage $batchMessage, Message $carrierMessage): void + /** + * @param Message[] $messages + */ + private function publishMessages(array $messages): void { - $entryMessages = []; - foreach ($batchMessage->getEntries() as $entry) { - $entryMessages[] = MessageBuilder::withPayload($entry['payload']) - ->setMultipleHeaders($entry['headers']) - ->build(); - } - $context = $this->connectionFactory->createContext(); if ($context instanceof AmqpLibContext) { - $this->publishBatchThroughSingleWrite($entryMessages, $context); - } else { - foreach ($entryMessages as $entryMessage) { - $this->publish($entryMessage); - } - } - - if ($this->canPublishAsynchronously()) { - $this->registerPendingDelivery($entryMessages); + $this->publishThroughSingleBatchWrite($messages, $context); return; } - $this->awaitPublisherConfirmsSynchronously(); + foreach ($messages as $message) { + $this->publish($message); + } } /** * @param Message[] $messages */ - private function publishBatchThroughSingleWrite(array $messages, AmqpLibContext $context): void + private function publishThroughSingleBatchWrite(array $messages, AmqpLibContext $context): void { $preparedEntries = []; $delayedMessages = []; @@ -128,12 +124,7 @@ private function publishBatchThroughSingleWrite(array $messages, AmqpLibContext continue; } - $amqpProperties = $interopMessage->getHeaders(); - if ($applicationProperties = $interopMessage->getProperties()) { - $amqpProperties['application_headers'] = new AMQPTable($applicationProperties); - } - - $preparedEntries[] = [new LibAMQPMessage($interopMessage->getBody(), $amqpProperties), $exchangeName, $interopMessage->getRoutingKey() ?? '']; + $preparedEntries[] = [$this->convertToLibMessage($interopMessage), $exchangeName, $interopMessage->getRoutingKey() ?? '', (bool) ($interopMessage->getFlags() & AmqpMessage::FLAG_MANDATORY)]; } foreach ($delayedMessages as $delayedMessage) { @@ -141,8 +132,8 @@ private function publishBatchThroughSingleWrite(array $messages, AmqpLibContext } $libChannel = $context->getLibChannel(); - foreach ($preparedEntries as [$libMessage, $exchangeName, $routingKey]) { - $libChannel->batch_basic_publish($libMessage, $exchangeName, $routingKey, mandatory: $this->publisherConfirms); + foreach ($preparedEntries as [$libMessage, $exchangeName, $routingKey, $mandatory]) { + $libChannel->batch_basic_publish($libMessage, $exchangeName, $routingKey, mandatory: $mandatory); } if ($preparedEntries !== []) { @@ -150,6 +141,16 @@ private function publishBatchThroughSingleWrite(array $messages, AmqpLibContext } } + private function convertToLibMessage(AmqpMessage $interopMessage): LibAMQPMessage + { + $amqpProperties = $interopMessage->getHeaders(); + if ($applicationProperties = $interopMessage->getProperties()) { + $amqpProperties['application_headers'] = new AMQPTable($applicationProperties); + } + + return new LibAMQPMessage($interopMessage->getBody(), $amqpProperties); + } + private function publish(Message $message): void { [$messageToSend, $exchangeName, $deliveryDelay, $timeToLive] = $this->prepareInteropMessage($message); diff --git a/packages/Dbal/src/DbalOutboundChannelAdapter.php b/packages/Dbal/src/DbalOutboundChannelAdapter.php index 7e93721cd..20cf2f273 100644 --- a/packages/Dbal/src/DbalOutboundChannelAdapter.php +++ b/packages/Dbal/src/DbalOutboundChannelAdapter.php @@ -11,6 +11,7 @@ use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Conversion\ConversionService; +use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageHeaders; use Enqueue\Dbal\DbalContext; use Enqueue\Dbal\DbalDestination; @@ -57,6 +58,14 @@ public function initialize(): void $context->createQueue($this->queueName); } + protected function sendSingleMessage(Message $message, Context $context): void + { + $this->handleBatch( + BatchMessage::constructEmpty()->append($message->getPayload(), $message->getHeaders()->headers()), + $context, + ); + } + protected function handleBatch(BatchMessage $batchMessage, Context $context): void { $messagesToSend = []; From 0863701bae464cfa27d2c0c3bf3ad24a5b9833b1 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 18 Jul 2026 08:00:00 +0200 Subject: [PATCH 30/38] fix: correct kafka queue-full retry and tighten async publishing lifecycle - catch \RdKafka\Exception instead of the undefined KafkaException so producer queue-full backpressure actually polls and retries rather than failing the publish on the first full-queue signal - require AsyncPublishingRegistry in the SQS and Enqueue outbound adapters, matching DBAL and Redis, removing the nullable dereference - drop the register_shutdown_function auto-flush; unresolved deliveries are now flushed via explicit flushUnawaitedDeliveries and the backlog limit --- .../Channel/AsyncPublishing/AsyncPublishingRegistry.php | 7 ------- packages/Enqueue/src/EnqueueOutboundChannelAdapter.php | 4 ++-- .../Kafka/src/Outbound/KafkaOutboundChannelAdapter.php | 2 +- packages/Sqs/src/SqsOutboundChannelAdapter.php | 2 +- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php index 741a880bf..f642c79cd 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php @@ -24,8 +24,6 @@ final class AsyncPublishingRegistry private bool $scopeActive = false; - private bool $shutdownFlushRegistered = false; - public function openScope(): void { $this->scopeActive = true; @@ -82,11 +80,6 @@ public function register(string $channelName, PendingDelivery $pendingDelivery): $this->registrationsSinceLastPrune = 0; } $this->pendingDeliveries[$this->nextRegistrationIndex++] = ['channelName' => $channelName, 'pendingDelivery' => $pendingDelivery, 'scopeOwned' => $this->scopeActive]; - - if (! $this->shutdownFlushRegistered) { - register_shutdown_function(fn () => $this->flushUnawaitedDeliveries()); - $this->shutdownFlushRegistered = true; - } } public function collectionPoint(): int diff --git a/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php b/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php index a0da428c5..3ead04540 100644 --- a/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php +++ b/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php @@ -32,7 +32,7 @@ public function __construct( protected bool $autoDeclare, protected OutboundMessageConverter $outboundMessageConverter, private ConversionService $conversionService, - private ?AsyncPublishingRegistry $asyncPublishingRegistry = null, + private AsyncPublishingRegistry $asyncPublishingRegistry, private bool $asyncPublishing = false, private string $asyncPublishingChannelName = '', ) { @@ -70,7 +70,7 @@ protected function createOutboundContext(): Context protected function registerSynchronouslyConfirmedDelivery(): void { - if (! $this->asyncPublishing || $this->asyncPublishingRegistry === null || ! $this->asyncPublishingRegistry->isScopeActive()) { + if (! $this->asyncPublishing || ! $this->asyncPublishingRegistry->isScopeActive()) { return; } diff --git a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php index 2d6b6008e..0fdac5716 100644 --- a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php +++ b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php @@ -130,7 +130,7 @@ private function produce(Message $message, ProducerTopic $topic, bool $trackDeli $this->produceTracked($topic, $outboundMessage, $partitionKey, $headers, $deliveryId); break; - } catch (KafkaException $exception) { + } catch (\RdKafka\Exception $exception) { if ($exception->getCode() !== RD_KAFKA_RESP_ERR__QUEUE_FULL || microtime(true) >= $retryDeadline) { throw $exception; } diff --git a/packages/Sqs/src/SqsOutboundChannelAdapter.php b/packages/Sqs/src/SqsOutboundChannelAdapter.php index 232835dc4..f2a4ce04b 100644 --- a/packages/Sqs/src/SqsOutboundChannelAdapter.php +++ b/packages/Sqs/src/SqsOutboundChannelAdapter.php @@ -33,7 +33,7 @@ public function __construct( bool $autoDeclare, OutboundMessageConverter $outboundMessageConverter, ConversionService $conversionService, - private ?AsyncPublishingRegistry $asyncPublishingRegistry = null, + private AsyncPublishingRegistry $asyncPublishingRegistry, private bool $asyncPublishing = false, private ?int $asyncPublishingTimeout = null, ) { From 163941a5f7ede22a0974a7989506c8473ec97194 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 18 Jul 2026 08:00:00 +0200 Subject: [PATCH 31/38] fix: per-message confirm attribution for amqp, granular batch retry and error channel routing, shutdown flush of unawaited deliveries --- .../src/AmqpExtPublisherConfirmations.php | 61 ------- .../Amqp/src/AmqpOutboundChannelAdapter.php | 133 +++++++++------ .../src/AmqpOutboundChannelAdapterBuilder.php | 4 +- packages/Amqp/src/AmqpPendingDelivery.php | 103 ++++++++---- .../Amqp/src/AmqpPublisherConfirmations.php | 152 ++++++++++++++++++ .../AmqpReconnectableConnectionFactory.php | 51 ++++-- .../AmqpMessagePublisherConfiguration.php | 2 + .../AsyncPublishingReliabilityTest.php | 139 +++++++++++++++- .../AsyncPublishingRegistry.php | 43 ++++- .../SendRetryChannelInterceptor.php | 34 +++- .../RegisterSingletonMessagingServices.php | 3 +- .../InMemoryAsyncOutboundAdapter.php | 39 ++++- .../InMemoryAsyncPublishingChannel.php | 58 ++++++- .../InMemoryPendingDelivery.php | 13 +- .../AsyncPublishingReliabilityTest.php | 48 +++++- .../AsyncPublishingScenariosTest.php | 29 ++++ .../SendRetryOfFailedBatchDeliveriesTest.php | 104 ++++++++++++ .../Channel/KafkaMessageChannelBuilder.php | 2 + .../KafkaPublisherConfiguration.php | 2 + .../Outbound/KafkaOutboundChannelAdapter.php | 9 +- .../SqsMessagePublisherConfiguration.php | 2 + .../Sqs/src/SqsOutboundChannelAdapter.php | 6 +- .../src/SqsOutboundChannelAdapterBuilder.php | 6 +- 23 files changed, 856 insertions(+), 187 deletions(-) delete mode 100644 packages/Amqp/src/AmqpExtPublisherConfirmations.php create mode 100644 packages/Amqp/src/AmqpPublisherConfirmations.php create mode 100644 packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/SendRetryOfFailedBatchDeliveriesTest.php diff --git a/packages/Amqp/src/AmqpExtPublisherConfirmations.php b/packages/Amqp/src/AmqpExtPublisherConfirmations.php deleted file mode 100644 index 40f1bce29..000000000 --- a/packages/Amqp/src/AmqpExtPublisherConfirmations.php +++ /dev/null @@ -1,61 +0,0 @@ - */ - private array $individuallyConfirmedTags = []; - - public function recordPublishedMessage(): void - { - $this->publishedCount++; - } - - public function recordConfirmation(int $deliveryTag, bool $multiple): void - { - if ($multiple) { - $this->highestConfirmedTag = max($this->highestConfirmedTag, $deliveryTag); - foreach ($this->individuallyConfirmedTags as $tag => $confirmed) { - if ($tag <= $this->highestConfirmedTag) { - unset($this->individuallyConfirmedTags[$tag]); - } - } - - return; - } - - if ($deliveryTag > $this->highestConfirmedTag) { - $this->individuallyConfirmedTags[$deliveryTag] = true; - } - } - - public function hasOutstandingConfirmations(): bool - { - return $this->publishedCount > ($this->highestConfirmedTag + count($this->individuallyConfirmedTags)); - } - - public function reset(): void - { - $this->epoch++; - $this->publishedCount = 0; - $this->highestConfirmedTag = 0; - $this->individuallyConfirmedTags = []; - } - - public function getEpoch(): int - { - return $this->epoch; - } -} diff --git a/packages/Amqp/src/AmqpOutboundChannelAdapter.php b/packages/Amqp/src/AmqpOutboundChannelAdapter.php index 16473b878..d523eaa51 100644 --- a/packages/Amqp/src/AmqpOutboundChannelAdapter.php +++ b/packages/Amqp/src/AmqpOutboundChannelAdapter.php @@ -7,7 +7,9 @@ use Ecotone\Amqp\Transaction\AmqpTransactionInterceptor; use Ecotone\Enqueue\CachedConnectionFactory; use Ecotone\Messaging\BatchMessage; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; +use Ecotone\Messaging\Channel\AsyncPublishing\FailedDelivery; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Conversion\ConversionService; use Ecotone\Messaging\Message; @@ -50,8 +52,8 @@ public function __construct( private OutboundMessageConverter $outboundMessageConverter, private ConversionService $conversionService, private AmqpTransactionInterceptor $amqpTransactionInterceptor, + private AsyncPublishingRegistry $asyncPublishingRegistry, private ?DelayStrategy $delayStrategy = null, - private ?AsyncPublishingRegistry $asyncPublishingRegistry = null, private bool $asyncPublishing = false, private int $asyncPublishingTimeout = AmqpOutboundChannelAdapterBuilder::DEFAULT_ASYNC_PUBLISHING_TIMEOUT, private string $channelName = '', @@ -75,15 +77,15 @@ public function handle(Message $message): void return; } - $this->publishMessages($messagesToPublish); + $publishRecords = $this->publishMessages($messagesToPublish); - if ($this->canPublishAsynchronously()) { - $this->registerPendingDelivery($messagesToPublish); + if ($publishRecords !== [] && $this->canPublishAsynchronously()) { + $this->registerPendingDelivery($publishRecords); return; } - $this->awaitPublisherConfirmsSynchronously(); + $this->awaitPublisherConfirmsSynchronously($publishRecords); } public function isAsyncPublishingEnabled(): bool @@ -93,25 +95,28 @@ public function isAsyncPublishingEnabled(): bool /** * @param Message[] $messages + * @return array */ - private function publishMessages(array $messages): void + private function publishMessages(array $messages): array { $context = $this->connectionFactory->createContext(); if ($context instanceof AmqpLibContext) { - $this->publishThroughSingleBatchWrite($messages, $context); - - return; + return $this->publishThroughSingleBatchWrite($messages, $context); } + $publishRecords = []; foreach ($messages as $message) { - $this->publish($message); + $publishRecords[] = $this->publish($message); } + + return array_values(array_filter($publishRecords)); } /** * @param Message[] $messages + * @return array */ - private function publishThroughSingleBatchWrite(array $messages, AmqpLibContext $context): void + private function publishThroughSingleBatchWrite(array $messages, AmqpLibContext $context): array { $preparedEntries = []; $delayedMessages = []; @@ -124,21 +129,30 @@ private function publishThroughSingleBatchWrite(array $messages, AmqpLibContext continue; } - $preparedEntries[] = [$this->convertToLibMessage($interopMessage), $exchangeName, $interopMessage->getRoutingKey() ?? '', (bool) ($interopMessage->getFlags() & AmqpMessage::FLAG_MANDATORY)]; + $preparedEntries[] = [$message, $interopMessage, $exchangeName]; } + $publishRecords = []; foreach ($delayedMessages as $delayedMessage) { - $this->publish($delayedMessage); + $publishRecords[] = $this->publish($delayedMessage); } $libChannel = $context->getLibChannel(); - foreach ($preparedEntries as [$libMessage, $exchangeName, $routingKey, $mandatory]) { - $libChannel->batch_basic_publish($libMessage, $exchangeName, $routingKey, mandatory: $mandatory); + foreach ($preparedEntries as [$message, $interopMessage, $exchangeName]) { + $libChannel->batch_basic_publish( + $this->convertToLibMessage($interopMessage), + $exchangeName, + $interopMessage->getRoutingKey() ?? '', + mandatory: (bool) ($interopMessage->getFlags() & AmqpMessage::FLAG_MANDATORY), + ); + $publishRecords[] = $this->recordPublishedMessage($message, $interopMessage); } if ($preparedEntries !== []) { $libChannel->publish_batch(); } + + return array_values(array_filter($publishRecords)); } private function convertToLibMessage(AmqpMessage $interopMessage): LibAMQPMessage @@ -151,7 +165,10 @@ private function convertToLibMessage(AmqpMessage $interopMessage): LibAMQPMessag return new LibAMQPMessage($interopMessage->getBody(), $amqpProperties); } - private function publish(Message $message): void + /** + * @return array{message: Message, deliveryTag: int, correlationId: string}|null + */ + private function publish(Message $message): ?array { [$messageToSend, $exchangeName, $deliveryDelay, $timeToLive] = $this->prepareInteropMessage($message); @@ -162,26 +179,39 @@ private function publish(Message $message): void // this allow for having queue per delay instead of queue per delay + exchangeName ->send(new AmqpTopic($exchangeName), $messageToSend); - $this->recordExtPublishedMessage(); + return $this->recordPublishedMessage($message, $messageToSend); } - private function recordExtPublishedMessage(): void + /** + * @return array{message: Message, deliveryTag: int, correlationId: string}|null + */ + private function recordPublishedMessage(Message $message, AmqpMessage $interopMessage): ?array { if (! $this->publisherConfirms) { - return; + return null; } - if ($this->connectionFactory->createContext() instanceof AmqpExtContext) { - $this->getExtPublisherConfirmations()?->recordPublishedMessage(); + $confirmations = $this->getPublisherConfirmations(); + if ($confirmations === null) { + return null; } + + $correlationId = (string) $interopMessage->getProperty(AmqpPublisherConfirmations::PUBLISH_BATCH_ID_PROPERTY, ''); + $resolveTagThroughCorrelation = $this->connectionFactory->createContext() instanceof AmqpLibContext; + + return [ + 'message' => $message, + 'deliveryTag' => $confirmations->recordPublishedMessage($resolveTagThroughCorrelation ? $correlationId : ''), + 'correlationId' => $correlationId, + ]; } - private function getExtPublisherConfirmations(): ?AmqpExtPublisherConfirmations + private function getPublisherConfirmations(): ?AmqpPublisherConfirmations { $innerConnectionFactory = $this->connectionFactory->getInnerConnectionFactory(); return $innerConnectionFactory instanceof AmqpReconnectableConnectionFactory - ? $innerConnectionFactory->getExtPublisherConfirmations() + ? $innerConnectionFactory->getPublisherConfirmations() : null; } @@ -227,6 +257,7 @@ private function prepareInteropMessage(Message $message): array if ($this->publisherConfirms) { Assert::isFalse($this->amqpTransactionInterceptor->isRunningInTransaction(), 'Cannot use publisher acknowledgments together with transactions. Please disable one of them.'); $messageToSend->addFlag(AmqpMessage::FLAG_MANDATORY); + $messageToSend->setProperty(AmqpPublisherConfirmations::PUBLISH_BATCH_ID_PROPERTY, bin2hex(random_bytes(8))); } return [$messageToSend, $exchangeName, $outboundMessage->getDeliveryDelay(), $timeToLive]; @@ -236,54 +267,66 @@ private function canPublishAsynchronously(): bool { return $this->asyncPublishing && $this->publisherConfirms - && $this->asyncPublishingRegistry !== null && $this->asyncPublishingRegistry->isScopeActive(); } /** - * @param Message[] $publishedMessages + * @param array $publishRecords */ - private function registerPendingDelivery(array $publishedMessages): void + private function registerPendingDelivery(array $publishRecords): void { $this->asyncPublishingRegistry->register( $this->channelName, new AmqpPendingDelivery( $this->connectionFactory->createContext(), - $publishedMessages, + $publishRecords, $this->asyncPublishingTimeout, $this->channelName, - $this->getExtPublisherConfirmations(), + $this->getPublisherConfirmations(), ), ); } - private function awaitPublisherConfirmsSynchronously(): void + /** + * @param array $publishRecords + */ + private function awaitPublisherConfirmsSynchronously(array $publishRecords): void { if (! $this->publisherConfirms || $this->amqpTransactionInterceptor->isRunningInTransaction()) { return; } - $timeoutInSeconds = $this->asyncPublishingTimeout / 1000; $context = $this->connectionFactory->createContext(); - if ($context instanceof AmqpLibContext) { - $context->getLibChannel()->wait_for_pending_acks_returns($timeoutInSeconds); - } elseif ($context instanceof AmqpExtContext) { - $extPublisherConfirmations = $this->getExtPublisherConfirmations(); - if ($extPublisherConfirmations === null) { + if ($publishRecords === []) { + $timeoutInSeconds = $this->asyncPublishingTimeout / 1000; + if ($context instanceof AmqpLibContext) { + $context->getLibChannel()->wait_for_pending_acks_returns($timeoutInSeconds); + } elseif ($context instanceof AmqpExtContext) { $context->getExtChannel()->waitForConfirm($timeoutInSeconds); - - return; } - $deadline = microtime(true) + $timeoutInSeconds; - while ($extPublisherConfirmations->hasOutstandingConfirmations()) { - $remainingSeconds = $deadline - microtime(true); - if ($remainingSeconds <= 0) { - throw new RuntimeException('Timed out awaiting publisher confirms from RabbitMQ instance.'); - } + return; + } - $context->getExtChannel()->waitForConfirm($remainingSeconds); - } + $deliveryResult = (new AmqpPendingDelivery( + $context, + $publishRecords, + $this->asyncPublishingTimeout, + $this->channelName, + $this->getPublisherConfirmations(), + ))->awaitDelivery(); + + if ($deliveryResult->isSuccessful()) { + return; } + + if ($this->asyncPublishing) { + throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + } + + throw new RuntimeException(implode('; ', array_unique(array_map( + fn (FailedDelivery $failedDelivery): string => $failedDelivery->getFailureReason(), + $deliveryResult->getFailedDeliveries(), + )))); } } diff --git a/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php b/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php index d1adac772..002b5a577 100644 --- a/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php +++ b/packages/Amqp/src/AmqpOutboundChannelAdapterBuilder.php @@ -23,7 +23,7 @@ class AmqpOutboundChannelAdapterBuilder extends EnqueueOutboundChannelAdapterBui { private const DEFAULT_PERSISTENT_MODE = true; - public const DEFAULT_ASYNC_PUBLISHING_TIMEOUT = 5000; + public const DEFAULT_ASYNC_PUBLISHING_TIMEOUT = 12000; private string $amqpConnectionFactoryReferenceName; private string $defaultRoutingKey = ''; @@ -186,8 +186,8 @@ public function compile(MessagingContainerBuilder $builder): Definition $outboundMessageConverter, new Reference(ConversionService::REFERENCE_NAME), Reference::to(AmqpTransactionInterceptor::class), - $this->delayStrategyReferenceName ? new Reference($this->delayStrategyReferenceName) : null, new Reference(AsyncPublishingRegistry::class), + $this->delayStrategyReferenceName ? new Reference($this->delayStrategyReferenceName) : null, $this->asyncPublishing, $this->asyncPublishingTimeout, $this->asyncPublishingChannelName ?? $this->exchangeName, diff --git a/packages/Amqp/src/AmqpPendingDelivery.php b/packages/Amqp/src/AmqpPendingDelivery.php index be7b3b48a..5c04e7b7a 100644 --- a/packages/Amqp/src/AmqpPendingDelivery.php +++ b/packages/Amqp/src/AmqpPendingDelivery.php @@ -11,7 +11,7 @@ use Enqueue\AmqpExt\AmqpContext as AmqpExtContext; use Enqueue\AmqpLib\AmqpContext as AmqpLibContext; use Interop\Amqp\AmqpContext; -use RuntimeException; +use PhpAmqpLib\Exception\AMQPTimeoutException; use Throwable; /** @@ -19,6 +19,10 @@ */ final class AmqpPendingDelivery implements PendingDelivery { + private const REJECTED_FAILURE_REASON = 'Message was rejected (nack) by RabbitMQ instance. Check RabbitMQ server logs.'; + private const TIMED_OUT_FAILURE_REASON = 'Timed out awaiting publisher confirms from RabbitMQ instance.'; + private const CONNECTION_RESET_FAILURE_REASON = 'AMQP connection was reset while awaiting publisher confirms. Delivery confirmation is unknown.'; + private bool $awaited = false; private ?DeliveryResult $deliveryResult = null; @@ -26,16 +30,16 @@ final class AmqpPendingDelivery implements PendingDelivery private int $confirmationsEpoch; /** - * @param Message[] $trackedMessages + * @param array $publishRecords */ public function __construct( private AmqpContext $context, - private array $trackedMessages, + private array $publishRecords, private int $timeoutInMilliseconds, private string $channelName, - private ?AmqpExtPublisherConfirmations $extPublisherConfirmations = null, + private AmqpPublisherConfirmations $confirmations, ) { - $this->confirmationsEpoch = $this->extPublisherConfirmations?->getEpoch() ?? 0; + $this->confirmationsEpoch = $confirmations->getEpoch(); } public function awaitDelivery(): DeliveryResult @@ -45,53 +49,82 @@ public function awaitDelivery(): DeliveryResult } $this->awaited = true; - $timeoutInSeconds = $this->timeoutInMilliseconds / 1000; + $deadline = microtime(true) + $this->timeoutInMilliseconds / 1000; + $unsettledFailureReason = self::TIMED_OUT_FAILURE_REASON; + + while (! $this->allRecordsSettled()) { + if ($this->confirmations->getEpoch() !== $this->confirmationsEpoch) { + $unsettledFailureReason = self::CONNECTION_RESET_FAILURE_REASON; + + break; + } - try { - if ($this->extPublisherConfirmations !== null && $this->extPublisherConfirmations->getEpoch() !== $this->confirmationsEpoch) { - throw new RuntimeException('AMQP connection was reset while awaiting publisher confirms. Delivery confirmation is unknown.'); + $remainingSeconds = $deadline - microtime(true); + if ($remainingSeconds <= 0) { + break; } - if ($this->context instanceof AmqpLibContext) { - $this->context->getLibChannel()->wait_for_pending_acks_returns($timeoutInSeconds); - } elseif ($this->context instanceof AmqpExtContext) { - $this->awaitAllExtConfirmations($timeoutInSeconds); + try { + $this->pumpConfirmations($remainingSeconds); + } catch (AMQPTimeoutException) { + continue; + } catch (Throwable $exception) { + $unsettledFailureReason = $exception->getMessage(); + + break; } - } catch (Throwable $exception) { - return $this->deliveryResult = DeliveryResult::withFailedDeliveries(array_map( - fn (Message $message) => new FailedDelivery($message, $exception->getMessage(), $this->channelName), - $this->trackedMessages, - )); } - return $this->deliveryResult = DeliveryResult::successful(); + return $this->deliveryResult = $this->collectResult($unsettledFailureReason); } - private function awaitAllExtConfirmations(float $timeoutInSeconds): void + public function isAwaited(): bool { - if ($this->extPublisherConfirmations === null) { - $this->context->getExtChannel()->waitForConfirm($timeoutInSeconds); - - return; - } + return $this->awaited; + } - $deadline = microtime(true) + $timeoutInSeconds; - while ($this->extPublisherConfirmations->hasOutstandingConfirmations()) { - if ($this->extPublisherConfirmations->getEpoch() !== $this->confirmationsEpoch) { - throw new RuntimeException('AMQP connection was reset while awaiting publisher confirms. Delivery confirmation is unknown.'); + private function allRecordsSettled(): bool + { + foreach ($this->publishRecords as $publishRecord) { + if (! $this->confirmations->isSettled($publishRecord['deliveryTag'])) { + return false; } + } - $remainingSeconds = $deadline - microtime(true); - if ($remainingSeconds <= 0) { - throw new RuntimeException('Timed out awaiting publisher confirms from RabbitMQ instance.'); - } + return true; + } + private function pumpConfirmations(float $remainingSeconds): void + { + if ($this->context instanceof AmqpLibContext) { + $this->context->getLibChannel()->wait_for_pending_acks_returns($remainingSeconds); + } elseif ($this->context instanceof AmqpExtContext) { $this->context->getExtChannel()->waitForConfirm($remainingSeconds); } } - public function isAwaited(): bool + private function collectResult(string $unsettledFailureReason): DeliveryResult { - return $this->awaited; + $failedDeliveries = []; + foreach ($this->publishRecords as $publishRecord) { + $returnReason = $this->confirmations->takeReturnReason($publishRecord['correlationId']); + if ($returnReason !== null) { + $failedDeliveries[] = new FailedDelivery($publishRecord['message'], $returnReason, $this->channelName); + + continue; + } + + if ($this->confirmations->takeRejection($publishRecord['deliveryTag'])) { + $failedDeliveries[] = new FailedDelivery($publishRecord['message'], self::REJECTED_FAILURE_REASON, $this->channelName); + + continue; + } + + if (! $this->confirmations->isSettled($publishRecord['deliveryTag'])) { + $failedDeliveries[] = new FailedDelivery($publishRecord['message'], $unsettledFailureReason, $this->channelName); + } + } + + return $failedDeliveries === [] ? DeliveryResult::successful() : DeliveryResult::withFailedDeliveries($failedDeliveries); } } diff --git a/packages/Amqp/src/AmqpPublisherConfirmations.php b/packages/Amqp/src/AmqpPublisherConfirmations.php new file mode 100644 index 000000000..cc5a172e9 --- /dev/null +++ b/packages/Amqp/src/AmqpPublisherConfirmations.php @@ -0,0 +1,152 @@ + */ + private array $individuallySettledTags = []; + + /** @var array */ + private array $rejectedTags = []; + + /** @var array */ + private array $deliveryTagsByCorrelationId = []; + + /** @var array */ + private array $returnReasonsByCorrelationId = []; + + public function recordPublishedMessage(string $correlationId = ''): int + { + $deliveryTag = ++$this->lastPublishedDeliveryTag; + if ($correlationId !== '') { + $this->deliveryTagsByCorrelationId[$correlationId] = $deliveryTag; + } + + return $deliveryTag; + } + + public function recordConfirmation(int $deliveryTag, bool $multiple): void + { + $this->settle($deliveryTag, $multiple); + } + + public function recordConfirmationForCorrelation(string $correlationId): void + { + $deliveryTag = $this->takeDeliveryTagForCorrelation($correlationId); + if ($deliveryTag !== null) { + $this->settle($deliveryTag, multiple: false); + } + } + + public function recordRejection(int $deliveryTag, bool $multiple): void + { + if ($multiple) { + for ($rejectedTag = $this->settledWatermark + 1; $rejectedTag <= $deliveryTag; $rejectedTag++) { + if (! isset($this->individuallySettledTags[$rejectedTag])) { + $this->rejectedTags[$rejectedTag] = true; + } + } + } else { + $this->rejectedTags[$deliveryTag] = true; + } + + $this->settle($deliveryTag, $multiple); + } + + public function recordRejectionForCorrelation(string $correlationId): void + { + $deliveryTag = $this->takeDeliveryTagForCorrelation($correlationId); + if ($deliveryTag !== null) { + $this->recordRejection($deliveryTag, multiple: false); + } + } + + public function recordReturnedMessage(string $correlationId, string $reason): void + { + if ($correlationId !== '') { + $this->returnReasonsByCorrelationId[$correlationId] = $reason; + } + } + + public function isSettled(int $deliveryTag): bool + { + return $deliveryTag <= $this->settledWatermark || isset($this->individuallySettledTags[$deliveryTag]); + } + + public function takeRejection(int $deliveryTag): bool + { + $wasRejected = isset($this->rejectedTags[$deliveryTag]); + unset($this->rejectedTags[$deliveryTag]); + + return $wasRejected; + } + + public function takeReturnReason(string $correlationId): ?string + { + $reason = $this->returnReasonsByCorrelationId[$correlationId] ?? null; + unset($this->returnReasonsByCorrelationId[$correlationId]); + + return $reason; + } + + public function hasOutstandingConfirmations(): bool + { + return $this->lastPublishedDeliveryTag > $this->settledWatermark + count($this->individuallySettledTags); + } + + public function reset(): void + { + $this->epoch++; + $this->lastPublishedDeliveryTag = 0; + $this->settledWatermark = 0; + $this->individuallySettledTags = []; + $this->rejectedTags = []; + $this->deliveryTagsByCorrelationId = []; + $this->returnReasonsByCorrelationId = []; + } + + public function getEpoch(): int + { + return $this->epoch; + } + + private function takeDeliveryTagForCorrelation(string $correlationId): ?int + { + $deliveryTag = $this->deliveryTagsByCorrelationId[$correlationId] ?? null; + unset($this->deliveryTagsByCorrelationId[$correlationId]); + + return $deliveryTag; + } + + private function settle(int $deliveryTag, bool $multiple): void + { + if ($multiple) { + $this->settledWatermark = max($this->settledWatermark, $deliveryTag); + foreach ($this->individuallySettledTags as $settledTag => $settled) { + if ($settledTag <= $this->settledWatermark) { + unset($this->individuallySettledTags[$settledTag]); + } + } + + return; + } + + if ($deliveryTag > $this->settledWatermark) { + $this->individuallySettledTags[$deliveryTag] = true; + } + } +} diff --git a/packages/Amqp/src/AmqpReconnectableConnectionFactory.php b/packages/Amqp/src/AmqpReconnectableConnectionFactory.php index 47671ed2a..0bf384182 100644 --- a/packages/Amqp/src/AmqpReconnectableConnectionFactory.php +++ b/packages/Amqp/src/AmqpReconnectableConnectionFactory.php @@ -2,6 +2,7 @@ namespace Ecotone\Amqp; +use AMQPBasicProperties; use AMQPConnection; use Ecotone\Enqueue\ReconnectableConnectionFactory; use Enqueue\AmqpExt\AmqpConnectionFactory as AmqpExtConnectionFactory; @@ -16,9 +17,10 @@ use Interop\Queue\SubscriptionConsumer; use PhpAmqpLib\Channel\AMQPChannel as LibAMQPChannel; use PhpAmqpLib\Connection\AMQPLazyConnection; +use PhpAmqpLib\Message\AMQPMessage as LibAMQPMessage; +use PhpAmqpLib\Wire\AMQPTable; use ReflectionClass; use ReflectionProperty; -use RuntimeException; /** * licence Apache-2.0 @@ -28,7 +30,7 @@ class AmqpReconnectableConnectionFactory implements ReconnectableConnectionFacto private string $connectionInstanceId; private AmqpConnectionFactory $connectionFactory; private ?SubscriptionConsumer $subscriptionConsumer = null; - private ?AmqpExtPublisherConfirmations $extPublisherConfirmations = null; + private ?AmqpPublisherConfirmations $publisherConfirmations = null; public function __construct(AmqpExtConnectionFactory|AmqpLibConnectionFactory $connectionFactory, ?string $connectionInstanceId = null, private bool $publisherConfirms = false) { @@ -50,22 +52,39 @@ public function createContext(): Context $context = $this->connectionFactory->createContext(); if ($this->publisherConfirms) { + $confirmations = $this->getPublisherConfirmations(); + $confirmations->reset(); if ($context instanceof AmqpLibContext) { $context->getLibChannel()->confirm_select(); - $context->getLibChannel()->set_nack_handler(fn () => throw new RuntimeException('Message was rejected (nack) by RabbitMQ instance. Check RabbitMQ server logs.')); - $context->getLibChannel()->set_return_listener(fn (int $replyCode, string $replyText, string $exchange, string $routingKey) => throw new RuntimeException(sprintf('Message was returned as unroutable by RabbitMQ instance (%d %s) for exchange `%s` and routing key `%s`.', $replyCode, $replyText, $exchange, $routingKey))); + $context->getLibChannel()->set_ack_handler(fn (LibAMQPMessage $message) => $confirmations->recordConfirmationForCorrelation(self::publishCorrelationIdFrom($message))); + $context->getLibChannel()->set_nack_handler(fn (LibAMQPMessage $message) => $confirmations->recordRejectionForCorrelation(self::publishCorrelationIdFrom($message))); + $context->getLibChannel()->set_return_listener(function (int $replyCode, string $replyText, string $exchange, string $routingKey, LibAMQPMessage $message) use ($confirmations): void { + $confirmations->recordReturnedMessage( + self::publishCorrelationIdFrom($message), + sprintf('Message was returned as unroutable by RabbitMQ instance (%d %s) for exchange `%s` and routing key `%s`.', $replyCode, $replyText, $exchange, $routingKey), + ); + }); } elseif ($context instanceof AmqpExtContext) { - $confirmations = $this->getExtPublisherConfirmations(); - $confirmations->reset(); $context->getExtChannel()->confirmSelect(); - $context->getExtChannel()->setReturnCallback(fn (int $replyCode, string $replyText, string $exchange, string $routingKey) => throw new RuntimeException(sprintf('Message was returned as unroutable by RabbitMQ instance (%d %s) for exchange `%s` and routing key `%s`.', $replyCode, $replyText, $exchange, $routingKey))); + $context->getExtChannel()->setReturnCallback(function (int $replyCode, string $replyText, string $exchange, string $routingKey, AMQPBasicProperties $properties) use ($confirmations): bool { + $confirmations->recordReturnedMessage( + (string) ($properties->getHeaders()[AmqpPublisherConfirmations::PUBLISH_BATCH_ID_PROPERTY] ?? ''), + sprintf('Message was returned as unroutable by RabbitMQ instance (%d %s) for exchange `%s` and routing key `%s`.', $replyCode, $replyText, $exchange, $routingKey), + ); + + return true; + }); $context->getExtChannel()->setConfirmCallback( function (int $deliveryTag, bool $multiple) use ($confirmations): bool { $confirmations->recordConfirmation($deliveryTag, $multiple); return $confirmations->hasOutstandingConfirmations(); }, - fn () => throw new RuntimeException('Message was failed to be persisted in RabbitMQ instance. Check RabbitMQ server logs.') + function (int $deliveryTag, bool $multiple) use ($confirmations): bool { + $confirmations->recordRejection($deliveryTag, $multiple); + + return $confirmations->hasOutstandingConfirmations(); + } ); } } @@ -73,9 +92,21 @@ function (int $deliveryTag, bool $multiple) use ($confirmations): bool { return $context; } - public function getExtPublisherConfirmations(): AmqpExtPublisherConfirmations + public function getPublisherConfirmations(): AmqpPublisherConfirmations { - return $this->extPublisherConfirmations ??= new AmqpExtPublisherConfirmations(); + return $this->publisherConfirmations ??= new AmqpPublisherConfirmations(); + } + + private static function publishCorrelationIdFrom(LibAMQPMessage $message): string + { + $applicationHeaders = $message->get_properties()['application_headers'] ?? null; + if ($applicationHeaders instanceof AMQPTable) { + $applicationHeaders = $applicationHeaders->getNativeData(); + } + + $applicationHeaders = (array) $applicationHeaders; + + return (string) ($applicationHeaders[AmqpPublisherConfirmations::PUBLISH_BATCH_ID_PROPERTY] ?? ''); } public function getConnectionInstanceId(): string diff --git a/packages/Amqp/src/Publisher/AmqpMessagePublisherConfiguration.php b/packages/Amqp/src/Publisher/AmqpMessagePublisherConfiguration.php index 695604fe3..e3fa17a60 100644 --- a/packages/Amqp/src/Publisher/AmqpMessagePublisherConfiguration.php +++ b/packages/Amqp/src/Publisher/AmqpMessagePublisherConfiguration.php @@ -3,6 +3,7 @@ namespace Ecotone\Amqp\Publisher; use Ecotone\Messaging\MessagePublisher; +use Ecotone\Messaging\Support\Assert; use Enqueue\AmqpExt\AmqpConnectionFactory; /** @@ -156,6 +157,7 @@ public function getDefaultPersistentDelivery(): bool public function withAsyncPublishing(bool $enabled = true, ?int $timeoutInMilliseconds = null): AmqpMessagePublisherConfiguration { + Assert::isTrue($timeoutInMilliseconds === null || $timeoutInMilliseconds > 0, 'Async publishing timeout must be a positive amount of milliseconds.'); $this->asyncPublishing = $enabled; if ($timeoutInMilliseconds !== null) { $this->asyncPublishingTimeout = $timeoutInMilliseconds; diff --git a/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php index f9639ca10..b8b9cb4a0 100644 --- a/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php +++ b/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php @@ -4,7 +4,7 @@ namespace Test\Ecotone\Amqp\Integration; -use Ecotone\Amqp\AmqpExtPublisherConfirmations; +use Ecotone\Amqp\AmqpPublisherConfirmations; use Ecotone\Amqp\Publisher\AmqpMessagePublisherConfiguration; use Ecotone\Lite\EcotoneLite; use Ecotone\Messaging\BatchMessage; @@ -57,7 +57,7 @@ public function test_nacked_message_fails_delivery_confirmation_over_amqp_ext(): public function test_ext_publisher_confirmations_track_outstanding_until_all_confirmed(): void { - $confirmations = new AmqpExtPublisherConfirmations(); + $confirmations = new AmqpPublisherConfirmations(); $confirmations->recordPublishedMessage(); $confirmations->recordPublishedMessage(); @@ -73,7 +73,7 @@ public function test_ext_publisher_confirmations_track_outstanding_until_all_con public function test_ext_publisher_confirmations_handle_multiple_flag_covering_individual_confirmations(): void { - $confirmations = new AmqpExtPublisherConfirmations(); + $confirmations = new AmqpPublisherConfirmations(); $confirmations->recordPublishedMessage(); $confirmations->recordPublishedMessage(); @@ -105,9 +105,108 @@ public function test_unroutable_message_fails_delivery_confirmation_over_amqp_ex $publisher->asyncPublish('order that routes nowhere')->resolve(); } + public function test_each_future_reports_outcome_of_its_own_message_when_sharing_channel(): void + { + $libConnectionFactory = new AmqpLibConnection(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']); + $queueName = $this->declareQueue($libConnectionFactory); + $publisher = $this->bootstrapPublisherWithRoutingKeyFromHeader($libConnectionFactory); + + $routableFuture = $publisher->asyncPublish('order that reaches the queue', metadata: ['routingKey' => $queueName]); + $unroutableFuture = $publisher->asyncPublish('order that routes nowhere', metadata: ['routingKey' => Uuid::v7()->toRfc4122()]); + + $routableFuture->resolve(); + + $this->expectException(AsyncPublishingFailedException::class); + + $unroutableFuture->resolve(); + } + + public function test_each_future_reports_outcome_of_its_own_message_when_sharing_channel_over_amqp_ext(): void + { + $extConnectionFactory = new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']); + $queueName = $this->declareQueue($extConnectionFactory); + $publisher = $this->bootstrapPublisherWithRoutingKeyFromHeader($extConnectionFactory); + + $routableFuture = $publisher->asyncPublish('order that reaches the queue', metadata: ['routingKey' => $queueName]); + $unroutableFuture = $publisher->asyncPublish('order that routes nowhere', metadata: ['routingKey' => Uuid::v7()->toRfc4122()]); + + $routableFuture->resolve(); + + $this->expectException(AsyncPublishingFailedException::class); + + $unroutableFuture->resolve(); + } + + public function test_nack_arriving_during_other_future_await_fails_only_nacked_future_over_amqp_lib(): void + { + $libConnectionFactory = new AmqpLibConnection(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']); + $normalQueue = $this->declareQueue($libConnectionFactory); + $overflowQueue = $this->declareQueueRejectingOverflow($libConnectionFactory); + $publisher = $this->bootstrapPublisherWithRoutingKeyFromHeader($libConnectionFactory); + + $publisher->asyncPublish('filler order', metadata: ['routingKey' => $overflowQueue])->resolve(); + + $deliveredFuture = $publisher->asyncPublish('delivered order', metadata: ['routingKey' => $normalQueue]); + $nackedFuture = $publisher->asyncPublish('nacked order', metadata: ['routingKey' => $overflowQueue]); + + $deliveredFuture->resolve(); + + $this->expectException(AsyncPublishingFailedException::class); + + $nackedFuture->resolve(); + } + + public function test_nack_arriving_during_other_future_await_fails_only_nacked_future_over_amqp_ext(): void + { + $extConnectionFactory = new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']); + $normalQueue = $this->declareQueue($extConnectionFactory); + $overflowQueue = $this->declareQueueRejectingOverflow($extConnectionFactory); + $publisher = $this->bootstrapPublisherWithRoutingKeyFromHeader($extConnectionFactory); + + $publisher->asyncPublish('filler order', metadata: ['routingKey' => $overflowQueue])->resolve(); + + $deliveredFuture = $publisher->asyncPublish('delivered order', metadata: ['routingKey' => $normalQueue]); + $nackedFuture = $publisher->asyncPublish('nacked order', metadata: ['routingKey' => $overflowQueue]); + + $deliveredFuture->resolve(); + + $this->expectException(AsyncPublishingFailedException::class); + + $nackedFuture->resolve(); + } + + public function test_only_failing_message_from_batch_is_reported_with_per_message_granularity(): void + { + $libConnectionFactory = new AmqpLibConnection(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']); + $queueName = $this->declareQueue($libConnectionFactory); + $publisher = $this->bootstrapPublisherWithRoutingKeyFromHeader($libConnectionFactory); + + $future = $publisher->asyncPublish( + BatchMessage::constructEmpty() + ->append('first delivered order', ['routingKey' => $queueName]) + ->append('order that routes nowhere', ['routingKey' => Uuid::v7()->toRfc4122()]) + ->append('second delivered order', ['routingKey' => $queueName]) + ); + + try { + $future->resolve(); + $this->fail('Expected unroutable batch entry to fail the delivery'); + } catch (AsyncPublishingFailedException $exception) { + $failedDeliveries = $exception->getFailedDeliveries(); + $this->assertCount(1, $failedDeliveries); + $this->assertSame('order that routes nowhere', $failedDeliveries[0]->getMessage()->getPayload()); + $this->assertStringContainsString('NO_ROUTE', $failedDeliveries[0]->getFailureReason()); + } + + $context = $libConnectionFactory->createContext(); + $consumer = $context->createConsumer($context->createQueue($queueName)); + $this->assertNotNull($consumer->receive(2000)); + $this->assertNotNull($consumer->receive(2000)); + } + public function test_ext_confirmations_reset_while_awaiting_is_detectable_through_epoch(): void { - $confirmations = new AmqpExtPublisherConfirmations(); + $confirmations = new AmqpPublisherConfirmations(); $epochBeforeReset = $confirmations->getEpoch(); $confirmations->recordPublishedMessage(); @@ -117,6 +216,38 @@ public function test_ext_confirmations_reset_while_awaiting_is_detectable_throug $this->assertFalse($confirmations->hasOutstandingConfirmations()); } + private function declareQueue(AmqpLibConnection|AmqpConnectionFactory $connectionFactory): string + { + $queueName = Uuid::v7()->toRfc4122(); + $context = $connectionFactory->createContext(); + $queue = $context->createQueue($queueName); + $queue->addFlag(AmqpQueue::FLAG_DURABLE); + $context->declareQueue($queue); + + return $queueName; + } + + private function bootstrapPublisherWithRoutingKeyFromHeader(AmqpLibConnection|AmqpConnectionFactory $connectionFactory): MessagePublisher + { + $messaging = EcotoneLite::bootstrapFlowTesting( + [], + [ + AmqpConnectionFactory::class => $connectionFactory, + ], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::AMQP_PACKAGE])) + ->withExtensionObjects([ + AmqpMessagePublisherConfiguration::create() + ->withAutoDeclareQueueOnSend(false) + ->withRoutingKeyFromHeader('routingKey') + ->withAsyncPublishing(timeoutInMilliseconds: 3000), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + return $messaging->getGateway(MessagePublisher::class); + } + private function declareQueueRejectingOverflow(AmqpLibConnection|AmqpConnectionFactory $connectionFactory): string { $queueName = Uuid::v7()->toRfc4122(); diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php index f642c79cd..bedf05ed7 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingRegistry.php @@ -4,7 +4,9 @@ namespace Ecotone\Messaging\Channel\AsyncPublishing; +use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Throwable; +use WeakMap; /** * licence Enterprise @@ -14,6 +16,13 @@ final class AsyncPublishingRegistry /** @var array */ private array $pendingDeliveries = []; + /** @var WeakMap|null */ + private static ?WeakMap $registriesFlushedOnShutdown = null; + + public function __construct(private LoggingGateway $logger) + { + } + private const PRUNE_INTERVAL = 256; private const MAX_UNAWAITED_BACKLOG = 1024; @@ -75,6 +84,7 @@ public function awaitAll(): DeliveryResult public function register(string $channelName, PendingDelivery $pendingDelivery): void { + $this->flushUnawaitedDeliveriesOnShutdown(); if (++$this->registrationsSinceLastPrune >= self::PRUNE_INTERVAL) { $this->pruneAwaitedDeliveries(); $this->registrationsSinceLastPrune = 0; @@ -102,6 +112,20 @@ public function registeredSince(int $collectionPoint): array return $registered; } + private function flushUnawaitedDeliveriesOnShutdown(): void + { + if (self::$registriesFlushedOnShutdown === null) { + self::$registriesFlushedOnShutdown = new WeakMap(); + register_shutdown_function(static function (): void { + foreach (self::$registriesFlushedOnShutdown as $registry => $awaitingFlush) { + $registry->flushUnawaitedDeliveries(); + } + }); + } + + self::$registriesFlushedOnShutdown[$this] = true; + } + public function flushUnawaitedDeliveries(): void { foreach ($this->pendingDeliveries as $registration) { @@ -149,16 +173,23 @@ private function awaitAndLogFailures(PendingDelivery $pendingDelivery): void try { $deliveryResult = $pendingDelivery->awaitDelivery(); } catch (Throwable $exception) { - error_log(sprintf('Ecotone async publishing: awaiting unresolved delivery failed: %s', $exception->getMessage())); + $this->logger->error( + sprintf('Async publishing: awaiting unresolved delivery failed: %s', $exception->getMessage()), + [], + ['exception' => $exception], + ); return; } foreach ($deliveryResult->getFailedDeliveries() as $failedDelivery) { - error_log(sprintf( - 'Ecotone async publishing: unresolved delivery for channel `%s` failed confirmation: %s', - $failedDelivery->getChannelName(), - $failedDelivery->getFailureReason(), - )); + $this->logger->error( + sprintf( + 'Async publishing: unresolved delivery for channel `%s` failed confirmation: %s', + $failedDelivery->getChannelName(), + $failedDelivery->getFailureReason(), + ), + $failedDelivery->getMessage(), + ); } } } diff --git a/packages/Ecotone/src/Messaging/Channel/PollableChannel/SendRetries/SendRetryChannelInterceptor.php b/packages/Ecotone/src/Messaging/Channel/PollableChannel/SendRetries/SendRetryChannelInterceptor.php index f75378f20..2167d46b2 100644 --- a/packages/Ecotone/src/Messaging/Channel/PollableChannel/SendRetries/SendRetryChannelInterceptor.php +++ b/packages/Ecotone/src/Messaging/Channel/PollableChannel/SendRetries/SendRetryChannelInterceptor.php @@ -4,7 +4,9 @@ namespace Ecotone\Messaging\Channel\PollableChannel\SendRetries; +use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AbstractChannelInterceptor; +use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\ChannelInterceptor; use Ecotone\Messaging\Config\ConfiguredMessagingSystem; use Ecotone\Messaging\Handler\Gateway\ErrorChannelService; @@ -12,6 +14,7 @@ use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageChannel; use Ecotone\Messaging\Scheduling\EcotoneClockInterface; +use Ecotone\Messaging\Support\MessageBuilder; use Exception; use Psr\Log\LoggerInterface; use Throwable; @@ -38,6 +41,8 @@ public function afterSendCompletion(Message $message, MessageChannel $messageCha return false; } + $messageToRedeliver = $this->messageContainingOnlyFailedDeliveries($message, $exception); + if ($exception !== null) { $attempt = 1; while ($this->retryTemplate->canBeCalledNextTime($attempt)) { @@ -49,10 +54,11 @@ public function afterSendCompletion(Message $message, MessageChannel $messageCha try { $this->clock->sleep($this->retryTemplate->durationToNextRetry($attempt)); - $messageChannel->send($message); + $messageChannel->send($messageToRedeliver); return true; } catch (Exception $exception) { + $messageToRedeliver = $this->messageContainingOnlyFailedDeliveries($messageToRedeliver, $exception); $attempt++; } } @@ -65,7 +71,7 @@ public function afterSendCompletion(Message $message, MessageChannel $messageCha if ($this->deadLetterChannel !== null) { $this->errorChannelService->handle( - $message, + $messageToRedeliver, $exception, $this->configuredMessagingSystem->getMessageChannelByName($this->deadLetterChannel), $this->relatedChannel, @@ -76,4 +82,28 @@ public function afterSendCompletion(Message $message, MessageChannel $messageCha return false; } + + private function messageContainingOnlyFailedDeliveries(Message $message, Throwable $exception): Message + { + if (! $exception instanceof AsyncPublishingFailedException || ! $message->getPayload() instanceof BatchMessage) { + return $message; + } + + $failedDeliveries = $exception->getFailedDeliveries(); + if ($failedDeliveries === []) { + return $message; + } + + $batchOfFailedDeliveries = BatchMessage::constructEmpty(); + foreach ($failedDeliveries as $failedDelivery) { + $batchOfFailedDeliveries = $batchOfFailedDeliveries->append( + $failedDelivery->getMessage()->getPayload(), + $failedDelivery->getMessage()->getHeaders()->headers(), + ); + } + + return MessageBuilder::fromMessage($message) + ->setPayload($batchOfFailedDeliveries) + ->build(); + } } diff --git a/packages/Ecotone/src/Messaging/Config/Container/Compiler/RegisterSingletonMessagingServices.php b/packages/Ecotone/src/Messaging/Config/Container/Compiler/RegisterSingletonMessagingServices.php index d082ce614..35652a393 100644 --- a/packages/Ecotone/src/Messaging/Config/Container/Compiler/RegisterSingletonMessagingServices.php +++ b/packages/Ecotone/src/Messaging/Config/Container/Compiler/RegisterSingletonMessagingServices.php @@ -21,6 +21,7 @@ use Ecotone\Messaging\Handler\Enricher\PropertyReaderAccessor; use Ecotone\Messaging\Handler\ExpressionEvaluationService; use Ecotone\Messaging\Handler\Gateway\ProxyFactory; +use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Ecotone\Messaging\Handler\ReferenceSearchService; use Ecotone\Messaging\Handler\SymfonyExpressionEvaluationAdapter; use Ecotone\Messaging\NullableMessageChannel; @@ -60,7 +61,7 @@ public function process(ContainerBuilder $builder): void $this->registerDefault($builder, ConfiguredMessagingSystem::class, new Definition(MessagingSystemContainer::class, [new Reference(ContainerInterface::class), [], []])); $this->registerDefault($builder, EventMapper::class, new Definition(EventMapper::class, factory: 'createEmpty')); $this->registerDefault($builder, LicenceDecider::class, new Definition(LicenceDecider::class, [$this->serviceConfiguration->isRunningForEnterprise()])); - $this->registerDefault($builder, AsyncPublishingRegistry::class, new Definition(AsyncPublishingRegistry::class)); + $this->registerDefault($builder, AsyncPublishingRegistry::class, new Definition(AsyncPublishingRegistry::class, [new Reference(LoggingGateway::class)])); } private function registerDefault(ContainerBuilder $builder, string $id, Definition|Reference $definition): void diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncOutboundAdapter.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncOutboundAdapter.php index 67d9caef1..23767a13e 100644 --- a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncOutboundAdapter.php +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncOutboundAdapter.php @@ -8,6 +8,7 @@ use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Message; +use Ecotone\Messaging\Support\MessageBuilder; /** * licence Apache-2.0 @@ -34,7 +35,7 @@ public function handle(Message $message, #[Reference] AsyncPublishingRegistry $a return; } - $pendingDelivery = new InMemoryPendingDelivery($message, $this->resolveFailureReason($message)); + $pendingDelivery = new InMemoryPendingDelivery($message, $this->resolveFailureReason($message), failedMessages: $this->resolveFailedMessages($message)); $this->pendingDeliveries[] = $pendingDelivery; $asyncPublishingRegistry->register(InMemoryAsyncPublisherModule::PUBLISHER_REFERENCE, $pendingDelivery); } @@ -82,18 +83,40 @@ private function resolveFailureReason(Message $message): ?string return $this->deliveryFailureReason; } + return $this->resolveFailedMessages($message) === [] ? null : $this->deliveryFailureReason; + } + + /** + * @return Message[] + */ + private function resolveFailedMessages(Message $message): array + { + if ($this->failingPayloadFragment === null) { + return []; + } + $payload = $message->getPayload(); - $payloadsToInspect = $payload instanceof BatchMessage - ? array_column($payload->getEntries(), 'payload') - : [$payload]; + if (! $payload instanceof BatchMessage) { + return $this->matchesFailingFragment($payload) ? [$message] : []; + } - foreach ($payloadsToInspect as $payloadToInspect) { - if (is_string($payloadToInspect) && str_contains($payloadToInspect, $this->failingPayloadFragment)) { - return $this->deliveryFailureReason; + $failedMessages = []; + foreach ($payload->getEntries() as $entry) { + if ($this->matchesFailingFragment($entry['payload'])) { + $failedMessages[] = MessageBuilder::withPayload($entry['payload']) + ->setMultipleHeaders($entry['headers']) + ->build(); } } - return null; + return $failedMessages; + } + + private function matchesFailingFragment(mixed $payload): bool + { + $payloadAsString = is_string($payload) ? $payload : json_encode($payload); + + return is_string($payloadAsString) && str_contains($payloadAsString, $this->failingPayloadFragment); } public function actAsSynchronousPublisher(): void diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php index fd853b35c..8a9bbbbd6 100644 --- a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php @@ -23,6 +23,8 @@ final class InMemoryAsyncPublishingChannel implements PollableChannel, BatchSupp private ?string $deliveryFailureReason = null; + private ?string $failingPayloadFragment = null; + public function __construct( private string $channelName, private AsyncPublishingRegistry $asyncPublishingRegistry, @@ -46,7 +48,13 @@ public function send(Message $message): void $this->queue[] = $message; } - $pendingDelivery = new InMemoryPendingDelivery($message, $this->deliveryFailureReason, $this->operationsLog, $this->channelName); + $pendingDelivery = new InMemoryPendingDelivery( + $message, + $this->resolveFailureReason($message), + $this->operationsLog, + $this->channelName, + failedMessages: $this->resolveFailedMessages($message), + ); if (! $this->asyncPublishingRegistry->isScopeActive()) { $deliveryResult = $pendingDelivery->awaitDelivery(); @@ -83,4 +91,52 @@ public function failDeliveriesWith(string $failureReason): void { $this->deliveryFailureReason = $failureReason; } + + public function failDeliveriesContaining(string $payloadFragment, string $failureReason): void + { + $this->failingPayloadFragment = $payloadFragment; + $this->deliveryFailureReason = $failureReason; + } + + private function resolveFailureReason(Message $message): ?string + { + if ($this->failingPayloadFragment === null) { + return $this->deliveryFailureReason; + } + + return $this->resolveFailedMessages($message) === [] ? null : $this->deliveryFailureReason; + } + + /** + * @return Message[] + */ + private function resolveFailedMessages(Message $message): array + { + if ($this->failingPayloadFragment === null) { + return []; + } + + $payload = $message->getPayload(); + if (! $payload instanceof BatchMessage) { + return $this->matchesFailingFragment($payload) ? [$message] : []; + } + + $failedMessages = []; + foreach ($payload->getEntries() as $entry) { + if ($this->matchesFailingFragment($entry['payload'])) { + $failedMessages[] = MessageBuilder::withPayload($entry['payload']) + ->setMultipleHeaders($entry['headers']) + ->build(); + } + } + + return $failedMessages; + } + + private function matchesFailingFragment(mixed $payload): bool + { + $payloadAsString = is_string($payload) ? $payload : json_encode($payload); + + return is_string($payloadAsString) && str_contains($payloadAsString, $this->failingPayloadFragment); + } } diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php index 2902accb1..03c733022 100644 --- a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryPendingDelivery.php @@ -19,12 +19,16 @@ final class InMemoryPendingDelivery implements PendingDelivery private ?DeliveryResult $deliveryResult = null; + /** + * @param Message[] $failedMessages + */ public function __construct( private Message $message, private ?string $failureReason = null, private ?OperationsLog $operationsLog = null, private string $channelName = 'in_memory_channel', private bool $throwOnAwait = false, + private array $failedMessages = [], ) { } @@ -42,9 +46,12 @@ public function awaitDelivery(): DeliveryResult } if ($this->failureReason !== null) { - return $this->deliveryResult = DeliveryResult::withFailedDeliveries([ - new FailedDelivery($this->message, $this->failureReason, $this->channelName), - ]); + $messagesToFail = $this->failedMessages !== [] ? $this->failedMessages : [$this->message]; + + return $this->deliveryResult = DeliveryResult::withFailedDeliveries(array_map( + fn (Message $failedMessage): FailedDelivery => new FailedDelivery($failedMessage, $this->failureReason, $this->channelName), + $messagesToFail, + )); } return $this->deliveryResult = DeliveryResult::successful(); diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php index d4451b3d8..fffec2827 100644 --- a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php @@ -12,6 +12,7 @@ use Ecotone\Messaging\Channel\AsyncPublishing\DeliveryFuture; use Ecotone\Messaging\Channel\MessageChannelInterceptorAdapter; use Ecotone\Messaging\Config\ServiceConfiguration; +use Ecotone\Messaging\Handler\Logger\LoggingService; use Ecotone\Messaging\MessagePublisher; use Ecotone\Messaging\Support\MessageBuilder; use Ecotone\Modelling\Attribute\CommandHandler; @@ -109,9 +110,52 @@ public function test_future_of_delivery_flushed_by_backlog_limit_still_reports_f $firstFuture->resolve(); } + public function test_unawaited_deliveries_of_all_registries_are_flushed_on_script_shutdown(): void + { + $scriptPath = tempnam(sys_get_temp_dir(), 'async_publishing_shutdown_flush_'); + file_put_contents($scriptPath, <<<'PHP' + awaited = true; + echo 'flushed;'; + + return \Ecotone\Messaging\Channel\AsyncPublishing\DeliveryResult::successful(); + } + + public function isAwaited(): bool + { + return $this->awaited; + } + }; + + $firstRegistry = new \Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry(new \Ecotone\Messaging\Handler\Logger\LoggingService()); + $firstRegistry->register('orders', $unawaitedDelivery); + $secondRegistry = new \Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry(new \Ecotone\Messaging\Handler\Logger\LoggingService()); + $secondRegistry->register('shipments', clone $unawaitedDelivery); + + echo 'script finished;'; + PHP); + + $output = shell_exec(sprintf( + 'php %s %s', + escapeshellarg($scriptPath), + escapeshellarg(dirname(__DIR__, 7) . '/vendor/autoload.php'), + )); + unlink($scriptPath); + + $this->assertSame('script finished;flushed;flushed;', $output); + } + public function test_shutdown_flush_continues_when_one_delivery_throws(): void { - $registry = new AsyncPublishingRegistry(); + $registry = new AsyncPublishingRegistry(new LoggingService()); $throwingDelivery = new InMemoryPendingDelivery(MessageBuilder::withPayload('first order')->build(), throwOnAwait: true); $followingDelivery = new InMemoryPendingDelivery(MessageBuilder::withPayload('second order')->build()); $registry->register('orders', $throwingDelivery); @@ -124,7 +168,7 @@ public function test_shutdown_flush_continues_when_one_delivery_throws(): void public function test_closing_scope_awaits_deliveries_left_unawaited_when_execution_fails_before_await(): void { - $registry = new AsyncPublishingRegistry(); + $registry = new AsyncPublishingRegistry(new LoggingService()); $registry->openScope(); $delivery = new InMemoryPendingDelivery(MessageBuilder::withPayload('order')->build()); $registry->register('orders', $delivery); diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingScenariosTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingScenariosTest.php index ad7299621..e86d34619 100644 --- a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingScenariosTest.php +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingScenariosTest.php @@ -128,6 +128,35 @@ public function test_failed_deliveries_are_routed_to_error_channel_and_transacti $this->assertNull($ecotoneLite->receiveMessageFrom('failure_channel')); } + public function test_only_failed_message_from_batch_is_routed_to_error_channel(): void + { + $operationsLog = new OperationsLog(); + $ecotoneLite = EcotoneLite::bootstrapFlowTesting( + [OrderService::class, AsyncOrderSubscriber::class, FakeTransactionModule::class], + [new OrderService($operationsLog), new AsyncOrderSubscriber(), OperationsLog::class => $operationsLog], + ServiceConfiguration::createWithDefaults()->withExtensionObjects([ + GlobalPollableChannelConfiguration::createWithDefaults()->withErrorChannel('failure_channel'), + PollableChannelConfiguration::neverRetry('async_orders')->withCollector(false)->withErrorChannel('failure_channel'), + ]), + enableAsynchronousProcessing: [ + InMemoryAsyncPublishingChannelBuilder::create('async_orders'), + SimpleMessageChannelBuilder::createQueueChannel('failure_channel'), + ], + ); + $channel = $ecotoneLite->getMessageChannel('async_orders'); + assert($channel instanceof MessageChannelInterceptorAdapter); + $channel->getInternalMessageChannel()->failDeliveriesContaining('espresso-2', 'broker rejected message'); + + $ecotoneLite->sendCommandWithRoutingKey('order.place', 'espresso'); + + $this->assertSame('transaction committed', $operationsLog->getOperations()[count($operationsLog->getOperations()) - 1]); + + $failedMessage = $ecotoneLite->receiveMessageFrom('failure_channel'); + $this->assertStringContainsString('espresso-2', $failedMessage->getPayload()); + $this->assertStringContainsString('broker rejected message', $failedMessage->getHeaders()->get(ErrorContext::EXCEPTION_MESSAGE)); + $this->assertNull($ecotoneLite->receiveMessageFrom('failure_channel')); + } + private function bootstrapEcotone(OperationsLog $operationsLog): FlowTestSupport { return EcotoneLite::bootstrapFlowTesting( diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/SendRetryOfFailedBatchDeliveriesTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/SendRetryOfFailedBatchDeliveriesTest.php new file mode 100644 index 000000000..9e988444b --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/SendRetryOfFailedBatchDeliveriesTest.php @@ -0,0 +1,104 @@ +createRecordingChannel(); + $interceptor = $this->createInterceptor(); + + $batchMessage = MessageBuilder::withPayload( + BatchMessage::constructEmpty() + ->append('first delivered order') + ->append('poison order') + ->append('second delivered order') + )->build(); + + $interceptor->afterSendCompletion( + $batchMessage, + $channel, + AsyncPublishingFailedException::withFailedDeliveries([ + new FailedDelivery(MessageBuilder::withPayload('poison order')->build(), 'nacked by broker', 'orders'), + ]), + ); + + $this->assertSame([['poison order']], $channel->sentBatchPayloads); + } + + public function test_retry_of_single_message_failure_redelivers_original_message(): void + { + $channel = $this->createRecordingChannel(); + $interceptor = $this->createInterceptor(); + + $singleMessage = MessageBuilder::withPayload('failed order')->build(); + + $interceptor->afterSendCompletion( + $singleMessage, + $channel, + AsyncPublishingFailedException::withFailedDeliveries([ + new FailedDelivery($singleMessage, 'nacked by broker', 'orders'), + ]), + ); + + $this->assertSame([['failed order']], $channel->sentBatchPayloads); + } + + private function createInterceptor(): SendRetryChannelInterceptor + { + return new SendRetryChannelInterceptor( + 'orders', + RetryTemplateBuilder::fixedBackOff(1)->maxRetryAttempts(1)->build(), + null, + new ErrorChannelService( + new LoggingService(), + $this->createStub(OutboundMessageConverter::class), + $this->createStub(ConversionService::class), + new MessageHeadersPropagatorInterceptor(), + ), + $this->createStub(ConfiguredMessagingSystem::class), + new NullLogger(), + StubUTCClock::createWithCurrentTime('2025-01-01 00:00:00'), + ); + } + + private function createRecordingChannel(): MessageChannel + { + return new class implements MessageChannel { + /** @var array> */ + public array $sentBatchPayloads = []; + + public function send(Message $message): void + { + $payload = $message->getPayload(); + $this->sentBatchPayloads[] = $payload instanceof BatchMessage + ? array_map(fn (array $entry): mixed => $entry['payload'], $payload->getEntries()) + : [$payload]; + } + }; + } +} diff --git a/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php b/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php index 838fa9a97..90de43b80 100644 --- a/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php +++ b/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php @@ -15,6 +15,7 @@ use Ecotone\Messaging\Endpoint\FinalFailureStrategy; use Ecotone\Messaging\MessageConverter\DefaultHeaderMapper; use Ecotone\Messaging\MessageConverter\HeaderMapper; +use Ecotone\Messaging\Support\Assert; /** * licence Enterprise @@ -120,6 +121,7 @@ public function withDefaultConversionMediaType(string $mediaType): self public function withAsyncPublishing(bool $enabled = true, ?int $timeoutInMilliseconds = null): self { + Assert::isTrue($timeoutInMilliseconds === null || $timeoutInMilliseconds > 0, 'Async publishing timeout must be a positive amount of milliseconds.'); $this->asyncPublishing = $enabled; if ($timeoutInMilliseconds !== null) { $this->asyncPublishingTimeout = $timeoutInMilliseconds; diff --git a/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php b/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php index 016bfe080..c14098d6e 100644 --- a/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php +++ b/packages/Kafka/src/Configuration/KafkaPublisherConfiguration.php @@ -9,6 +9,7 @@ use Ecotone\Messaging\MessageConverter\DefaultHeaderMapper; use Ecotone\Messaging\MessageConverter\HeaderMapper; use Ecotone\Messaging\MessagePublisher; +use Ecotone\Messaging\Support\Assert; use RdKafka\Conf; /** @@ -107,6 +108,7 @@ public function getHeaderMapper(): HeaderMapper public function withAsyncPublishing(bool $asyncPublishing = true, ?int $timeoutInMilliseconds = null): self { + Assert::isTrue($timeoutInMilliseconds === null || $timeoutInMilliseconds > 0, 'Async publishing timeout must be a positive amount of milliseconds.'); $this->asyncPublishing = $asyncPublishing; if ($timeoutInMilliseconds !== null) { $this->asyncPublishingTimeout = $timeoutInMilliseconds; diff --git a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php index 0fdac5716..29c0fb6d9 100644 --- a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php +++ b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php @@ -10,6 +10,7 @@ use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; +use Ecotone\Messaging\Channel\AsyncPublishing\FailedDelivery; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Conversion\ConversionService; use Ecotone\Messaging\Message; @@ -208,8 +209,12 @@ private function flushSynchronously(Producer $producer, array $deliveryIds = []) } throw MessagePublishingException::create(sprintf( - 'Failed to send message to Kafka: %s', - $deliveryResult->getFailedDeliveries()[0]->getFailureReason(), + 'Failed to deliver %d message(s) to Kafka: %s', + count($deliveryResult->getFailedDeliveries()), + implode('; ', array_unique(array_map( + fn (FailedDelivery $failedDelivery): string => $failedDelivery->getFailureReason(), + $deliveryResult->getFailedDeliveries(), + ))), )); } } diff --git a/packages/Sqs/src/Configuration/SqsMessagePublisherConfiguration.php b/packages/Sqs/src/Configuration/SqsMessagePublisherConfiguration.php index d809efee7..f38f4a1e3 100644 --- a/packages/Sqs/src/Configuration/SqsMessagePublisherConfiguration.php +++ b/packages/Sqs/src/Configuration/SqsMessagePublisherConfiguration.php @@ -5,6 +5,7 @@ namespace Ecotone\Sqs\Configuration; use Ecotone\Messaging\MessagePublisher; +use Ecotone\Messaging\Support\Assert; use Enqueue\Sqs\SqsConnectionFactory; /** @@ -76,6 +77,7 @@ public function getReferenceName(): string public function withAsyncPublishing(bool $asyncPublishing = true, ?int $timeoutInMilliseconds = null): self { + Assert::isTrue($timeoutInMilliseconds === null || $timeoutInMilliseconds > 0, 'Async publishing timeout must be a positive amount of milliseconds.'); $this->asyncPublishing = $asyncPublishing; if ($timeoutInMilliseconds !== null) { $this->asyncPublishingTimeout = $timeoutInMilliseconds; diff --git a/packages/Sqs/src/SqsOutboundChannelAdapter.php b/packages/Sqs/src/SqsOutboundChannelAdapter.php index f2a4ce04b..a374870fd 100644 --- a/packages/Sqs/src/SqsOutboundChannelAdapter.php +++ b/packages/Sqs/src/SqsOutboundChannelAdapter.php @@ -35,7 +35,7 @@ public function __construct( ConversionService $conversionService, private AsyncPublishingRegistry $asyncPublishingRegistry, private bool $asyncPublishing = false, - private ?int $asyncPublishingTimeout = null, + private int $asyncPublishingTimeout = SqsOutboundChannelAdapterBuilder::DEFAULT_ASYNC_PUBLISHING_TIMEOUT, ) { $this->requestDispatchPool = new SqsRequestDispatchPool(); parent::__construct( @@ -176,9 +176,7 @@ private function buildBatchRequest(SqsDestination $destination, string $queueUrl 'Entries' => $entries, ]; - if ($this->asyncPublishingTimeout !== null) { - $arguments['@http'] = ['timeout' => $this->asyncPublishingTimeout / 1000]; - } + $arguments['@http'] = ['timeout' => $this->asyncPublishingTimeout / 1000]; return ['arguments' => $arguments, 'trackedMessages' => $trackedMessages]; } diff --git a/packages/Sqs/src/SqsOutboundChannelAdapterBuilder.php b/packages/Sqs/src/SqsOutboundChannelAdapterBuilder.php index 0d900282c..84b3638c7 100644 --- a/packages/Sqs/src/SqsOutboundChannelAdapterBuilder.php +++ b/packages/Sqs/src/SqsOutboundChannelAdapterBuilder.php @@ -13,6 +13,7 @@ use Ecotone\Messaging\Config\Container\MessagingContainerBuilder; use Ecotone\Messaging\Config\Container\Reference; use Ecotone\Messaging\Conversion\ConversionService; +use Ecotone\Messaging\Support\Assert; use Ecotone\Messaging\Support\LicensingException; use Enqueue\Sqs\SqsConnectionFactory; @@ -21,8 +22,10 @@ */ final class SqsOutboundChannelAdapterBuilder extends EnqueueOutboundChannelAdapterBuilder { + public const DEFAULT_ASYNC_PUBLISHING_TIMEOUT = 12000; + private bool $asyncPublishing = false; - private ?int $asyncPublishingTimeout = null; + private int $asyncPublishingTimeout = self::DEFAULT_ASYNC_PUBLISHING_TIMEOUT; private function __construct(private string $queueName, private string $connectionFactoryReferenceName) { @@ -36,6 +39,7 @@ public static function create(string $queueName, string $connectionFactoryRefere public function withAsyncPublishing(bool $asyncPublishing = true, ?int $timeoutInMilliseconds = null): self { + Assert::isTrue($timeoutInMilliseconds === null || $timeoutInMilliseconds > 0, 'Async publishing timeout must be a positive amount of milliseconds.'); $this->asyncPublishing = $asyncPublishing; if ($timeoutInMilliseconds !== null) { $this->asyncPublishingTimeout = $timeoutInMilliseconds; From f69e918f4e4bbeb5ef3edeada1df16485cd42208 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Sat, 18 Jul 2026 08:00:00 +0200 Subject: [PATCH 32/38] fix: resolve nearest composer autoload in shutdown flush test so split package testing passes --- .../AsyncPublishingReliabilityTest.php | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php index fffec2827..615b69ac7 100644 --- a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php @@ -144,15 +144,26 @@ public function isAwaited(): bool PHP); $output = shell_exec(sprintf( - 'php %s %s', + 'php %s %s 2>&1', escapeshellarg($scriptPath), - escapeshellarg(dirname(__DIR__, 7) . '/vendor/autoload.php'), + escapeshellarg($this->nearestComposerAutoloadPath()), )); unlink($scriptPath); $this->assertSame('script finished;flushed;flushed;', $output); } + private function nearestComposerAutoloadPath(): string + { + for ($directory = __DIR__; $directory !== dirname($directory); $directory = dirname($directory)) { + if (file_exists($directory . '/vendor/autoload.php')) { + return $directory . '/vendor/autoload.php'; + } + } + + $this->fail('No composer autoload found above ' . __DIR__); + } + public function test_shutdown_flush_continues_when_one_delivery_throws(): void { $registry = new AsyncPublishingRegistry(new LoggingService()); From 34ed05c3b1b29daee47ac5e44f3ee67e9722f73a Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Mon, 3 Aug 2026 17:59:04 +0200 Subject: [PATCH 33/38] feat: enterprise licence gating for BatchMessage with channel-level batch handling BatchMessage publishing is now an explicit channel capability instead of a transparent interceptor fallback: - In-memory queue channels split batches into individual messages under enterprise licence and throw LicensingException without one, so tests mirror production behaviour - Broker outbound adapters reject BatchMessage when async publishing is not enabled, closing the direct MessagePublisher bypass - Collector combines into a batch only for batch-supporting channels and flattens already-batched payloads, keeping per-message flow and channel spies intact everywhere else - SendingInterceptorAdapter no longer splits batches - Kafka message channel licence test covers the batch-enabled channel variant --- .../Amqp/src/AmqpOutboundChannelAdapter.php | 5 ++ .../tests/Integration/AsyncPublishingTest.php | 25 +++++++++ .../Collector/CollectorSenderInterceptor.php | 36 ++++++++++-- .../Channel/DelayableQueueChannel.php | 30 ++++++++-- .../src/Messaging/Channel/QueueChannel.php | 31 ++++++++-- .../Channel/SendingInterceptorAdapter.php | 26 --------- .../Channel/SimpleMessageChannelBuilder.php | 7 +++ .../Unit/Channel/BatchMessageSendingTest.php | 56 +++++++++++++++++++ .../Unit/Channel/TestQueueChannel.php | 2 +- .../src/EnqueueOutboundChannelAdapter.php | 5 ++ .../Outbound/KafkaOutboundChannelAdapter.php | 5 ++ .../tests/Integration/AsyncPublishingTest.php | 19 +++++++ .../Sqs/src/SqsOutboundChannelAdapter.php | 5 ++ 13 files changed, 213 insertions(+), 39 deletions(-) diff --git a/packages/Amqp/src/AmqpOutboundChannelAdapter.php b/packages/Amqp/src/AmqpOutboundChannelAdapter.php index d523eaa51..a16b06605 100644 --- a/packages/Amqp/src/AmqpOutboundChannelAdapter.php +++ b/packages/Amqp/src/AmqpOutboundChannelAdapter.php @@ -11,6 +11,7 @@ use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\AsyncPublishing\FailedDelivery; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; +use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Conversion\ConversionService; use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageHandler; @@ -66,6 +67,10 @@ public function __construct( public function handle(Message $message): void { $payload = $message->getPayload(); + if ($payload instanceof BatchMessage && ! $this->asyncPublishing) { + throw ConfigurationException::create(sprintf('Sending BatchMessage over `%s` requires async publishing to be enabled. Enable it with withAsyncPublishing(), available as part of Ecotone Enterprise.', $this->channelName !== '' ? $this->channelName : $this->exchangeName)); + } + $messagesToPublish = $payload instanceof BatchMessage ? array_map( fn (array $entry): Message => MessageBuilder::withPayload($entry['payload'])->setMultipleHeaders($entry['headers'])->build(), diff --git a/packages/Dbal/tests/Integration/AsyncPublishingTest.php b/packages/Dbal/tests/Integration/AsyncPublishingTest.php index 7ebbb0890..2a5921b4b 100644 --- a/packages/Dbal/tests/Integration/AsyncPublishingTest.php +++ b/packages/Dbal/tests/Integration/AsyncPublishingTest.php @@ -12,6 +12,7 @@ use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; @@ -21,6 +22,7 @@ use Ecotone\Messaging\MessagePublisher; use Ecotone\Messaging\PollableChannel; use Ecotone\Messaging\Support\LicensingException; +use Ecotone\Messaging\Support\MessageBuilder; use Ecotone\Modelling\Attribute\CommandHandler; use Ecotone\Modelling\Attribute\EventHandler; use Ecotone\Modelling\Attribute\QueryHandler; @@ -121,6 +123,29 @@ public function test_message_publisher_async_publish_confirms_delivery_on_future $this->assertSame(['first order', 'second order', 'single order'], $receivedPayloads); } + public function test_sending_batch_message_over_channel_without_async_publishing_throws(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: false); + + $this->expectException(ConfigurationException::class); + + $messaging->getMessageChannel($queueName)->send( + MessageBuilder::withPayload(BatchMessage::constructEmpty()->append('first order'))->build() + ); + } + + public function test_sending_batch_message_via_publisher_without_async_publishing_throws(): void + { + $queueName = Uuid::v7()->toRfc4122(); + $messaging = $this->bootstrapPublisher($queueName, asyncPublishing: false); + $publisher = $messaging->getGateway(MessagePublisher::class); + + $this->expectException(ConfigurationException::class); + + $publisher->convertAndSend(BatchMessage::constructEmpty()->append('first order')); + } + public function test_batch_message_published_synchronously_from_command_handler_is_delivered(): void { $queueName = Uuid::v7()->toRfc4122(); diff --git a/packages/Ecotone/src/Messaging/Channel/Collector/CollectorSenderInterceptor.php b/packages/Ecotone/src/Messaging/Channel/Collector/CollectorSenderInterceptor.php index c4eae5a19..3564754e2 100644 --- a/packages/Ecotone/src/Messaging/Channel/Collector/CollectorSenderInterceptor.php +++ b/packages/Ecotone/src/Messaging/Channel/Collector/CollectorSenderInterceptor.php @@ -7,6 +7,8 @@ use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\Attribute\WithoutMessageCollector; use Ecotone\Messaging\BatchMessage; +use Ecotone\Messaging\Channel\BatchSupportingMessageChannel; +use Ecotone\Messaging\Channel\MessageChannelInterceptorAdapter; use Ecotone\Messaging\Config\ConfiguredMessagingSystem; use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Ecotone\Messaging\Handler\Processor\MethodInvoker\MethodInvocation; @@ -43,9 +45,17 @@ public function send( $result = $methodInvocation->proceed(); $collectedMessages = $this->collectorStorage->releaseMessages($logger, $message); if ($collectedMessages !== []) { - $this->getTargetChannel($configuredMessagingSystem)->send( - MessageBuilder::withPayload($this->combineIntoBatch($collectedMessages))->build() - ); + $messageChannel = $this->getTargetChannel($configuredMessagingSystem); + + if ($this->supportsBatchMessages($messageChannel)) { + $messageChannel->send( + MessageBuilder::withPayload($this->combineIntoBatch($collectedMessages))->build() + ); + } else { + foreach ($collectedMessages as $collectedMessage) { + $messageChannel->send($collectedMessage); + } + } } } finally { $this->collectorStorage->disable(); @@ -59,6 +69,15 @@ private function getTargetChannel(ConfiguredMessagingSystem $configuredMessaging return $configuredMessagingSystem->getMessageChannelByName($this->targetChannel); } + private function supportsBatchMessages(MessageChannel $messageChannel): bool + { + if ($messageChannel instanceof MessageChannelInterceptorAdapter) { + $messageChannel = $messageChannel->getInternalMessageChannel(); + } + + return $messageChannel instanceof BatchSupportingMessageChannel && $messageChannel->supportsBatchMessages(); + } + /** * @param Message[] $collectedMessages */ @@ -66,8 +85,17 @@ private function combineIntoBatch(array $collectedMessages): BatchMessage { $batchMessage = BatchMessage::constructEmpty(); foreach ($collectedMessages as $collectedMessage) { + $payload = $collectedMessage->getPayload(); + if ($payload instanceof BatchMessage) { + foreach ($payload->getEntries() as $entry) { + $batchMessage = $batchMessage->append($entry['payload'], $entry['headers']); + } + + continue; + } + $batchMessage = $batchMessage->append( - $collectedMessage->getPayload(), + $payload, $collectedMessage->getHeaders()->headers() ); } diff --git a/packages/Ecotone/src/Messaging/Channel/DelayableQueueChannel.php b/packages/Ecotone/src/Messaging/Channel/DelayableQueueChannel.php index ff1a5e6de..7a735ebbd 100644 --- a/packages/Ecotone/src/Messaging/Channel/DelayableQueueChannel.php +++ b/packages/Ecotone/src/Messaging/Channel/DelayableQueueChannel.php @@ -5,6 +5,7 @@ namespace Ecotone\Messaging\Channel; use DateTimeInterface; +use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Config\Container\DefinedObject; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Endpoint\PollingMetadata; @@ -13,6 +14,7 @@ use Ecotone\Messaging\PollableChannel; use Ecotone\Messaging\Scheduling\DatePoint; use Ecotone\Messaging\Scheduling\TimeSpan; +use Ecotone\Messaging\Support\LicensingException; use Ecotone\Messaging\Support\MessageBuilder; /** @@ -23,13 +25,18 @@ final class DelayableQueueChannel implements PollableChannel, DefinedObject /** * @param Message[] $queue */ - public function __construct(private string $name, private array $queue = [], private int|DateTimeInterface $releaseMessagesAwaitingFor = 0) + public function __construct(private string $name, private array $queue = [], private int|DateTimeInterface $releaseMessagesAwaitingFor = 0, private bool $batchMessagesSupport = false) { } - public static function create(string $name): self + public static function create(string $name, bool $batchMessagesSupport = false): self { - return new self($name); + return new self($name, batchMessagesSupport: $batchMessagesSupport); + } + + public function enableBatchMessagesSupport(): void + { + $this->batchMessagesSupport = true; } /** @@ -37,6 +44,21 @@ public static function create(string $name): self */ public function send(Message $message): void { + $payload = $message->getPayload(); + if ($payload instanceof BatchMessage) { + if (! $this->batchMessagesSupport) { + throw LicensingException::create('Sending BatchMessage is available only with Ecotone Enterprise licence.'); + } + + foreach ($payload->getEntries() as $entry) { + $this->queue[] = MessageBuilder::withPayload($entry['payload']) + ->setMultipleHeaders($entry['headers']) + ->build(); + } + + return; + } + $this->queue[] = $message; } @@ -97,7 +119,7 @@ public function __toString() public function getDefinition(): Definition { - return new Definition(self::class, [$this->name], 'create'); + return new Definition(self::class, [$this->name, $this->batchMessagesSupport], 'create'); } public function getCurrentDeliveryTimeShift(Message $message): int diff --git a/packages/Ecotone/src/Messaging/Channel/QueueChannel.php b/packages/Ecotone/src/Messaging/Channel/QueueChannel.php index 7f501c3ef..be534ca26 100644 --- a/packages/Ecotone/src/Messaging/Channel/QueueChannel.php +++ b/packages/Ecotone/src/Messaging/Channel/QueueChannel.php @@ -2,11 +2,14 @@ namespace Ecotone\Messaging\Channel; +use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Config\Container\DefinedObject; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Endpoint\PollingMetadata; use Ecotone\Messaging\Message; use Ecotone\Messaging\PollableChannel; +use Ecotone\Messaging\Support\LicensingException; +use Ecotone\Messaging\Support\MessageBuilder; /** * licence Apache-2.0 @@ -18,13 +21,18 @@ class QueueChannel implements PollableChannel, DefinedObject */ private array $queue = []; - public function __construct(private string $name) + public function __construct(private string $name, private bool $batchMessagesSupport = false) { } - public static function create(string $name = 'unknown'): self + public static function create(string $name = 'unknown', bool $batchMessagesSupport = false): self { - return new self($name); + return new self($name, $batchMessagesSupport); + } + + public function enableBatchMessagesSupport(): void + { + $this->batchMessagesSupport = true; } /** @@ -32,6 +40,21 @@ public static function create(string $name = 'unknown'): self */ public function send(Message $message): void { + $payload = $message->getPayload(); + if ($payload instanceof BatchMessage) { + if (! $this->batchMessagesSupport) { + throw LicensingException::create('Sending BatchMessage is available only with Ecotone Enterprise licence.'); + } + + foreach ($payload->getEntries() as $entry) { + $this->queue[] = MessageBuilder::withPayload($entry['payload']) + ->setMultipleHeaders($entry['headers']) + ->build(); + } + + return; + } + $this->queue[] = $message; } @@ -68,6 +91,6 @@ public function __toString() public function getDefinition(): Definition { - return new Definition(self::class, [$this->name]); + return new Definition(self::class, [$this->name, $this->batchMessagesSupport]); } } diff --git a/packages/Ecotone/src/Messaging/Channel/SendingInterceptorAdapter.php b/packages/Ecotone/src/Messaging/Channel/SendingInterceptorAdapter.php index f95cc0704..91a887bd4 100644 --- a/packages/Ecotone/src/Messaging/Channel/SendingInterceptorAdapter.php +++ b/packages/Ecotone/src/Messaging/Channel/SendingInterceptorAdapter.php @@ -4,11 +4,9 @@ namespace Ecotone\Messaging\Channel; -use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageChannel; use Ecotone\Messaging\Support\Assert; -use Ecotone\Messaging\Support\MessageBuilder; use Throwable; /** @@ -50,12 +48,6 @@ public function __construct(MessageChannel $messageChannel, array $sortedChannel */ public function send(Message $message): void { - if ($message->getPayload() instanceof BatchMessage && ! $this->targetChannelSupportsBatchMessages()) { - $this->sendEachMessageFromBatch($message->getPayload()); - - return; - } - $messageToSend = $message; $executedInterceptors = []; $isMessageDropped = false; @@ -110,24 +102,6 @@ public function send(Message $message): void $this->executePostSend($messageToSend, $executedInterceptors, $firstCleanupFailure); } - private function targetChannelSupportsBatchMessages(): bool - { - $targetChannel = $this->getInternalMessageChannel(); - - return $targetChannel instanceof BatchSupportingMessageChannel && $targetChannel->supportsBatchMessages(); - } - - private function sendEachMessageFromBatch(BatchMessage $batchMessage): void - { - foreach ($batchMessage->getEntries() as $entry) { - $this->send( - MessageBuilder::withPayload($entry['payload']) - ->setMultipleHeaders($entry['headers']) - ->build() - ); - } - } - /** * @param ChannelInterceptor[] $executedInterceptors */ diff --git a/packages/Ecotone/src/Messaging/Channel/SimpleMessageChannelBuilder.php b/packages/Ecotone/src/Messaging/Channel/SimpleMessageChannelBuilder.php index d8db4f5e7..0990d99d1 100644 --- a/packages/Ecotone/src/Messaging/Channel/SimpleMessageChannelBuilder.php +++ b/packages/Ecotone/src/Messaging/Channel/SimpleMessageChannelBuilder.php @@ -163,6 +163,13 @@ public function compile(MessagingContainerBuilder $builder): Definition ]); } + if ( + ($this->messageChannel instanceof QueueChannel || $this->messageChannel instanceof DelayableQueueChannel) + && $builder->getServiceConfiguration()->isRunningForEnterprise() + ) { + $this->messageChannel->enableBatchMessagesSupport(); + } + return new DefinedObjectWrapper($this->messageChannel); } diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/BatchMessageSendingTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/BatchMessageSendingTest.php index cc20a9eee..b1849fd5b 100644 --- a/packages/Ecotone/tests/Messaging/Unit/Channel/BatchMessageSendingTest.php +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/BatchMessageSendingTest.php @@ -7,7 +7,10 @@ use Ecotone\Lite\EcotoneLite; use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; +use Ecotone\Messaging\Support\LicensingException; use Ecotone\Messaging\Support\MessageBuilder; +use Ecotone\Modelling\Attribute\CommandHandler; +use Ecotone\Test\LicenceTesting; use PHPUnit\Framework\TestCase; /** @@ -22,6 +25,7 @@ public function test_batch_message_sent_to_pollable_channel_is_delivered_as_indi enableAsynchronousProcessing: [ SimpleMessageChannelBuilder::createQueueChannel('orders'), ], + licenceKey: LicenceTesting::VALID_LICENCE, ); $batch = BatchMessage::constructEmpty() @@ -41,12 +45,64 @@ public function test_batch_message_sent_to_pollable_channel_is_delivered_as_indi $this->assertNull($ecotoneLite->receiveMessageFrom('orders')); } + public function test_batch_message_sent_to_handler_output_channel_is_split_into_individual_messages(): void + { + $orderProcessor = $this->createOrderProcessor(); + $ecotoneLite = EcotoneLite::bootstrapFlowTesting( + [$orderProcessor::class], + [$orderProcessor], + enableAsynchronousProcessing: [ + SimpleMessageChannelBuilder::createQueueChannel('orders'), + ], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $ecotoneLite->sendCommandWithRoutingKey('order.placeAll', ['espresso', 'latte']); + + $this->assertSame('espresso', $ecotoneLite->receiveMessageFrom('orders')->getPayload()); + $this->assertSame('latte', $ecotoneLite->receiveMessageFrom('orders')->getPayload()); + $this->assertNull($ecotoneLite->receiveMessageFrom('orders')); + } + + public function test_batch_message_sent_to_handler_output_channel_requires_enterprise_licence(): void + { + $orderProcessor = $this->createOrderProcessor(); + $ecotoneLite = EcotoneLite::bootstrapFlowTesting( + [$orderProcessor::class], + [$orderProcessor], + enableAsynchronousProcessing: [ + SimpleMessageChannelBuilder::createQueueChannel('orders'), + ], + ); + + $this->expectException(LicensingException::class); + + $ecotoneLite->sendCommandWithRoutingKey('order.placeAll', ['espresso', 'latte']); + } + + private function createOrderProcessor(): object + { + return new class () { + #[CommandHandler('order.placeAll', outputChannelName: 'orders')] + public function placeOrders(array $orders): BatchMessage + { + $batch = BatchMessage::constructEmpty(); + foreach ($orders as $order) { + $batch = $batch->append($order); + } + + return $batch; + } + }; + } + public function test_empty_batch_message_delivers_nothing(): void { $ecotoneLite = EcotoneLite::bootstrapFlowTesting( enableAsynchronousProcessing: [ SimpleMessageChannelBuilder::createQueueChannel('orders'), ], + licenceKey: LicenceTesting::VALID_LICENCE, ); $ecotoneLite->getMessageChannel('orders')->send( diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/TestQueueChannel.php b/packages/Ecotone/tests/Messaging/Unit/Channel/TestQueueChannel.php index 1ab3c7786..05e70f1b7 100644 --- a/packages/Ecotone/tests/Messaging/Unit/Channel/TestQueueChannel.php +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/TestQueueChannel.php @@ -24,7 +24,7 @@ public function __construct(string $name = 'unknown', bool $throwException = fal $this->messageToReturn = $messageToReturn; } - public static function create(string $name = 'unknown'): self + public static function create(string $name = 'unknown', bool $batchMessagesSupport = false): self { return new self($name); } diff --git a/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php b/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php index 3ead04540..289eae2d2 100644 --- a/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php +++ b/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php @@ -9,6 +9,7 @@ use Ecotone\Messaging\Channel\AsyncPublishing\ConfirmedDelivery; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessage; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; +use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Conversion\ConversionService; use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageHandler; @@ -42,6 +43,10 @@ abstract public function initialize(): void; public function handle(Message $message): void { + if ($message->getPayload() instanceof BatchMessage && ! $this->asyncPublishing) { + throw ConfigurationException::create(sprintf('Sending BatchMessage over `%s` requires async publishing to be enabled. Enable it with withAsyncPublishing(), available as part of Ecotone Enterprise.', $this->asyncPublishingChannelName)); + } + $context = $this->createOutboundContext(); if ($message->getPayload() instanceof BatchMessage) { diff --git a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php index 29c0fb6d9..bcf0d8435 100644 --- a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php +++ b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php @@ -12,6 +12,7 @@ use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\AsyncPublishing\FailedDelivery; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; +use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Conversion\ConversionService; use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageHandler; @@ -44,6 +45,10 @@ public function __construct( */ public function handle(Message $message): void { + if ($message->getPayload() instanceof BatchMessage && ! $this->isAsyncPublishingEnabled()) { + throw ConfigurationException::create(sprintf('Sending BatchMessage over `%s` requires async publishing to be enabled. Enable it with withAsyncPublishing(), available as part of Ecotone Enterprise.', $this->referenceName)); + } + $producer = $this->kafkaAdmin->getProducer($this->referenceName); $topic = $this->kafkaAdmin->getTopicForProducer($this->referenceName); diff --git a/packages/Kafka/tests/Integration/AsyncPublishingTest.php b/packages/Kafka/tests/Integration/AsyncPublishingTest.php index 329ac18ef..ddc14cb77 100644 --- a/packages/Kafka/tests/Integration/AsyncPublishingTest.php +++ b/packages/Kafka/tests/Integration/AsyncPublishingTest.php @@ -85,6 +85,25 @@ public function test_async_publishing_requires_enterprise_licence(): void ); } + public function test_async_publishing_via_message_channel_requires_enterprise_licence(): void + { + $this->expectException(LicensingException::class); + + EcotoneLite::bootstrapFlowTesting( + [], + [KafkaBrokerConfiguration::class => ConnectionTestCase::getConnection()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::KAFKA_PACKAGE])) + ->withExtensionObjects([ + KafkaMessageChannelBuilder::create( + 'async_orders', + topicName: $uniqueId = Uuid::v7()->toRfc4122(), + messageGroupId: $uniqueId, + )->withAsyncPublishing(), + ]), + ); + } + public function test_message_publisher_async_publish_confirms_delivery_on_future_resolve(): void { $messaging = EcotoneLite::bootstrapFlowTesting( diff --git a/packages/Sqs/src/SqsOutboundChannelAdapter.php b/packages/Sqs/src/SqsOutboundChannelAdapter.php index a374870fd..c861fffef 100644 --- a/packages/Sqs/src/SqsOutboundChannelAdapter.php +++ b/packages/Sqs/src/SqsOutboundChannelAdapter.php @@ -10,6 +10,7 @@ use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; +use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Conversion\ConversionService; use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageHeaders; @@ -60,6 +61,10 @@ public function initialize(): void public function handle(Message $message): void { + if ($message->getPayload() instanceof BatchMessage && ! $this->asyncPublishing) { + throw ConfigurationException::create(sprintf('Sending BatchMessage over `%s` requires async publishing to be enabled. Enable it with withAsyncPublishing(), available as part of Ecotone Enterprise.', $this->queueName)); + } + /** @var SqsContext $context */ $context = $this->createOutboundContext(); From aab4ae1c4288bac81a6814d6350870531c2188aa Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Mon, 3 Aug 2026 08:00:00 +0200 Subject: [PATCH 34/38] fix: align TestQueueChannel with batch-aware QueueChannel signature and give reactive quickstart consumer realistic time budget --- packages/DataProtection/tests/TestQueueChannel.php | 8 ++++---- .../RefactorToReactiveSystem/run_example.php | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/DataProtection/tests/TestQueueChannel.php b/packages/DataProtection/tests/TestQueueChannel.php index e5e55743f..dff14e493 100644 --- a/packages/DataProtection/tests/TestQueueChannel.php +++ b/packages/DataProtection/tests/TestQueueChannel.php @@ -14,14 +14,14 @@ class TestQueueChannel extends QueueChannel { private ?Message $lastSentMessage = null; - public function __construct(string $name = 'unknown') + public function __construct(string $name = 'unknown', bool $batchMessagesSupport = false) { - parent::__construct($name); + parent::__construct($name, $batchMessagesSupport); } - public static function create(string $name = 'unknown'): self + public static function create(string $name = 'unknown', bool $batchMessagesSupport = false): self { - return new self($name); + return new self($name, $batchMessagesSupport); } public function send(Message $message): void diff --git a/quickstart-examples/RefactorToReactiveSystem/run_example.php b/quickstart-examples/RefactorToReactiveSystem/run_example.php index e62ef78cd..045153b7d 100644 --- a/quickstart-examples/RefactorToReactiveSystem/run_example.php +++ b/quickstart-examples/RefactorToReactiveSystem/run_example.php @@ -34,5 +34,5 @@ ]))); if ($stageToRun !== 'Stage_1') { - $messagingSystem->run("asynchronous", ExecutionPollingMetadata::createWithDefaults()->withTestingSetup(2)); + $messagingSystem->run("asynchronous", ExecutionPollingMetadata::createWithDefaults()->withTestingSetup(amountOfMessagesToHandle: 2, maxExecutionTimeInMilliseconds: 60000)); } \ No newline at end of file From 31365675d384c5d2b3669ff5f9595f2ecd7b9071 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Mon, 3 Aug 2026 08:00:00 +0200 Subject: [PATCH 35/38] test: widen timing margins in async projection and amqp delay tests for loaded ci runners --- .../Amqp/tests/Integration/AmqpChannelAdapterTest.php | 4 ++-- .../tests/InMemory/ProjectionMetadataPropagationTest.php | 8 ++++---- .../tests/Projecting/Global/MultiTenantProjectionTest.php | 2 +- .../Partitioned/AsynchronousEventDrivenProjectionTest.php | 4 ++-- .../Projecting/Partitioned/MultiTenantProjectionTest.php | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/Amqp/tests/Integration/AmqpChannelAdapterTest.php b/packages/Amqp/tests/Integration/AmqpChannelAdapterTest.php index f1caab836..28d4219af 100644 --- a/packages/Amqp/tests/Integration/AmqpChannelAdapterTest.php +++ b/packages/Amqp/tests/Integration/AmqpChannelAdapterTest.php @@ -723,13 +723,13 @@ public function test_delaying_the_message() $messageChannel = $ecotoneLite->getMessageChannelByName($queueName); $messageChannel->send( MessageBuilder::withPayload('some') - ->setHeader(MessageHeaders::DELIVERY_DELAY, 250) + ->setHeader(MessageHeaders::DELIVERY_DELAY, 1000) ->build() ); $this->assertNull($messageChannel->receiveWithTimeout(PollingMetadata::create('test')->setExecutionTimeLimitInMilliseconds(200))); - $this->assertNotNull($messageChannel->receiveWithTimeout(PollingMetadata::create('test')->setExecutionTimeLimitInMilliseconds(1000))); + $this->assertNotNull($messageChannel->receiveWithTimeout(PollingMetadata::create('test')->setExecutionTimeLimitInMilliseconds(5000))); } public function test_receiving_from_dead_letter_queue() diff --git a/packages/PdoEventSourcing/tests/InMemory/ProjectionMetadataPropagationTest.php b/packages/PdoEventSourcing/tests/InMemory/ProjectionMetadataPropagationTest.php index ffe6e6c8f..01123a90f 100644 --- a/packages/PdoEventSourcing/tests/InMemory/ProjectionMetadataPropagationTest.php +++ b/packages/PdoEventSourcing/tests/InMemory/ProjectionMetadataPropagationTest.php @@ -62,7 +62,7 @@ public function test_metadata_propagation_with_async_projection_when_catching_up $ecotoneLite->sendCommandWithRoutingKey(routingKey: 'order.create', command: 2, metadata: ['eventId' => 2]); $ecotoneLite->sendCommandWithRoutingKey(routingKey: 'order.create', command: 3, metadata: ['foo' => 'baz', 'eventId' => 3]); - $ecotoneLite->run(name: OrderProjection::CHANNEL, executionPollingMetadata: ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 15, maxExecutionTimeInMilliseconds: 1000)); + $ecotoneLite->run(name: OrderProjection::CHANNEL, executionPollingMetadata: ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 15, maxExecutionTimeInMilliseconds: 5000)); self::assertEquals(expected: 4, actual: $ecotoneLite->sendQueryWithRouting('foo_orders.count')); self::assertEquals(expected: 4, actual: $ecotoneLite->sendQueryWithRouting('getNotificationCountWithFoo')); @@ -80,19 +80,19 @@ public function test_metadata_propagation_with_async_projection_when_populated_d ); $ecotoneLite->sendCommandWithRoutingKey(routingKey: 'order.create', command: 1, metadata: ['foo' => 'bar']); - $ecotoneLite->run(name: OrderProjection::CHANNEL, executionPollingMetadata: ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 4, maxExecutionTimeInMilliseconds: 1000)); + $ecotoneLite->run(name: OrderProjection::CHANNEL, executionPollingMetadata: ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 4, maxExecutionTimeInMilliseconds: 5000)); self::assertEquals(expected: 2, actual: $ecotoneLite->sendQueryWithRouting('foo_orders.count')); self::assertEquals(expected: 2, actual: $ecotoneLite->sendQueryWithRouting('getNotificationCountWithFoo')); $ecotoneLite->sendCommandWithRoutingKey(routingKey: 'order.create', command: 2); - $ecotoneLite->run(name: OrderProjection::CHANNEL, executionPollingMetadata: ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 4, maxExecutionTimeInMilliseconds: 1000)); + $ecotoneLite->run(name: OrderProjection::CHANNEL, executionPollingMetadata: ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 4, maxExecutionTimeInMilliseconds: 5000)); self::assertEquals(expected: 2, actual: $ecotoneLite->sendQueryWithRouting('foo_orders.count')); self::assertEquals(expected: 2, actual: $ecotoneLite->sendQueryWithRouting('getNotificationCountWithFoo')); $ecotoneLite->sendCommandWithRoutingKey(routingKey: 'order.create', command: 3, metadata: ['foo' => 'baz']); - $ecotoneLite->run(name: OrderProjection::CHANNEL, executionPollingMetadata: ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 4, maxExecutionTimeInMilliseconds: 1000)); + $ecotoneLite->run(name: OrderProjection::CHANNEL, executionPollingMetadata: ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 4, maxExecutionTimeInMilliseconds: 5000)); self::assertEquals(expected: 4, actual: $ecotoneLite->sendQueryWithRouting('foo_orders.count')); self::assertEquals(expected: 4, actual: $ecotoneLite->sendQueryWithRouting('getNotificationCountWithFoo')); diff --git a/packages/PdoEventSourcing/tests/Projecting/Global/MultiTenantProjectionTest.php b/packages/PdoEventSourcing/tests/Projecting/Global/MultiTenantProjectionTest.php index df78cab02..12e9ba2e0 100644 --- a/packages/PdoEventSourcing/tests/Projecting/Global/MultiTenantProjectionTest.php +++ b/packages/PdoEventSourcing/tests/Projecting/Global/MultiTenantProjectionTest.php @@ -149,7 +149,7 @@ classesToResolve: [get_class($projection), Ticket::class, TicketEventConverter:: SimpleMessageChannelBuilder::createQueueChannel('async_projection_channel'), PollingMetadata::create('async_projection_channel') ->setExecutionAmountLimit(3) - ->setExecutionTimeLimitInMilliseconds(300), + ->setExecutionTimeLimitInMilliseconds(5000), ]), runForProductionEventStore: true, licenceKey: LicenceTesting::VALID_LICENCE, diff --git a/packages/PdoEventSourcing/tests/Projecting/Partitioned/AsynchronousEventDrivenProjectionTest.php b/packages/PdoEventSourcing/tests/Projecting/Partitioned/AsynchronousEventDrivenProjectionTest.php index 7001795b4..66b7ef68b 100644 --- a/packages/PdoEventSourcing/tests/Projecting/Partitioned/AsynchronousEventDrivenProjectionTest.php +++ b/packages/PdoEventSourcing/tests/Projecting/Partitioned/AsynchronousEventDrivenProjectionTest.php @@ -182,8 +182,8 @@ classesToResolve: [$projection::class, Ticket::class, TicketEventConverter::clas $ecotone->run($projection::CHANNEL); $finishTime = microtime(true); - // around ~300 ms as default testing setup is 100ms (however connection and set up might take longer) - self::assertLessThan(300, ($finishTime - $currentTime) * 1000); + // well below the default 1s polling timeout, proving the run does not wait for it (CI runners can be slow) + self::assertLessThan(800, ($finishTime - $currentTime) * 1000); self::assertEquals([['ticket_id' => '123', 'ticket_type' => 'alert']], $ecotone->sendQueryWithRouting('getInProgressTickets')); } diff --git a/packages/PdoEventSourcing/tests/Projecting/Partitioned/MultiTenantProjectionTest.php b/packages/PdoEventSourcing/tests/Projecting/Partitioned/MultiTenantProjectionTest.php index a8f6655dc..81a6b302d 100644 --- a/packages/PdoEventSourcing/tests/Projecting/Partitioned/MultiTenantProjectionTest.php +++ b/packages/PdoEventSourcing/tests/Projecting/Partitioned/MultiTenantProjectionTest.php @@ -148,7 +148,7 @@ classesToResolve: [get_class($projection), Ticket::class, TicketEventConverter:: SimpleMessageChannelBuilder::createQueueChannel('async_projection_channel'), PollingMetadata::create('async_projection_channel') ->setExecutionAmountLimit(3) - ->setExecutionTimeLimitInMilliseconds(300), + ->setExecutionTimeLimitInMilliseconds(5000), ]), runForProductionEventStore: true, licenceKey: LicenceTesting::VALID_LICENCE, From cf087691c5745fd38f421478d4c6a5ad15d2a61f Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Mon, 3 Aug 2026 08:00:00 +0200 Subject: [PATCH 36/38] fix: address review findings for async publishing reliability - capture amqp confirm epoch pre-publish and fail all records on reconnect so stale delivery tags can never settle against a fresh channel - redis batch publishing reports exactly which entries were not pushed so retries do not duplicate already delivered ones - dead letter each failed batch delivery as separate message in send retries exhaustion path - rename AsyncPublishingFailedException to PublishingFailedException as it serves sync and async publishing - make BatchMessage immutable with fromEntries bulk construction - raise sqs publishing timeout default to 25s as pre-batching sends had no http timeout --- .../Amqp/src/AmqpOutboundChannelAdapter.php | 60 +++++----- packages/Amqp/src/AmqpPendingDelivery.php | 22 +++- .../AsyncPublishingReliabilityTest.php | 20 ++-- .../tests/Integration/AsyncPublishingTest.php | 4 +- .../AmqpPendingDeliveryStaleEpochTest.php | 43 +++++++ .../AsyncPublishingTestChannel.php | 4 +- .../tests/Integration/AsyncPublishingTest.php | 4 +- .../AsyncPublishingTransactionTest.php | 4 +- .../Ecotone/src/Messaging/BatchMessage.php | 16 ++- .../AsyncPublishingGateway.php | 4 +- .../AsyncPublishingWaiterInterceptor.php | 4 +- .../AsyncPublishing/DeliveryFuture.php | 8 +- ...tion.php => PublishingFailedException.php} | 4 +- .../Collector/CollectorSenderInterceptor.php | 11 +- .../SendRetryChannelInterceptor.php | 52 +++++--- .../InMemoryAsyncPublishingChannel.php | 4 +- .../tests/Messaging/Unit/BatchMessageTest.php | 26 ++++ .../AsyncPublishingChannelTest.php | 6 +- .../AsyncPublishingReliabilityTest.php | 12 +- .../DeadLetterOfFailedBatchDeliveriesTest.php | 113 ++++++++++++++++++ .../MessagePublisherAsyncPublishTest.php | 6 +- .../SendRetryOfFailedBatchDeliveriesTest.php | 8 +- .../Outbound/KafkaOutboundChannelAdapter.php | 4 +- .../AsyncPublishingReliabilityTest.php | 6 +- .../tests/Integration/AsyncPublishingTest.php | 4 +- .../Redis/src/RedisOutboundChannelAdapter.php | 35 +++++- .../AsyncPublishingReliabilityTest.php | 96 +++++++++++++++ .../tests/Integration/AsyncPublishingTest.php | 4 +- .../Sqs/src/SqsOutboundChannelAdapter.php | 4 +- .../src/SqsOutboundChannelAdapterBuilder.php | 2 +- .../AsyncPublishingReliabilityTest.php | 8 +- .../tests/Integration/AsyncPublishingTest.php | 4 +- 32 files changed, 473 insertions(+), 129 deletions(-) create mode 100644 packages/Amqp/tests/Unit/AmqpPendingDeliveryStaleEpochTest.php rename packages/Ecotone/src/Messaging/Channel/AsyncPublishing/{AsyncPublishingFailedException.php => PublishingFailedException.php} (89%) create mode 100644 packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/DeadLetterOfFailedBatchDeliveriesTest.php create mode 100644 packages/Redis/tests/Integration/AsyncPublishingReliabilityTest.php diff --git a/packages/Amqp/src/AmqpOutboundChannelAdapter.php b/packages/Amqp/src/AmqpOutboundChannelAdapter.php index a16b06605..5bbb01105 100644 --- a/packages/Amqp/src/AmqpOutboundChannelAdapter.php +++ b/packages/Amqp/src/AmqpOutboundChannelAdapter.php @@ -7,9 +7,9 @@ use Ecotone\Amqp\Transaction\AmqpTransactionInterceptor; use Ecotone\Enqueue\CachedConnectionFactory; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\AsyncPublishing\FailedDelivery; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Conversion\ConversionService; @@ -20,6 +20,7 @@ use Enqueue\AmqpExt\AmqpContext as AmqpExtContext; use Enqueue\AmqpLib\AmqpContext as AmqpLibContext; use Enqueue\AmqpTools\DelayStrategy; +use Interop\Amqp\AmqpContext as InteropAmqpContext; use Interop\Amqp\AmqpMessage; use Interop\Amqp\Impl\AmqpTopic; use PhpAmqpLib\Message\AMQPMessage as LibAMQPMessage; @@ -82,15 +83,19 @@ public function handle(Message $message): void return; } - $publishRecords = $this->publishMessages($messagesToPublish); + $context = $this->connectionFactory->createContext(); + $confirmations = $this->getPublisherConfirmations(); + $prePublishConfirmationsEpoch = $confirmations?->getEpoch() ?? 0; + + $publishRecords = $this->publishMessages($messagesToPublish, $context, $confirmations); - if ($publishRecords !== [] && $this->canPublishAsynchronously()) { - $this->registerPendingDelivery($publishRecords); + if ($publishRecords !== [] && $confirmations !== null && $this->canPublishAsynchronously()) { + $this->registerPendingDelivery($publishRecords, $context, $confirmations, $prePublishConfirmationsEpoch); return; } - $this->awaitPublisherConfirmsSynchronously($publishRecords); + $this->awaitPublisherConfirmsSynchronously($publishRecords, $context, $confirmations, $prePublishConfirmationsEpoch); } public function isAsyncPublishingEnabled(): bool @@ -102,16 +107,15 @@ public function isAsyncPublishingEnabled(): bool * @param Message[] $messages * @return array */ - private function publishMessages(array $messages): array + private function publishMessages(array $messages, InteropAmqpContext $context, ?AmqpPublisherConfirmations $confirmations): array { - $context = $this->connectionFactory->createContext(); if ($context instanceof AmqpLibContext) { - return $this->publishThroughSingleBatchWrite($messages, $context); + return $this->publishThroughSingleBatchWrite($messages, $context, $confirmations); } $publishRecords = []; foreach ($messages as $message) { - $publishRecords[] = $this->publish($message); + $publishRecords[] = $this->publish($message, $context, $confirmations); } return array_values(array_filter($publishRecords)); @@ -121,7 +125,7 @@ private function publishMessages(array $messages): array * @param Message[] $messages * @return array */ - private function publishThroughSingleBatchWrite(array $messages, AmqpLibContext $context): array + private function publishThroughSingleBatchWrite(array $messages, AmqpLibContext $context, ?AmqpPublisherConfirmations $confirmations): array { $preparedEntries = []; $delayedMessages = []; @@ -139,7 +143,7 @@ private function publishThroughSingleBatchWrite(array $messages, AmqpLibContext $publishRecords = []; foreach ($delayedMessages as $delayedMessage) { - $publishRecords[] = $this->publish($delayedMessage); + $publishRecords[] = $this->publish($delayedMessage, $context, $confirmations); } $libChannel = $context->getLibChannel(); @@ -150,7 +154,7 @@ private function publishThroughSingleBatchWrite(array $messages, AmqpLibContext $interopMessage->getRoutingKey() ?? '', mandatory: (bool) ($interopMessage->getFlags() & AmqpMessage::FLAG_MANDATORY), ); - $publishRecords[] = $this->recordPublishedMessage($message, $interopMessage); + $publishRecords[] = $this->recordPublishedMessage($message, $interopMessage, $context, $confirmations); } if ($preparedEntries !== []) { @@ -173,7 +177,7 @@ private function convertToLibMessage(AmqpMessage $interopMessage): LibAMQPMessag /** * @return array{message: Message, deliveryTag: int, correlationId: string}|null */ - private function publish(Message $message): ?array + private function publish(Message $message, InteropAmqpContext $context, ?AmqpPublisherConfirmations $confirmations): ?array { [$messageToSend, $exchangeName, $deliveryDelay, $timeToLive] = $this->prepareInteropMessage($message); @@ -184,25 +188,20 @@ private function publish(Message $message): ?array // this allow for having queue per delay instead of queue per delay + exchangeName ->send(new AmqpTopic($exchangeName), $messageToSend); - return $this->recordPublishedMessage($message, $messageToSend); + return $this->recordPublishedMessage($message, $messageToSend, $context, $confirmations); } /** * @return array{message: Message, deliveryTag: int, correlationId: string}|null */ - private function recordPublishedMessage(Message $message, AmqpMessage $interopMessage): ?array + private function recordPublishedMessage(Message $message, AmqpMessage $interopMessage, InteropAmqpContext $context, ?AmqpPublisherConfirmations $confirmations): ?array { - if (! $this->publisherConfirms) { - return null; - } - - $confirmations = $this->getPublisherConfirmations(); - if ($confirmations === null) { + if (! $this->publisherConfirms || $confirmations === null) { return null; } $correlationId = (string) $interopMessage->getProperty(AmqpPublisherConfirmations::PUBLISH_BATCH_ID_PROPERTY, ''); - $resolveTagThroughCorrelation = $this->connectionFactory->createContext() instanceof AmqpLibContext; + $resolveTagThroughCorrelation = $context instanceof AmqpLibContext; return [ 'message' => $message, @@ -278,16 +277,17 @@ private function canPublishAsynchronously(): bool /** * @param array $publishRecords */ - private function registerPendingDelivery(array $publishRecords): void + private function registerPendingDelivery(array $publishRecords, InteropAmqpContext $context, AmqpPublisherConfirmations $confirmations, int $prePublishConfirmationsEpoch): void { $this->asyncPublishingRegistry->register( $this->channelName, new AmqpPendingDelivery( - $this->connectionFactory->createContext(), + $context, $publishRecords, $this->asyncPublishingTimeout, $this->channelName, - $this->getPublisherConfirmations(), + $confirmations, + $prePublishConfirmationsEpoch, ), ); } @@ -295,14 +295,13 @@ private function registerPendingDelivery(array $publishRecords): void /** * @param array $publishRecords */ - private function awaitPublisherConfirmsSynchronously(array $publishRecords): void + private function awaitPublisherConfirmsSynchronously(array $publishRecords, InteropAmqpContext $context, ?AmqpPublisherConfirmations $confirmations, int $prePublishConfirmationsEpoch): void { if (! $this->publisherConfirms || $this->amqpTransactionInterceptor->isRunningInTransaction()) { return; } - $context = $this->connectionFactory->createContext(); - if ($publishRecords === []) { + if ($publishRecords === [] || $confirmations === null) { $timeoutInSeconds = $this->asyncPublishingTimeout / 1000; if ($context instanceof AmqpLibContext) { $context->getLibChannel()->wait_for_pending_acks_returns($timeoutInSeconds); @@ -318,7 +317,8 @@ private function awaitPublisherConfirmsSynchronously(array $publishRecords): voi $publishRecords, $this->asyncPublishingTimeout, $this->channelName, - $this->getPublisherConfirmations(), + $confirmations, + $prePublishConfirmationsEpoch, ))->awaitDelivery(); if ($deliveryResult->isSuccessful()) { @@ -326,7 +326,7 @@ private function awaitPublisherConfirmsSynchronously(array $publishRecords): voi } if ($this->asyncPublishing) { - throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + throw PublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); } throw new RuntimeException(implode('; ', array_unique(array_map( diff --git a/packages/Amqp/src/AmqpPendingDelivery.php b/packages/Amqp/src/AmqpPendingDelivery.php index 5c04e7b7a..4c010a9a0 100644 --- a/packages/Amqp/src/AmqpPendingDelivery.php +++ b/packages/Amqp/src/AmqpPendingDelivery.php @@ -27,8 +27,6 @@ final class AmqpPendingDelivery implements PendingDelivery private ?DeliveryResult $deliveryResult = null; - private int $confirmationsEpoch; - /** * @param array $publishRecords */ @@ -38,8 +36,8 @@ public function __construct( private int $timeoutInMilliseconds, private string $channelName, private AmqpPublisherConfirmations $confirmations, + private int $confirmationsEpoch, ) { - $this->confirmationsEpoch = $confirmations->getEpoch(); } public function awaitDelivery(): DeliveryResult @@ -52,10 +50,12 @@ public function awaitDelivery(): DeliveryResult $deadline = microtime(true) + $this->timeoutInMilliseconds / 1000; $unsettledFailureReason = self::TIMED_OUT_FAILURE_REASON; - while (! $this->allRecordsSettled()) { + while (true) { if ($this->confirmations->getEpoch() !== $this->confirmationsEpoch) { - $unsettledFailureReason = self::CONNECTION_RESET_FAILURE_REASON; + return $this->deliveryResult = $this->failAllRecords(self::CONNECTION_RESET_FAILURE_REASON); + } + if ($this->allRecordsSettled()) { break; } @@ -78,6 +78,18 @@ public function awaitDelivery(): DeliveryResult return $this->deliveryResult = $this->collectResult($unsettledFailureReason); } + private function failAllRecords(string $failureReason): DeliveryResult + { + if ($this->publishRecords === []) { + return DeliveryResult::successful(); + } + + return DeliveryResult::withFailedDeliveries(array_map( + fn (array $publishRecord): FailedDelivery => new FailedDelivery($publishRecord['message'], $failureReason, $this->channelName), + $this->publishRecords, + )); + } + public function isAwaited(): bool { return $this->awaited; diff --git a/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php index b8b9cb4a0..c566efbfa 100644 --- a/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php +++ b/packages/Amqp/tests/Integration/AsyncPublishingReliabilityTest.php @@ -8,7 +8,7 @@ use Ecotone\Amqp\Publisher\AmqpMessagePublisherConfiguration; use Ecotone\Lite\EcotoneLite; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\MessagePublisher; @@ -31,7 +31,7 @@ public function test_nacked_message_fails_delivery_confirmation_over_amqp_lib(): $queueName = $this->declareQueueRejectingOverflow($libConnectionFactory); $publisher = $this->bootstrapPublisher($libConnectionFactory, $queueName); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $publisher->asyncPublish( BatchMessage::constructEmpty() @@ -46,7 +46,7 @@ public function test_nacked_message_fails_delivery_confirmation_over_amqp_ext(): $queueName = $this->declareQueueRejectingOverflow($extConnectionFactory); $publisher = $this->bootstrapPublisher($extConnectionFactory, $queueName); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $publisher->asyncPublish( BatchMessage::constructEmpty() @@ -90,7 +90,7 @@ public function test_unroutable_message_fails_delivery_confirmation_over_amqp_li $libConnectionFactory = new AmqpLibConnection(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']); $publisher = $this->bootstrapPublisher($libConnectionFactory, Uuid::v7()->toRfc4122()); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $publisher->asyncPublish('order that routes nowhere')->resolve(); } @@ -100,7 +100,7 @@ public function test_unroutable_message_fails_delivery_confirmation_over_amqp_ex $extConnectionFactory = new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f']); $publisher = $this->bootstrapPublisher($extConnectionFactory, Uuid::v7()->toRfc4122()); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $publisher->asyncPublish('order that routes nowhere')->resolve(); } @@ -116,7 +116,7 @@ public function test_each_future_reports_outcome_of_its_own_message_when_sharing $routableFuture->resolve(); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $unroutableFuture->resolve(); } @@ -132,7 +132,7 @@ public function test_each_future_reports_outcome_of_its_own_message_when_sharing $routableFuture->resolve(); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $unroutableFuture->resolve(); } @@ -151,7 +151,7 @@ public function test_nack_arriving_during_other_future_await_fails_only_nacked_f $deliveredFuture->resolve(); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $nackedFuture->resolve(); } @@ -170,7 +170,7 @@ public function test_nack_arriving_during_other_future_await_fails_only_nacked_f $deliveredFuture->resolve(); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $nackedFuture->resolve(); } @@ -191,7 +191,7 @@ public function test_only_failing_message_from_batch_is_reported_with_per_messag try { $future->resolve(); $this->fail('Expected unroutable batch entry to fail the delivery'); - } catch (AsyncPublishingFailedException $exception) { + } catch (PublishingFailedException $exception) { $failedDeliveries = $exception->getFailedDeliveries(); $this->assertCount(1, $failedDeliveries); $this->assertSame('order that routes nowhere', $failedDeliveries[0]->getMessage()->getPayload()); diff --git a/packages/Amqp/tests/Integration/AsyncPublishingTest.php b/packages/Amqp/tests/Integration/AsyncPublishingTest.php index 2489e6cd5..799b4b32d 100644 --- a/packages/Amqp/tests/Integration/AsyncPublishingTest.php +++ b/packages/Amqp/tests/Integration/AsyncPublishingTest.php @@ -11,7 +11,7 @@ use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; @@ -100,7 +100,7 @@ public function test_async_publish_on_publisher_without_async_configuration_thro $publishFailed = false; try { $publisher->asyncPublish('order that must not be published'); - } catch (AsyncPublishingFailedException) { + } catch (PublishingFailedException) { $publishFailed = true; } diff --git a/packages/Amqp/tests/Unit/AmqpPendingDeliveryStaleEpochTest.php b/packages/Amqp/tests/Unit/AmqpPendingDeliveryStaleEpochTest.php new file mode 100644 index 000000000..9e28dc5f2 --- /dev/null +++ b/packages/Amqp/tests/Unit/AmqpPendingDeliveryStaleEpochTest.php @@ -0,0 +1,43 @@ +getEpoch(); + $deliveryTag = $confirmations->recordPublishedMessage(); + + $confirmations->reset(); + $freshChannelDeliveryTag = $confirmations->recordPublishedMessage(); + $confirmations->recordConfirmation($freshChannelDeliveryTag, multiple: true); + + $pendingDelivery = new AmqpPendingDelivery( + $this->createStub(AmqpContext::class), + [['message' => MessageBuilder::withPayload('order published before reconnect')->build(), 'deliveryTag' => $deliveryTag, 'correlationId' => '']], + timeoutInMilliseconds: 1000, + channelName: 'orders', + confirmations: $confirmations, + confirmationsEpoch: $prePublishEpoch, + ); + + $deliveryResult = $pendingDelivery->awaitDelivery(); + + $this->assertFalse($deliveryResult->isSuccessful()); + $this->assertStringContainsString('connection was reset', $deliveryResult->getFailedDeliveries()[0]->getFailureReason()); + } +} diff --git a/packages/Dbal/tests/Fixture/AsyncPublishing/AsyncPublishingTestChannel.php b/packages/Dbal/tests/Fixture/AsyncPublishing/AsyncPublishingTestChannel.php index cbe5a9632..ccfad310a 100644 --- a/packages/Dbal/tests/Fixture/AsyncPublishing/AsyncPublishingTestChannel.php +++ b/packages/Dbal/tests/Fixture/AsyncPublishing/AsyncPublishingTestChannel.php @@ -5,8 +5,8 @@ namespace Test\Ecotone\Dbal\Fixture\AsyncPublishing; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Channel\BatchSupportingMessageChannel; use Ecotone\Messaging\Endpoint\PollingMetadata; use Ecotone\Messaging\Message; @@ -47,7 +47,7 @@ public function send(Message $message): void if (! $this->asyncPublishingRegistry->isScopeActive()) { $deliveryResult = $pendingDelivery->awaitDelivery(); if (! $deliveryResult->isSuccessful()) { - throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + throw PublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); } return; diff --git a/packages/Dbal/tests/Integration/AsyncPublishingTest.php b/packages/Dbal/tests/Integration/AsyncPublishingTest.php index 2a5921b4b..ad13337cd 100644 --- a/packages/Dbal/tests/Integration/AsyncPublishingTest.php +++ b/packages/Dbal/tests/Integration/AsyncPublishingTest.php @@ -11,7 +11,7 @@ use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; @@ -91,7 +91,7 @@ public function test_async_publish_on_publisher_without_async_configuration_thro $publishFailed = false; try { $publisher->asyncPublish('order that must not be published'); - } catch (AsyncPublishingFailedException) { + } catch (PublishingFailedException) { $publishFailed = true; } diff --git a/packages/Dbal/tests/Integration/AsyncPublishingTransactionTest.php b/packages/Dbal/tests/Integration/AsyncPublishingTransactionTest.php index 7843e3d7b..71b962ad1 100644 --- a/packages/Dbal/tests/Integration/AsyncPublishingTransactionTest.php +++ b/packages/Dbal/tests/Integration/AsyncPublishingTransactionTest.php @@ -7,7 +7,7 @@ use Ecotone\Dbal\Configuration\DbalConfiguration; use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Channel\PollableChannel\GlobalPollableChannelConfiguration; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; use Ecotone\Messaging\Config\ModulePackageList; @@ -50,7 +50,7 @@ public function test_failed_delivery_confirmation_rolls_back_database_transactio $deliveryFailed = false; try { $ecotoneLite->sendCommand(new RegisterPerson(100, 'Johny')); - } catch (AsyncPublishingFailedException) { + } catch (PublishingFailedException) { $deliveryFailed = true; } $this->assertTrue($deliveryFailed); diff --git a/packages/Ecotone/src/Messaging/BatchMessage.php b/packages/Ecotone/src/Messaging/BatchMessage.php index 6e9f44ecf..fd4163b80 100644 --- a/packages/Ecotone/src/Messaging/BatchMessage.php +++ b/packages/Ecotone/src/Messaging/BatchMessage.php @@ -23,14 +23,26 @@ public static function constructEmpty(): self return new self(); } + /** + * @param array}> $entries + */ + public static function fromEntries(array $entries): self + { + $batchMessage = new self(); + $batchMessage->entries = array_values($entries); + + return $batchMessage; + } + /** * @param array $headers */ public function append(mixed $payload, array $headers = []): self { - $this->entries[] = ['payload' => $payload, 'headers' => $headers]; + $appended = clone $this; + $appended->entries[] = ['payload' => $payload, 'headers' => $headers]; - return $this; + return $appended; } /** diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php index f2c8c8515..d89645210 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingGateway.php @@ -27,7 +27,7 @@ public function __construct( public function publish(Message $message): Future { if (! $this->asyncPublishingEnabled) { - throw AsyncPublishingFailedException::publisherNotConfiguredForAsyncPublishing($this->publisherReference); + throw PublishingFailedException::publisherNotConfiguredForAsyncPublishing($this->publisherReference); } $payload = $message->getPayload(); @@ -61,7 +61,7 @@ public function publish(Message $message): Future } if ($pendingDeliveries === []) { - throw AsyncPublishingFailedException::publisherNotConfiguredForAsyncPublishing($this->publisherReference); + throw PublishingFailedException::publisherNotConfiguredForAsyncPublishing($this->publisherReference); } return DeliveryFuture::forPendingDeliveries($pendingDeliveries); diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php index 8021ade5a..6de7aca01 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingWaiterInterceptor.php @@ -45,7 +45,7 @@ public function await(MethodInvocation $methodInvocation): mixed $errorChannelDeliveryResult = $this->asyncPublishingRegistry->awaitAll(); $remainingFailedDeliveries = array_merge($unroutedFailedDeliveries, $errorChannelDeliveryResult->getFailedDeliveries()); if ($remainingFailedDeliveries !== []) { - throw AsyncPublishingFailedException::withFailedDeliveries($remainingFailedDeliveries); + throw PublishingFailedException::withFailedDeliveries($remainingFailedDeliveries); } } } finally { @@ -76,7 +76,7 @@ private function handleFailedDeliveries(array $failedDeliveries): array foreach ($this->unpackFailedMessages($failedDelivery->getMessage()) as $failedMessage) { $this->errorChannelService->handle( $failedMessage, - AsyncPublishingFailedException::withFailedDeliveries([$failedDelivery]), + PublishingFailedException::withFailedDeliveries([$failedDelivery]), $this->configuredMessagingSystem->getMessageChannelByName($errorChannelName), $failedDelivery->getChannelName(), ); diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php index 549672992..e73f0dfae 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/DeliveryFuture.php @@ -14,7 +14,7 @@ final class DeliveryFuture implements Future { private bool $resolved = false; - private ?AsyncPublishingFailedException $failure = null; + private ?PublishingFailedException $failure = null; /** * @param PendingDelivery[] $pendingDeliveries @@ -59,15 +59,15 @@ public function resolve() } if ($awaitFailure !== null) { - $this->failure = $awaitFailure instanceof AsyncPublishingFailedException && $failedDeliveries === [] + $this->failure = $awaitFailure instanceof PublishingFailedException && $failedDeliveries === [] ? $awaitFailure - : new AsyncPublishingFailedException(sprintf('Awaiting delivery confirmation failed: %s', $awaitFailure->getMessage()), 0, $awaitFailure); + : new PublishingFailedException(sprintf('Awaiting delivery confirmation failed: %s', $awaitFailure->getMessage()), 0, $awaitFailure); throw $this->failure; } if ($failedDeliveries !== []) { - $this->failure = AsyncPublishingFailedException::withFailedDeliveries($failedDeliveries); + $this->failure = PublishingFailedException::withFailedDeliveries($failedDeliveries); throw $this->failure; } diff --git a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingFailedException.php b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/PublishingFailedException.php similarity index 89% rename from packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingFailedException.php rename to packages/Ecotone/src/Messaging/Channel/AsyncPublishing/PublishingFailedException.php index 9e4b03163..ddd84d828 100644 --- a/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/AsyncPublishingFailedException.php +++ b/packages/Ecotone/src/Messaging/Channel/AsyncPublishing/PublishingFailedException.php @@ -9,7 +9,7 @@ /** * licence Enterprise */ -final class AsyncPublishingFailedException extends MessagingException +final class PublishingFailedException extends MessagingException { /** * @param FailedDelivery[] $failedDeliveries @@ -22,7 +22,7 @@ public static function withFailedDeliveries(array $failedDeliveries): self ); $exception = new self(sprintf( - 'Failed to deliver %d asynchronously published message(s): %s', + 'Failed to deliver %d published message(s): %s', count($failedDeliveries), implode('; ', array_unique($failureReasons)), )); diff --git a/packages/Ecotone/src/Messaging/Channel/Collector/CollectorSenderInterceptor.php b/packages/Ecotone/src/Messaging/Channel/Collector/CollectorSenderInterceptor.php index 3564754e2..6c2ae5803 100644 --- a/packages/Ecotone/src/Messaging/Channel/Collector/CollectorSenderInterceptor.php +++ b/packages/Ecotone/src/Messaging/Channel/Collector/CollectorSenderInterceptor.php @@ -83,23 +83,20 @@ private function supportsBatchMessages(MessageChannel $messageChannel): bool */ private function combineIntoBatch(array $collectedMessages): BatchMessage { - $batchMessage = BatchMessage::constructEmpty(); + $entries = []; foreach ($collectedMessages as $collectedMessage) { $payload = $collectedMessage->getPayload(); if ($payload instanceof BatchMessage) { foreach ($payload->getEntries() as $entry) { - $batchMessage = $batchMessage->append($entry['payload'], $entry['headers']); + $entries[] = $entry; } continue; } - $batchMessage = $batchMessage->append( - $payload, - $collectedMessage->getHeaders()->headers() - ); + $entries[] = ['payload' => $payload, 'headers' => $collectedMessage->getHeaders()->headers()]; } - return $batchMessage; + return BatchMessage::fromEntries($entries); } } diff --git a/packages/Ecotone/src/Messaging/Channel/PollableChannel/SendRetries/SendRetryChannelInterceptor.php b/packages/Ecotone/src/Messaging/Channel/PollableChannel/SendRetries/SendRetryChannelInterceptor.php index 2167d46b2..bf27eb9fb 100644 --- a/packages/Ecotone/src/Messaging/Channel/PollableChannel/SendRetries/SendRetryChannelInterceptor.php +++ b/packages/Ecotone/src/Messaging/Channel/PollableChannel/SendRetries/SendRetryChannelInterceptor.php @@ -6,7 +6,8 @@ use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AbstractChannelInterceptor; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Channel\AsyncPublishing\FailedDelivery; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Channel\ChannelInterceptor; use Ecotone\Messaging\Config\ConfiguredMessagingSystem; use Ecotone\Messaging\Handler\Gateway\ErrorChannelService; @@ -70,12 +71,15 @@ public function afterSendCompletion(Message $message, MessageChannel $messageCha ]); if ($this->deadLetterChannel !== null) { - $this->errorChannelService->handle( - $messageToRedeliver, - $exception, - $this->configuredMessagingSystem->getMessageChannelByName($this->deadLetterChannel), - $this->relatedChannel, - ); + $deadLetterChannel = $this->configuredMessagingSystem->getMessageChannelByName($this->deadLetterChannel); + foreach ($this->unpackMessages($messageToRedeliver) as $failedMessage) { + $this->errorChannelService->handle( + $failedMessage, + $exception, + $deadLetterChannel, + $this->relatedChannel, + ); + } return true; } @@ -83,9 +87,27 @@ public function afterSendCompletion(Message $message, MessageChannel $messageCha return false; } + /** + * @return Message[] + */ + private function unpackMessages(Message $message): array + { + $payload = $message->getPayload(); + if (! $payload instanceof BatchMessage) { + return [$message]; + } + + return array_map( + static fn (array $entry): Message => MessageBuilder::withPayload($entry['payload']) + ->setMultipleHeaders($entry['headers']) + ->build(), + $payload->getEntries(), + ); + } + private function messageContainingOnlyFailedDeliveries(Message $message, Throwable $exception): Message { - if (! $exception instanceof AsyncPublishingFailedException || ! $message->getPayload() instanceof BatchMessage) { + if (! $exception instanceof PublishingFailedException || ! $message->getPayload() instanceof BatchMessage) { return $message; } @@ -94,13 +116,13 @@ private function messageContainingOnlyFailedDeliveries(Message $message, Throwab return $message; } - $batchOfFailedDeliveries = BatchMessage::constructEmpty(); - foreach ($failedDeliveries as $failedDelivery) { - $batchOfFailedDeliveries = $batchOfFailedDeliveries->append( - $failedDelivery->getMessage()->getPayload(), - $failedDelivery->getMessage()->getHeaders()->headers(), - ); - } + $batchOfFailedDeliveries = BatchMessage::fromEntries(array_map( + static fn (FailedDelivery $failedDelivery): array => [ + 'payload' => $failedDelivery->getMessage()->getPayload(), + 'headers' => $failedDelivery->getMessage()->getHeaders()->headers(), + ], + $failedDeliveries, + )); return MessageBuilder::fromMessage($message) ->setPayload($batchOfFailedDeliveries) diff --git a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php index 8a9bbbbd6..f4ec8cb11 100644 --- a/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php +++ b/packages/Ecotone/tests/Messaging/Fixture/AsyncPublishing/InMemoryAsyncPublishingChannel.php @@ -5,8 +5,8 @@ namespace Test\Ecotone\Messaging\Fixture\AsyncPublishing; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Channel\BatchSupportingMessageChannel; use Ecotone\Messaging\Endpoint\PollingMetadata; use Ecotone\Messaging\Message; @@ -59,7 +59,7 @@ public function send(Message $message): void if (! $this->asyncPublishingRegistry->isScopeActive()) { $deliveryResult = $pendingDelivery->awaitDelivery(); if (! $deliveryResult->isSuccessful()) { - throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + throw PublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); } return; diff --git a/packages/Ecotone/tests/Messaging/Unit/BatchMessageTest.php b/packages/Ecotone/tests/Messaging/Unit/BatchMessageTest.php index d6748e263..cfb232165 100644 --- a/packages/Ecotone/tests/Messaging/Unit/BatchMessageTest.php +++ b/packages/Ecotone/tests/Messaging/Unit/BatchMessageTest.php @@ -36,4 +36,30 @@ public function test_counting_appended_messages(): void $this->assertCount(0, BatchMessage::constructEmpty()); $this->assertCount(2, BatchMessage::constructEmpty()->append('one')->append('two')); } + + public function test_appending_returns_new_instance_keeping_original_untouched(): void + { + $original = BatchMessage::constructEmpty()->append('first payload'); + + $extended = $original->append('second payload'); + + $this->assertCount(1, $original); + $this->assertCount(2, $extended); + } + + public function test_constructing_from_entries(): void + { + $batch = BatchMessage::fromEntries([ + ['payload' => 'first payload', 'headers' => []], + ['payload' => 'second payload', 'headers' => ['priority' => 5]], + ]); + + $this->assertSame( + [ + ['payload' => 'first payload', 'headers' => []], + ['payload' => 'second payload', 'headers' => ['priority' => 5]], + ], + $batch->getEntries(), + ); + } } diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingChannelTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingChannelTest.php index d0ac80717..fc87f03da 100644 --- a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingChannelTest.php +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingChannelTest.php @@ -6,7 +6,7 @@ use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Channel\MessageChannelInterceptorAdapter; use PHPUnit\Framework\TestCase; use Test\Ecotone\Messaging\Fixture\AsyncPublishing\AsyncOrderSubscriber; @@ -51,11 +51,11 @@ public function test_failed_delivery_confirmation_fails_command_execution_and_ro $commandException = null; try { $ecotoneLite->sendCommandWithRoutingKey('order.place', 'espresso'); - } catch (AsyncPublishingFailedException $exception) { + } catch (PublishingFailedException $exception) { $commandException = $exception; } - $this->assertInstanceOf(AsyncPublishingFailedException::class, $commandException); + $this->assertInstanceOf(PublishingFailedException::class, $commandException); $this->assertStringContainsString('broker not available', $commandException->getMessage()); $this->assertSame( [ diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php index 615b69ac7..dbabab250 100644 --- a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/AsyncPublishingReliabilityTest.php @@ -7,9 +7,9 @@ use Ecotone\Lite\EcotoneLite; use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\AsyncPublishing\DeliveryFuture; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Channel\MessageChannelInterceptorAdapter; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Handler\Logger\LoggingService; @@ -43,12 +43,12 @@ public function test_future_throwing_during_await_keeps_reporting_failure_on_sub $firstResolveException = null; try { $future->resolve(); - } catch (AsyncPublishingFailedException $exception) { + } catch (PublishingFailedException $exception) { $firstResolveException = $exception; } $this->assertNotNull($firstResolveException); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $future->resolve(); } @@ -105,7 +105,7 @@ public function test_future_of_delivery_flushed_by_backlog_limit_still_reports_f } $this->assertGreaterThan(0, $outboundAdapter->awaitedDeliveriesCount()); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $firstFuture->resolve(); } @@ -197,7 +197,7 @@ public function test_future_awaits_remaining_deliveries_when_earlier_delivery_th try { $future->resolve(); - } catch (AsyncPublishingFailedException) { + } catch (PublishingFailedException) { } $this->assertTrue($followingDelivery->isAwaited()); @@ -225,7 +225,7 @@ public function handle(string $order, #[Reference(InMemoryAsyncPublisherModule:: $commandFailed = false; try { $ecotoneLite->sendCommandWithRoutingKey('order.placeAllBatches', 'espresso'); - } catch (AsyncPublishingFailedException) { + } catch (PublishingFailedException) { $commandFailed = true; } diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/DeadLetterOfFailedBatchDeliveriesTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/DeadLetterOfFailedBatchDeliveriesTest.php new file mode 100644 index 000000000..426a3bf33 --- /dev/null +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/DeadLetterOfFailedBatchDeliveriesTest.php @@ -0,0 +1,113 @@ +bootstrapWithDeadLetterChannel(); + $ordersChannel = $ecotoneLite->getMessageChannel('async_orders'); + assert($ordersChannel instanceof MessageChannelInterceptorAdapter); + $ordersChannel->getInternalMessageChannel()->failDeliveriesContaining('poison', 'nacked by broker'); + + $ordersChannel->send(MessageBuilder::withPayload( + BatchMessage::constructEmpty() + ->append('first poison order') + ->append('delivered order') + ->append('second poison order') + )->build()); + + $this->assertSame( + ['first poison order', 'second poison order'], + $this->receiveAllPayloads($ecotoneLite->getMessageChannel('dead_letters')), + ); + } + + public function test_failed_async_batch_deliveries_are_stored_as_separate_dead_letters(): void + { + $operationsLog = new OperationsLog(); + $ecotoneLite = EcotoneLite::bootstrapFlowTesting( + [OrderService::class, AsyncOrderSubscriber::class], + [new OrderService($operationsLog), new AsyncOrderSubscriber(), OperationsLog::class => $operationsLog], + ServiceConfiguration::createWithDefaults()->withExtensionObjects([ + PollableChannelConfiguration::create('async_orders', RetryTemplateBuilder::fixedBackOff(1)->maxRetryAttempts(1)->build()) + ->withErrorChannel('dead_letters'), + ]), + enableAsynchronousProcessing: [ + InMemoryAsyncPublishingChannelBuilder::create('async_orders'), + SimpleMessageChannelBuilder::createQueueChannel('dead_letters'), + ], + ); + $ordersChannel = $ecotoneLite->getMessageChannel('async_orders'); + assert($ordersChannel instanceof MessageChannelInterceptorAdapter); + $ordersChannel->getInternalMessageChannel()->failDeliveriesWith('broker not available'); + + $ecotoneLite->sendCommandWithRoutingKey('order.place', 'espresso'); + + $deadLetteredPayloads = $this->receiveAllPayloads($ecotoneLite->getMessageChannel('dead_letters')); + $this->assertCount(2, $deadLetteredPayloads); + $this->assertStringContainsString('espresso-1', $deadLetteredPayloads[0]); + $this->assertStringNotContainsString('espresso-2', $deadLetteredPayloads[0]); + $this->assertStringContainsString('espresso-2', $deadLetteredPayloads[1]); + $this->assertStringNotContainsString('espresso-1', $deadLetteredPayloads[1]); + } + + private function bootstrapWithDeadLetterChannel(): \Ecotone\Lite\Test\FlowTestSupport + { + $operationsLog = new OperationsLog(); + + return EcotoneLite::bootstrapFlowTesting( + [OrderService::class], + [new OrderService($operationsLog), OperationsLog::class => $operationsLog], + ServiceConfiguration::createWithDefaults()->withExtensionObjects([ + PollableChannelConfiguration::create('async_orders', RetryTemplateBuilder::fixedBackOff(1)->maxRetryAttempts(1)->build()) + ->withErrorChannel('dead_letters'), + ]), + enableAsynchronousProcessing: [ + InMemoryAsyncPublishingChannelBuilder::create('async_orders'), + SimpleMessageChannelBuilder::createQueueChannel('dead_letters'), + ], + ); + } + + /** + * @return array + */ + private function receiveAllPayloads(PollableChannel $deadLetterChannel): array + { + $payloads = []; + while ($deadLetter = $deadLetterChannel->receive()) { + $payloads[] = $this->unwrapPayload($deadLetter); + } + + return $payloads; + } + + private function unwrapPayload(Message $deadLetter): mixed + { + return $deadLetter->getPayload(); + } +} diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/MessagePublisherAsyncPublishTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/MessagePublisherAsyncPublishTest.php index 5f96dfa38..2272b17d5 100644 --- a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/MessagePublisherAsyncPublishTest.php +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/MessagePublisherAsyncPublishTest.php @@ -6,8 +6,8 @@ use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\MessagePublisher; use PHPUnit\Framework\TestCase; use Test\Ecotone\Messaging\Fixture\AsyncPublishing\InMemoryAsyncOutboundAdapter; @@ -54,7 +54,7 @@ public function test_resolving_future_throws_when_delivery_failed(): void $future = $publisher->asyncPublish('order was placed'); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $future->resolve(); } @@ -65,7 +65,7 @@ public function test_async_publish_on_synchronous_publisher_throws_clear_excepti $outboundAdapter->actAsSynchronousPublisher(); $publisher = $this->bootstrapPublisher($outboundAdapter); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $this->expectExceptionMessageMatches('/not configured for asynchronous publishing/'); $publisher->asyncPublish('order was placed'); diff --git a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/SendRetryOfFailedBatchDeliveriesTest.php b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/SendRetryOfFailedBatchDeliveriesTest.php index 9e988444b..79040a1eb 100644 --- a/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/SendRetryOfFailedBatchDeliveriesTest.php +++ b/packages/Ecotone/tests/Messaging/Unit/Channel/AsyncPublishing/SendRetryOfFailedBatchDeliveriesTest.php @@ -5,8 +5,8 @@ namespace Test\Ecotone\Messaging\Unit\Channel\AsyncPublishing; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\FailedDelivery; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Channel\PollableChannel\SendRetries\SendRetryChannelInterceptor; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Config\ConfiguredMessagingSystem; @@ -42,7 +42,7 @@ public function test_retry_redelivers_only_failed_messages_from_batch(): void $interceptor->afterSendCompletion( $batchMessage, $channel, - AsyncPublishingFailedException::withFailedDeliveries([ + PublishingFailedException::withFailedDeliveries([ new FailedDelivery(MessageBuilder::withPayload('poison order')->build(), 'nacked by broker', 'orders'), ]), ); @@ -60,7 +60,7 @@ public function test_retry_of_single_message_failure_redelivers_original_message $interceptor->afterSendCompletion( $singleMessage, $channel, - AsyncPublishingFailedException::withFailedDeliveries([ + PublishingFailedException::withFailedDeliveries([ new FailedDelivery($singleMessage, 'nacked by broker', 'orders'), ]), ); @@ -88,7 +88,7 @@ private function createInterceptor(): SendRetryChannelInterceptor private function createRecordingChannel(): MessageChannel { - return new class implements MessageChannel { + return new class () implements MessageChannel { /** @var array> */ public array $sentBatchPayloads = []; diff --git a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php index bcf0d8435..1dedee701 100644 --- a/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php +++ b/packages/Kafka/src/Outbound/KafkaOutboundChannelAdapter.php @@ -8,9 +8,9 @@ use Ecotone\Kafka\Configuration\KafkaAdmin; use Ecotone\Kafka\Configuration\KafkaPublisherConfiguration; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; use Ecotone\Messaging\Channel\AsyncPublishing\FailedDelivery; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Conversion\ConversionService; @@ -210,7 +210,7 @@ private function flushSynchronously(Producer $producer, array $deliveryIds = []) $deliveryResult = $this->kafkaAdmin->getDeliveryTracker($this->referenceName)->collectResult($deliveryIds, $this->referenceName); if (! $deliveryResult->isSuccessful()) { if ($this->isAsyncPublishingEnabled()) { - throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + throw PublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); } throw MessagePublishingException::create(sprintf( diff --git a/packages/Kafka/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Kafka/tests/Integration/AsyncPublishingReliabilityTest.php index b0c8d94d1..c36069602 100644 --- a/packages/Kafka/tests/Integration/AsyncPublishingReliabilityTest.php +++ b/packages/Kafka/tests/Integration/AsyncPublishingReliabilityTest.php @@ -8,7 +8,7 @@ use Ecotone\Kafka\Configuration\KafkaPublisherConfiguration; use Ecotone\Kafka\Outbound\MessagePublishingException; use Ecotone\Lite\EcotoneLite; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\MessagePublisher; @@ -29,7 +29,7 @@ public function test_broker_rejected_message_fails_synchronous_fallback_of_async { $publisher = $this->bootstrapPublisher(); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $publisher->send(str_repeat('x', 2_000_000)); } @@ -40,7 +40,7 @@ public function test_broker_rejected_message_fails_async_publish_on_future_resol $future = $publisher->asyncPublish(str_repeat('x', 2_000_000)); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $future->resolve(); } diff --git a/packages/Kafka/tests/Integration/AsyncPublishingTest.php b/packages/Kafka/tests/Integration/AsyncPublishingTest.php index ddc14cb77..54fc2a094 100644 --- a/packages/Kafka/tests/Integration/AsyncPublishingTest.php +++ b/packages/Kafka/tests/Integration/AsyncPublishingTest.php @@ -14,7 +14,7 @@ use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; @@ -64,7 +64,7 @@ public function test_failing_to_deliver_asynchronously_published_messages_throws asyncPublishingTimeout: 500, ); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $messaging->sendCommandWithRoutingKey('order.place', 'espresso'); } diff --git a/packages/Redis/src/RedisOutboundChannelAdapter.php b/packages/Redis/src/RedisOutboundChannelAdapter.php index 131931709..a115f6921 100644 --- a/packages/Redis/src/RedisOutboundChannelAdapter.php +++ b/packages/Redis/src/RedisOutboundChannelAdapter.php @@ -8,6 +8,8 @@ use Ecotone\Enqueue\EnqueueOutboundChannelAdapter; use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; +use Ecotone\Messaging\Channel\AsyncPublishing\FailedDelivery; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Conversion\ConversionService; use Ecotone\Messaging\Message; @@ -28,12 +30,18 @@ final class RedisOutboundChannelAdapter extends EnqueueOutboundChannelAdapter local pushed = 0 local immediateAmount = tonumber(ARGV[1]) for argumentIndex = 2, immediateAmount + 1 do - redis.call("lpush", KEYS[1], ARGV[argumentIndex]) + local result = redis.pcall("lpush", KEYS[1], ARGV[argumentIndex]) + if type(result) == "table" and result.err then + return {pushed, result.err} + end pushed = pushed + 1 end local argumentIndex = immediateAmount + 2 while argumentIndex <= #ARGV do - redis.call("zadd", KEYS[2], ARGV[argumentIndex], ARGV[argumentIndex + 1]) + local result = redis.pcall("zadd", KEYS[2], ARGV[argumentIndex], ARGV[argumentIndex + 1]) + if type(result) == "table" and result.err then + return {pushed, result.err} + end pushed = pushed + 1 argumentIndex = argumentIndex + 2 end @@ -84,9 +92,12 @@ protected function handleBatch(BatchMessage $batchMessage, Context $context): vo /** @var RedisContext $context */ $immediatePayloads = []; + $immediateMessages = []; $delayedEntries = []; + $delayedMessages = []; foreach ($batchMessage->getEntries() as $entry) { - $outboundMessage = $this->prepareOutboundMessage($this->convertBatchEntryToMessage($entry)); + $originalMessage = $this->convertBatchEntryToMessage($entry); + $outboundMessage = $this->prepareOutboundMessage($originalMessage); $headers = $outboundMessage->getHeaders(); $headers[MessageHeaders::CONTENT_TYPE] = $outboundMessage->getContentType(); @@ -104,8 +115,10 @@ protected function handleBatch(BatchMessage $batchMessage, Context $context): vo if ($outboundMessage->getDeliveryDelay()) { $delayedEntries[] = ['score' => time() + $outboundMessage->getDeliveryDelay() / 1000, 'payload' => $payload]; + $delayedMessages[] = $originalMessage; } else { $immediatePayloads[] = $payload; + $immediateMessages[] = $originalMessage; } } @@ -133,18 +146,28 @@ protected function handleBatch(BatchMessage $batchMessage, Context $context): vo $arguments[] = $delayedEntry['payload']; } - $pushedMessages = $context->getRedis()->eval( + $batchPublishResult = $context->getRedis()->eval( self::BATCH_PUBLISH_SCRIPT, [$this->queueName, $this->queueName . ':delayed'], $arguments, ); - if ($pushedMessages !== count($batchMessage)) { + if (is_array($batchPublishResult)) { + $pushedMessages = (int) ($batchPublishResult[0] ?? 0); + $failureReason = (string) ($batchPublishResult[1] ?? 'Redis rejected publishing'); + + throw PublishingFailedException::withFailedDeliveries(array_map( + fn (Message $unpublishedMessage): FailedDelivery => new FailedDelivery($unpublishedMessage, $failureReason, $this->queueName), + array_slice([...$immediateMessages, ...$delayedMessages], $pushedMessages), + )); + } + + if ((int) $batchPublishResult !== count($batchMessage)) { throw new RuntimeException(sprintf( 'Redis did not confirm publishing whole batch to queue %s. Expected %d published messages, got %s.', $this->queueName, count($batchMessage), - var_export($pushedMessages, true), + var_export($batchPublishResult, true), )); } } diff --git a/packages/Redis/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Redis/tests/Integration/AsyncPublishingReliabilityTest.php new file mode 100644 index 000000000..9c6f57af6 --- /dev/null +++ b/packages/Redis/tests/Integration/AsyncPublishingReliabilityTest.php @@ -0,0 +1,96 @@ +redis(); + $redis->del(self::CHANNEL_NAME); + $redis->del(self::CHANNEL_NAME . ':delayed'); + } + + public function test_mid_batch_failure_reports_only_unpushed_entries_so_retry_does_not_duplicate_delivered_ones(): void + { + $this->makeDelayedStorageRejectWrites(); + $messaging = $this->bootstrapEcotoneWithRetryingChannel(); + $channel = $messaging->getMessageChannel(self::CHANNEL_NAME); + + $caughtException = null; + try { + $channel->send(MessageBuilder::withPayload( + BatchMessage::constructEmpty() + ->append('delivered order') + ->append('order for broken delayed storage', [MessageHeaders::DELIVERY_DELAY => 60000]) + )->build()); + } catch (Throwable $exception) { + $caughtException = $exception; + } + + $this->assertInstanceOf(PublishingFailedException::class, $caughtException); + $this->assertCount(1, $caughtException->getFailedDeliveries()); + $this->assertSame('order for broken delayed storage', $caughtException->getFailedDeliveries()[0]->getMessage()->getPayload()); + $this->assertSame(1, $this->queueLength()); + } + + private function makeDelayedStorageRejectWrites(): void + { + $this->redis()->lpush(self::CHANNEL_NAME . ':delayed', 'occupying delayed storage with wrong type'); + } + + private function bootstrapEcotoneWithRetryingChannel(): FlowTestSupport + { + return EcotoneLite::bootstrapFlowTesting( + [], + [RedisConnectionFactory::class => $this->getConnectionFactory()], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::REDIS_PACKAGE, ModulePackageList::ASYNCHRONOUS_PACKAGE])) + ->withExtensionObjects([ + PollableChannelConfiguration::create(self::CHANNEL_NAME, RetryTemplateBuilder::fixedBackOff(1)->maxRetryAttempts(1)->build()), + ]), + enableAsynchronousProcessing: [ + RedisBackedMessageChannelBuilder::create(self::CHANNEL_NAME)->withAsyncPublishing(), + ], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + + private function queueLength(): int + { + return (int) $this->redis()->eval('return redis.call("llen", KEYS[1])', [self::CHANNEL_NAME], []); + } + + private function redis(): \Enqueue\Redis\Redis + { + /** @var RedisContext $context */ + $context = $this->getConnectionFactory()->createContext(); + + return $context->getRedis(); + } +} diff --git a/packages/Redis/tests/Integration/AsyncPublishingTest.php b/packages/Redis/tests/Integration/AsyncPublishingTest.php index 42f9437e0..d38992d85 100644 --- a/packages/Redis/tests/Integration/AsyncPublishingTest.php +++ b/packages/Redis/tests/Integration/AsyncPublishingTest.php @@ -9,7 +9,7 @@ use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; @@ -95,7 +95,7 @@ public function test_async_publish_on_publisher_without_async_configuration_thro $publishFailed = false; try { $publisher->asyncPublish('order that must not be published'); - } catch (AsyncPublishingFailedException) { + } catch (PublishingFailedException) { $publishFailed = true; } diff --git a/packages/Sqs/src/SqsOutboundChannelAdapter.php b/packages/Sqs/src/SqsOutboundChannelAdapter.php index c861fffef..4cd75e82c 100644 --- a/packages/Sqs/src/SqsOutboundChannelAdapter.php +++ b/packages/Sqs/src/SqsOutboundChannelAdapter.php @@ -7,8 +7,8 @@ use Ecotone\Enqueue\CachedConnectionFactory; use Ecotone\Enqueue\EnqueueOutboundChannelAdapter; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingRegistry; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Channel\PollableChannel\Serialization\OutboundMessageConverter; use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Conversion\ConversionService; @@ -96,7 +96,7 @@ public function handle(Message $message): void $deliveryResult = $pendingDelivery->awaitDelivery(); if (! $deliveryResult->isSuccessful()) { - throw AsyncPublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); + throw PublishingFailedException::withFailedDeliveries($deliveryResult->getFailedDeliveries()); } } diff --git a/packages/Sqs/src/SqsOutboundChannelAdapterBuilder.php b/packages/Sqs/src/SqsOutboundChannelAdapterBuilder.php index 84b3638c7..d3b1cbe42 100644 --- a/packages/Sqs/src/SqsOutboundChannelAdapterBuilder.php +++ b/packages/Sqs/src/SqsOutboundChannelAdapterBuilder.php @@ -22,7 +22,7 @@ */ final class SqsOutboundChannelAdapterBuilder extends EnqueueOutboundChannelAdapterBuilder { - public const DEFAULT_ASYNC_PUBLISHING_TIMEOUT = 12000; + public const DEFAULT_ASYNC_PUBLISHING_TIMEOUT = 25000; private bool $asyncPublishing = false; private int $asyncPublishingTimeout = self::DEFAULT_ASYNC_PUBLISHING_TIMEOUT; diff --git a/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php index 4c3cce4f6..0e409934e 100644 --- a/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php +++ b/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php @@ -8,7 +8,7 @@ use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\MessagePublisher; @@ -38,7 +38,7 @@ public function test_broker_rejected_batch_fails_async_publish_on_future_resolve ->append(str_repeat('x', 300_000)) ); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $future->resolve(); } @@ -48,7 +48,7 @@ public function test_broker_rejected_batch_sent_without_active_scope_throws_imme $queueName = Uuid::v7()->toRfc4122(); $messaging = $this->bootstrapChannel($queueName); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $messaging->getMessageChannel($queueName)->send( MessageBuilder::withPayload( @@ -80,7 +80,7 @@ public function handle(string $order, #[Reference(MessagePublisher::class)] Mess licenceKey: LicenceTesting::VALID_LICENCE, ); - $this->expectException(AsyncPublishingFailedException::class); + $this->expectException(PublishingFailedException::class); $messaging->sendCommandWithRoutingKey('order.placeAllBatches', 'espresso'); } diff --git a/packages/Sqs/tests/Integration/AsyncPublishingTest.php b/packages/Sqs/tests/Integration/AsyncPublishingTest.php index cd5cba5ea..ae441c798 100644 --- a/packages/Sqs/tests/Integration/AsyncPublishingTest.php +++ b/packages/Sqs/tests/Integration/AsyncPublishingTest.php @@ -9,7 +9,7 @@ use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\Attribute\Parameter\Reference; use Ecotone\Messaging\BatchMessage; -use Ecotone\Messaging\Channel\AsyncPublishing\AsyncPublishingFailedException; +use Ecotone\Messaging\Channel\AsyncPublishing\PublishingFailedException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; @@ -84,7 +84,7 @@ public function test_async_publish_on_publisher_without_async_configuration_thro $publishFailed = false; try { $publisher->asyncPublish('order that must not be published'); - } catch (AsyncPublishingFailedException) { + } catch (PublishingFailedException) { $publishFailed = true; } From c2b220a568068b0b959bba568edd1202a7e59821 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Mon, 3 Aug 2026 08:00:00 +0200 Subject: [PATCH 37/38] test: bound stream channel distributed bus consumers by expected message count with generous time budget --- packages/Amqp/tests/Integration/AmqpStreamChannelTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/Amqp/tests/Integration/AmqpStreamChannelTest.php b/packages/Amqp/tests/Integration/AmqpStreamChannelTest.php index e7c345849..ec155e4f7 100644 --- a/packages/Amqp/tests/Integration/AmqpStreamChannelTest.php +++ b/packages/Amqp/tests/Integration/AmqpStreamChannelTest.php @@ -1363,8 +1363,8 @@ public function getConsumed(): array $publisherService->getDistributedBus()->publishEvent('distributed.event', 'event3'); // Both consumers should receive all events independently - $consumerService1->run($channelName, ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 10)); - $consumerService2->run($channelName, ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 10)); + $consumerService1->run($channelName, ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 3, maxExecutionTimeInMilliseconds: 10000)); + $consumerService2->run($channelName, ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 3, maxExecutionTimeInMilliseconds: 10000)); $this->assertEquals(['event1', 'event2', 'event3'], $consumerService1->getQueryBus()->sendWithRouting('getConsumed1')); $this->assertEquals(['event1', 'event2', 'event3'], $consumerService2->getQueryBus()->sendWithRouting('getConsumed2')); From f0da42e7a834f2f040af7df9baa771f7f12606ab Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 38/38] test: give kafka deduplication consumer runs realistic time budget matching existing in-file pattern --- .../KafkaConsumerDeduplicationTest.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Monorepo/CrossModuleTests/Tests/MessageConsumer/KafkaConsumerDeduplicationTest.php b/Monorepo/CrossModuleTests/Tests/MessageConsumer/KafkaConsumerDeduplicationTest.php index 822e2f6ee..a657718eb 100644 --- a/Monorepo/CrossModuleTests/Tests/MessageConsumer/KafkaConsumerDeduplicationTest.php +++ b/Monorepo/CrossModuleTests/Tests/MessageConsumer/KafkaConsumerDeduplicationTest.php @@ -150,7 +150,7 @@ public function test_deduplicating_with_default_message_id_kafka_consumer(): voi ); // Run consumer - $ecotoneLite->run('kafka_default_deduplication_consumer', ExecutionPollingMetadata::createWithTestingSetup()); + $ecotoneLite->run('kafka_default_deduplication_consumer', ExecutionPollingMetadata::createWithTestingSetup()->withExecutionTimeLimitInMilliseconds(10_000)); // Verify message processed $this->assertEquals(['test-payload-1'], $ecotoneLite->sendQueryWithRouting('kafka.getDefaultProcessedMessages')); @@ -163,7 +163,7 @@ public function test_deduplicating_with_default_message_id_kafka_consumer(): voi ); // Run consumer again - $ecotoneLite->run('kafka_default_deduplication_consumer', ExecutionPollingMetadata::createWithTestingSetup()); + $ecotoneLite->run('kafka_default_deduplication_consumer', ExecutionPollingMetadata::createWithTestingSetup()->withExecutionTimeLimitInMilliseconds(10_000)); // Verify message NOT processed again (still only one message) $this->assertEquals(['test-payload-1'], $ecotoneLite->sendQueryWithRouting('kafka.getDefaultProcessedMessages')); @@ -176,7 +176,7 @@ public function test_deduplicating_with_default_message_id_kafka_consumer(): voi ); // Run consumer - $ecotoneLite->run('kafka_default_deduplication_consumer', ExecutionPollingMetadata::createWithTestingSetup()); + $ecotoneLite->run('kafka_default_deduplication_consumer', ExecutionPollingMetadata::createWithTestingSetup()->withExecutionTimeLimitInMilliseconds(10_000)); // Verify new message IS processed $this->assertEquals(['test-payload-1', 'test-payload-2'], $ecotoneLite->sendQueryWithRouting('kafka.getDefaultProcessedMessages')); @@ -227,13 +227,13 @@ public function test_deduplication_works_independently_across_different_consumer ); // Run consumer to process first message - $ecotoneLite->run('kafka_deduplication_consumer', ExecutionPollingMetadata::createWithTestingSetup()->withHandledMessageLimit(1)); + $ecotoneLite->run('kafka_deduplication_consumer', ExecutionPollingMetadata::createWithTestingSetup()->withExecutionTimeLimitInMilliseconds(10_000)->withHandledMessageLimit(1)); // Verify first message processed $this->assertEquals(['test-payload-1'], $ecotoneLite->sendQueryWithRouting('kafka.getProcessedMessages')); // Run consumer to process second message - $ecotoneLite->run('kafka_deduplication_consumer', ExecutionPollingMetadata::createWithTestingSetup()->withHandledMessageLimit(1)); + $ecotoneLite->run('kafka_deduplication_consumer', ExecutionPollingMetadata::createWithTestingSetup()->withExecutionTimeLimitInMilliseconds(10_000)->withHandledMessageLimit(1)); // Verify both messages processed (different custom header values) $this->assertEquals(['test-payload-1', 'test-payload-2'], $ecotoneLite->sendQueryWithRouting('kafka.getProcessedMessages')); @@ -252,7 +252,7 @@ public function test_deduplication_works_independently_across_different_consumer ); // Run consumer again - $ecotoneLite->run('kafka_deduplication_consumer', ExecutionPollingMetadata::createWithTestingSetup()->withHandledMessageLimit(2)); + $ecotoneLite->run('kafka_deduplication_consumer', ExecutionPollingMetadata::createWithTestingSetup()->withExecutionTimeLimitInMilliseconds(10_000)->withHandledMessageLimit(2)); // Verify no duplicate processing occurred (still only 2 messages) $this->assertEquals(['test-payload-1', 'test-payload-2'], $ecotoneLite->sendQueryWithRouting('kafka.getProcessedMessages'));