From 6b30acdc48cdd96841d01944bec21620fae6ea94 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 01/25] refactor: rename channel builder withAsyncPublishing to withBatchedNonBlockingDelivery --- .../Benchmark/AsyncPublishingBenchmark.php | 10 +-- .../src/AmqpBackedMessageChannelBuilder.php | 2 +- .../tests/Integration/AsyncPublishingTest.php | 4 +- .../src/DbalBackedMessageChannelBuilder.php | 4 +- .../tests/Integration/AsyncPublishingTest.php | 2 +- .../CombinedChannelBatchForwardingTest.php | 79 +++++++++++++++++++ .../Channel/KafkaMessageChannelBuilder.php | 4 +- .../Kafka/src/Configuration/KafkaModule.php | 2 +- .../tests/Integration/AsyncPublishingTest.php | 4 +- .../src/RedisBackedMessageChannelBuilder.php | 4 +- .../AsyncPublishingReliabilityTest.php | 2 +- .../tests/Integration/AsyncPublishingTest.php | 2 +- .../src/SqsBackedMessageChannelBuilder.php | 4 +- .../AsyncPublishingReliabilityTest.php | 2 +- .../tests/Integration/AsyncPublishingTest.php | 2 +- 15 files changed, 103 insertions(+), 24 deletions(-) create mode 100644 packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php diff --git a/Monorepo/Benchmark/AsyncPublishingBenchmark.php b/Monorepo/Benchmark/AsyncPublishingBenchmark.php index 63a00ca13..548b4f647 100644 --- a/Monorepo/Benchmark/AsyncPublishingBenchmark.php +++ b/Monorepo/Benchmark/AsyncPublishingBenchmark.php @@ -72,7 +72,7 @@ public function setUpAmqpBatchChannel(): void { $this->batchChannel = $this->bootstrapBatchChannel( ModulePackageList::AMQP_PACKAGE, - AmqpBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withAsyncPublishing(), + AmqpBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withBatchedNonBlockingDelivery(), [AmqpConnectionFactory::class => new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f'])], ); $this->warmUpBatchChannel(); @@ -95,7 +95,7 @@ public function setUpKafkaBatchChannel(): void $uniqueId = uniqid('benchmark_orders_'); $this->batchChannel = $this->bootstrapBatchChannel( ModulePackageList::KAFKA_PACKAGE, - KafkaMessageChannelBuilder::create($uniqueId, topicName: $uniqueId, messageGroupId: $uniqueId)->withAsyncPublishing(), + KafkaMessageChannelBuilder::create($uniqueId, topicName: $uniqueId, messageGroupId: $uniqueId)->withBatchedNonBlockingDelivery(), [KafkaBrokerConfiguration::class => KafkaBrokerConfiguration::createWithDefaults([getenv('KAFKA_DSN') ?: 'localhost:9094'])], ); $this->warmUpBatchChannel(); @@ -111,7 +111,7 @@ public function setUpDbalBatchChannel(): void { $this->batchChannel = $this->bootstrapBatchChannel( ModulePackageList::DBAL_PACKAGE, - DbalBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withAsyncPublishing(), + DbalBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withBatchedNonBlockingDelivery(), [DbalConnectionFactory::class => new DbalConnectionFactory(getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone')], ); $this->warmUpBatchChannel(); @@ -127,7 +127,7 @@ public function setUpRedisBatchChannel(): void { $this->batchChannel = $this->bootstrapBatchChannel( ModulePackageList::REDIS_PACKAGE, - RedisBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withAsyncPublishing(), + RedisBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withBatchedNonBlockingDelivery(), [RedisConnectionFactory::class => new RedisConnectionFactory(getenv('REDIS_DSN') ?: 'redis://localhost:6379')], ); $this->warmUpBatchChannel(); @@ -149,7 +149,7 @@ public function setUpSqsBatchChannel(): void { $this->batchChannel = $this->bootstrapBatchChannel( ModulePackageList::SQS_PACKAGE, - SqsBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withAsyncPublishing(), + SqsBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withBatchedNonBlockingDelivery(), [SqsConnectionFactory::class => new SqsConnectionFactory(getenv('SQS_DSN') ?: 'sqs:?key=key&secret=secret®ion=us-east-1&endpoint=http://localhost:4566&version=latest')], ); $this->warmUpBatchChannel(); diff --git a/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php b/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php index 3aa4a822b..84bae43ec 100644 --- a/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php +++ b/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php @@ -72,7 +72,7 @@ public function withPublisherConfirms(bool $enabled): self return $this; } - public function withAsyncPublishing(bool $enabled = true, ?int $timeoutInMilliseconds = null): self + public function withBatchedNonBlockingDelivery(bool $enabled = true, ?int $timeoutInMilliseconds = null): self { $this->getAmqpOutboundChannelAdapter()->withAsyncPublishing($enabled, $timeoutInMilliseconds); diff --git a/packages/Amqp/tests/Integration/AsyncPublishingTest.php b/packages/Amqp/tests/Integration/AsyncPublishingTest.php index 799b4b32d..c31ff8840 100644 --- a/packages/Amqp/tests/Integration/AsyncPublishingTest.php +++ b/packages/Amqp/tests/Integration/AsyncPublishingTest.php @@ -220,7 +220,7 @@ public function test_batch_published_over_amqp_lib_connection_is_delivered(): vo ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::AMQP_PACKAGE])) ->withExtensionObjects([ AmqpBackedMessageChannelBuilder::create('asyncOrdersChannel', queueName: $channelName) - ->withAsyncPublishing(), + ->withBatchedNonBlockingDelivery(), ]), licenceKey: LicenceTesting::VALID_LICENCE, ); @@ -309,7 +309,7 @@ private function bootstrapEcotone(string $channelName, object $orderService, ?st ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::AMQP_PACKAGE])) ->withExtensionObjects([ AmqpBackedMessageChannelBuilder::create('asyncOrdersChannel', queueName: $channelName) - ->withAsyncPublishing(), + ->withBatchedNonBlockingDelivery(), ]), licenceKey: $licenceKey, ); diff --git a/packages/Dbal/src/DbalBackedMessageChannelBuilder.php b/packages/Dbal/src/DbalBackedMessageChannelBuilder.php index 29adc9994..b9fb2b216 100644 --- a/packages/Dbal/src/DbalBackedMessageChannelBuilder.php +++ b/packages/Dbal/src/DbalBackedMessageChannelBuilder.php @@ -31,9 +31,9 @@ public static function create(string $channelName, string $connectionReferenceNa return new self($channelName, $connectionReferenceName); } - public function withAsyncPublishing(bool $asyncPublishing = true): self + public function withBatchedNonBlockingDelivery(bool $enabled = true): self { - $this->getDbalOutboundChannelAdapter()->withAsyncPublishing($asyncPublishing); + $this->getDbalOutboundChannelAdapter()->withAsyncPublishing($enabled); return $this; } diff --git a/packages/Dbal/tests/Integration/AsyncPublishingTest.php b/packages/Dbal/tests/Integration/AsyncPublishingTest.php index ad13337cd..a7ca883af 100644 --- a/packages/Dbal/tests/Integration/AsyncPublishingTest.php +++ b/packages/Dbal/tests/Integration/AsyncPublishingTest.php @@ -286,7 +286,7 @@ private function bootstrapEcotoneWithChannel(object $orderService, ?string $lice ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ DbalBackedMessageChannelBuilder::create('asyncOrdersChannel') - ->withAsyncPublishing(), + ->withBatchedNonBlockingDelivery(), ]), licenceKey: $licenceKey, ); diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php new file mode 100644 index 000000000..6d9da44e0 --- /dev/null +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -0,0 +1,79 @@ +orders[] = $order; + } + + #[QueryHandler('order.getRegistered')] + public function getRegistered(): array + { + return $this->orders; + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $this->assertCount(3, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); + $this->assertNull($messaging->getMessageChannel('outbox')->receive()); + } + + private function receiveAllFrom(PollableChannel $channel): array + { + $messages = []; + while ($message = $channel->receive()) { + $messages[] = $message; + } + + return $messages; + } +} diff --git a/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php b/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php index 90de43b80..baf26168d 100644 --- a/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php +++ b/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php @@ -119,7 +119,7 @@ public function withDefaultConversionMediaType(string $mediaType): self return $this; } - public function withAsyncPublishing(bool $enabled = true, ?int $timeoutInMilliseconds = null): self + public function withBatchedNonBlockingDelivery(bool $enabled = true, ?int $timeoutInMilliseconds = null): self { Assert::isTrue($timeoutInMilliseconds === null || $timeoutInMilliseconds > 0, 'Async publishing timeout must be a positive amount of milliseconds.'); $this->asyncPublishing = $enabled; @@ -130,7 +130,7 @@ public function withAsyncPublishing(bool $enabled = true, ?int $timeoutInMillise return $this; } - public function isAsyncPublishingEnabled(): bool + public function isBatchedNonBlockingDeliveryEnabled(): bool { return $this->asyncPublishing; } diff --git a/packages/Kafka/src/Configuration/KafkaModule.php b/packages/Kafka/src/Configuration/KafkaModule.php index f4bd3a87d..6ba7ba15d 100644 --- a/packages/Kafka/src/Configuration/KafkaModule.php +++ b/packages/Kafka/src/Configuration/KafkaModule.php @@ -113,7 +113,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO MessagePublisher::class . '::' . $extensionObject->getMessageChannelName(), ) ->withHeaderMapper($extensionObject->getHeaderMapper()) - ->withAsyncPublishing($extensionObject->isAsyncPublishingEnabled(), $extensionObject->getAsyncPublishingTimeout()); + ->withAsyncPublishing($extensionObject->isBatchedNonBlockingDeliveryEnabled(), $extensionObject->getAsyncPublishingTimeout()); } } diff --git a/packages/Kafka/tests/Integration/AsyncPublishingTest.php b/packages/Kafka/tests/Integration/AsyncPublishingTest.php index 54fc2a094..7d47c1e1d 100644 --- a/packages/Kafka/tests/Integration/AsyncPublishingTest.php +++ b/packages/Kafka/tests/Integration/AsyncPublishingTest.php @@ -220,10 +220,10 @@ private function bootstrapEcotone(string $channelName, object $orderService, Kaf $channelName, topicName: $uniqueId = Uuid::v7()->toRfc4122(), messageGroupId: $uniqueId, - )->withAsyncPublishing(); + )->withBatchedNonBlockingDelivery(); if ($asyncPublishingTimeout !== null) { - $channelBuilder = $channelBuilder->withAsyncPublishing(timeoutInMilliseconds: $asyncPublishingTimeout); + $channelBuilder = $channelBuilder->withBatchedNonBlockingDelivery(timeoutInMilliseconds: $asyncPublishingTimeout); } return EcotoneLite::bootstrapFlowTesting( diff --git a/packages/Redis/src/RedisBackedMessageChannelBuilder.php b/packages/Redis/src/RedisBackedMessageChannelBuilder.php index df716069f..b97720421 100644 --- a/packages/Redis/src/RedisBackedMessageChannelBuilder.php +++ b/packages/Redis/src/RedisBackedMessageChannelBuilder.php @@ -33,9 +33,9 @@ public static function create(string $channelName, string $connectionReferenceNa return new self($channelName, $connectionReferenceName); } - public function withAsyncPublishing(bool $asyncPublishing = true): self + public function withBatchedNonBlockingDelivery(bool $enabled = true): self { - $this->getRedisOutboundChannelAdapter()->withAsyncPublishing($asyncPublishing); + $this->getRedisOutboundChannelAdapter()->withAsyncPublishing($enabled); return $this; } diff --git a/packages/Redis/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Redis/tests/Integration/AsyncPublishingReliabilityTest.php index 9c6f57af6..6349aa975 100644 --- a/packages/Redis/tests/Integration/AsyncPublishingReliabilityTest.php +++ b/packages/Redis/tests/Integration/AsyncPublishingReliabilityTest.php @@ -75,7 +75,7 @@ private function bootstrapEcotoneWithRetryingChannel(): FlowTestSupport PollableChannelConfiguration::create(self::CHANNEL_NAME, RetryTemplateBuilder::fixedBackOff(1)->maxRetryAttempts(1)->build()), ]), enableAsynchronousProcessing: [ - RedisBackedMessageChannelBuilder::create(self::CHANNEL_NAME)->withAsyncPublishing(), + RedisBackedMessageChannelBuilder::create(self::CHANNEL_NAME)->withBatchedNonBlockingDelivery(), ], licenceKey: LicenceTesting::VALID_LICENCE, ); diff --git a/packages/Redis/tests/Integration/AsyncPublishingTest.php b/packages/Redis/tests/Integration/AsyncPublishingTest.php index d38992d85..b861f19ab 100644 --- a/packages/Redis/tests/Integration/AsyncPublishingTest.php +++ b/packages/Redis/tests/Integration/AsyncPublishingTest.php @@ -293,7 +293,7 @@ private function bootstrapEcotoneWithChannel(object $orderService, ?string $lice ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::REDIS_PACKAGE])) ->withExtensionObjects([ RedisBackedMessageChannelBuilder::create('asyncOrdersChannel') - ->withAsyncPublishing(), + ->withBatchedNonBlockingDelivery(), ]), licenceKey: $licenceKey, ); diff --git a/packages/Sqs/src/SqsBackedMessageChannelBuilder.php b/packages/Sqs/src/SqsBackedMessageChannelBuilder.php index c12326cba..385d6b63f 100644 --- a/packages/Sqs/src/SqsBackedMessageChannelBuilder.php +++ b/packages/Sqs/src/SqsBackedMessageChannelBuilder.php @@ -33,9 +33,9 @@ public static function create(string $channelName, string $connectionReferenceNa return new self($channelName, $connectionReferenceName); } - public function withAsyncPublishing(bool $asyncPublishing = true, ?int $timeoutInMilliseconds = null): self + public function withBatchedNonBlockingDelivery(bool $enabled = true, ?int $timeoutInMilliseconds = null): self { - $this->getSqsOutboundChannelAdapter()->withAsyncPublishing($asyncPublishing, $timeoutInMilliseconds); + $this->getSqsOutboundChannelAdapter()->withAsyncPublishing($enabled, $timeoutInMilliseconds); return $this; } diff --git a/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php index 0e409934e..9788e6637 100644 --- a/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php +++ b/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php @@ -109,7 +109,7 @@ private function bootstrapChannel(string $channelName): FlowTestSupport ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::SQS_PACKAGE])) ->withExtensionObjects([ SqsBackedMessageChannelBuilder::create($channelName) - ->withAsyncPublishing(), + ->withBatchedNonBlockingDelivery(), ]), licenceKey: LicenceTesting::VALID_LICENCE, ); diff --git a/packages/Sqs/tests/Integration/AsyncPublishingTest.php b/packages/Sqs/tests/Integration/AsyncPublishingTest.php index ae441c798..7faf20e86 100644 --- a/packages/Sqs/tests/Integration/AsyncPublishingTest.php +++ b/packages/Sqs/tests/Integration/AsyncPublishingTest.php @@ -240,7 +240,7 @@ private function bootstrapEcotoneWithChannel(object $orderService, ?string $lice ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::SQS_PACKAGE])) ->withExtensionObjects([ SqsBackedMessageChannelBuilder::create('asyncOrdersChannel') - ->withAsyncPublishing(), + ->withBatchedNonBlockingDelivery(), ]), licenceKey: $licenceKey, ); From db81ef84f0e3ef428931e5746f328b98ac7fc139 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 02/25] feat: batched drain-and-group forwarding for combined channel relays (Enterprise) --- .../Config/MessagingSystemConfiguration.php | 15 +- .../Handler/Bridge/BatchForwardingBridge.php | 216 ++++++++++++++++++ 2 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php diff --git a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php index 58b4652e6..ccf368342 100644 --- a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php +++ b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php @@ -44,7 +44,9 @@ use Ecotone\Messaging\Endpoint\PollingConsumer\AsyncHandlerAnnotationRegistry; use Ecotone\Messaging\Endpoint\PollingMetadata; use Ecotone\Messaging\Gateway\MessagingEntrypointService; +use Ecotone\Messaging\Handler\Bridge\BatchForwardingBridge; use Ecotone\Messaging\Handler\Bridge\BridgeBuilder; +use Ecotone\Messaging\Handler\ChannelResolver; use Ecotone\Messaging\Handler\Gateway\GatewayProxyBuilder; use Ecotone\Messaging\Handler\InterceptedEndpoint; use Ecotone\Messaging\Handler\InterfaceToCall; @@ -57,6 +59,7 @@ use Ecotone\Messaging\Handler\Processor\MethodInvoker\InterceptorWithPointCut; use Ecotone\Messaging\Handler\Processor\MethodInvoker\MethodInterceptorBuilder; use Ecotone\Messaging\Handler\Recoverability\RetryTemplateBuilder; +use Ecotone\Messaging\Handler\ServiceActivator\ServiceActivatorBuilder; use Ecotone\Messaging\Handler\ServiceActivator\UninterruptibleServiceActivator; use Ecotone\Messaging\Handler\Transformer\RoutingSlipPrepender; use Ecotone\Messaging\Handler\Type; @@ -84,6 +87,8 @@ */ final class MessagingSystemConfiguration implements Configuration { + public const DEFAULT_FORWARDING_BATCH_SIZE = 100; + /** * @var MessageChannelBuilder[] */ @@ -536,7 +541,15 @@ private function configureAsynchronousEndpoints(InterfaceToCallRegistry $interfa * This is Bridge that will fetch the message and make use of routing_slip to target it * message handler. */ - $this->messageHandlerBuilders[$asynchronousChannel] = BridgeBuilder::create() + $this->messageHandlerBuilders[$asynchronousChannel] = ServiceActivatorBuilder::createWithDefinition( + new Definition(BatchForwardingBridge::class, [ + new ChannelReference($asynchronousChannel), + new Reference(ChannelResolver::class), + $this->isRunningForEnterpriseLicence, + self::DEFAULT_FORWARDING_BATCH_SIZE, + ]), + 'handle', + ) ->withInputChannelName($asynchronousChannel) ->withEndpointId($asynchronousChannel); } diff --git a/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php b/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php new file mode 100644 index 000000000..77e44b014 --- /dev/null +++ b/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php @@ -0,0 +1,216 @@ +batchForwardingEnabled) { + return $message; + } + + $targetChannelName = $this->nextRoutingSlipChannel($message); + if ($targetChannelName === null || ! $this->isPollable($this->channelResolver->resolve($targetChannelName))) { + return $message; + } + + $drainedMessages = $this->drainSourceChannel(); + if ($drainedMessages === []) { + return $message; + } + + foreach ($this->groupByTargetChannel($message, $drainedMessages) as $groupTargetChannelName => $groupedMessages) { + $this->forwardGroup((string) $groupTargetChannelName, $groupedMessages, $message); + } + + return null; + } + + /** + * @return Message[] + */ + private function drainSourceChannel(): array + { + $drainedMessages = []; + while (count($drainedMessages) < $this->maxBatchSize - 1) { + $nextMessage = $this->sourceChannel->receive(); + if ($nextMessage === null) { + break; + } + $drainedMessages[] = $nextMessage; + } + + return $drainedMessages; + } + + /** + * @param Message[] $drainedMessages + * @return array + */ + private function groupByTargetChannel(Message $polledMessage, array $drainedMessages): array + { + $polledMessageTargetChannelName = $this->nextRoutingSlipChannel($polledMessage); + $groups = [$polledMessageTargetChannelName => [$polledMessage]]; + foreach ($drainedMessages as $drainedMessage) { + $targetChannelName = $this->nextRoutingSlipChannel($drainedMessage); + if ($targetChannelName === null) { + $this->releaseWithoutForwarding($drainedMessage); + + continue; + } + $groups[$targetChannelName][] = $drainedMessage; + } + + $polledMessageGroup = $groups[$polledMessageTargetChannelName]; + unset($groups[$polledMessageTargetChannelName]); + $groups[$polledMessageTargetChannelName] = $polledMessageGroup; + + return $groups; + } + + /** + * @param Message[] $groupedMessages + */ + private function forwardGroup(string $targetChannelName, array $groupedMessages, Message $polledMessage): void + { + $targetChannel = $this->channelResolver->resolve($targetChannelName); + $messagesToForward = array_map(fn (Message $groupedMessage) => $this->advanceRoutingSlip($groupedMessage), $groupedMessages); + + if ($this->supportsBatchMessages($targetChannel)) { + $targetChannel->send(MessageBuilder::withPayload($this->combineIntoBatch($messagesToForward))->build()); + } else { + foreach ($messagesToForward as $messageToForward) { + $targetChannel->send($messageToForward); + } + } + + foreach ($groupedMessages as $groupedMessage) { + if ($groupedMessage !== $polledMessage) { + $this->acknowledge($groupedMessage); + } + } + } + + /** + * @param Message[] $messages + */ + private function combineIntoBatch(array $messages): BatchMessage + { + $entries = []; + foreach ($messages as $message) { + $entries[] = ['payload' => $message->getPayload(), 'headers' => $this->transferableHeaders($message)]; + } + + return BatchMessage::fromEntries($entries); + } + + /** + * @return array + */ + private function transferableHeaders(Message $message): array + { + $headers = $message->getHeaders()->headers(); + if (isset($headers[MessageHeaders::CONSUMER_ACK_HEADER_LOCATION])) { + unset($headers[$headers[MessageHeaders::CONSUMER_ACK_HEADER_LOCATION]], $headers[MessageHeaders::CONSUMER_ACK_HEADER_LOCATION]); + } + unset($headers[MessageHeaders::POLLED_CHANNEL_NAME], $headers[MessageHeaders::CONSUMER_POLLING_METADATA]); + + return $headers; + } + + private function advanceRoutingSlip(Message $message): Message + { + $routingSlipChannels = explode(',', (string) $message->getHeaders()->get(MessageHeaders::ROUTING_SLIP)); + array_shift($routingSlipChannels); + $messageBuilder = MessageBuilder::fromMessage($message); + if ($routingSlipChannels === []) { + $messageBuilder->removeHeader(MessageHeaders::ROUTING_SLIP); + } else { + $messageBuilder->setHeader(MessageHeaders::ROUTING_SLIP, implode(',', $routingSlipChannels)); + } + + return $messageBuilder->build(); + } + + private function nextRoutingSlipChannel(Message $message): ?string + { + if (! $message->getHeaders()->containsKey(MessageHeaders::ROUTING_SLIP)) { + return null; + } + $routingSlip = (string) $message->getHeaders()->get(MessageHeaders::ROUTING_SLIP); + if ($routingSlip === '') { + return null; + } + + return explode(',', $routingSlip)[0]; + } + + private function isPollable(MessageChannel $channel): bool + { + return $this->unwrap($channel) instanceof PollableChannel; + } + + private function supportsBatchMessages(MessageChannel $channel): bool + { + $unwrappedChannel = $this->unwrap($channel); + + return $unwrappedChannel instanceof BatchSupportingMessageChannel && $unwrappedChannel->supportsBatchMessages(); + } + + private function unwrap(MessageChannel $channel): MessageChannel + { + if ($channel instanceof MessageChannelInterceptorAdapter) { + return $channel->getInternalMessageChannel(); + } + + return $channel; + } + + private function acknowledge(Message $message): void + { + $acknowledgementCallback = $this->acknowledgementCallbackOf($message); + if ($acknowledgementCallback?->isAutoAcked()) { + $acknowledgementCallback->accept(); + } + } + + private function releaseWithoutForwarding(Message $message): void + { + $this->acknowledgementCallbackOf($message)?->release(); + } + + private function acknowledgementCallbackOf(Message $message): ?AcknowledgementCallback + { + $headers = $message->getHeaders(); + if (! $headers->containsKey(MessageHeaders::CONSUMER_ACK_HEADER_LOCATION)) { + return null; + } + + return $headers->get($headers->get(MessageHeaders::CONSUMER_ACK_HEADER_LOCATION)); + } +} From d765c17790d5fd25126ae8eab12151af7e7d557f Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 03/25] feat: configurable max forwarding batch size on message channel builders --- .../CombinedChannelBatchForwardingTest.php | 34 +++++++++++++++++++ .../Channel/ForwardingBatchSizeAware.php | 13 +++++++ .../Config/MessagingSystemConfiguration.php | 4 ++- .../src/EnqueueMessageChannelBuilder.php | 19 ++++++++++- 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 packages/Ecotone/src/Messaging/Channel/ForwardingBatchSizeAware.php diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php index 6d9da44e0..14773d325 100644 --- a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -67,6 +67,40 @@ public function getRegistered(): array $this->assertNull($messaging->getMessageChannel('outbox')->receive()); } + public function test_single_run_moves_no_more_messages_than_configured_forwarding_batch_size(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + DbalBackedMessageChannelBuilder::create('outbox') + ->withMaxForwardingBatchSize(2), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); + $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); + } + private function receiveAllFrom(PollableChannel $channel): array { $messages = []; diff --git a/packages/Ecotone/src/Messaging/Channel/ForwardingBatchSizeAware.php b/packages/Ecotone/src/Messaging/Channel/ForwardingBatchSizeAware.php new file mode 100644 index 000000000..34b67ae57 --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/ForwardingBatchSizeAware.php @@ -0,0 +1,13 @@ +channelBuilders[$asynchronousChannel]; $this->messageHandlerBuilders[$asynchronousChannel] = ServiceActivatorBuilder::createWithDefinition( new Definition(BatchForwardingBridge::class, [ new ChannelReference($asynchronousChannel), new Reference(ChannelResolver::class), $this->isRunningForEnterpriseLicence, - self::DEFAULT_FORWARDING_BATCH_SIZE, + $channelBuilder instanceof ForwardingBatchSizeAware ? $channelBuilder->getMaxForwardingBatchSize() : self::DEFAULT_FORWARDING_BATCH_SIZE, ]), 'handle', ) diff --git a/packages/Enqueue/src/EnqueueMessageChannelBuilder.php b/packages/Enqueue/src/EnqueueMessageChannelBuilder.php index 1b962fe81..474eb7131 100644 --- a/packages/Enqueue/src/EnqueueMessageChannelBuilder.php +++ b/packages/Enqueue/src/EnqueueMessageChannelBuilder.php @@ -2,20 +2,24 @@ namespace Ecotone\Enqueue; +use Ecotone\Messaging\Channel\ForwardingBatchSizeAware; use Ecotone\Messaging\Channel\MessageChannelWithSerializationBuilder; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\MessagingContainerBuilder; +use Ecotone\Messaging\Config\MessagingSystemConfiguration; use Ecotone\Messaging\Conversion\MediaType; use Ecotone\Messaging\Endpoint\FinalFailureStrategy; use Ecotone\Messaging\MessageConverter\HeaderMapper; +use Ecotone\Messaging\Support\Assert; /** * licence Apache-2.0 */ -abstract class EnqueueMessageChannelBuilder implements MessageChannelWithSerializationBuilder +abstract class EnqueueMessageChannelBuilder implements MessageChannelWithSerializationBuilder, ForwardingBatchSizeAware { protected EnqueueInboundChannelAdapterBuilder $inboundChannelAdapter; protected EnqueueOutboundChannelAdapterBuilder $outboundChannelAdapter; + protected int $maxForwardingBatchSize = MessagingSystemConfiguration::DEFAULT_FORWARDING_BATCH_SIZE; public function __construct(EnqueueInboundChannelAdapterBuilder $inboundChannelAdapterBuilder, EnqueueOutboundChannelAdapterBuilder $outboundChannelAdapterBuilder) { @@ -45,6 +49,19 @@ public function isStreamingChannel(): bool return false; } + public function withMaxForwardingBatchSize(int $maxForwardingBatchSize): self + { + Assert::isTrue($maxForwardingBatchSize > 0, 'Max forwarding batch size must be a positive number.'); + $this->maxForwardingBatchSize = $maxForwardingBatchSize; + + return $this; + } + + public function getMaxForwardingBatchSize(): int + { + return $this->maxForwardingBatchSize; + } + public function withHeaderMapping(string $headerMapper): self { $this->getInboundChannelAdapter()->withHeaderMapper($headerMapper); From f4f5597c17ac0aa77f9cbaa320562068aab0f8c8 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 04/25] test: batch forwarding into target with batched non-blocking delivery delivers all messages in order --- .../CombinedChannelBatchForwardingTest.php | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php index 14773d325..d552b5045 100644 --- a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -67,6 +67,52 @@ public function getRegistered(): array $this->assertNull($messaging->getMessageChannel('outbox')->receive()); } + public function test_forwarding_as_single_batch_to_target_with_batched_non_blocking_delivery_delivers_all_messages(): void + { + $orderService = new class () { + /** @var string[] */ + private array $orders = []; + + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + $this->orders[] = $order; + } + + #[QueryHandler('order.getRegistered')] + public function getRegistered(): array + { + return $this->orders; + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing') + ->withBatchedNonBlockingDelivery(), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + $this->assertNull($messaging->getMessageChannel('outbox')->receive()); + + $messaging->run('orderProcessing', ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 3, maxExecutionTimeInMilliseconds: 10000)); + + $this->assertSame(['espresso', 'latte', 'cappuccino'], $messaging->sendQueryWithRouting('order.getRegistered')); + } + public function test_single_run_moves_no_more_messages_than_configured_forwarding_batch_size(): void { $orderService = new class () { From e412e5ded7c5fb14ef03c0bd09af889663442501 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 05/25] test: non-enterprise relay keeps one message per consumer run --- .../CombinedChannelBatchForwardingTest.php | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php index d552b5045..feae8c439 100644 --- a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -147,6 +147,38 @@ public function register(string $order): void $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); } + public function test_single_run_without_enterprise_licence_moves_one_message_only(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + ]), + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); + $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); + } + private function receiveAllFrom(PollableChannel $channel): array { $messages = []; From 8e0587d543b371323c4d488752bd6f06507a6f34 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 06/25] test: shared outbox groups drained messages per routing slip target --- .../CombinedChannelBatchForwardingTest.php | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php index feae8c439..a41509392 100644 --- a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -179,6 +179,49 @@ public function register(string $order): void $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); } + public function test_messages_for_different_targets_on_shared_outbox_reach_their_own_channels(): void + { + $orderService = new class () { + #[Asynchronous('standardOrders')] + #[CommandHandler('order.registerStandard', endpointId: 'standardOrderEndpoint')] + public function registerStandard(string $order): void + { + } + + #[Asynchronous('priorityOrders')] + #[CommandHandler('order.registerPriority', endpointId: 'priorityOrderEndpoint')] + public function registerPriority(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('standardOrders', ['outbox', 'standardProcessing']), + CombinedMessageChannel::create('priorityOrders', ['outbox', 'priorityProcessing']), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('standardProcessing'), + DbalBackedMessageChannelBuilder::create('priorityProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.registerStandard', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.registerPriority', 'flat white'); + $messaging->sendCommandWithRoutingKey('order.registerStandard', 'latte'); + $messaging->sendCommandWithRoutingKey('order.registerPriority', 'cortado'); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('standardProcessing'))); + $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('priorityProcessing'))); + $this->assertNull($messaging->getMessageChannel('outbox')->receive()); + } + private function receiveAllFrom(PollableChannel $channel): array { $messages = []; From ec532b864a6bf1ac7c5b86a0c9de6b683a217d5d Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 07/25] test: failed forwarding keeps all drained messages available on outbox --- .../FailingPollableChannel.php | 31 ++++++++++++++ .../CombinedChannelBatchForwardingTest.php | 41 +++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 packages/Dbal/tests/Fixture/BatchForwarding/FailingPollableChannel.php diff --git a/packages/Dbal/tests/Fixture/BatchForwarding/FailingPollableChannel.php b/packages/Dbal/tests/Fixture/BatchForwarding/FailingPollableChannel.php new file mode 100644 index 000000000..acbdc1bfd --- /dev/null +++ b/packages/Dbal/tests/Fixture/BatchForwarding/FailingPollableChannel.php @@ -0,0 +1,31 @@ +assertNull($messaging->getMessageChannel('outbox')->receive()); } + public function test_failed_forwarding_keeps_all_messages_available_on_outbox(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'failingProcessing']), + DbalBackedMessageChannelBuilder::create('outbox'), + SimpleMessageChannelBuilder::create('failingProcessing', new FailingPollableChannel()), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $forwardingFailed = false; + try { + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + } catch (RuntimeException) { + $forwardingFailed = true; + } + + $this->assertTrue($forwardingFailed); + $this->assertCount(3, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); + } + private function receiveAllFrom(PollableChannel $channel): array { $messages = []; From 80d6269fa43f3c5e5e47ae5211193289f9456d02 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 08/25] feat: benchmark outbox relay batched vs message-by-message for AMQP and Kafka --- Monorepo/Benchmark/OutboxRelayBenchmark.php | 148 ++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 Monorepo/Benchmark/OutboxRelayBenchmark.php diff --git a/Monorepo/Benchmark/OutboxRelayBenchmark.php b/Monorepo/Benchmark/OutboxRelayBenchmark.php new file mode 100644 index 000000000..5733a9ebd --- /dev/null +++ b/Monorepo/Benchmark/OutboxRelayBenchmark.php @@ -0,0 +1,148 @@ +messaging = $this->bootstrapOutboxWithTarget( + ModulePackageList::AMQP_PACKAGE, + AmqpBackedMessageChannelBuilder::create(uniqid('benchmark_relay_')), + [AmqpConnectionFactory::class => new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f'])], + licenceKey: null, + ); + $this->fillOutbox(); + } + + public function setUpAmqpRelayBatched(): void + { + $this->messaging = $this->bootstrapOutboxWithTarget( + ModulePackageList::AMQP_PACKAGE, + AmqpBackedMessageChannelBuilder::create(uniqid('benchmark_relay_'))->withBatchedNonBlockingDelivery(), + [AmqpConnectionFactory::class => new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f'])], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $this->fillOutbox(); + } + + public function setUpKafkaRelayMessageByMessage(): void + { + $uniqueId = uniqid('benchmark_relay_'); + $this->messaging = $this->bootstrapOutboxWithTarget( + ModulePackageList::KAFKA_PACKAGE, + KafkaMessageChannelBuilder::create($uniqueId, topicName: $uniqueId, messageGroupId: $uniqueId), + [KafkaBrokerConfiguration::class => KafkaBrokerConfiguration::createWithDefaults([getenv('KAFKA_DSN') ?: 'localhost:9094'])], + licenceKey: null, + ); + $this->fillOutbox(); + } + + public function setUpKafkaRelayBatched(): void + { + $uniqueId = uniqid('benchmark_relay_'); + $this->messaging = $this->bootstrapOutboxWithTarget( + ModulePackageList::KAFKA_PACKAGE, + KafkaMessageChannelBuilder::create($uniqueId, topicName: $uniqueId, messageGroupId: $uniqueId)->withBatchedNonBlockingDelivery(), + [KafkaBrokerConfiguration::class => KafkaBrokerConfiguration::createWithDefaults([getenv('KAFKA_DSN') ?: 'localhost:9094'])], + licenceKey: LicenceTesting::VALID_LICENCE, + ); + $this->fillOutbox(); + } + + #[BeforeMethods('setUpAmqpRelayMessageByMessage')] + public function bench_amqp_outbox_relay_message_by_message(): void + { + $this->relayWholeOutbox(); + } + + #[BeforeMethods('setUpAmqpRelayBatched')] + public function bench_amqp_outbox_relay_batched(): void + { + $this->relayWholeOutbox(); + } + + #[BeforeMethods('setUpKafkaRelayMessageByMessage')] + public function bench_kafka_outbox_relay_message_by_message(): void + { + $this->relayWholeOutbox(); + } + + #[BeforeMethods('setUpKafkaRelayBatched')] + public function bench_kafka_outbox_relay_batched(): void + { + $this->relayWholeOutbox(); + } + + private function relayWholeOutbox(): void + { + $this->messaging->run('benchmark_outbox', ExecutionPollingMetadata::createWithFinishWhenNoMessages()); + } + + private function fillOutbox(): void + { + for ($messageNumber = 0; $messageNumber < self::AMOUNT_OF_RELAYED_MESSAGES; $messageNumber++) { + $this->messaging->sendCommandWithRoutingKey('benchmark.relayOrder', self::MESSAGE_PAYLOAD); + } + } + + private function bootstrapOutboxWithTarget(string $modulePackage, object $targetChannelBuilder, array $services, ?string $licenceKey): FlowTestSupport + { + $orderService = new class () { + #[Asynchronous('benchmark_relay_orders')] + #[CommandHandler('benchmark.relayOrder', endpointId: 'benchmarkRelayOrderEndpoint')] + public function handle(string $order): void + { + } + }; + + return EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + array_merge($services, [ + DbalConnectionFactory::class => new DbalConnectionFactory(getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone'), + $orderService, + ]), + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE, $modulePackage])) + ->withExtensionObjects([ + CombinedMessageChannel::create('benchmark_relay_orders', ['benchmark_outbox', $targetChannelBuilder->getMessageChannelName()]), + DbalBackedMessageChannelBuilder::create('benchmark_outbox'), + $targetChannelBuilder, + ]), + licenceKey: $licenceKey, + ); + } +} From a4565d9c29ce207629b815c19d7535ab8ae13d01 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 09/25] feat: per-message delivery failure isolation with release for redelivery, connection failures abort cycle for transactional retry --- .../FailOnceOnPayloadChannelInterceptor.php | 32 ++++++++ .../CombinedChannelBatchForwardingTest.php | 78 +++++++++++++++++++ .../Config/MessagingSystemConfiguration.php | 6 +- .../InterceptedPollingConsumerBuilder.php | 2 + .../Handler/Bridge/BatchForwardingBridge.php | 58 ++++++++++++-- 5 files changed, 167 insertions(+), 9 deletions(-) create mode 100644 packages/Dbal/tests/Fixture/BatchForwarding/FailOnceOnPayloadChannelInterceptor.php diff --git a/packages/Dbal/tests/Fixture/BatchForwarding/FailOnceOnPayloadChannelInterceptor.php b/packages/Dbal/tests/Fixture/BatchForwarding/FailOnceOnPayloadChannelInterceptor.php new file mode 100644 index 000000000..6e57c9583 --- /dev/null +++ b/packages/Dbal/tests/Fixture/BatchForwarding/FailOnceOnPayloadChannelInterceptor.php @@ -0,0 +1,32 @@ +alreadyFailed && $message->getPayload() === $this->failingPayload) { + $this->alreadyFailed = true; + + throw new ($this->exceptionClass)('Delivery of ' . $this->failingPayload . ' failed'); + } + + return $message; + } +} diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php index a4b8a73cd..685dded8c 100644 --- a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -6,12 +6,15 @@ use Ecotone\Dbal\DbalBackedMessageChannelBuilder; use Ecotone\Lite\EcotoneLite; +use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\Channel\CombinedMessageChannel; +use Ecotone\Messaging\Channel\SimpleChannelInterceptorBuilder; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; +use Ecotone\Messaging\Endpoint\PollingConsumer\ConnectionException; use Ecotone\Messaging\PollableChannel; use Ecotone\Modelling\Attribute\CommandHandler; use Ecotone\Modelling\Attribute\QueryHandler; @@ -20,6 +23,7 @@ use RuntimeException; use Test\Ecotone\Dbal\DbalMessagingTestCase; use Test\Ecotone\Dbal\Fixture\BatchForwarding\FailingPollableChannel; +use Test\Ecotone\Dbal\Fixture\BatchForwarding\FailOnceOnPayloadChannelInterceptor; /** * licence Apache-2.0 @@ -263,6 +267,80 @@ public function register(string $order): void $this->assertCount(3, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); } + public function test_failed_delivery_of_single_message_releases_only_that_message_without_duplicates(): void + { + $messaging = $this->bootstrapWithFailingTargetInterceptor(RuntimeException::class); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $this->assertSame(['espresso', 'latte'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('orderProcessing')))); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $this->assertSame(['cappuccino'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('orderProcessing')))); + $this->assertNull($messaging->getMessageChannel('outbox')->receive()); + } + + public function test_connection_failure_during_delivery_is_recovered_without_duplicates(): void + { + $messaging = $this->bootstrapWithFailingTargetInterceptor(ConnectionException::class); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + try { + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + } catch (ConnectionException) { + } + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $this->assertSame(['espresso', 'latte', 'cappuccino'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('orderProcessing')))); + $this->assertNull($messaging->getMessageChannel('outbox')->receive()); + } + + private function bootstrapWithFailingTargetInterceptor(string $exceptionClass): FlowTestSupport + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + return EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [ + DbalConnectionFactory::class => $this->getConnectionFactory(), + $orderService, + 'failingDeliveryInterceptor' => new FailOnceOnPayloadChannelInterceptor('cappuccino', $exceptionClass), + ], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + SimpleChannelInterceptorBuilder::create('orderProcessing', 'failingDeliveryInterceptor'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + + /** + * @param \Ecotone\Messaging\Message[] $messages + * @return string[] + */ + private function payloadsOf(array $messages): array + { + return array_map(fn ($message) => $message->getPayload(), $messages); + } + private function receiveAllFrom(PollableChannel $channel): array { $messages = []; diff --git a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php index 55e79857a..88c825a8e 100644 --- a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php +++ b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php @@ -11,8 +11,10 @@ use Ecotone\Lite\Test\TestConfiguration; use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\Attribute\AsynchronousRunningEndpoint; +use Ecotone\Messaging\Attribute\WithoutMessageCollector; use Ecotone\Messaging\Channel\ChannelInterceptorBuilder; use Ecotone\Messaging\Channel\EventDrivenChannelInterceptorAdapter; +use Ecotone\Messaging\Channel\ForwardingBatchSizeAware; use Ecotone\Messaging\Channel\MessageChannelBuilder; use Ecotone\Messaging\Channel\PollableChannelInterceptorAdapter; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; @@ -44,9 +46,7 @@ use Ecotone\Messaging\Endpoint\PollingConsumer\AsyncHandlerAnnotationRegistry; use Ecotone\Messaging\Endpoint\PollingMetadata; use Ecotone\Messaging\Gateway\MessagingEntrypointService; -use Ecotone\Messaging\Channel\ForwardingBatchSizeAware; use Ecotone\Messaging\Handler\Bridge\BatchForwardingBridge; -use Ecotone\Messaging\Handler\Bridge\BridgeBuilder; use Ecotone\Messaging\Handler\ChannelResolver; use Ecotone\Messaging\Handler\Gateway\GatewayProxyBuilder; use Ecotone\Messaging\Handler\InterceptedEndpoint; @@ -547,11 +547,13 @@ private function configureAsynchronousEndpoints(InterfaceToCallRegistry $interfa new Definition(BatchForwardingBridge::class, [ new ChannelReference($asynchronousChannel), new Reference(ChannelResolver::class), + new Reference(LoggingGateway::class), $this->isRunningForEnterpriseLicence, $channelBuilder instanceof ForwardingBatchSizeAware ? $channelBuilder->getMaxForwardingBatchSize() : self::DEFAULT_FORWARDING_BATCH_SIZE, ]), 'handle', ) + ->withEndpointAnnotations([AttributeDefinition::fromObject(new WithoutMessageCollector())]) ->withInputChannelName($asynchronousChannel) ->withEndpointId($asynchronousChannel); } diff --git a/packages/Ecotone/src/Messaging/Endpoint/PollingConsumer/InterceptedPollingConsumerBuilder.php b/packages/Ecotone/src/Messaging/Endpoint/PollingConsumer/InterceptedPollingConsumerBuilder.php index a8685e32f..3ee071a66 100644 --- a/packages/Ecotone/src/Messaging/Endpoint/PollingConsumer/InterceptedPollingConsumerBuilder.php +++ b/packages/Ecotone/src/Messaging/Endpoint/PollingConsumer/InterceptedPollingConsumerBuilder.php @@ -3,6 +3,7 @@ namespace Ecotone\Messaging\Endpoint\PollingConsumer; use Ecotone\Messaging\Attribute\AsynchronousRunningEndpoint; +use Ecotone\Messaging\Handler\InterceptedEndpoint; use Ecotone\Messaging\Channel\DirectChannel; use Ecotone\Messaging\Channel\DynamicChannel\DynamicMessageChannelBuilder; use Ecotone\Messaging\Channel\MessageChannelBuilder; @@ -96,6 +97,7 @@ public function registerConsumer(MessagingContainerBuilder $builder, MessageHand ); $gatewayBuilder->withEndpointAnnotations(array_merge( $this->endpointAnnotations, + $messageHandlerBuilder instanceof InterceptedEndpoint ? $messageHandlerBuilder->getEndpointAnnotations() : [], [new AttributeDefinition(AsynchronousRunningEndpoint::class, [$endpointId])] )); $gatewayBuilder diff --git a/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php b/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php index 77e44b014..503be3343 100644 --- a/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php +++ b/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php @@ -8,12 +8,15 @@ use Ecotone\Messaging\Channel\BatchSupportingMessageChannel; use Ecotone\Messaging\Channel\MessageChannelInterceptorAdapter; use Ecotone\Messaging\Endpoint\AcknowledgementCallback; +use Ecotone\Messaging\Endpoint\PollingConsumer\ConnectionException; use Ecotone\Messaging\Handler\ChannelResolver; +use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageChannel; use Ecotone\Messaging\MessageHeaders; use Ecotone\Messaging\PollableChannel; use Ecotone\Messaging\Support\MessageBuilder; +use Throwable; /** * licence Enterprise @@ -23,6 +26,7 @@ final class BatchForwardingBridge public function __construct( private PollableChannel $sourceChannel, private ChannelResolver $channelResolver, + private LoggingGateway $logger, private bool $batchForwardingEnabled, private int $maxBatchSize, ) { @@ -86,10 +90,6 @@ private function groupByTargetChannel(Message $polledMessage, array $drainedMess $groups[$targetChannelName][] = $drainedMessage; } - $polledMessageGroup = $groups[$polledMessageTargetChannelName]; - unset($groups[$polledMessageTargetChannelName]); - $groups[$polledMessageTargetChannelName] = $polledMessageGroup; - return $groups; } @@ -102,13 +102,57 @@ private function forwardGroup(string $targetChannelName, array $groupedMessages, $messagesToForward = array_map(fn (Message $groupedMessage) => $this->advanceRoutingSlip($groupedMessage), $groupedMessages); if ($this->supportsBatchMessages($targetChannel)) { - $targetChannel->send(MessageBuilder::withPayload($this->combineIntoBatch($messagesToForward))->build()); - } else { - foreach ($messagesToForward as $messageToForward) { + try { + $targetChannel->send(MessageBuilder::withPayload($this->combineIntoBatch($messagesToForward))->build()); + } catch (Throwable $exception) { + $this->releaseFailedDelivery($groupedMessages, $polledMessage, $targetChannelName, $exception); + + return; + } + $this->acknowledgeAllExcept($groupedMessages, $polledMessage); + + return; + } + + foreach ($messagesToForward as $messageIndex => $messageToForward) { + $groupedMessage = $groupedMessages[$messageIndex]; + try { $targetChannel->send($messageToForward); + } catch (Throwable $exception) { + $this->releaseFailedDelivery([$groupedMessage], $polledMessage, $targetChannelName, $exception); + + continue; + } + if ($groupedMessage !== $polledMessage) { + $this->acknowledge($groupedMessage); } } + } + /** + * @param Message[] $failedMessages + */ + private function releaseFailedDelivery(array $failedMessages, Message $polledMessage, string $targetChannelName, Throwable $exception): void + { + if ($exception instanceof ConnectionException || in_array($polledMessage, $failedMessages, true)) { + throw $exception; + } + + foreach ($failedMessages as $failedMessage) { + $this->acknowledgementCallbackOf($failedMessage)?->release(); + $this->logger->info( + sprintf('Message with id `%s` released back to source channel, as delivery to `%s` failed. Due to %s', $failedMessage->getHeaders()->getMessageId(), $targetChannelName, $exception->getMessage()), + $failedMessage, + ['exception' => $exception], + ); + } + } + + /** + * @param Message[] $groupedMessages + */ + private function acknowledgeAllExcept(array $groupedMessages, Message $polledMessage): void + { foreach ($groupedMessages as $groupedMessage) { if ($groupedMessage !== $polledMessage) { $this->acknowledge($groupedMessage); From e2dc0b95f9d11efd2bd4b23b5f2e5455e96cbf04 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 10/25] fix: relay forwards bypass message collector per message, keeping collector protection for handler-published messages --- .../Collector/MessageCollectorChannelInterceptor.php | 8 ++++++++ .../Config/MessagingSystemConfiguration.php | 2 -- .../InterceptedPollingConsumerBuilder.php | 2 -- .../Handler/Bridge/BatchForwardingBridge.php | 12 +++++++++++- packages/Ecotone/src/Messaging/MessageHeaders.php | 4 ++++ 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/packages/Ecotone/src/Messaging/Channel/Collector/MessageCollectorChannelInterceptor.php b/packages/Ecotone/src/Messaging/Channel/Collector/MessageCollectorChannelInterceptor.php index 3f646cb0f..d1196a2bd 100644 --- a/packages/Ecotone/src/Messaging/Channel/Collector/MessageCollectorChannelInterceptor.php +++ b/packages/Ecotone/src/Messaging/Channel/Collector/MessageCollectorChannelInterceptor.php @@ -9,6 +9,8 @@ use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageChannel; +use Ecotone\Messaging\MessageHeaders; +use Ecotone\Messaging\Support\MessageBuilder; /** * licence Apache-2.0 @@ -23,6 +25,12 @@ public function __construct( public function preSend(Message $message, MessageChannel $messageChannel): ?Message { + if ($message->getHeaders()->containsKey(MessageHeaders::COLLECTOR_BYPASS)) { + return MessageBuilder::fromMessage($message) + ->removeHeader(MessageHeaders::COLLECTOR_BYPASS) + ->build(); + } + if ($this->collectorStorage->isEnabled()) { $this->collectorStorage->collect($message, $this->logger); diff --git a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php index 88c825a8e..a50d26aa2 100644 --- a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php +++ b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php @@ -11,7 +11,6 @@ use Ecotone\Lite\Test\TestConfiguration; use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\Attribute\AsynchronousRunningEndpoint; -use Ecotone\Messaging\Attribute\WithoutMessageCollector; use Ecotone\Messaging\Channel\ChannelInterceptorBuilder; use Ecotone\Messaging\Channel\EventDrivenChannelInterceptorAdapter; use Ecotone\Messaging\Channel\ForwardingBatchSizeAware; @@ -553,7 +552,6 @@ private function configureAsynchronousEndpoints(InterfaceToCallRegistry $interfa ]), 'handle', ) - ->withEndpointAnnotations([AttributeDefinition::fromObject(new WithoutMessageCollector())]) ->withInputChannelName($asynchronousChannel) ->withEndpointId($asynchronousChannel); } diff --git a/packages/Ecotone/src/Messaging/Endpoint/PollingConsumer/InterceptedPollingConsumerBuilder.php b/packages/Ecotone/src/Messaging/Endpoint/PollingConsumer/InterceptedPollingConsumerBuilder.php index 3ee071a66..a8685e32f 100644 --- a/packages/Ecotone/src/Messaging/Endpoint/PollingConsumer/InterceptedPollingConsumerBuilder.php +++ b/packages/Ecotone/src/Messaging/Endpoint/PollingConsumer/InterceptedPollingConsumerBuilder.php @@ -3,7 +3,6 @@ namespace Ecotone\Messaging\Endpoint\PollingConsumer; use Ecotone\Messaging\Attribute\AsynchronousRunningEndpoint; -use Ecotone\Messaging\Handler\InterceptedEndpoint; use Ecotone\Messaging\Channel\DirectChannel; use Ecotone\Messaging\Channel\DynamicChannel\DynamicMessageChannelBuilder; use Ecotone\Messaging\Channel\MessageChannelBuilder; @@ -97,7 +96,6 @@ public function registerConsumer(MessagingContainerBuilder $builder, MessageHand ); $gatewayBuilder->withEndpointAnnotations(array_merge( $this->endpointAnnotations, - $messageHandlerBuilder instanceof InterceptedEndpoint ? $messageHandlerBuilder->getEndpointAnnotations() : [], [new AttributeDefinition(AsynchronousRunningEndpoint::class, [$endpointId])] )); $gatewayBuilder diff --git a/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php b/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php index 503be3343..3ce31c2ba 100644 --- a/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php +++ b/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php @@ -103,7 +103,11 @@ private function forwardGroup(string $targetChannelName, array $groupedMessages, if ($this->supportsBatchMessages($targetChannel)) { try { - $targetChannel->send(MessageBuilder::withPayload($this->combineIntoBatch($messagesToForward))->build()); + $targetChannel->send( + MessageBuilder::withPayload($this->combineIntoBatch($messagesToForward)) + ->setHeader(MessageHeaders::COLLECTOR_BYPASS, true) + ->build() + ); } catch (Throwable $exception) { $this->releaseFailedDelivery($groupedMessages, $polledMessage, $targetChannelName, $exception); @@ -114,8 +118,14 @@ private function forwardGroup(string $targetChannelName, array $groupedMessages, return; } + $bypassCollector = $this->isPollable($targetChannel); foreach ($messagesToForward as $messageIndex => $messageToForward) { $groupedMessage = $groupedMessages[$messageIndex]; + if ($bypassCollector) { + $messageToForward = MessageBuilder::fromMessage($messageToForward) + ->setHeader(MessageHeaders::COLLECTOR_BYPASS, true) + ->build(); + } try { $targetChannel->send($messageToForward); } catch (Throwable $exception) { diff --git a/packages/Ecotone/src/Messaging/MessageHeaders.php b/packages/Ecotone/src/Messaging/MessageHeaders.php index 2159a126c..25217fbae 100644 --- a/packages/Ecotone/src/Messaging/MessageHeaders.php +++ b/packages/Ecotone/src/Messaging/MessageHeaders.php @@ -96,6 +96,10 @@ final class MessageHeaders * Consumed channel name (set when the Message originates from a pollable Message Channel) */ public const POLLED_CHANNEL_NAME = 'polledChannelName'; + /** + * Marks a Message forwarded directly between Message Channels, so it skips the Message Collector buffering + */ + public const COLLECTOR_BYPASS = 'collectorBypass'; /** * Inbound Channel Adapter request channel name (set when the Message originates from an Inbound Channel Adapter * such as #[KafkaConsumer], AMQP inbound, #[Scheduled]). Carries the user-facing request channel where the Message From 7f1c6db25bcccfe7769c5e399bb6e959e7ac1b86 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 11/25] refactor: rename missed kafka channel builder toggle call site --- packages/Kafka/tests/Integration/AsyncPublishingTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/Kafka/tests/Integration/AsyncPublishingTest.php b/packages/Kafka/tests/Integration/AsyncPublishingTest.php index 7d47c1e1d..b187d4969 100644 --- a/packages/Kafka/tests/Integration/AsyncPublishingTest.php +++ b/packages/Kafka/tests/Integration/AsyncPublishingTest.php @@ -99,7 +99,7 @@ public function test_async_publishing_via_message_channel_requires_enterprise_li 'async_orders', topicName: $uniqueId = Uuid::v7()->toRfc4122(), messageGroupId: $uniqueId, - )->withAsyncPublishing(), + )->withBatchedNonBlockingDelivery(), ]), ); } From aa2b6cdde0491d0ac80b196d2cb0096b07fe70a0 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 12/25] refactor: rename channel builder toggle to withHighThroughputPublishing --- Monorepo/Benchmark/AsyncPublishingBenchmark.php | 10 +++++----- Monorepo/Benchmark/OutboxRelayBenchmark.php | 4 ++-- packages/Amqp/src/AmqpBackedMessageChannelBuilder.php | 2 +- .../Amqp/tests/Integration/AsyncPublishingTest.php | 4 ++-- packages/Dbal/src/DbalBackedMessageChannelBuilder.php | 2 +- .../Dbal/tests/Integration/AsyncPublishingTest.php | 2 +- .../Integration/CombinedChannelBatchForwardingTest.php | 4 ++-- .../Kafka/src/Channel/KafkaMessageChannelBuilder.php | 4 ++-- packages/Kafka/src/Configuration/KafkaModule.php | 2 +- .../Kafka/tests/Integration/AsyncPublishingTest.php | 6 +++--- .../Redis/src/RedisBackedMessageChannelBuilder.php | 2 +- .../Integration/AsyncPublishingReliabilityTest.php | 2 +- .../Redis/tests/Integration/AsyncPublishingTest.php | 2 +- packages/Sqs/src/SqsBackedMessageChannelBuilder.php | 2 +- .../Integration/AsyncPublishingReliabilityTest.php | 2 +- packages/Sqs/tests/Integration/AsyncPublishingTest.php | 2 +- 16 files changed, 26 insertions(+), 26 deletions(-) diff --git a/Monorepo/Benchmark/AsyncPublishingBenchmark.php b/Monorepo/Benchmark/AsyncPublishingBenchmark.php index 548b4f647..4814fc83b 100644 --- a/Monorepo/Benchmark/AsyncPublishingBenchmark.php +++ b/Monorepo/Benchmark/AsyncPublishingBenchmark.php @@ -72,7 +72,7 @@ public function setUpAmqpBatchChannel(): void { $this->batchChannel = $this->bootstrapBatchChannel( ModulePackageList::AMQP_PACKAGE, - AmqpBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withBatchedNonBlockingDelivery(), + AmqpBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withHighThroughputPublishing(), [AmqpConnectionFactory::class => new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f'])], ); $this->warmUpBatchChannel(); @@ -95,7 +95,7 @@ public function setUpKafkaBatchChannel(): void $uniqueId = uniqid('benchmark_orders_'); $this->batchChannel = $this->bootstrapBatchChannel( ModulePackageList::KAFKA_PACKAGE, - KafkaMessageChannelBuilder::create($uniqueId, topicName: $uniqueId, messageGroupId: $uniqueId)->withBatchedNonBlockingDelivery(), + KafkaMessageChannelBuilder::create($uniqueId, topicName: $uniqueId, messageGroupId: $uniqueId)->withHighThroughputPublishing(), [KafkaBrokerConfiguration::class => KafkaBrokerConfiguration::createWithDefaults([getenv('KAFKA_DSN') ?: 'localhost:9094'])], ); $this->warmUpBatchChannel(); @@ -111,7 +111,7 @@ public function setUpDbalBatchChannel(): void { $this->batchChannel = $this->bootstrapBatchChannel( ModulePackageList::DBAL_PACKAGE, - DbalBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withBatchedNonBlockingDelivery(), + DbalBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withHighThroughputPublishing(), [DbalConnectionFactory::class => new DbalConnectionFactory(getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone')], ); $this->warmUpBatchChannel(); @@ -127,7 +127,7 @@ public function setUpRedisBatchChannel(): void { $this->batchChannel = $this->bootstrapBatchChannel( ModulePackageList::REDIS_PACKAGE, - RedisBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withBatchedNonBlockingDelivery(), + RedisBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withHighThroughputPublishing(), [RedisConnectionFactory::class => new RedisConnectionFactory(getenv('REDIS_DSN') ?: 'redis://localhost:6379')], ); $this->warmUpBatchChannel(); @@ -149,7 +149,7 @@ public function setUpSqsBatchChannel(): void { $this->batchChannel = $this->bootstrapBatchChannel( ModulePackageList::SQS_PACKAGE, - SqsBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withBatchedNonBlockingDelivery(), + SqsBackedMessageChannelBuilder::create(uniqid('benchmark_orders_'))->withHighThroughputPublishing(), [SqsConnectionFactory::class => new SqsConnectionFactory(getenv('SQS_DSN') ?: 'sqs:?key=key&secret=secret®ion=us-east-1&endpoint=http://localhost:4566&version=latest')], ); $this->warmUpBatchChannel(); diff --git a/Monorepo/Benchmark/OutboxRelayBenchmark.php b/Monorepo/Benchmark/OutboxRelayBenchmark.php index 5733a9ebd..004e4d975 100644 --- a/Monorepo/Benchmark/OutboxRelayBenchmark.php +++ b/Monorepo/Benchmark/OutboxRelayBenchmark.php @@ -52,7 +52,7 @@ public function setUpAmqpRelayBatched(): void { $this->messaging = $this->bootstrapOutboxWithTarget( ModulePackageList::AMQP_PACKAGE, - AmqpBackedMessageChannelBuilder::create(uniqid('benchmark_relay_'))->withBatchedNonBlockingDelivery(), + AmqpBackedMessageChannelBuilder::create(uniqid('benchmark_relay_'))->withHighThroughputPublishing(), [AmqpConnectionFactory::class => new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f'])], licenceKey: LicenceTesting::VALID_LICENCE, ); @@ -76,7 +76,7 @@ public function setUpKafkaRelayBatched(): void $uniqueId = uniqid('benchmark_relay_'); $this->messaging = $this->bootstrapOutboxWithTarget( ModulePackageList::KAFKA_PACKAGE, - KafkaMessageChannelBuilder::create($uniqueId, topicName: $uniqueId, messageGroupId: $uniqueId)->withBatchedNonBlockingDelivery(), + KafkaMessageChannelBuilder::create($uniqueId, topicName: $uniqueId, messageGroupId: $uniqueId)->withHighThroughputPublishing(), [KafkaBrokerConfiguration::class => KafkaBrokerConfiguration::createWithDefaults([getenv('KAFKA_DSN') ?: 'localhost:9094'])], licenceKey: LicenceTesting::VALID_LICENCE, ); diff --git a/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php b/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php index 84bae43ec..cfa770b15 100644 --- a/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php +++ b/packages/Amqp/src/AmqpBackedMessageChannelBuilder.php @@ -72,7 +72,7 @@ public function withPublisherConfirms(bool $enabled): self return $this; } - public function withBatchedNonBlockingDelivery(bool $enabled = true, ?int $timeoutInMilliseconds = null): self + public function withHighThroughputPublishing(bool $enabled = true, ?int $timeoutInMilliseconds = null): self { $this->getAmqpOutboundChannelAdapter()->withAsyncPublishing($enabled, $timeoutInMilliseconds); diff --git a/packages/Amqp/tests/Integration/AsyncPublishingTest.php b/packages/Amqp/tests/Integration/AsyncPublishingTest.php index c31ff8840..7b131ccb9 100644 --- a/packages/Amqp/tests/Integration/AsyncPublishingTest.php +++ b/packages/Amqp/tests/Integration/AsyncPublishingTest.php @@ -220,7 +220,7 @@ public function test_batch_published_over_amqp_lib_connection_is_delivered(): vo ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::AMQP_PACKAGE])) ->withExtensionObjects([ AmqpBackedMessageChannelBuilder::create('asyncOrdersChannel', queueName: $channelName) - ->withBatchedNonBlockingDelivery(), + ->withHighThroughputPublishing(), ]), licenceKey: LicenceTesting::VALID_LICENCE, ); @@ -309,7 +309,7 @@ private function bootstrapEcotone(string $channelName, object $orderService, ?st ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::AMQP_PACKAGE])) ->withExtensionObjects([ AmqpBackedMessageChannelBuilder::create('asyncOrdersChannel', queueName: $channelName) - ->withBatchedNonBlockingDelivery(), + ->withHighThroughputPublishing(), ]), licenceKey: $licenceKey, ); diff --git a/packages/Dbal/src/DbalBackedMessageChannelBuilder.php b/packages/Dbal/src/DbalBackedMessageChannelBuilder.php index b9fb2b216..e08f96825 100644 --- a/packages/Dbal/src/DbalBackedMessageChannelBuilder.php +++ b/packages/Dbal/src/DbalBackedMessageChannelBuilder.php @@ -31,7 +31,7 @@ public static function create(string $channelName, string $connectionReferenceNa return new self($channelName, $connectionReferenceName); } - public function withBatchedNonBlockingDelivery(bool $enabled = true): self + public function withHighThroughputPublishing(bool $enabled = true): self { $this->getDbalOutboundChannelAdapter()->withAsyncPublishing($enabled); diff --git a/packages/Dbal/tests/Integration/AsyncPublishingTest.php b/packages/Dbal/tests/Integration/AsyncPublishingTest.php index a7ca883af..bdb4b0fbf 100644 --- a/packages/Dbal/tests/Integration/AsyncPublishingTest.php +++ b/packages/Dbal/tests/Integration/AsyncPublishingTest.php @@ -286,7 +286,7 @@ private function bootstrapEcotoneWithChannel(object $orderService, ?string $lice ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ DbalBackedMessageChannelBuilder::create('asyncOrdersChannel') - ->withBatchedNonBlockingDelivery(), + ->withHighThroughputPublishing(), ]), licenceKey: $licenceKey, ); diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php index 685dded8c..232ea722a 100644 --- a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -74,7 +74,7 @@ public function getRegistered(): array $this->assertNull($messaging->getMessageChannel('outbox')->receive()); } - public function test_forwarding_as_single_batch_to_target_with_batched_non_blocking_delivery_delivers_all_messages(): void + public function test_forwarding_as_single_batch_to_target_with_high_throughput_publishing_delivers_all_messages(): void { $orderService = new class () { /** @var string[] */ @@ -103,7 +103,7 @@ public function getRegistered(): array CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing') - ->withBatchedNonBlockingDelivery(), + ->withHighThroughputPublishing(), ]), licenceKey: LicenceTesting::VALID_LICENCE, ); diff --git a/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php b/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php index baf26168d..fcabf14ba 100644 --- a/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php +++ b/packages/Kafka/src/Channel/KafkaMessageChannelBuilder.php @@ -119,7 +119,7 @@ public function withDefaultConversionMediaType(string $mediaType): self return $this; } - public function withBatchedNonBlockingDelivery(bool $enabled = true, ?int $timeoutInMilliseconds = null): self + public function withHighThroughputPublishing(bool $enabled = true, ?int $timeoutInMilliseconds = null): self { Assert::isTrue($timeoutInMilliseconds === null || $timeoutInMilliseconds > 0, 'Async publishing timeout must be a positive amount of milliseconds.'); $this->asyncPublishing = $enabled; @@ -130,7 +130,7 @@ public function withBatchedNonBlockingDelivery(bool $enabled = true, ?int $timeo return $this; } - public function isBatchedNonBlockingDeliveryEnabled(): bool + public function isHighThroughputPublishingEnabled(): bool { return $this->asyncPublishing; } diff --git a/packages/Kafka/src/Configuration/KafkaModule.php b/packages/Kafka/src/Configuration/KafkaModule.php index 6ba7ba15d..ad1883bfe 100644 --- a/packages/Kafka/src/Configuration/KafkaModule.php +++ b/packages/Kafka/src/Configuration/KafkaModule.php @@ -113,7 +113,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO MessagePublisher::class . '::' . $extensionObject->getMessageChannelName(), ) ->withHeaderMapper($extensionObject->getHeaderMapper()) - ->withAsyncPublishing($extensionObject->isBatchedNonBlockingDeliveryEnabled(), $extensionObject->getAsyncPublishingTimeout()); + ->withAsyncPublishing($extensionObject->isHighThroughputPublishingEnabled(), $extensionObject->getAsyncPublishingTimeout()); } } diff --git a/packages/Kafka/tests/Integration/AsyncPublishingTest.php b/packages/Kafka/tests/Integration/AsyncPublishingTest.php index b187d4969..40a65ea87 100644 --- a/packages/Kafka/tests/Integration/AsyncPublishingTest.php +++ b/packages/Kafka/tests/Integration/AsyncPublishingTest.php @@ -99,7 +99,7 @@ public function test_async_publishing_via_message_channel_requires_enterprise_li 'async_orders', topicName: $uniqueId = Uuid::v7()->toRfc4122(), messageGroupId: $uniqueId, - )->withBatchedNonBlockingDelivery(), + )->withHighThroughputPublishing(), ]), ); } @@ -220,10 +220,10 @@ private function bootstrapEcotone(string $channelName, object $orderService, Kaf $channelName, topicName: $uniqueId = Uuid::v7()->toRfc4122(), messageGroupId: $uniqueId, - )->withBatchedNonBlockingDelivery(); + )->withHighThroughputPublishing(); if ($asyncPublishingTimeout !== null) { - $channelBuilder = $channelBuilder->withBatchedNonBlockingDelivery(timeoutInMilliseconds: $asyncPublishingTimeout); + $channelBuilder = $channelBuilder->withHighThroughputPublishing(timeoutInMilliseconds: $asyncPublishingTimeout); } return EcotoneLite::bootstrapFlowTesting( diff --git a/packages/Redis/src/RedisBackedMessageChannelBuilder.php b/packages/Redis/src/RedisBackedMessageChannelBuilder.php index b97720421..c0096ac6b 100644 --- a/packages/Redis/src/RedisBackedMessageChannelBuilder.php +++ b/packages/Redis/src/RedisBackedMessageChannelBuilder.php @@ -33,7 +33,7 @@ public static function create(string $channelName, string $connectionReferenceNa return new self($channelName, $connectionReferenceName); } - public function withBatchedNonBlockingDelivery(bool $enabled = true): self + public function withHighThroughputPublishing(bool $enabled = true): self { $this->getRedisOutboundChannelAdapter()->withAsyncPublishing($enabled); diff --git a/packages/Redis/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Redis/tests/Integration/AsyncPublishingReliabilityTest.php index 6349aa975..181039554 100644 --- a/packages/Redis/tests/Integration/AsyncPublishingReliabilityTest.php +++ b/packages/Redis/tests/Integration/AsyncPublishingReliabilityTest.php @@ -75,7 +75,7 @@ private function bootstrapEcotoneWithRetryingChannel(): FlowTestSupport PollableChannelConfiguration::create(self::CHANNEL_NAME, RetryTemplateBuilder::fixedBackOff(1)->maxRetryAttempts(1)->build()), ]), enableAsynchronousProcessing: [ - RedisBackedMessageChannelBuilder::create(self::CHANNEL_NAME)->withBatchedNonBlockingDelivery(), + RedisBackedMessageChannelBuilder::create(self::CHANNEL_NAME)->withHighThroughputPublishing(), ], licenceKey: LicenceTesting::VALID_LICENCE, ); diff --git a/packages/Redis/tests/Integration/AsyncPublishingTest.php b/packages/Redis/tests/Integration/AsyncPublishingTest.php index b861f19ab..0116072ca 100644 --- a/packages/Redis/tests/Integration/AsyncPublishingTest.php +++ b/packages/Redis/tests/Integration/AsyncPublishingTest.php @@ -293,7 +293,7 @@ private function bootstrapEcotoneWithChannel(object $orderService, ?string $lice ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::REDIS_PACKAGE])) ->withExtensionObjects([ RedisBackedMessageChannelBuilder::create('asyncOrdersChannel') - ->withBatchedNonBlockingDelivery(), + ->withHighThroughputPublishing(), ]), licenceKey: $licenceKey, ); diff --git a/packages/Sqs/src/SqsBackedMessageChannelBuilder.php b/packages/Sqs/src/SqsBackedMessageChannelBuilder.php index 385d6b63f..cb6df98e4 100644 --- a/packages/Sqs/src/SqsBackedMessageChannelBuilder.php +++ b/packages/Sqs/src/SqsBackedMessageChannelBuilder.php @@ -33,7 +33,7 @@ public static function create(string $channelName, string $connectionReferenceNa return new self($channelName, $connectionReferenceName); } - public function withBatchedNonBlockingDelivery(bool $enabled = true, ?int $timeoutInMilliseconds = null): self + public function withHighThroughputPublishing(bool $enabled = true, ?int $timeoutInMilliseconds = null): self { $this->getSqsOutboundChannelAdapter()->withAsyncPublishing($enabled, $timeoutInMilliseconds); diff --git a/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php b/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php index 9788e6637..ed9707cac 100644 --- a/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php +++ b/packages/Sqs/tests/Integration/AsyncPublishingReliabilityTest.php @@ -109,7 +109,7 @@ private function bootstrapChannel(string $channelName): FlowTestSupport ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::SQS_PACKAGE])) ->withExtensionObjects([ SqsBackedMessageChannelBuilder::create($channelName) - ->withBatchedNonBlockingDelivery(), + ->withHighThroughputPublishing(), ]), licenceKey: LicenceTesting::VALID_LICENCE, ); diff --git a/packages/Sqs/tests/Integration/AsyncPublishingTest.php b/packages/Sqs/tests/Integration/AsyncPublishingTest.php index 7faf20e86..492c746e1 100644 --- a/packages/Sqs/tests/Integration/AsyncPublishingTest.php +++ b/packages/Sqs/tests/Integration/AsyncPublishingTest.php @@ -240,7 +240,7 @@ private function bootstrapEcotoneWithChannel(object $orderService, ?string $lice ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::SQS_PACKAGE])) ->withExtensionObjects([ SqsBackedMessageChannelBuilder::create('asyncOrdersChannel') - ->withBatchedNonBlockingDelivery(), + ->withHighThroughputPublishing(), ]), licenceKey: $licenceKey, ); From c21551eb2634c113348dfed64d000b79be6db665 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 13/25] fix: use enterprise licence key for Kafka message-by-message outbox relay benchmark Kafka module requires an Ecotone Enterprise licence unconditionally, unlike AMQP where only high-throughput publishing needs one. setUpKafkaRelayMessageByMessage passed licenceKey: null, causing a LicensingException fatal error in CI. --- Monorepo/Benchmark/OutboxRelayBenchmark.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Monorepo/Benchmark/OutboxRelayBenchmark.php b/Monorepo/Benchmark/OutboxRelayBenchmark.php index 004e4d975..8ba737534 100644 --- a/Monorepo/Benchmark/OutboxRelayBenchmark.php +++ b/Monorepo/Benchmark/OutboxRelayBenchmark.php @@ -66,7 +66,7 @@ public function setUpKafkaRelayMessageByMessage(): void ModulePackageList::KAFKA_PACKAGE, KafkaMessageChannelBuilder::create($uniqueId, topicName: $uniqueId, messageGroupId: $uniqueId), [KafkaBrokerConfiguration::class => KafkaBrokerConfiguration::createWithDefaults([getenv('KAFKA_DSN') ?: 'localhost:9094'])], - licenceKey: null, + licenceKey: LicenceTesting::VALID_LICENCE, ); $this->fillOutbox(); } From 40d3d097d95e2d0b32d6905e8104a32b682669be Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 14/25] refactor: confine batched forwarding to combined channel relays with batch size on CombinedMessageChannel, skip draining non-auto-acked sources --- .../FailOnceOnPayloadPollableChannel.php | 37 ++++++ .../CombinedChannelBatchForwardingTest.php | 116 +++++++++++++++++- ...CombinedChannelForwardingConfiguration.php | 25 ++++ .../Channel/CombinedMessageChannel.php | 15 +++ .../Channel/ForwardingBatchSizeAware.php | 13 -- .../AsynchronousModule.php | 25 ++++ .../Config/MessagingSystemConfiguration.php | 48 +++++--- .../Handler/Bridge/BatchForwardingBridge.php | 12 +- .../src/EnqueueMessageChannelBuilder.php | 19 +-- 9 files changed, 261 insertions(+), 49 deletions(-) create mode 100644 packages/Dbal/tests/Fixture/BatchForwarding/FailOnceOnPayloadPollableChannel.php create mode 100644 packages/Ecotone/src/Messaging/Channel/CombinedChannelForwardingConfiguration.php delete mode 100644 packages/Ecotone/src/Messaging/Channel/ForwardingBatchSizeAware.php diff --git a/packages/Dbal/tests/Fixture/BatchForwarding/FailOnceOnPayloadPollableChannel.php b/packages/Dbal/tests/Fixture/BatchForwarding/FailOnceOnPayloadPollableChannel.php new file mode 100644 index 000000000..671ce1f18 --- /dev/null +++ b/packages/Dbal/tests/Fixture/BatchForwarding/FailOnceOnPayloadPollableChannel.php @@ -0,0 +1,37 @@ +failuresLeft > 0 && $message->getPayload() === $this->failingPayload) { + $this->failuresLeft--; + + throw new RuntimeException('Delivery of ' . $this->failingPayload . ' failed'); + } + + parent::send($message); + } + + public function getDefinition(): Definition + { + return new Definition(self::class, [$this->failingPayload, $this->failuresLeft]); + } +} diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php index 232ea722a..ea26d2262 100644 --- a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -9,12 +9,14 @@ use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\Channel\CombinedMessageChannel; +use Ecotone\Messaging\Channel\PollableChannel\PollableChannelConfiguration; use Ecotone\Messaging\Channel\SimpleChannelInterceptorBuilder; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; use Ecotone\Messaging\Endpoint\PollingConsumer\ConnectionException; +use Ecotone\Messaging\Handler\Recoverability\RetryTemplateBuilder; use Ecotone\Messaging\PollableChannel; use Ecotone\Modelling\Attribute\CommandHandler; use Ecotone\Modelling\Attribute\QueryHandler; @@ -24,6 +26,7 @@ use Test\Ecotone\Dbal\DbalMessagingTestCase; use Test\Ecotone\Dbal\Fixture\BatchForwarding\FailingPollableChannel; use Test\Ecotone\Dbal\Fixture\BatchForwarding\FailOnceOnPayloadChannelInterceptor; +use Test\Ecotone\Dbal\Fixture\BatchForwarding\FailOnceOnPayloadPollableChannel; /** * licence Apache-2.0 @@ -136,9 +139,9 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - DbalBackedMessageChannelBuilder::create('outbox') + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']) ->withMaxForwardingBatchSize(2), + DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), ]), licenceKey: LicenceTesting::VALID_LICENCE, @@ -229,6 +232,115 @@ public function registerPriority(string $order): void $this->assertNull($messaging->getMessageChannel('outbox')->receive()); } + public function test_plain_asynchronous_dbal_channel_keeps_one_message_per_handled_message(): void + { + $orderService = new class () { + /** @var string[] */ + private array $orders = []; + + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + $this->orders[] = $order; + } + + #[QueryHandler('order.getRegistered')] + public function getRegistered(): array + { + return $this->orders; + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + DbalBackedMessageChannelBuilder::create('orders'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + + $messaging->run('orders', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $this->assertSame(['espresso'], $messaging->sendQueryWithRouting('order.getRegistered')); + } + + public function test_non_auto_acked_source_is_not_drained(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + SimpleMessageChannelBuilder::createQueueChannel('outbox', isAutoAcked: false), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); + } + + public function test_failed_send_of_single_message_on_target_channel_releases_only_that_message(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'failingProcessing']), + DbalBackedMessageChannelBuilder::create('outbox'), + SimpleMessageChannelBuilder::create('failingProcessing', new FailOnceOnPayloadPollableChannel('cappuccino')), + PollableChannelConfiguration::create('failingProcessing', RetryTemplateBuilder::fixedBackOff(1)->maxRetryAttempts(1)->build()), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $this->assertSame(['espresso', 'latte'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('failingProcessing')))); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $this->assertSame(['cappuccino'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('failingProcessing')))); + $this->assertNull($messaging->getMessageChannel('outbox')->receive()); + } + public function test_failed_forwarding_keeps_all_messages_available_on_outbox(): void { $orderService = new class () { diff --git a/packages/Ecotone/src/Messaging/Channel/CombinedChannelForwardingConfiguration.php b/packages/Ecotone/src/Messaging/Channel/CombinedChannelForwardingConfiguration.php new file mode 100644 index 000000000..f379774aa --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/CombinedChannelForwardingConfiguration.php @@ -0,0 +1,25 @@ + $maxForwardingBatchSizes indexed by relay source channel name + */ + public function __construct(private array $maxForwardingBatchSizes = []) + { + } + + public function getMaxForwardingBatchSizeFor(string $channelName): int + { + return $this->maxForwardingBatchSizes[$channelName] ?? self::DEFAULT_MAX_BATCH_SIZE; + } +} diff --git a/packages/Ecotone/src/Messaging/Channel/CombinedMessageChannel.php b/packages/Ecotone/src/Messaging/Channel/CombinedMessageChannel.php index bdf05f368..218eb0f4f 100644 --- a/packages/Ecotone/src/Messaging/Channel/CombinedMessageChannel.php +++ b/packages/Ecotone/src/Messaging/Channel/CombinedMessageChannel.php @@ -11,6 +11,8 @@ */ final class CombinedMessageChannel { + private ?int $maxForwardingBatchSize = null; + private function __construct(private string $referenceName, private array $combinedChannels) { Assert::notNull($referenceName, 'Reference name can not be null'); @@ -25,6 +27,19 @@ public static function create(string $referenceName, array $combinedChannels): s return new self($referenceName, $combinedChannels); } + public function withMaxForwardingBatchSize(int $maxForwardingBatchSize): self + { + Assert::isTrue($maxForwardingBatchSize > 0, 'Max forwarding batch size must be a positive number.'); + $this->maxForwardingBatchSize = $maxForwardingBatchSize; + + return $this; + } + + public function getMaxForwardingBatchSize(): ?int + { + return $this->maxForwardingBatchSize; + } + public function getReferenceName(): string { return $this->referenceName; diff --git a/packages/Ecotone/src/Messaging/Channel/ForwardingBatchSizeAware.php b/packages/Ecotone/src/Messaging/Channel/ForwardingBatchSizeAware.php deleted file mode 100644 index 34b67ae57..000000000 --- a/packages/Ecotone/src/Messaging/Channel/ForwardingBatchSizeAware.php +++ /dev/null @@ -1,13 +0,0 @@ -resolveChannels($extensionObjects); + $this->registerCombinedChannelForwardingConfiguration($messagingConfiguration, $extensionObjects); $serviceConfiguration = ExtensionObjectResolver::resolveUnique(ServiceConfiguration::class, $extensionObjects, ServiceConfiguration::createWithDefaults()); $pollingMetadata = ExtensionObjectResolver::resolve(PollingMetadata::class, $extensionObjects); $polingChannelBuilders = ExtensionObjectResolver::resolve(SimpleMessageChannelBuilder::class, $extensionObjects); @@ -240,6 +243,28 @@ public function resolveChannels(array $extensionObjects): array return $endpointChannels; } + private function registerCombinedChannelForwardingConfiguration(Configuration $messagingConfiguration, array $extensionObjects): void + { + $maxForwardingBatchSizes = []; + /** @var CombinedMessageChannel $combinedMessageChannel */ + foreach (ExtensionObjectResolver::resolve(CombinedMessageChannel::class, $extensionObjects) as $combinedMessageChannel) { + $maxForwardingBatchSize = $combinedMessageChannel->getMaxForwardingBatchSize(); + if ($maxForwardingBatchSize === null) { + continue; + } + + $relaySourceChannel = $combinedMessageChannel->getCombinedChannels()[0]; + $maxForwardingBatchSizes[$relaySourceChannel] = isset($maxForwardingBatchSizes[$relaySourceChannel]) + ? min($maxForwardingBatchSizes[$relaySourceChannel], $maxForwardingBatchSize) + : $maxForwardingBatchSize; + } + + $messagingConfiguration->registerServiceDefinition( + CombinedChannelForwardingConfiguration::class, + new Definition(CombinedChannelForwardingConfiguration::class, [$maxForwardingBatchSizes]) + ); + } + public function handleRoutingEvent(RoutingEvent $event): void { $registration = $event->getRegistration(); diff --git a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php index a50d26aa2..0db98bc5e 100644 --- a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php +++ b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php @@ -13,7 +13,7 @@ use Ecotone\Messaging\Attribute\AsynchronousRunningEndpoint; use Ecotone\Messaging\Channel\ChannelInterceptorBuilder; use Ecotone\Messaging\Channel\EventDrivenChannelInterceptorAdapter; -use Ecotone\Messaging\Channel\ForwardingBatchSizeAware; +use Ecotone\Messaging\Channel\CombinedChannelForwardingConfiguration; use Ecotone\Messaging\Channel\MessageChannelBuilder; use Ecotone\Messaging\Channel\PollableChannelInterceptorAdapter; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; @@ -46,6 +46,7 @@ use Ecotone\Messaging\Endpoint\PollingMetadata; use Ecotone\Messaging\Gateway\MessagingEntrypointService; use Ecotone\Messaging\Handler\Bridge\BatchForwardingBridge; +use Ecotone\Messaging\Handler\Bridge\BridgeBuilder; use Ecotone\Messaging\Handler\ChannelResolver; use Ecotone\Messaging\Handler\Gateway\GatewayProxyBuilder; use Ecotone\Messaging\Handler\InterceptedEndpoint; @@ -87,8 +88,6 @@ */ final class MessagingSystemConfiguration implements Configuration { - public const DEFAULT_FORWARDING_BATCH_SIZE = 100; - /** * @var MessageChannelBuilder[] */ @@ -534,6 +533,19 @@ private function configureAsynchronousEndpoints(InterfaceToCallRegistry $interfa ) ); + $relaySourceChannels = []; + foreach ($this->asynchronousEndpoints as $asynchronousMessageChannels) { + foreach (array_slice($asynchronousMessageChannels, 0, -1) as $relaySourceChannel) { + $relaySourceChannels[$relaySourceChannel] = true; + } + } + if ($relaySourceChannels !== []) { + $this->registerServiceDefinition( + CombinedChannelForwardingConfiguration::class, + new Definition(CombinedChannelForwardingConfiguration::class, [[]]) + ); + } + foreach ($asynchronousChannels as $asynchronousChannel) { Assert::isTrue($this->channelBuilders[$asynchronousChannel]->isPollable(), "Asynchronous Message Channel {$asynchronousChannel} must be Pollable"); // needed for correct around intercepting, otherwise requestReply is outside of around interceptor scope @@ -541,17 +553,25 @@ private function configureAsynchronousEndpoints(InterfaceToCallRegistry $interfa * This is Bridge that will fetch the message and make use of routing_slip to target it * message handler. */ - $channelBuilder = $this->channelBuilders[$asynchronousChannel]; - $this->messageHandlerBuilders[$asynchronousChannel] = ServiceActivatorBuilder::createWithDefinition( - new Definition(BatchForwardingBridge::class, [ - new ChannelReference($asynchronousChannel), - new Reference(ChannelResolver::class), - new Reference(LoggingGateway::class), - $this->isRunningForEnterpriseLicence, - $channelBuilder instanceof ForwardingBatchSizeAware ? $channelBuilder->getMaxForwardingBatchSize() : self::DEFAULT_FORWARDING_BATCH_SIZE, - ]), - 'handle', - ) + if (isset($relaySourceChannels[$asynchronousChannel])) { + $this->messageHandlerBuilders[$asynchronousChannel] = ServiceActivatorBuilder::createWithDefinition( + new Definition(BatchForwardingBridge::class, [ + new ChannelReference($asynchronousChannel), + new Reference(ChannelResolver::class), + new Reference(LoggingGateway::class), + $this->isRunningForEnterpriseLicence, + $asynchronousChannel, + new Reference(CombinedChannelForwardingConfiguration::class), + ]), + 'handle', + ) + ->withInputChannelName($asynchronousChannel) + ->withEndpointId($asynchronousChannel); + + continue; + } + + $this->messageHandlerBuilders[$asynchronousChannel] = BridgeBuilder::create() ->withInputChannelName($asynchronousChannel) ->withEndpointId($asynchronousChannel); } diff --git a/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php b/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php index 3ce31c2ba..4802a9a98 100644 --- a/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php +++ b/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php @@ -6,6 +6,7 @@ use Ecotone\Messaging\BatchMessage; use Ecotone\Messaging\Channel\BatchSupportingMessageChannel; +use Ecotone\Messaging\Channel\CombinedChannelForwardingConfiguration; use Ecotone\Messaging\Channel\MessageChannelInterceptorAdapter; use Ecotone\Messaging\Endpoint\AcknowledgementCallback; use Ecotone\Messaging\Endpoint\PollingConsumer\ConnectionException; @@ -28,7 +29,8 @@ public function __construct( private ChannelResolver $channelResolver, private LoggingGateway $logger, private bool $batchForwardingEnabled, - private int $maxBatchSize, + private string $sourceChannelName, + private CombinedChannelForwardingConfiguration $forwardingConfiguration, ) { } @@ -43,6 +45,11 @@ public function handle(Message $message): ?Message return $message; } + $acknowledgementCallback = $this->acknowledgementCallbackOf($message); + if ($acknowledgementCallback !== null && ! $acknowledgementCallback->isAutoAcked()) { + return $message; + } + $drainedMessages = $this->drainSourceChannel(); if ($drainedMessages === []) { return $message; @@ -60,8 +67,9 @@ public function handle(Message $message): ?Message */ private function drainSourceChannel(): array { + $maxBatchSize = $this->forwardingConfiguration->getMaxForwardingBatchSizeFor($this->sourceChannelName); $drainedMessages = []; - while (count($drainedMessages) < $this->maxBatchSize - 1) { + while (count($drainedMessages) < $maxBatchSize - 1) { $nextMessage = $this->sourceChannel->receive(); if ($nextMessage === null) { break; diff --git a/packages/Enqueue/src/EnqueueMessageChannelBuilder.php b/packages/Enqueue/src/EnqueueMessageChannelBuilder.php index 474eb7131..1b962fe81 100644 --- a/packages/Enqueue/src/EnqueueMessageChannelBuilder.php +++ b/packages/Enqueue/src/EnqueueMessageChannelBuilder.php @@ -2,24 +2,20 @@ namespace Ecotone\Enqueue; -use Ecotone\Messaging\Channel\ForwardingBatchSizeAware; use Ecotone\Messaging\Channel\MessageChannelWithSerializationBuilder; use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\Container\MessagingContainerBuilder; -use Ecotone\Messaging\Config\MessagingSystemConfiguration; use Ecotone\Messaging\Conversion\MediaType; use Ecotone\Messaging\Endpoint\FinalFailureStrategy; use Ecotone\Messaging\MessageConverter\HeaderMapper; -use Ecotone\Messaging\Support\Assert; /** * licence Apache-2.0 */ -abstract class EnqueueMessageChannelBuilder implements MessageChannelWithSerializationBuilder, ForwardingBatchSizeAware +abstract class EnqueueMessageChannelBuilder implements MessageChannelWithSerializationBuilder { protected EnqueueInboundChannelAdapterBuilder $inboundChannelAdapter; protected EnqueueOutboundChannelAdapterBuilder $outboundChannelAdapter; - protected int $maxForwardingBatchSize = MessagingSystemConfiguration::DEFAULT_FORWARDING_BATCH_SIZE; public function __construct(EnqueueInboundChannelAdapterBuilder $inboundChannelAdapterBuilder, EnqueueOutboundChannelAdapterBuilder $outboundChannelAdapterBuilder) { @@ -49,19 +45,6 @@ public function isStreamingChannel(): bool return false; } - public function withMaxForwardingBatchSize(int $maxForwardingBatchSize): self - { - Assert::isTrue($maxForwardingBatchSize > 0, 'Max forwarding batch size must be a positive number.'); - $this->maxForwardingBatchSize = $maxForwardingBatchSize; - - return $this; - } - - public function getMaxForwardingBatchSize(): int - { - return $this->maxForwardingBatchSize; - } - public function withHeaderMapping(string $headerMapper): self { $this->getInboundChannelAdapter()->withHeaderMapper($headerMapper); From ebddf028ede1fff317075806000d4ccc748ba274 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 15/25] fix: pin kafka relay baseline benchmark to batch size one under enterprise licence --- Monorepo/Benchmark/OutboxRelayBenchmark.php | 10 ++++++++-- .../Messaging/Config/MessagingSystemConfiguration.php | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Monorepo/Benchmark/OutboxRelayBenchmark.php b/Monorepo/Benchmark/OutboxRelayBenchmark.php index 8ba737534..3880fc1b0 100644 --- a/Monorepo/Benchmark/OutboxRelayBenchmark.php +++ b/Monorepo/Benchmark/OutboxRelayBenchmark.php @@ -67,6 +67,7 @@ public function setUpKafkaRelayMessageByMessage(): void KafkaMessageChannelBuilder::create($uniqueId, topicName: $uniqueId, messageGroupId: $uniqueId), [KafkaBrokerConfiguration::class => KafkaBrokerConfiguration::createWithDefaults([getenv('KAFKA_DSN') ?: 'localhost:9094'])], licenceKey: LicenceTesting::VALID_LICENCE, + maxForwardingBatchSize: 1, ); $this->fillOutbox(); } @@ -119,7 +120,7 @@ private function fillOutbox(): void } } - private function bootstrapOutboxWithTarget(string $modulePackage, object $targetChannelBuilder, array $services, ?string $licenceKey): FlowTestSupport + private function bootstrapOutboxWithTarget(string $modulePackage, object $targetChannelBuilder, array $services, ?string $licenceKey, ?int $maxForwardingBatchSize = null): FlowTestSupport { $orderService = new class () { #[Asynchronous('benchmark_relay_orders')] @@ -129,6 +130,11 @@ public function handle(string $order): void } }; + $combinedMessageChannel = CombinedMessageChannel::create('benchmark_relay_orders', ['benchmark_outbox', $targetChannelBuilder->getMessageChannelName()]); + if ($maxForwardingBatchSize !== null) { + $combinedMessageChannel = $combinedMessageChannel->withMaxForwardingBatchSize($maxForwardingBatchSize); + } + return EcotoneLite::bootstrapFlowTesting( [$orderService::class], array_merge($services, [ @@ -138,7 +144,7 @@ public function handle(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE, $modulePackage])) ->withExtensionObjects([ - CombinedMessageChannel::create('benchmark_relay_orders', ['benchmark_outbox', $targetChannelBuilder->getMessageChannelName()]), + $combinedMessageChannel, DbalBackedMessageChannelBuilder::create('benchmark_outbox'), $targetChannelBuilder, ]), diff --git a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php index 0db98bc5e..d34de281b 100644 --- a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php +++ b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php @@ -12,8 +12,8 @@ use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\Attribute\AsynchronousRunningEndpoint; use Ecotone\Messaging\Channel\ChannelInterceptorBuilder; -use Ecotone\Messaging\Channel\EventDrivenChannelInterceptorAdapter; use Ecotone\Messaging\Channel\CombinedChannelForwardingConfiguration; +use Ecotone\Messaging\Channel\EventDrivenChannelInterceptorAdapter; use Ecotone\Messaging\Channel\MessageChannelBuilder; use Ecotone\Messaging\Channel\PollableChannelInterceptorAdapter; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; From 5b9b6ffb39d29ed71e7f762fe8e90c928a04cea8 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 16/25] fix: no-wait drain receive and final failure strategy honoured for drained messages --- .../src/DbalInboundChannelAdapterBuilder.php | 1 + .../AlwaysFailOnPayloadChannelInterceptor.php | 29 +++++ .../CombinedChannelBatchForwardingTest.php | 119 ++++++++++++++++++ .../Handler/Bridge/BatchForwardingBridge.php | 27 +++- .../src/RedisInboundChannelAdapterBuilder.php | 1 + .../src/SqsInboundChannelAdapterBuilder.php | 1 + 6 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 packages/Dbal/tests/Fixture/BatchForwarding/AlwaysFailOnPayloadChannelInterceptor.php diff --git a/packages/Dbal/src/DbalInboundChannelAdapterBuilder.php b/packages/Dbal/src/DbalInboundChannelAdapterBuilder.php index 91a5b19b2..26471a8f3 100644 --- a/packages/Dbal/src/DbalInboundChannelAdapterBuilder.php +++ b/packages/Dbal/src/DbalInboundChannelAdapterBuilder.php @@ -38,6 +38,7 @@ public function compile(MessagingContainerBuilder $builder): Definition DefaultHeaderMapper::createWith($this->headerMapper, []), EnqueueHeader::HEADER_ACKNOWLEDGE, Reference::to(LoggingGateway::class), + $this->finalFailureStrategy, ]); return new Definition(DbalInboundChannelAdapter::class, [ diff --git a/packages/Dbal/tests/Fixture/BatchForwarding/AlwaysFailOnPayloadChannelInterceptor.php b/packages/Dbal/tests/Fixture/BatchForwarding/AlwaysFailOnPayloadChannelInterceptor.php new file mode 100644 index 000000000..a6c9914f2 --- /dev/null +++ b/packages/Dbal/tests/Fixture/BatchForwarding/AlwaysFailOnPayloadChannelInterceptor.php @@ -0,0 +1,29 @@ +getPayload() === $this->failingPayload) { + throw new RuntimeException('Delivery of ' . $this->failingPayload . ' failed'); + } + + return $message; + } +} diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php index ea26d2262..1edba225e 100644 --- a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -15,6 +15,7 @@ use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; +use Ecotone\Messaging\Endpoint\FinalFailureStrategy; use Ecotone\Messaging\Endpoint\PollingConsumer\ConnectionException; use Ecotone\Messaging\Handler\Recoverability\RetryTemplateBuilder; use Ecotone\Messaging\PollableChannel; @@ -24,6 +25,7 @@ use Enqueue\Dbal\DbalConnectionFactory; use RuntimeException; use Test\Ecotone\Dbal\DbalMessagingTestCase; +use Test\Ecotone\Dbal\Fixture\BatchForwarding\AlwaysFailOnPayloadChannelInterceptor; use Test\Ecotone\Dbal\Fixture\BatchForwarding\FailingPollableChannel; use Test\Ecotone\Dbal\Fixture\BatchForwarding\FailOnceOnPayloadChannelInterceptor; use Test\Ecotone\Dbal\Fixture\BatchForwarding\FailOnceOnPayloadPollableChannel; @@ -341,6 +343,123 @@ public function register(string $order): void $this->assertNull($messaging->getMessageChannel('outbox')->receive()); } + public function test_single_message_is_forwarded_without_waiting_for_source_receive_timeout(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + DbalBackedMessageChannelBuilder::create('outbox') + ->withReceiveTimeout(3000), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + + $startedAt = microtime(true); + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 10000)); + $elapsedInMilliseconds = (microtime(true) - $startedAt) * 1000; + + $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); + $this->assertLessThan(3000, $elapsedInMilliseconds); + } + + public function test_drained_message_honours_ignore_final_failure_strategy(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [ + DbalConnectionFactory::class => $this->getConnectionFactory(), + $orderService, + 'alwaysFailingDelivery' => new AlwaysFailOnPayloadChannelInterceptor('cappuccino'), + ], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + DbalBackedMessageChannelBuilder::create('outbox') + ->withFinalFailureStrategy(FinalFailureStrategy::IGNORE), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + SimpleChannelInterceptorBuilder::create('orderProcessing', 'alwaysFailingDelivery'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $this->assertSame(['espresso', 'latte'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('orderProcessing')))); + $this->assertNull($messaging->getMessageChannel('outbox')->receive()); + } + + public function test_drained_message_with_stop_failure_strategy_stops_consumer_without_message_loss(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [ + DbalConnectionFactory::class => $this->getConnectionFactory(), + $orderService, + 'alwaysFailingDelivery' => new AlwaysFailOnPayloadChannelInterceptor('cappuccino'), + ], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + DbalBackedMessageChannelBuilder::create('outbox') + ->withFinalFailureStrategy(FinalFailureStrategy::STOP), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + SimpleChannelInterceptorBuilder::create('orderProcessing', 'alwaysFailingDelivery'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $consumerStopped = false; + try { + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + } catch (RuntimeException) { + $consumerStopped = true; + } + + $this->assertTrue($consumerStopped); + $this->assertCount(3, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); + } + public function test_failed_forwarding_keeps_all_messages_available_on_outbox(): void { $orderService = new class () { diff --git a/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php b/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php index 4802a9a98..c97765f5d 100644 --- a/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php +++ b/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php @@ -9,7 +9,9 @@ use Ecotone\Messaging\Channel\CombinedChannelForwardingConfiguration; use Ecotone\Messaging\Channel\MessageChannelInterceptorAdapter; use Ecotone\Messaging\Endpoint\AcknowledgementCallback; +use Ecotone\Messaging\Endpoint\FinalFailureStrategy; use Ecotone\Messaging\Endpoint\PollingConsumer\ConnectionException; +use Ecotone\Messaging\Endpoint\PollingMetadata; use Ecotone\Messaging\Handler\ChannelResolver; use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Ecotone\Messaging\Message; @@ -68,9 +70,11 @@ public function handle(Message $message): ?Message private function drainSourceChannel(): array { $maxBatchSize = $this->forwardingConfiguration->getMaxForwardingBatchSizeFor($this->sourceChannelName); + $withoutWaitingPollingMetadata = PollingMetadata::create($this->sourceChannelName) + ->setExecutionTimeLimitInMilliseconds(1); $drainedMessages = []; while (count($drainedMessages) < $maxBatchSize - 1) { - $nextMessage = $this->sourceChannel->receive(); + $nextMessage = $this->sourceChannel->receiveWithTimeout($withoutWaitingPollingMetadata); if ($nextMessage === null) { break; } @@ -156,14 +160,31 @@ private function releaseFailedDelivery(array $failedMessages, Message $polledMes throw $exception; } + $shouldStopConsumer = false; foreach ($failedMessages as $failedMessage) { - $this->acknowledgementCallbackOf($failedMessage)?->release(); + $acknowledgementCallback = $this->acknowledgementCallbackOf($failedMessage); + if ($acknowledgementCallback === null) { + continue; + } + + $failureStrategy = $acknowledgementCallback->getFailureStrategy(); + match ($failureStrategy) { + FinalFailureStrategy::STOP => $acknowledgementCallback->release(), + FinalFailureStrategy::IGNORE => $acknowledgementCallback->reject(), + FinalFailureStrategy::RELEASE => $acknowledgementCallback->release(), + FinalFailureStrategy::RESEND => $acknowledgementCallback->resend(), + }; + $shouldStopConsumer = $shouldStopConsumer || $failureStrategy === FinalFailureStrategy::STOP; $this->logger->info( - sprintf('Message with id `%s` released back to source channel, as delivery to `%s` failed. Due to %s', $failedMessage->getHeaders()->getMessageId(), $targetChannelName, $exception->getMessage()), + sprintf('Message with id `%s` handled with `%s` failure strategy, as delivery to `%s` failed. Due to %s', $failedMessage->getHeaders()->getMessageId(), $failureStrategy->value, $targetChannelName, $exception->getMessage()), $failedMessage, ['exception' => $exception], ); } + + if ($shouldStopConsumer) { + throw $exception; + } } /** diff --git a/packages/Redis/src/RedisInboundChannelAdapterBuilder.php b/packages/Redis/src/RedisInboundChannelAdapterBuilder.php index 59a88461e..524d69a58 100644 --- a/packages/Redis/src/RedisInboundChannelAdapterBuilder.php +++ b/packages/Redis/src/RedisInboundChannelAdapterBuilder.php @@ -40,6 +40,7 @@ public function compile(MessagingContainerBuilder $builder): Definition DefaultHeaderMapper::createWith($this->headerMapper, []), EnqueueHeader::HEADER_ACKNOWLEDGE, Reference::to(LoggingGateway::class), + $this->finalFailureStrategy, ]); return new Definition(RedisInboundChannelAdapter::class, [ diff --git a/packages/Sqs/src/SqsInboundChannelAdapterBuilder.php b/packages/Sqs/src/SqsInboundChannelAdapterBuilder.php index 574501fbc..59c9f58af 100644 --- a/packages/Sqs/src/SqsInboundChannelAdapterBuilder.php +++ b/packages/Sqs/src/SqsInboundChannelAdapterBuilder.php @@ -40,6 +40,7 @@ public function compile(MessagingContainerBuilder $builder): Definition DefaultHeaderMapper::createWith($this->headerMapper, []), EnqueueHeader::HEADER_ACKNOWLEDGE, Reference::to(LoggingGateway::class), + $this->finalFailureStrategy, ]); return new Definition(SqsInboundChannelAdapter::class, [ From ac9cdfd5d2c976703a0193155edb37ffcca54b56 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Tue, 4 Aug 2026 08:00:00 +0200 Subject: [PATCH 17/25] fix: gate drain sources to marker-capable channel builders and disable collector for relay targets at configuration time --- .../src/DbalBackedMessageChannelBuilder.php | 3 +- .../CombinedChannelBatchForwardingTest.php | 72 +++++++++++++++++ .../Channel/BatchForwardingSourceChannel.php | 15 ++++ .../Collector/Config/CollectorModule.php | 14 ++++ .../MessageCollectorChannelInterceptor.php | 8 -- .../Config/MessagingSystemConfiguration.php | 3 +- .../Handler/Bridge/BatchForwardingBridge.php | 12 +-- .../Ecotone/src/Messaging/MessageHeaders.php | 4 - .../CombinedChannelForwardingTest.php | 77 +++++++++++++++++++ 9 files changed, 183 insertions(+), 25 deletions(-) create mode 100644 packages/Ecotone/src/Messaging/Channel/BatchForwardingSourceChannel.php create mode 100644 packages/Kafka/tests/Integration/CombinedChannelForwardingTest.php diff --git a/packages/Dbal/src/DbalBackedMessageChannelBuilder.php b/packages/Dbal/src/DbalBackedMessageChannelBuilder.php index e08f96825..444200916 100644 --- a/packages/Dbal/src/DbalBackedMessageChannelBuilder.php +++ b/packages/Dbal/src/DbalBackedMessageChannelBuilder.php @@ -3,12 +3,13 @@ namespace Ecotone\Dbal; use Ecotone\Enqueue\EnqueueMessageChannelBuilder; +use Ecotone\Messaging\Channel\BatchForwardingSourceChannel; use Enqueue\Dbal\DbalConnectionFactory; /** * licence Apache-2.0 */ -class DbalBackedMessageChannelBuilder extends EnqueueMessageChannelBuilder +class DbalBackedMessageChannelBuilder extends EnqueueMessageChannelBuilder implements BatchForwardingSourceChannel { private function __construct(string $channelName, string $connectionReferenceName) { diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php index 1edba225e..62fddf967 100644 --- a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -9,6 +9,7 @@ use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\Channel\CombinedMessageChannel; +use Ecotone\Messaging\Channel\PollableChannel\GlobalPollableChannelConfiguration; use Ecotone\Messaging\Channel\PollableChannel\PollableChannelConfiguration; use Ecotone\Messaging\Channel\SimpleChannelInterceptorBuilder; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; @@ -460,6 +461,77 @@ public function register(string $order): void $this->assertCount(3, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); } + public function test_batched_forwarding_does_not_apply_to_in_memory_source_channels(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [$orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['inMemoryOutbox', 'inMemoryProcessing']), + SimpleMessageChannelBuilder::createQueueChannel('inMemoryOutbox'), + SimpleMessageChannelBuilder::createQueueChannel('inMemoryProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $messaging->run('inMemoryOutbox', ExecutionPollingMetadata::createWithTestingSetup()); + + $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('inMemoryProcessing'))); + $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('inMemoryOutbox'))); + } + + public function test_forwarded_message_does_not_carry_internal_collector_header(): void + { + foreach ([true, false] as $collectorEnabled) { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + GlobalPollableChannelConfiguration::createWithDefaults()->withCollector($collectorEnabled), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $forwardedMessages = $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing')); + $this->assertCount(2, $forwardedMessages); + foreach ($forwardedMessages as $forwardedMessage) { + $this->assertFalse($forwardedMessage->getHeaders()->containsKey('collectorBypass')); + } + } + } + public function test_failed_forwarding_keeps_all_messages_available_on_outbox(): void { $orderService = new class () { diff --git a/packages/Ecotone/src/Messaging/Channel/BatchForwardingSourceChannel.php b/packages/Ecotone/src/Messaging/Channel/BatchForwardingSourceChannel.php new file mode 100644 index 000000000..603a2ec03 --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/BatchForwardingSourceChannel.php @@ -0,0 +1,15 @@ +getCombinedChannels(), 1) as $relayTargetChannel) { + $combinedChannelRelayTargets[$relayTargetChannel] = true; + } + } + $takenChannelNames = []; foreach ($pollableChannelConfigurations as $pollableChannelConfiguration) { if (in_array($pollableChannelConfiguration->getChannelName(), $takenChannelNames)) { @@ -52,6 +61,10 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO foreach ($pollableMessageChannels as $pollableMessageChannel) { + if (isset($combinedChannelRelayTargets[$pollableMessageChannel->getMessageChannelName()])) { + continue; + } + $channelConfiguration = $globalPollableChannelConfiguration; foreach ($pollableChannelConfigurations as $pollableChannelConfiguration) { @@ -103,6 +116,7 @@ public function canHandle($extensionObject): bool return $extensionObject instanceof PollableChannelConfiguration || $extensionObject instanceof GlobalPollableChannelConfiguration + || $extensionObject instanceof CombinedMessageChannel /** Dynamic and RoundRobin are proxies, therefore should not be intercepted */ || ($extensionObject instanceof MessageChannelBuilder && $extensionObject->isPollable() && ! ($extensionObject instanceof DynamicMessageChannelBuilder)); } diff --git a/packages/Ecotone/src/Messaging/Channel/Collector/MessageCollectorChannelInterceptor.php b/packages/Ecotone/src/Messaging/Channel/Collector/MessageCollectorChannelInterceptor.php index d1196a2bd..3f646cb0f 100644 --- a/packages/Ecotone/src/Messaging/Channel/Collector/MessageCollectorChannelInterceptor.php +++ b/packages/Ecotone/src/Messaging/Channel/Collector/MessageCollectorChannelInterceptor.php @@ -9,8 +9,6 @@ use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Ecotone\Messaging\Message; use Ecotone\Messaging\MessageChannel; -use Ecotone\Messaging\MessageHeaders; -use Ecotone\Messaging\Support\MessageBuilder; /** * licence Apache-2.0 @@ -25,12 +23,6 @@ public function __construct( public function preSend(Message $message, MessageChannel $messageChannel): ?Message { - if ($message->getHeaders()->containsKey(MessageHeaders::COLLECTOR_BYPASS)) { - return MessageBuilder::fromMessage($message) - ->removeHeader(MessageHeaders::COLLECTOR_BYPASS) - ->build(); - } - if ($this->collectorStorage->isEnabled()) { $this->collectorStorage->collect($message, $this->logger); diff --git a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php index d34de281b..0e72100e7 100644 --- a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php +++ b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php @@ -11,6 +11,7 @@ use Ecotone\Lite\Test\TestConfiguration; use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\Attribute\AsynchronousRunningEndpoint; +use Ecotone\Messaging\Channel\BatchForwardingSourceChannel; use Ecotone\Messaging\Channel\ChannelInterceptorBuilder; use Ecotone\Messaging\Channel\CombinedChannelForwardingConfiguration; use Ecotone\Messaging\Channel\EventDrivenChannelInterceptorAdapter; @@ -553,7 +554,7 @@ private function configureAsynchronousEndpoints(InterfaceToCallRegistry $interfa * This is Bridge that will fetch the message and make use of routing_slip to target it * message handler. */ - if (isset($relaySourceChannels[$asynchronousChannel])) { + if (isset($relaySourceChannels[$asynchronousChannel]) && $this->channelBuilders[$asynchronousChannel] instanceof BatchForwardingSourceChannel) { $this->messageHandlerBuilders[$asynchronousChannel] = ServiceActivatorBuilder::createWithDefinition( new Definition(BatchForwardingBridge::class, [ new ChannelReference($asynchronousChannel), diff --git a/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php b/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php index c97765f5d..370588d4a 100644 --- a/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php +++ b/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php @@ -115,11 +115,7 @@ private function forwardGroup(string $targetChannelName, array $groupedMessages, if ($this->supportsBatchMessages($targetChannel)) { try { - $targetChannel->send( - MessageBuilder::withPayload($this->combineIntoBatch($messagesToForward)) - ->setHeader(MessageHeaders::COLLECTOR_BYPASS, true) - ->build() - ); + $targetChannel->send(MessageBuilder::withPayload($this->combineIntoBatch($messagesToForward))->build()); } catch (Throwable $exception) { $this->releaseFailedDelivery($groupedMessages, $polledMessage, $targetChannelName, $exception); @@ -130,14 +126,8 @@ private function forwardGroup(string $targetChannelName, array $groupedMessages, return; } - $bypassCollector = $this->isPollable($targetChannel); foreach ($messagesToForward as $messageIndex => $messageToForward) { $groupedMessage = $groupedMessages[$messageIndex]; - if ($bypassCollector) { - $messageToForward = MessageBuilder::fromMessage($messageToForward) - ->setHeader(MessageHeaders::COLLECTOR_BYPASS, true) - ->build(); - } try { $targetChannel->send($messageToForward); } catch (Throwable $exception) { diff --git a/packages/Ecotone/src/Messaging/MessageHeaders.php b/packages/Ecotone/src/Messaging/MessageHeaders.php index 25217fbae..2159a126c 100644 --- a/packages/Ecotone/src/Messaging/MessageHeaders.php +++ b/packages/Ecotone/src/Messaging/MessageHeaders.php @@ -96,10 +96,6 @@ final class MessageHeaders * Consumed channel name (set when the Message originates from a pollable Message Channel) */ public const POLLED_CHANNEL_NAME = 'polledChannelName'; - /** - * Marks a Message forwarded directly between Message Channels, so it skips the Message Collector buffering - */ - public const COLLECTOR_BYPASS = 'collectorBypass'; /** * Inbound Channel Adapter request channel name (set when the Message originates from an Inbound Channel Adapter * such as #[KafkaConsumer], AMQP inbound, #[Scheduled]). Carries the user-facing request channel where the Message diff --git a/packages/Kafka/tests/Integration/CombinedChannelForwardingTest.php b/packages/Kafka/tests/Integration/CombinedChannelForwardingTest.php new file mode 100644 index 000000000..9a144c590 --- /dev/null +++ b/packages/Kafka/tests/Integration/CombinedChannelForwardingTest.php @@ -0,0 +1,77 @@ +toRfc4122(); + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [KafkaBrokerConfiguration::class => ConnectionTestCase::getConnection(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::KAFKA_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['kafkaOutbox', 'orderProcessing']), + KafkaMessageChannelBuilder::create('kafkaOutbox', topicName: $uniqueId, messageGroupId: $uniqueId), + SimpleMessageChannelBuilder::createQueueChannel('orderProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $messaging->run('kafkaOutbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 30000)); + + $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); + } + + /** + * @return Message[] + */ + private function receiveAllFrom(PollableChannel $channel): array + { + $messages = []; + while ($message = $channel->receive()) { + $messages[] = $message; + } + + return $messages; + } +} From 5adabdd67ce81697141a82cbd5671d9836d4150a Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Wed, 5 Aug 2026 08:00:00 +0200 Subject: [PATCH 18/25] refactor: standalone direct-SQL batch publishing endpoint for combined channel outbox relays Batch forwarding no longer consumes the outbox through a message channel consumer. An opt-in BatchForwardingConfiguration replaces the channel consumer with a polling endpoint executing SQL directly per tick: claim up to batch size rows (FOR UPDATE SKIP LOCKED on PostgreSQL, claim markers elsewhere), group by routing slip target, publish groups, delete delivered and release failed rows inside an explicit per channel transaction. Rows relay in wire format without deserialization. Multiple outbox channels can share one publishing process via withEndpointId, covering outboxes living in different databases. Unconsumed configurations, execution channel and output channel usage of the outbox fail at compile time. Warmed benchmark relays 10k messages in ~1s into a durable Dbal target against ~82s message by message. --- Monorepo/Benchmark/OutboxRelayBenchmark.php | 138 +++--- .../DbalBatchForwardingModule.php | 108 +++++ .../BatchForwarding/DbalBatchPublisher.php | 43 ++ .../BatchForwarding/DbalOutboxPublisher.php | 374 ++++++++++++++++ .../src/DbalBackedMessageChannelBuilder.php | 3 +- .../CombinedChannelBatchForwardingTest.php | 398 ++++++++++++++++-- .../Channel/BatchForwardingConfiguration.php | 67 +++ .../Channel/BatchForwardingSourceChannel.php | 15 - .../Collector/Config/CollectorModule.php | 14 - ...CombinedChannelForwardingConfiguration.php | 25 -- .../Channel/CombinedMessageChannel.php | 15 - .../AsynchronousModule.php | 25 -- .../src/Messaging/Config/Configuration.php | 6 + .../Config/MessagingSystemConfiguration.php | 92 ++-- .../src/Messaging/Config/ModuleClassList.php | 2 + .../InboundChannelAdapterBuilder.php | 10 +- .../InterceptedChannelAdapterBuilder.php | 4 +- .../Handler/Bridge/BatchForwardingBridge.php | 289 ------------- .../EnqueueInboundChannelAdapterBuilder.php | 5 + .../CombinedChannelForwardingTest.php | 33 +- 20 files changed, 1127 insertions(+), 539 deletions(-) create mode 100644 packages/Dbal/src/BatchForwarding/DbalBatchForwardingModule.php create mode 100644 packages/Dbal/src/BatchForwarding/DbalBatchPublisher.php create mode 100644 packages/Dbal/src/BatchForwarding/DbalOutboxPublisher.php create mode 100644 packages/Ecotone/src/Messaging/Channel/BatchForwardingConfiguration.php delete mode 100644 packages/Ecotone/src/Messaging/Channel/BatchForwardingSourceChannel.php delete mode 100644 packages/Ecotone/src/Messaging/Channel/CombinedChannelForwardingConfiguration.php delete mode 100644 packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php diff --git a/Monorepo/Benchmark/OutboxRelayBenchmark.php b/Monorepo/Benchmark/OutboxRelayBenchmark.php index 3880fc1b0..2a63392fc 100644 --- a/Monorepo/Benchmark/OutboxRelayBenchmark.php +++ b/Monorepo/Benchmark/OutboxRelayBenchmark.php @@ -4,20 +4,18 @@ namespace Monorepo\Benchmark; -use Ecotone\Amqp\AmqpBackedMessageChannelBuilder; use Ecotone\Dbal\DbalBackedMessageChannelBuilder; -use Ecotone\Kafka\Channel\KafkaMessageChannelBuilder; -use Ecotone\Kafka\Configuration\KafkaBrokerConfiguration; use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; +use Ecotone\Messaging\Channel\BatchForwardingConfiguration; use Ecotone\Messaging\Channel\CombinedMessageChannel; +use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; use Ecotone\Modelling\Attribute\CommandHandler; use Ecotone\Test\LicenceTesting; -use Enqueue\AmqpExt\AmqpConnectionFactory; use Enqueue\Dbal\DbalConnectionFactory; use PhpBench\Attributes\BeforeMethods; use PhpBench\Attributes\Iterations; @@ -25,94 +23,93 @@ use PhpBench\Attributes\Warmup; /** - * Compares outbox relay throughput over a combined channel (DBAL outbox in front of a broker channel): - * message-by-message forwarding (no enterprise licence) against batched drain-and-group forwarding (enterprise). + * Measures how fast the whole DBAL outbox is drained and handed over to the next channel of a combined channel: + * message-by-message forwarding (no enterprise licence) against batched SQL drain-and-forward (enterprise). + * The consumer is warmed up on an empty outbox before messages are published, so only steady-state relay work is measured. + * The in-memory target subjects isolate the producing side of the relay; the high throughput target subject shows + * the full path into a Dbal backed channel receiving whole batches at once. */ #[Warmup(0), Revs(1), Iterations(5)] class OutboxRelayBenchmark { - private const AMOUNT_OF_RELAYED_MESSAGES = 200; + private const AMOUNT_OF_RELAYED_MESSAGES = 10_000; private const MESSAGE_PAYLOAD = 'benchmark order payload for outbox relay comparison'; private FlowTestSupport $messaging; - public function setUpAmqpRelayMessageByMessage(): void + public function setUpRelayMessageByMessage(): void { - $this->messaging = $this->bootstrapOutboxWithTarget( - ModulePackageList::AMQP_PACKAGE, - AmqpBackedMessageChannelBuilder::create(uniqid('benchmark_relay_')), - [AmqpConnectionFactory::class => new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f'])], - licenceKey: null, - ); + $this->messaging = $this->bootstrapOutbox(licenceKey: null); + $this->warmUpConsumerOnEmptyOutbox(); $this->fillOutbox(); } - public function setUpAmqpRelayBatched(): void + public function setUpRelayBatched(): void { - $this->messaging = $this->bootstrapOutboxWithTarget( - ModulePackageList::AMQP_PACKAGE, - AmqpBackedMessageChannelBuilder::create(uniqid('benchmark_relay_'))->withHighThroughputPublishing(), - [AmqpConnectionFactory::class => new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f'])], - licenceKey: LicenceTesting::VALID_LICENCE, - ); + $this->messaging = $this->bootstrapOutbox(licenceKey: LicenceTesting::VALID_LICENCE); + $this->warmUpConsumerOnEmptyOutbox(); $this->fillOutbox(); } - public function setUpKafkaRelayMessageByMessage(): void + public function setUpRelaySingleBatch(): void { - $uniqueId = uniqid('benchmark_relay_'); - $this->messaging = $this->bootstrapOutboxWithTarget( - ModulePackageList::KAFKA_PACKAGE, - KafkaMessageChannelBuilder::create($uniqueId, topicName: $uniqueId, messageGroupId: $uniqueId), - [KafkaBrokerConfiguration::class => KafkaBrokerConfiguration::createWithDefaults([getenv('KAFKA_DSN') ?: 'localhost:9094'])], - licenceKey: LicenceTesting::VALID_LICENCE, - maxForwardingBatchSize: 1, - ); + $this->messaging = $this->bootstrapOutbox(licenceKey: LicenceTesting::VALID_LICENCE, maxForwardingBatchSize: self::AMOUNT_OF_RELAYED_MESSAGES); + $this->warmUpConsumerOnEmptyOutbox(); $this->fillOutbox(); } - public function setUpKafkaRelayBatched(): void + public function setUpRelayBatchedIntoHighThroughputTarget(): void { - $uniqueId = uniqid('benchmark_relay_'); - $this->messaging = $this->bootstrapOutboxWithTarget( - ModulePackageList::KAFKA_PACKAGE, - KafkaMessageChannelBuilder::create($uniqueId, topicName: $uniqueId, messageGroupId: $uniqueId)->withHighThroughputPublishing(), - [KafkaBrokerConfiguration::class => KafkaBrokerConfiguration::createWithDefaults([getenv('KAFKA_DSN') ?: 'localhost:9094'])], - licenceKey: LicenceTesting::VALID_LICENCE, - ); + $this->messaging = $this->bootstrapOutbox(licenceKey: LicenceTesting::VALID_LICENCE, highThroughputTarget: true); + $this->warmUpConsumerOnEmptyOutbox(); $this->fillOutbox(); } - #[BeforeMethods('setUpAmqpRelayMessageByMessage')] - public function bench_amqp_outbox_relay_message_by_message(): void + #[BeforeMethods('setUpRelayMessageByMessage')] + public function bench_dbal_outbox_drain_message_by_message(): void { - $this->relayWholeOutbox(); + $this->drainWholeOutbox(); } - #[BeforeMethods('setUpAmqpRelayBatched')] - public function bench_amqp_outbox_relay_batched(): void + #[BeforeMethods('setUpRelayBatched')] + public function bench_dbal_outbox_drain_batched(): void { - $this->relayWholeOutbox(); + $this->drainWholeOutbox(); } - #[BeforeMethods('setUpKafkaRelayMessageByMessage')] - public function bench_kafka_outbox_relay_message_by_message(): void + #[BeforeMethods('setUpRelaySingleBatch')] + public function bench_dbal_outbox_drain_as_single_batch(): void { - $this->relayWholeOutbox(); + $this->drainWholeOutbox(); } - #[BeforeMethods('setUpKafkaRelayBatched')] - public function bench_kafka_outbox_relay_batched(): void + #[BeforeMethods('setUpRelayBatchedIntoHighThroughputTarget')] + public function bench_dbal_outbox_drain_batched_into_high_throughput_dbal_target(): void { - $this->relayWholeOutbox(); + $this->drainWholeOutbox(); + } + + private function warmUpConsumerOnEmptyOutbox(): void + { + $context = (new DbalConnectionFactory(self::databaseDsn()))->createContext(); + $context->createDataBaseTable(); + $context->purgeQueue($context->createQueue('benchmark_outbox')); + $context->purgeQueue($context->createQueue('benchmark_target')); + + $this->messaging->run('benchmark_outbox', ExecutionPollingMetadata::createWithFinishWhenNoMessages()); } - private function relayWholeOutbox(): void + private function drainWholeOutbox(): void { $this->messaging->run('benchmark_outbox', ExecutionPollingMetadata::createWithFinishWhenNoMessages()); } + private static function databaseDsn(): string + { + return getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone'; + } + private function fillOutbox(): void { for ($messageNumber = 0; $messageNumber < self::AMOUNT_OF_RELAYED_MESSAGES; $messageNumber++) { @@ -120,8 +117,21 @@ private function fillOutbox(): void } } - private function bootstrapOutboxWithTarget(string $modulePackage, object $targetChannelBuilder, array $services, ?string $licenceKey, ?int $maxForwardingBatchSize = null): FlowTestSupport + private function bootstrapOutbox(?string $licenceKey, ?int $maxForwardingBatchSize = null, bool $highThroughputTarget = false): FlowTestSupport { + $batchForwardingExtensions = []; + if ($licenceKey !== null) { + $batchForwardingConfiguration = BatchForwardingConfiguration::create('benchmark_outbox'); + if ($maxForwardingBatchSize !== null) { + $batchForwardingConfiguration = $batchForwardingConfiguration->withMaxForwardingBatchSize($maxForwardingBatchSize); + } + $batchForwardingExtensions[] = $batchForwardingConfiguration; + } + + $targetChannel = $highThroughputTarget + ? DbalBackedMessageChannelBuilder::create('benchmark_target')->withHighThroughputPublishing() + : SimpleMessageChannelBuilder::createQueueChannel('benchmark_target'); + $orderService = new class () { #[Asynchronous('benchmark_relay_orders')] #[CommandHandler('benchmark.relayOrder', endpointId: 'benchmarkRelayOrderEndpoint')] @@ -130,24 +140,20 @@ public function handle(string $order): void } }; - $combinedMessageChannel = CombinedMessageChannel::create('benchmark_relay_orders', ['benchmark_outbox', $targetChannelBuilder->getMessageChannelName()]); - if ($maxForwardingBatchSize !== null) { - $combinedMessageChannel = $combinedMessageChannel->withMaxForwardingBatchSize($maxForwardingBatchSize); - } - return EcotoneLite::bootstrapFlowTesting( [$orderService::class], - array_merge($services, [ - DbalConnectionFactory::class => new DbalConnectionFactory(getenv('DATABASE_DSN') ?: 'pgsql://ecotone:secret@localhost:5432/ecotone'), + [ + DbalConnectionFactory::class => new DbalConnectionFactory(self::databaseDsn()), $orderService, - ]), + ], ServiceConfiguration::createWithDefaults() - ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE, $modulePackage])) - ->withExtensionObjects([ - $combinedMessageChannel, - DbalBackedMessageChannelBuilder::create('benchmark_outbox'), - $targetChannelBuilder, - ]), + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects(array_merge([ + CombinedMessageChannel::create('benchmark_relay_orders', ['benchmark_outbox', 'benchmark_target']), + DbalBackedMessageChannelBuilder::create('benchmark_outbox') + ->withReceiveTimeout(20), + $targetChannel, + ], $batchForwardingExtensions)), licenceKey: $licenceKey, ); } diff --git a/packages/Dbal/src/BatchForwarding/DbalBatchForwardingModule.php b/packages/Dbal/src/BatchForwarding/DbalBatchForwardingModule.php new file mode 100644 index 000000000..aa8ed8a4b --- /dev/null +++ b/packages/Dbal/src/BatchForwarding/DbalBatchForwardingModule.php @@ -0,0 +1,108 @@ +getMessageChannelName()] = $channelBuilder; + } + + $outboxesPerEndpointId = []; + /** @var BatchForwardingConfiguration $batchForwardingConfiguration */ + foreach (ExtensionObjectResolver::resolve(BatchForwardingConfiguration::class, $extensionObjects) as $batchForwardingConfiguration) { + $channelBuilder = $channelBuilders[$batchForwardingConfiguration->getChannelName()] ?? null; + if (! $batchForwardingConfiguration->isEnabled() || $channelBuilder === null) { + continue; + } + + $messagingConfiguration->registerBatchForwardingSourceChannel($channelBuilder->getMessageChannelName()); + $outboxesPerEndpointId[$batchForwardingConfiguration->getEndpointId()][] = $this->createOutboxPublisherDefinition($channelBuilder, $batchForwardingConfiguration); + } + + foreach ($outboxesPerEndpointId as $endpointId => $outboxPublishers) { + $batchPublisherReference = 'dbal.batch_forwarding.' . $endpointId; + $messagingConfiguration->registerServiceDefinition( + $batchPublisherReference, + new Definition(DbalBatchPublisher::class, [$outboxPublishers]), + ); + $messagingConfiguration->registerConsumer( + InboundChannelAdapterBuilder::create( + NullableMessageChannel::CHANNEL_NAME, + $batchPublisherReference, + $interfaceToCallRegistry->getFor(DbalBatchPublisher::class, 'publish'), + ) + ->withEndpointId((string) $endpointId) + ->withContinuousPolling() + ->withEndpointAnnotations([new AttributeDefinition(WithoutMessageCollector::class)]), + ); + } + } + + public function canHandle($extensionObject): bool + { + return $extensionObject instanceof BatchForwardingConfiguration + || $extensionObject instanceof DbalBackedMessageChannelBuilder; + } + + public function getModulePackageName(): string + { + return ModulePackageList::DBAL_PACKAGE; + } + + private function createOutboxPublisherDefinition(DbalBackedMessageChannelBuilder $channelBuilder, BatchForwardingConfiguration $batchForwardingConfiguration): Definition + { + $inboundChannelAdapter = $channelBuilder->getInboundChannelAdapter(); + + return new Definition(DbalOutboxPublisher::class, [ + new Definition(CachedConnectionFactory::class, [ + new Definition(DbalReconnectableConnectionFactory::class, [ + new Reference($inboundChannelAdapter->getConnectionReferenceName()), + ]), + ], 'createFor'), + $channelBuilder->getMessageChannelName(), + new Reference(ChannelResolver::class), + new Reference(LoggingGateway::class), + new Reference(EcotoneClockInterface::class), + $batchForwardingConfiguration->getMaxForwardingBatchSize(), + $inboundChannelAdapter->getFinalFailureStrategy(), + ]); + } +} diff --git a/packages/Dbal/src/BatchForwarding/DbalBatchPublisher.php b/packages/Dbal/src/BatchForwarding/DbalBatchPublisher.php new file mode 100644 index 000000000..c687616e8 --- /dev/null +++ b/packages/Dbal/src/BatchForwarding/DbalBatchPublisher.php @@ -0,0 +1,43 @@ +outboxPublishers as $outboxPublisher) { + $publishedAmount += $outboxPublisher->publishBatch(); + } + + if ($publishedAmount > 0) { + $this->previousPublishFoundNothing = false; + + return $publishedAmount; + } + + if ($this->previousPublishFoundNothing) { + usleep(self::IDLE_POLL_INTERVAL_IN_MILLISECONDS * 1000); + } + $this->previousPublishFoundNothing = true; + + return null; + } +} diff --git a/packages/Dbal/src/BatchForwarding/DbalOutboxPublisher.php b/packages/Dbal/src/BatchForwarding/DbalOutboxPublisher.php new file mode 100644 index 000000000..aff89a02b --- /dev/null +++ b/packages/Dbal/src/BatchForwarding/DbalOutboxPublisher.php @@ -0,0 +1,374 @@ +connectionFactory->createContext(); + $connection = $context->getDbalConnection(); + + $connection->beginTransaction(); + try { + $drainedRecords = $this->claimPendingRecords($context, $connection); + if ($drainedRecords === []) { + $this->markExpiredClaimsForRedelivery($context, $connection); + $drainedRecords = $this->claimPendingRecords($context, $connection); + } + if ($drainedRecords === []) { + $connection->commit(); + + return 0; + } + + $deliveredRowIds = []; + $releasedRowIds = []; + foreach ($this->groupByTargetChannel($drainedRecords, $releasedRowIds) as $groupTargetChannelName => $groupedRecords) { + $this->forwardGroup((string) $groupTargetChannelName, $groupedRecords, $deliveredRowIds, $releasedRowIds); + } + + $this->deleteRows($context, $connection, $deliveredRowIds); + $this->releaseRows($context, $connection, $releasedRowIds); + $connection->commit(); + + return count($deliveredRowIds); + } catch (Throwable $exception) { + $connection->rollBack(); + + throw $exception; + } + } + + private function markExpiredClaimsForRedelivery(DbalContext $context, Connection $connection): void + { + $connection->createQueryBuilder() + ->update($context->getTableName()) + ->set('delivery_id', ':deliveryId') + ->set('redelivered', ':redelivered') + ->andWhere('queue = :queue') + ->andWhere('redeliver_after < :now') + ->andWhere('delivery_id IS NOT NULL') + ->setParameter('queue', $this->queueName) + ->setParameter('now', $this->clock->now()->unixTime()->inSeconds(), DbalType::BIGINT) + ->setParameter('deliveryId', null, DbalType::GUID) + ->setParameter('redelivered', true, DbalType::BOOLEAN) + ->executeStatement(); + } + + /** + * @return array}> + */ + private function claimPendingRecords(DbalContext $context, Connection $connection): array + { + $nowInSeconds = $this->clock->now()->unixTime()->inSeconds(); + + $claimedRows = $this->supportsLockedFetch($connection) + ? $this->fetchPendingRowsWithLock($context, $connection, $nowInSeconds) + : $this->fetchPendingRowsWithClaimMarker($context, $connection, $nowInSeconds); + if ($claimedRows === []) { + return []; + } + + $expiredRowIds = []; + $records = []; + foreach ($claimedRows as $claimedRow) { + if (! ($claimedRow['redelivered'] || empty($claimedRow['time_to_live']) || $claimedRow['time_to_live'] > $nowInSeconds)) { + $expiredRowIds[] = $claimedRow['id']; + + continue; + } + + $records[] = $this->convertRowToForwardableRecord($claimedRow); + } + $this->deleteRows($context, $connection, $expiredRowIds); + + return $records; + } + + /** + * @param array $claimedRow + * @return array{rowId: string, targetChannelName: string|null, payload: string, headers: array} + */ + private function convertRowToForwardableRecord(array $claimedRow): array + { + $headers = $claimedRow['properties'] ? json_decode((string) $claimedRow['properties'], true, 512, JSON_THROW_ON_ERROR) : []; + unset($headers[MessageHeaders::POLLED_CHANNEL_NAME], $headers[MessageHeaders::CONSUMER_POLLING_METADATA], $headers[MessageHeaders::CONSUMER_ACK_HEADER_LOCATION]); + + $routingSlipChannels = array_filter(explode(',', (string) ($headers[MessageHeaders::ROUTING_SLIP] ?? ''))); + $targetChannelName = array_shift($routingSlipChannels); + if ($routingSlipChannels === []) { + unset($headers[MessageHeaders::ROUTING_SLIP]); + } else { + $headers[MessageHeaders::ROUTING_SLIP] = implode(',', $routingSlipChannels); + } + + return [ + 'rowId' => $claimedRow['id'], + 'targetChannelName' => $targetChannelName, + 'payload' => (string) $claimedRow['body'], + 'headers' => $headers, + ]; + } + + /** + * @return array> + */ + private function fetchPendingRowsWithLock(DbalContext $context, Connection $connection, int $nowInSeconds): array + { + $lockedFetchSql = sprintf( + 'SELECT * FROM %s WHERE queue = :queue AND (delayed_until IS NULL OR delayed_until <= :now) AND delivery_id IS NULL ORDER BY priority ASC, published_at ASC LIMIT %d FOR UPDATE SKIP LOCKED', + $context->getTableName(), + $this->maxBatchSize, + ); + + return $connection->executeQuery( + $lockedFetchSql, + ['queue' => $this->queueName, 'now' => $nowInSeconds], + ['now' => DbalType::INTEGER], + )->fetchAllAssociative(); + } + + /** + * @return array> + */ + private function fetchPendingRowsWithClaimMarker(DbalContext $context, Connection $connection, int $nowInSeconds): array + { + $selectedRows = $connection->createQueryBuilder() + ->select('*') + ->from($context->getTableName()) + ->andWhere('queue = :queue') + ->andWhere('delayed_until IS NULL OR delayed_until <= :now') + ->andWhere('delivery_id IS NULL') + ->addOrderBy('priority', 'asc') + ->addOrderBy('published_at', 'asc') + ->setMaxResults($this->maxBatchSize) + ->setParameter('queue', $this->queueName) + ->setParameter('now', $nowInSeconds, DbalType::INTEGER) + ->executeQuery() + ->fetchAllAssociative(); + if ($selectedRows === []) { + return []; + } + + $batchDeliveryId = Uuid::v7()->toRfc4122(); + $claimedAmount = $connection->createQueryBuilder() + ->update($context->getTableName()) + ->set('delivery_id', ':deliveryId') + ->set('redeliver_after', ':redeliverAfter') + ->andWhere('id IN (:rowIds)') + ->andWhere('delivery_id IS NULL') + ->setParameter('deliveryId', $batchDeliveryId, DbalType::GUID) + ->setParameter('redeliverAfter', $nowInSeconds + self::CLAIM_REDELIVERY_SAFETY_WINDOW_IN_SECONDS, DbalType::BIGINT) + ->setParameter('rowIds', array_column($selectedRows, 'id'), $this->arrayOfStringsParameterType()) + ->executeStatement(); + if ($claimedAmount === 0) { + return []; + } + if ($claimedAmount === count($selectedRows)) { + return $selectedRows; + } + + return $connection->createQueryBuilder() + ->select('*') + ->from($context->getTableName()) + ->andWhere('delivery_id = :deliveryId') + ->addOrderBy('priority', 'asc') + ->addOrderBy('published_at', 'asc') + ->setParameter('deliveryId', $batchDeliveryId, DbalType::GUID) + ->executeQuery() + ->fetchAllAssociative(); + } + + private function supportsLockedFetch(Connection $connection): bool + { + return $connection->getDatabasePlatform() instanceof PostgreSQLPlatform; + } + + /** + * @param array}> $drainedRecords + * @param string[] $releasedRowIds + * @return array}>> + */ + private function groupByTargetChannel(array $drainedRecords, array &$releasedRowIds): array + { + $groups = []; + foreach ($drainedRecords as $drainedRecord) { + $targetChannelName = $drainedRecord['targetChannelName']; + if ($targetChannelName === null) { + $releasedRowIds[] = $drainedRecord['rowId']; + $this->logger->error( + sprintf('Message with id `%s` inside outbox Channel `%s` has no routing slip to determine the forwarding target. It was released for redelivery.', $drainedRecord['headers'][MessageHeaders::MESSAGE_ID] ?? 'unknown', $this->queueName), + ); + + continue; + } + $groups[$targetChannelName][] = $drainedRecord; + } + + return $groups; + } + + /** + * @param array}> $groupedRecords + * @param string[] $deliveredRowIds + * @param string[] $releasedRowIds + */ + private function forwardGroup(string $targetChannelName, array $groupedRecords, array &$deliveredRowIds, array &$releasedRowIds): void + { + $targetChannel = $this->channelResolver->resolve($targetChannelName); + + if ($this->supportsBatchMessages($targetChannel)) { + try { + $targetChannel->send( + MessageBuilder::withPayload( + BatchMessage::fromEntries(array_map( + fn (array $groupedRecord) => ['payload' => $groupedRecord['payload'], 'headers' => $groupedRecord['headers']], + $groupedRecords, + )), + )->build(), + ); + } catch (Throwable $exception) { + $this->handleFailedDelivery($groupedRecords, $targetChannelName, $exception, $deliveredRowIds, $releasedRowIds); + + return; + } + foreach ($groupedRecords as $groupedRecord) { + $deliveredRowIds[] = $groupedRecord['rowId']; + } + + return; + } + + foreach ($groupedRecords as $groupedRecord) { + try { + $targetChannel->send( + MessageBuilder::withPayload($groupedRecord['payload']) + ->setMultipleHeaders($groupedRecord['headers']) + ->build(), + ); + } catch (Throwable $exception) { + $this->handleFailedDelivery([$groupedRecord], $targetChannelName, $exception, $deliveredRowIds, $releasedRowIds); + + continue; + } + $deliveredRowIds[] = $groupedRecord['rowId']; + } + } + + /** + * @param array}> $failedRecords + * @param string[] $deliveredRowIds + * @param string[] $releasedRowIds + */ + private function handleFailedDelivery(array $failedRecords, string $targetChannelName, Throwable $exception, array &$deliveredRowIds, array &$releasedRowIds): void + { + if ($exception instanceof ConnectionException || $this->finalFailureStrategy === FinalFailureStrategy::STOP) { + throw $exception; + } + + foreach ($failedRecords as $failedRecord) { + if ($this->finalFailureStrategy === FinalFailureStrategy::IGNORE) { + $deliveredRowIds[] = $failedRecord['rowId']; + } else { + $releasedRowIds[] = $failedRecord['rowId']; + } + $this->logger->info( + sprintf('Message with id `%s` handled with `%s` failure strategy, as delivery to `%s` failed. Due to %s', $failedRecord['headers'][MessageHeaders::MESSAGE_ID] ?? 'unknown', $this->finalFailureStrategy->value, $targetChannelName, $exception->getMessage()), + ['exception' => $exception], + ); + } + } + + /** + * @param string[] $rowIds + */ + private function deleteRows(DbalContext $context, Connection $connection, array $rowIds): void + { + if ($rowIds === []) { + return; + } + + $connection->createQueryBuilder() + ->delete($context->getTableName()) + ->andWhere('id IN (:rowIds)') + ->setParameter('rowIds', $rowIds, $this->arrayOfStringsParameterType()) + ->executeStatement(); + } + + /** + * @param string[] $rowIds + */ + private function releaseRows(DbalContext $context, Connection $connection, array $rowIds): void + { + if ($rowIds === []) { + return; + } + + $connection->createQueryBuilder() + ->update($context->getTableName()) + ->set('delivery_id', ':deliveryId') + ->set('redelivered', ':redelivered') + ->andWhere('id IN (:rowIds)') + ->setParameter('deliveryId', null, DbalType::GUID) + ->setParameter('redelivered', true, DbalType::BOOLEAN) + ->setParameter('rowIds', $rowIds, $this->arrayOfStringsParameterType()) + ->executeStatement(); + } + + private function supportsBatchMessages(MessageChannel $channel): bool + { + $unwrappedChannel = $channel instanceof MessageChannelInterceptorAdapter ? $channel->getInternalMessageChannel() : $channel; + + return $unwrappedChannel instanceof BatchSupportingMessageChannel && $unwrappedChannel->supportsBatchMessages(); + } + + private function arrayOfStringsParameterType(): mixed + { + return class_exists('\Doctrine\DBAL\ArrayParameterType') + ? \Doctrine\DBAL\ArrayParameterType::STRING + : (defined('\Doctrine\DBAL\Connection::PARAM_STR_ARRAY') ? Connection::PARAM_STR_ARRAY : 'string[]'); + } +} diff --git a/packages/Dbal/src/DbalBackedMessageChannelBuilder.php b/packages/Dbal/src/DbalBackedMessageChannelBuilder.php index 444200916..e08f96825 100644 --- a/packages/Dbal/src/DbalBackedMessageChannelBuilder.php +++ b/packages/Dbal/src/DbalBackedMessageChannelBuilder.php @@ -3,13 +3,12 @@ namespace Ecotone\Dbal; use Ecotone\Enqueue\EnqueueMessageChannelBuilder; -use Ecotone\Messaging\Channel\BatchForwardingSourceChannel; use Enqueue\Dbal\DbalConnectionFactory; /** * licence Apache-2.0 */ -class DbalBackedMessageChannelBuilder extends EnqueueMessageChannelBuilder implements BatchForwardingSourceChannel +class DbalBackedMessageChannelBuilder extends EnqueueMessageChannelBuilder { private function __construct(string $channelName, string $connectionReferenceName) { diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php index 62fddf967..a5d2db035 100644 --- a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -8,11 +8,13 @@ use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; +use Ecotone\Messaging\Channel\BatchForwardingConfiguration; use Ecotone\Messaging\Channel\CombinedMessageChannel; use Ecotone\Messaging\Channel\PollableChannel\GlobalPollableChannelConfiguration; use Ecotone\Messaging\Channel\PollableChannel\PollableChannelConfiguration; use Ecotone\Messaging\Channel\SimpleChannelInterceptorBuilder; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; +use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; @@ -20,6 +22,7 @@ use Ecotone\Messaging\Endpoint\PollingConsumer\ConnectionException; use Ecotone\Messaging\Handler\Recoverability\RetryTemplateBuilder; use Ecotone\Messaging\PollableChannel; +use Ecotone\Messaging\Support\LicensingException; use Ecotone\Modelling\Attribute\CommandHandler; use Ecotone\Modelling\Attribute\QueryHandler; use Ecotone\Test\LicenceTesting; @@ -64,6 +67,7 @@ public function getRegistered(): array ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), ]), @@ -107,6 +111,7 @@ public function getRegistered(): array ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing') ->withHighThroughputPublishing(), @@ -142,7 +147,8 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']) + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox') ->withMaxForwardingBatchSize(2), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), @@ -160,7 +166,33 @@ public function register(string $order): void $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); } - public function test_single_run_without_enterprise_licence_moves_one_message_only(): void + public function test_batch_forwarding_configuration_requires_enterprise_licence(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $this->expectException(LicensingException::class); + + EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + ]), + ); + } + + public function test_combined_channel_without_licence_keeps_one_message_per_run(): void { $orderService = new class () { #[Asynchronous('orders')] @@ -216,6 +248,7 @@ public function registerPriority(string $order): void ->withExtensionObjects([ CombinedMessageChannel::create('standardOrders', ['outbox', 'standardProcessing']), CombinedMessageChannel::create('priorityOrders', ['outbox', 'priorityProcessing']), + BatchForwardingConfiguration::create('outbox'), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('standardProcessing'), DbalBackedMessageChannelBuilder::create('priorityProcessing'), @@ -274,12 +307,18 @@ public function getRegistered(): array $this->assertSame(['espresso'], $messaging->sendQueryWithRouting('order.getRegistered')); } - public function test_non_auto_acked_source_is_not_drained(): void + public function test_multiple_outbox_channels_are_published_by_single_shared_endpoint(): void { $orderService = new class () { - #[Asynchronous('orders')] - #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] - public function register(string $order): void + #[Asynchronous('standardOrders')] + #[CommandHandler('order.registerStandard', endpointId: 'standardOrderEndpoint')] + public function registerStandard(string $order): void + { + } + + #[Asynchronous('priorityOrders')] + #[CommandHandler('order.registerPriority', endpointId: 'priorityOrderEndpoint')] + public function registerPriority(string $order): void { } }; @@ -290,20 +329,30 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - SimpleMessageChannelBuilder::createQueueChannel('outbox', isAutoAcked: false), - DbalBackedMessageChannelBuilder::create('orderProcessing'), + CombinedMessageChannel::create('standardOrders', ['standardOutbox', 'standardProcessing']), + CombinedMessageChannel::create('priorityOrders', ['priorityOutbox', 'priorityProcessing']), + BatchForwardingConfiguration::create('standardOutbox') + ->withEndpointId('sharedOutboxPublisher'), + BatchForwardingConfiguration::create('priorityOutbox') + ->withEndpointId('sharedOutboxPublisher'), + DbalBackedMessageChannelBuilder::create('standardOutbox'), + DbalBackedMessageChannelBuilder::create('priorityOutbox'), + DbalBackedMessageChannelBuilder::create('standardProcessing'), + DbalBackedMessageChannelBuilder::create('priorityProcessing'), ]), licenceKey: LicenceTesting::VALID_LICENCE, ); - $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); - $messaging->sendCommandWithRoutingKey('order.register', 'latte'); - $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + $messaging->sendCommandWithRoutingKey('order.registerStandard', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.registerPriority', 'flat white'); + $messaging->sendCommandWithRoutingKey('order.registerStandard', 'latte'); - $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + $messaging->run('sharedOutboxPublisher', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); - $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); + $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('standardProcessing'))); + $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('priorityProcessing'))); + $this->assertNull($messaging->getMessageChannel('standardOutbox')->receive()); + $this->assertNull($messaging->getMessageChannel('priorityOutbox')->receive()); } public function test_failed_send_of_single_message_on_target_channel_releases_only_that_message(): void @@ -323,6 +372,7 @@ public function register(string $order): void ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ CombinedMessageChannel::create('orders', ['outbox', 'failingProcessing']), + BatchForwardingConfiguration::create('outbox'), DbalBackedMessageChannelBuilder::create('outbox'), SimpleMessageChannelBuilder::create('failingProcessing', new FailOnceOnPayloadPollableChannel('cappuccino')), PollableChannelConfiguration::create('failingProcessing', RetryTemplateBuilder::fixedBackOff(1)->maxRetryAttempts(1)->build()), @@ -361,6 +411,7 @@ public function register(string $order): void ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), DbalBackedMessageChannelBuilder::create('outbox') ->withReceiveTimeout(3000), DbalBackedMessageChannelBuilder::create('orderProcessing'), @@ -399,6 +450,7 @@ public function register(string $order): void ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), DbalBackedMessageChannelBuilder::create('outbox') ->withFinalFailureStrategy(FinalFailureStrategy::IGNORE), DbalBackedMessageChannelBuilder::create('orderProcessing'), @@ -438,6 +490,7 @@ public function register(string $order): void ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), DbalBackedMessageChannelBuilder::create('outbox') ->withFinalFailureStrategy(FinalFailureStrategy::STOP), DbalBackedMessageChannelBuilder::create('orderProcessing'), @@ -461,7 +514,7 @@ public function register(string $order): void $this->assertCount(3, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); } - public function test_batched_forwarding_does_not_apply_to_in_memory_source_channels(): void + public function test_batch_forwarding_configuration_for_non_dbal_channel_fails_at_compile_time(): void { $orderService = new class () { #[Asynchronous('orders')] @@ -471,27 +524,48 @@ public function register(string $order): void } }; - $messaging = EcotoneLite::bootstrapFlowTesting( + $this->expectException(ConfigurationException::class); + + EcotoneLite::bootstrapFlowTesting( [$orderService::class], [$orderService], ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE])) ->withExtensionObjects([ CombinedMessageChannel::create('orders', ['inMemoryOutbox', 'inMemoryProcessing']), + BatchForwardingConfiguration::create('inMemoryOutbox'), SimpleMessageChannelBuilder::createQueueChannel('inMemoryOutbox'), SimpleMessageChannelBuilder::createQueueChannel('inMemoryProcessing'), ]), licenceKey: LicenceTesting::VALID_LICENCE, ); + } - $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); - $messaging->sendCommandWithRoutingKey('order.register', 'latte'); - $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + public function test_batch_forwarding_configuration_for_unknown_channel_fails_at_compile_time(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; - $messaging->run('inMemoryOutbox', ExecutionPollingMetadata::createWithTestingSetup()); + $this->expectException(ConfigurationException::class); - $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('inMemoryProcessing'))); - $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('inMemoryOutbox'))); + EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('misspelled_outbox'), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); } public function test_forwarded_message_does_not_carry_internal_collector_header(): void @@ -512,6 +586,7 @@ public function register(string $order): void ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), GlobalPollableChannelConfiguration::createWithDefaults()->withCollector($collectorEnabled), @@ -532,7 +607,131 @@ public function register(string $order): void } } - public function test_failed_forwarding_keeps_all_messages_available_on_outbox(): void + public function test_using_batched_outbox_directly_as_asynchronous_channel_fails_at_compile_time(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + + #[Asynchronous('outbox')] + #[CommandHandler('order.registerDirectly', endpointId: 'orderRegisterDirectlyEndpoint')] + public function registerDirectly(string $order): void + { + } + }; + + $this->expectException(ConfigurationException::class); + + EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + + public function test_using_batched_outbox_directly_as_asynchronous_channel_without_combined_channel_fails_at_compile_time(): void + { + $orderService = new class () { + #[Asynchronous('outbox')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $this->expectException(ConfigurationException::class); + + EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + BatchForwardingConfiguration::create('outbox'), + DbalBackedMessageChannelBuilder::create('outbox'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + + public function test_using_batched_outbox_as_output_channel_fails_at_compile_time(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + + #[CommandHandler('order.startFlow', outputChannelName: 'outbox')] + public function startFlow(string $order): string + { + return $order; + } + }; + + $this->expectException(ConfigurationException::class); + + EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + + public function test_combined_channel_without_batch_forwarding_configuration_keeps_one_message_per_run(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + + $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); + $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); + } + + public function test_failed_forwarding_releases_all_messages_back_to_outbox(): void { $orderService = new class () { #[Asynchronous('orders')] @@ -549,6 +748,7 @@ public function register(string $order): void ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ CombinedMessageChannel::create('orders', ['outbox', 'failingProcessing']), + BatchForwardingConfiguration::create('outbox'), DbalBackedMessageChannelBuilder::create('outbox'), SimpleMessageChannelBuilder::create('failingProcessing', new FailingPollableChannel()), ]), @@ -559,15 +759,158 @@ public function register(string $order): void $messaging->sendCommandWithRoutingKey('order.register', 'latte'); $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); - $forwardingFailed = false; + $messaging->run('outbox', ExecutionPollingMetadata::createWithFinishWhenNoMessages()); + + $this->assertCount(3, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); + } + + public function test_error_channel_is_not_involved_in_failed_forwarding_for_release_and_ignore_strategies(): void + { + foreach ([FinalFailureStrategy::RELEASE, FinalFailureStrategy::IGNORE] as $finalFailureStrategy) { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [ + DbalConnectionFactory::class => $this->getConnectionFactory(), + $orderService, + 'alwaysFailingDelivery' => new AlwaysFailOnPayloadChannelInterceptor('cappuccino'), + ], + ServiceConfiguration::createWithDefaults() + ->withDefaultErrorChannel('customErrorChannel') + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), + DbalBackedMessageChannelBuilder::create('outbox') + ->withFinalFailureStrategy($finalFailureStrategy), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + SimpleMessageChannelBuilder::createQueueChannel('customErrorChannel'), + SimpleChannelInterceptorBuilder::create('orderProcessing', 'alwaysFailingDelivery'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithFinishWhenNoMessages()); + + $this->assertNull($messaging->getMessageChannel('customErrorChannel')->receive()); + $this->assertSame(['espresso'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('orderProcessing')))); + $expectedRemainingOnOutbox = $finalFailureStrategy === FinalFailureStrategy::RELEASE ? 1 : 0; + $this->assertCount($expectedRemainingOnOutbox, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); + } + } + + public function test_error_channel_is_not_involved_in_failed_forwarding_for_stop_strategy(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [ + DbalConnectionFactory::class => $this->getConnectionFactory(), + $orderService, + 'alwaysFailingDelivery' => new AlwaysFailOnPayloadChannelInterceptor('cappuccino'), + ], + ServiceConfiguration::createWithDefaults() + ->withDefaultErrorChannel('customErrorChannel') + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), + DbalBackedMessageChannelBuilder::create('outbox') + ->withFinalFailureStrategy(FinalFailureStrategy::STOP), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + SimpleMessageChannelBuilder::createQueueChannel('customErrorChannel'), + SimpleChannelInterceptorBuilder::create('orderProcessing', 'alwaysFailingDelivery'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); + + $consumerStopped = false; try { $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); } catch (RuntimeException) { - $forwardingFailed = true; + $consumerStopped = true; } - $this->assertTrue($forwardingFailed); - $this->assertCount(3, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); + $this->assertTrue($consumerStopped); + $this->assertNull($messaging->getMessageChannel('customErrorChannel')->receive()); + $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); + } + + public function test_messages_claimed_by_another_process_are_not_published_until_claim_expires(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); + $messaging->sendCommandWithRoutingKey('order.register', 'latte'); + + $this->claimSingleOutboxRowAsAnotherProcess('outbox', claimValidForSeconds: 3600); + $messaging->run('outbox', ExecutionPollingMetadata::createWithFinishWhenNoMessages()); + + $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); + + $this->expireForeignOutboxClaims('outbox'); + $messaging->run('outbox', ExecutionPollingMetadata::createWithFinishWhenNoMessages()); + + $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); + $this->assertNull($messaging->getMessageChannel('outbox')->receive()); + } + + private function claimSingleOutboxRowAsAnotherProcess(string $channelName, int $claimValidForSeconds): void + { + $connection = $this->getConnection(); + $rowId = $connection->fetchOne('SELECT id FROM enqueue WHERE queue = ? AND delivery_id IS NULL ORDER BY published_at ASC LIMIT 1', [$channelName]); + $connection->executeStatement( + 'UPDATE enqueue SET delivery_id = ?, redeliver_after = ? WHERE id = ?', + ['019890ab-0000-7000-8000-000000000001', time() + $claimValidForSeconds, $rowId], + ); + } + + private function expireForeignOutboxClaims(string $channelName): void + { + $this->getConnection()->executeStatement( + 'UPDATE enqueue SET redeliver_after = ? WHERE queue = ? AND delivery_id IS NOT NULL', + [time() - 10, $channelName], + ); } public function test_failed_delivery_of_single_message_releases_only_that_message_without_duplicates(): void @@ -627,6 +970,7 @@ public function register(string $order): void ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), SimpleChannelInterceptorBuilder::create('orderProcessing', 'failingDeliveryInterceptor'), diff --git a/packages/Ecotone/src/Messaging/Channel/BatchForwardingConfiguration.php b/packages/Ecotone/src/Messaging/Channel/BatchForwardingConfiguration.php new file mode 100644 index 000000000..6afb61cde --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/BatchForwardingConfiguration.php @@ -0,0 +1,67 @@ + 0, 'Max forwarding batch size must be a positive number.'); + $this->maxForwardingBatchSize = $maxForwardingBatchSize; + + return $this; + } + + public function withEndpointId(string $endpointId): self + { + Assert::notNullAndEmpty($endpointId, 'Endpoint id for batch forwarding can not be empty.'); + $this->endpointId = $endpointId; + + return $this; + } + + public function getEndpointId(): string + { + return $this->endpointId ?? $this->channelName; + } + + public function isEnabled(): bool + { + return $this->enabled; + } + + public function getChannelName(): string + { + return $this->channelName; + } + + public function getMaxForwardingBatchSize(): int + { + return $this->maxForwardingBatchSize; + } +} diff --git a/packages/Ecotone/src/Messaging/Channel/BatchForwardingSourceChannel.php b/packages/Ecotone/src/Messaging/Channel/BatchForwardingSourceChannel.php deleted file mode 100644 index 603a2ec03..000000000 --- a/packages/Ecotone/src/Messaging/Channel/BatchForwardingSourceChannel.php +++ /dev/null @@ -1,15 +0,0 @@ -getCombinedChannels(), 1) as $relayTargetChannel) { - $combinedChannelRelayTargets[$relayTargetChannel] = true; - } - } - $takenChannelNames = []; foreach ($pollableChannelConfigurations as $pollableChannelConfiguration) { if (in_array($pollableChannelConfiguration->getChannelName(), $takenChannelNames)) { @@ -61,10 +52,6 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO foreach ($pollableMessageChannels as $pollableMessageChannel) { - if (isset($combinedChannelRelayTargets[$pollableMessageChannel->getMessageChannelName()])) { - continue; - } - $channelConfiguration = $globalPollableChannelConfiguration; foreach ($pollableChannelConfigurations as $pollableChannelConfiguration) { @@ -116,7 +103,6 @@ public function canHandle($extensionObject): bool return $extensionObject instanceof PollableChannelConfiguration || $extensionObject instanceof GlobalPollableChannelConfiguration - || $extensionObject instanceof CombinedMessageChannel /** Dynamic and RoundRobin are proxies, therefore should not be intercepted */ || ($extensionObject instanceof MessageChannelBuilder && $extensionObject->isPollable() && ! ($extensionObject instanceof DynamicMessageChannelBuilder)); } diff --git a/packages/Ecotone/src/Messaging/Channel/CombinedChannelForwardingConfiguration.php b/packages/Ecotone/src/Messaging/Channel/CombinedChannelForwardingConfiguration.php deleted file mode 100644 index f379774aa..000000000 --- a/packages/Ecotone/src/Messaging/Channel/CombinedChannelForwardingConfiguration.php +++ /dev/null @@ -1,25 +0,0 @@ - $maxForwardingBatchSizes indexed by relay source channel name - */ - public function __construct(private array $maxForwardingBatchSizes = []) - { - } - - public function getMaxForwardingBatchSizeFor(string $channelName): int - { - return $this->maxForwardingBatchSizes[$channelName] ?? self::DEFAULT_MAX_BATCH_SIZE; - } -} diff --git a/packages/Ecotone/src/Messaging/Channel/CombinedMessageChannel.php b/packages/Ecotone/src/Messaging/Channel/CombinedMessageChannel.php index 218eb0f4f..bdf05f368 100644 --- a/packages/Ecotone/src/Messaging/Channel/CombinedMessageChannel.php +++ b/packages/Ecotone/src/Messaging/Channel/CombinedMessageChannel.php @@ -11,8 +11,6 @@ */ final class CombinedMessageChannel { - private ?int $maxForwardingBatchSize = null; - private function __construct(private string $referenceName, private array $combinedChannels) { Assert::notNull($referenceName, 'Reference name can not be null'); @@ -27,19 +25,6 @@ public static function create(string $referenceName, array $combinedChannels): s return new self($referenceName, $combinedChannels); } - public function withMaxForwardingBatchSize(int $maxForwardingBatchSize): self - { - Assert::isTrue($maxForwardingBatchSize > 0, 'Max forwarding batch size must be a positive number.'); - $this->maxForwardingBatchSize = $maxForwardingBatchSize; - - return $this; - } - - public function getMaxForwardingBatchSize(): ?int - { - return $this->maxForwardingBatchSize; - } - public function getReferenceName(): string { return $this->referenceName; diff --git a/packages/Ecotone/src/Messaging/Config/Annotation/ModuleConfiguration/AsynchronousModule.php b/packages/Ecotone/src/Messaging/Config/Annotation/ModuleConfiguration/AsynchronousModule.php index b53cf8f45..596911573 100644 --- a/packages/Ecotone/src/Messaging/Config/Annotation/ModuleConfiguration/AsynchronousModule.php +++ b/packages/Ecotone/src/Messaging/Config/Annotation/ModuleConfiguration/AsynchronousModule.php @@ -9,7 +9,6 @@ use Ecotone\Messaging\Attribute\EndpointAnnotation; use Ecotone\Messaging\Attribute\ModuleAnnotation; use Ecotone\Messaging\Attribute\StreamBasedSource; -use Ecotone\Messaging\Channel\CombinedChannelForwardingConfiguration; use Ecotone\Messaging\Channel\CombinedMessageChannel; use Ecotone\Messaging\Channel\MessageChannelBuilder; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; @@ -17,7 +16,6 @@ use Ecotone\Messaging\Config\Annotation\AnnotationModule; use Ecotone\Messaging\Config\Configuration; use Ecotone\Messaging\Config\ConfigurationException; -use Ecotone\Messaging\Config\Container\Definition; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ModuleReferenceSearchService; use Ecotone\Messaging\Config\ServiceConfiguration; @@ -141,7 +139,6 @@ public function canHandle($extensionObject): bool public function prepare(Configuration $messagingConfiguration, array $extensionObjects, ModuleReferenceSearchService $moduleReferenceSearchService, InterfaceToCallRegistry $interfaceToCallRegistry): void { $endpointChannels = $this->resolveChannels($extensionObjects); - $this->registerCombinedChannelForwardingConfiguration($messagingConfiguration, $extensionObjects); $serviceConfiguration = ExtensionObjectResolver::resolveUnique(ServiceConfiguration::class, $extensionObjects, ServiceConfiguration::createWithDefaults()); $pollingMetadata = ExtensionObjectResolver::resolve(PollingMetadata::class, $extensionObjects); $polingChannelBuilders = ExtensionObjectResolver::resolve(SimpleMessageChannelBuilder::class, $extensionObjects); @@ -243,28 +240,6 @@ public function resolveChannels(array $extensionObjects): array return $endpointChannels; } - private function registerCombinedChannelForwardingConfiguration(Configuration $messagingConfiguration, array $extensionObjects): void - { - $maxForwardingBatchSizes = []; - /** @var CombinedMessageChannel $combinedMessageChannel */ - foreach (ExtensionObjectResolver::resolve(CombinedMessageChannel::class, $extensionObjects) as $combinedMessageChannel) { - $maxForwardingBatchSize = $combinedMessageChannel->getMaxForwardingBatchSize(); - if ($maxForwardingBatchSize === null) { - continue; - } - - $relaySourceChannel = $combinedMessageChannel->getCombinedChannels()[0]; - $maxForwardingBatchSizes[$relaySourceChannel] = isset($maxForwardingBatchSizes[$relaySourceChannel]) - ? min($maxForwardingBatchSizes[$relaySourceChannel], $maxForwardingBatchSize) - : $maxForwardingBatchSize; - } - - $messagingConfiguration->registerServiceDefinition( - CombinedChannelForwardingConfiguration::class, - new Definition(CombinedChannelForwardingConfiguration::class, [$maxForwardingBatchSizes]) - ); - } - public function handleRoutingEvent(RoutingEvent $event): void { $registration = $event->getRegistration(); diff --git a/packages/Ecotone/src/Messaging/Config/Configuration.php b/packages/Ecotone/src/Messaging/Config/Configuration.php index 085be7f9b..560d24fb6 100644 --- a/packages/Ecotone/src/Messaging/Config/Configuration.php +++ b/packages/Ecotone/src/Messaging/Config/Configuration.php @@ -67,6 +67,12 @@ public function registerChannelInterceptor(ChannelInterceptorBuilder $channelInt */ public function registerAsynchronousEndpoint(array|string $asynchronousChannelNames, string $targetEndpointId): Configuration; + /** + * Marks given Message Channel as batched forwarding outbox, which removes its standard Message Channel consumer + * and ensures the Channel is not used as execution or output channel. + */ + public function registerBatchForwardingSourceChannel(string $channelName): Configuration; + /** * @param MethodInterceptorBuilder $methodInterceptor * @return Configuration diff --git a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php index 0e72100e7..e8c797db6 100644 --- a/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php +++ b/packages/Ecotone/src/Messaging/Config/MessagingSystemConfiguration.php @@ -11,9 +11,8 @@ use Ecotone\Lite\Test\TestConfiguration; use Ecotone\Messaging\Attribute\Asynchronous; use Ecotone\Messaging\Attribute\AsynchronousRunningEndpoint; -use Ecotone\Messaging\Channel\BatchForwardingSourceChannel; +use Ecotone\Messaging\Channel\BatchForwardingConfiguration; use Ecotone\Messaging\Channel\ChannelInterceptorBuilder; -use Ecotone\Messaging\Channel\CombinedChannelForwardingConfiguration; use Ecotone\Messaging\Channel\EventDrivenChannelInterceptorAdapter; use Ecotone\Messaging\Channel\MessageChannelBuilder; use Ecotone\Messaging\Channel\PollableChannelInterceptorAdapter; @@ -46,9 +45,7 @@ use Ecotone\Messaging\Endpoint\PollingConsumer\AsyncHandlerAnnotationRegistry; use Ecotone\Messaging\Endpoint\PollingMetadata; use Ecotone\Messaging\Gateway\MessagingEntrypointService; -use Ecotone\Messaging\Handler\Bridge\BatchForwardingBridge; use Ecotone\Messaging\Handler\Bridge\BridgeBuilder; -use Ecotone\Messaging\Handler\ChannelResolver; use Ecotone\Messaging\Handler\Gateway\GatewayProxyBuilder; use Ecotone\Messaging\Handler\InterceptedEndpoint; use Ecotone\Messaging\Handler\InterfaceToCall; @@ -61,7 +58,6 @@ use Ecotone\Messaging\Handler\Processor\MethodInvoker\InterceptorWithPointCut; use Ecotone\Messaging\Handler\Processor\MethodInvoker\MethodInterceptorBuilder; use Ecotone\Messaging\Handler\Recoverability\RetryTemplateBuilder; -use Ecotone\Messaging\Handler\ServiceActivator\ServiceActivatorBuilder; use Ecotone\Messaging\Handler\ServiceActivator\UninterruptibleServiceActivator; use Ecotone\Messaging\Handler\Transformer\RoutingSlipPrepender; use Ecotone\Messaging\Handler\Type; @@ -146,6 +142,14 @@ final class MessagingSystemConfiguration implements Configuration private array $messageConverterReferenceNames = []; private ?ModuleReferenceSearchService $moduleReferenceSearchService; private array $asynchronousEndpoints = []; + /** + * @var array + */ + private array $batchForwardingSourceChannels = []; + /** + * @var string[] + */ + private array $declaredBatchForwardingChannels = []; private ServiceConfiguration $applicationConfiguration; /** * @var string[] @@ -224,6 +228,12 @@ function ($extensionObject) { $this->isRunningForTest = ExtensionObjectResolver::contains(TestConfiguration::class, $extensionObjects); + foreach (ExtensionObjectResolver::resolve(BatchForwardingConfiguration::class, $extensionObjects) as $batchForwardingConfiguration) { + if ($batchForwardingConfiguration->isEnabled()) { + $this->declaredBatchForwardingChannels[] = $batchForwardingConfiguration->getChannelName(); + } + } + $extensionObjects[] = $serviceConfiguration; if ($serviceConfiguration->getLicenceKey() !== null) { @@ -414,6 +424,16 @@ private function configureAsynchronousEndpoints(InterfaceToCallRegistry $interfa /** @var array $asyncHandlerAnnotations */ $asyncHandlerAnnotations = []; + if ($this->batchForwardingSourceChannels !== [] && ! $this->isRunningForEnterpriseLicence) { + throw LicensingException::create(sprintf('Batch forwarding for Message Channel `%s` is available only with Ecotone Enterprise licence.', array_key_first($this->batchForwardingSourceChannels))); + } + foreach ($this->declaredBatchForwardingChannels as $declaredBatchForwardingChannel) { + if (! isset($this->batchForwardingSourceChannels[$declaredBatchForwardingChannel])) { + throw ConfigurationException::create("Batch forwarding was configured for Message Channel `{$declaredBatchForwardingChannel}`, yet no module enabled it for that channel. Batch forwarding requires a Dbal backed Message Channel with matching name and the Dbal package enabled."); + } + } + $this->verifyBatchForwardingSourceChannelsAreNotUsedForExecution($this->batchForwardingSourceChannels); + foreach ($this->asynchronousEndpoints as $targetEndpointId => $asynchronousMessageChannels) { $asynchronousMessageChannel = array_shift($asynchronousMessageChannels); if (! isset($this->channelBuilders[$asynchronousMessageChannel]) && ! isset($this->defaultChannelBuilders[$asynchronousMessageChannel])) { @@ -534,19 +554,6 @@ private function configureAsynchronousEndpoints(InterfaceToCallRegistry $interfa ) ); - $relaySourceChannels = []; - foreach ($this->asynchronousEndpoints as $asynchronousMessageChannels) { - foreach (array_slice($asynchronousMessageChannels, 0, -1) as $relaySourceChannel) { - $relaySourceChannels[$relaySourceChannel] = true; - } - } - if ($relaySourceChannels !== []) { - $this->registerServiceDefinition( - CombinedChannelForwardingConfiguration::class, - new Definition(CombinedChannelForwardingConfiguration::class, [[]]) - ); - } - foreach ($asynchronousChannels as $asynchronousChannel) { Assert::isTrue($this->channelBuilders[$asynchronousChannel]->isPollable(), "Asynchronous Message Channel {$asynchronousChannel} must be Pollable"); // needed for correct around intercepting, otherwise requestReply is outside of around interceptor scope @@ -554,20 +561,10 @@ private function configureAsynchronousEndpoints(InterfaceToCallRegistry $interfa * This is Bridge that will fetch the message and make use of routing_slip to target it * message handler. */ - if (isset($relaySourceChannels[$asynchronousChannel]) && $this->channelBuilders[$asynchronousChannel] instanceof BatchForwardingSourceChannel) { - $this->messageHandlerBuilders[$asynchronousChannel] = ServiceActivatorBuilder::createWithDefinition( - new Definition(BatchForwardingBridge::class, [ - new ChannelReference($asynchronousChannel), - new Reference(ChannelResolver::class), - new Reference(LoggingGateway::class), - $this->isRunningForEnterpriseLicence, - $asynchronousChannel, - new Reference(CombinedChannelForwardingConfiguration::class), - ]), - 'handle', - ) - ->withInputChannelName($asynchronousChannel) - ->withEndpointId($asynchronousChannel); + if (isset($this->batchForwardingSourceChannels[$asynchronousChannel])) { + if (! isset($this->channelAdapters[$asynchronousChannel])) { + unset($this->pollingMetadata[$asynchronousChannel]); + } continue; } @@ -580,6 +577,29 @@ private function configureAsynchronousEndpoints(InterfaceToCallRegistry $interfa $this->asynchronousEndpoints = []; } + /** + * @param array $batchForwardingSourceChannels + */ + private function verifyBatchForwardingSourceChannelsAreNotUsedForExecution(array $batchForwardingSourceChannels): void + { + if ($batchForwardingSourceChannels === []) { + return; + } + + foreach ($this->asynchronousEndpoints as $targetEndpointId => $asynchronousMessageChannels) { + $executionChannel = $asynchronousMessageChannels[array_key_last($asynchronousMessageChannels)]; + if (isset($batchForwardingSourceChannels[$executionChannel])) { + throw ConfigurationException::create("Channel `{$executionChannel}` is a batched forwarding outbox, which only pushes messages forward to the next channel. It can not be used as execution channel for endpoint `{$targetEndpointId}`. Point the endpoint at the Combined Message Channel instead."); + } + } + + foreach ($this->messageHandlerBuilders as $messageHandlerBuilder) { + if ($messageHandlerBuilder instanceof MessageHandlerBuilderWithOutputChannel && isset($batchForwardingSourceChannels[$messageHandlerBuilder->getOutputMessageChannelName()])) { + throw ConfigurationException::create("Channel `{$messageHandlerBuilder->getOutputMessageChannelName()}` is a batched forwarding outbox, which only pushes messages forward to the next channel. It can not be used as output channel of {$messageHandlerBuilder}. Point the output at the Combined Message Channel instead."); + } + } + } + /** * @return void */ @@ -792,6 +812,14 @@ public function registerAroundMethodInterceptor(AroundInterceptorBuilder $around /** * @inheritDoc */ + public function registerBatchForwardingSourceChannel(string $channelName): Configuration + { + Assert::isTrue(! isset($this->batchForwardingSourceChannels[$channelName]), "Batch forwarding for Message Channel `{$channelName}` is already registered."); + $this->batchForwardingSourceChannels[$channelName] = true; + + return $this; + } + public function registerAsynchronousEndpoint(array|string $asynchronousChannelNames, string $targetEndpointId): Configuration { $this->asynchronousEndpoints[$targetEndpointId] = is_string($asynchronousChannelNames) ? [$asynchronousChannelNames] : $asynchronousChannelNames; diff --git a/packages/Ecotone/src/Messaging/Config/ModuleClassList.php b/packages/Ecotone/src/Messaging/Config/ModuleClassList.php index 1046e033c..227e45b5e 100644 --- a/packages/Ecotone/src/Messaging/Config/ModuleClassList.php +++ b/packages/Ecotone/src/Messaging/Config/ModuleClassList.php @@ -9,6 +9,7 @@ use Ecotone\Amqp\Publisher\AmqpMessagePublisherModule; use Ecotone\Amqp\Transaction\AmqpTransactionModule; use Ecotone\DataProtection\Configuration\DataProtectionModule; +use Ecotone\Dbal\BatchForwarding\DbalBatchForwardingModule; use Ecotone\Dbal\Configuration\DbalConnectionModule; use Ecotone\Dbal\Configuration\DbalPublisherModule; use Ecotone\Dbal\Database\DatabaseSetupModule; @@ -155,6 +156,7 @@ class ModuleClassList ]; public const DBAL_MODULES = [ + DbalBatchForwardingModule::class, DbalConnectionModule::class, DbalDeadLetterModule::class, ObjectManagerModule::class, diff --git a/packages/Ecotone/src/Messaging/Endpoint/InboundChannelAdapter/InboundChannelAdapterBuilder.php b/packages/Ecotone/src/Messaging/Endpoint/InboundChannelAdapter/InboundChannelAdapterBuilder.php index 442ab8b6c..8ceea41b3 100644 --- a/packages/Ecotone/src/Messaging/Endpoint/InboundChannelAdapter/InboundChannelAdapterBuilder.php +++ b/packages/Ecotone/src/Messaging/Endpoint/InboundChannelAdapter/InboundChannelAdapterBuilder.php @@ -30,6 +30,7 @@ class InboundChannelAdapterBuilder extends InterceptedChannelAdapterBuilder private string $referenceName; private string $requestChannelName; private ?object $directObject = null; + private bool $continuousPolling = false; private function __construct(string $requestChannelName, string $referenceName, private InterfaceToCall $interfaceToCall) { @@ -115,9 +116,16 @@ public function withRequiredInterceptorNames(iterable $interceptorNames): self return $this; } + public function withContinuousPolling(bool $continuousPolling = true): self + { + $this->continuousPolling = $continuousPolling; + + return $this; + } + protected function withContinuesPolling(): bool { - return false; + return $this->continuousPolling; } public function compile(MessagingContainerBuilder $builder): Definition diff --git a/packages/Ecotone/src/Messaging/Endpoint/InterceptedChannelAdapterBuilder.php b/packages/Ecotone/src/Messaging/Endpoint/InterceptedChannelAdapterBuilder.php index e8c0f9811..7e6039cb9 100644 --- a/packages/Ecotone/src/Messaging/Endpoint/InterceptedChannelAdapterBuilder.php +++ b/packages/Ecotone/src/Messaging/Endpoint/InterceptedChannelAdapterBuilder.php @@ -44,7 +44,7 @@ abstract class InterceptedChannelAdapterBuilder implements ChannelAdapterConsume protected function withContinuesPolling(): bool { - return true; + return false; } abstract protected function getInterceptedInterface(InterfaceToCallRegistry $interfaceToCallRegistry): InterfaceToCall; @@ -76,7 +76,7 @@ public function registerConsumer(MessagingContainerBuilder $builder): void new Reference(AsyncHandlerAnnotationRegistry::class), new Reference(AsyncEndpointAnnotationContext::class), ]); - $builder->registerPollingEndpoint($this->endpointId, $consumerRunner); + $builder->registerPollingEndpoint($this->endpointId, $consumerRunner, $this->withContinuesPolling()); } private function getErrorInterceptorReference(MessagingContainerBuilder $builder): AroundInterceptorBuilder diff --git a/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php b/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php deleted file mode 100644 index 370588d4a..000000000 --- a/packages/Ecotone/src/Messaging/Handler/Bridge/BatchForwardingBridge.php +++ /dev/null @@ -1,289 +0,0 @@ -batchForwardingEnabled) { - return $message; - } - - $targetChannelName = $this->nextRoutingSlipChannel($message); - if ($targetChannelName === null || ! $this->isPollable($this->channelResolver->resolve($targetChannelName))) { - return $message; - } - - $acknowledgementCallback = $this->acknowledgementCallbackOf($message); - if ($acknowledgementCallback !== null && ! $acknowledgementCallback->isAutoAcked()) { - return $message; - } - - $drainedMessages = $this->drainSourceChannel(); - if ($drainedMessages === []) { - return $message; - } - - foreach ($this->groupByTargetChannel($message, $drainedMessages) as $groupTargetChannelName => $groupedMessages) { - $this->forwardGroup((string) $groupTargetChannelName, $groupedMessages, $message); - } - - return null; - } - - /** - * @return Message[] - */ - private function drainSourceChannel(): array - { - $maxBatchSize = $this->forwardingConfiguration->getMaxForwardingBatchSizeFor($this->sourceChannelName); - $withoutWaitingPollingMetadata = PollingMetadata::create($this->sourceChannelName) - ->setExecutionTimeLimitInMilliseconds(1); - $drainedMessages = []; - while (count($drainedMessages) < $maxBatchSize - 1) { - $nextMessage = $this->sourceChannel->receiveWithTimeout($withoutWaitingPollingMetadata); - if ($nextMessage === null) { - break; - } - $drainedMessages[] = $nextMessage; - } - - return $drainedMessages; - } - - /** - * @param Message[] $drainedMessages - * @return array - */ - private function groupByTargetChannel(Message $polledMessage, array $drainedMessages): array - { - $polledMessageTargetChannelName = $this->nextRoutingSlipChannel($polledMessage); - $groups = [$polledMessageTargetChannelName => [$polledMessage]]; - foreach ($drainedMessages as $drainedMessage) { - $targetChannelName = $this->nextRoutingSlipChannel($drainedMessage); - if ($targetChannelName === null) { - $this->releaseWithoutForwarding($drainedMessage); - - continue; - } - $groups[$targetChannelName][] = $drainedMessage; - } - - return $groups; - } - - /** - * @param Message[] $groupedMessages - */ - private function forwardGroup(string $targetChannelName, array $groupedMessages, Message $polledMessage): void - { - $targetChannel = $this->channelResolver->resolve($targetChannelName); - $messagesToForward = array_map(fn (Message $groupedMessage) => $this->advanceRoutingSlip($groupedMessage), $groupedMessages); - - if ($this->supportsBatchMessages($targetChannel)) { - try { - $targetChannel->send(MessageBuilder::withPayload($this->combineIntoBatch($messagesToForward))->build()); - } catch (Throwable $exception) { - $this->releaseFailedDelivery($groupedMessages, $polledMessage, $targetChannelName, $exception); - - return; - } - $this->acknowledgeAllExcept($groupedMessages, $polledMessage); - - return; - } - - foreach ($messagesToForward as $messageIndex => $messageToForward) { - $groupedMessage = $groupedMessages[$messageIndex]; - try { - $targetChannel->send($messageToForward); - } catch (Throwable $exception) { - $this->releaseFailedDelivery([$groupedMessage], $polledMessage, $targetChannelName, $exception); - - continue; - } - if ($groupedMessage !== $polledMessage) { - $this->acknowledge($groupedMessage); - } - } - } - - /** - * @param Message[] $failedMessages - */ - private function releaseFailedDelivery(array $failedMessages, Message $polledMessage, string $targetChannelName, Throwable $exception): void - { - if ($exception instanceof ConnectionException || in_array($polledMessage, $failedMessages, true)) { - throw $exception; - } - - $shouldStopConsumer = false; - foreach ($failedMessages as $failedMessage) { - $acknowledgementCallback = $this->acknowledgementCallbackOf($failedMessage); - if ($acknowledgementCallback === null) { - continue; - } - - $failureStrategy = $acknowledgementCallback->getFailureStrategy(); - match ($failureStrategy) { - FinalFailureStrategy::STOP => $acknowledgementCallback->release(), - FinalFailureStrategy::IGNORE => $acknowledgementCallback->reject(), - FinalFailureStrategy::RELEASE => $acknowledgementCallback->release(), - FinalFailureStrategy::RESEND => $acknowledgementCallback->resend(), - }; - $shouldStopConsumer = $shouldStopConsumer || $failureStrategy === FinalFailureStrategy::STOP; - $this->logger->info( - sprintf('Message with id `%s` handled with `%s` failure strategy, as delivery to `%s` failed. Due to %s', $failedMessage->getHeaders()->getMessageId(), $failureStrategy->value, $targetChannelName, $exception->getMessage()), - $failedMessage, - ['exception' => $exception], - ); - } - - if ($shouldStopConsumer) { - throw $exception; - } - } - - /** - * @param Message[] $groupedMessages - */ - private function acknowledgeAllExcept(array $groupedMessages, Message $polledMessage): void - { - foreach ($groupedMessages as $groupedMessage) { - if ($groupedMessage !== $polledMessage) { - $this->acknowledge($groupedMessage); - } - } - } - - /** - * @param Message[] $messages - */ - private function combineIntoBatch(array $messages): BatchMessage - { - $entries = []; - foreach ($messages as $message) { - $entries[] = ['payload' => $message->getPayload(), 'headers' => $this->transferableHeaders($message)]; - } - - return BatchMessage::fromEntries($entries); - } - - /** - * @return array - */ - private function transferableHeaders(Message $message): array - { - $headers = $message->getHeaders()->headers(); - if (isset($headers[MessageHeaders::CONSUMER_ACK_HEADER_LOCATION])) { - unset($headers[$headers[MessageHeaders::CONSUMER_ACK_HEADER_LOCATION]], $headers[MessageHeaders::CONSUMER_ACK_HEADER_LOCATION]); - } - unset($headers[MessageHeaders::POLLED_CHANNEL_NAME], $headers[MessageHeaders::CONSUMER_POLLING_METADATA]); - - return $headers; - } - - private function advanceRoutingSlip(Message $message): Message - { - $routingSlipChannels = explode(',', (string) $message->getHeaders()->get(MessageHeaders::ROUTING_SLIP)); - array_shift($routingSlipChannels); - $messageBuilder = MessageBuilder::fromMessage($message); - if ($routingSlipChannels === []) { - $messageBuilder->removeHeader(MessageHeaders::ROUTING_SLIP); - } else { - $messageBuilder->setHeader(MessageHeaders::ROUTING_SLIP, implode(',', $routingSlipChannels)); - } - - return $messageBuilder->build(); - } - - private function nextRoutingSlipChannel(Message $message): ?string - { - if (! $message->getHeaders()->containsKey(MessageHeaders::ROUTING_SLIP)) { - return null; - } - $routingSlip = (string) $message->getHeaders()->get(MessageHeaders::ROUTING_SLIP); - if ($routingSlip === '') { - return null; - } - - return explode(',', $routingSlip)[0]; - } - - private function isPollable(MessageChannel $channel): bool - { - return $this->unwrap($channel) instanceof PollableChannel; - } - - private function supportsBatchMessages(MessageChannel $channel): bool - { - $unwrappedChannel = $this->unwrap($channel); - - return $unwrappedChannel instanceof BatchSupportingMessageChannel && $unwrappedChannel->supportsBatchMessages(); - } - - private function unwrap(MessageChannel $channel): MessageChannel - { - if ($channel instanceof MessageChannelInterceptorAdapter) { - return $channel->getInternalMessageChannel(); - } - - return $channel; - } - - private function acknowledge(Message $message): void - { - $acknowledgementCallback = $this->acknowledgementCallbackOf($message); - if ($acknowledgementCallback?->isAutoAcked()) { - $acknowledgementCallback->accept(); - } - } - - private function releaseWithoutForwarding(Message $message): void - { - $this->acknowledgementCallbackOf($message)?->release(); - } - - private function acknowledgementCallbackOf(Message $message): ?AcknowledgementCallback - { - $headers = $message->getHeaders(); - if (! $headers->containsKey(MessageHeaders::CONSUMER_ACK_HEADER_LOCATION)) { - return null; - } - - return $headers->get($headers->get(MessageHeaders::CONSUMER_ACK_HEADER_LOCATION)); - } -} diff --git a/packages/Enqueue/src/EnqueueInboundChannelAdapterBuilder.php b/packages/Enqueue/src/EnqueueInboundChannelAdapterBuilder.php index 75871b5f1..a999854ff 100644 --- a/packages/Enqueue/src/EnqueueInboundChannelAdapterBuilder.php +++ b/packages/Enqueue/src/EnqueueInboundChannelAdapterBuilder.php @@ -168,6 +168,11 @@ public function withDeclareOnStartup(bool $declareOnStartup): self return $this; } + public function getFinalFailureStrategy(): FinalFailureStrategy + { + return $this->finalFailureStrategy; + } + /** * @return string */ diff --git a/packages/Kafka/tests/Integration/CombinedChannelForwardingTest.php b/packages/Kafka/tests/Integration/CombinedChannelForwardingTest.php index 9a144c590..676275a4e 100644 --- a/packages/Kafka/tests/Integration/CombinedChannelForwardingTest.php +++ b/packages/Kafka/tests/Integration/CombinedChannelForwardingTest.php @@ -8,13 +8,12 @@ use Ecotone\Kafka\Configuration\KafkaBrokerConfiguration; use Ecotone\Lite\EcotoneLite; use Ecotone\Messaging\Attribute\Asynchronous; +use Ecotone\Messaging\Channel\BatchForwardingConfiguration; use Ecotone\Messaging\Channel\CombinedMessageChannel; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; +use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; -use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; -use Ecotone\Messaging\Message; -use Ecotone\Messaging\PollableChannel; use Ecotone\Modelling\Attribute\CommandHandler; use Ecotone\Test\LicenceTesting; use PHPUnit\Framework\Attributes\RunTestsInSeparateProcesses; @@ -29,7 +28,7 @@ #[RunTestsInSeparateProcesses] final class CombinedChannelForwardingTest extends TestCase { - public function test_kafka_source_channel_keeps_one_message_per_handled_message(): void + public function test_batch_forwarding_configuration_for_kafka_source_channel_fails_at_compile_time(): void { $orderService = new class () { #[Asynchronous('orders')] @@ -39,39 +38,21 @@ public function register(string $order): void } }; + $this->expectException(ConfigurationException::class); + $uniqueId = Uuid::v7()->toRfc4122(); - $messaging = EcotoneLite::bootstrapFlowTesting( + EcotoneLite::bootstrapFlowTesting( [$orderService::class], [KafkaBrokerConfiguration::class => ConnectionTestCase::getConnection(), $orderService], ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::KAFKA_PACKAGE])) ->withExtensionObjects([ CombinedMessageChannel::create('orders', ['kafkaOutbox', 'orderProcessing']), + BatchForwardingConfiguration::create('kafkaOutbox'), KafkaMessageChannelBuilder::create('kafkaOutbox', topicName: $uniqueId, messageGroupId: $uniqueId), SimpleMessageChannelBuilder::createQueueChannel('orderProcessing'), ]), licenceKey: LicenceTesting::VALID_LICENCE, ); - - $messaging->sendCommandWithRoutingKey('order.register', 'espresso'); - $messaging->sendCommandWithRoutingKey('order.register', 'latte'); - $messaging->sendCommandWithRoutingKey('order.register', 'cappuccino'); - - $messaging->run('kafkaOutbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 30000)); - - $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); - } - - /** - * @return Message[] - */ - private function receiveAllFrom(PollableChannel $channel): array - { - $messages = []; - while ($message = $channel->receive()) { - $messages[] = $message; - } - - return $messages; } } From 9ac7b73b9ee2a3d75514ab17c0c96a31cfd19687 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Wed, 5 Aug 2026 08:00:00 +0200 Subject: [PATCH 19/25] perf: wire-format batch entries and low-overhead batch inserts for pollable channels Batch entries whose payload is already a string carrying own message id and timestamp skip the intermediate Message construction on the outbound path, falling back to full preparation otherwise. Header mapping short-circuits scalar values and the match-all mapping before entering type analysis. The Dbal batch insert binds parameters with direct ParameterType bindings and inlines constant columns, bypassing the doctrine type registry per value. --- .../Dbal/src/DbalOutboundChannelAdapter.php | 2 +- .../Dbal/src/EnqueueDbal/DbalProducer.php | 40 +++++----- .../OutboundMessageConverter.php | 74 ++++++++++++++++--- .../MessageConverter/DefaultHeaderMapper.php | 10 ++- .../src/EnqueueOutboundChannelAdapter.php | 8 ++ 5 files changed, 103 insertions(+), 31 deletions(-) diff --git a/packages/Dbal/src/DbalOutboundChannelAdapter.php b/packages/Dbal/src/DbalOutboundChannelAdapter.php index 20cf2f273..7dc72077a 100644 --- a/packages/Dbal/src/DbalOutboundChannelAdapter.php +++ b/packages/Dbal/src/DbalOutboundChannelAdapter.php @@ -70,7 +70,7 @@ protected function handleBatch(BatchMessage $batchMessage, Context $context): vo { $messagesToSend = []; foreach ($batchMessage->getEntries() as $entry) { - $outboundMessage = $this->prepareOutboundMessage($this->convertBatchEntryToMessage($entry)); + $outboundMessage = $this->prepareOutboundMessageFromBatchEntry($entry); $headers = $outboundMessage->getHeaders(); $headers[MessageHeaders::CONTENT_TYPE] = $outboundMessage->getContentType(); diff --git a/packages/Dbal/src/EnqueueDbal/DbalProducer.php b/packages/Dbal/src/EnqueueDbal/DbalProducer.php index ad3121d48..ed803255d 100644 --- a/packages/Dbal/src/EnqueueDbal/DbalProducer.php +++ b/packages/Dbal/src/EnqueueDbal/DbalProducer.php @@ -4,6 +4,7 @@ namespace Enqueue\Dbal; +use Doctrine\DBAL\ParameterType; use Ecotone\Messaging\Scheduling\Duration; use Interop\Queue\Destination; use Interop\Queue\Exception\Exception; @@ -22,21 +23,21 @@ class DbalProducer implements Producer { private const BATCH_INSERT_CHUNK_SIZE = 250; - private const COLUMN_TYPES = [ - 'id' => DbalType::GUID, - 'published_at' => DbalType::INTEGER, - 'body' => DbalType::TEXT, - 'headers' => DbalType::TEXT, - 'properties' => DbalType::TEXT, - 'priority' => DbalType::SMALLINT, - 'queue' => DbalType::STRING, - 'redelivered' => DbalType::SMALLINT, - 'delivery_id' => DbalType::STRING, - 'redeliver_after' => DbalType::BIGINT, - 'delayed_until' => DbalType::INTEGER, - 'time_to_live' => DbalType::INTEGER, + private const PARAMETERIZED_COLUMN_BINDINGS = [ + 'id' => ParameterType::STRING, + 'published_at' => ParameterType::INTEGER, + 'body' => ParameterType::STRING, + 'headers' => ParameterType::STRING, + 'properties' => ParameterType::STRING, + 'priority' => ParameterType::INTEGER, + 'queue' => ParameterType::STRING, + 'delayed_until' => ParameterType::INTEGER, + 'time_to_live' => ParameterType::INTEGER, ]; + private const CONSTANT_COLUMNS = 'redelivered, delivery_id, redeliver_after'; + private const CONSTANT_COLUMN_VALUES = 'FALSE, NULL, NULL'; + /** * @var int|null */ @@ -105,22 +106,23 @@ public function sendBatch(Destination $destination, array $messages): void private function insertRecords(array $records): int { - $columns = array_keys(self::COLUMN_TYPES); - $rowPlaceholders = '(' . implode(', ', array_fill(0, count($columns), '?')) . ')'; + $parameterizedColumns = array_keys(self::PARAMETERIZED_COLUMN_BINDINGS); + $rowPlaceholders = '(' . implode(', ', array_fill(0, count($parameterizedColumns), '?')) . ', ' . self::CONSTANT_COLUMN_VALUES . ')'; $sql = sprintf( - 'INSERT INTO %s (%s) VALUES %s', + 'INSERT INTO %s (%s, %s) VALUES %s', $this->context->getTableName(), - implode(', ', $columns), + implode(', ', $parameterizedColumns), + self::CONSTANT_COLUMNS, implode(', ', array_fill(0, count($records), $rowPlaceholders)), ); $parameters = []; $types = []; foreach ($records as $record) { - foreach ($columns as $column) { + foreach (self::PARAMETERIZED_COLUMN_BINDINGS as $column => $bindingType) { $parameters[] = $record[$column]; - $types[] = self::COLUMN_TYPES[$column]; + $types[] = $record[$column] === null ? ParameterType::NULL : $bindingType; } } diff --git a/packages/Ecotone/src/Messaging/Channel/PollableChannel/Serialization/OutboundMessageConverter.php b/packages/Ecotone/src/Messaging/Channel/PollableChannel/Serialization/OutboundMessageConverter.php index 13924da72..2d00572cf 100644 --- a/packages/Ecotone/src/Messaging/Channel/PollableChannel/Serialization/OutboundMessageConverter.php +++ b/packages/Ecotone/src/Messaging/Channel/PollableChannel/Serialization/OutboundMessageConverter.php @@ -13,6 +13,7 @@ use Ecotone\Messaging\Scheduling\DatePoint; use Ecotone\Messaging\Scheduling\Duration; use Ecotone\Messaging\Scheduling\TimeSpan; +use Ecotone\Messaging\Support\MessageBuilder; /** * licence Apache-2.0 @@ -29,6 +30,51 @@ public function __construct( ) { } + /** + * Behaves identically to preparing a full Message built from the entry: string payloads carrying own + * message id and timestamp skip the Message construction, as no payload conversion applies to them. + * + * @param array{payload: mixed, headers: array} $batchEntry + */ + public function prepareFromBatchEntry(array $batchEntry, ConversionService $conversionService): OutboundMessage + { + $payload = $batchEntry['payload']; + $headers = $batchEntry['headers']; + + if (! is_string($payload) || ! isset($headers[MessageHeaders::MESSAGE_ID], $headers[MessageHeaders::TIMESTAMP])) { + return $this->prepare( + MessageBuilder::withPayload($payload)->setMultipleHeaders($headers)->build(), + $conversionService, + ); + } + + $applicationHeaders = MessageHeaders::unsetAggregateKeys($headers); + $applicationHeaders = MessageHeaders::unsetEnqueueMetadata($applicationHeaders); + $applicationHeaders = $this->headerMapper->mapFromMessageHeaders($applicationHeaders, $conversionService); + $applicationHeaders[MessageHeaders::MESSAGE_ID] = $headers[MessageHeaders::MESSAGE_ID]; + $applicationHeaders[MessageHeaders::TIMESTAMP] = $headers[MessageHeaders::TIMESTAMP]; + + if (isset($headers[MessageHeaders::ROUTING_SLIP])) { + $applicationHeaders[MessageHeaders::ROUTING_SLIP] = $headers[MessageHeaders::ROUTING_SLIP]; + } + $contentType = isset($headers[MessageHeaders::CONTENT_TYPE]) ? MediaType::parseMediaType($headers[MessageHeaders::CONTENT_TYPE])->toString() : null; + $applicationHeaders[MessageHeaders::CONTENT_TYPE] = $contentType; + + $deliveryDelay = $this->normalizeDeliveryDelay( + array_key_exists(MessageHeaders::DELIVERY_DELAY, $headers) ? $headers[MessageHeaders::DELIVERY_DELAY] : $this->defaultDeliveryDelay, + $headers[MessageHeaders::TIMESTAMP], + ); + + return new OutboundMessage( + $payload, + array_merge($applicationHeaders, $this->staticHeadersToAdd), + $contentType, + $deliveryDelay, + array_key_exists(MessageHeaders::TIME_TO_LIVE, $headers) ? $headers[MessageHeaders::TIME_TO_LIVE] : $this->defaultTimeToLive, + array_key_exists(MessageHeaders::PRIORITY, $headers) ? $headers[MessageHeaders::PRIORITY] : $this->defaultPriority, + ); + } + public function prepare(Message $messageToConvert, ConversionService $conversionService): OutboundMessage { $messagePayload = $messageToConvert->getPayload(); @@ -97,10 +143,25 @@ public function prepare(Message $messageToConvert, ConversionService $conversion } $applicationHeaders[MessageHeaders::CONTENT_TYPE] = $sourceMediaType?->toString(); - $deliveryDelay = $messageToConvert->getHeaders()->containsKey(MessageHeaders::DELIVERY_DELAY) ? $messageToConvert->getHeaders()->get(MessageHeaders::DELIVERY_DELAY) : $this->defaultDeliveryDelay; + $deliveryDelay = $this->normalizeDeliveryDelay( + $messageToConvert->getHeaders()->containsKey(MessageHeaders::DELIVERY_DELAY) ? $messageToConvert->getHeaders()->get(MessageHeaders::DELIVERY_DELAY) : $this->defaultDeliveryDelay, + $messageToConvert->getHeaders()->getTimestamp(), + ); + return new OutboundMessage( + $messagePayload, + array_merge($applicationHeaders, $this->staticHeadersToAdd), + $applicationHeaders[MessageHeaders::CONTENT_TYPE], + $deliveryDelay, + $messageToConvert->getHeaders()->containsKey(MessageHeaders::TIME_TO_LIVE) ? $messageToConvert->getHeaders()->get(MessageHeaders::TIME_TO_LIVE) : $this->defaultTimeToLive, + $messageToConvert->getHeaders()->containsKey(MessageHeaders::PRIORITY) ? $messageToConvert->getHeaders()->get(MessageHeaders::PRIORITY) : $this->defaultPriority, + ); + } + + private function normalizeDeliveryDelay(mixed $deliveryDelay, int $messageTimestamp): ?int + { if ($deliveryDelay instanceof DateTimeInterface) { - $deliveryDelay = DatePoint::createFromInterface($deliveryDelay)->durationSince(DatePoint::createFromTimestamp($messageToConvert->getHeaders()->getTimestamp())); + $deliveryDelay = DatePoint::createFromInterface($deliveryDelay)->durationSince(DatePoint::createFromTimestamp($messageTimestamp)); } if ($deliveryDelay instanceof Duration) { @@ -115,14 +176,7 @@ public function prepare(Message $messageToConvert, ConversionService $conversion $deliveryDelay = null; } - return new OutboundMessage( - $messagePayload, - array_merge($applicationHeaders, $this->staticHeadersToAdd), - $applicationHeaders[MessageHeaders::CONTENT_TYPE], - $deliveryDelay, - $messageToConvert->getHeaders()->containsKey(MessageHeaders::TIME_TO_LIVE) ? $messageToConvert->getHeaders()->get(MessageHeaders::TIME_TO_LIVE) : $this->defaultTimeToLive, - $messageToConvert->getHeaders()->containsKey(MessageHeaders::PRIORITY) ? $messageToConvert->getHeaders()->get(MessageHeaders::PRIORITY) : $this->defaultPriority, - ); + return $deliveryDelay; } private function doesRequireConversion( diff --git a/packages/Ecotone/src/Messaging/MessageConverter/DefaultHeaderMapper.php b/packages/Ecotone/src/Messaging/MessageConverter/DefaultHeaderMapper.php index 89bc167ec..546208737 100644 --- a/packages/Ecotone/src/Messaging/MessageConverter/DefaultHeaderMapper.php +++ b/packages/Ecotone/src/Messaging/MessageConverter/DefaultHeaderMapper.php @@ -120,6 +120,14 @@ private function mapHeaders(array $mappingHeaders, array $sourceHeaders, Convers continue; } + if ($mappedHeader === '.*') { + foreach ($convertedSourceHeaders as $sourceHeaderName => $value) { + $targetHeaders = $this->convertToStoreableFormat($sourceHeaderName, $value, $targetHeaders, $conversionService); + } + + continue; + } + foreach ($convertedSourceHeaders as $sourceHeaderName => $value) { if (preg_match("#{$mappedHeader}#", $sourceHeaderName)) { $targetHeaders = $this->convertToStoreableFormat($sourceHeaderName, $value, $targetHeaders, $conversionService); @@ -138,7 +146,7 @@ private function mapHeaders(array $mappingHeaders, array $sourceHeaders, Convers */ private function isScalarType($headerValue): bool { - return (Type::createFromVariable($headerValue))->isScalar(); + return is_scalar($headerValue) || (Type::createFromVariable($headerValue))->isScalar(); } /** diff --git a/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php b/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php index 289eae2d2..addbe7121 100644 --- a/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php +++ b/packages/Enqueue/src/EnqueueOutboundChannelAdapter.php @@ -101,6 +101,14 @@ protected function prepareOutboundMessage(Message $message): OutboundMessage return $this->outboundMessageConverter->prepare($message, $this->conversionService); } + /** + * @param array{payload: mixed, headers: array} $entry + */ + protected function prepareOutboundMessageFromBatchEntry(array $entry): OutboundMessage + { + return $this->outboundMessageConverter->prepareFromBatchEntry($entry, $this->conversionService); + } + protected function sendSingleMessage(Message $message, Context $context): void { $outboundMessage = $this->prepareOutboundMessage($message); From c87bf8f4bb7b7b1bc9277a03342296e053b57528 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Wed, 5 Aug 2026 08:00:00 +0200 Subject: [PATCH 20/25] fix: bump stale pdo-event-sourcing dev pin in JmsConverter to current release line Release 1.322.2 bumped the path repository branch alias, making the ~1.320.0 pin unresolvable against the canonical path repo and failing Split Testing for every branch created since. All sibling packages already pin ~1.322.2. --- packages/JmsConverter/composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/JmsConverter/composer.json b/packages/JmsConverter/composer.json index 004ad531f..794f65f32 100644 --- a/packages/JmsConverter/composer.json +++ b/packages/JmsConverter/composer.json @@ -54,7 +54,7 @@ "symfony/cache": "^6.4|^7.0|^8.0" }, "require-dev": { - "ecotone/pdo-event-sourcing": "~1.320.0", + "ecotone/pdo-event-sourcing": "~1.322.2", "phpunit/phpunit": "^10.5|^11.0", "phpstan/phpstan": "^1.8" }, From 019851a44157f21b5a6c284b4a3b6b992d61f804 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Wed, 5 Aug 2026 08:00:00 +0200 Subject: [PATCH 21/25] test: consumer execution limit maps one handled execution to one forwarding batch --- .../CombinedChannelBatchForwardingTest.php | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php index a5d2db035..e6c09222d 100644 --- a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -166,6 +166,41 @@ public function register(string $order): void $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); } + public function test_execution_limit_of_two_moves_exactly_two_batches(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + }; + + $messaging = EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox') + ->withMaxForwardingBatchSize(100), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + for ($messageNumber = 0; $messageNumber < 300; $messageNumber++) { + $messaging->sendCommandWithRoutingKey('order.register', 'order-' . $messageNumber); + } + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 2, maxExecutionTimeInMilliseconds: 30000)); + + $this->assertCount(200, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); + $this->assertCount(100, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); + } + public function test_batch_forwarding_configuration_requires_enterprise_licence(): void { $orderService = new class () { From b24b3271cc3e1f88c11e90e05d70b6aeb99c4c4b Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Wed, 5 Aug 2026 08:00:00 +0200 Subject: [PATCH 22/25] feat: configurable final failure strategy on batch forwarding with per-run delivery assertions Failure strategy is configured on BatchForwardingConfiguration itself and inherits from the outbox channel when not set. Relay tests assert exact payload routing per run and inspect the backing store for release flags, claim restoration and remaining rows instead of counting received messages. --- .../DbalBatchForwardingModule.php | 5 +- .../CombinedChannelBatchForwardingTest.php | 85 ++++++++++++++----- .../Channel/BatchForwardingConfiguration.php | 14 +++ 3 files changed, 80 insertions(+), 24 deletions(-) diff --git a/packages/Dbal/src/BatchForwarding/DbalBatchForwardingModule.php b/packages/Dbal/src/BatchForwarding/DbalBatchForwardingModule.php index aa8ed8a4b..4b0c3281b 100644 --- a/packages/Dbal/src/BatchForwarding/DbalBatchForwardingModule.php +++ b/packages/Dbal/src/BatchForwarding/DbalBatchForwardingModule.php @@ -9,6 +9,7 @@ use Ecotone\Dbal\DbalReconnectableConnectionFactory; use Ecotone\Enqueue\CachedConnectionFactory; use Ecotone\Messaging\Attribute\ModuleAnnotation; +use Ecotone\Messaging\Attribute\WithoutDatabaseTransaction; use Ecotone\Messaging\Attribute\WithoutMessageCollector; use Ecotone\Messaging\Channel\BatchForwardingConfiguration; use Ecotone\Messaging\Config\Annotation\AnnotationModule; @@ -71,7 +72,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO ) ->withEndpointId((string) $endpointId) ->withContinuousPolling() - ->withEndpointAnnotations([new AttributeDefinition(WithoutMessageCollector::class)]), + ->withEndpointAnnotations([new AttributeDefinition(WithoutMessageCollector::class), new AttributeDefinition(WithoutDatabaseTransaction::class)]), ); } } @@ -102,7 +103,7 @@ private function createOutboxPublisherDefinition(DbalBackedMessageChannelBuilder new Reference(LoggingGateway::class), new Reference(EcotoneClockInterface::class), $batchForwardingConfiguration->getMaxForwardingBatchSize(), - $inboundChannelAdapter->getFinalFailureStrategy(), + $batchForwardingConfiguration->getFinalFailureStrategy() ?? $inboundChannelAdapter->getFinalFailureStrategy(), ]); } } diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php index e6c09222d..1e20a0f42 100644 --- a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -80,8 +80,8 @@ public function getRegistered(): array $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); - $this->assertCount(3, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); - $this->assertNull($messaging->getMessageChannel('outbox')->receive()); + $this->assertSame(['espresso', 'latte', 'cappuccino'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('orderProcessing')))); + $this->assertSame(0, $this->amountOfRowsOn('outbox')); } public function test_forwarding_as_single_batch_to_target_with_high_throughput_publishing_delivers_all_messages(): void @@ -162,8 +162,8 @@ public function register(string $order): void $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); - $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); - $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); + $this->assertSame(['espresso', 'latte'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('orderProcessing')))); + $this->assertSame(['cappuccino'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('outbox')))); } public function test_execution_limit_of_two_moves_exactly_two_batches(): void @@ -197,8 +197,10 @@ public function register(string $order): void $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 2, maxExecutionTimeInMilliseconds: 30000)); - $this->assertCount(200, $this->receiveAllFrom($messaging->getMessageChannel('orderProcessing'))); - $this->assertCount(100, $this->receiveAllFrom($messaging->getMessageChannel('outbox'))); + $expectedMoved = array_map(fn (int $messageNumber) => 'order-' . $messageNumber, range(0, 199)); + $expectedRemaining = array_map(fn (int $messageNumber) => 'order-' . $messageNumber, range(200, 299)); + $this->assertSame($expectedMoved, $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('orderProcessing')))); + $this->assertSame($expectedRemaining, $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('outbox')))); } public function test_batch_forwarding_configuration_requires_enterprise_licence(): void @@ -283,7 +285,8 @@ public function registerPriority(string $order): void ->withExtensionObjects([ CombinedMessageChannel::create('standardOrders', ['outbox', 'standardProcessing']), CombinedMessageChannel::create('priorityOrders', ['outbox', 'priorityProcessing']), - BatchForwardingConfiguration::create('outbox'), + BatchForwardingConfiguration::create('outbox') + ->withMaxForwardingBatchSize(2), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('standardProcessing'), DbalBackedMessageChannelBuilder::create('priorityProcessing'), @@ -292,15 +295,21 @@ public function registerPriority(string $order): void ); $messaging->sendCommandWithRoutingKey('order.registerStandard', 'espresso'); - $messaging->sendCommandWithRoutingKey('order.registerPriority', 'flat white'); $messaging->sendCommandWithRoutingKey('order.registerStandard', 'latte'); + $messaging->sendCommandWithRoutingKey('order.registerPriority', 'flat white'); $messaging->sendCommandWithRoutingKey('order.registerPriority', 'cortado'); - $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 1, maxExecutionTimeInMilliseconds: 5000)); - $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('standardProcessing'))); - $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('priorityProcessing'))); - $this->assertNull($messaging->getMessageChannel('outbox')->receive()); + $this->assertSame(['espresso', 'latte'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('standardProcessing')))); + $this->assertSame(0, $this->amountOfRowsOn('priorityProcessing')); + $this->assertSame(2, $this->amountOfRowsOn('outbox')); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 1, maxExecutionTimeInMilliseconds: 5000)); + + $this->assertSame(['flat white', 'cortado'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('priorityProcessing')))); + $this->assertSame(0, $this->amountOfPendingRowsOn('standardProcessing')); + $this->assertSame(0, $this->amountOfRowsOn('outbox')); } public function test_plain_asynchronous_dbal_channel_keeps_one_message_per_handled_message(): void @@ -384,10 +393,10 @@ public function registerPriority(string $order): void $messaging->run('sharedOutboxPublisher', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); - $this->assertCount(2, $this->receiveAllFrom($messaging->getMessageChannel('standardProcessing'))); - $this->assertCount(1, $this->receiveAllFrom($messaging->getMessageChannel('priorityProcessing'))); - $this->assertNull($messaging->getMessageChannel('standardOutbox')->receive()); - $this->assertNull($messaging->getMessageChannel('priorityOutbox')->receive()); + $this->assertSame(['espresso', 'latte'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('standardProcessing')))); + $this->assertSame(['flat white'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('priorityProcessing')))); + $this->assertSame(0, $this->amountOfRowsOn('standardOutbox')); + $this->assertSame(0, $this->amountOfRowsOn('priorityOutbox')); } public function test_failed_send_of_single_message_on_target_channel_releases_only_that_message(): void @@ -422,11 +431,13 @@ public function register(string $order): void $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); $this->assertSame(['espresso', 'latte'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('failingProcessing')))); + $this->assertSame(1, $this->amountOfRowsOn('outbox')); + $this->assertSame([true], $this->redeliveredFlagsOn('outbox')); $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); $this->assertSame(['cappuccino'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('failingProcessing')))); - $this->assertNull($messaging->getMessageChannel('outbox')->receive()); + $this->assertSame(0, $this->amountOfRowsOn('outbox')); } public function test_single_message_is_forwarded_without_waiting_for_source_receive_timeout(): void @@ -485,9 +496,9 @@ public function register(string $order): void ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), - DbalBackedMessageChannelBuilder::create('outbox') + BatchForwardingConfiguration::create('outbox') ->withFinalFailureStrategy(FinalFailureStrategy::IGNORE), + DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), SimpleChannelInterceptorBuilder::create('orderProcessing', 'alwaysFailingDelivery'), ]), @@ -525,9 +536,9 @@ public function register(string $order): void ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), - DbalBackedMessageChannelBuilder::create('outbox') + BatchForwardingConfiguration::create('outbox') ->withFinalFailureStrategy(FinalFailureStrategy::STOP), + DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), SimpleChannelInterceptorBuilder::create('orderProcessing', 'alwaysFailingDelivery'), ]), @@ -978,10 +989,14 @@ public function test_connection_failure_during_delivery_is_recovered_without_dup $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); } catch (ConnectionException) { } + + $this->assertSame(3, $this->amountOfRowsOn('outbox')); + $this->assertSame(0, $this->amountOfClaimedRowsOn('outbox')); + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(maxExecutionTimeInMilliseconds: 5000)); $this->assertSame(['espresso', 'latte', 'cappuccino'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('orderProcessing')))); - $this->assertNull($messaging->getMessageChannel('outbox')->receive()); + $this->assertSame(0, $this->amountOfRowsOn('outbox')); } private function bootstrapWithFailingTargetInterceptor(string $exceptionClass): FlowTestSupport @@ -1014,6 +1029,32 @@ public function register(string $order): void ); } + private function amountOfRowsOn(string $channelName): int + { + return (int) $this->getConnection()->fetchOne('SELECT COUNT(*) FROM enqueue WHERE queue = ?', [$channelName]); + } + + private function amountOfClaimedRowsOn(string $channelName): int + { + return (int) $this->getConnection()->fetchOne('SELECT COUNT(*) FROM enqueue WHERE queue = ? AND delivery_id IS NOT NULL', [$channelName]); + } + + private function amountOfPendingRowsOn(string $channelName): int + { + return (int) $this->getConnection()->fetchOne('SELECT COUNT(*) FROM enqueue WHERE queue = ? AND delivery_id IS NULL', [$channelName]); + } + + /** + * @return bool[] + */ + private function redeliveredFlagsOn(string $channelName): array + { + return array_map( + fn ($redelivered) => (bool) $redelivered, + $this->getConnection()->fetchFirstColumn('SELECT redelivered FROM enqueue WHERE queue = ? ORDER BY published_at ASC', [$channelName]), + ); + } + /** * @param \Ecotone\Messaging\Message[] $messages * @return string[] diff --git a/packages/Ecotone/src/Messaging/Channel/BatchForwardingConfiguration.php b/packages/Ecotone/src/Messaging/Channel/BatchForwardingConfiguration.php index 6afb61cde..8a3d15180 100644 --- a/packages/Ecotone/src/Messaging/Channel/BatchForwardingConfiguration.php +++ b/packages/Ecotone/src/Messaging/Channel/BatchForwardingConfiguration.php @@ -4,6 +4,7 @@ namespace Ecotone\Messaging\Channel; +use Ecotone\Messaging\Endpoint\FinalFailureStrategy; use Ecotone\Messaging\Support\Assert; /** @@ -19,6 +20,7 @@ final class BatchForwardingConfiguration private int $maxForwardingBatchSize = self::DEFAULT_MAX_FORWARDING_BATCH_SIZE; private bool $enabled = true; private ?string $endpointId = null; + private ?FinalFailureStrategy $finalFailureStrategy = null; private function __construct(private string $channelName) { @@ -50,6 +52,18 @@ public function getEndpointId(): string return $this->endpointId ?? $this->channelName; } + public function withFinalFailureStrategy(FinalFailureStrategy $finalFailureStrategy): self + { + $this->finalFailureStrategy = $finalFailureStrategy; + + return $this; + } + + public function getFinalFailureStrategy(): ?FinalFailureStrategy + { + return $this->finalFailureStrategy; + } + public function isEnabled(): bool { return $this->enabled; From 0da4f2be5ec9b597531b9f44062c2ffe186e2723 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Wed, 5 Aug 2026 08:00:00 +0200 Subject: [PATCH 23/25] feat: round-robin multi-tenant outbox publishing Each publisher run drains the next tenant outbox in round robin, reusing the polling consumer tenant propagation, so target sends route to the drained tenant. The batch publishing endpoint carries WithoutDatabaseTransaction, now honoured consistently by the object manager and deduplication interceptors alongside the transaction interceptor, which also removes the per tick deduplication insert every relay was paying. --- .../DeduplicationInterceptor.php | 7 +- .../ObjectManagerInterceptor.php | 7 +- .../BatchForwardingMultiTenantTest.php | 107 ++++++++++++++++++ 3 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 packages/Dbal/tests/Integration/MultiTenant/BatchForwardingMultiTenantTest.php diff --git a/packages/Dbal/src/Deduplication/DeduplicationInterceptor.php b/packages/Dbal/src/Deduplication/DeduplicationInterceptor.php index a12dbfd71..8d6a7108e 100644 --- a/packages/Dbal/src/Deduplication/DeduplicationInterceptor.php +++ b/packages/Dbal/src/Deduplication/DeduplicationInterceptor.php @@ -11,6 +11,7 @@ use Ecotone\Messaging\Attribute\AsynchronousRunningEndpoint; use Ecotone\Messaging\Attribute\Deduplicated; use Ecotone\Messaging\Attribute\IdentifiedAnnotation; +use Ecotone\Messaging\Attribute\WithoutDatabaseTransaction; use Ecotone\Messaging\Handler\ClosureExpression\AttributeExpressionExecutor; use Ecotone\Messaging\Handler\ClosureExpression\ExecutorFor; use Ecotone\Messaging\Handler\Logger\LoggingGateway; @@ -50,8 +51,12 @@ public function __construct( } } - public function deduplicate(MethodInvocation $methodInvocation, Message $message, #[ExecutorFor(Deduplicated::class)] ?AttributeExpressionExecutor $deduplicated, ?IdentifiedAnnotation $identifiedAnnotation, ?AsynchronousRunningEndpoint $asynchronousRunningEndpoint): mixed + public function deduplicate(MethodInvocation $methodInvocation, Message $message, #[ExecutorFor(Deduplicated::class)] ?AttributeExpressionExecutor $deduplicated, ?IdentifiedAnnotation $identifiedAnnotation, ?AsynchronousRunningEndpoint $asynchronousRunningEndpoint, ?WithoutDatabaseTransaction $withoutDatabaseTransaction = null): mixed { + if ($withoutDatabaseTransaction !== null) { + return $methodInvocation->proceed(); + } + $connectionFactory = CachedConnectionFactory::createFor(new DbalReconnectableConnectionFactory($this->connection)); $contextId = spl_object_id($connectionFactory->createContext()); diff --git a/packages/Dbal/src/ObjectManager/ObjectManagerInterceptor.php b/packages/Dbal/src/ObjectManager/ObjectManagerInterceptor.php index 4d991f6e6..68b4191d9 100644 --- a/packages/Dbal/src/ObjectManager/ObjectManagerInterceptor.php +++ b/packages/Dbal/src/ObjectManager/ObjectManagerInterceptor.php @@ -7,6 +7,7 @@ use Ecotone\Dbal\EcotoneManagerRegistryConnectionFactory; use Ecotone\Dbal\MultiTenant\MultiTenantConnectionFactory; use Ecotone\Messaging\Attribute\Parameter\Reference; +use Ecotone\Messaging\Attribute\WithoutDatabaseTransaction; use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Ecotone\Messaging\Handler\Processor\MethodInvoker\MethodInvocation; use Ecotone\Messaging\Message; @@ -27,8 +28,12 @@ public function __construct(private array $managerRegistryConnectionFactories) { } - public function transactional(MethodInvocation $methodInvocation, Message $message, #[Reference] LoggingGateway $logger) + public function transactional(MethodInvocation $methodInvocation, Message $message, #[Reference] LoggingGateway $logger, ?WithoutDatabaseTransaction $withoutDatabaseTransaction = null) { + if ($withoutDatabaseTransaction !== null) { + return $methodInvocation->proceed(); + } + /** @var ManagerRegistry[] $managerRegistries */ $managerRegistries = []; diff --git a/packages/Dbal/tests/Integration/MultiTenant/BatchForwardingMultiTenantTest.php b/packages/Dbal/tests/Integration/MultiTenant/BatchForwardingMultiTenantTest.php new file mode 100644 index 000000000..6b2bbda71 --- /dev/null +++ b/packages/Dbal/tests/Integration/MultiTenant/BatchForwardingMultiTenantTest.php @@ -0,0 +1,107 @@ + $this->connectionForTenantA(), + 'tenant_b_connection' => $this->connectionForTenantB(), + ], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + MultiTenantConfiguration::create( + 'tenant', + ['tenant_a' => 'tenant_a_connection', 'tenant_b' => 'tenant_b_connection'], + ), + CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), + BatchForwardingConfiguration::create('outbox'), + DbalBackedMessageChannelBuilder::create('outbox'), + SimpleMessageChannelBuilder::createQueueChannel('orderProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + + $messaging->sendCommandWithRoutingKey('order.register', 'espresso', metadata: ['tenant' => 'tenant_a']); + $messaging->sendCommandWithRoutingKey('order.register', 'latte', metadata: ['tenant' => 'tenant_a']); + $messaging->sendCommandWithRoutingKey('order.register', 'flat white', metadata: ['tenant' => 'tenant_b']); + + $this->assertSame(2, $this->amountOfOutboxRowsFor($this->connectionForTenantA())); + $this->assertSame(1, $this->amountOfOutboxRowsFor($this->connectionForTenantB())); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 1, maxExecutionTimeInMilliseconds: 5000)); + + $this->assertSame(['espresso', 'latte'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('orderProcessing')))); + $this->assertSame(0, $this->amountOfOutboxRowsFor($this->connectionForTenantA())); + $this->assertSame(1, $this->amountOfOutboxRowsFor($this->connectionForTenantB())); + + $messaging->run('outbox', ExecutionPollingMetadata::createWithTestingSetup(amountOfMessagesToHandle: 1, maxExecutionTimeInMilliseconds: 5000)); + + $this->assertSame(['flat white'], $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('orderProcessing')))); + $this->assertSame(0, $this->amountOfOutboxRowsFor($this->connectionForTenantB())); + } + + private function amountOfOutboxRowsFor(ConnectionFactory $connectionFactory): int + { + $connection = $connectionFactory->createContext()->getDbalConnection(); + if (! self::checkIfTableExists($connection, 'enqueue')) { + return 0; + } + + return (int) $connection->fetchOne('SELECT COUNT(*) FROM enqueue WHERE queue = ?', ['outbox']); + } + + /** + * @param \Ecotone\Messaging\Message[] $messages + * @return string[] + */ + private function payloadsOf(array $messages): array + { + return array_map(fn ($message) => $message->getPayload(), $messages); + } + + private function receiveAllFrom(\Ecotone\Messaging\PollableChannel $channel): array + { + $messages = []; + while ($message = $channel->receive()) { + $messages[] = $message; + } + + return $messages; + } +} From a35ab1f775d599d4811f07e394f78d09723b3118 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Thu, 6 Aug 2026 08:00:00 +0200 Subject: [PATCH 24/25] bench: outbox relay draining into RabbitMQ, Kafka, Redis and SQS targets Provider subjects reuse the end-to-end warm up, so queue and topic creation stays outside the measured drain. Broker targets receive whole batches via high throughput publishing. --- Monorepo/Benchmark/OutboxRelayBenchmark.php | 131 ++++++++++++++++---- 1 file changed, 110 insertions(+), 21 deletions(-) diff --git a/Monorepo/Benchmark/OutboxRelayBenchmark.php b/Monorepo/Benchmark/OutboxRelayBenchmark.php index 2a63392fc..39e842475 100644 --- a/Monorepo/Benchmark/OutboxRelayBenchmark.php +++ b/Monorepo/Benchmark/OutboxRelayBenchmark.php @@ -4,7 +4,10 @@ namespace Monorepo\Benchmark; +use Ecotone\Amqp\AmqpBackedMessageChannelBuilder; use Ecotone\Dbal\DbalBackedMessageChannelBuilder; +use Ecotone\Kafka\Channel\KafkaMessageChannelBuilder; +use Ecotone\Kafka\Configuration\KafkaBrokerConfiguration; use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; @@ -15,8 +18,13 @@ use Ecotone\Messaging\Config\ServiceConfiguration; use Ecotone\Messaging\Endpoint\ExecutionPollingMetadata; use Ecotone\Modelling\Attribute\CommandHandler; +use Ecotone\Redis\RedisBackedMessageChannelBuilder; +use Ecotone\Sqs\SqsBackedMessageChannelBuilder; use Ecotone\Test\LicenceTesting; +use Enqueue\AmqpExt\AmqpConnectionFactory; use Enqueue\Dbal\DbalConnectionFactory; +use Enqueue\Redis\RedisConnectionFactory; +use Enqueue\Sqs\SqsConnectionFactory; use PhpBench\Attributes\BeforeMethods; use PhpBench\Attributes\Iterations; use PhpBench\Attributes\Revs; @@ -25,9 +33,9 @@ /** * Measures how fast the whole DBAL outbox is drained and handed over to the next channel of a combined channel: * message-by-message forwarding (no enterprise licence) against batched SQL drain-and-forward (enterprise). - * The consumer is warmed up on an empty outbox before messages are published, so only steady-state relay work is measured. - * The in-memory target subjects isolate the producing side of the relay; the high throughput target subject shows - * the full path into a Dbal backed channel receiving whole batches at once. + * The consumer is warmed up before messages are published, so only steady-state relay work is measured. + * The in-memory target subjects isolate the producing side of the relay; the provider subjects show the full + * path into real brokers receiving whole batches at once. */ #[Warmup(0), Revs(1), Iterations(5)] class OutboxRelayBenchmark @@ -41,28 +49,56 @@ class OutboxRelayBenchmark public function setUpRelayMessageByMessage(): void { $this->messaging = $this->bootstrapOutbox(licenceKey: null); - $this->warmUpConsumerOnEmptyOutbox(); + $this->warmUpConsumer(); $this->fillOutbox(); } public function setUpRelayBatched(): void { $this->messaging = $this->bootstrapOutbox(licenceKey: LicenceTesting::VALID_LICENCE); - $this->warmUpConsumerOnEmptyOutbox(); + $this->warmUpConsumer(); $this->fillOutbox(); } public function setUpRelaySingleBatch(): void { $this->messaging = $this->bootstrapOutbox(licenceKey: LicenceTesting::VALID_LICENCE, maxForwardingBatchSize: self::AMOUNT_OF_RELAYED_MESSAGES); - $this->warmUpConsumerOnEmptyOutbox(); + $this->warmUpConsumer(); $this->fillOutbox(); } - public function setUpRelayBatchedIntoHighThroughputTarget(): void + public function setUpRelayBatchedIntoDbalTarget(): void { - $this->messaging = $this->bootstrapOutbox(licenceKey: LicenceTesting::VALID_LICENCE, highThroughputTarget: true); - $this->warmUpConsumerOnEmptyOutbox(); + $this->messaging = $this->bootstrapOutbox(licenceKey: LicenceTesting::VALID_LICENCE, targetProvider: 'dbal'); + $this->warmUpConsumer(); + $this->fillOutbox(); + } + + public function setUpRelayBatchedIntoAmqpTarget(): void + { + $this->messaging = $this->bootstrapOutbox(licenceKey: LicenceTesting::VALID_LICENCE, targetProvider: 'amqp'); + $this->warmUpConsumer(); + $this->fillOutbox(); + } + + public function setUpRelayBatchedIntoKafkaTarget(): void + { + $this->messaging = $this->bootstrapOutbox(licenceKey: LicenceTesting::VALID_LICENCE, targetProvider: 'kafka'); + $this->warmUpConsumer(); + $this->fillOutbox(); + } + + public function setUpRelayBatchedIntoRedisTarget(): void + { + $this->messaging = $this->bootstrapOutbox(licenceKey: LicenceTesting::VALID_LICENCE, targetProvider: 'redis'); + $this->warmUpConsumer(); + $this->fillOutbox(); + } + + public function setUpRelayBatchedIntoSqsTarget(): void + { + $this->messaging = $this->bootstrapOutbox(licenceKey: LicenceTesting::VALID_LICENCE, targetProvider: 'sqs'); + $this->warmUpConsumer(); $this->fillOutbox(); } @@ -84,19 +120,44 @@ public function bench_dbal_outbox_drain_as_single_batch(): void $this->drainWholeOutbox(); } - #[BeforeMethods('setUpRelayBatchedIntoHighThroughputTarget')] + #[BeforeMethods('setUpRelayBatchedIntoDbalTarget')] public function bench_dbal_outbox_drain_batched_into_high_throughput_dbal_target(): void { $this->drainWholeOutbox(); } - private function warmUpConsumerOnEmptyOutbox(): void + #[BeforeMethods('setUpRelayBatchedIntoAmqpTarget'), Iterations(3)] + public function bench_dbal_outbox_drain_batched_into_rabbitmq_target(): void + { + $this->drainWholeOutbox(); + } + + #[BeforeMethods('setUpRelayBatchedIntoKafkaTarget'), Iterations(3)] + public function bench_dbal_outbox_drain_batched_into_kafka_target(): void + { + $this->drainWholeOutbox(); + } + + #[BeforeMethods('setUpRelayBatchedIntoRedisTarget'), Iterations(3)] + public function bench_dbal_outbox_drain_batched_into_redis_target(): void + { + $this->drainWholeOutbox(); + } + + #[BeforeMethods('setUpRelayBatchedIntoSqsTarget'), Iterations(3)] + public function bench_dbal_outbox_drain_batched_into_sqs_target(): void + { + $this->drainWholeOutbox(); + } + + private function warmUpConsumer(): void { $context = (new DbalConnectionFactory(self::databaseDsn()))->createContext(); $context->createDataBaseTable(); $context->purgeQueue($context->createQueue('benchmark_outbox')); $context->purgeQueue($context->createQueue('benchmark_target')); + $this->messaging->sendCommandWithRoutingKey('benchmark.relayOrder', self::MESSAGE_PAYLOAD); $this->messaging->run('benchmark_outbox', ExecutionPollingMetadata::createWithFinishWhenNoMessages()); } @@ -117,7 +178,7 @@ private function fillOutbox(): void } } - private function bootstrapOutbox(?string $licenceKey, ?int $maxForwardingBatchSize = null, bool $highThroughputTarget = false): FlowTestSupport + private function bootstrapOutbox(?string $licenceKey, ?int $maxForwardingBatchSize = null, string $targetProvider = 'in_memory'): FlowTestSupport { $batchForwardingExtensions = []; if ($licenceKey !== null) { @@ -128,9 +189,31 @@ private function bootstrapOutbox(?string $licenceKey, ?int $maxForwardingBatchSi $batchForwardingExtensions[] = $batchForwardingConfiguration; } - $targetChannel = $highThroughputTarget - ? DbalBackedMessageChannelBuilder::create('benchmark_target')->withHighThroughputPublishing() - : SimpleMessageChannelBuilder::createQueueChannel('benchmark_target'); + $targetName = in_array($targetProvider, ['in_memory', 'dbal'], true) ? 'benchmark_target' : uniqid('benchmark_target_'); + [$targetChannel, $targetServices, $targetPackage] = match ($targetProvider) { + 'in_memory' => [SimpleMessageChannelBuilder::createQueueChannel($targetName), [], null], + 'dbal' => [DbalBackedMessageChannelBuilder::create($targetName)->withHighThroughputPublishing(), [], null], + 'amqp' => [ + AmqpBackedMessageChannelBuilder::create($targetName)->withHighThroughputPublishing(), + [AmqpConnectionFactory::class => new AmqpConnectionFactory(['dsn' => getenv('RABBIT_HOST') ?: 'amqp://guest:guest@localhost:5672/%2f'])], + ModulePackageList::AMQP_PACKAGE, + ], + 'kafka' => [ + KafkaMessageChannelBuilder::create($targetName, topicName: $targetName, messageGroupId: $targetName)->withHighThroughputPublishing(), + [KafkaBrokerConfiguration::class => KafkaBrokerConfiguration::createWithDefaults([getenv('KAFKA_DSN') ?: 'localhost:9094'])], + ModulePackageList::KAFKA_PACKAGE, + ], + 'redis' => [ + RedisBackedMessageChannelBuilder::create($targetName)->withHighThroughputPublishing(), + [RedisConnectionFactory::class => new RedisConnectionFactory(getenv('REDIS_DSN') ?: 'redis://localhost:6379')], + ModulePackageList::REDIS_PACKAGE, + ], + 'sqs' => [ + SqsBackedMessageChannelBuilder::create($targetName)->withHighThroughputPublishing(), + [SqsConnectionFactory::class => new SqsConnectionFactory(getenv('SQS_DSN') ?: 'sqs:?key=key&secret=secret®ion=us-east-1&endpoint=http://localhost:4566&version=latest')], + ModulePackageList::SQS_PACKAGE, + ], + }; $orderService = new class () { #[Asynchronous('benchmark_relay_orders')] @@ -142,14 +225,20 @@ public function handle(string $order): void return EcotoneLite::bootstrapFlowTesting( [$orderService::class], - [ - DbalConnectionFactory::class => new DbalConnectionFactory(self::databaseDsn()), - $orderService, - ], + array_merge( + [ + DbalConnectionFactory::class => new DbalConnectionFactory(self::databaseDsn()), + $orderService, + ], + $targetServices, + ), ServiceConfiguration::createWithDefaults() - ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept(array_merge( + [ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE], + $targetPackage !== null ? [$targetPackage] : [], + ))) ->withExtensionObjects(array_merge([ - CombinedMessageChannel::create('benchmark_relay_orders', ['benchmark_outbox', 'benchmark_target']), + CombinedMessageChannel::create('benchmark_relay_orders', ['benchmark_outbox', $targetName]), DbalBackedMessageChannelBuilder::create('benchmark_outbox') ->withReceiveTimeout(20), $targetChannel, From c67415463fd0a5845ff1cd33dd48714c2ebd96c6 Mon Sep 17 00:00:00 2001 From: Dariusz Gafka Date: Thu, 6 Aug 2026 08:00:00 +0200 Subject: [PATCH 25/25] feat: OutboxForwardingMessageChannel as the single definition of an outbox relay The Dbal owned channel type replaces BatchForwardingConfiguration: it extends CombinedMessageChannel with exactly one Dbal backed source and one target, carries batch size, endpoint id and failure strategy, and may embed the source channel builder itself so the outbox cannot be misconfigured by name. Messaging core keeps only the OutboxForwardingChannel contract, unwraps embedded source builders into extension objects and guards at compile time that the source is claimed by a forwarding module and not reused inside plain Combined Message Channels. Forwarding channels sharing one outbox must agree on their settings. --- Monorepo/Benchmark/OutboxRelayBenchmark.php | 19 ++- .../DbalBatchForwardingModule.php | 40 +++++-- .../src/OutboxForwardingMessageChannel.php | 101 ++++++++++++++++ .../CombinedChannelBatchForwardingTest.php | 109 ++++++++++-------- .../BatchForwardingMultiTenantTest.php | 6 +- .../Channel/BatchForwardingConfiguration.php | 81 ------------- .../Channel/CombinedMessageChannel.php | 4 +- .../Channel/OutboxForwardingChannel.php | 21 ++++ .../Config/MessagingSystemConfiguration.php | 25 +++- .../CombinedChannelForwardingTest.php | 6 +- 10 files changed, 247 insertions(+), 165 deletions(-) create mode 100644 packages/Dbal/src/OutboxForwardingMessageChannel.php delete mode 100644 packages/Ecotone/src/Messaging/Channel/BatchForwardingConfiguration.php create mode 100644 packages/Ecotone/src/Messaging/Channel/OutboxForwardingChannel.php diff --git a/Monorepo/Benchmark/OutboxRelayBenchmark.php b/Monorepo/Benchmark/OutboxRelayBenchmark.php index 39e842475..61b314b89 100644 --- a/Monorepo/Benchmark/OutboxRelayBenchmark.php +++ b/Monorepo/Benchmark/OutboxRelayBenchmark.php @@ -6,12 +6,12 @@ use Ecotone\Amqp\AmqpBackedMessageChannelBuilder; use Ecotone\Dbal\DbalBackedMessageChannelBuilder; +use Ecotone\Dbal\OutboxForwardingMessageChannel; use Ecotone\Kafka\Channel\KafkaMessageChannelBuilder; use Ecotone\Kafka\Configuration\KafkaBrokerConfiguration; use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; -use Ecotone\Messaging\Channel\BatchForwardingConfiguration; use Ecotone\Messaging\Channel\CombinedMessageChannel; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; use Ecotone\Messaging\Config\ModulePackageList; @@ -180,16 +180,15 @@ private function fillOutbox(): void private function bootstrapOutbox(?string $licenceKey, ?int $maxForwardingBatchSize = null, string $targetProvider = 'in_memory'): FlowTestSupport { - $batchForwardingExtensions = []; + $targetName = in_array($targetProvider, ['in_memory', 'dbal'], true) ? 'benchmark_target' : uniqid('benchmark_target_'); if ($licenceKey !== null) { - $batchForwardingConfiguration = BatchForwardingConfiguration::create('benchmark_outbox'); + $relayChannel = OutboxForwardingMessageChannel::create('benchmark_relay_orders', 'benchmark_outbox', $targetName); if ($maxForwardingBatchSize !== null) { - $batchForwardingConfiguration = $batchForwardingConfiguration->withMaxForwardingBatchSize($maxForwardingBatchSize); + $relayChannel = $relayChannel->withMaxForwardingBatchSize($maxForwardingBatchSize); } - $batchForwardingExtensions[] = $batchForwardingConfiguration; + } else { + $relayChannel = CombinedMessageChannel::create('benchmark_relay_orders', ['benchmark_outbox', $targetName]); } - - $targetName = in_array($targetProvider, ['in_memory', 'dbal'], true) ? 'benchmark_target' : uniqid('benchmark_target_'); [$targetChannel, $targetServices, $targetPackage] = match ($targetProvider) { 'in_memory' => [SimpleMessageChannelBuilder::createQueueChannel($targetName), [], null], 'dbal' => [DbalBackedMessageChannelBuilder::create($targetName)->withHighThroughputPublishing(), [], null], @@ -237,12 +236,12 @@ public function handle(string $order): void [ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE], $targetPackage !== null ? [$targetPackage] : [], ))) - ->withExtensionObjects(array_merge([ - CombinedMessageChannel::create('benchmark_relay_orders', ['benchmark_outbox', $targetName]), + ->withExtensionObjects([ + $relayChannel, DbalBackedMessageChannelBuilder::create('benchmark_outbox') ->withReceiveTimeout(20), $targetChannel, - ], $batchForwardingExtensions)), + ]), licenceKey: $licenceKey, ); } diff --git a/packages/Dbal/src/BatchForwarding/DbalBatchForwardingModule.php b/packages/Dbal/src/BatchForwarding/DbalBatchForwardingModule.php index 4b0c3281b..6a6bfdd8c 100644 --- a/packages/Dbal/src/BatchForwarding/DbalBatchForwardingModule.php +++ b/packages/Dbal/src/BatchForwarding/DbalBatchForwardingModule.php @@ -7,11 +7,11 @@ use Ecotone\AnnotationFinder\AnnotationFinder; use Ecotone\Dbal\DbalBackedMessageChannelBuilder; use Ecotone\Dbal\DbalReconnectableConnectionFactory; +use Ecotone\Dbal\OutboxForwardingMessageChannel; use Ecotone\Enqueue\CachedConnectionFactory; use Ecotone\Messaging\Attribute\ModuleAnnotation; use Ecotone\Messaging\Attribute\WithoutDatabaseTransaction; use Ecotone\Messaging\Attribute\WithoutMessageCollector; -use Ecotone\Messaging\Channel\BatchForwardingConfiguration; use Ecotone\Messaging\Config\Annotation\AnnotationModule; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\ExtensionObjectResolver; use Ecotone\Messaging\Config\Annotation\ModuleConfiguration\NoExternalConfigurationModule; @@ -27,6 +27,7 @@ use Ecotone\Messaging\Handler\Logger\LoggingGateway; use Ecotone\Messaging\NullableMessageChannel; use Ecotone\Messaging\Scheduling\EcotoneClockInterface; +use Ecotone\Messaging\Support\Assert; #[ModuleAnnotation] /** @@ -46,16 +47,33 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO $channelBuilders[$channelBuilder->getMessageChannelName()] = $channelBuilder; } + $forwardingChannelsPerSource = []; + /** @var OutboxForwardingMessageChannel $outboxForwardingChannel */ + foreach (ExtensionObjectResolver::resolve(OutboxForwardingMessageChannel::class, $extensionObjects) as $outboxForwardingChannel) { + $sourceChannelName = $outboxForwardingChannel->getSourceChannelName(); + if (isset($forwardingChannelsPerSource[$sourceChannelName])) { + $alreadyRegistered = $forwardingChannelsPerSource[$sourceChannelName]; + Assert::isTrue( + $alreadyRegistered->getMaxForwardingBatchSize() === $outboxForwardingChannel->getMaxForwardingBatchSize() + && $alreadyRegistered->getEndpointId() === $outboxForwardingChannel->getEndpointId() + && $alreadyRegistered->getFinalFailureStrategy() === $outboxForwardingChannel->getFinalFailureStrategy(), + "Outbox forwarding Message Channels sharing source `{$sourceChannelName}` must configure the same batch size, endpoint id and failure strategy.", + ); + + continue; + } + $forwardingChannelsPerSource[$sourceChannelName] = $outboxForwardingChannel; + } + $outboxesPerEndpointId = []; - /** @var BatchForwardingConfiguration $batchForwardingConfiguration */ - foreach (ExtensionObjectResolver::resolve(BatchForwardingConfiguration::class, $extensionObjects) as $batchForwardingConfiguration) { - $channelBuilder = $channelBuilders[$batchForwardingConfiguration->getChannelName()] ?? null; - if (! $batchForwardingConfiguration->isEnabled() || $channelBuilder === null) { + foreach ($forwardingChannelsPerSource as $sourceChannelName => $outboxForwardingChannel) { + $channelBuilder = $channelBuilders[$sourceChannelName] ?? null; + if ($channelBuilder === null) { continue; } - $messagingConfiguration->registerBatchForwardingSourceChannel($channelBuilder->getMessageChannelName()); - $outboxesPerEndpointId[$batchForwardingConfiguration->getEndpointId()][] = $this->createOutboxPublisherDefinition($channelBuilder, $batchForwardingConfiguration); + $messagingConfiguration->registerBatchForwardingSourceChannel($sourceChannelName); + $outboxesPerEndpointId[$outboxForwardingChannel->getEndpointId()][] = $this->createOutboxPublisherDefinition($channelBuilder, $outboxForwardingChannel); } foreach ($outboxesPerEndpointId as $endpointId => $outboxPublishers) { @@ -79,7 +97,7 @@ public function prepare(Configuration $messagingConfiguration, array $extensionO public function canHandle($extensionObject): bool { - return $extensionObject instanceof BatchForwardingConfiguration + return $extensionObject instanceof OutboxForwardingMessageChannel || $extensionObject instanceof DbalBackedMessageChannelBuilder; } @@ -88,7 +106,7 @@ public function getModulePackageName(): string return ModulePackageList::DBAL_PACKAGE; } - private function createOutboxPublisherDefinition(DbalBackedMessageChannelBuilder $channelBuilder, BatchForwardingConfiguration $batchForwardingConfiguration): Definition + private function createOutboxPublisherDefinition(DbalBackedMessageChannelBuilder $channelBuilder, OutboxForwardingMessageChannel $outboxForwardingChannel): Definition { $inboundChannelAdapter = $channelBuilder->getInboundChannelAdapter(); @@ -102,8 +120,8 @@ private function createOutboxPublisherDefinition(DbalBackedMessageChannelBuilder new Reference(ChannelResolver::class), new Reference(LoggingGateway::class), new Reference(EcotoneClockInterface::class), - $batchForwardingConfiguration->getMaxForwardingBatchSize(), - $batchForwardingConfiguration->getFinalFailureStrategy() ?? $inboundChannelAdapter->getFinalFailureStrategy(), + $outboxForwardingChannel->getMaxForwardingBatchSize(), + $outboxForwardingChannel->getFinalFailureStrategy() ?? $inboundChannelAdapter->getFinalFailureStrategy(), ]); } } diff --git a/packages/Dbal/src/OutboxForwardingMessageChannel.php b/packages/Dbal/src/OutboxForwardingMessageChannel.php new file mode 100644 index 000000000..d035f1ba8 --- /dev/null +++ b/packages/Dbal/src/OutboxForwardingMessageChannel.php @@ -0,0 +1,101 @@ +getMessageChannelName(); + } + + $combinedChannels = is_string($sourceChannelName) ? [$sourceChannelName, $targetChannelName] : $sourceChannelName; + Assert::isTrue(count($combinedChannels) === 2, "Outbox forwarding Message Channel `{$referenceName}` requires exactly one source outbox channel and one target channel."); + [$outboxChannelName, $forwardingTargetChannelName] = $combinedChannels; + Assert::isTrue(is_string($outboxChannelName) && is_string($forwardingTargetChannelName), "Outbox forwarding Message Channel `{$referenceName}` requires channel names to be strings."); + Assert::isTrue($outboxChannelName !== $forwardingTargetChannelName, "Outbox forwarding Message Channel `{$referenceName}` requires source and target to be different channels."); + + $outboxForwardingChannel = new static($referenceName, $combinedChannels); + $outboxForwardingChannel->embeddedSourceChannelBuilder = $embeddedSourceChannelBuilder; + + return $outboxForwardingChannel; + } + + public function withMaxForwardingBatchSize(int $maxForwardingBatchSize): self + { + Assert::isTrue($maxForwardingBatchSize > 0, 'Max forwarding batch size must be a positive number.'); + $this->maxForwardingBatchSize = $maxForwardingBatchSize; + + return $this; + } + + public function withEndpointId(string $endpointId): self + { + Assert::notNullAndEmpty($endpointId, 'Endpoint id for outbox forwarding can not be empty.'); + $this->endpointId = $endpointId; + + return $this; + } + + public function withFinalFailureStrategy(FinalFailureStrategy $finalFailureStrategy): self + { + $this->finalFailureStrategy = $finalFailureStrategy; + + return $this; + } + + public function getSourceChannelName(): string + { + return $this->getCombinedChannels()[0]; + } + + public function getTargetChannelName(): string + { + return $this->getCombinedChannels()[1]; + } + + public function getEmbeddedSourceChannelBuilder(): ?MessageChannelBuilder + { + return $this->embeddedSourceChannelBuilder; + } + + public function getEndpointId(): string + { + return $this->endpointId ?? $this->getSourceChannelName(); + } + + public function getMaxForwardingBatchSize(): int + { + return $this->maxForwardingBatchSize; + } + + public function getFinalFailureStrategy(): ?FinalFailureStrategy + { + return $this->finalFailureStrategy; + } +} diff --git a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php index 1e20a0f42..d9a625542 100644 --- a/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php +++ b/packages/Dbal/tests/Integration/CombinedChannelBatchForwardingTest.php @@ -5,10 +5,10 @@ namespace Test\Ecotone\Dbal\Integration; use Ecotone\Dbal\DbalBackedMessageChannelBuilder; +use Ecotone\Dbal\OutboxForwardingMessageChannel; use Ecotone\Lite\EcotoneLite; use Ecotone\Lite\Test\FlowTestSupport; use Ecotone\Messaging\Attribute\Asynchronous; -use Ecotone\Messaging\Channel\BatchForwardingConfiguration; use Ecotone\Messaging\Channel\CombinedMessageChannel; use Ecotone\Messaging\Channel\PollableChannel\GlobalPollableChannelConfiguration; use Ecotone\Messaging\Channel\PollableChannel\PollableChannelConfiguration; @@ -66,9 +66,7 @@ public function getRegistered(): array ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), - DbalBackedMessageChannelBuilder::create('outbox'), + OutboxForwardingMessageChannel::create('orders', DbalBackedMessageChannelBuilder::create('outbox'), 'orderProcessing'), DbalBackedMessageChannelBuilder::create('orderProcessing'), ]), licenceKey: LicenceTesting::VALID_LICENCE, @@ -110,8 +108,7 @@ public function getRegistered(): array ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing'), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing') ->withHighThroughputPublishing(), @@ -147,8 +144,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox') + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing') ->withMaxForwardingBatchSize(2), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), @@ -182,8 +178,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox') + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing') ->withMaxForwardingBatchSize(100), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), @@ -203,6 +198,40 @@ public function register(string $order): void $this->assertSame($expectedRemaining, $this->payloadsOf($this->receiveAllFrom($messaging->getMessageChannel('outbox')))); } + public function test_plain_combined_channel_reusing_outbox_forwarding_source_fails_at_compile_time(): void + { + $orderService = new class () { + #[Asynchronous('orders')] + #[CommandHandler('order.register', endpointId: 'orderRegisterEndpoint')] + public function register(string $order): void + { + } + + #[Asynchronous('otherOrders')] + #[CommandHandler('order.registerOther', endpointId: 'orderRegisterOtherEndpoint')] + public function registerOther(string $order): void + { + } + }; + + $this->expectException(ConfigurationException::class); + + EcotoneLite::bootstrapFlowTesting( + [$orderService::class], + [DbalConnectionFactory::class => $this->getConnectionFactory(), $orderService], + ServiceConfiguration::createWithDefaults() + ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) + ->withExtensionObjects([ + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing'), + CombinedMessageChannel::create('otherOrders', ['outbox', 'otherProcessing']), + DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing'), + DbalBackedMessageChannelBuilder::create('otherProcessing'), + ]), + licenceKey: LicenceTesting::VALID_LICENCE, + ); + } + public function test_batch_forwarding_configuration_requires_enterprise_licence(): void { $orderService = new class () { @@ -221,8 +250,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing'), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), ]), @@ -283,9 +311,9 @@ public function registerPriority(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('standardOrders', ['outbox', 'standardProcessing']), - CombinedMessageChannel::create('priorityOrders', ['outbox', 'priorityProcessing']), - BatchForwardingConfiguration::create('outbox') + OutboxForwardingMessageChannel::create('standardOrders', 'outbox', 'standardProcessing') + ->withMaxForwardingBatchSize(2), + OutboxForwardingMessageChannel::create('priorityOrders', 'outbox', 'priorityProcessing') ->withMaxForwardingBatchSize(2), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('standardProcessing'), @@ -373,11 +401,9 @@ public function registerPriority(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('standardOrders', ['standardOutbox', 'standardProcessing']), - CombinedMessageChannel::create('priorityOrders', ['priorityOutbox', 'priorityProcessing']), - BatchForwardingConfiguration::create('standardOutbox') + OutboxForwardingMessageChannel::create('standardOrders', 'standardOutbox', 'standardProcessing') ->withEndpointId('sharedOutboxPublisher'), - BatchForwardingConfiguration::create('priorityOutbox') + OutboxForwardingMessageChannel::create('priorityOrders', 'priorityOutbox', 'priorityProcessing') ->withEndpointId('sharedOutboxPublisher'), DbalBackedMessageChannelBuilder::create('standardOutbox'), DbalBackedMessageChannelBuilder::create('priorityOutbox'), @@ -415,8 +441,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'failingProcessing']), - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'failingProcessing'), DbalBackedMessageChannelBuilder::create('outbox'), SimpleMessageChannelBuilder::create('failingProcessing', new FailOnceOnPayloadPollableChannel('cappuccino')), PollableChannelConfiguration::create('failingProcessing', RetryTemplateBuilder::fixedBackOff(1)->maxRetryAttempts(1)->build()), @@ -456,8 +481,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing'), DbalBackedMessageChannelBuilder::create('outbox') ->withReceiveTimeout(3000), DbalBackedMessageChannelBuilder::create('orderProcessing'), @@ -495,8 +519,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox') + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing') ->withFinalFailureStrategy(FinalFailureStrategy::IGNORE), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), @@ -535,8 +558,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox') + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing') ->withFinalFailureStrategy(FinalFailureStrategy::STOP), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), @@ -578,8 +600,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['inMemoryOutbox', 'inMemoryProcessing']), - BatchForwardingConfiguration::create('inMemoryOutbox'), + OutboxForwardingMessageChannel::create('orders', 'inMemoryOutbox', 'inMemoryProcessing'), SimpleMessageChannelBuilder::createQueueChannel('inMemoryOutbox'), SimpleMessageChannelBuilder::createQueueChannel('inMemoryProcessing'), ]), @@ -605,8 +626,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('misspelled_outbox'), + OutboxForwardingMessageChannel::create('orders', 'misspelled_outbox', 'orderProcessing'), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), ]), @@ -631,8 +651,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing'), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), GlobalPollableChannelConfiguration::createWithDefaults()->withCollector($collectorEnabled), @@ -677,8 +696,7 @@ public function registerDirectly(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing'), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), ]), @@ -704,8 +722,9 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing'), DbalBackedMessageChannelBuilder::create('outbox'), + DbalBackedMessageChannelBuilder::create('orderProcessing'), ]), licenceKey: LicenceTesting::VALID_LICENCE, ); @@ -735,8 +754,7 @@ public function startFlow(string $order): string ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing'), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), ]), @@ -793,8 +811,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'failingProcessing']), - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'failingProcessing'), DbalBackedMessageChannelBuilder::create('outbox'), SimpleMessageChannelBuilder::create('failingProcessing', new FailingPollableChannel()), ]), @@ -832,8 +849,7 @@ public function register(string $order): void ->withDefaultErrorChannel('customErrorChannel') ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing'), DbalBackedMessageChannelBuilder::create('outbox') ->withFinalFailureStrategy($finalFailureStrategy), DbalBackedMessageChannelBuilder::create('orderProcessing'), @@ -876,8 +892,7 @@ public function register(string $order): void ->withDefaultErrorChannel('customErrorChannel') ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing'), DbalBackedMessageChannelBuilder::create('outbox') ->withFinalFailureStrategy(FinalFailureStrategy::STOP), DbalBackedMessageChannelBuilder::create('orderProcessing'), @@ -918,8 +933,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing'), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), ]), @@ -1019,8 +1033,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::DBAL_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing'), DbalBackedMessageChannelBuilder::create('outbox'), DbalBackedMessageChannelBuilder::create('orderProcessing'), SimpleChannelInterceptorBuilder::create('orderProcessing', 'failingDeliveryInterceptor'), diff --git a/packages/Dbal/tests/Integration/MultiTenant/BatchForwardingMultiTenantTest.php b/packages/Dbal/tests/Integration/MultiTenant/BatchForwardingMultiTenantTest.php index 6b2bbda71..196566dd4 100644 --- a/packages/Dbal/tests/Integration/MultiTenant/BatchForwardingMultiTenantTest.php +++ b/packages/Dbal/tests/Integration/MultiTenant/BatchForwardingMultiTenantTest.php @@ -6,10 +6,9 @@ use Ecotone\Dbal\DbalBackedMessageChannelBuilder; use Ecotone\Dbal\MultiTenant\MultiTenantConfiguration; +use Ecotone\Dbal\OutboxForwardingMessageChannel; use Ecotone\Lite\EcotoneLite; use Ecotone\Messaging\Attribute\Asynchronous; -use Ecotone\Messaging\Channel\BatchForwardingConfiguration; -use Ecotone\Messaging\Channel\CombinedMessageChannel; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; use Ecotone\Messaging\Config\ModulePackageList; use Ecotone\Messaging\Config\ServiceConfiguration; @@ -49,8 +48,7 @@ public function register(string $order): void 'tenant', ['tenant_a' => 'tenant_a_connection', 'tenant_b' => 'tenant_b_connection'], ), - CombinedMessageChannel::create('orders', ['outbox', 'orderProcessing']), - BatchForwardingConfiguration::create('outbox'), + OutboxForwardingMessageChannel::create('orders', 'outbox', 'orderProcessing'), DbalBackedMessageChannelBuilder::create('outbox'), SimpleMessageChannelBuilder::createQueueChannel('orderProcessing'), ]), diff --git a/packages/Ecotone/src/Messaging/Channel/BatchForwardingConfiguration.php b/packages/Ecotone/src/Messaging/Channel/BatchForwardingConfiguration.php deleted file mode 100644 index 8a3d15180..000000000 --- a/packages/Ecotone/src/Messaging/Channel/BatchForwardingConfiguration.php +++ /dev/null @@ -1,81 +0,0 @@ - 0, 'Max forwarding batch size must be a positive number.'); - $this->maxForwardingBatchSize = $maxForwardingBatchSize; - - return $this; - } - - public function withEndpointId(string $endpointId): self - { - Assert::notNullAndEmpty($endpointId, 'Endpoint id for batch forwarding can not be empty.'); - $this->endpointId = $endpointId; - - return $this; - } - - public function getEndpointId(): string - { - return $this->endpointId ?? $this->channelName; - } - - public function withFinalFailureStrategy(FinalFailureStrategy $finalFailureStrategy): self - { - $this->finalFailureStrategy = $finalFailureStrategy; - - return $this; - } - - public function getFinalFailureStrategy(): ?FinalFailureStrategy - { - return $this->finalFailureStrategy; - } - - public function isEnabled(): bool - { - return $this->enabled; - } - - public function getChannelName(): string - { - return $this->channelName; - } - - public function getMaxForwardingBatchSize(): int - { - return $this->maxForwardingBatchSize; - } -} diff --git a/packages/Ecotone/src/Messaging/Channel/CombinedMessageChannel.php b/packages/Ecotone/src/Messaging/Channel/CombinedMessageChannel.php index bdf05f368..73498d957 100644 --- a/packages/Ecotone/src/Messaging/Channel/CombinedMessageChannel.php +++ b/packages/Ecotone/src/Messaging/Channel/CombinedMessageChannel.php @@ -9,9 +9,9 @@ /** * licence Apache-2.0 */ -final class CombinedMessageChannel +class CombinedMessageChannel { - private function __construct(private string $referenceName, private array $combinedChannels) + protected function __construct(private string $referenceName, private array $combinedChannels) { Assert::notNull($referenceName, 'Reference name can not be null'); Assert::notNullAndEmpty($this->combinedChannels, 'Combined channels can not be empty'); diff --git a/packages/Ecotone/src/Messaging/Channel/OutboxForwardingChannel.php b/packages/Ecotone/src/Messaging/Channel/OutboxForwardingChannel.php new file mode 100644 index 000000000..a2d36ff64 --- /dev/null +++ b/packages/Ecotone/src/Messaging/Channel/OutboxForwardingChannel.php @@ -0,0 +1,21 @@ + + */ + private array $plainCombinedChannelUsages = []; private ServiceConfiguration $applicationConfiguration; /** * @var string[] @@ -228,9 +233,14 @@ function ($extensionObject) { $this->isRunningForTest = ExtensionObjectResolver::contains(TestConfiguration::class, $extensionObjects); - foreach (ExtensionObjectResolver::resolve(BatchForwardingConfiguration::class, $extensionObjects) as $batchForwardingConfiguration) { - if ($batchForwardingConfiguration->isEnabled()) { - $this->declaredBatchForwardingChannels[] = $batchForwardingConfiguration->getChannelName(); + foreach (ExtensionObjectResolver::resolve(CombinedMessageChannel::class, $extensionObjects) as $combinedMessageChannel) { + if ($combinedMessageChannel instanceof OutboxForwardingChannel) { + $this->declaredBatchForwardingChannels[] = $combinedMessageChannel->getSourceChannelName(); + if ($combinedMessageChannel->getEmbeddedSourceChannelBuilder() !== null) { + $extensionObjects[] = $combinedMessageChannel->getEmbeddedSourceChannelBuilder(); + } + } else { + $this->plainCombinedChannelUsages[$combinedMessageChannel->getReferenceName()] = $combinedMessageChannel->getCombinedChannels(); } } @@ -429,7 +439,12 @@ private function configureAsynchronousEndpoints(InterfaceToCallRegistry $interfa } foreach ($this->declaredBatchForwardingChannels as $declaredBatchForwardingChannel) { if (! isset($this->batchForwardingSourceChannels[$declaredBatchForwardingChannel])) { - throw ConfigurationException::create("Batch forwarding was configured for Message Channel `{$declaredBatchForwardingChannel}`, yet no module enabled it for that channel. Batch forwarding requires a Dbal backed Message Channel with matching name and the Dbal package enabled."); + throw ConfigurationException::create("Outbox forwarding was configured with source Message Channel `{$declaredBatchForwardingChannel}`, yet no module enabled it for that channel. Outbox forwarding requires a Dbal backed Message Channel as the source and the Dbal package enabled."); + } + foreach ($this->plainCombinedChannelUsages as $combinedChannelReferenceName => $referencedChannels) { + if (in_array($declaredBatchForwardingChannel, $referencedChannels, true)) { + throw ConfigurationException::create("Message Channel `{$declaredBatchForwardingChannel}` is the outbox source of an Outbox forwarding Message Channel and can not be reused inside Combined Message Channel `{$combinedChannelReferenceName}`. Define that flow with OutboxForwardingMessageChannel as well."); + } } } $this->verifyBatchForwardingSourceChannelsAreNotUsedForExecution($this->batchForwardingSourceChannels); diff --git a/packages/Kafka/tests/Integration/CombinedChannelForwardingTest.php b/packages/Kafka/tests/Integration/CombinedChannelForwardingTest.php index 676275a4e..ba0b5dc98 100644 --- a/packages/Kafka/tests/Integration/CombinedChannelForwardingTest.php +++ b/packages/Kafka/tests/Integration/CombinedChannelForwardingTest.php @@ -4,12 +4,11 @@ namespace Test\Ecotone\Kafka\Integration; +use Ecotone\Dbal\OutboxForwardingMessageChannel; use Ecotone\Kafka\Channel\KafkaMessageChannelBuilder; use Ecotone\Kafka\Configuration\KafkaBrokerConfiguration; use Ecotone\Lite\EcotoneLite; use Ecotone\Messaging\Attribute\Asynchronous; -use Ecotone\Messaging\Channel\BatchForwardingConfiguration; -use Ecotone\Messaging\Channel\CombinedMessageChannel; use Ecotone\Messaging\Channel\SimpleMessageChannelBuilder; use Ecotone\Messaging\Config\ConfigurationException; use Ecotone\Messaging\Config\ModulePackageList; @@ -47,8 +46,7 @@ public function register(string $order): void ServiceConfiguration::createWithDefaults() ->withSkippedModulePackageNames(ModulePackageList::allPackagesExcept([ModulePackageList::ASYNCHRONOUS_PACKAGE, ModulePackageList::KAFKA_PACKAGE])) ->withExtensionObjects([ - CombinedMessageChannel::create('orders', ['kafkaOutbox', 'orderProcessing']), - BatchForwardingConfiguration::create('kafkaOutbox'), + OutboxForwardingMessageChannel::create('orders', 'kafkaOutbox', 'orderProcessing'), KafkaMessageChannelBuilder::create('kafkaOutbox', topicName: $uniqueId, messageGroupId: $uniqueId), SimpleMessageChannelBuilder::createQueueChannel('orderProcessing'), ]),