diff --git a/.github/workflows/deploy-pm4.yml b/.github/workflows/deploy-pm4.yml index 0da93fde5c..8350a66d4f 100644 --- a/.github/workflows/deploy-pm4.yml +++ b/.github/workflows/deploy-pm4.yml @@ -9,5 +9,5 @@ on: jobs: run: name: Run PM4-workflow - uses: processmaker/.github/.github/workflows/deploy-pm4.yml@main + uses: processmaker/.github/.github/workflows/deploy-pm4.yml@better-cleanup secrets: inherit diff --git a/ProcessMaker/Console/Commands/ScheduleMultitenantRun.php b/ProcessMaker/Console/Commands/ScheduleMultitenantRun.php new file mode 100644 index 0000000000..c83ec57eae --- /dev/null +++ b/ProcessMaker/Console/Commands/ScheduleMultitenantRun.php @@ -0,0 +1,111 @@ +get()) { + $message = 'schedule:multitenant-run is already running; skipping this invocation.'; + Log::error($message); + $this->error($message); + + return self::FAILURE; + } + + $startedAt = microtime(true); + + try { + return $this->runSchedules(); + } catch (Throwable $e) { + Log::error('schedule:multitenant-run failed: ' . $e->getMessage(), [ + 'exception' => $e, + ]); + + throw $e; + } finally { + $lock->release(); + } + } + + private function runSchedules(): int + { + if (config('app.multitenancy') === false) { + return $this->call('schedule:run'); + } + + $tenants = Tenant::query()->cursor(); + + $ranForTenant = false; + + foreach ($tenants as $tenant) { + $ranForTenant = true; + $this->info("Running schedule for tenant [{$tenant->id}] {$tenant->name}"); + + $tenant->makeCurrent(); + + try { + // Schedule holds sticky CacheEventMutex/CacheSchedulingMutex instances. + // Forget it after makeCurrent so locks use this tenant's cache prefix. + // $this->forgetSchedule(); + $this->call('schedule:run'); + } finally { + Tenant::forgetCurrent(); + // $this->forgetSchedule(); + } + } + + if (!$ranForTenant) { + $this->info('No tenants found.'); + } + + return self::SUCCESS; + } + + /** + * Drop the Schedule singleton so the next resolve gets fresh mutexes/cache bindings. + */ + private function forgetSchedule(): void + { + app()->forgetInstance(Schedule::class); + } +} diff --git a/ProcessMaker/Console/Kernel.php b/ProcessMaker/Console/Kernel.php index 03fc408949..cd7424b8a2 100644 --- a/ProcessMaker/Console/Kernel.php +++ b/ProcessMaker/Console/Kernel.php @@ -4,6 +4,7 @@ use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Console\Kernel as ConsoleKernel; +use ProcessMaker\Console\Scheduling\FastSchedule; class Kernel extends ConsoleKernel { @@ -14,6 +15,18 @@ class Kernel extends ConsoleKernel */ protected $commands = []; + /** + * Resolve a console schedule instance that runs Artisan commands in-process. + * + * @return Schedule + */ + public function resolveConsoleSchedule() + { + return tap(new FastSchedule($this->scheduleTimezone()), function ($schedule) { + $this->schedule($schedule->useCache($this->scheduleCache())); + }); + } + /** * Define the application's command schedule. * @@ -24,6 +37,7 @@ protected function schedule(Schedule $schedule) { $schedule->command('bpmn:timer') ->everyMinute() + ->name('bpmn:timer') ->onOneServer() ->withoutOverlapping(config('app.scheduler.bpmn_timer_overlap_minutes', 5)); diff --git a/ProcessMaker/Console/Scheduling/FastCommandEvent.php b/ProcessMaker/Console/Scheduling/FastCommandEvent.php new file mode 100644 index 0000000000..589e7fcfee --- /dev/null +++ b/ProcessMaker/Console/Scheduling/FastCommandEvent.php @@ -0,0 +1,60 @@ + + */ + protected array $artisanParameters; + + /** + * @param array $artisanParameters + * @param \DateTimeZone|string|null $timezone + */ + public function __construct( + EventMutex $mutex, + string $shellCommand, + string $artisanCommand, + array $artisanParameters = [], + $timezone = null + ) { + parent::__construct($mutex, $shellCommand, $timezone); + + $this->artisanCommand = $artisanCommand; + $this->artisanParameters = $artisanParameters; + } + + public function artisanCommandName(): string + { + return (new StringInput($this->artisanCommand))->getFirstArgument() ?? $this->artisanCommand; + } + + /** + * Run the command in-process, or shell out when runInBackground is set. + * + * @param \Illuminate\Contracts\Container\Container $container + * @return int + */ + protected function execute($container) + { + if ($this->runInBackground) { + return parent::execute($container); + } + + return Artisan::call($this->artisanCommand, $this->artisanParameters); + } +} diff --git a/ProcessMaker/Console/Scheduling/FastSchedule.php b/ProcessMaker/Console/Scheduling/FastSchedule.php new file mode 100644 index 0000000000..6f820fd004 --- /dev/null +++ b/ProcessMaker/Console/Scheduling/FastSchedule.php @@ -0,0 +1,86 @@ +make(get_class($command)); + $commandDescription = $command->getDescription(); + $artisanCommand = $command->getName(); + } elseif (is_string($command) && class_exists($command)) { + $command = Container::getInstance()->make($command); + $commandDescription = $command->getDescription(); + $artisanCommand = $command->getName(); + } else { + $artisanCommand = $command; + } + + $artisanSignature = $artisanCommand; + if ($parameters !== []) { + $artisanSignature .= ' ' . $this->compileParameters($parameters); + } + + if ($this->containsShellMetacharacters($artisanSignature)) { + $event = parent::command($artisanCommand, $parameters); + + if ($commandDescription !== null) { + $event->description($commandDescription); + } + + return $event; + } + + $shellCommand = Application::formatCommandString($artisanCommand); + if ($parameters !== []) { + $shellCommand .= ' ' . $this->compileParameters($parameters); + } + + $this->events[] = $event = new FastCommandEvent( + $this->eventMutex, + $shellCommand, + $artisanCommand, + $parameters, + $this->timezone + ); + + $this->mergePendingAttributes($event); + + if ($commandDescription !== null) { + $event->description($commandDescription); + } + + if (empty($event->description)) { + $event->name($artisanSignature); + } + + return $event; + } + + /** + * Determine if the artisan signature requires a shell (redirects, pipes, etc.). + */ + protected function containsShellMetacharacters(string $command): bool + { + return (bool) preg_match('/(&&|&|\||>>|>)/', $command); + } +} diff --git a/ProcessMaker/Facades/Metrics.php b/ProcessMaker/Facades/Metrics.php index 878526722e..ba331ddf32 100644 --- a/ProcessMaker/Facades/Metrics.php +++ b/ProcessMaker/Facades/Metrics.php @@ -6,14 +6,14 @@ use ProcessMaker\Services\MetricsService; /** - * @method static \Prometheus\Counter counter(string $name, string $help = null, array $labels = []) - * @method static \Prometheus\Gauge gauge(string $name, string $help = null, array $labels = []) - * @method static \Prometheus\Histogram histogram(string $name, string $help = null, array $labels = [], array $buckets = [0.1, 1, 5, 10]) + * @method static \Prometheus\Counter counter(string $name, string|null $help = null, array $labels = []) + * @method static \Prometheus\Gauge gauge(string $name, string|null $help = null, array $labels = []) + * @method static \Prometheus\Histogram histogram(string $name, string|null $help = null, array $labels = [], array $buckets = []) * @method static void setGauge(string $name, float $value, array $labelValues = []) * @method static string renderMetrics() * @method static \Prometheus\CollectorRegistry getCollectionRegistry() - * @method static void counterInc(string $name, string $help = null, array $labels = []) - * @method static void histogramObserve(string $name, string $help = null, array $labels = [], array $buckets = [0.1, 1, 5, 10], float $executionTime) + * @method static void counterInc(string $name, string|null $help = null, array $labels = []) + * @method static void histogramObserve(string $name, string|null $help = null, array $labels = [], array $buckets = [], float $executionTime = 0) * @method static void clearMetrics() */ class Metrics extends Facade diff --git a/ProcessMaker/Listeners/ScheduledTaskMetricsSubscriber.php b/ProcessMaker/Listeners/ScheduledTaskMetricsSubscriber.php new file mode 100644 index 0000000000..b3fe9b0a3e --- /dev/null +++ b/ProcessMaker/Listeners/ScheduledTaskMetricsSubscriber.php @@ -0,0 +1,113 @@ + + */ + private WeakMap $startedAt; + + public function __construct() + { + $this->startedAt = new WeakMap(); + } + + public function handleStarting(ScheduledTaskStarting $event): void + { + $this->startedAt[$event->task] = microtime(true); + + if (Tenant::current() !== null) { + Metrics::clearResolvedInstance(MetricsService::class); + app()->forgetInstance(MetricsService::class); + } + } + + public function handleFinished(ScheduledTaskFinished $event): void + { + $job = $this->jobName($event->task); + + Metrics::gauge( + self::DURATION_SECONDS, + 'Duration of the last scheduled job run in seconds', + ['job'] + )->set($event->runtime, [$job]); + + unset($this->startedAt[$event->task]); + + if ($event->task->runInBackground || $event->task->exitCode !== 0) { + return; + } + + $this->recordResult($job, 'success'); + } + + public function handleFailed(ScheduledTaskFailed $event): void + { + $job = $this->jobName($event->task); + + if (isset($this->startedAt[$event->task])) { + Metrics::gauge( + self::DURATION_SECONDS, + 'Duration of the last scheduled job run in seconds', + ['job'] + )->set(microtime(true) - $this->startedAt[$event->task], [$job]); + + unset($this->startedAt[$event->task]); + } + + $this->recordResult($job, 'failure'); + } + + private function recordResult(string $job, string $status): void + { + $timestampMetric = $status === 'success' + ? self::LAST_SUCCESS_TIMESTAMP + : self::LAST_FAILURE_TIMESTAMP; + $resultDescription = $status === 'success' ? 'successful' : 'failed'; + + Metrics::gauge( + $timestampMetric, + "Unix timestamp of the last {$resultDescription} scheduled job run", + ['job'] + )->set(now()->timestamp, [$job]); + + Metrics::counter( + self::RUNS_TOTAL, + 'Total number of scheduled job runs', + ['job', 'status'] + )->inc([$job, $status]); + } + + private function jobName(Event $task): string + { + if ($task instanceof FastCommandEvent) { + return $task->artisanCommandName(); + } + + if (preg_match('/(?:^|\s)[\'"]?artisan[\'"]?\s+([^\s\'"]+)/', $task->command, $matches)) { + return $matches[1]; + } + + return $task->getSummaryForDisplay(); + } +} diff --git a/ProcessMaker/Multitenancy/PrefixCacheTask.php b/ProcessMaker/Multitenancy/PrefixCacheTask.php index 30aa0c127a..121e4876d1 100644 --- a/ProcessMaker/Multitenancy/PrefixCacheTask.php +++ b/ProcessMaker/Multitenancy/PrefixCacheTask.php @@ -2,11 +2,14 @@ namespace ProcessMaker\Multitenancy; +use Prometheus\Storage\Redis as PrometheusRedis; use Spatie\Multitenancy\Contracts\IsTenant; use Spatie\Multitenancy\Tasks\PrefixCacheTask as SpatiePrefixCacheTask; class PrefixCacheTask extends SpatiePrefixCacheTask { + private const LANDLORD_PROMETHEUS_PREFIX = 'PROMETHEUS_'; + private $originalSettingsPrefix; public function makeCurrent(IsTenant $tenant): void @@ -19,6 +22,8 @@ public function makeCurrent(IsTenant $tenant): void config()->set('cache.stores.cache_settings.prefix', $tenantSettingsPrefix); $this->storeName = 'cache_settings'; $this->setCachePrefix($cachePrefix); + + PrometheusRedis::setPrefix($cachePrefix . self::LANDLORD_PROMETHEUS_PREFIX); } public function forgetCurrent(): void @@ -28,5 +33,7 @@ public function forgetCurrent(): void config()->set('cache.stores.cache_settings.prefix', $this->originalSettingsPrefix); $this->storeName = 'cache_settings'; $this->setCachePrefix($this->originalPrefix); + + PrometheusRedis::setPrefix(self::LANDLORD_PROMETHEUS_PREFIX); } } diff --git a/ProcessMaker/Multitenancy/Services/TenantSchedulingService.php b/ProcessMaker/Multitenancy/Services/TenantSchedulingService.php index de7acb76f4..444bc75df5 100644 --- a/ProcessMaker/Multitenancy/Services/TenantSchedulingService.php +++ b/ProcessMaker/Multitenancy/Services/TenantSchedulingService.php @@ -3,9 +3,7 @@ namespace ProcessMaker\Multitenancy\Services; use Illuminate\Console\Scheduling\Schedule; -use Illuminate\Support\Facades\Artisan; use Illuminate\Support\Facades\Log; -use Illuminate\Support\Facades\Request; use ProcessMaker\Multitenancy\Tenant; class TenantSchedulingService @@ -58,11 +56,12 @@ public function registerScheduledTasksForTenant(Schedule $schedule, Tenant $tena */ protected function createScheduledEvent(Schedule $schedule, Tenant $tenant, string $command) { - $fullCommand = "tenants:artisan {$command} --tenant={$tenant->id}"; + // Run the tenant command directly. schedule:multitenant-run already + // makeCurrent()'s the tenant before schedule:run, and FastSchedule + // executes commands in-process so tenant context is preserved. + $event = $schedule->command($command); - $event = $schedule->command($fullCommand); - - Log::info("Created scheduled event for tenant {$tenant->id}: {$fullCommand}"); + Log::info("Created scheduled event for tenant {$tenant->id}: {$command}"); return $event; } diff --git a/ProcessMaker/Multitenancy/SwitchTenant.php b/ProcessMaker/Multitenancy/SwitchTenant.php index e85c1421f7..46fb90b86d 100644 --- a/ProcessMaker/Multitenancy/SwitchTenant.php +++ b/ProcessMaker/Multitenancy/SwitchTenant.php @@ -140,7 +140,7 @@ private function overrideConfigs(Application $app, IsTenant $tenant) // url() helper app(UrlGenerator::class)->useOrigin($tenant->config['app.url']); - // NOTE: Cache prefix and cache settings prefix are handled in PrefixCacheTask + // NOTE: Cache, cache settings, and Prometheus prefixes are handled in PrefixCacheTask if (!isset($tenant->config['app.docker_host_url'])) { // There is no specific override in the tenant's config so set it to the app url diff --git a/ProcessMaker/Providers/ProcessMakerServiceProvider.php b/ProcessMaker/Providers/ProcessMakerServiceProvider.php index c19ded9d5f..61d5112dcd 100644 --- a/ProcessMaker/Providers/ProcessMakerServiceProvider.php +++ b/ProcessMaker/Providers/ProcessMakerServiceProvider.php @@ -43,6 +43,7 @@ use ProcessMaker\ImportExport\SignalHelper; use ProcessMaker\Jobs\SmartInbox; use ProcessMaker\LicensedPackageManifest; +use ProcessMaker\Listeners\ScheduledTaskMetricsSubscriber; use ProcessMaker\Managers; use ProcessMaker\Managers\MenuManager; use ProcessMaker\Managers\ScreenCompiledManager; @@ -130,6 +131,8 @@ public function register(): void return new Managers\LoginManager(); }); + $this->app->singleton(ScheduledTaskMetricsSubscriber::class); + /* * Maps our Index Manager as a singleton. The Index Manager is used * to manage customizations to the search indexer. diff --git a/ProcessMaker/Services/MetricsService.php b/ProcessMaker/Services/MetricsService.php index 824f12b6cc..710a43c178 100644 --- a/ProcessMaker/Services/MetricsService.php +++ b/ProcessMaker/Services/MetricsService.php @@ -7,14 +7,12 @@ use Laravel\Horizon\Contracts\MetricsRepository; use Laravel\Horizon\Contracts\WorkloadRepository; use ProcessMaker\Facades\Metrics; -use ProcessMaker\Multitenancy\Tenant; use Prometheus\CollectorRegistry; use Prometheus\Counter; use Prometheus\Gauge; use Prometheus\Histogram; use Prometheus\RenderTextFormat; use Prometheus\Storage\Redis as PrometheusRedis; -use Redis; use RuntimeException; class MetricsService @@ -46,10 +44,6 @@ public function __construct(private $adapter = null) if ($adapter === null) { $redis = app('redis')->client(); $adapter = PrometheusRedis::fromExistingConnection($redis); - if (app()->has(Tenant::BOOTSTRAPPED_TENANT)) { - $tenantInfo = app(Tenant::BOOTSTRAPPED_TENANT); - $adapter->setPrefix('tenant_' . $tenantInfo['id'] . ':PROMETHEUS_'); - } } $this->collectionRegistry = new CollectorRegistry($adapter); } catch (Exception $e) { diff --git a/tests/Feature/Console/FastScheduleTest.php b/tests/Feature/Console/FastScheduleTest.php new file mode 100644 index 0000000000..76d42562d6 --- /dev/null +++ b/tests/Feature/Console/FastScheduleTest.php @@ -0,0 +1,94 @@ +assertInstanceOf(FastSchedule::class, app(Schedule::class)); + } + + public function testCommandCreatesFastCommandEventAndAutoNames() + { + $schedule = new FastSchedule(); + $event = $schedule->command('foo:bar --queue'); + + $this->assertInstanceOf(FastCommandEvent::class, $event); + $this->assertSame('foo:bar --queue', $event->description); + } + + public function testCommandPreservesExplicitName() + { + $schedule = new FastSchedule(); + $event = $schedule->command('foo:bar')->name('custom-name'); + + $this->assertInstanceOf(FastCommandEvent::class, $event); + $this->assertSame('custom-name', $event->description); + } + + public function testCommandFallsBackToShellEventForRedirects() + { + $schedule = new FastSchedule(); + $event = $schedule->command('cache:metrics --format=json > storage/logs/metrics.json'); + + $this->assertInstanceOf(Event::class, $event); + $this->assertNotInstanceOf(FastCommandEvent::class, $event); + } + + public function testRunInBackgroundKeepsShellFallbackFlag() + { + $schedule = new FastSchedule(); + $event = $schedule->command('cases:retention:evaluate')->runInBackground(); + + $this->assertInstanceOf(FastCommandEvent::class, $event); + $this->assertTrue($event->runInBackground); + } + + public function testCommandRunsInProcessViaArtisanCall() + { + Artisan::registerCommand(app(FastScheduleProbeCommand::class)); + + app()->instance('fast-schedule-probe-token', 'in-process'); + FastScheduleProbeCommand::$seenToken = null; + + $schedule = new FastSchedule(); + $event = $schedule->command('fast-schedule:probe'); + + $this->assertInstanceOf(FastCommandEvent::class, $event); + + $event->run(app()); + + $this->assertSame( + 'in-process', + FastScheduleProbeCommand::$seenToken, + 'Command should see container state from the current process' + ); + } +} + +class FastScheduleProbeCommand extends Command +{ + public static ?string $seenToken = null; + + protected $signature = 'fast-schedule:probe'; + + protected $description = 'Probe command for FastSchedule tests'; + + public function handle(): int + { + self::$seenToken = app()->bound('fast-schedule-probe-token') + ? app('fast-schedule-probe-token') + : null; + + return self::SUCCESS; + } +} diff --git a/tests/Feature/Console/ScheduleMultitenantRunTest.php b/tests/Feature/Console/ScheduleMultitenantRunTest.php new file mode 100644 index 0000000000..a6ff04dd09 --- /dev/null +++ b/tests/Feature/Console/ScheduleMultitenantRunTest.php @@ -0,0 +1,139 @@ +assertTrue( + array_key_exists('schedule:multitenant-run', Artisan::all()) + ); + } + + public function testCommandFallsBackToScheduleRunWhenMultitenancyDisabled() + { + config(['app.multitenancy' => false]); + + $this->artisan('schedule:multitenant-run') + ->assertSuccessful(); + } + + public function testCommandSkipsWhenLockIsAlreadyHeld() + { + config(['app.multitenancy' => false]); + + $lock = Cache::lock(ScheduleMultitenantRun::LOCK_KEY, ScheduleMultitenantRun::LOCK_SECONDS); + $this->assertTrue($lock->get()); + + try { + Log::shouldReceive('error') + ->once() + ->withArgs(function (string $message) { + return str_contains($message, 'already running'); + }); + + $this->artisan('schedule:multitenant-run') + ->expectsOutputToContain('already running') + ->assertFailed(); + } finally { + $lock->release(); + } + } + + public function testCommandReleasesLockAfterSuccessfulRun() + { + config(['app.multitenancy' => false]); + + $this->artisan('schedule:multitenant-run') + ->assertSuccessful(); + + $lock = Cache::lock(ScheduleMultitenantRun::LOCK_KEY, ScheduleMultitenantRun::LOCK_SECONDS); + $this->assertTrue($lock->get(), 'Lock should be available after a successful run'); + $lock->release(); + } + + public function testCommandRecordsDurationHistogram() + { + config(['app.multitenancy' => false]); + + $this->artisan('schedule:multitenant-run') + ->assertSuccessful(); + + $ns = config('app.prometheus_namespace', 'app'); + $histogram = Metrics::getCollectionRegistry()->getHistogram( + $ns, + ScheduleMultitenantRun::DURATION_METRIC + ); + + $this->assertInstanceOf(Histogram::class, $histogram); + $this->assertStringContainsString( + ScheduleMultitenantRun::DURATION_METRIC, + Metrics::renderMetrics() + ); + } + + public function testCommandDoesNotRecordDurationWhenLockIsAlreadyHeld() + { + config(['app.multitenancy' => false]); + + $lock = Cache::lock(ScheduleMultitenantRun::LOCK_KEY, ScheduleMultitenantRun::LOCK_SECONDS); + $this->assertTrue($lock->get()); + + try { + Log::shouldReceive('error') + ->once() + ->withArgs(function (string $message) { + return str_contains($message, 'already running'); + }); + + $this->artisan('schedule:multitenant-run') + ->assertFailed(); + + $this->assertStringNotContainsString( + ScheduleMultitenantRun::DURATION_METRIC, + Metrics::renderMetrics() + ); + } finally { + $lock->release(); + } + } + + public function testForgetScheduleRebindsSingletonWithFreshMutexes() + { + $command = app(ScheduleMultitenantRun::class); + $forgetSchedule = new ReflectionMethod($command, 'forgetSchedule'); + + $first = app(Schedule::class); + $firstMutexProperty = new ReflectionProperty(Schedule::class, 'eventMutex'); + $firstMutex = $firstMutexProperty->getValue($first); + + $forgetSchedule->invoke($command); + + $second = app(Schedule::class); + $secondMutex = $firstMutexProperty->getValue($second); + + $this->assertNotSame($first, $second); + $this->assertNotSame($firstMutex, $secondMutex); + } +} diff --git a/tests/Feature/Console/ScheduledTaskMetricsSubscriberTest.php b/tests/Feature/Console/ScheduledTaskMetricsSubscriberTest.php new file mode 100644 index 0000000000..fced5c465a --- /dev/null +++ b/tests/Feature/Console/ScheduledTaskMetricsSubscriberTest.php @@ -0,0 +1,116 @@ + false]); + + App::instance(MetricsService::class, new MetricsService(new InMemory())); + } + + public function testRecordsSuccessfulScheduledTaskMetrics(): void + { + $task = (new FastSchedule())->command('emails:send --queue'); + + Event::dispatch(new ScheduledTaskStarting($task)); + $task->exitCode = 0; + Event::dispatch(new ScheduledTaskFinished($task, 2.41)); + + $metrics = Metrics::renderMetrics(); + + $this->assertStringContainsString( + ScheduledTaskMetricsSubscriber::LAST_SUCCESS_TIMESTAMP . '{job="emails:send"}', + $metrics + ); + $this->assertStringContainsString( + ScheduledTaskMetricsSubscriber::DURATION_SECONDS . '{job="emails:send"} 2.41', + $metrics + ); + $this->assertStringContainsString( + ScheduledTaskMetricsSubscriber::RUNS_TOTAL . '{job="emails:send",status="success"} 1', + $metrics + ); + } + + public function testRecordsFailedScheduledTaskMetricsAndDuration(): void + { + $task = (new FastSchedule())->command('emails:send'); + + Event::dispatch(new ScheduledTaskStarting($task)); + Event::dispatch(new ScheduledTaskFailed($task, new RuntimeException('Sending failed'))); + + $metrics = Metrics::renderMetrics(); + + $this->assertStringContainsString( + ScheduledTaskMetricsSubscriber::LAST_FAILURE_TIMESTAMP . '{job="emails:send"}', + $metrics + ); + $this->assertStringContainsString( + ScheduledTaskMetricsSubscriber::DURATION_SECONDS . '{job="emails:send"}', + $metrics + ); + $this->assertStringContainsString( + ScheduledTaskMetricsSubscriber::RUNS_TOTAL . '{job="emails:send",status="failure"} 1', + $metrics + ); + $this->assertStringNotContainsString( + ScheduledTaskMetricsSubscriber::LAST_SUCCESS_TIMESTAMP . '{job="emails:send"}', + $metrics + ); + } + + public function testNonZeroExitIsOnlyRecordedAsFailure(): void + { + $task = (new FastSchedule())->command('emails:send'); + + Event::dispatch(new ScheduledTaskStarting($task)); + $task->exitCode = 1; + Event::dispatch(new ScheduledTaskFinished($task, 1.25)); + Event::dispatch(new ScheduledTaskFailed($task, new RuntimeException('Exit code 1'))); + + $metrics = Metrics::renderMetrics(); + + $this->assertStringContainsString( + ScheduledTaskMetricsSubscriber::RUNS_TOTAL . '{job="emails:send",status="failure"} 1', + $metrics + ); + $this->assertStringNotContainsString( + ScheduledTaskMetricsSubscriber::RUNS_TOTAL . '{job="emails:send",status="success"}', + $metrics + ); + } + + public function testTenantMetricsServiceIsResolvedAfterTenantBecomesCurrent(): void + { + config(['app.multitenancy' => true]); + app()->instance(config('multitenancy.current_tenant_container_key'), new Tenant(['id' => 123])); + + app(ScheduledTaskMetricsSubscriber::class)->handleStarting( + new ScheduledTaskStarting((new FastSchedule())->command('emails:send')) + ); + + $instances = (new ReflectionProperty(Container::class, 'instances'))->getValue(app()); + $this->assertArrayNotHasKey(MetricsService::class, $instances); + + app()->forgetInstance(config('multitenancy.current_tenant_container_key')); + } +}