diff --git a/.env.example b/.env.example index 7bef07fe..c372c3f0 100644 --- a/.env.example +++ b/.env.example @@ -89,6 +89,14 @@ AI_TIMEOUT=300s # set it to override for proxies/self-hosted gateways, e.g. AI_PROVIDER=ollama # AI_PROVIDER= +# OTLP endpoint traces and metrics are exported to. +#OTEL_EXPORTER_ENDPOINT=http://localhost:4317 +# Prometheus scrape endpoint: the same metrics in Prometheus text format at GET /metrics on the +# main HTTP port (on by default, no collector needed). Unauthenticated — it reveals model names, +# token counts, and spend, so restrict it at the proxy/firewall or disable it when the bot is +# internet-facing. +#PROMETHEUS_METRICS_ENABLED=false + # Per-model AI settings (thrillhousebot.ai.models..*, keyed by the AI_MODEL name like the # pricing map). Keep entries for several models and switch AI_MODEL freely — only the active # model's entry is read. max-input-tokens is the model's input hard cap (context window): the diff --git a/CHANGELOG.md b/CHANGELOG.md index 16465a16..dc7538a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to ThrillhouseBot. ## [Unreleased] +### Added + +- **Prometheus metrics endpoint**: the metrics exported over OTLP — token usage, latency, AI cost, and the new `thrillhouse.reviews.total` finished-reviews-by-outcome counter — are also served in Prometheus text format at `GET /metrics` on the main HTTP port, so self-hosters can scrape directly without an OTLP collector. On by default; the endpoint is unauthenticated, so shield it at the proxy/firewall or disable it with `PROMETHEUS_METRICS_ENABLED=false` (scrapes then return `404`; OTLP export is unaffected) + ## [0.5.0] — 2026-07-26 Review precision: confidence now decides where a finding lands, newly-added parsers and regexes are stress-tested for their own failure modes, and several classes of false positive are guarded at both the generator and the verifier. Operators gain configurable CI-gating and blocking strictness, structured skip reasons, and per-model generation parameters. diff --git a/README.md b/README.md index d6f6eb71..fd31f4db 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ guide, configuration reference, architecture, comparison, and the hosted - A summary comment on the first run, with a risk breakdown and a changed-files walkthrough - Operable from the PR with comment commands — `/help`, `/review`, `/summary`, `/describe`, `/changelog`, `/add-docs`, `/resolve`, `/pause`, `/resume` - Live dashboard (Next.js) with a WebSocket activity feed, cost charts, and token tracking -- OpenTelemetry traces, token histograms, cost counters, and latency metrics +- OpenTelemetry traces, token histograms, cost counters, and latency and review-outcome metrics — exported via OTLP and scrapeable in Prometheus format at `/metrics` - Optional reasoning-effort dial and per-model generation/budget caps for OpenAI-compatible endpoints - Reads per-repo instructions from `.github/thrillhousebot.md`, falling back to Copilot/Claude/Agents files - Compiles ahead-of-time with GraalVM/Mandrel, so it starts fast and stays small @@ -268,6 +268,8 @@ will change per provider: | `HTTP_CONNECT_TIMEOUT` | Outbound HTTP connect timeout (GitHub API, OAuth) | `10s` | | `HTTP_REQUEST_TIMEOUT` | Outbound HTTP request timeout (GitHub API, OAuth) | `10s` | | `WEBSOCKET_KEEPALIVE_MS` | Dashboard WebSocket keepalive interval in ms; `0` or negative disables it (and stale replay-buffer eviction) | `25000` | +| `OTEL_EXPORTER_ENDPOINT` | OTLP endpoint traces and metrics are exported to | `http://localhost:4317` | +| `PROMETHEUS_METRICS_ENABLED` | Serve the same metrics in Prometheus text format at `GET /metrics` on the main HTTP port; the endpoint is unauthenticated, so disable or shield it on internet-facing deployments | `true` | ### AI call budget @@ -444,7 +446,8 @@ Labelling is best-effort — a failure here never blocks or fails the review. ## Observability -All telemetry is exported via OTLP: +All telemetry is exported via OTLP (`OTEL_EXPORTER_ENDPOINT`, default +`http://localhost:4317`): | Signal | Metric | |---|---| @@ -453,12 +456,34 @@ All telemetry is exported via OTLP: | `gen_ai.client.operation.duration` | Histogram: latency in seconds | | `thrillhouse.ai.cost.total` | Counter: USD cost by model | | `thrillhouse.review.skips` | Counter: automatic reviews skipped, tagged with `reason` and `repository` | +| `thrillhouse.reviews.total` | Counter: finished reviews by `outcome` (`completed`/`failed`) | Spans and metrics are tagged with `gen_ai.provider.name`, derived from `AI_BASE_URL` (e.g. `deepseek`, `openai`, `groq`, `openrouter`). Loopback and unrecognized endpoints report `unknown`; set `AI_PROVIDER` to label them (e.g. a local `ollama` or `vllm` server, a proxy, or a self-hosted gateway). +### Prometheus + +The same metrics are also served in Prometheus text format at `GET /metrics` on +the main HTTP port — no OTLP collector needed: + +```yaml +scrape_configs: + - job_name: thrillhousebot + static_configs: + - targets: ["your-bot-host:8080"] +``` + +Metric names follow the Prometheus conversion of the OTel names above (e.g. +`thrillhouse_ai_cost_total`, `gen_ai_client_token_usage`, +`thrillhouse_reviews_total`). + +The endpoint is unauthenticated and reveals model names, token counts, and +spend. On an internet-facing deployment, restrict `/metrics` at your reverse +proxy or firewall, or disable it with `PROMETHEUS_METRICS_ENABLED=false` +(scrapes then return `404`; OTLP export is unaffected). + ## Troubleshooting ### PR opened but no review posted diff --git a/pom.xml b/pom.xml index 2fd128a5..7f999414 100644 --- a/pom.xml +++ b/pom.xml @@ -22,6 +22,8 @@ 1.12.0 1.62.0 1.62.0-alpha + + 1.8.0 2.22.1 true @@ -141,6 +143,24 @@ io.quarkus quarkus-opentelemetry + + + io.opentelemetry + opentelemetry-exporter-prometheus + + + io.prometheus + prometheus-metrics-model + ${prometheus-metrics.version} + + + io.prometheus + prometheus-metrics-exposition-textformats + ${prometheus-metrics.version} + diff --git a/src/main/java/dev/thiagogonzaga/thrillhousebot/config/ThrillhouseConfig.java b/src/main/java/dev/thiagogonzaga/thrillhousebot/config/ThrillhouseConfig.java index 41b986d4..56801cbb 100644 --- a/src/main/java/dev/thiagogonzaga/thrillhousebot/config/ThrillhouseConfig.java +++ b/src/main/java/dev/thiagogonzaga/thrillhousebot/config/ThrillhouseConfig.java @@ -53,6 +53,19 @@ public interface ThrillhouseConfig { AiPricingConfig ai(); + MetricsConfig metrics(); + + interface MetricsConfig { + /** + * Serves the OpenTelemetry metrics in Prometheus text format at {@code GET /metrics} on the + * main HTTP port, alongside the OTLP export. The endpoint is unauthenticated; disable it (or + * shield it at the proxy) on internet-facing deployments. + */ + @WithDefault("true") + @WithName("prometheus-enabled") + boolean prometheusEnabled(); + } + interface GitHubConfig { @WithName("app-id") String appId(); diff --git a/src/main/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsRegistrar.java b/src/main/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsRegistrar.java new file mode 100644 index 00000000..84fbadc7 --- /dev/null +++ b/src/main/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsRegistrar.java @@ -0,0 +1,63 @@ +/* + * Copyright 2026 Thiago Gonzaga + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.thiagogonzaga.thrillhousebot.observability; + +import dev.thiagogonzaga.thrillhousebot.config.ThrillhouseConfig; +import io.opentelemetry.exporter.prometheus.PrometheusMetricReader; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdkBuilder; +import io.quarkus.opentelemetry.runtime.AutoConfiguredOpenTelemetrySdkBuilderCustomizer; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Registers a {@link PrometheusMetricReader} on the OpenTelemetry SDK so the metrics exported over + * OTLP can also be scraped in Prometheus text format from {@code GET /metrics} (served by {@link + * PrometheusMetricsResource}). Quarkus invokes every CDI bean implementing {@link + * AutoConfiguredOpenTelemetrySdkBuilderCustomizer} while building the SDK; the reader stays unset — + * and the endpoint answers 404 — when {@code thrillhousebot.metrics.prometheus-enabled=false} or + * the OTel SDK itself is disabled. + */ +@ApplicationScoped +public class PrometheusMetricsRegistrar implements AutoConfiguredOpenTelemetrySdkBuilderCustomizer { + + private final boolean enabled; + private final AtomicReference reader = new AtomicReference<>(); + + @Inject + public PrometheusMetricsRegistrar(ThrillhouseConfig config) { + this.enabled = config.metrics().prometheusEnabled(); + } + + @Override + public void customize(AutoConfiguredOpenTelemetrySdkBuilder builder) { + if (!enabled) { + return; + } + builder.addMeterProviderCustomizer( + (meterProviderBuilder, configProperties) -> { + var prometheusReader = PrometheusMetricReader.create(); + reader.set(prometheusReader); + return meterProviderBuilder.registerMetricReader(prometheusReader); + }); + } + + /** The registered reader, or empty when the endpoint is disabled or the SDK never started. */ + Optional reader() { + return Optional.ofNullable(reader.get()); + } +} diff --git a/src/main/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsResource.java b/src/main/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsResource.java new file mode 100644 index 00000000..de3c27a7 --- /dev/null +++ b/src/main/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsResource.java @@ -0,0 +1,58 @@ +/* + * Copyright 2026 Thiago Gonzaga + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.thiagogonzaga.thrillhousebot.observability; + +import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter; +import jakarta.inject.Inject; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * Prometheus scrape endpoint on the main HTTP port. Serves whatever the {@link + * PrometheusMetricsRegistrar} collected — token usage, latency, cost, and review-outcome metrics — + * in Prometheus text format; answers 404 while no reader is registered (endpoint disabled via + * {@code PROMETHEUS_METRICS_ENABLED=false} or OTel SDK disabled). + */ +@Path("/metrics") +public class PrometheusMetricsResource { + + private static final PrometheusTextFormatWriter WRITER = PrometheusTextFormatWriter.create(); + + private final PrometheusMetricsRegistrar registrar; + + @Inject + public PrometheusMetricsResource(PrometheusMetricsRegistrar registrar) { + this.registrar = registrar; + } + + @GET + public Response scrape() throws IOException { + var reader = registrar.reader(); + if (reader.isEmpty()) { + return Response.status(Response.Status.NOT_FOUND) + .type(MediaType.TEXT_PLAIN) + .entity("Prometheus metrics are disabled (PROMETHEUS_METRICS_ENABLED=false).") + .build(); + } + var buffer = new ByteArrayOutputStream(); + WRITER.write(buffer, reader.get().collect()); + return Response.ok(buffer.toByteArray(), WRITER.getContentType()).build(); + } +} diff --git a/src/main/java/dev/thiagogonzaga/thrillhousebot/observability/ReviewOutcomeMetrics.java b/src/main/java/dev/thiagogonzaga/thrillhousebot/observability/ReviewOutcomeMetrics.java new file mode 100644 index 00000000..0840cee8 --- /dev/null +++ b/src/main/java/dev/thiagogonzaga/thrillhousebot/observability/ReviewOutcomeMetrics.java @@ -0,0 +1,53 @@ +/* + * Copyright 2026 Thiago Gonzaga + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.thiagogonzaga.thrillhousebot.observability; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +/** + * Counts finished PR reviews by outcome. Recorded once per review at its terminal transition (the + * orchestrator's completed/failed paths), never per AI call — retried calls inside one review must + * not inflate the count. + */ +@ApplicationScoped +public class ReviewOutcomeMetrics { + + private static final AttributeKey OUTCOME = AttributeKey.stringKey("outcome"); + + private final LongCounter reviews; + + @Inject + public ReviewOutcomeMetrics(OpenTelemetry otel) { + this.reviews = + otel.getMeter("thrillhousebot") + .counterBuilder("thrillhouse.reviews.total") + .setDescription("Finished PR reviews by outcome") + .build(); + } + + public void recordCompleted() { + reviews.add(1, Attributes.of(OUTCOME, "completed")); + } + + public void recordFailed() { + reviews.add(1, Attributes.of(OUTCOME, "failed")); + } +} diff --git a/src/main/java/dev/thiagogonzaga/thrillhousebot/review/ReviewOrchestrator.java b/src/main/java/dev/thiagogonzaga/thrillhousebot/review/ReviewOrchestrator.java index 73f887b4..2f5d8038 100644 --- a/src/main/java/dev/thiagogonzaga/thrillhousebot/review/ReviewOrchestrator.java +++ b/src/main/java/dev/thiagogonzaga/thrillhousebot/review/ReviewOrchestrator.java @@ -21,6 +21,7 @@ import dev.thiagogonzaga.thrillhousebot.dashboard.ReviewSessionPersistence; import dev.thiagogonzaga.thrillhousebot.dashboard.SessionEventBroadcaster; import dev.thiagogonzaga.thrillhousebot.github.*; +import dev.thiagogonzaga.thrillhousebot.observability.ReviewOutcomeMetrics; import io.quarkus.logging.Log; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.context.control.ActivateRequestContext; @@ -60,6 +61,8 @@ public class ReviewOrchestrator { private final FindingPipeline findingPipeline; + private final ReviewOutcomeMetrics outcomeMetrics; + private final FindingFeedbackCaptureService findingFeedbackCapture; private final ExecutorService reviewExecutor; @@ -176,6 +179,7 @@ public ReviewOrchestrator( ReviewPublisher reviewPublisher, VerdictBuilder verdictBuilder, FindingPipeline findingPipeline, + ReviewOutcomeMetrics outcomeMetrics, FindingFeedbackCaptureService findingFeedbackCapture, @ReviewExecutor ExecutorService reviewExecutor) { this.config = config; @@ -190,6 +194,7 @@ public ReviewOrchestrator( this.reviewPublisher = reviewPublisher; this.verdictBuilder = verdictBuilder; this.findingPipeline = findingPipeline; + this.outcomeMetrics = outcomeMetrics; this.findingFeedbackCapture = findingFeedbackCapture; this.reviewExecutor = reviewExecutor; } @@ -469,6 +474,7 @@ void applyReviewResult(ReviewSession session, ReviewResult result) { s.setAiResponseJson(session.getAiResponseJson()); } }); + outcomeMetrics.recordCompleted(); } /** Applies failure fields to the in-memory session and persisted entity together. */ @@ -479,6 +485,7 @@ void applyReviewFailure(ReviewSession session, String errorMessage) { s.setStatus(ReviewSession.STATUS_FAILED); s.setErrorMessage(errorMessage); }); + outcomeMetrics.recordFailed(); } private void applySessionState(ReviewSession session, Consumer mutator) { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 1ef456e0..83aff242 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -111,6 +111,12 @@ quarkus.langchain4j.openai.chat-model.log-responses=false # OpenTelemetry quarkus.otel.exporter.otlp.traces.endpoint=${OTEL_EXPORTER_ENDPOINT:http://localhost:4317} quarkus.otel.service.name=thrillhousebot +# Metrics are recorded via the OpenTelemetry API; enabling them exports over OTLP alongside traces +# (build-time flag, baked into the artifact). +quarkus.otel.metrics.enabled=true +# Prometheus scrape endpoint: the same metrics, served in Prometheus text format at GET /metrics +# on the main HTTP port. Unauthenticated — restrict or disable it on internet-facing deployments. +thrillhousebot.metrics.prometheus-enabled=${PROMETHEUS_METRICS_ENABLED:true} # Review settings thrillhousebot.review.max-review-comments=50 diff --git a/src/test/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsEndpointDisabledTest.java b/src/test/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsEndpointDisabledTest.java new file mode 100644 index 00000000..7d0131e6 --- /dev/null +++ b/src/test/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsEndpointDisabledTest.java @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Thiago Gonzaga + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.thiagogonzaga.thrillhousebot.observability; + +import static io.restassured.RestAssured.given; + +import io.quarkus.test.junit.QuarkusTest; +import org.junit.jupiter.api.Test; + +/** + * With the OTel SDK disabled (the shared test config default) no Prometheus reader is registered, + * and the scrape endpoint must answer 404 rather than an empty 200 a scraper would treat as + * healthy. + */ +@QuarkusTest +class PrometheusMetricsEndpointDisabledTest { + + @Test + void metricsEndpointAnswers404WithoutRegisteredReader() { + given().when().get("/metrics").then().statusCode(404); + } +} diff --git a/src/test/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsEndpointTest.java b/src/test/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsEndpointTest.java new file mode 100644 index 00000000..ac13cbfd --- /dev/null +++ b/src/test/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsEndpointTest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2026 Thiago Gonzaga + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.thiagogonzaga.thrillhousebot.observability; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.containsString; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * End-to-end check that Quarkus invokes the registrar while building the OTel SDK and that the + * scrape endpoint serves what the application records. Runs with the SDK enabled (the shared test + * config disables it), so it pays a separate augmentation, which is exactly the wiring under test. + */ +@QuarkusTest +@TestProfile(PrometheusMetricsEndpointTest.OtelSdkEnabledProfile.class) +class PrometheusMetricsEndpointTest { + + @Inject ReviewOutcomeMetrics outcomeMetrics; + + @Test + void metricsEndpointServesRecordedMetrics() { + outcomeMetrics.recordCompleted(); + + given() + .when() + .get("/metrics") + .then() + .statusCode(200) + .header("Content-Type", containsString("text/plain")) + .body(containsString("thrillhouse_reviews_total")) + .body(containsString("outcome=\"completed\"")); + } + + public static class OtelSdkEnabledProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of("quarkus.otel.sdk.disabled", "false"); + } + } +} diff --git a/src/test/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsResourceTest.java b/src/test/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsResourceTest.java new file mode 100644 index 00000000..5ea34451 --- /dev/null +++ b/src/test/java/dev/thiagogonzaga/thrillhousebot/observability/PrometheusMetricsResourceTest.java @@ -0,0 +1,83 @@ +/* + * Copyright 2026 Thiago Gonzaga + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dev.thiagogonzaga.thrillhousebot.observability; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import dev.thiagogonzaga.thrillhousebot.config.ThrillhouseConfig; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Exercises the registrar + resource pair against a real OpenTelemetry SDK built through the same + * autoconfigure builder Quarkus uses, so the OTel→Prometheus naming conversion is covered without + * booting the application. + */ +class PrometheusMetricsResourceTest { + + private static ThrillhouseConfig configWithPrometheusEnabled(boolean enabled) { + var metrics = mock(ThrillhouseConfig.MetricsConfig.class); + when(metrics.prometheusEnabled()).thenReturn(enabled); + var config = mock(ThrillhouseConfig.class); + when(config.metrics()).thenReturn(metrics); + return config; + } + + @Test + void scrapeServesOtelMetricsInPrometheusTextFormat() throws IOException { + var registrar = new PrometheusMetricsRegistrar(configWithPrometheusEnabled(true)); + var builder = + AutoConfiguredOpenTelemetrySdk.builder() + .disableShutdownHook() + .addPropertiesSupplier( + () -> + Map.of( + "otel.traces.exporter", "none", + "otel.metrics.exporter", "none", + "otel.logs.exporter", "none")); + registrar.customize(builder); + var sdk = builder.build().getOpenTelemetrySdk(); + try { + new ReviewOutcomeMetrics(sdk).recordCompleted(); + + var response = new PrometheusMetricsResource(registrar).scrape(); + + assertEquals(200, response.getStatus()); + assertTrue(response.getMediaType().toString().startsWith("text/plain")); + var body = new String((byte[]) response.getEntity(), StandardCharsets.UTF_8); + assertTrue(body.contains("thrillhouse_reviews_total"), body); + assertTrue(body.contains("outcome=\"completed\""), body); + } finally { + sdk.close(); + } + } + + @Test + void scrapeReturns404WhenPrometheusDisabled() throws IOException { + var registrar = new PrometheusMetricsRegistrar(configWithPrometheusEnabled(false)); + var builder = AutoConfiguredOpenTelemetrySdk.builder(); + registrar.customize(builder); + + assertTrue(registrar.reader().isEmpty()); + assertEquals(404, new PrometheusMetricsResource(registrar).scrape().getStatus()); + } +} diff --git a/src/test/java/dev/thiagogonzaga/thrillhousebot/review/ReviewOrchestratorTest.java b/src/test/java/dev/thiagogonzaga/thrillhousebot/review/ReviewOrchestratorTest.java index 52b2e654..9211ddba 100644 --- a/src/test/java/dev/thiagogonzaga/thrillhousebot/review/ReviewOrchestratorTest.java +++ b/src/test/java/dev/thiagogonzaga/thrillhousebot/review/ReviewOrchestratorTest.java @@ -27,10 +27,12 @@ import dev.thiagogonzaga.thrillhousebot.dashboard.ReviewSessionPersistence; import dev.thiagogonzaga.thrillhousebot.dashboard.SessionEventBroadcaster; import dev.thiagogonzaga.thrillhousebot.github.*; +import dev.thiagogonzaga.thrillhousebot.observability.ReviewOutcomeMetrics; import dev.thiagogonzaga.thrillhousebot.review.ai.AiReviewService; import dev.thiagogonzaga.thrillhousebot.review.ai.FindingVerificationService; import dev.thiagogonzaga.thrillhousebot.review.ai.ReviewResponse; import dev.thiagogonzaga.thrillhousebot.review.ai.TokenCounter; +import io.opentelemetry.api.OpenTelemetry; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; @@ -240,6 +242,7 @@ diffFormatter, new TokenCounter(), config, new ActiveModelSettings(config, "m")) reviewPublisher, verdictBuilder, findingPipeline, + new ReviewOutcomeMetrics(OpenTelemetry.noop()), mock(FindingFeedbackCaptureService.class), reviewExecutor); }