diff --git a/.github/workflows/promql-differential.yml b/.github/workflows/promql-differential.yml new file mode 100644 index 0000000..e968d1c --- /dev/null +++ b/.github/workflows/promql-differential.yml @@ -0,0 +1,68 @@ +name: PromQL Differential Tests + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + branches: [main] + paths: + - 'promql-compliance/**' + - 'asap-query-engine/**' + - 'asap-planner-rs/**' + - 'asap-common/**' + - 'Cargo.toml' + - 'Cargo.lock' + - '.github/workflows/promql-differential.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + packages: read + +jobs: + differential: + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR for Docker layer cache + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.25.x' + + - name: Run seeder tests + working-directory: promql-compliance/seeder + run: go test ./... + + - name: Run runner tests + working-directory: promql-compliance/runner + run: go test ./... + + - name: Run live PromQL differential suite + working-directory: promql-compliance/runner + run: make run + + - name: Upload differential report and service logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: promql-differential-results + path: | + promql-compliance/runner/differential-report.json + /tmp/asapquery-differential-* + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 3fb8e4e..5374e70 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ asap-quickstart/bin/ # roborev snapshots /.roborev/ +# Differential test output +promql-compliance/runner/differential-report.json /.agent/ /**/*.log diff --git a/promql-compliance/ARCHITECTURE.md b/promql-compliance/ARCHITECTURE.md new file mode 100644 index 0000000..5603fae --- /dev/null +++ b/promql-compliance/ARCHITECTURE.md @@ -0,0 +1,106 @@ +# Differential experiment architecture + +The differential runner answers one question: + +> Given the same samples and the same PromQL request, does ASAPQuery produce +> the same result as Prometheus? + +The experiment uses Prometheus as the reference implementation. ASAPQuery is +the system under test. + +## Stack + +```text + query requests + ┌─────────────────────┐ + │ differential runner│ + └─────────┬───────────┘ + │ + ┌─────────────┴─────────────┐ + │ │ + Prometheus reference ASAPQuery query API + host :19090 host :18088 + ▲ ▲ + │ │ + remote-write data remote-write ingest + │ host :19091 + │ ▲ + └───────────┬───────────────┘ + │ + identical samples + + ASAPQuery planner ──► shared planner-output volume ──► query engine +``` + +The Compose services are: + +| Service | Role | Host ports | +| --- | --- | --- | +| `prometheus` | Reference PromQL implementation and backend | `19090` | +| `planner` | Generates ASAPQuery inference and streaming configuration | none | +| `queryengine` | Runs ASAPQuery in precompute mode | query `18088`, ingest `19091` | + +Inside the Compose network, the query engine reaches Prometheus at +`http://prometheus:9090`. The engine configuration sets +`forward_unsupported_queries: false`, so an unsupported query is rejected +instead of being answered by Prometheus. + +## Run lifecycle + +The runner performs these stages: + +1. Load a dataset fixture and a query suite. +2. Choose a base timestamp. Dataset sample offsets and suite evaluation times + are relative to this timestamp. +3. Generate temporary planner and engine configuration. +4. Start Prometheus and wait for its health check. +5. Run the planner. Its output is placed in a named shared volume. +6. Start the ASAPQuery query engine after the planner completes. +7. Send the same encoded remote-write batches to Prometheus and ASAPQuery. +8. Wait for both query APIs to be ready and for the first probe query to return + samples. +9. Execute every configured range and instant query against both targets. +10. Write the JSON report and, by default, remove the containers, network, and + named volumes. + +Use `--keep-services` when inspecting logs or making manual requests after the +runner exits. + +## Configuration boundaries + +The checked-in dataset and suite are inputs to the runner, not service +configuration. The runner generates these temporary files: + +- `controller-config.yaml`: metrics, query groups, planner timing, and cleanup + policy. +- `engine_config.yaml`: HTTP ports, backend, ingestion, logging, and paths to + planner output. + +The planner receives metric and label hints from the dataset, so it does not +need to discover them from a separate live data source in this workflow. + +## Comparison model + +For each query, the runner can perform four related checks: + +- Range comparison: Prometheus range result versus ASAPQuery range result. +- Instant comparison: Prometheus instant result versus ASAPQuery instant result. +- Reference parity: Prometheus range-at-t versus Prometheus instant-at-t. +- Test parity: ASAPQuery range-at-t versus ASAPQuery instant-at-t. + +The report passes only if every configured comparison passes. An unexpected +HTTP/query error from either target fails the comparison, even if both targets +fail. The exception is a query explicitly marked `expect_error: true`, where +both targets must return an error. + +Equal successful empty results are still equal results; use a dataset and +probe query that should contain samples when testing ingestion readiness. + +## Current limitations + +The runner’s readiness check currently probes the first suite query and its +first evaluation time. It does not yet expose a general ingestion watermark or +drain signal for proving that every asynchronous batch has finished processing. +For the same reason, suites should currently use a supported, non-empty first +query as their readiness probe. These are tracked as follow-up synchronization +work. diff --git a/promql-compliance/HOW_TO.md b/promql-compliance/HOW_TO.md new file mode 100644 index 0000000..68dffe5 --- /dev/null +++ b/promql-compliance/HOW_TO.md @@ -0,0 +1,236 @@ +# Differential experiment how-to guide + +All commands below assume the current directory is +`promql-compliance/runner`. + +## Add a dataset + +Create a YAML fixture under `promql-compliance/datasets/`: + +```yaml +name: cpu-example +series: + - metric: cpu_usage_seconds_total + labels: + host: server-a + mode: user + samples: + - {offset_seconds: 0, value: 0} + - {offset_seconds: 60, value: 10} + - {offset_seconds: 120, value: 25} +``` + +Each `series` entry is one labeled time series. Add another entry with +different labels to represent another series of the same metric. Sample +timestamps are seconds relative to the run’s `baseTime`; values are written to +both targets without changing the payload. + +Smoke-test the stack with the checked-in fixture and matching temporal suite: + +```bash +go run ./cmd/differential-runner \ + --dataset ../datasets/single-rate.yaml \ + --suite ../suites/temporal.yaml \ + --compose-file ../docker-compose.yml +``` + +The runner derives the planner’s metric and label hints from the fixture. No +source-code change is needed to add a dataset. To run the `cpu-example` above, +create a suite whose expressions use `cpu_usage_seconds_total`, as shown in +the next section. + +## Add or change queries + +Create or edit a suite under `promql-compliance/suites/`: + +```yaml +name: cpu-queries +comparison_defaults: + value_tolerance: + relative: 0 + absolute: 0.000001 + +queries: + - name: cpu-rate + expr: rate(cpu_usage_seconds_total[5m]) + instant_offsets_seconds: [300, 600] + range: + start_offset_seconds: 300 + end_offset_seconds: 600 + step_seconds: 60 + + - name: user-cpu-at-end + expr: cpu_usage_seconds_total{mode="user"} + instant_offsets_seconds: [600] +``` + +For each query: + +- `expr` is the PromQL expression sent to both targets. +- `instant_offsets_seconds` selects instant-query times relative to + `baseTime`. +- `range` selects a range-query start, end, and evaluation step. +- Instant offsets must lie inside the configured range when both are present. +- A query may contain only instant evaluations, only a range, or both. + +Run it with: + +```bash +go run ./cmd/differential-runner \ + --dataset ../datasets/cpu-example.yaml \ + --suite ../suites/cpu-queries.yaml \ + --compose-file ../docker-compose.yml +``` + +## Change query timing + +`range.step_seconds` controls the evaluation cadence of a range query. It is +not the same as the planner’s repetition delay. + +The runner generates one planner query group per suite query. For each group, +it derives a compatible repetition delay from the largest range selector in +the expression, caps it at five minutes, and adjusts it to work with the +range step. Queries without a range selector use the one-second ingestion +interval. The query’s range step is also passed to the planner as `step_ms`. + +There is currently no YAML field for an arbitrary per-query repetition delay. +To change the five-minute cap or one-second ingestion assumption, change the +runner’s generated-config logic in `runner/run.go`. Do not edit the generated +configuration: it is temporary and is deleted after the run. + +## Configure tolerance + +Set a suite-wide default: + +```yaml +comparison_defaults: + value_tolerance: + relative: 0.001 + absolute: 0.000001 +``` + +Override either value for one query: + +```yaml +queries: + - name: approximate-query + expr: some_query + instant_offsets_seconds: [600] + comparison: + value_tolerance: + relative: 0.01 +``` + +If tolerance is omitted, values are compared exactly. A tolerance should be +small and justified; a broad tolerance can hide a correctness bug. + +## Test expected failures + +Use this only when the query is deliberately expected to fail on both +targets: + +```yaml +- name: intentionally-unsupported + expr: unsupported_expression + instant_offsets_seconds: [600] + expect_error: true +``` + +For ordinary queries, an error from either target makes the comparison fail. +Two matching errors do not accidentally pass. With `expect_error: true`, both +targets must return errors; a success from either target fails the comparison. + +This also makes the suite useful for finding unsupported ASAPQuery queries: +leave `expect_error` unset for a query that Prometheus supports. A Prometheus +success paired with an ASAPQuery error produces `passed: false` and records the +ASAPQuery error in `testError`. + +## Run with DEBUG logging + +The default Compose stack uses `INFO`. Temporarily change the query engine +service in `promql-compliance/docker-compose.yml`: + +```yaml +environment: + RUST_LOG: DEBUG +``` + +Then retain the services while running: + +```bash +go run ./cmd/differential-runner \ + --dataset ../datasets/single-rate.yaml \ + --suite ../suites/temporal.yaml \ + --compose-file ../docker-compose.yml \ + --keep-services \ + --output /tmp/differential-report-debug.json +``` + +Inspect the query-engine container: + +```bash +docker logs asapquery-differential-queryengine-1 2>&1 \ + | rg "destination=asap|destination=prometheus|none_unsupported|QUERY ENGINE SUCCESS" +``` + +Interpret the destination markers as follows: + +- `destination=asap`: ASAPQuery answered locally. +- `destination=prometheus`: the query was forwarded to the backend. +- `destination=none_unsupported`: the query was unsupported and forwarding was + disabled. +- `QUERY ENGINE SUCCESS`: the local query engine produced a result. + +Clean up the retained stack afterward: + +```bash +ASAP_TEST_CONFIG_DIR=/tmp docker compose \ + --project-name asapquery-differential \ + --file ../docker-compose.yml \ + down --volumes --remove-orphans +``` + +## Use already-running targets + +When Prometheus and ASAPQuery are managed outside Compose, omit +`--compose-file` and provide their query and write endpoints: + +```bash +go run ./cmd/differential-runner \ + --dataset ../datasets/cpu-example.yaml \ + --suite ../suites/cpu-queries.yaml \ + --reference-url http://localhost:9090 \ + --test-url http://localhost:8088 \ + --reference-write-url http://localhost:9090 \ + --test-write-url http://localhost:9091 +``` + +In this mode, the services’ existing configurations are authoritative. The +runner only writes the fixture data and sends the comparison queries. + +## Read the report + +The JSON report contains: + +- `range`: Prometheus range result versus ASAPQuery range result. +- `instant`: cross-target comparison at each configured instant time. +- `referenceParity`: Prometheus range-at-t versus Prometheus instant-at-t. +- `testParity`: ASAPQuery range-at-t versus ASAPQuery instant-at-t. +- `passed`: the conjunction of every comparison in the report. + +On failure, `referenceError`, `testError`, or `diff` identifies the failure +class. The current report records differences and errors, but does not include +the complete raw response bodies. + +## Diagnose common failures + +- Planner fails during startup: inspect the generated timing error and check + the query lookback and range step. +- Readiness times out: check that the first suite query is supported and + should return samples at its first instant time. +- Both results are empty: verify metric names, labels, base time, and sample + offsets. +- Results disagree immediately after ingestion: allow the ASAPQuery + precompute pipeline to process the remote-write batches, then rerun. +- `destination=prometheus` appears unexpectedly: inspect the ASAPQuery backend + configuration and `forward_unsupported_queries` setting. diff --git a/promql-compliance/QUICK_START.md b/promql-compliance/QUICK_START.md new file mode 100644 index 0000000..4564af1 --- /dev/null +++ b/promql-compliance/QUICK_START.md @@ -0,0 +1,42 @@ +# PromQL differential testing quick start + +This is the recommended workflow for comparing Prometheus and ASAPQuery on +the same deterministic data. + +## Requirements + +- Docker with Compose. +- Go 1.25 or newer. + +## Run the checked-in smoke test + +```bash +cd promql-compliance/runner +make run +``` + +The command: + +1. Loads `../datasets/single-rate.yaml` and `../suites/temporal.yaml`. +2. Generates planner and query-engine configuration in a temporary directory. +3. Starts Prometheus, the ASAPQuery planner, and the ASAPQuery query engine. +4. Sends identical remote-write data to Prometheus and ASAPQuery. +5. Waits for both targets to expose the data. +6. Runs the configured instant and range queries against both targets. +7. Compares Prometheus with ASAPQuery and checks range-at-t against instant-at-t. +8. Writes `differential-report.json` and tears down the stack. + +The command exits with status 1 if any comparison fails. + +## Run a different dataset or suite + +```bash +go run ./cmd/differential-runner \ + --dataset ../datasets/sparse-checkout.yaml \ + --suite ../suites/temporal.yaml \ + --compose-file ../docker-compose.yml +``` + +See [ARCHITECTURE.md](ARCHITECTURE.md) for the stack and +[HOW_TO.md](HOW_TO.md) for extending datasets, queries, timing, tolerances, +and logging. diff --git a/promql-compliance/config/prometheus.yml b/promql-compliance/config/prometheus.yml new file mode 100644 index 0000000..0cd3af5 --- /dev/null +++ b/promql-compliance/config/prometheus.yml @@ -0,0 +1,3 @@ +global: + scrape_interval: 1s + evaluation_interval: 1s diff --git a/promql-compliance/datasets/single-rate.yaml b/promql-compliance/datasets/single-rate.yaml new file mode 100644 index 0000000..8bc52bf --- /dev/null +++ b/promql-compliance/datasets/single-rate.yaml @@ -0,0 +1,27 @@ +name: single-rate +series: + - metric: http_requests_total + labels: + host: a + samples: + - {offset_seconds: 0, value: 0} + - {offset_seconds: 60, value: 60} + - {offset_seconds: 120, value: 120} + - {offset_seconds: 180, value: 180} + - {offset_seconds: 240, value: 240} + - {offset_seconds: 300, value: 300} + - {offset_seconds: 360, value: 360} + - {offset_seconds: 420, value: 420} + - {offset_seconds: 480, value: 480} + - {offset_seconds: 540, value: 540} + - {offset_seconds: 600, value: 600} + - {offset_seconds: 660, value: 660} + - {offset_seconds: 720, value: 720} + - {offset_seconds: 780, value: 780} + - {offset_seconds: 840, value: 840} + - {offset_seconds: 900, value: 900} + - {offset_seconds: 960, value: 960} + - {offset_seconds: 1020, value: 1020} + - {offset_seconds: 1080, value: 1080} + - {offset_seconds: 1140, value: 1140} + - {offset_seconds: 1200, value: 1200} diff --git a/promql-compliance/datasets/sparse-checkout.yaml b/promql-compliance/datasets/sparse-checkout.yaml new file mode 100644 index 0000000..d9b71da --- /dev/null +++ b/promql-compliance/datasets/sparse-checkout.yaml @@ -0,0 +1,74 @@ +name: sparse-checkout +series: + - metric: http_requests_total + labels: + host: a + samples: + - {offset_seconds: 0, value: 0} + - {offset_seconds: 60, value: 60} + - {offset_seconds: 120, value: 120} + - {offset_seconds: 180, value: 180} + - {offset_seconds: 240, value: 240} + - {offset_seconds: 300, value: 300} + - {offset_seconds: 360, value: 360} + - {offset_seconds: 420, value: 420} + - {offset_seconds: 480, value: 480} + - {offset_seconds: 540, value: 540} + - {offset_seconds: 600, value: 600} + - {offset_seconds: 660, value: 660} + - {offset_seconds: 720, value: 720} + - {offset_seconds: 780, value: 780} + - {offset_seconds: 840, value: 840} + - {offset_seconds: 900, value: 900} + - {offset_seconds: 960, value: 960} + - {offset_seconds: 1020, value: 1020} + - {offset_seconds: 1080, value: 1080} + - {offset_seconds: 1140, value: 1140} + - {offset_seconds: 1200, value: 1200} + - metric: http_requests_total + labels: + host: b + samples: + - {offset_seconds: 0, value: 1000} + - {offset_seconds: 60, value: 1120} + - {offset_seconds: 120, value: 1240} + - {offset_seconds: 180, value: 1360} + - {offset_seconds: 240, value: 1480} + - {offset_seconds: 300, value: 1600} + - {offset_seconds: 360, value: 1720} + - {offset_seconds: 420, value: 1840} + - {offset_seconds: 480, value: 1960} + - {offset_seconds: 540, value: 2080} + - {offset_seconds: 600, value: 2200} + - {offset_seconds: 660, value: 2320} + - {offset_seconds: 720, value: 2440} + - {offset_seconds: 780, value: 2560} + - {offset_seconds: 840, value: 2680} + - {offset_seconds: 900, value: 2800} + - {offset_seconds: 960, value: 2920} + - {offset_seconds: 1020, value: 3040} + - {offset_seconds: 1080, value: 3160} + - {offset_seconds: 1140, value: 3280} + - {offset_seconds: 1200, value: 3400} + - metric: checkout_up + labels: + service: checkout + region: us-east + samples: + - {offset_seconds: 0, value: 1} + - {offset_seconds: 60, value: 1} + - {offset_seconds: 120, value: 1} + - {offset_seconds: 180, value: 1} + - {offset_seconds: 240, value: 1} + - {offset_seconds: 300, value: 1} + - metric: checkout_up + labels: + service: checkout + region: us-west + samples: + - {offset_seconds: 900, value: 1} + - {offset_seconds: 960, value: 1} + - {offset_seconds: 1020, value: 1} + - {offset_seconds: 1080, value: 1} + - {offset_seconds: 1140, value: 1} + - {offset_seconds: 1200, value: 1} diff --git a/promql-compliance/docker-compose.yml b/promql-compliance/docker-compose.yml new file mode 100644 index 0000000..0a20620 --- /dev/null +++ b/promql-compliance/docker-compose.yml @@ -0,0 +1,65 @@ +name: asapquery-differential + +volumes: + differential-planner-output: + differential-prometheus-data: + +services: + prometheus: + image: prom/prometheus:v3.9.1 + ports: + - "${PROMETHEUS_PORT:-19090}:9090" + volumes: + - differential-prometheus-data:/prometheus + - ./config/prometheus.yml:/etc/prometheus/prometheus.yml:ro + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--web.enable-remote-write-receiver" + healthcheck: + test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:9090/-/ready || exit 1"] + interval: 2s + timeout: 2s + retries: 60 + + planner: + build: + context: .. + dockerfile: asap-planner-rs/Dockerfile + cache_from: + - type=registry,ref=ghcr.io/projectasap/asap-planner-rs:buildcache + volumes: + - "${ASAP_TEST_CONFIG_DIR:?ASAP_TEST_CONFIG_DIR is required}/controller-config.yaml:/config/controller-config.yaml:ro" + - differential-planner-output:/asap-planner-output + command: + - "--input_config=/config/controller-config.yaml" + - "--output_dir=/asap-planner-output" + - "--data-ingestion-interval-ms=1000" + - "--streaming_engine=precompute" + - "--range-duration-ms=1800000" + - "--step-ms=60000" + depends_on: + prometheus: + condition: service_healthy + + queryengine: + build: + context: .. + dockerfile: asap-query-engine/Dockerfile + cache_from: + - type=registry,ref=ghcr.io/projectasap/asap-query-engine:buildcache + ports: + - "${ASAP_QUERY_PORT:-18088}:8088" + - "${ASAP_INGEST_PORT:-19091}:9091" + environment: + RUST_LOG: INFO + RUST_BACKTRACE: "1" + volumes: + - differential-planner-output:/asap-planner-output:ro + - "${ASAP_TEST_CONFIG_DIR:?ASAP_TEST_CONFIG_DIR is required}/engine_config.yaml:/config/engine_config.yaml:ro" + command: ["--config-file", "/config/engine_config.yaml"] + depends_on: + planner: + condition: service_completed_successfully + prometheus: + condition: service_healthy diff --git a/promql-compliance/runner/Makefile b/promql-compliance/runner/Makefile new file mode 100644 index 0000000..ff88249 --- /dev/null +++ b/promql-compliance/runner/Makefile @@ -0,0 +1,10 @@ +.PHONY: test run + +test: + go test ./... + +run: + go run ./cmd/differential-runner \ + --dataset ../datasets/single-rate.yaml \ + --suite ../suites/temporal.yaml \ + --compose-file ../docker-compose.yml diff --git a/promql-compliance/runner/README.md b/promql-compliance/runner/README.md new file mode 100644 index 0000000..1c5194e --- /dev/null +++ b/promql-compliance/runner/README.md @@ -0,0 +1,36 @@ +# Differential runner + +The runner executes one dataset/query-suite pair against Prometheus and +ASAPQuery. It sends the same remote-write bytes to both targets, evaluates +each configured instant and range query, and also compares range-at-t against +instant-at-t within each target. + +Start with: + +- [`../QUICK_START.md`](../QUICK_START.md) for the shortest working example. +- [`../ARCHITECTURE.md`](../ARCHITECTURE.md) for the stack and data/query flow. +- [`../HOW_TO.md`](../HOW_TO.md) for adding datasets and queries, timing, + tolerance, DEBUG logging, and troubleshooting. + +Run the isolated Compose stack and execute the checked-in fixture/suite: + +```bash +make run +``` + +Run against already-running targets: + +```bash +go run ./cmd/differential-runner \ + --dataset ../datasets/sparse-checkout.yaml \ + --suite ../suites/temporal.yaml \ + --reference-url http://localhost:9090 \ + --test-url http://localhost:8088 \ + --reference-write-url http://localhost:9090 \ + --test-write-url http://localhost:9091 +``` + +Add repeated `--compose-file` flags to have the runner start and tear down an +isolated Docker Compose project. The runner generates the planner and engine +configuration from the fixture and suite. The report is written as JSON and +the process exits with status 1 when any comparison fails. diff --git a/promql-compliance/runner/cmd/differential-runner/main.go b/promql-compliance/runner/cmd/differential-runner/main.go new file mode 100644 index 0000000..85435d8 --- /dev/null +++ b/promql-compliance/runner/cmd/differential-runner/main.go @@ -0,0 +1,80 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "time" + + "github.com/ProjectASAP/ASAPQuery/promql-compliance/runner" +) + +type stringList []string + +func (values *stringList) String() string { return fmt.Sprint([]string(*values)) } + +func (values *stringList) Set(value string) error { + *values = append(*values, value) + return nil +} + +func main() { + var composeFiles stringList + datasetPath := flag.String("dataset", "", "dataset fixture YAML path") + suitePath := flag.String("suite", "", "query suite YAML path") + referenceURL := flag.String("reference-url", "http://localhost:19090", "Prometheus query URL") + testURL := flag.String("test-url", "http://localhost:18088", "ASAPQuery query URL") + referenceWriteURL := flag.String("reference-write-url", "http://localhost:19090", "Prometheus remote-write base URL") + testWriteURL := flag.String("test-write-url", "http://localhost:19091", "ASAPQuery remote-write base URL") + baseTimeMS := flag.Int64("base-time-ms", 0, "dataset base Unix time in milliseconds; default is now minus 30 minutes") + composeProject := flag.String("compose-project", "asapquery-differential", "Docker Compose project name") + outputPath := flag.String("output", "differential-report.json", "JSON report path") + keepServices := flag.Bool("keep-services", false, "leave Compose services running after the run") + flag.Var(&composeFiles, "compose-file", "Compose file to start; may be repeated") + flag.Parse() + + if *datasetPath == "" || *suitePath == "" { + flag.Usage() + os.Exit(2) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + report, err := runner.Run(ctx, runner.RunOptions{ + DatasetPath: *datasetPath, + SuitePath: *suitePath, + ReferenceURL: *referenceURL, + TestURL: *testURL, + ReferenceWriteURL: *referenceWriteURL, + TestWriteURL: *testWriteURL, + ComposeFiles: composeFiles, + ComposeProject: *composeProject, + BaseTimeMS: *baseTimeMS, + KeepServices: *keepServices, + }) + if err != nil { + log.Fatal(err) + } + + file, err := os.Create(*outputPath) + if err != nil { + log.Fatalf("create report %q: %v", *outputPath, err) + } + encoder := json.NewEncoder(file) + encoder.SetIndent("", " ") + if err := encoder.Encode(report); err != nil { + _ = file.Close() + log.Fatalf("write report %q: %v", *outputPath, err) + } + if err := file.Close(); err != nil { + log.Fatalf("close report %q: %v", *outputPath, err) + } + + fmt.Printf("dataset=%s suite=%s base-time=%s passed=%t report=%s\n", report.Dataset, report.Suite, report.BaseTime.Format(time.RFC3339), report.Passed, *outputPath) + if !report.Passed { + os.Exit(1) + } +} diff --git a/promql-compliance/runner/compare.go b/promql-compliance/runner/compare.go new file mode 100644 index 0000000..f5f5a6a --- /dev/null +++ b/promql-compliance/runner/compare.go @@ -0,0 +1,327 @@ +package runner + +import ( + "context" + "encoding/json" + "fmt" + "math" + "sort" + "strings" + "time" + + clientv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" +) + +// QueryAPI is the small seam implemented by both Prometheus and ASAPQuery. +// The comparison engine does not know whether a target is local, remote, or +// running in Docker. +type QueryAPI interface { + Query(context.Context, string, time.Time, ...clientv1.Option) (model.Value, clientv1.Warnings, error) + QueryRange(context.Context, string, clientv1.Range, ...clientv1.Option) (model.Value, clientv1.Warnings, error) +} + +type QueryReport struct { + Name string `json:"name"` + Expr string `json:"expr"` + Tolerance ComparisonPolicy `json:"tolerance"` + Range *ComparisonOutcome `json:"range,omitempty"` + Instant []InstantComparison `json:"instant,omitempty"` + ReferenceParity []ParityComparison `json:"referenceParity,omitempty"` + TestParity []ParityComparison `json:"testParity,omitempty"` + Passed bool `json:"passed"` +} + +type ComparisonOutcome struct { + Passed bool `json:"passed"` + Diff string `json:"diff,omitempty"` + ReferenceError string `json:"referenceError,omitempty"` + TestError string `json:"testError,omitempty"` +} + +type InstantComparison struct { + OffsetSeconds float64 `json:"offsetSeconds"` + Time time.Time `json:"time"` + Comparison ComparisonOutcome `json:"comparison"` +} + +type ParityComparison struct { + OffsetSeconds float64 `json:"offsetSeconds"` + Time time.Time `json:"time"` + Comparison ComparisonOutcome `json:"comparison"` +} + +// CompareQuery runs every configured query shape at every configured time. +// It compares the two targets and also checks range-at-t against +// instant-at-t within each target. +func CompareQuery(ctx context.Context, reference, test QueryAPI, query QueryCase, base time.Time, defaults ComparisonPolicy) (QueryReport, error) { + effective := query.EffectiveTolerance(defaults) + report := QueryReport{ + Name: query.Name, + Expr: query.Expr, + Tolerance: effective, + Passed: true, + } + + var referenceRange, testRange model.Value + var referenceRangeErr, testRangeErr error + if query.Range != nil { + rng, err := query.RangeAt(base) + if err != nil { + return QueryReport{}, err + } + referenceRange, _, referenceRangeErr = reference.QueryRange(ctx, query.Expr, rng) + testRange, _, testRangeErr = test.QueryRange(ctx, query.Expr, rng) + rangeOutcome := responseComparison(referenceRange, testRange, referenceRangeErr, testRangeErr, effective, query.ExpectError) + report.Range = &rangeOutcome + report.Passed = report.Passed && rangeOutcome.Passed + } + + instantTimes := query.InstantTimes(base) + for index, instantTime := range instantTimes { + referenceInstant, _, referenceErr := reference.Query(ctx, query.Expr, instantTime) + testInstant, _, testErr := test.Query(ctx, query.Expr, instantTime) + outcome := responseComparison(referenceInstant, testInstant, referenceErr, testErr, effective, query.ExpectError) + report.Instant = append(report.Instant, InstantComparison{ + OffsetSeconds: query.InstantOffsetsSeconds[index], + Time: instantTime, + Comparison: outcome, + }) + report.Passed = report.Passed && outcome.Passed + + if query.Range == nil || referenceRangeErr != nil || testRangeErr != nil || referenceErr != nil || testErr != nil || query.ExpectError { + continue + } + referenceParity := parityComparison(referenceRange, referenceInstant, instantTime, effective) + testParity := parityComparison(testRange, testInstant, instantTime, effective) + report.ReferenceParity = append(report.ReferenceParity, ParityComparison{ + OffsetSeconds: query.InstantOffsetsSeconds[index], + Time: instantTime, + Comparison: referenceParity, + }) + report.TestParity = append(report.TestParity, ParityComparison{ + OffsetSeconds: query.InstantOffsetsSeconds[index], + Time: instantTime, + Comparison: testParity, + }) + report.Passed = report.Passed && referenceParity.Passed && testParity.Passed + } + + return report, nil +} + +func (q QueryCase) RangeAt(base time.Time) (clientv1.Range, error) { + if q.Range == nil { + return clientv1.Range{}, fmt.Errorf("query %q has no range", q.Name) + } + if err := q.Range.validate(); err != nil { + return clientv1.Range{}, err + } + return clientv1.Range{ + Start: addSeconds(base, q.Range.StartOffsetSeconds), + End: addSeconds(base, q.Range.EndOffsetSeconds), + Step: time.Duration(q.Range.StepSeconds * float64(time.Second)), + }, nil +} + +func responseComparison(reference, test model.Value, referenceErr, testErr error, tolerance ComparisonPolicy, expectError bool) ComparisonOutcome { + outcome := ComparisonOutcome{} + if referenceErr != nil { + outcome.ReferenceError = referenceErr.Error() + } + if testErr != nil { + outcome.TestError = testErr.Error() + } + if expectError { + outcome.Passed = referenceErr != nil && testErr != nil + return outcome + } + if referenceErr != nil || testErr != nil { + outcome.Passed = false + return outcome + } + outcome.Diff = compareValues(reference, test, tolerance) + outcome.Passed = outcome.Diff == "" + return outcome +} + +func parityComparison(rangeValue, instantValue model.Value, timestamp time.Time, tolerance ComparisonPolicy) ComparisonOutcome { + rangeSnapshot, err := normalizeAt(rangeValue, timestamp) + if err != nil { + return ComparisonOutcome{Diff: err.Error()} + } + instantSnapshot, err := normalizeAt(instantValue, timestamp) + if err != nil { + return ComparisonOutcome{Diff: err.Error()} + } + diff := compareNormalized(rangeSnapshot, instantSnapshot, tolerance) + return ComparisonOutcome{Passed: diff == "", Diff: diff} +} + +func compareValues(reference, test model.Value, tolerance ComparisonPolicy) string { + referenceNormalized, err := normalize(reference) + if err != nil { + return fmt.Sprintf("reference result cannot be compared: %v", err) + } + testNormalized, err := normalize(test) + if err != nil { + return fmt.Sprintf("test result cannot be compared: %v", err) + } + return compareNormalized(referenceNormalized, testNormalized, tolerance) +} + +type normalizedValue struct { + Type string `json:"type"` + Samples []normalizedSample `json:"samples,omitempty"` + Scalar *float64 `json:"scalar,omitempty"` + String *string `json:"string,omitempty"` +} + +type normalizedSample struct { + Metric string `json:"metric"` + Timestamp int64 `json:"timestamp"` + Value float64 `json:"value"` +} + +func normalize(value model.Value) (normalizedValue, error) { + switch value := value.(type) { + case model.Vector: + return normalizedVector(value), nil + case model.Matrix: + return normalizedMatrix(value), nil + case *model.Scalar: + return normalizedValue{Type: "scalar", Scalar: floatPtr(float64(value.Value))}, nil + case *model.String: + return normalizedValue{Type: "string", String: stringPtr(value.Value)}, nil + default: + return normalizedValue{}, fmt.Errorf("unsupported Prometheus result type %T", value) + } +} + +func normalizeAt(value model.Value, timestamp time.Time) (normalizedValue, error) { + requested := timestamp.UnixMilli() + switch value := value.(type) { + case model.Matrix: + samples := make([]normalizedSample, 0) + for _, stream := range value { + for _, sample := range stream.Values { + if int64(sample.Timestamp) == requested { + samples = append(samples, normalizedSample{Metric: metricString(stream.Metric), Timestamp: requested, Value: float64(sample.Value)}) + } + } + } + result := normalizedValue{Type: "vector", Samples: samples} + sortNormalizedSamples(result.Samples) + return result, nil + case model.Vector: + result := normalizedVector(value) + for index := range result.Samples { + result.Samples[index].Timestamp = requested + } + return result, nil + default: + return normalize(value) + } +} + +func normalizedVector(value model.Vector) normalizedValue { + samples := make([]normalizedSample, 0, len(value)) + for _, sample := range value { + samples = append(samples, normalizedSample{Metric: metricString(sample.Metric), Timestamp: int64(sample.Timestamp), Value: float64(sample.Value)}) + } + sortNormalizedSamples(samples) + return normalizedValue{Type: "vector", Samples: samples} +} + +func normalizedMatrix(value model.Matrix) normalizedValue { + samples := make([]normalizedSample, 0) + for _, stream := range value { + for _, sample := range stream.Values { + samples = append(samples, normalizedSample{Metric: metricString(stream.Metric), Timestamp: int64(sample.Timestamp), Value: float64(sample.Value)}) + } + } + sortNormalizedSamples(samples) + return normalizedValue{Type: "matrix", Samples: samples} +} + +func compareNormalized(reference, test normalizedValue, tolerance ComparisonPolicy) string { + if reference.Type != test.Type { + return describeDiff(reference, test, "result type differs") + } + if reference.Scalar != nil || test.Scalar != nil { + if reference.Scalar == nil || test.Scalar == nil || !equalFloat(*reference.Scalar, *test.Scalar, tolerance.ValueTolerance) { + return describeDiff(reference, test, "scalar differs") + } + return "" + } + if reference.String != nil || test.String != nil { + if reference.String == nil || test.String == nil || *reference.String != *test.String { + return describeDiff(reference, test, "string differs") + } + return "" + } + if len(reference.Samples) != len(test.Samples) { + return describeDiff(reference, test, "sample count differs") + } + for index := range reference.Samples { + left, right := reference.Samples[index], test.Samples[index] + if left.Metric != right.Metric || left.Timestamp != right.Timestamp { + return describeDiff(reference, test, "metric or timestamp differs") + } + if !equalFloat(left.Value, right.Value, tolerance.ValueTolerance) { + return describeDiff(reference, test, "sample value differs") + } + } + return "" +} + +func equalFloat(left, right float64, tolerance *Tolerance) bool { + if math.IsNaN(left) || math.IsNaN(right) { + return math.IsNaN(left) && math.IsNaN(right) + } + if math.IsInf(left, 0) || math.IsInf(right, 0) { + return left == right + } + relative, absolute := 0.0, 0.0 + if tolerance != nil { + if tolerance.Relative != nil { + relative = *tolerance.Relative + } + if tolerance.Absolute != nil { + absolute = *tolerance.Absolute + } + } + limit := absolute + relative*math.Max(math.Abs(left), math.Abs(right)) + return math.Abs(left-right) <= limit +} + +func sortNormalizedSamples(samples []normalizedSample) { + sort.Slice(samples, func(left, right int) bool { + if samples[left].Metric != samples[right].Metric { + return samples[left].Metric < samples[right].Metric + } + return samples[left].Timestamp < samples[right].Timestamp + }) +} + +func metricString(metric model.Metric) string { + keys := make([]string, 0, len(metric)) + for key := range metric { + keys = append(keys, string(key)) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, key := range keys { + parts = append(parts, key+"="+fmt.Sprintf("%q", metric[model.LabelName(key)])) + } + return strings.Join(parts, ",") +} + +func describeDiff(reference, test normalizedValue, reason string) string { + left, _ := json.Marshal(reference) + right, _ := json.Marshal(test) + return fmt.Sprintf("%s\nreference: %s\ntest: %s", reason, left, right) +} + +func floatPtr(value float64) *float64 { return &value } +func stringPtr(value string) *string { return &value } diff --git a/promql-compliance/runner/compare_test.go b/promql-compliance/runner/compare_test.go new file mode 100644 index 0000000..7de9ffe --- /dev/null +++ b/promql-compliance/runner/compare_test.go @@ -0,0 +1,155 @@ +package runner + +import ( + "context" + "errors" + "testing" + "time" + + clientv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" +) + +func TestCompareQueryRejectsUnexpectedSharedErrors(t *testing.T) { + err := errors.New("query failed") + outcome := responseComparison(nil, nil, err, err, ComparisonPolicy{}, false) + + if outcome.Passed { + t.Fatal("comparison passed even though both targets failed unexpectedly") + } + if outcome.ReferenceError != err.Error() || outcome.TestError != err.Error() { + t.Fatalf("errors = %#v, want both target errors recorded", outcome) + } +} + +func TestCompareQueryAcceptsSharedExpectedErrors(t *testing.T) { + err := errors.New("query failed") + outcome := responseComparison(nil, nil, err, err, ComparisonPolicy{}, true) + + if !outcome.Passed { + t.Fatal("comparison rejected matching expected errors") + } +} + +type fakeTarget struct { + rangeValue model.Value + instantByMS map[int64]model.Value +} + +type errorTarget struct{ err error } + +func (target errorTarget) Query(context.Context, string, time.Time, ...clientv1.Option) (model.Value, clientv1.Warnings, error) { + return nil, nil, target.err +} + +func (target errorTarget) QueryRange(context.Context, string, clientv1.Range, ...clientv1.Option) (model.Value, clientv1.Warnings, error) { + return nil, nil, target.err +} + +func (f fakeTarget) Query(_ context.Context, _ string, ts time.Time, _ ...clientv1.Option) (model.Value, clientv1.Warnings, error) { + return f.instantByMS[ts.UnixMilli()], nil, nil +} + +func (f fakeTarget) QueryRange(_ context.Context, _ string, _ clientv1.Range, _ ...clientv1.Option) (model.Value, clientv1.Warnings, error) { + return f.rangeValue, nil, nil +} + +func TestCompareQueryDoesNotPassWhenBothTargetsFailUnexpectedly(t *testing.T) { + err := errors.New("query failed") + query := QueryCase{ + Name: "failing-query", + Expr: "rate(up[5m])", + InstantOffsetsSeconds: []float64{0}, + } + + report, compareErr := CompareQuery( + context.Background(), errorTarget{err: err}, errorTarget{err: err}, query, + time.Unix(1_700_000_000, 0).UTC(), ComparisonPolicy{}, + ) + if compareErr != nil { + t.Fatalf("CompareQuery: %v", compareErr) + } + if report.Passed { + t.Fatalf("query passed despite both targets failing: %#v", report.Instant[0].Comparison) + } +} + +func TestCompareQueryChecksEveryInstantTimeAndTargetParity(t *testing.T) { + base := time.UnixMilli(1_700_000_000_000).UTC() + first := model.Time(base.UnixMilli()) + second := model.Time(base.Add(time.Minute).UnixMilli()) + + rangeValue := model.Matrix{&model.SampleStream{ + Metric: model.Metric{"__name__": "up"}, + Values: []model.SamplePair{ + {Timestamp: first, Value: 1}, + {Timestamp: second, Value: 1}, + }, + }} + ref := fakeTarget{ + rangeValue: rangeValue, + instantByMS: map[int64]model.Value{ + base.UnixMilli(): model.Vector{&model.Sample{Metric: model.Metric{"__name__": "up"}, Value: 1, Timestamp: first}}, + base.Add(time.Minute).UnixMilli(): model.Vector{&model.Sample{Metric: model.Metric{"__name__": "up"}, Value: 1, Timestamp: second}}, + }, + } + testTarget := fakeTarget{ + rangeValue: rangeValue, + instantByMS: map[int64]model.Value{ + base.UnixMilli(): model.Vector{&model.Sample{Metric: model.Metric{"__name__": "up"}, Value: 1, Timestamp: first}}, + base.Add(time.Minute).UnixMilli(): model.Vector{&model.Sample{Metric: model.Metric{"__name__": "up"}, Value: 2, Timestamp: second}}, + }, + } + query := QueryCase{ + Name: "up-at-both-steps", + Expr: "up", + InstantOffsetsSeconds: []float64{0, 60}, + Range: &RangeSpec{StartOffsetSeconds: 0, EndOffsetSeconds: 60, StepSeconds: 60}, + } + + report, err := CompareQuery(context.Background(), ref, testTarget, query, base, ComparisonPolicy{}) + if err != nil { + t.Fatalf("CompareQuery: %v", err) + } + if len(report.Instant) != 2 { + t.Fatalf("instant comparisons = %d, want 2", len(report.Instant)) + } + if !report.Instant[0].Comparison.Passed { + t.Fatalf("first instant comparison failed: %#v", report.Instant[0]) + } + if report.Instant[1].Comparison.Passed { + t.Fatal("second instant comparison passed despite target mismatch") + } + if report.TestParity[1].Comparison.Passed { + t.Fatal("ASAPQuery range/instant parity passed despite second-step mismatch") + } + if !report.ReferenceParity[0].Comparison.Passed { + t.Fatalf("Prometheus parity failed unexpectedly: %#v", report.ReferenceParity[0]) + } +} + +func TestCompareValuesHonorsExplicitToleranceOnlyForValues(t *testing.T) { + left := model.Vector{&model.Sample{ + Metric: model.Metric{"__name__": "up"}, + Timestamp: model.Time(1000), + Value: 100, + }} + right := model.Vector{&model.Sample{ + Metric: model.Metric{"__name__": "up"}, + Timestamp: model.Time(1000), + Value: 101, + }} + tolerance := ComparisonPolicy{ValueTolerance: &Tolerance{Relative: floatPtr(0.02)}} + if diff := compareValues(left, right, tolerance); diff != "" { + t.Fatalf("comparison rejected explicit tolerance: %s", diff) + } + + differentLabels := model.Vector{&model.Sample{ + Metric: model.Metric{"__name__": "other"}, + Timestamp: model.Time(1000), + Value: 101, + }} + if diff := compareValues(left, differentLabels, tolerance); diff == "" { + t.Fatal("comparison accepted different labels because values were within tolerance") + } +} diff --git a/promql-compliance/runner/config.go b/promql-compliance/runner/config.go new file mode 100644 index 0000000..95a8d73 --- /dev/null +++ b/promql-compliance/runner/config.go @@ -0,0 +1,184 @@ +package runner + +import ( + "bytes" + "fmt" + "math" + "os" + "time" + + "gopkg.in/yaml.v3" +) + +// Suite is a data-independent collection of PromQL cases. All timestamps are +// offsets from the dataset base time selected for a run. +type Suite struct { + Name string `yaml:"name" json:"name"` + ComparisonDefaults ComparisonPolicy `yaml:"comparison_defaults" json:"comparisonDefaults"` + Queries []QueryCase `yaml:"queries" json:"queries"` +} + +type QueryCase struct { + Name string `yaml:"name" json:"name"` + Expr string `yaml:"expr" json:"expr"` + InstantOffsetsSeconds []float64 `yaml:"instant_offsets_seconds" json:"instantOffsetsSeconds"` + Range *RangeSpec `yaml:"range" json:"range"` + Comparison *ComparisonPolicy `yaml:"comparison" json:"comparison"` + ExpectError bool `yaml:"expect_error" json:"expectError"` +} + +type RangeSpec struct { + StartOffsetSeconds float64 `yaml:"start_offset_seconds" json:"startOffsetSeconds"` + EndOffsetSeconds float64 `yaml:"end_offset_seconds" json:"endOffsetSeconds"` + StepSeconds float64 `yaml:"step_seconds" json:"stepSeconds"` +} + +// ComparisonPolicy is intentionally pointer-valued: omitted means exact +// comparison, while zero is a valid explicit tolerance. +type ComparisonPolicy struct { + ValueTolerance *Tolerance `yaml:"value_tolerance" json:"valueTolerance"` +} + +type Tolerance struct { + Relative *float64 `yaml:"relative" json:"relative"` + Absolute *float64 `yaml:"absolute" json:"absolute"` +} + +// LoadSuite parses and validates a query suite. It rejects incomplete cases +// rather than silently selecting wall-clock defaults. +func LoadSuite(contents []byte) (Suite, error) { + var suite Suite + decoder := yaml.NewDecoder(bytes.NewReader(contents)) + decoder.KnownFields(true) + if err := decoder.Decode(&suite); err != nil { + return Suite{}, fmt.Errorf("parse query suite: %w", err) + } + if suite.Name == "" { + return Suite{}, fmt.Errorf("query suite has no name") + } + if len(suite.Queries) == 0 { + return Suite{}, fmt.Errorf("query suite %q has no queries", suite.Name) + } + for i := range suite.Queries { + query := &suite.Queries[i] + if query.Name == "" { + return Suite{}, fmt.Errorf("query %d has no name", i) + } + if query.Expr == "" { + return Suite{}, fmt.Errorf("query %q has no expr", query.Name) + } + if len(query.InstantOffsetsSeconds) == 0 && query.Range == nil { + return Suite{}, fmt.Errorf("query %q has neither instant times nor a range", query.Name) + } + if query.Range != nil { + if err := query.Range.validate(); err != nil { + return Suite{}, fmt.Errorf("query %q: %w", query.Name, err) + } + for _, offset := range query.InstantOffsetsSeconds { + if !finite(offset) || !query.Range.contains(offset) { + return Suite{}, fmt.Errorf("query %q instant offset %v is outside its range", query.Name, offset) + } + if !query.Range.containsGridOffset(offset) { + return Suite{}, fmt.Errorf( + "query %q instant offset %v is not aligned to the range start and step", + query.Name, + offset, + ) + } + } + } else { + for _, offset := range query.InstantOffsetsSeconds { + if !finite(offset) { + return Suite{}, fmt.Errorf("query %q has a non-finite instant offset", query.Name) + } + } + } + if err := validateTolerance(query.EffectiveTolerance(suite.ComparisonDefaults).ValueTolerance); err != nil { + return Suite{}, fmt.Errorf("query %q: %w", query.Name, err) + } + } + return suite, nil +} + +// LoadSuiteFile reads a query suite from disk. +func LoadSuiteFile(path string) (Suite, error) { + contents, err := os.ReadFile(path) + if err != nil { + return Suite{}, fmt.Errorf("read query suite %q: %w", path, err) + } + return LoadSuite(contents) +} + +func (q QueryCase) InstantTimes(base time.Time) []time.Time { + result := make([]time.Time, 0, len(q.InstantOffsetsSeconds)) + for _, offset := range q.InstantOffsetsSeconds { + result = append(result, addSeconds(base, offset)) + } + return result +} + +func (q QueryCase) EffectiveTolerance(defaults ComparisonPolicy) ComparisonPolicy { + if q.Comparison == nil || q.Comparison.ValueTolerance == nil { + return defaults + } + result := defaults + if result.ValueTolerance == nil { + result.ValueTolerance = &Tolerance{} + } + merged := *result.ValueTolerance + if q.Comparison.ValueTolerance.Relative != nil { + merged.Relative = q.Comparison.ValueTolerance.Relative + } + if q.Comparison.ValueTolerance.Absolute != nil { + merged.Absolute = q.Comparison.ValueTolerance.Absolute + } + result.ValueTolerance = &merged + return result +} + +func (r RangeSpec) validate() error { + if !finite(r.StartOffsetSeconds) || !finite(r.EndOffsetSeconds) || !finite(r.StepSeconds) { + return fmt.Errorf("range offsets and step must be finite") + } + if r.EndOffsetSeconds <= r.StartOffsetSeconds { + return fmt.Errorf("range end must be after range start") + } + if r.EndOffsetSeconds-r.StartOffsetSeconds < float64(time.Millisecond)/float64(time.Second) { + return fmt.Errorf("range duration must be at least 1ms") + } + if r.StepSeconds <= 0 { + return fmt.Errorf("range step must be positive") + } + return nil +} + +func (r RangeSpec) contains(offset float64) bool { + return offset >= r.StartOffsetSeconds && offset <= r.EndOffsetSeconds +} + +func (r RangeSpec) containsGridOffset(offset float64) bool { + steps := (offset - r.StartOffsetSeconds) / r.StepSeconds + nearestStep := math.Round(steps) + return math.Abs(steps-nearestStep) <= 1e-9 +} + +func finite(value float64) bool { + return !math.IsNaN(value) && !math.IsInf(value, 0) +} + +func validateTolerance(tolerance *Tolerance) error { + if tolerance == nil { + return nil + } + if tolerance.Relative != nil && (*tolerance.Relative < 0 || math.IsNaN(*tolerance.Relative) || math.IsInf(*tolerance.Relative, 0)) { + return fmt.Errorf("relative tolerance must be a finite non-negative number") + } + if tolerance.Absolute != nil && (*tolerance.Absolute < 0 || math.IsNaN(*tolerance.Absolute) || math.IsInf(*tolerance.Absolute, 0)) { + return fmt.Errorf("absolute tolerance must be a finite non-negative number") + } + return nil +} + +func addSeconds(base time.Time, seconds float64) time.Time { + return base.Add(time.Duration(seconds * float64(time.Second))) +} diff --git a/promql-compliance/runner/config_test.go b/promql-compliance/runner/config_test.go new file mode 100644 index 0000000..ee65e2d --- /dev/null +++ b/promql-compliance/runner/config_test.go @@ -0,0 +1,120 @@ +package runner + +import ( + "testing" + "time" +) + +func TestLoadSuiteResolvesExplicitPerQueryTimesAndTolerance(t *testing.T) { + suite, err := LoadSuite([]byte(`name: temporal +comparison_defaults: + value_tolerance: + relative: 0.01 +queries: + - name: gap + expr: checkout_up + instant_offsets_seconds: [300, 660, 900] + range: + start_offset_seconds: 0 + end_offset_seconds: 1200 + step_seconds: 60 + - name: exact + expr: sum(up) + instant_offsets_seconds: [600] + comparison: + value_tolerance: + absolute: 0.000001 +`)) + if err != nil { + t.Fatalf("LoadSuite: %v", err) + } + + base := time.UnixMilli(1_700_000_000_000).UTC() + first := suite.Queries[0] + times := first.InstantTimes(base) + if got, want := len(times), 3; got != want { + t.Fatalf("instant time count = %d, want %d", got, want) + } + if got, want := times[1], base.Add(660*time.Second); !got.Equal(want) { + t.Fatalf("gap time = %s, want %s", got, want) + } + + rng, err := first.RangeAt(base) + if err != nil { + t.Fatalf("Range: %v", err) + } + if got, want := rng.Start, base; !got.Equal(want) { + t.Fatalf("range start = %s, want %s", got, want) + } + if got, want := rng.End, base.Add(1200*time.Second); !got.Equal(want) { + t.Fatalf("range end = %s, want %s", got, want) + } + + if got := first.EffectiveTolerance(suite.ComparisonDefaults); got.ValueTolerance == nil || got.ValueTolerance.Relative == nil || *got.ValueTolerance.Relative != 0.01 { + t.Fatalf("global tolerance was not inherited: %#v", got) + } + if got := suite.Queries[1].EffectiveTolerance(suite.ComparisonDefaults); got.ValueTolerance == nil || got.ValueTolerance.Absolute == nil || *got.ValueTolerance.Absolute != 0.000001 { + t.Fatalf("per-query tolerance was not applied: %#v", got) + } + if got := suite.Queries[1].EffectiveTolerance(suite.ComparisonDefaults); got.ValueTolerance == nil || got.ValueTolerance.Relative == nil || *got.ValueTolerance.Relative != 0.01 { + t.Fatalf("per-query tolerance did not inherit global relative value: %#v", got) + } +} + +func TestLoadSuiteRejectsImplicitEvaluationWindow(t *testing.T) { + _, err := LoadSuite([]byte(`name: incomplete +queries: + - name: missing-times + expr: up +`)) + if err == nil { + t.Fatal("LoadSuite accepted a query with no instant times or range") + } +} + +func TestLoadSuiteRejectsInstantOutsideRange(t *testing.T) { + _, err := LoadSuite([]byte(`name: outside +queries: + - name: invalid-time + expr: up + instant_offsets_seconds: [120] + range: + start_offset_seconds: 0 + end_offset_seconds: 60 + step_seconds: 60 +`)) + if err == nil { + t.Fatal("LoadSuite accepted an instant time outside its range") + } +} + +func TestLoadSuiteRejectsInstantOffsetOffRangeGrid(t *testing.T) { + _, err := LoadSuite([]byte(`name: off-grid +queries: + - name: invalid-time + expr: up + instant_offsets_seconds: [330] + range: + start_offset_seconds: 300 + end_offset_seconds: 600 + step_seconds: 60 +`)) + if err == nil { + t.Fatal("LoadSuite accepted an instant offset that is not on the range grid") + } +} + +func TestLoadSuiteRejectsSubMillisecondRange(t *testing.T) { + _, err := LoadSuite([]byte(`name: sub-millisecond +queries: + - name: too-small + expr: up + range: + start_offset_seconds: 0 + end_offset_seconds: 0.0005 + step_seconds: 0.0005 +`)) + if err == nil { + t.Fatal("LoadSuite accepted a sub-millisecond range") + } +} diff --git a/promql-compliance/runner/go.mod b/promql-compliance/runner/go.mod new file mode 100644 index 0000000..ea2e429 --- /dev/null +++ b/promql-compliance/runner/go.mod @@ -0,0 +1,32 @@ +module github.com/ProjectASAP/ASAPQuery/promql-compliance/runner + +go 1.25.8 + +require ( + github.com/ProjectASAP/ASAPQuery/promql-compliance/seeder v0.0.0 + github.com/prometheus/client_golang v1.24.1 + github.com/prometheus/common v0.70.1 + github.com/prometheus/prometheus v0.314.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dennwc/varint v1.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect +) + +replace github.com/ProjectASAP/ASAPQuery/promql-compliance/seeder => ../seeder diff --git a/promql-compliance/runner/go.sum b/promql-compliance/runner/go.sum new file mode 100644 index 0000000..35cdfc0 --- /dev/null +++ b/promql-compliance/runner/go.sum @@ -0,0 +1,210 @@ +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= +github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= +github.com/aws/aws-sdk-go-v2 v1.43.0 h1:fharf/WhbRAVZ1du0QL7roNFxZ6T/sWr+4Ni617bwSI= +github.com/aws/aws-sdk-go-v2 v1.43.0/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= +github.com/aws/aws-sdk-go-v2/config v1.32.31 h1:n4nY9O3QKoHIkL85EX+V8RcMFtOhlpTFhGArg915PXk= +github.com/aws/aws-sdk-go-v2/config v1.32.31/go.mod h1:PN0NYDCCoOpGGsZ2+elDUidmHfQBPyYzN2GCgl8HEBs= +github.com/aws/aws-sdk-go-v2/credentials v1.19.30 h1:TTCvvzFU6gXa4iJecNG/0F/B0oYTiazoRECr2XyLHrY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.30/go.mod h1:jKxAp2AEncnliinzpgOSZDFv6+VjvWhjw/AtbfsWT9U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 h1:kfVL5wAunCJycL6MOQ6aNh6PlAYEymflcjuKmrWUA0o= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31/go.mod h1:nWfRNDAppujCQgOUd43lKT4yeLv9z3nJ3bw1G3BgQKo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 h1:Z8F3hfCY33IGpJjFAnv0wvtv1FIKj1GHmRDEYqy64tw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31/go.mod h1:aVyUoytEyOViR6jhq6jula0xkc5NfBE2hgeF6BvOrao= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 h1:hyOxUyXdh3AyjE93gBgsfziJag9ACwcs+ZpDBLzi8mw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31/go.mod h1:OERqI9k0draSLB8O8woxY3q25ZWTELRK4RRoLMuMZFo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 h1:0MrUL35H/Y4kdFfItoR5jCgtDQ4Z/8LudAoIHRfA4hE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32/go.mod h1:2tNZkuWz54arj8mHVf+8Y7cKkcD8Wr/fBpENgEXpjLc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 h1:w2SIhW92DZPFrSL4ksVCr8IYff5OZwIcxg8+95tzvAI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31/go.mod h1:wAhpCQbkov+IcvjozJbd2xRCoZybUEHNkcFunssNACg= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 h1:OHH5iTQvVGmfHjX/5Q+vFuA/Rf2x6/95aJ/75QCQSm4= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.0/go.mod h1:mCF3AK9PpL49oOrhniUXWAfhVBVQ/XbytoE5eccZUIs= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 h1:CaJyYhxBE0M/HJX/YvSaSmQlsI91VHB0lKU8LtLxL3A= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.0/go.mod h1:+e6BMRMPjBQoCw/WovYR9GLy2IU0z4Q77smOB1DraSg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 h1:tC323YV77QdafeBr6LUhLDTsboyuyHLNRwAyCP44kGU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0/go.mod h1:SfLK1sgviHmbI+MozR9iDwDjL4cdCVZtahsjoR+z7wg= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 h1:Pd6PNlp4t8PTXxqzstICl52Wsy78vpjFZ7PRUj44mJc= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.0/go.mod h1:rmQ0TnHzuLPmabgjPcsywhsSOmaBDgzR4zvDxSPsGdg= +github.com/aws/smithy-go v1.27.4 h1:JQcphmBN4f0q/sPqXqROIItRNV/hy10cgu7CsFy616M= +github.com/aws/smithy-go v1.27.4/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 h1:6df1vn4bBlDDo4tARvBm7l6KA9iVMnE3NWizDeWSrps= +github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3/go.mod h1:CIWtjkly68+yqLPbvwwR/fjNJA/idrtULjZWh2v1ys0= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dennwc/varint v1.0.0 h1:kGNFFSSw8ToIy3obO/kKr8U9GZYUAxQEVuix4zfDWzE= +github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.18 h1:hvVi34VucdrV1IIsiWuqYM8kutw/92MxNEFxCJZEh0k= +github.com/googleapis/enterprise-certificate-proxy v0.3.18/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= +github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 h1:cLN4IBkmkYZNnk7EAJ0BHIethd+J6LqxFNw5mSiI2bM= +github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid/v2 v2.1.2 h1:IEclFb9JNvzYA6MW2SCxbLzcHTVsfqm3PrqGQJH5zec= +github.com/oklog/ulid/v2 v2.1.2/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_golang/exp v0.0.0-20260724065723-ecdb8254ba61 h1:SgKx/5u9SwqzZ27E1T4bfuisjTOkI3GagC6WtdEE5lg= +github.com/prometheus/client_golang/exp v0.0.0-20260724065723-ecdb8254ba61/go.mod h1:CoLfLGxCH1vzpdmZ+p2uaUGH43j+99HYmnK1Wak6rS4= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEoIwkU+A6qos= +github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/prometheus/prometheus v0.314.0 h1:YjsimqsIi6/mOtzZcrPEYUALO6zpfaht9O5sXqDz2vg= +github.com/prometheus/prometheus v0.314.0/go.mod h1:zjg3pMTAkY0/JG8jy/h8/YgSQUVB+aCXMhUqN6l64jg= +github.com/prometheus/sigv4 v0.4.1 h1:EIc3j+8NBea9u1iV6O5ZAN8uvPq2xOIUPcqCTivHuXs= +github.com/prometheus/sigv4 v0.4.1/go.mod h1:eu+ZbRvsc5TPiHwqh77OWuCnWK73IdkETYY46P4dXOU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597 h1:qLvzZeaANDgyVOA8pyHCOStGlXn0rseXma+GQjeuv2g= +golang.org/x/exp v0.0.0-20260709172345-9ea1abe57597/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.290.0 h1:eMw0Xo+IfbbMlKmW7aHvpyQRv9RCXuWx/vs8AD+0x9A= +google.golang.org/api v0.290.0/go.mod h1:weJZ3lldHFYI0DBFNKpJelUDNnusTt5YaOEgxvt8ci8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260720155508-bb71a54f79dc h1:3TtNq/QbJNrSY1nVdjcikfBw6ujnaNbdrd88wNr1OW4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260720155508-bb71a54f79dc/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.35.3 h1:MeaUwQCV3tjKP4bcwWGgZ/cp/vpsRnQzqO6J6tJyoF8= +k8s.io/apimachinery v0.35.3/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/client-go v0.35.3 h1:s1lZbpN4uI6IxeTM2cpdtrwHcSOBML1ODNTCCfsP1pg= +k8s.io/client-go v0.35.3/go.mod h1:RzoXkc0mzpWIDvBrRnD+VlfXP+lRzqQjCmKtiwZ8Q9c= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= diff --git a/promql-compliance/runner/report.go b/promql-compliance/runner/report.go new file mode 100644 index 0000000..d7457d0 --- /dev/null +++ b/promql-compliance/runner/report.go @@ -0,0 +1,35 @@ +package runner + +import ( + "context" + "time" +) + +type Report struct { + Suite string `json:"suite"` + Dataset string `json:"dataset"` + BaseTime time.Time `json:"baseTime"` + Queries []QueryReport `json:"queries"` + Passed bool `json:"passed"` +} + +// CompareSuite evaluates a suite against both targets. It returns a report +// even when individual queries differ; infrastructure failures are returned as +// errors so callers can distinguish a failed test from a broken environment. +func CompareSuite(ctx context.Context, reference, test QueryAPI, suite Suite, datasetName string, base time.Time) (Report, error) { + report := Report{ + Suite: suite.Name, + Dataset: datasetName, + BaseTime: base, + Passed: true, + } + for _, query := range suite.Queries { + result, err := CompareQuery(ctx, reference, test, query, base, suite.ComparisonDefaults) + if err != nil { + return report, err + } + report.Queries = append(report.Queries, result) + report.Passed = report.Passed && result.Passed + } + return report, nil +} diff --git a/promql-compliance/runner/run.go b/promql-compliance/runner/run.go new file mode 100644 index 0000000..88ce069 --- /dev/null +++ b/promql-compliance/runner/run.go @@ -0,0 +1,447 @@ +package runner + +import ( + "context" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/ProjectASAP/ASAPQuery/promql-compliance/seeder" + "github.com/prometheus/common/model" + promqlparser "github.com/prometheus/prometheus/promql/parser" + "gopkg.in/yaml.v3" +) + +const ( + defaultDatasetAge = 30 * time.Minute + defaultPlannerWindow = 5 * time.Minute + defaultPlannerWindowMS = int(defaultPlannerWindow / time.Millisecond) + defaultDataIngestionIntervalMS = 1000 + defaultPlannerStepMS = 60_000 + serviceReadyTimeout = 3 * time.Minute + dataReadyTimeout = 3 * time.Minute + readinessPollInterval = time.Second + composeShutdownTimeout = time.Minute +) + +type RunOptions struct { + DatasetPath string + SuitePath string + ReferenceURL string + TestURL string + ReferenceWriteURL string + TestWriteURL string + ComposeFiles []string + ComposeProject string + BaseTimeMS int64 + KeepServices bool +} + +// Run executes one isolated dataset/suite run. If ComposeFiles is empty, the +// caller owns the target processes; otherwise this function starts and stops +// an isolated Docker Compose project around the run. +func Run(ctx context.Context, options RunOptions) (Report, error) { + fixture, err := seeder.LoadFixture(options.DatasetPath) + if err != nil { + return Report{}, err + } + suite, err := LoadSuiteFile(options.SuitePath) + if err != nil { + return Report{}, err + } + if options.ReferenceURL == "" || options.TestURL == "" { + return Report{}, fmt.Errorf("reference and test query URLs are required") + } + if options.ReferenceWriteURL == "" { + options.ReferenceWriteURL = options.ReferenceURL + } + if options.TestWriteURL == "" { + return Report{}, fmt.Errorf("test write URL is required") + } + + base := runBaseTime(options.BaseTimeMS) + runDirectory, err := os.MkdirTemp("", "asapquery-differential-") + if err != nil { + return Report{}, fmt.Errorf("create run directory: %w", err) + } + defer os.RemoveAll(runDirectory) + if err := writeGeneratedConfigs(runDirectory, fixture, suite); err != nil { + return Report{}, err + } + + lifecycle := composeLifecycle{ + files: options.ComposeFiles, + project: options.ComposeProject, + env: []string{"ASAP_TEST_CONFIG_DIR=" + runDirectory}, + } + if len(options.ComposeFiles) > 0 { + if err := lifecycle.Start(ctx); err != nil { + return Report{}, err + } + if !options.KeepServices { + defer lifecycle.Stop() + } + } + + reference, err := NewHTTPQueryTarget(options.ReferenceURL) + if err != nil { + return Report{}, err + } + test, err := NewHTTPQueryTarget(options.TestURL) + if err != nil { + return Report{}, err + } + probeQuery := suite.Queries[0].Expr + if err := waitForHTTPReady(ctx, options.ReferenceURL); err != nil { + return Report{}, fmt.Errorf("reference target is not ready: %w", err) + } + if err := waitForHTTPReady(ctx, options.TestURL); err != nil { + return Report{}, fmt.Errorf("test target is not ready: %w", err) + } + + requests := seeder.BuildWriteRequestsFromFixtureBatches(base.UnixMilli(), fixture) + for index, request := range requests { + body, err := seeder.EncodeSnappy(request) + if err != nil { + return Report{}, fmt.Errorf("encode dataset %q batch %d: %w", fixture.Name, index, err) + } + if err := seeder.PushEncoded(ctx, options.ReferenceWriteURL, body); err != nil { + return Report{}, fmt.Errorf("seed reference target batch %d: %w", index, err) + } + if err := seeder.PushEncoded(ctx, options.TestWriteURL, body); err != nil { + return Report{}, fmt.Errorf("seed test target batch %d: %w", index, err) + } + } + probeTime, err := dataProbeTime(suite.Queries[0], base) + if err != nil { + return Report{}, err + } + if err := waitForData(ctx, reference, probeQuery, probeTime); err != nil { + return Report{}, fmt.Errorf("reference target did not expose seeded data: %w", err) + } + if err := waitForData(ctx, test, probeQuery, probeTime); err != nil { + return Report{}, fmt.Errorf("test target did not expose seeded data: %w", err) + } + + return CompareSuite(ctx, reference, test, suite, fixture.Name, base) +} + +func runBaseTime(baseTimeMS int64) time.Time { + if baseTimeMS != 0 { + return time.UnixMilli(baseTimeMS).UTC() + } + return time.Now().UTC().Truncate(defaultPlannerWindow).Add(-defaultDatasetAge) +} + +func dataProbeTime(query QueryCase, base time.Time) (time.Time, error) { + if offsets := query.InstantTimes(base); len(offsets) > 0 { + return offsets[0], nil + } + interval, err := query.RangeAt(base) + if err != nil { + return time.Time{}, fmt.Errorf("select data probe time: %w", err) + } + return interval.End, nil +} + +func waitForHTTPReady(ctx context.Context, baseURL string) error { + deadline, cancel := context.WithTimeout(ctx, serviceReadyTimeout) + defer cancel() + healthURL := strings.TrimRight(baseURL, "/") + "/api/v1/status/runtimeinfo" + client := &http.Client{} + var lastErr error + for { + request, err := http.NewRequestWithContext(deadline, http.MethodGet, healthURL, nil) + if err == nil { + response, requestErr := client.Do(request) + if requestErr == nil { + _ = response.Body.Close() + if response.StatusCode >= http.StatusOK && response.StatusCode < http.StatusMultipleChoices { + return nil + } + lastErr = fmt.Errorf("HTTP %s", response.Status) + } else { + lastErr = requestErr + } + } else { + lastErr = err + } + select { + case <-deadline.Done(): + if lastErr != nil { + return fmt.Errorf("last query error: %w", lastErr) + } + return deadline.Err() + case <-time.After(readinessPollInterval): + } + } +} + +func waitForData(ctx context.Context, target QueryAPI, query string, timestamp time.Time) error { + deadline, cancel := context.WithTimeout(ctx, dataReadyTimeout) + defer cancel() + var lastErr error + for { + value, _, err := target.Query(deadline, query, timestamp) + if err != nil { + lastErr = err + } else if hasSamples(value) { + return nil + } + select { + case <-deadline.Done(): + if lastErr != nil { + return fmt.Errorf("last query error: %w", lastErr) + } + return fmt.Errorf("query %q is still empty", query) + case <-time.After(readinessPollInterval): + } + } +} + +func hasSamples(value model.Value) bool { + switch value := value.(type) { + case model.Vector: + return len(value) > 0 + case model.Matrix: + for _, stream := range value { + if len(stream.Values) > 0 { + return true + } + } + return false + case *model.Scalar, *model.String: + return true + default: + return false + } +} + +type composeLifecycle struct { + files []string + project string + env []string + started bool +} + +func (l *composeLifecycle) Start(ctx context.Context) error { + if l.project == "" { + l.project = "asapquery-differential" + } + args := l.composeArgs() + args = append(args, "up", "-d", "--build") + command := exec.CommandContext(ctx, "docker", args...) + command.Env = append(os.Environ(), l.env...) + output, err := command.CombinedOutput() + if err != nil { + return fmt.Errorf("start compose project %q: %w\n%s", l.project, err, output) + } + l.started = true + return nil +} + +func (l *composeLifecycle) Stop() { + if !l.started { + return + } + ctx, cancel := context.WithTimeout(context.Background(), composeShutdownTimeout) + defer cancel() + args := l.composeArgs() + args = append(args, "down", "--volumes", "--remove-orphans") + command := exec.CommandContext(ctx, "docker", args...) + command.Env = append(os.Environ(), l.env...) + _ = command.Run() +} + +func (l *composeLifecycle) composeArgs() []string { + args := []string{"compose"} + if l.project != "" { + args = append(args, "--project-name", l.project) + } + for _, file := range l.files { + args = append(args, "--file", file) + } + return args +} + +type plannerConfig struct { + QueryGroups []plannerQueryGroup `yaml:"query_groups"` + Metrics []plannerMetric `yaml:"metrics"` + Cleanup plannerCleanup `yaml:"aggregate_cleanup"` +} + +type plannerQueryGroup struct { + ID int `yaml:"id"` + Queries []string `yaml:"queries"` + RepetitionDelayMS int `yaml:"repetition_delay_ms"` + ControllerOptions plannerController `yaml:"controller_options"` + StepMS *int `yaml:"step_ms,omitempty"` + RangeDurationMS *int `yaml:"range_duration_ms,omitempty"` +} + +type plannerController struct { + AccuracySLA float64 `yaml:"accuracy_sla"` + LatencySLA float64 `yaml:"latency_sla"` +} + +type plannerMetric struct { + Metric string `yaml:"metric"` + Labels []string `yaml:"labels"` +} + +type plannerCleanup struct { + Policy string `yaml:"policy"` +} + +func writeGeneratedConfigs(directory string, fixture seeder.Fixture, suite Suite) error { + metrics := make(map[string]map[string]struct{}) + for _, series := range fixture.Series { + if metrics[series.Metric] == nil { + metrics[series.Metric] = make(map[string]struct{}) + } + for label := range series.Labels { + metrics[series.Metric][label] = struct{}{} + } + } + metricNames := make([]string, 0, len(metrics)) + for metric := range metrics { + metricNames = append(metricNames, metric) + } + sort.Strings(metricNames) + plannerMetrics := make([]plannerMetric, 0, len(metricNames)) + for _, metric := range metricNames { + labels := make([]string, 0, len(metrics[metric])) + for label := range metrics[metric] { + labels = append(labels, label) + } + sort.Strings(labels) + plannerMetrics = append(plannerMetrics, plannerMetric{Metric: metric, Labels: labels}) + } + queryGroups := make([]plannerQueryGroup, 0, len(suite.Queries)) + for index, query := range suite.Queries { + repetitionDelayMS, err := plannerRepetitionDelayMS(query) + if err != nil { + return fmt.Errorf("query %q: %w", query.Name, err) + } + group := plannerQueryGroup{ + ID: index + 1, Queries: []string{query.Expr}, RepetitionDelayMS: repetitionDelayMS, + ControllerOptions: plannerController{AccuracySLA: 0.99, LatencySLA: 1}, + } + if query.Range != nil { + stepMS := int(query.Range.StepSeconds * float64(time.Second/time.Millisecond)) + group.StepMS = &stepMS + rangeDurationMS := int((query.Range.EndOffsetSeconds - query.Range.StartOffsetSeconds) * float64(time.Second/time.Millisecond)) + group.RangeDurationMS = &rangeDurationMS + } + queryGroups = append(queryGroups, group) + } + config := plannerConfig{ + QueryGroups: queryGroups, + Metrics: plannerMetrics, + Cleanup: plannerCleanup{Policy: "read_based"}, + } + contents, err := yaml.Marshal(config) + if err != nil { + return fmt.Errorf("marshal generated planner config: %w", err) + } + if err := os.WriteFile(filepath.Join(directory, "controller-config.yaml"), contents, 0o600); err != nil { + return fmt.Errorf("write generated planner config: %w", err) + } + engineConfig := []byte(`output_dir: "/app/outputs" +log_level: "INFO" +data_ingestion_interval_ms: 1000 +streaming_engine: "precompute" +http_server: + port: 8088 +backend: + type: "prometheus" + server: "http://prometheus:9090" + forward_unsupported_queries: false +store: + lock_strategy: "per-key" +ingest: + type: "http_remote_write" + port: 9091 +inference_config: "/asap-planner-output/inference_config.yaml" +streaming_config: "/asap-planner-output/streaming_config.yaml" +`) + if err := os.WriteFile(filepath.Join(directory, "engine_config.yaml"), engineConfig, 0o600); err != nil { + return fmt.Errorf("write generated engine config: %w", err) + } + return nil +} + +func plannerRepetitionDelayMS(query QueryCase) (int, error) { + lookbackMS := defaultDataIngestionIntervalMS + if expr, err := promqlparser.NewParser(promqlparser.Options{}).ParseExpr(query.Expr); err == nil { + maxRange := time.Duration(0) + promqlparser.Inspect(expr, func(node promqlparser.Node, _ []promqlparser.Node) error { + switch node := node.(type) { + case *promqlparser.MatrixSelector: + maxRange = maxDuration(maxRange, node.Range) + case *promqlparser.SubqueryExpr: + maxRange = maxDuration(maxRange, node.Range) + } + return nil + }) + if maxRange > 0 { + lookbackMS = int(maxRange / time.Millisecond) + } + } + + delayMS := minInt(defaultPlannerWindowMS, lookbackMS) + if delayMS < defaultDataIngestionIntervalMS { + delayMS = defaultDataIngestionIntervalMS + } + + stepMS := defaultPlannerStepMS + if query.Range != nil { + stepMS = int(query.Range.StepSeconds * float64(time.Second/time.Millisecond)) + } + if stepMS > 0 && delayMS < stepMS && stepMS%delayMS != 0 { + delayMS = largestDivisorAtMost(stepMS, delayMS, defaultDataIngestionIntervalMS) + if delayMS == 0 { + return 0, fmt.Errorf( + "no repetition delay at least %dms divides evaluation step %dms within lookback %dms", + defaultDataIngestionIntervalMS, stepMS, lookbackMS, + ) + } + } + return delayMS, nil +} + +func maxDuration(left, right time.Duration) time.Duration { + if right > left { + return right + } + return left +} + +func minInt(left, right int) int { + if right < left { + return right + } + return left +} + +func largestDivisorAtMost(number, upper, lower int) int { + best := 0 + for divisor := 1; divisor <= number/divisor; divisor++ { + if number%divisor != 0 { + continue + } + if divisor >= lower && divisor <= upper && divisor > best { + best = divisor + } + paired := number / divisor + if paired >= lower && paired <= upper && paired > best { + best = paired + } + } + return best +} diff --git a/promql-compliance/runner/run_test.go b/promql-compliance/runner/run_test.go new file mode 100644 index 0000000..aa0136f --- /dev/null +++ b/promql-compliance/runner/run_test.go @@ -0,0 +1,133 @@ +package runner + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/ProjectASAP/ASAPQuery/promql-compliance/seeder" + "gopkg.in/yaml.v3" +) + +func TestDataProbeTimeUsesConfiguredEvaluationTime(t *testing.T) { + base := time.UnixMilli(1_700_000_000_000).UTC() + probe, err := dataProbeTime(QueryCase{ + InstantOffsetsSeconds: []float64{300, 600}, + }, base) + if err != nil { + t.Fatalf("dataProbeTime: %v", err) + } + if got, want := probe, base.Add(300*time.Second); !got.Equal(want) { + t.Fatalf("probe time = %s, want %s", got, want) + } +} + +func TestGeneratedConfigsUseCurrentPlannerAndEngineSchema(t *testing.T) { + directory := t.TempDir() + fixture := seeder.Fixture{ + Name: "single-series", + Series: []seeder.FixtureSeries{{ + Metric: "up", + Labels: map[string]string{"job": "test"}, + Samples: []seeder.FixtureSample{{OffsetSeconds: 0, Value: 1}}, + }}, + } + suite := Suite{ + Name: "single-query", + Queries: []QueryCase{{Name: "up", Expr: "up"}}, + } + + if err := writeGeneratedConfigs(directory, fixture, suite); err != nil { + t.Fatalf("writeGeneratedConfigs: %v", err) + } + + plannerContents, err := os.ReadFile(filepath.Join(directory, "controller-config.yaml")) + if err != nil { + t.Fatalf("read planner config: %v", err) + } + var planner struct { + QueryGroups []struct { + RepetitionDelayMS int `yaml:"repetition_delay_ms"` + } `yaml:"query_groups"` + } + if err := yaml.Unmarshal(plannerContents, &planner); err != nil { + t.Fatalf("parse planner config: %v", err) + } + if got, want := planner.QueryGroups[0].RepetitionDelayMS, defaultDataIngestionIntervalMS; got != want { + t.Fatalf("planner repetition delay = %d, want %d", got, want) + } + + engineContents, err := os.ReadFile(filepath.Join(directory, "engine_config.yaml")) + if err != nil { + t.Fatalf("read engine config: %v", err) + } + var engine struct { + DataIngestionIntervalMS int `yaml:"data_ingestion_interval_ms"` + } + if err := yaml.Unmarshal(engineContents, &engine); err != nil { + t.Fatalf("parse engine config: %v", err) + } + if got, want := engine.DataIngestionIntervalMS, 1000; got != want { + t.Fatalf("engine ingestion interval = %d, want %d", got, want) + } +} + +func TestGeneratedPlannerGroupsUseCompatibleQueryTiming(t *testing.T) { + directory := t.TempDir() + fixture := seeder.Fixture{ + Name: "single-series", + Series: []seeder.FixtureSeries{{ + Metric: "up", + Labels: map[string]string{"job": "test"}, + Samples: []seeder.FixtureSample{{OffsetSeconds: 0, Value: 1}}, + }}, + } + suite := Suite{ + Name: "mixed-query-timing", + Queries: []QueryCase{ + {Name: "instant", Expr: "up", InstantOffsetsSeconds: []float64{0}}, + { + Name: "short-rate", + Expr: "rate(up[1500ms])", + InstantOffsetsSeconds: []float64{60}, + Range: &RangeSpec{StartOffsetSeconds: 60, EndOffsetSeconds: 120, StepSeconds: 2}, + }, + }, + } + + if err := writeGeneratedConfigs(directory, fixture, suite); err != nil { + t.Fatalf("writeGeneratedConfigs: %v", err) + } + + plannerContents, err := os.ReadFile(filepath.Join(directory, "controller-config.yaml")) + if err != nil { + t.Fatalf("read planner config: %v", err) + } + var planner struct { + QueryGroups []struct { + Queries []string `yaml:"queries"` + RepetitionDelayMS int `yaml:"repetition_delay_ms"` + RangeDurationMS *int `yaml:"range_duration_ms"` + StepMS *int `yaml:"step_ms"` + } `yaml:"query_groups"` + } + if err := yaml.Unmarshal(plannerContents, &planner); err != nil { + t.Fatalf("parse planner config: %v", err) + } + if got, want := len(planner.QueryGroups), 2; got != want { + t.Fatalf("planner query groups = %d, want %d", got, want) + } + if got, want := planner.QueryGroups[0].RepetitionDelayMS, defaultDataIngestionIntervalMS; got != want { + t.Fatalf("instant repetition delay = %d, want %d", got, want) + } + if got, want := planner.QueryGroups[1].RepetitionDelayMS, 1_000; got != want { + t.Fatalf("short-rate repetition delay = %d, want %d", got, want) + } + if planner.QueryGroups[1].StepMS == nil || *planner.QueryGroups[1].StepMS != 2_000 { + t.Fatalf("short-rate step_ms = %v, want 2000", planner.QueryGroups[1].StepMS) + } + if planner.QueryGroups[1].RangeDurationMS == nil || *planner.QueryGroups[1].RangeDurationMS != 60_000 { + t.Fatalf("short-rate range_duration_ms = %v, want 60000", planner.QueryGroups[1].RangeDurationMS) + } +} diff --git a/promql-compliance/runner/target.go b/promql-compliance/runner/target.go new file mode 100644 index 0000000..3ac8f04 --- /dev/null +++ b/promql-compliance/runner/target.go @@ -0,0 +1,40 @@ +package runner + +import ( + "context" + "fmt" + "net/http" + "time" + + client "github.com/prometheus/client_golang/api" + clientv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" +) + +// HTTPQueryTarget adapts any Prometheus-compatible HTTP query endpoint to the +// runner's small QueryAPI seam. +type HTTPQueryTarget struct { + api clientv1.API +} + +func NewHTTPQueryTarget(url string) (*HTTPQueryTarget, error) { + if url == "" { + return nil, fmt.Errorf("query target URL is empty") + } + apiClient, err := client.NewClient(client.Config{ + Address: url, + RoundTripper: http.DefaultTransport, + }) + if err != nil { + return nil, fmt.Errorf("create query target %q: %w", url, err) + } + return &HTTPQueryTarget{api: clientv1.NewAPI(apiClient)}, nil +} + +func (t *HTTPQueryTarget) Query(ctx context.Context, query string, timestamp time.Time, options ...clientv1.Option) (model.Value, clientv1.Warnings, error) { + return t.api.Query(ctx, query, timestamp, options...) +} + +func (t *HTTPQueryTarget) QueryRange(ctx context.Context, query string, interval clientv1.Range, options ...clientv1.Option) (model.Value, clientv1.Warnings, error) { + return t.api.QueryRange(ctx, query, interval, options...) +} diff --git a/promql-compliance/seeder/README.md b/promql-compliance/seeder/README.md new file mode 100644 index 0000000..e34261c --- /dev/null +++ b/promql-compliance/seeder/README.md @@ -0,0 +1,105 @@ +# promql-compliance seeder + +Part of [#594](https://github.com/ProjectASAP/ASAPQuery/issues/594): pushes a +fixed, hand-authored dataset via Prometheus remote-write to two targets — a +real Prometheus (started with `--web.enable-remote-write-receiver`) and +ASAPQuery's own remote-write ingest endpoint (`asap-query-engine/src/drivers/ingest/prometheus_remote_write.rs`, +served at `POST /api/v1/write`, snappy + protobuf, same wire format). Using +the exact same code path against both targets means there's no risk of two +different ingestion mechanisms producing false diffs in the differential +runner that consumes this. + +This is a standalone Go module (`go.mod` in this directory) — there is no +other Go code in the ASAPQuery repo. + +## Build / test + +``` +go build ./... +go test ./... +``` + +No live Prometheus/ASAPQuery instance is required to build or test: the unit +tests build a `WriteRequest`, snappy-encode it, decode it back, and assert +round-trip equality; the HTTP path is tested against an `httptest.Server` +instead of a real endpoint. + +## Usage + +``` +go run ./cmd/seed --reference-url=http://localhost:9090 --test-url=http://localhost:9091 +``` + +Each URL is the target's base URL; the seeder POSTs to `/api/v1/write`. +Prints the base timestamp (Unix ms) it anchored the dataset to — the +differential runner needs this to compute absolute query timestamps (see +below). + +Optionally pass `--base-time-ms` to pin the anchor instead of using +"now minus 30 minutes" (the default). Real Prometheus rejects samples that +are too old or too far in the future relative to wall-clock time, so the +dataset's timestamps are expressed as **offsets in seconds from a base time +chosen at seed time**, not fixed absolute timestamps. The *values* are fully +deterministic; only the absolute wall-clock placement moves on each run. + +## Dataset shape + +Defined in `dataset.go`. 20-minute window, sampled every 60s: offsets +0, 60, 120, ..., 1200 (seconds) from the run's base time. Six series across +three metrics: + +| Metric | Labels | Kind | Range | +|---|---|---|---| +| `http_requests_total` | `host="a"` | counter, +1/s | offsets 0..1200 (21 samples) | +| `http_requests_total` | `host="b"` | counter, +2/s | offsets 0..1200 (21 samples) | +| `node_memory_used_bytes` | `host="a"` | gauge, triangle 500->1000->500 | offsets 0..1200 (21 samples) | +| `node_memory_used_bytes` | `host="b"` | gauge, flat 2000 | offsets 0..1200 (21 samples) | +| `checkout_up` | `service="checkout",region="us-east"` | gauge, value 1 | offsets 0..300 only (6 samples) | +| `checkout_up` | `service="checkout",region="us-west"` | gauge, value 1 | offsets 900..1200 only (6 samples) | + +The `checkout_up` pair is deliberate: the us-east series stops at offset 300 +and us-west doesn't start until offset 900, a 600s gap — comfortably past +PromQL's default 5m (300s) staleness/lookback window. This is built to +exercise the same class of instant-vs-range divergence bugs as #589/#583/#584. + +## Hand-computed expected values + +Let `base` = the printed base-time-ms. All timestamps below are +`base + offset_seconds * 1000`. + +- `rate(http_requests_total{host="a"}[5m])` at any `t` in `[base+300s, base+1200s]` = **1.0** exactly. +- `rate(http_requests_total{host="b"}[5m])` at the same `t` = **2.0** exactly. +- `sum(rate(http_requests_total[5m]))` at `t = base+600s` = **3.0**. +- `increase(http_requests_total{host="a"}[5m])` at `t = base+600s` = **300**. +- `http_requests_total{host="a"}` instant value at `t = base+1200s` = **1200**. +- `max_over_time(node_memory_used_bytes{host="a"}[20m])` = **1000**; `min_over_time(...)` = **500**. +- `node_memory_used_bytes{host="a"}` instant value at `t = base+600s` (the peak) = **1000**. +- `avg_over_time(node_memory_used_bytes{host="b"}[20m])` = **2000** exactly (flat series). +- `sum(node_memory_used_bytes)` at `t = base+600s` = **3000** (1000 + 2000). +- Instant query `checkout_up{service="checkout"}` at `t = base+660s` = **empty result vector** + (us-east's last sample at offset 300 is 360s stale, > 5m lookback; us-west's + first sample isn't until offset 900). This is the instant/range divergence probe. +- Range query `checkout_up{service="checkout"}[20m]` evaluated at `t = base+1200s` + returns a matrix with **two series**: us-east (6 samples, offsets 0..300) and + us-west (6 samples, offsets 900..1200) — i.e. the range query surfaces both + series even though no single instant in the gap does. + +See the doc comment at the top of `dataset.go` for the full derivation. + +## Known limitations + +- `go mod init`/`go get` pulled the real `github.com/prometheus/prometheus/prompb` + and `github.com/golang/snappy` packages from the public Go module proxy — + network access was available in this environment, so no vendoring fallback + was needed. `prompb` messages use `github.com/gogo/protobuf/proto` for + marshal/unmarshal (matches upstream `prometheus/prometheus`), which is an + explicit dependency here (`push.go`). +- Building `prometheus/prometheus` bumped the module's Go toolchain + requirement to 1.25.8 (from the system's 1.21.13); `go` auto-downloaded and + used the newer toolchain per `go.mod`'s `go 1.25.8` directive. This only + affects `promql-compliance/seeder`'s own module — it does not touch the + Rust workspace or any other part of the repo. +- Not run against a live Prometheus or ASAPQuery instance — none was + available in this environment. `go build ./...` and `go test ./...` both + pass; the HTTP POST path (headers, path, body encoding) is covered via + `httptest.Server` in `push_test.go`. diff --git a/promql-compliance/seeder/cmd/seed/main.go b/promql-compliance/seeder/cmd/seed/main.go new file mode 100644 index 0000000..18284c8 --- /dev/null +++ b/promql-compliance/seeder/cmd/seed/main.go @@ -0,0 +1,58 @@ +// Command seed pushes a dataset defined in package seeder or a fixture to two +// Prometheus-remote-write-compatible endpoints: a real Prometheus (started +// with --web.enable-remote-write-receiver) and ASAPQuery's own remote-write +// ingest endpoint. Using the same WriteRequest bytes against both means +// there's no risk of the two ingestion mechanisms disagreeing and producing +// false diffs in the differential PromQL runner (see issue +// #594). +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "time" + + "github.com/ProjectASAP/ASAPQuery/promql-compliance/seeder" +) + +func main() { + referenceURL := flag.String("reference-url", "", "base URL of the reference target (real Prometheus), e.g. http://localhost:9090") + testURL := flag.String("test-url", "", "base URL of the test target (ASAPQuery), e.g. http://localhost:9091") + datasetPath := flag.String("dataset", "", "optional YAML dataset fixture; defaults to the built-in dataset") + baseTimeFlag := flag.Int64("base-time-ms", 0, "base Unix time in ms to anchor the dataset's offsets to (default: now, floored to the minute, minus 30 minutes so the whole 20-minute window is safely in the past)") + flag.Parse() + + if *referenceURL == "" || *testURL == "" { + fmt.Fprintln(os.Stderr, "usage: seed --reference-url= --test-url= [--base-time-ms=]") + os.Exit(2) + } + + baseTimeMs := *baseTimeFlag + if baseTimeMs == 0 { + now := time.Now().UTC().Truncate(time.Minute) + baseTimeMs = now.Add(-30 * time.Minute).UnixMilli() + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + var err error + if *datasetPath == "" { + err = seeder.PushDataset(ctx, baseTimeMs, *referenceURL, *testURL) + } else { + fixture, loadErr := seeder.LoadFixture(*datasetPath) + if loadErr != nil { + log.Fatalf("loading dataset failed: %v", loadErr) + } + err = seeder.PushFixture(ctx, baseTimeMs, fixture, *referenceURL, *testURL) + } + if err != nil { + log.Fatalf("seeding failed: %v", err) + } + + fmt.Printf("seeded dataset to %s and %s\n", *referenceURL, *testURL) + fmt.Printf("base-time-ms=%d\n", baseTimeMs) +} diff --git a/promql-compliance/seeder/dataset.go b/promql-compliance/seeder/dataset.go new file mode 100644 index 0000000..ccc53d8 --- /dev/null +++ b/promql-compliance/seeder/dataset.go @@ -0,0 +1,150 @@ +package seeder + +// This file defines the seeder's one fixed, hand-authored dataset. +// +// # Shape +// +// The dataset spans a 20-minute window sampled every 60s: 21 timestamps at +// offsets (in seconds from a base time chosen at seed time, see BaseTimeMs +// in push.go) of 0, 60, 120, ..., 1200. +// +// It contains three metrics, six series total: +// +// 1. http_requests_total{host="a"|"b"} — a counter-like metric (strictly +// increasing), present for the whole window, to exercise rate()/increase(). +// 2. node_memory_used_bytes{host="a"|"b"} — a gauge-like metric, present for +// the whole window, to exercise min/max/avg_over_time(). +// 3. checkout_up{service="checkout", region="us-east"|"us-west"} — a series +// whose label set changes partway through the window: the us-east series +// only has samples in the first 5 minutes, the us-west series only has +// samples in the last 5 minutes, with a >5m silent gap in between. This +// is deliberately built to exercise instant-vs-range divergence bugs +// (see #589/#583/#584): an instant query evaluated inside the gap must +// see an empty result (both series are stale/not-yet-started under the +// default 5m lookback), while a range query covering the same window +// returns raw matrix samples for both series. +// +// # Hand-computed expected values +// +// Let base = the base time (ms) the seeder used for this run (printed by +// cmd/seed on push). All timestamps below are "base + offset seconds". +// +// - http_requests_total{host="a"}: value(offset) = offset (seconds). +// So value(0)=0, value(600)=600, value(1200)=1200. +// rate(http_requests_total{host="a"}[5m]) at any t in [base+300, base+1200] +// = 1.0 exactly (1 unit/second). +// increase(http_requests_total{host="a"}[5m]) at those same t = 300. +// +// - http_requests_total{host="b"}: value(offset) = 1000 + 2*offset. +// value(0)=1000, value(600)=2200, value(1200)=3400. +// rate(http_requests_total{host="b"}[5m]) at any t in [base+300, base+1200] +// = 2.0 exactly. +// +// - sum(rate(http_requests_total[5m])) at t=base+600 = 1.0 + 2.0 = 3.0. +// +// - node_memory_used_bytes{host="a"}: a triangle wave. Rises from 500 to +// 1000 in steps of 50 over offsets 0..600 (11 points), then falls back +// from 950 to 500 in steps of 50 over offsets 660..1200 (10 points). +// max_over_time(node_memory_used_bytes{host="a"}[20m]) at t=base+1200 = 1000. +// min_over_time(node_memory_used_bytes{host="a"}[20m]) at t=base+1200 = 500. +// Instant value at t=base+600 = 1000 (the peak). +// +// - node_memory_used_bytes{host="b"}: flat 2000 for every sample. +// avg_over_time(node_memory_used_bytes{host="b"}[20m]) = 2000 exactly. +// +// - sum(node_memory_used_bytes) at t=base+600 (instant) = 1000 + 2000 = 3000. +// +// - checkout_up{service="checkout",region="us-east"}: value=1 at offsets +// 0,60,120,180,240,300, then no more samples. +// checkout_up{service="checkout",region="us-west"}: value=1 at offsets +// 900,960,1020,1080,1140,1200, no samples before that. +// At t=base+660 (360s after the last us-east sample, i.e. > the 5m +// default lookback, and 240s before the first us-west sample): an +// instant query for checkout_up{service="checkout"} must return an +// EMPTY result vector (both series are absent/stale). A range query +// for checkout_up{service="checkout"}[20m] evaluated at t=base+1200 +// must return a matrix with two series: us-east with 6 samples +// (offsets 0..300) and us-west with 6 samples (offsets 900..1200). +// This is the instant-vs-range divergence case #594 is meant to catch. +// +// Point counts: http_requests_total and node_memory_used_bytes each have 21 +// samples per series (offsets 0,60,...,1200). checkout_up has 6 samples per +// series (12 total), deliberately sparse and non-overlapping in time. + +// Dataset returns the fixed set of series pushed by the seeder. It is a +// plain Go literal (built with small loops below for the repetitive parts) +// rather than data read from a file, so the values above are exactly what +// gets pushed — no external format to keep in sync. +func Dataset() []SeriesDef { + offsets := make([]int64, 0, 21) + for o := int64(0); o <= 1200; o += 60 { + offsets = append(offsets, o) + } + + httpRequestsA := SeriesDef{ + Name: "http_requests_total", + Labels: map[string]string{"host": "a"}, + } + httpRequestsB := SeriesDef{ + Name: "http_requests_total", + Labels: map[string]string{"host": "b"}, + } + for _, o := range offsets { + httpRequestsA.Samples = append(httpRequestsA.Samples, Sample{ + OffsetSeconds: o, + Value: float64(o), // 1 unit/sec + }) + httpRequestsB.Samples = append(httpRequestsB.Samples, Sample{ + OffsetSeconds: o, + Value: 1000 + 2*float64(o), // 2 units/sec + }) + } + + memA := SeriesDef{ + Name: "node_memory_used_bytes", + Labels: map[string]string{"host": "a"}, + } + memB := SeriesDef{ + Name: "node_memory_used_bytes", + Labels: map[string]string{"host": "b"}, + } + for _, o := range offsets { + var v float64 + switch { + case o <= 600: + // Rising leg: 500 at o=0 up to 1000 at o=600, step 50 per 60s. + v = 500 + 50*float64(o/60) + default: + // Falling leg: 950 at o=660 down to 500 at o=1200, step 50 per 60s. + stepsPastPeak := (o - 600) / 60 + v = 1000 - 50*float64(stepsPastPeak) + } + memA.Samples = append(memA.Samples, Sample{OffsetSeconds: o, Value: v}) + memB.Samples = append(memB.Samples, Sample{OffsetSeconds: o, Value: 2000}) + } + + checkoutUpEast := SeriesDef{ + Name: "checkout_up", + Labels: map[string]string{"service": "checkout", "region": "us-east"}, + } + for o := int64(0); o <= 300; o += 60 { + checkoutUpEast.Samples = append(checkoutUpEast.Samples, Sample{OffsetSeconds: o, Value: 1}) + } + + checkoutUpWest := SeriesDef{ + Name: "checkout_up", + Labels: map[string]string{"service": "checkout", "region": "us-west"}, + } + for o := int64(900); o <= 1200; o += 60 { + checkoutUpWest.Samples = append(checkoutUpWest.Samples, Sample{OffsetSeconds: o, Value: 1}) + } + + return []SeriesDef{ + httpRequestsA, + httpRequestsB, + memA, + memB, + checkoutUpEast, + checkoutUpWest, + } +} diff --git a/promql-compliance/seeder/fixture.go b/promql-compliance/seeder/fixture.go new file mode 100644 index 0000000..a6356b9 --- /dev/null +++ b/promql-compliance/seeder/fixture.go @@ -0,0 +1,111 @@ +package seeder + +import ( + "bytes" + "fmt" + "os" + "sort" + + "github.com/prometheus/prometheus/prompb" + "gopkg.in/yaml.v3" +) + +// Fixture is a versioned, human-authored dataset. Sample timestamps are +// offsets from the base timestamp selected for a run, so the same fixture can +// be replayed against Prometheus without becoming too old or too far ahead. +type Fixture struct { + Name string `yaml:"name"` + Series []FixtureSeries `yaml:"series"` +} + +type FixtureSeries struct { + Metric string `yaml:"metric"` + Labels map[string]string `yaml:"labels"` + Samples []FixtureSample `yaml:"samples"` +} + +type FixtureSample struct { + OffsetSeconds int64 `yaml:"offset_seconds"` + Value float64 `yaml:"value"` +} + +// LoadFixture reads and validates a dataset fixture from YAML. +func LoadFixture(path string) (Fixture, error) { + contents, err := os.ReadFile(path) + if err != nil { + return Fixture{}, fmt.Errorf("read dataset fixture %q: %w", path, err) + } + + var fixture Fixture + decoder := yaml.NewDecoder(bytes.NewReader(contents)) + decoder.KnownFields(true) + if err := decoder.Decode(&fixture); err != nil { + return Fixture{}, fmt.Errorf("parse dataset fixture %q: %w", path, err) + } + if fixture.Name == "" { + return Fixture{}, fmt.Errorf("dataset fixture %q has no name", path) + } + if len(fixture.Series) == 0 { + return Fixture{}, fmt.Errorf("dataset fixture %q has no series", path) + } + for i, series := range fixture.Series { + if series.Metric == "" { + return Fixture{}, fmt.Errorf("dataset fixture %q series %d has no metric", path, i) + } + if len(series.Samples) == 0 { + return Fixture{}, fmt.Errorf("dataset fixture %q series %q has no samples", path, series.Metric) + } + } + return fixture, nil +} + +// BuildWriteRequestFromFixture converts a fixture into the same canonical +// remote-write representation used by the hand-authored Dataset. +func BuildWriteRequestFromFixture(baseTimeMs int64, fixture Fixture) *prompb.WriteRequest { + series := make([]SeriesDef, 0, len(fixture.Series)) + for _, input := range fixture.Series { + samples := make([]Sample, 0, len(input.Samples)) + for _, sample := range input.Samples { + samples = append(samples, Sample{ + OffsetSeconds: sample.OffsetSeconds, + Value: sample.Value, + }) + } + series = append(series, SeriesDef{ + Name: input.Metric, + Labels: input.Labels, + Samples: samples, + }) + } + return BuildWriteRequest(baseTimeMs, series) +} + +// BuildWriteRequestsFromFixtureBatches converts a fixture into timestamp-ordered +// remote-write batches. Sending one event-time slice at a time lets streaming +// engines advance and close windows as they would during normal ingestion. +func BuildWriteRequestsFromFixtureBatches(baseTimeMs int64, fixture Fixture) []*prompb.WriteRequest { + byOffset := make(map[int64][]SeriesDef) + for _, input := range fixture.Series { + for _, sample := range input.Samples { + byOffset[sample.OffsetSeconds] = append(byOffset[sample.OffsetSeconds], SeriesDef{ + Name: input.Metric, + Labels: input.Labels, + Samples: []Sample{{ + OffsetSeconds: sample.OffsetSeconds, + Value: sample.Value, + }}, + }) + } + } + offsets := make([]int64, 0, len(byOffset)) + for offset := range byOffset { + offsets = append(offsets, offset) + } + sort.Slice(offsets, func(i, j int) bool { return offsets[i] < offsets[j] }) + + requests := make([]*prompb.WriteRequest, 0, len(offsets)) + for _, offset := range offsets { + requests = append(requests, BuildWriteRequest(baseTimeMs, byOffset[offset])) + } + return requests +} diff --git a/promql-compliance/seeder/fixture_test.go b/promql-compliance/seeder/fixture_test.go new file mode 100644 index 0000000..ad546ba --- /dev/null +++ b/promql-compliance/seeder/fixture_test.go @@ -0,0 +1,77 @@ +package seeder + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadFixtureBuildsRelativeTimestamps(t *testing.T) { + path := filepath.Join(t.TempDir(), "dataset.yaml") + contents := []byte(`name: sparse-checkout +series: + - metric: checkout_up + labels: + service: checkout + region: us-east + samples: + - offset_seconds: 0 + value: 1 + - offset_seconds: 300 + value: 0 +`) + if err := os.WriteFile(path, contents, 0o600); err != nil { + t.Fatal(err) + } + + fixture, err := LoadFixture(path) + if err != nil { + t.Fatalf("LoadFixture: %v", err) + } + if fixture.Name != "sparse-checkout" { + t.Fatalf("fixture name = %q, want sparse-checkout", fixture.Name) + } + + request := BuildWriteRequestFromFixture(1_700_000_000_000, fixture) + if got := len(request.Timeseries); got != 1 { + t.Fatalf("timeseries = %d, want 1", got) + } + if got := request.Timeseries[0].Samples[1].Timestamp; got != 1_700_000_300_000 { + t.Fatalf("timestamp = %d, want 1700000300000", got) + } +} + +func TestLoadFixtureRejectsInvalidShape(t *testing.T) { + path := filepath.Join(t.TempDir(), "dataset.yaml") + if err := os.WriteFile(path, []byte(`name: missing-series`), 0o600); err != nil { + t.Fatal(err) + } + + if _, err := LoadFixture(path); err == nil { + t.Fatal("LoadFixture succeeded for a fixture without series") + } +} + +func TestBuildWriteRequestsFromFixtureBatchesOrdersEventTime(t *testing.T) { + fixture := Fixture{ + Name: "ordered", + Series: []FixtureSeries{{ + Metric: "up", + Samples: []FixtureSample{ + {OffsetSeconds: 60, Value: 2}, + {OffsetSeconds: 0, Value: 1}, + }, + }}, + } + + requests := BuildWriteRequestsFromFixtureBatches(1_700_000_000_000, fixture) + if got, want := len(requests), 2; got != want { + t.Fatalf("batch count = %d, want %d", got, want) + } + if got, want := requests[0].Timeseries[0].Samples[0].Timestamp, int64(1_700_000_000_000); got != want { + t.Fatalf("first batch timestamp = %d, want %d", got, want) + } + if got, want := requests[1].Timeseries[0].Samples[0].Timestamp, int64(1_700_000_060_000); got != want { + t.Fatalf("second batch timestamp = %d, want %d", got, want) + } +} diff --git a/promql-compliance/seeder/go.mod b/promql-compliance/seeder/go.mod new file mode 100644 index 0000000..583abfb --- /dev/null +++ b/promql-compliance/seeder/go.mod @@ -0,0 +1,19 @@ +module github.com/ProjectASAP/ASAPQuery/promql-compliance/seeder + +go 1.25.8 + +require ( + github.com/gogo/protobuf v1.3.2 + github.com/golang/snappy v1.0.0 + github.com/prometheus/prometheus v0.314.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.70.1 // indirect + golang.org/x/text v0.40.0 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect +) diff --git a/promql-compliance/seeder/go.sum b/promql-compliance/seeder/go.sum new file mode 100644 index 0000000..ef4738c --- /dev/null +++ b/promql-compliance/seeder/go.sum @@ -0,0 +1,59 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 h1:cLN4IBkmkYZNnk7EAJ0BHIethd+J6LqxFNw5mSiI2bM= +github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/prometheus v0.314.0 h1:YjsimqsIi6/mOtzZcrPEYUALO6zpfaht9O5sXqDz2vg= +github.com/prometheus/prometheus v0.314.0/go.mod h1:zjg3pMTAkY0/JG8jy/h8/YgSQUVB+aCXMhUqN6l64jg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/promql-compliance/seeder/push.go b/promql-compliance/seeder/push.go new file mode 100644 index 0000000..601fdce --- /dev/null +++ b/promql-compliance/seeder/push.go @@ -0,0 +1,169 @@ +// Package seeder builds Prometheus remote-write WriteRequests from a fixed, +// hand-authored dataset and pushes them to one or more remote-write +// endpoints. It exists to seed a real Prometheus and ASAPQuery's own +// remote-write ingest endpoint with the exact same bytes, so a differential +// PromQL compliance test (see GitHub issue #594) has no risk of the two +// ingestion mechanisms disagreeing and producing false diffs. +package seeder + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "sort" + "strings" + + "github.com/gogo/protobuf/proto" + "github.com/golang/snappy" + "github.com/prometheus/prometheus/prompb" +) + +// Sample is one (offset, value) point in a SeriesDef. OffsetSeconds is +// relative to a base time supplied at push time (see BuildWriteRequest), +// not an absolute Unix timestamp — this keeps the dataset's values fully +// deterministic while letting the actual wall-clock timestamps be chosen +// fresh on every run, which real Prometheus requires (it rejects samples +// that are too far in the past or future relative to "now"). +type Sample struct { + OffsetSeconds int64 + Value float64 +} + +// SeriesDef is one time series: a metric name, a label set (NOT including +// __name__), and its samples. +type SeriesDef struct { + Name string + Labels map[string]string + Samples []Sample +} + +// BuildWriteRequest converts a set of SeriesDef into a prompb.WriteRequest, +// resolving each sample's absolute timestamp as baseTimeMs + +// sample.OffsetSeconds*1000. +func BuildWriteRequest(baseTimeMs int64, series []SeriesDef) *prompb.WriteRequest { + wr := &prompb.WriteRequest{ + Timeseries: make([]prompb.TimeSeries, 0, len(series)), + } + + for _, s := range series { + labels := make([]prompb.Label, 0, len(s.Labels)+1) + labels = append(labels, prompb.Label{Name: "__name__", Value: s.Name}) + for k, v := range s.Labels { + labels = append(labels, prompb.Label{Name: k, Value: v}) + } + // Prometheus's remote-write receiver requires labels to be sorted + // by name (excluding this, some implementations reject the write). + sort.Slice(labels, func(i, j int) bool { return labels[i].Name < labels[j].Name }) + + samples := make([]prompb.Sample, 0, len(s.Samples)) + for _, sm := range s.Samples { + samples = append(samples, prompb.Sample{ + Value: sm.Value, + Timestamp: baseTimeMs + sm.OffsetSeconds*1000, + }) + } + + wr.Timeseries = append(wr.Timeseries, prompb.TimeSeries{ + Labels: labels, + Samples: samples, + }) + } + + return wr +} + +// EncodeSnappy protobuf-marshals a WriteRequest and snappy-compresses the +// result, i.e. produces exactly the body Prometheus remote-write expects. +func EncodeSnappy(wr *prompb.WriteRequest) ([]byte, error) { + data, err := proto.Marshal(wr) + if err != nil { + return nil, fmt.Errorf("marshal WriteRequest: %w", err) + } + return snappy.Encode(nil, data), nil +} + +// DecodeSnappy reverses EncodeSnappy: snappy-decompresses and +// protobuf-unmarshals a remote-write body back into a WriteRequest. It is +// primarily useful for tests that want to assert on what was actually sent. +func DecodeSnappy(body []byte) (*prompb.WriteRequest, error) { + decompressed, err := snappy.Decode(nil, body) + if err != nil { + return nil, fmt.Errorf("snappy decode: %w", err) + } + wr := &prompb.WriteRequest{} + if err := proto.Unmarshal(decompressed, wr); err != nil { + return nil, fmt.Errorf("unmarshal WriteRequest: %w", err) + } + return wr, nil +} + +// Push POSTs a WriteRequest to url + "/api/v1/write" using the standard +// Prometheus remote-write wire format: snappy-compressed protobuf, with the +// headers a remote-write receiver expects. +func Push(ctx context.Context, url string, wr *prompb.WriteRequest) error { + body, err := EncodeSnappy(wr) + if err != nil { + return err + } + return PushEncoded(ctx, url, body) +} + +// PushEncoded sends an already-encoded remote-write body. The caller can +// encode once and send the exact same bytes to every target. +func PushEncoded(ctx context.Context, url string, body []byte) error { + endpoint := strings.TrimRight(url, "/") + "/api/v1/write" + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("build request for %s: %w", url, err) + } + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Content-Encoding", "snappy") + req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("POST %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode/100 != 2 { + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("POST %s: unexpected status %s: %s", url, resp.Status, string(respBody)) + } + return nil +} + +// PushDataset builds the write request for the fixed Dataset() at the given +// base time and pushes it to every URL in urls, stopping at the first +// error. +func PushDataset(ctx context.Context, baseTimeMs int64, urls ...string) error { + return PushSeries(ctx, baseTimeMs, Dataset(), urls...) +} + +// PushFixture sends a YAML fixture to every target using one encoded body. +func PushFixture(ctx context.Context, baseTimeMs int64, fixture Fixture, urls ...string) error { + wr := BuildWriteRequestFromFixture(baseTimeMs, fixture) + return PushEncodedRequest(ctx, wr, urls...) +} + +// PushSeries sends one canonical encoded request to every target. +func PushSeries(ctx context.Context, baseTimeMs int64, series []SeriesDef, urls ...string) error { + wr := BuildWriteRequest(baseTimeMs, series) + return PushEncodedRequest(ctx, wr, urls...) +} + +func PushEncodedRequest(ctx context.Context, wr *prompb.WriteRequest, urls ...string) error { + body, err := EncodeSnappy(wr) + if err != nil { + return err + } + for _, u := range urls { + if err := PushEncoded(ctx, u, body); err != nil { + return err + } + } + return nil +} diff --git a/promql-compliance/seeder/push_test.go b/promql-compliance/seeder/push_test.go new file mode 100644 index 0000000..a9782bf --- /dev/null +++ b/promql-compliance/seeder/push_test.go @@ -0,0 +1,264 @@ +package seeder + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "reflect" + "sort" + "testing" + + "github.com/prometheus/prometheus/prompb" +) + +func TestBuildWriteRequest_TimestampsAndLabels(t *testing.T) { + series := []SeriesDef{ + { + Name: "test_metric", + Labels: map[string]string{"region": "us-east-1", "host": "a"}, + Samples: []Sample{ + {OffsetSeconds: 0, Value: 1.5}, + {OffsetSeconds: 60, Value: 2.5}, + }, + }, + } + + const base int64 = 1_700_000_000_000 + wr := BuildWriteRequest(base, series) + + if len(wr.Timeseries) != 1 { + t.Fatalf("expected 1 timeseries, got %d", len(wr.Timeseries)) + } + ts := wr.Timeseries[0] + + if len(ts.Samples) != 2 { + t.Fatalf("expected 2 samples, got %d", len(ts.Samples)) + } + if ts.Samples[0].Timestamp != base { + t.Errorf("sample 0 timestamp = %d, want %d", ts.Samples[0].Timestamp, base) + } + if ts.Samples[1].Timestamp != base+60_000 { + t.Errorf("sample 1 timestamp = %d, want %d", ts.Samples[1].Timestamp, base+60_000) + } + if ts.Samples[0].Value != 1.5 || ts.Samples[1].Value != 2.5 { + t.Errorf("unexpected sample values: %+v", ts.Samples) + } + + // Labels must include __name__ and be sorted by name. + names := make([]string, len(ts.Labels)) + for i, l := range ts.Labels { + names[i] = l.Name + } + if !sort.StringsAreSorted(names) { + t.Errorf("labels not sorted by name: %v", names) + } + + got := map[string]string{} + for _, l := range ts.Labels { + got[l.Name] = l.Value + } + want := map[string]string{"__name__": "test_metric", "region": "us-east-1", "host": "a"} + for k, v := range want { + if got[k] != v { + t.Errorf("label %q = %q, want %q", k, got[k], v) + } + } +} + +func TestEncodeDecodeSnappyRoundTrip(t *testing.T) { + wr := BuildWriteRequest(1000, []SeriesDef{ + { + Name: "roundtrip_metric", + Labels: map[string]string{"env": "prod"}, + Samples: []Sample{ + {OffsetSeconds: 0, Value: 42}, + {OffsetSeconds: 5, Value: 43.5}, + }, + }, + }) + + body, err := EncodeSnappy(wr) + if err != nil { + t.Fatalf("EncodeSnappy: %v", err) + } + + decoded, err := DecodeSnappy(body) + if err != nil { + t.Fatalf("DecodeSnappy: %v", err) + } + + if len(decoded.Timeseries) != 1 { + t.Fatalf("expected 1 timeseries after roundtrip, got %d", len(decoded.Timeseries)) + } + if !reflect.DeepEqual(decoded.Timeseries[0].Labels, wr.Timeseries[0].Labels) || + !reflect.DeepEqual(decoded.Timeseries[0].Samples, wr.Timeseries[0].Samples) { + t.Errorf("roundtripped timeseries mismatch:\ngot: %+v\nwant: %+v", decoded.Timeseries[0], wr.Timeseries[0]) + } +} + +func TestDatasetBuildsAndEncodesCleanly(t *testing.T) { + wr := BuildWriteRequest(1_700_000_000_000, Dataset()) + + if len(wr.Timeseries) != 6 { + t.Fatalf("expected 6 series in Dataset(), got %d", len(wr.Timeseries)) + } + + totalSamples := 0 + for _, ts := range wr.Timeseries { + totalSamples += len(ts.Samples) + } + // 4 series x 21 samples + 2 series x 6 samples = 96. + if want := 4*21 + 2*6; totalSamples != want { + t.Errorf("total samples = %d, want %d", totalSamples, want) + } + + if _, err := EncodeSnappy(wr); err != nil { + t.Fatalf("EncodeSnappy(Dataset): %v", err) + } +} + +func TestDatasetHandComputedValues(t *testing.T) { + series := Dataset() + + find := func(name string, labels map[string]string) SeriesDef { + for _, s := range series { + if s.Name != name || len(s.Labels) != len(labels) { + continue + } + match := true + for k, v := range labels { + if s.Labels[k] != v { + match = false + break + } + } + if match { + return s + } + } + t.Fatalf("series %s%v not found in Dataset()", name, labels) + return SeriesDef{} + } + + sampleAt := func(s SeriesDef, offset int64) (float64, bool) { + for _, sm := range s.Samples { + if sm.OffsetSeconds == offset { + return sm.Value, true + } + } + return 0, false + } + + httpA := find("http_requests_total", map[string]string{"host": "a"}) + if v, ok := sampleAt(httpA, 1200); !ok || v != 1200 { + t.Errorf("http_requests_total{host=a} at offset 1200 = %v (ok=%v), want 1200", v, ok) + } + + httpB := find("http_requests_total", map[string]string{"host": "b"}) + if v, ok := sampleAt(httpB, 600); !ok || v != 2200 { + t.Errorf("http_requests_total{host=b} at offset 600 = %v (ok=%v), want 2200", v, ok) + } + + memA := find("node_memory_used_bytes", map[string]string{"host": "a"}) + if v, ok := sampleAt(memA, 600); !ok || v != 1000 { + t.Errorf("node_memory_used_bytes{host=a} peak at offset 600 = %v (ok=%v), want 1000", v, ok) + } + if v, ok := sampleAt(memA, 1200); !ok || v != 500 { + t.Errorf("node_memory_used_bytes{host=a} at offset 1200 = %v (ok=%v), want 500", v, ok) + } + + memB := find("node_memory_used_bytes", map[string]string{"host": "b"}) + for _, sm := range memB.Samples { + if sm.Value != 2000 { + t.Errorf("node_memory_used_bytes{host=b} at offset %d = %v, want flat 2000", sm.OffsetSeconds, sm.Value) + } + } + + east := find("checkout_up", map[string]string{"service": "checkout", "region": "us-east"}) + if len(east.Samples) != 6 { + t.Errorf("checkout_up{region=us-east} has %d samples, want 6", len(east.Samples)) + } + for _, sm := range east.Samples { + if sm.OffsetSeconds > 300 { + t.Errorf("checkout_up{region=us-east} has sample at offset %d, want all <= 300", sm.OffsetSeconds) + } + } + + west := find("checkout_up", map[string]string{"service": "checkout", "region": "us-west"}) + if len(west.Samples) != 6 { + t.Errorf("checkout_up{region=us-west} has %d samples, want 6", len(west.Samples)) + } + for _, sm := range west.Samples { + if sm.OffsetSeconds < 900 { + t.Errorf("checkout_up{region=us-west} has sample at offset %d, want all >= 900", sm.OffsetSeconds) + } + } +} + +func TestPushPostsSnappyProtobufWithExpectedHeaders(t *testing.T) { + var gotContentType, gotContentEncoding, gotVersion, gotPath string + var gotBody []byte + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotContentType = r.Header.Get("Content-Type") + gotContentEncoding = r.Header.Get("Content-Encoding") + gotVersion = r.Header.Get("X-Prometheus-Remote-Write-Version") + b, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("reading request body: %v", err) + } + gotBody = b + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + wr := BuildWriteRequest(1000, []SeriesDef{ + { + Name: "push_test_metric", + Labels: map[string]string{"k": "v"}, + Samples: []Sample{{OffsetSeconds: 0, Value: 7}}, + }, + }) + + if err := Push(context.Background(), srv.URL, wr); err != nil { + t.Fatalf("Push: %v", err) + } + + if gotPath != "/api/v1/write" { + t.Errorf("path = %q, want /api/v1/write", gotPath) + } + if gotContentType != "application/x-protobuf" { + t.Errorf("Content-Type = %q, want application/x-protobuf", gotContentType) + } + if gotContentEncoding != "snappy" { + t.Errorf("Content-Encoding = %q, want snappy", gotContentEncoding) + } + if gotVersion != "0.1.0" { + t.Errorf("X-Prometheus-Remote-Write-Version = %q, want 0.1.0", gotVersion) + } + + decoded, err := DecodeSnappy(gotBody) + if err != nil { + t.Fatalf("DecodeSnappy(received body): %v", err) + } + if len(decoded.Timeseries) != 1 || + !reflect.DeepEqual(decoded.Timeseries[0].Labels, wr.Timeseries[0].Labels) || + !reflect.DeepEqual(decoded.Timeseries[0].Samples, wr.Timeseries[0].Samples) { + t.Errorf("received WriteRequest does not match what was built:\ngot: %+v\nwant: %+v", decoded, wr) + } +} + +func TestPushReturnsErrorOnNon2xx(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte("boom")) + })) + defer srv.Close() + + wr := &prompb.WriteRequest{} + if err := Push(context.Background(), srv.URL, wr); err == nil { + t.Fatal("expected error on 400 response, got nil") + } +} diff --git a/promql-compliance/suites/temporal.yaml b/promql-compliance/suites/temporal.yaml new file mode 100644 index 0000000..69740cf --- /dev/null +++ b/promql-compliance/suites/temporal.yaml @@ -0,0 +1,13 @@ +name: temporal +comparison_defaults: + value_tolerance: + relative: 0 + absolute: 0.000001 +queries: + - name: request-rate + expr: rate(http_requests_total[5m]) + instant_offsets_seconds: [300, 600, 1200] + range: + start_offset_seconds: 300 + end_offset_seconds: 1200 + step_seconds: 60