From 77dbad7e87e7fbc944932afe9f5f5994b96d84d3 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Fri, 24 Apr 2026 18:44:05 -0700 Subject: [PATCH 1/9] Integration tests Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/integration_test.yml | 46 +++++++++ integration/alertmanager-config.yaml | 4 + integration/cortex-config.yaml | 67 +++++++++++++ integration/integration_test.go | 127 +++++++++++++++++++++++++ 4 files changed, 244 insertions(+) create mode 100644 .github/workflows/integration_test.yml create mode 100644 integration/alertmanager-config.yaml create mode 100644 integration/cortex-config.yaml create mode 100644 integration/integration_test.go diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml new file mode 100644 index 000000000..f2d52b499 --- /dev/null +++ b/.github/workflows/integration_test.yml @@ -0,0 +1,46 @@ +name: Integration Tests +on: + pull_request: + paths-ignore: + - "docs/**" + - "*.md" +jobs: + integration: + name: Integration Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: 1.22 + - name: Start Cortex + run: | + docker run -d --name cortex \ + -p 9009:9009 \ + -v ${{ github.workspace }}/integration/cortex-config.yaml:/etc/cortex/config.yaml \ + cortexproject/cortex:v1.18.1 \ + -config.file=/etc/cortex/config.yaml \ + -target=all,alertmanager + - name: Wait for Cortex + run: | + for i in $(seq 1 30); do + if wget -qO- http://localhost:9009/ready > /dev/null 2>&1; then + echo "Cortex is ready" + exit 0 + fi + echo "Waiting for Cortex... ($i)" + sleep 2 + done + echo "Cortex failed to start" + docker logs cortex + exit 1 + - name: Run Integration Tests + env: + CORTEX_ADDRESS: http://localhost:9009 + run: go test -mod=vendor -tags=integration -v -count=1 ./integration/... + - name: Cortex Logs + if: failure() + run: docker logs cortex + - name: Cleanup + if: always() + run: docker stop cortex && docker rm cortex diff --git a/integration/alertmanager-config.yaml b/integration/alertmanager-config.yaml new file mode 100644 index 000000000..5e7be387a --- /dev/null +++ b/integration/alertmanager-config.yaml @@ -0,0 +1,4 @@ +route: + receiver: default +receivers: + - name: default diff --git a/integration/cortex-config.yaml b/integration/cortex-config.yaml new file mode 100644 index 000000000..b3dd86231 --- /dev/null +++ b/integration/cortex-config.yaml @@ -0,0 +1,67 @@ +auth_enabled: false + +server: + http_listen_port: 9009 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + grpc_server_max_concurrent_streams: 1000 + +distributor: + shard_by_all_labels: true + pool: + health_check_ingesters: true + +ingester_client: + grpc_client_config: + max_recv_msg_size: 104857600 + max_send_msg_size: 104857600 + grpc_compression: gzip + +ingester: + lifecycler: + min_ready_duration: 0s + final_sleep: 0s + num_tokens: 512 + ring: + kvstore: + store: inmemory + replication_factor: 1 + +blocks_storage: + tsdb: + dir: /tmp/cortex/tsdb + bucket_store: + sync_dir: /tmp/cortex/tsdb-sync + backend: filesystem + filesystem: + dir: /tmp/cortex/data/tsdb + +compactor: + data_dir: /tmp/cortex/compactor + sharding_ring: + kvstore: + store: inmemory + +frontend_worker: + match_max_concurrent: true + +ruler: + enable_api: true + +ruler_storage: + backend: filesystem + filesystem: + dir: /tmp/cortex/rules + +alertmanager: + enable_api: true + external_url: http://localhost:9009/alertmanager + sharding_ring: + kvstore: + store: inmemory + replication_factor: 1 + +alertmanager_storage: + backend: filesystem + filesystem: + dir: /tmp/cortex/alerts diff --git a/integration/integration_test.go b/integration/integration_test.go new file mode 100644 index 000000000..475997dfb --- /dev/null +++ b/integration/integration_test.go @@ -0,0 +1,127 @@ +//go:build integration + +package integration + +import ( + "context" + "errors" + "os" + "testing" + + "github.com/prometheus/prometheus/model/rulefmt" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/cortexproject/cortex-tools/pkg/client" + "github.com/cortexproject/cortex-tools/pkg/rules/rwrulefmt" +) + +func cortexAddress() string { + if addr := os.Getenv("CORTEX_ADDRESS"); addr != "" { + return addr + } + return "http://localhost:9009" +} + +func newClient(t *testing.T) *client.CortexClient { + t.Helper() + c, err := client.New(client.Config{ + Address: cortexAddress(), + ID: "fake", + }) + require.NoError(t, err) + return c +} + +func ruleNode(record, expr string) rulefmt.RuleNode { + return rulefmt.RuleNode{ + Record: yaml.Node{Kind: yaml.ScalarNode, Value: record}, + Expr: yaml.Node{Kind: yaml.ScalarNode, Value: expr}, + } +} + +func TestRulesLoadListDelete(t *testing.T) { + ctx := context.Background() + c := newClient(t) + + namespace := "test_namespace" + group := rwrulefmt.RuleGroup{} + group.Name = "test_rule_group" + group.Rules = []rulefmt.RuleNode{ + ruleNode("summed_up", "sum(up)"), + } + + err := c.CreateRuleGroup(ctx, namespace, group) + require.NoError(t, err, "CreateRuleGroup should succeed") + + ruleSet, err := c.ListRules(ctx, "") + require.NoError(t, err, "ListRules should succeed") + require.Contains(t, ruleSet, namespace, "namespace should exist") + + found := false + for _, g := range ruleSet[namespace] { + if g.Name == "test_rule_group" { + found = true + break + } + } + require.True(t, found, "rule group should be in list") + + rg, err := c.GetRuleGroup(ctx, namespace, "test_rule_group") + require.NoError(t, err, "GetRuleGroup should succeed") + require.Equal(t, "test_rule_group", rg.Name) + require.Len(t, rg.Rules, 1) + + err = c.DeleteRuleNamespace(ctx, namespace) + require.NoError(t, err, "DeleteRuleNamespace should succeed") + + ruleSet, err = c.ListRules(ctx, "") + if err != nil { + require.True(t, errors.Is(err, client.ErrResourceNotFound), "expected no rules or resource not found, got: %v", err) + } else { + require.NotContains(t, ruleSet, namespace, "namespace should be deleted") + } +} + +func TestRulesMultipleGroups(t *testing.T) { + ctx := context.Background() + c := newClient(t) + + namespace := "multi_group_namespace" + groups := []rwrulefmt.RuleGroup{ + {RuleGroup: rulefmt.RuleGroup{Name: "group_a", Rules: []rulefmt.RuleNode{ruleNode("metric_a", "sum(up)")}}}, + {RuleGroup: rulefmt.RuleGroup{Name: "group_b", Rules: []rulefmt.RuleNode{ruleNode("metric_b", "count(up)")}}}, + {RuleGroup: rulefmt.RuleGroup{Name: "group_c", Rules: []rulefmt.RuleNode{ruleNode("metric_c", "avg(up)")}}}, + } + + for _, g := range groups { + err := c.CreateRuleGroup(ctx, namespace, g) + require.NoError(t, err, "CreateRuleGroup %s should succeed", g.Name) + } + + ruleSet, err := c.ListRules(ctx, namespace) + require.NoError(t, err) + require.Contains(t, ruleSet, namespace) + require.Len(t, ruleSet[namespace], 3, "should have 3 rule groups") + + err = c.DeleteRuleNamespace(ctx, namespace) + require.NoError(t, err) +} + +func TestAlertmanagerLoadGet(t *testing.T) { + ctx := context.Background() + c := newClient(t) + + amConfig := "route:\n receiver: default\nreceivers:\n - name: default\n" + + err := c.CreateAlertmanagerConfig(ctx, amConfig, nil) + require.NoError(t, err, "CreateAlertmanagerConfig should succeed") + + cfg, templates, err := c.GetAlertmanagerConfig(ctx) + require.NoError(t, err, "GetAlertmanagerConfig should succeed") + require.Contains(t, cfg, "receiver: default") + require.Empty(t, templates) + + err = c.DeleteAlermanagerConfig(ctx) + require.NoError(t, err, "DeleteAlermanagerConfig should succeed") +} From 2836f87692a44f51f8fea3ce1d672ff1c4370e04 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Fri, 24 Apr 2026 18:57:23 -0700 Subject: [PATCH 2/9] Test Remote read too Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- integration/integration_test.go | 222 ++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) diff --git a/integration/integration_test.go b/integration/integration_test.go index 475997dfb..7d5f7eab4 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -3,15 +3,31 @@ package integration import ( + "bytes" "context" "errors" + "fmt" + "io" + "net/http" + "net/url" "os" + "path/filepath" "testing" + "time" + "github.com/gogo/protobuf/proto" + "github.com/golang/snappy" + config_util "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/rulefmt" + "github.com/prometheus/prometheus/prompb" + "github.com/prometheus/prometheus/promql/parser" + "github.com/prometheus/prometheus/storage/remote" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" + "github.com/cortexproject/cortex-tools/pkg/backfill" "github.com/cortexproject/cortex-tools/pkg/client" "github.com/cortexproject/cortex-tools/pkg/rules/rwrulefmt" ) @@ -125,3 +141,209 @@ func TestAlertmanagerLoadGet(t *testing.T) { err = c.DeleteAlermanagerConfig(ctx) require.NoError(t, err, "DeleteAlermanagerConfig should succeed") } + +func remoteWrite(t *testing.T, series []prompb.TimeSeries) { + t.Helper() + + writeReq := &prompb.WriteRequest{Timeseries: series} + data, err := proto.Marshal(writeReq) + require.NoError(t, err) + + compressed := snappy.Encode(nil, data) + + writeURL := cortexAddress() + "/api/v1/push" + req, err := http.NewRequest("POST", writeURL, bytes.NewReader(compressed)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Content-Encoding", "snappy") + req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0") + req.Header.Set("X-Scope-OrgID", "fake") + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode, "remote write should return 200") +} + +func TestRemoteRead(t *testing.T) { + now := time.Now() + + series := []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "integration_test_metric"}, + {Name: "job", Value: "test"}, + }, + Samples: []prompb.Sample{ + {Value: 42.0, Timestamp: now.UnixMilli()}, + {Value: 43.0, Timestamp: now.Add(-30 * time.Second).UnixMilli()}, + {Value: 44.0, Timestamp: now.Add(-60 * time.Second).UnixMilli()}, + }, + }, + } + + remoteWrite(t, series) + + _, result := remoteReadQuery(t, `integration_test_metric`, now.Add(-5*time.Minute), now.Add(time.Minute)) + require.NotEmpty(t, result.Timeseries, "should have at least one timeseries") + + ts := result.Timeseries[0] + foundMetricName := false + for _, l := range ts.Labels { + if l.Name == "__name__" && l.Value == "integration_test_metric" { + foundMetricName = true + } + } + require.True(t, foundMetricName, "timeseries should have the correct metric name") + require.GreaterOrEqual(t, len(ts.Samples), 1, "should have samples") + + fmt.Printf("Remote read returned %d timeseries with %d samples\n", len(result.Timeseries), len(ts.Samples)) +} + +func TestRemoteReadExport(t *testing.T) { + now := time.Now() + + series := []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "export_test_metric"}, + {Name: "job", Value: "export_test"}, + }, + Samples: []prompb.Sample{ + {Value: 100.0, Timestamp: now.UnixMilli()}, + }, + }, + } + + remoteWrite(t, series) + + // Read the data back + readClient, result := remoteReadQuery(t, `export_test_metric`, now.Add(-5*time.Minute), now.Add(time.Minute)) + _ = readClient + require.NotEmpty(t, result.Timeseries) + + // Export to TSDB using backfill.CreateBlocks + tsdbPath, err := os.MkdirTemp("", "cortex-integration-export-*") + require.NoError(t, err) + defer os.RemoveAll(tsdbPath) + + from := now.Add(-5 * time.Minute) + to := now.Add(time.Minute) + mint := int64(model.TimeFromUnixNano(from.UnixNano())) + maxt := int64(model.TimeFromUnixNano(to.UnixNano())) + + timeseries := result.Timeseries + iterator := func() backfill.Iterator { + return newTimeSeriesIterator(timeseries) + } + + err = backfill.CreateBlocks(iterator, mint, maxt, 1000, tsdbPath, false, io.Discard) + require.NoError(t, err, "CreateBlocks should succeed") + + // Verify TSDB blocks were created + entries, err := os.ReadDir(tsdbPath) + require.NoError(t, err) + blockCount := 0 + for _, e := range entries { + if e.IsDir() && e.Name() != "wal" { + blockCount++ + } + } + require.GreaterOrEqual(t, blockCount, 1, "should have created at least one TSDB block") + fmt.Printf("Export created %d TSDB blocks in %s\n", blockCount, tsdbPath) +} + +func remoteReadQuery(t *testing.T, selector string, from, to time.Time) (remote.ReadClient, *prompb.QueryResult) { + t.Helper() + + addressURL, err := url.Parse(cortexAddress()) + require.NoError(t, err) + addressURL.Path = filepath.Join(addressURL.Path, "/prometheus/api/v1/read") + + readClient, err := remote.NewReadClient("test", &remote.ClientConfig{ + URL: &config_util.URL{URL: addressURL}, + Timeout: model.Duration(30 * time.Second), + }) + require.NoError(t, err) + + rc := readClient.(*remote.Client) + rc.Client.Transport = &tenantIDTransport{ + RoundTripper: http.DefaultTransport, + tenantID: "fake", + } + + matchers, err := parser.ParseMetricSelector(selector) + require.NoError(t, err) + + pbQuery, err := remote.ToQuery( + int64(model.TimeFromUnixNano(from.UnixNano())), + int64(model.TimeFromUnixNano(to.UnixNano())), + matchers, + nil, + ) + require.NoError(t, err) + + result, err := readClient.Read(context.Background(), pbQuery) + require.NoError(t, err) + return readClient, result +} + +func newTimeSeriesIterator(ts []*prompb.TimeSeries) *timeSeriesIterator { + return &timeSeriesIterator{ + posSeries: 0, + posSample: -1, + labelsSeriesPos: -1, + ts: ts, + } +} + +type timeSeriesIterator struct { + posSeries int + posSample int + ts []*prompb.TimeSeries + labels labels.Labels + labelsSeriesPos int +} + +func (i *timeSeriesIterator) Next() error { + if i.posSeries >= len(i.ts) { + return io.EOF + } + i.posSample++ + if i.posSample >= len(i.ts[i.posSeries].Samples) { + i.posSample = -1 + i.posSeries++ + return i.Next() + } + return nil +} + +func (i *timeSeriesIterator) Labels() labels.Labels { + if i.posSeries == i.labelsSeriesPos { + return i.labels + } + series := i.ts[i.posSeries] + i.labels = make(labels.Labels, len(series.Labels)) + for idx := range series.Labels { + i.labels[idx].Name = series.Labels[idx].Name + i.labels[idx].Value = series.Labels[idx].Value + } + i.labelsSeriesPos = i.posSeries + return i.labels +} + +func (i *timeSeriesIterator) Sample() (int64, float64) { + series := i.ts[i.posSeries] + sample := series.Samples[i.posSample] + return sample.GetTimestamp(), sample.GetValue() +} + +type tenantIDTransport struct { + http.RoundTripper + tenantID string +} + +func (t *tenantIDTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("X-Scope-OrgID", t.tenantID) + return t.RoundTripper.RoundTrip(req) +} From 8169e16a761292f38b4f4a6e096d9c3e90979a69 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Fri, 24 Apr 2026 18:58:30 -0700 Subject: [PATCH 3/9] Potential fix for pull request finding 'CodeQL / Workflow does not contain permissions' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- .github/workflows/integration_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml index f2d52b499..eea1d825c 100644 --- a/.github/workflows/integration_test.yml +++ b/.github/workflows/integration_test.yml @@ -4,6 +4,8 @@ on: paths-ignore: - "docs/**" - "*.md" +permissions: + contents: read jobs: integration: name: Integration Tests From 4ac04be7a3a1ad4a7588dd74fdadd855eb9851e1 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Fri, 24 Apr 2026 19:08:18 -0700 Subject: [PATCH 4/9] test loadgen Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- integration/integration_test.go | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/integration/integration_test.go b/integration/integration_test.go index 7d5f7eab4..ef1ce34d0 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -28,6 +28,7 @@ import ( "gopkg.in/yaml.v3" "github.com/cortexproject/cortex-tools/pkg/backfill" + "github.com/cortexproject/cortex-tools/pkg/bench" "github.com/cortexproject/cortex-tools/pkg/client" "github.com/cortexproject/cortex-tools/pkg/rules/rwrulefmt" ) @@ -253,6 +254,43 @@ func TestRemoteReadExport(t *testing.T) { fmt.Printf("Export created %d TSDB blocks in %s\n", blockCount, tsdbPath) } +func TestLoadgen(t *testing.T) { + now := time.Now() + + seriesDescs := []bench.SeriesDesc{ + { + Name: "loadgen_test_metric", + Type: bench.GaugeRandom, + Labels: []bench.LabelDesc{ + {Name: "instance", ValuePrefix: "inst", UniqueValues: 2}, + }, + }, + } + + series, totalSeriesTypeMap := bench.SeriesDescToSeries(seriesDescs) + totalSeries := 0 + for _, n := range totalSeriesTypeMap { + totalSeries += n + } + + workload := &bench.WriteWorkload{ + Replicas: 1, + Series: series, + TotalSeries: totalSeries, + TotalSeriesTypeMap: totalSeriesTypeMap, + } + + timeseries := workload.GenerateTimeSeries("integration-test", now) + require.NotEmpty(t, timeseries, "workload should generate timeseries") + + remoteWrite(t, timeseries) + + _, result := remoteReadQuery(t, `{__name__="loadgen_test_metric"}`, now.Add(-5*time.Minute), now.Add(time.Minute)) + require.NotEmpty(t, result.Timeseries, "should read back loadgen timeseries") + + fmt.Printf("Loadgen wrote %d series, read back %d timeseries\n", len(timeseries), len(result.Timeseries)) +} + func remoteReadQuery(t *testing.T, selector string, from, to time.Time) (remote.ReadClient, *prompb.QueryResult) { t.Helper() From 7671961004de355862b4d0668394aa9c167d5d1d Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Fri, 24 Apr 2026 19:14:00 -0700 Subject: [PATCH 5/9] Delete alertmanager config Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- integration/integration_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/integration/integration_test.go b/integration/integration_test.go index ef1ce34d0..6b597e541 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -143,6 +143,32 @@ func TestAlertmanagerLoadGet(t *testing.T) { require.NoError(t, err, "DeleteAlermanagerConfig should succeed") } +func TestAlertmanagerWithTemplates(t *testing.T) { + ctx := context.Background() + c := newClient(t) + + amConfig := "route:\n receiver: default\nreceivers:\n - name: default\n" + tmpl := map[string]string{ + "slack.tmpl": "{{ define \"slack.title\" }}Alert: {{ .CommonLabels.alertname }}{{ end }}", + } + + err := c.CreateAlertmanagerConfig(ctx, amConfig, tmpl) + require.NoError(t, err, "CreateAlertmanagerConfig with templates should succeed") + + cfg, templates, err := c.GetAlertmanagerConfig(ctx) + require.NoError(t, err, "GetAlertmanagerConfig should succeed") + require.Contains(t, cfg, "receiver: default") + require.Contains(t, templates, "slack.tmpl") + require.Contains(t, templates["slack.tmpl"], "slack.title") + + err = c.DeleteAlermanagerConfig(ctx) + require.NoError(t, err, "DeleteAlermanagerConfig should succeed") + + // Verify config is gone + _, _, err = c.GetAlertmanagerConfig(ctx) + require.Error(t, err, "GetAlertmanagerConfig should fail after delete") +} + func remoteWrite(t *testing.T, series []prompb.TimeSeries) { t.Helper() From 767ba1f649541c35c42095a7bd181f730f99c60e Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Fri, 24 Apr 2026 19:15:42 -0700 Subject: [PATCH 6/9] More rules group Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- integration/integration_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/integration/integration_test.go b/integration/integration_test.go index 6b597e541..c21bed55a 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -121,6 +121,23 @@ func TestRulesMultipleGroups(t *testing.T) { require.Contains(t, ruleSet, namespace) require.Len(t, ruleSet[namespace], 3, "should have 3 rule groups") + // Delete a single group + err = c.DeleteRuleGroup(ctx, namespace, "group_b") + require.NoError(t, err, "DeleteRuleGroup should succeed") + + ruleSet, err = c.ListRules(ctx, namespace) + require.NoError(t, err) + require.Len(t, ruleSet[namespace], 2, "should have 2 rule groups after deleting one") + + groupNames := make([]string, 0, len(ruleSet[namespace])) + for _, g := range ruleSet[namespace] { + groupNames = append(groupNames, g.Name) + } + require.NotContains(t, groupNames, "group_b", "deleted group should be gone") + require.Contains(t, groupNames, "group_a") + require.Contains(t, groupNames, "group_c") + + // Clean up remaining err = c.DeleteRuleNamespace(ctx, namespace) require.NoError(t, err) } From 546036c4f9e554eea432f690b0260815a2fd27d9 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Fri, 24 Apr 2026 19:31:33 -0700 Subject: [PATCH 7/9] Test rules lint Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- integration/rules_offline_test.go | 208 ++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 integration/rules_offline_test.go diff --git a/integration/rules_offline_test.go b/integration/rules_offline_test.go new file mode 100644 index 000000000..e84e63ea9 --- /dev/null +++ b/integration/rules_offline_test.go @@ -0,0 +1,208 @@ +//go:build integration + +package integration + +import ( + "testing" + + "github.com/prometheus/prometheus/model/rulefmt" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/cortexproject/cortex-tools/pkg/rules" + "github.com/cortexproject/cortex-tools/pkg/rules/rwrulefmt" +) + +func TestRulesLint(t *testing.T) { + t.Run("valid expression is unchanged", func(t *testing.T) { + ns := rules.RuleNamespace{ + Groups: []rwrulefmt.RuleGroup{ + {RuleGroup: rulefmt.RuleGroup{ + Name: "test", + Rules: []rulefmt.RuleNode{ruleNode("test_metric", "sum(up)")}, + }}, + }, + } + + count, mod, err := ns.LintExpressions() + require.NoError(t, err) + require.Equal(t, 1, count, "should evaluate 1 rule") + require.Equal(t, 0, mod, "canonical expression should not be modified") + }) + + t.Run("expression is reformatted", func(t *testing.T) { + ns := rules.RuleNamespace{ + Groups: []rwrulefmt.RuleGroup{ + {RuleGroup: rulefmt.RuleGroup{ + Name: "test", + Rules: []rulefmt.RuleNode{ + ruleNode("test_metric", "sum by(job)(up)"), + }, + }}, + }, + } + + count, mod, err := ns.LintExpressions() + require.NoError(t, err) + require.Equal(t, 1, count) + require.Equal(t, 1, mod, "expression with extra whitespace should be reformatted") + require.Equal(t, "sum by (job) (up)", ns.Groups[0].Rules[0].Expr.Value) + }) + + t.Run("invalid expression returns error", func(t *testing.T) { + ns := rules.RuleNamespace{ + Groups: []rwrulefmt.RuleGroup{ + {RuleGroup: rulefmt.RuleGroup{ + Name: "test", + Rules: []rulefmt.RuleNode{ruleNode("test_metric", "not a valid expr !!!")}, + }}, + }, + } + + _, _, err := ns.LintExpressions() + require.Error(t, err) + }) +} + +func TestRulesCheck(t *testing.T) { + t.Run("valid recording rule name", func(t *testing.T) { + ns := rules.RuleNamespace{ + Groups: []rwrulefmt.RuleGroup{ + {RuleGroup: rulefmt.RuleGroup{ + Name: "test", + Rules: []rulefmt.RuleNode{ruleNode("level:metric:operation", "sum(up)")}, + }}, + }, + } + + count := ns.CheckRecordingRules(false) + require.Equal(t, 0, count, "valid recording rule name should pass") + }) + + t.Run("invalid recording rule name", func(t *testing.T) { + ns := rules.RuleNamespace{ + Groups: []rwrulefmt.RuleGroup{ + {RuleGroup: rulefmt.RuleGroup{ + Name: "test", + Rules: []rulefmt.RuleNode{ruleNode("no_colons_here", "sum(up)")}, + }}, + }, + } + + count := ns.CheckRecordingRules(false) + require.Equal(t, 1, count, "recording rule without colon should fail") + }) + + t.Run("strict mode requires two colons", func(t *testing.T) { + ns := rules.RuleNamespace{ + Groups: []rwrulefmt.RuleGroup{ + {RuleGroup: rulefmt.RuleGroup{ + Name: "test", + Rules: []rulefmt.RuleNode{ruleNode("level:metric", "sum(up)")}, + }}, + }, + } + + count := ns.CheckRecordingRules(true) + require.Equal(t, 1, count, "strict mode should require level:metric:operation format") + + countNonStrict := ns.CheckRecordingRules(false) + require.Equal(t, 0, countNonStrict, "non-strict should pass with one colon") + }) +} + +func TestRulesPrepare(t *testing.T) { + t.Run("adds aggregation label", func(t *testing.T) { + ns := rules.RuleNamespace{ + Groups: []rwrulefmt.RuleGroup{ + {RuleGroup: rulefmt.RuleGroup{ + Name: "test", + Rules: []rulefmt.RuleNode{ + { + Record: yaml.Node{Kind: yaml.ScalarNode, Value: "test_metric"}, + Expr: yaml.Node{Kind: yaml.ScalarNode, Value: "sum by (job) (up)"}, + }, + }, + }}, + }, + } + + count, mod, err := ns.AggregateBy("cluster", nil) + require.NoError(t, err) + require.Equal(t, 1, count) + require.Equal(t, 1, mod, "should modify expression to include cluster label") + require.Contains(t, ns.Groups[0].Rules[0].Expr.Value, "cluster") + }) + + t.Run("skips if label already present", func(t *testing.T) { + ns := rules.RuleNamespace{ + Groups: []rwrulefmt.RuleGroup{ + {RuleGroup: rulefmt.RuleGroup{ + Name: "test", + Rules: []rulefmt.RuleNode{ + { + Record: yaml.Node{Kind: yaml.ScalarNode, Value: "test_metric"}, + Expr: yaml.Node{Kind: yaml.ScalarNode, Value: "sum by (job, cluster) (up)"}, + }, + }, + }}, + }, + } + + count, mod, err := ns.AggregateBy("cluster", nil) + require.NoError(t, err) + require.Equal(t, 1, count) + require.Equal(t, 0, mod, "should not modify when label already present") + }) + + t.Run("respects applyTo filter", func(t *testing.T) { + ns := rules.RuleNamespace{ + Groups: []rwrulefmt.RuleGroup{ + {RuleGroup: rulefmt.RuleGroup{ + Name: "excluded_group", + Rules: []rulefmt.RuleNode{ + { + Record: yaml.Node{Kind: yaml.ScalarNode, Value: "test_metric"}, + Expr: yaml.Node{Kind: yaml.ScalarNode, Value: "sum by (job) (up)"}, + }, + }, + }}, + }, + } + + applyTo := func(group rwrulefmt.RuleGroup, _ rulefmt.RuleNode) bool { + return group.Name != "excluded_group" + } + + count, mod, err := ns.AggregateBy("cluster", applyTo) + require.NoError(t, err) + require.Equal(t, 1, count) + require.Equal(t, 0, mod, "excluded group should not be modified") + }) +} + +func TestRulesParseFile(t *testing.T) { + t.Run("parse basic namespace", func(t *testing.T) { + nss, errs := rules.Parse("../pkg/rules/testdata/basic_namespace.yaml") + require.Empty(t, errs) + require.Len(t, nss, 1) + require.Equal(t, "example_namespace", nss[0].Namespace) + require.Len(t, nss[0].Groups, 1) + require.Equal(t, "example_rule_group", nss[0].Groups[0].Name) + }) + + t.Run("parse multiple namespaces", func(t *testing.T) { + nss, errs := rules.Parse("../pkg/rules/testdata/multiple_namespace.yaml") + require.Empty(t, errs) + require.Len(t, nss, 2) + require.Equal(t, "example_namespace", nss[0].Namespace) + require.Equal(t, "other_example_namespace", nss[1].Namespace) + }) + + t.Run("validate catches errors", func(t *testing.T) { + content := []byte("groups:\n- name: \"\"\n rules:\n - expr: sum(up)\n record: test\n") + nss, errs := rules.ParseBytes(content) + require.NotEmpty(t, errs, "empty group name should fail validation") + require.Nil(t, nss) + }) +} From 2f102eff0dd88da1b23b47119d73f9ace91b894d Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Fri, 24 Apr 2026 19:33:24 -0700 Subject: [PATCH 8/9] Document integration tests Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- integration/README.md | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 integration/README.md diff --git a/integration/README.md b/integration/README.md new file mode 100644 index 000000000..5eb9da0c2 --- /dev/null +++ b/integration/README.md @@ -0,0 +1,46 @@ +# Integration Tests + +## Running + +```bash +# Start Cortex +docker run -d --name cortex-test -p 9009:9009 \ + -v $(pwd)/integration/cortex-config.yaml:/etc/cortex/config.yaml \ + cortexproject/cortex:v1.18.1 \ + -config.file=/etc/cortex/config.yaml \ + -target=all,alertmanager + +# Wait for ready +until curl -s http://localhost:9009/ready > /dev/null 2>&1; do sleep 2; done && echo "Ready" + +# Run tests +go test -mod=vendor -tags=integration -v -count=1 ./integration/... + +# Cleanup +docker stop cortex-test && docker rm cortex-test +``` + +## Coverage + +| Command | Needs Cortex? | Tested? | +|---------|:---:|:---:| +| **rules load/list/print/get** | Yes | Yes | +| **rules delete (group)** | Yes | Yes | +| **rules delete-namespace** | Yes | Yes | +| **rules lint** | No | Yes | +| **rules check** | No | Yes | +| **rules prepare** | No | Yes | +| **rules parse/validate** | No | Yes | +| **alertmanager load** | Yes | Yes | +| **alertmanager load (with templates)** | Yes | Yes | +| **alertmanager get** | Yes | Yes | +| **alertmanager delete** | Yes | Yes | +| **remote-read dump/stats** | Yes | Yes | +| **remote-read export** | Yes | Yes | +| **loadgen (write workload)** | Yes | Yes | +| **analyse** (grafana/ruler/dashboard/rule-file) | No | No | +| **acl** | No | No | +| **bucket-validation** | Needs object store | No | +| **config** (use-context) | No | No | +| **alert verify** | Needs Cortex + alerting | No | +| **push-gateway** | Needs push gateway | No | From 28d7afa8c3f16abb622bbb5a7763e7638422c5b7 Mon Sep 17 00:00:00 2001 From: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> Date: Fri, 24 Apr 2026 19:43:49 -0700 Subject: [PATCH 9/9] Removing duplicated tests Signed-off-by: Friedrich Gonzalez <1517449+friedrichg@users.noreply.github.com> --- integration/README.md | 41 +++--- integration/rules_offline_test.go | 208 ------------------------------ 2 files changed, 19 insertions(+), 230 deletions(-) delete mode 100644 integration/rules_offline_test.go diff --git a/integration/README.md b/integration/README.md index 5eb9da0c2..295e06080 100644 --- a/integration/README.md +++ b/integration/README.md @@ -22,25 +22,22 @@ docker stop cortex-test && docker rm cortex-test ## Coverage -| Command | Needs Cortex? | Tested? | -|---------|:---:|:---:| -| **rules load/list/print/get** | Yes | Yes | -| **rules delete (group)** | Yes | Yes | -| **rules delete-namespace** | Yes | Yes | -| **rules lint** | No | Yes | -| **rules check** | No | Yes | -| **rules prepare** | No | Yes | -| **rules parse/validate** | No | Yes | -| **alertmanager load** | Yes | Yes | -| **alertmanager load (with templates)** | Yes | Yes | -| **alertmanager get** | Yes | Yes | -| **alertmanager delete** | Yes | Yes | -| **remote-read dump/stats** | Yes | Yes | -| **remote-read export** | Yes | Yes | -| **loadgen (write workload)** | Yes | Yes | -| **analyse** (grafana/ruler/dashboard/rule-file) | No | No | -| **acl** | No | No | -| **bucket-validation** | Needs object store | No | -| **config** (use-context) | No | No | -| **alert verify** | Needs Cortex + alerting | No | -| **push-gateway** | Needs push gateway | No | +These integration tests require a running Cortex instance. Offline commands (lint, check, prepare, parse) are covered by unit tests in `pkg/rules/`. + +| Command | Tested? | +|---------|:---:| +| **rules load/list/print/get** | Yes | +| **rules delete (group)** | Yes | +| **rules delete-namespace** | Yes | +| **alertmanager load** | Yes | +| **alertmanager load (with templates)** | Yes | +| **alertmanager get** | Yes | +| **alertmanager delete** | Yes | +| **remote-read dump/stats** | Yes | +| **remote-read export** | Yes | +| **loadgen (write workload)** | Yes | +| **analyse** (grafana/ruler/dashboard/rule-file) | No | +| **bucket-validation** | No | +| **config** (use-context) | No | +| **alert verify** | No | +| **push-gateway** | No | diff --git a/integration/rules_offline_test.go b/integration/rules_offline_test.go deleted file mode 100644 index e84e63ea9..000000000 --- a/integration/rules_offline_test.go +++ /dev/null @@ -1,208 +0,0 @@ -//go:build integration - -package integration - -import ( - "testing" - - "github.com/prometheus/prometheus/model/rulefmt" - "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" - - "github.com/cortexproject/cortex-tools/pkg/rules" - "github.com/cortexproject/cortex-tools/pkg/rules/rwrulefmt" -) - -func TestRulesLint(t *testing.T) { - t.Run("valid expression is unchanged", func(t *testing.T) { - ns := rules.RuleNamespace{ - Groups: []rwrulefmt.RuleGroup{ - {RuleGroup: rulefmt.RuleGroup{ - Name: "test", - Rules: []rulefmt.RuleNode{ruleNode("test_metric", "sum(up)")}, - }}, - }, - } - - count, mod, err := ns.LintExpressions() - require.NoError(t, err) - require.Equal(t, 1, count, "should evaluate 1 rule") - require.Equal(t, 0, mod, "canonical expression should not be modified") - }) - - t.Run("expression is reformatted", func(t *testing.T) { - ns := rules.RuleNamespace{ - Groups: []rwrulefmt.RuleGroup{ - {RuleGroup: rulefmt.RuleGroup{ - Name: "test", - Rules: []rulefmt.RuleNode{ - ruleNode("test_metric", "sum by(job)(up)"), - }, - }}, - }, - } - - count, mod, err := ns.LintExpressions() - require.NoError(t, err) - require.Equal(t, 1, count) - require.Equal(t, 1, mod, "expression with extra whitespace should be reformatted") - require.Equal(t, "sum by (job) (up)", ns.Groups[0].Rules[0].Expr.Value) - }) - - t.Run("invalid expression returns error", func(t *testing.T) { - ns := rules.RuleNamespace{ - Groups: []rwrulefmt.RuleGroup{ - {RuleGroup: rulefmt.RuleGroup{ - Name: "test", - Rules: []rulefmt.RuleNode{ruleNode("test_metric", "not a valid expr !!!")}, - }}, - }, - } - - _, _, err := ns.LintExpressions() - require.Error(t, err) - }) -} - -func TestRulesCheck(t *testing.T) { - t.Run("valid recording rule name", func(t *testing.T) { - ns := rules.RuleNamespace{ - Groups: []rwrulefmt.RuleGroup{ - {RuleGroup: rulefmt.RuleGroup{ - Name: "test", - Rules: []rulefmt.RuleNode{ruleNode("level:metric:operation", "sum(up)")}, - }}, - }, - } - - count := ns.CheckRecordingRules(false) - require.Equal(t, 0, count, "valid recording rule name should pass") - }) - - t.Run("invalid recording rule name", func(t *testing.T) { - ns := rules.RuleNamespace{ - Groups: []rwrulefmt.RuleGroup{ - {RuleGroup: rulefmt.RuleGroup{ - Name: "test", - Rules: []rulefmt.RuleNode{ruleNode("no_colons_here", "sum(up)")}, - }}, - }, - } - - count := ns.CheckRecordingRules(false) - require.Equal(t, 1, count, "recording rule without colon should fail") - }) - - t.Run("strict mode requires two colons", func(t *testing.T) { - ns := rules.RuleNamespace{ - Groups: []rwrulefmt.RuleGroup{ - {RuleGroup: rulefmt.RuleGroup{ - Name: "test", - Rules: []rulefmt.RuleNode{ruleNode("level:metric", "sum(up)")}, - }}, - }, - } - - count := ns.CheckRecordingRules(true) - require.Equal(t, 1, count, "strict mode should require level:metric:operation format") - - countNonStrict := ns.CheckRecordingRules(false) - require.Equal(t, 0, countNonStrict, "non-strict should pass with one colon") - }) -} - -func TestRulesPrepare(t *testing.T) { - t.Run("adds aggregation label", func(t *testing.T) { - ns := rules.RuleNamespace{ - Groups: []rwrulefmt.RuleGroup{ - {RuleGroup: rulefmt.RuleGroup{ - Name: "test", - Rules: []rulefmt.RuleNode{ - { - Record: yaml.Node{Kind: yaml.ScalarNode, Value: "test_metric"}, - Expr: yaml.Node{Kind: yaml.ScalarNode, Value: "sum by (job) (up)"}, - }, - }, - }}, - }, - } - - count, mod, err := ns.AggregateBy("cluster", nil) - require.NoError(t, err) - require.Equal(t, 1, count) - require.Equal(t, 1, mod, "should modify expression to include cluster label") - require.Contains(t, ns.Groups[0].Rules[0].Expr.Value, "cluster") - }) - - t.Run("skips if label already present", func(t *testing.T) { - ns := rules.RuleNamespace{ - Groups: []rwrulefmt.RuleGroup{ - {RuleGroup: rulefmt.RuleGroup{ - Name: "test", - Rules: []rulefmt.RuleNode{ - { - Record: yaml.Node{Kind: yaml.ScalarNode, Value: "test_metric"}, - Expr: yaml.Node{Kind: yaml.ScalarNode, Value: "sum by (job, cluster) (up)"}, - }, - }, - }}, - }, - } - - count, mod, err := ns.AggregateBy("cluster", nil) - require.NoError(t, err) - require.Equal(t, 1, count) - require.Equal(t, 0, mod, "should not modify when label already present") - }) - - t.Run("respects applyTo filter", func(t *testing.T) { - ns := rules.RuleNamespace{ - Groups: []rwrulefmt.RuleGroup{ - {RuleGroup: rulefmt.RuleGroup{ - Name: "excluded_group", - Rules: []rulefmt.RuleNode{ - { - Record: yaml.Node{Kind: yaml.ScalarNode, Value: "test_metric"}, - Expr: yaml.Node{Kind: yaml.ScalarNode, Value: "sum by (job) (up)"}, - }, - }, - }}, - }, - } - - applyTo := func(group rwrulefmt.RuleGroup, _ rulefmt.RuleNode) bool { - return group.Name != "excluded_group" - } - - count, mod, err := ns.AggregateBy("cluster", applyTo) - require.NoError(t, err) - require.Equal(t, 1, count) - require.Equal(t, 0, mod, "excluded group should not be modified") - }) -} - -func TestRulesParseFile(t *testing.T) { - t.Run("parse basic namespace", func(t *testing.T) { - nss, errs := rules.Parse("../pkg/rules/testdata/basic_namespace.yaml") - require.Empty(t, errs) - require.Len(t, nss, 1) - require.Equal(t, "example_namespace", nss[0].Namespace) - require.Len(t, nss[0].Groups, 1) - require.Equal(t, "example_rule_group", nss[0].Groups[0].Name) - }) - - t.Run("parse multiple namespaces", func(t *testing.T) { - nss, errs := rules.Parse("../pkg/rules/testdata/multiple_namespace.yaml") - require.Empty(t, errs) - require.Len(t, nss, 2) - require.Equal(t, "example_namespace", nss[0].Namespace) - require.Equal(t, "other_example_namespace", nss[1].Namespace) - }) - - t.Run("validate catches errors", func(t *testing.T) { - content := []byte("groups:\n- name: \"\"\n rules:\n - expr: sum(up)\n record: test\n") - nss, errs := rules.ParseBytes(content) - require.NotEmpty(t, errs, "empty group name should fail validation") - require.Nil(t, nss) - }) -}