From 05b35dd4eeacc16ba7efb9974eda4f62caee1d98 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Sun, 19 Jul 2026 23:13:47 +0530 Subject: [PATCH 1/5] fix(clickhouseprometheus): merge fingerprints sharing a labelset instead of injecting a fingerprint label (#12152) #8563 worked around duplicate-labelset collisions (a label value going empty re-fingerprints a series; empty-valued labels are dropped at read) by appending a synthetic fingerprint label to every series and stripping it from results. The label participates in evaluation before any strip runs: without() grouped per fingerprint (duplicate identical-labeled output series) and unaggregated vector matching never matched. Resolve series identity at the adapter instead: drop empty-valued labels at unmarshal, merge fingerprints that share the resulting labelset (timestamp-ordered k-way merge, highest fingerprint wins equal timestamps), and delete the injection and every RemoveExtraLabels strip. Engine-semantic duplicates (e.g. rate over {__name__=~"a|b"}) still error, matching upstream Prometheus. Co-authored-by: Claude Fable 5 --- pkg/prometheus/clickhouseprometheus/client.go | 136 +++++++++++++++++- .../clickhouseprometheus/client_query_test.go | 90 +++++++++++- pkg/prometheus/clickhouseprometheus/json.go | 14 +- .../clickhouseprometheus/merge_test.go | 82 +++++++++++ pkg/prometheus/label.go | 39 ----- pkg/prometheus/prometheustest/utils_test.go | 107 -------------- .../app/clickhouseReader/reader.go | 8 -- pkg/query-service/rules/prom_rule.go | 5 - 8 files changed, 310 insertions(+), 171 deletions(-) create mode 100644 pkg/prometheus/clickhouseprometheus/merge_test.go delete mode 100644 pkg/prometheus/label.go delete mode 100644 pkg/prometheus/prometheustest/utils_test.go diff --git a/pkg/prometheus/clickhouseprometheus/client.go b/pkg/prometheus/clickhouseprometheus/client.go index b911ee7de57..821509e792e 100644 --- a/pkg/prometheus/clickhouseprometheus/client.go +++ b/pkg/prometheus/clickhouseprometheus/client.go @@ -14,6 +14,7 @@ import ( "github.com/SigNoz/signoz/pkg/types/ctxtypes" "github.com/SigNoz/signoz/pkg/types/instrumentationtypes" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/cespare/xxhash/v2" promValue "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/prompb" "github.com/prometheus/prometheus/storage" @@ -184,7 +185,7 @@ func (client *client) getFingerprintsFromClickhouseQuery(ctx context.Context, qu return nil, err } - labels, _, err := unmarshalLabels(labelString, fingerprint) + labels, _, err := unmarshalLabels(labelString) if err != nil { return nil, err } @@ -281,7 +282,138 @@ func (client *client) querySamples(ctx context.Context, start int64, end int64, return nil, err } - return res, nil + return mergeSeriesWithIdenticalLabels(res), nil +} + +// mergeSeriesWithIdenticalLabels collapses series sharing one labelset into +// one series each. Distinct fingerprints can map to one labelset: a label +// value goes empty over a series' lifetime (#8563), the fingerprint +// algorithm changes across exporter versions, or the env changes. The +// engine treats the labelset as series identity — duplicates raise +// "duplicate series", and #8563's workaround of injecting a synthetic +// fingerprint label silently broke without() and vector matching. Merging +// at the last point before hand-off keeps any future input-side label +// normalization collision-safe. Grouping is by an order-insensitive 64-bit +// hash so the common no-collision case costs one hash and one map insert +// per series; hash-equal groups are confirmed by exact labelset equality +// before any merge. Regression: +// TestClient_QuerySamplesMergesIdenticalLabelSets and +// tests/integration/tests/promqlconformance/02_fingerprint_probe.py. +func mergeSeriesWithIdenticalLabels(series []*prompb.TimeSeries) []*prompb.TimeSeries { + if len(series) < 2 { + return series + } + + groups := make(map[uint64][]*prompb.TimeSeries, len(series)) + order := make([]uint64, 0, len(series)) + for _, ts := range series { + key := labelsHash(ts.Labels) + if _, ok := groups[key]; !ok { + order = append(order, key) + } + groups[key] = append(groups[key], ts) + } + if len(order) == len(series) { + return series + } + + res := make([]*prompb.TimeSeries, 0, len(order)) + for _, key := range order { + group := groups[key] + if len(group) == 1 { + res = append(res, group[0]) + continue + } + for _, sub := range splitByLabelSet(group) { + if len(sub) == 1 { + res = append(res, sub[0]) + continue + } + res = append(res, mergeSamples(sub)) + } + } + return res +} + +var labelHashSep = []byte{0xff} + +// labelsHash combines per-label hashes commutatively, so the stored JSON's +// key order (not canonical across fingerprints) needs no sort. +func labelsHash(lbls []prompb.Label) uint64 { + var h uint64 + var d xxhash.Digest + for _, l := range lbls { + d.Reset() + _, _ = d.WriteString(l.Name) + _, _ = d.Write(labelHashSep) + _, _ = d.WriteString(l.Value) + h += d.Sum64() + } + return h +} + +// splitByLabelSet partitions a hash-equal group into sub-groups of exactly +// equal labelsets, preserving input order; series that merely collide on the +// 64-bit hash must not be merged. +func splitByLabelSet(group []*prompb.TimeSeries) [][]*prompb.TimeSeries { + var out [][]*prompb.TimeSeries +outer: + for _, ts := range group { + for i, sub := range out { + if labelSetsEqual(sub[0].Labels, ts.Labels) { + out[i] = append(out[i], ts) + continue outer + } + } + out = append(out, []*prompb.TimeSeries{ts}) + } + return out +} + +func labelSetsEqual(a, b []prompb.Label) bool { + if len(a) != len(b) { + return false + } + for _, la := range a { + found := false + for _, lb := range b { + if la.Name == lb.Name { + found = la.Value == lb.Value + break + } + } + if !found { + return false + } + } + return true +} + +// mergeSamples k-way merges sample streams that share one labelset. On +// equal timestamps the highest fingerprint wins: the input is in ascending +// fingerprint order (samples SQL), keeping the choice deterministic. +func mergeSamples(group []*prompb.TimeSeries) *prompb.TimeSeries { + merged := &prompb.TimeSeries{Labels: group[0].Labels} + idx := make([]int, len(group)) + for { + minTs := int64(math.MaxInt64) + for i, ts := range group { + if idx[i] < len(ts.Samples) && ts.Samples[idx[i]].Timestamp < minTs { + minTs = ts.Samples[idx[i]].Timestamp + } + } + if minTs == math.MaxInt64 { + return merged + } + var chosen prompb.Sample + for i, ts := range group { + if idx[i] < len(ts.Samples) && ts.Samples[idx[i]].Timestamp == minTs { + chosen = ts.Samples[idx[i]] + idx[i]++ + } + } + merged.Samples = append(merged.Samples, chosen) + } } func (client *client) queryRaw(ctx context.Context, query string, ts int64) (*prompb.QueryResult, error) { diff --git a/pkg/prometheus/clickhouseprometheus/client_query_test.go b/pkg/prometheus/clickhouseprometheus/client_query_test.go index 38e96b6f376..5417f575afb 100644 --- a/pkg/prometheus/clickhouseprometheus/client_query_test.go +++ b/pkg/prometheus/clickhouseprometheus/client_query_test.go @@ -152,21 +152,22 @@ func TestClient_getFingerprintsFromClickhouseQuery(t *testing.T) { setupMock: func(m cmock.ClickConnMockCommon, args ...any) { rows := [][]any{ {uint64(123), `{"t1":"s1","t2":"s2"}`}, - {uint64(234), `{"t1":"s1","t2":"s2"}`}, + {uint64(234), `{"t1":"s1","t2":"s2","empty":""}`}, } m.ExpectQuery(`SELECT fingerprint,labels`).WithArgs(args...).WillReturnRows( cmock.NewRows(cols, rows), ) }, + // No synthetic fingerprint label (#8563), empty-valued labels + // dropped: both fingerprints present one labelset for + // querySamples to merge. want: map[uint64][]prompb.Label{ 123: { - {Name: "fingerprint", Value: "123"}, {Name: "t1", Value: "s1"}, {Name: "t2", Value: "s2"}, }, 234: { - {Name: "fingerprint", Value: "234"}, {Name: "t1", Value: "s1"}, {Name: "t2", Value: "s2"}, }, @@ -208,3 +209,86 @@ func TestClient_getFingerprintsFromClickhouseQuery(t *testing.T) { }) } } + +// Regression for the duplicate-series class behind #8563: fingerprints +// sharing one labelset must come back as one merged series, the higher +// fingerprint winning equal timestamps. +func TestClient_QuerySamplesMergesIdenticalLabelSets(t *testing.T) { + ctx := context.Background() + cols := []cmock.ColumnType{ + {Name: "metric_name", Type: "String"}, + {Name: "fingerprint", Type: "UInt64"}, + {Name: "unix_milli", Type: "Int64"}, + {Name: "value", Type: "Float64"}, + {Name: "flags", Type: "UInt32"}, + } + + canary := []prompb.Label{ + {Name: "__name__", Value: "requests"}, + {Name: "group", Value: "canary"}, + } + production := []prompb.Label{ + {Name: "__name__", Value: "requests"}, + {Name: "group", Value: "production"}, + } + fingerprints := map[uint64][]prompb.Label{ + 100: canary, + 200: canary, + 300: production, + } + + telemetryStore := telemetrystoretest.New(telemetrystore.Config{Provider: "clickhouse"}, sqlmock.QueryMatcherRegexp) + // Rows arrive ordered by (fingerprint, unix_milli), matching the SQL. + values := [][]any{ + {"requests", uint64(100), int64(1000), float64(1.0), uint32(0)}, + {"requests", uint64(100), int64(2000), float64(2.0), uint32(0)}, + {"requests", uint64(200), int64(2000), float64(20.0), uint32(0)}, + {"requests", uint64(200), int64(3000), float64(30.0), uint32(0)}, + {"requests", uint64(300), int64(1500), float64(5.0), uint32(0)}, + } + telemetryStore.Mock().ExpectQuery("SELECT metric_name, fingerprint, unix_milli, value, flags"). + WithArgs("requests", int64(1000), int64(3000)). + WillReturnRows(cmock.NewRows(cols, values)) + + readClient := client{telemetryStore: telemetryStore} + result, err := readClient.querySamples(ctx, 1000, 3000, fingerprints, "requests", "SELECT metric_name, fingerprint, unix_milli, value, flags", nil) + require.NoError(t, err) + + assert.Equal(t, []*prompb.TimeSeries{ + { + Labels: canary, + Samples: []prompb.Sample{ + {Timestamp: 1000, Value: 1.0}, + {Timestamp: 2000, Value: 20.0}, + {Timestamp: 3000, Value: 30.0}, + }, + }, + { + Labels: production, + Samples: []prompb.Sample{ + {Timestamp: 1500, Value: 5.0}, + }, + }, + }, result) +} + +// Hash grouping must stay order-insensitive (stored JSON key order is not +// canonical across fingerprints), and a 64-bit hash collision between +// distinct labelsets must not merge them — splitByLabelSet is that guard. +func TestLabelsHashAndCollisionSplit(t *testing.T) { + lbls := []prompb.Label{ + {Name: "__name__", Value: "requests"}, + {Name: "job", Value: "api"}, + {Name: "instance", Value: "0"}, + } + reversed := []prompb.Label{lbls[2], lbls[1], lbls[0]} + assert.Equal(t, labelsHash(lbls), labelsHash(reversed)) + + a := &prompb.TimeSeries{Labels: []prompb.Label{{Name: "job", Value: "x"}}} + b := &prompb.TimeSeries{Labels: []prompb.Label{{Name: "job", Value: "y"}}} + c := &prompb.TimeSeries{Labels: []prompb.Label{{Name: "job", Value: "x"}}} + got := splitByLabelSet([]*prompb.TimeSeries{a, b, c}) + require.Len(t, got, 2) + assert.Equal(t, []*prompb.TimeSeries{a, c}, got[0]) + assert.Equal(t, []*prompb.TimeSeries{b}, got[1]) +} diff --git a/pkg/prometheus/clickhouseprometheus/json.go b/pkg/prometheus/clickhouseprometheus/json.go index ec478223edf..c9859c6be39 100644 --- a/pkg/prometheus/clickhouseprometheus/json.go +++ b/pkg/prometheus/clickhouseprometheus/json.go @@ -2,14 +2,15 @@ package clickhouseprometheus import ( "encoding/json" - "strconv" - "github.com/SigNoz/signoz/pkg/prometheus" "github.com/prometheus/prometheus/prompb" ) // Unmarshals JSON into Prometheus labels. It does not preserve order. -func unmarshalLabels(s string, fingerprint uint64) ([]prompb.Label, string, error) { +// Empty-valued labels are dropped: Prometheus treats them as absent, and +// keeping them lets two fingerprints present duplicate labelsets to the +// engine (the incident behind #8563). +func unmarshalLabels(s string) ([]prompb.Label, string, error) { var metricName string m := make(map[string]string) if err := json.Unmarshal([]byte(s), &m); err != nil { @@ -17,6 +18,9 @@ func unmarshalLabels(s string, fingerprint uint64) ([]prompb.Label, string, erro } res := make([]prompb.Label, 0, len(m)) for n, v := range m { + if v == "" { + continue + } if n == "__name__" { metricName = v } @@ -26,9 +30,5 @@ func unmarshalLabels(s string, fingerprint uint64) ([]prompb.Label, string, erro Value: v, }) } - res = append(res, prompb.Label{ - Name: prometheus.FingerprintAsPromLabelName, - Value: strconv.FormatUint(fingerprint, 10), - }) return res, metricName, nil } diff --git a/pkg/prometheus/clickhouseprometheus/merge_test.go b/pkg/prometheus/clickhouseprometheus/merge_test.go new file mode 100644 index 00000000000..e56f9ebdfa4 --- /dev/null +++ b/pkg/prometheus/clickhouseprometheus/merge_test.go @@ -0,0 +1,82 @@ +package clickhouseprometheus + +import ( + "testing" + + "github.com/prometheus/prometheus/prompb" + "github.com/stretchr/testify/assert" +) + +func mkSeries(value string, samples ...prompb.Sample) *prompb.TimeSeries { + return &prompb.TimeSeries{ + Labels: []prompb.Label{{Name: "job", Value: value}}, + Samples: samples, + } +} + +func TestMergeSeriesWithIdenticalLabels(t *testing.T) { + s := func(ts int64, v float64) prompb.Sample { return prompb.Sample{Timestamp: ts, Value: v} } + + t.Run("empty and single series pass through untouched", func(t *testing.T) { + assert.Nil(t, mergeSeriesWithIdenticalLabels(nil)) + one := []*prompb.TimeSeries{mkSeries("a", s(1, 1))} + got := mergeSeriesWithIdenticalLabels(one) + assert.Equal(t, one, got) + }) + + t.Run("no collisions returns the input slice as is", func(t *testing.T) { + in := []*prompb.TimeSeries{mkSeries("a", s(1, 1)), mkSeries("b", s(1, 2))} + got := mergeSeriesWithIdenticalLabels(in) + // same backing slice: the fast path must not rebuild anything + assert.Equal(t, &in[0], &got[0]) + }) + + t.Run("disjoint streams concatenate in timestamp order", func(t *testing.T) { + // the #8563 shape: the series transitioned fingerprints at a point + // in time, so the streams do not overlap at all + got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{ + mkSeries("a", s(1, 1), s(2, 2)), + mkSeries("a", s(3, 3), s(4, 4)), + }) + assert.Equal(t, []prompb.Sample{s(1, 1), s(2, 2), s(3, 3), s(4, 4)}, got[0].Samples) + }) + + t.Run("three fingerprints one labelset", func(t *testing.T) { + got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{ + mkSeries("a", s(1, 1), s(4, 4)), + mkSeries("a", s(2, 2)), + mkSeries("a", s(3, 3)), + }) + assert.Len(t, got, 1) + assert.Equal(t, []prompb.Sample{s(1, 1), s(2, 2), s(3, 3), s(4, 4)}, got[0].Samples) + }) + + t.Run("equal timestamps everywhere keep the last stream's value", func(t *testing.T) { + // input is in ascending fingerprint order; the highest wins each tie + got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{ + mkSeries("a", s(1, 1), s(2, 1)), + mkSeries("a", s(1, 9), s(2, 9)), + }) + assert.Equal(t, []prompb.Sample{s(1, 9), s(2, 9)}, got[0].Samples) + }) + + t.Run("zero-sample series in a group is harmless", func(t *testing.T) { + got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{ + mkSeries("a"), + mkSeries("a", s(1, 1)), + }) + assert.Len(t, got, 1) + assert.Equal(t, []prompb.Sample{s(1, 1)}, got[0].Samples) + }) + + t.Run("colliding and distinct series interleave without cross-talk", func(t *testing.T) { + got := mergeSeriesWithIdenticalLabels([]*prompb.TimeSeries{ + mkSeries("a", s(1, 1)), + mkSeries("b", s(1, 2)), + mkSeries("a", s(2, 3)), + }) + assert.Len(t, got, 2) + assert.Equal(t, []prompb.Sample{s(1, 1), s(2, 3)}, got[0].Samples) + assert.Equal(t, []prompb.Sample{s(1, 2)}, got[1].Samples) + }) +} diff --git a/pkg/prometheus/label.go b/pkg/prometheus/label.go deleted file mode 100644 index c0fb8899ae1..00000000000 --- a/pkg/prometheus/label.go +++ /dev/null @@ -1,39 +0,0 @@ -package prometheus - -import ( - "github.com/SigNoz/signoz/pkg/errors" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/promql" -) - -const FingerprintAsPromLabelName string = "fingerprint" - -func RemoveExtraLabels(res *promql.Result, labelsToRemove ...string) error { - if len(labelsToRemove) == 0 || res == nil { - return nil - } - - switch res.Value.(type) { - case promql.Vector: - value := res.Value.(promql.Vector) - for i := range value { - b := labels.NewBuilder(value[i].Metric) - b.Del(labelsToRemove...) - newLabels := b.Labels() - value[i].Metric = newLabels - } - case promql.Matrix: - value := res.Value.(promql.Matrix) - for i := range value { - b := labels.NewBuilder(value[i].Metric) - b.Del(labelsToRemove...) - newLabels := b.Labels() - value[i].Metric = newLabels - } - case promql.Scalar: - return nil - default: - return errors.NewInternalf(errors.CodeInternal, "rule result is not a vector or scalar or matrix") - } - return nil -} diff --git a/pkg/prometheus/prometheustest/utils_test.go b/pkg/prometheus/prometheustest/utils_test.go deleted file mode 100644 index 48d71bce9a2..00000000000 --- a/pkg/prometheus/prometheustest/utils_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package prometheustest - -import ( - "testing" - - "github.com/SigNoz/signoz/pkg/prometheus" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/promql" -) - -func TestRemoveExtraLabels(t *testing.T) { - - tests := []struct { - name string - res *promql.Result - remove []string - wantErr bool - verify func(t *testing.T, result *promql.Result, removed []string) - }{ - { - name: "vector – label removed", - res: &promql.Result{ - Value: promql.Vector{ - promql.Sample{ - Metric: labels.FromStrings( - "__name__", "http_requests_total", - "job", "demo", - "drop_me", "dropped", - ), - }, - }, - }, - remove: []string{"drop_me"}, - verify: func(t *testing.T, result *promql.Result, removed []string) { - k := result.Value.(promql.Vector) - for _, str := range removed { - get := k[0].Metric.Get(str) - if get != "" { - t.Fatalf("label not removed") - } - } - }, - }, - { - name: "scalar – nothing to strip", - res: &promql.Result{ - Value: promql.Scalar{V: 99, T: 1}, - }, - remove: []string{"irrelevant"}, - verify: func(t *testing.T, result *promql.Result, removed []string) { - sc := result.Value.(promql.Scalar) - if sc.V != 99 || sc.T != 1 { - t.Fatalf("scalar unexpectedly modified: got %+v", sc) - } - }, - }, - { - name: "matrix – label removed", - res: &promql.Result{ - Value: promql.Matrix{ - promql.Series{ - Metric: labels.FromStrings( - "__name__", "http_requests_total", - "pod", "p0", - "drop_me", "dropped", - ), - Floats: []promql.FPoint{{T: 0, F: 1}, {T: 1, F: 2}}, - }, - promql.Series{ - Metric: labels.FromStrings( - "__name__", "http_requests_total", - "pod", "p0", - "drop_me", "dropped", - ), - Floats: []promql.FPoint{{T: 0, F: 1}, {T: 1, F: 2}}, - }, - }, - }, - remove: []string{"drop_me"}, - verify: func(t *testing.T, result *promql.Result, removed []string) { - mat := result.Value.(promql.Matrix) - for _, str := range removed { - for _, k := range mat { - if k.Metric.Get(str) != "" { - t.Fatalf("label not removed") - } - } - } - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - err := prometheus.RemoveExtraLabels(tc.res, tc.remove...) - if tc.wantErr && err == nil { - t.Fatalf("expected error, got nil") - } - if !tc.wantErr && err != nil { - t.Fatalf("unexpected error: %v", err) - } - if tc.verify != nil { - tc.verify(t, tc.res, tc.remove) - } - }) - } -} diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index aceed4323f1..21558c42504 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -247,10 +247,6 @@ func (r *ClickHouseReader) GetInstantQueryMetricsResult(ctx context.Context, que } qry.Close() - err = prometheus.RemoveExtraLabels(res, prometheus.FingerprintAsPromLabelName) - if err != nil { - return nil, nil, &model.ApiError{Typ: model.ErrorInternal, Err: err} - } return res, &qs, nil } @@ -271,10 +267,6 @@ func (r *ClickHouseReader) GetQueryRangeResult(ctx context.Context, query *model } qry.Close() - err = prometheus.RemoveExtraLabels(res, prometheus.FingerprintAsPromLabelName) - if err != nil { - return nil, nil, &model.ApiError{Typ: model.ErrorInternal, Err: err} - } return res, &qs, nil } diff --git a/pkg/query-service/rules/prom_rule.go b/pkg/query-service/rules/prom_rule.go index 81c802b8149..f68bf43a148 100644 --- a/pkg/query-service/rules/prom_rule.go +++ b/pkg/query-service/rules/prom_rule.go @@ -387,11 +387,6 @@ func (r *PromRule) RunAlertQuery(ctx context.Context, qs string, start, end time return nil, res.Err } - err = prometheus.RemoveExtraLabels(res, prometheus.FingerprintAsPromLabelName) - if err != nil { - return nil, err - } - switch typ := res.Value.(type) { case promql.Vector: series := make([]promql.Series, 0, len(typ)) From 852ca779c96067ee65ce76a389ae9f76cf95cfec Mon Sep 17 00:00:00 2001 From: Tushar Vats Date: Mon, 20 Jul 2026 00:11:53 +0530 Subject: [PATCH 2/5] fix: convert key not found to warnings for logs (#12134) * fix: convert key not found to warnings * fix(logs): has-family body-only errors, not-found warnings, and condition-builder cleanup - has/hasAny/hasAll/hasToken on a non-body key now return "supports only body JSON search" (both modes); a not-found body path warns and queries the underlying data instead of 400 (JSON mode) - capture the two has-family 500s (hasToken separator/whitespace needle; hasAny/hasAll quoted int >= 2^32) as regression tests - use telemetrytypes.NewTelemetryFieldKey for derived keys (fresh identity-only keys, no stale resolved metadata) instead of struct copies of the field key - rename ConditionForKeys -> ConditionFor and inline conditionsForKeys into it across the builders; rename private conditionFor -> conditionForResolvedKey - move the trace_noise fixture from queriertraces/conftest.py to fixtures/traces.py * fix: convert trace_noise fixture to util --- .../implrawdataexport/handler.go | 8 + .../implrulestatehistory/condition_builder.go | 6 +- .../implrulestatehistory/field_mapper.go | 2 +- pkg/querier/signozquerier/provider.go | 6 +- pkg/query-service/rules/setups_test.go | 4 +- .../rules/threshold_rule_test.go | 4 +- pkg/querybuilder/agg_rewrite.go | 20 +- pkg/querybuilder/exists_expr.go | 100 + pkg/querybuilder/fallback_expr.go | 115 +- pkg/querybuilder/key_resolution.go | 22 +- pkg/querybuilder/where_clause_visitor.go | 31 +- pkg/querybuilder/where_clause_visitor_test.go | 29 +- pkg/telemetryaudit/condition_builder.go | 86 +- pkg/telemetryaudit/field_mapper.go | 24 + pkg/telemetryaudit/statement_builder.go | 17 +- pkg/telemetryaudit/statement_builder_test.go | 27 +- pkg/telemetrylogs/condition_builder.go | 207 +- pkg/telemetrylogs/condition_builder_test.go | 119 +- pkg/telemetrylogs/field_mapper.go | 329 ++- pkg/telemetrylogs/field_mapper_test.go | 4 +- pkg/telemetrylogs/filter_expr_logs_test.go | 580 ++--- pkg/telemetrylogs/groupby_unknown_key_test.go | 72 + pkg/telemetrylogs/json_planless_test.go | 86 + pkg/telemetrylogs/json_stmt_builder_test.go | 28 +- pkg/telemetrylogs/json_string.go | 24 +- pkg/telemetrylogs/statement_builder.go | 109 +- pkg/telemetrylogs/stmt_builder_test.go | 77 +- pkg/telemetrymetadata/condition_builder.go | 9 +- .../condition_builder_test.go | 2 +- pkg/telemetrymetadata/field_mapper.go | 1 + pkg/telemetrymetadata/metadata.go | 6 +- pkg/telemetrymeter/statement_builder.go | 6 +- pkg/telemetrymetrics/condition_builder.go | 6 +- .../condition_builder_test.go | 4 +- pkg/telemetrymetrics/field_mapper.go | 1 + pkg/telemetrymetrics/statement_builder.go | 4 +- .../condition_builder.go | 41 +- .../condition_builder_test.go | 2 +- pkg/telemetryresourcefilter/field_mapper.go | 1 + pkg/telemetrytraces/condition_builder.go | 203 +- pkg/telemetrytraces/condition_builder_test.go | 134 +- pkg/telemetrytraces/field_mapper.go | 190 +- pkg/telemetrytraces/field_mapper_test.go | 4 +- pkg/telemetrytraces/statement_builder.go | 8 +- pkg/telemetrytraces/stmt_builder_test.go | 70 +- .../trace_operator_cte_builder.go | 10 +- .../trace_operator_cte_builder_test.go | 8 +- pkg/telemetrytraces/trace_time_range_test.go | 2 +- .../querybuildertypesv5/evolution.go | 3 + .../querybuildertypesv5/qb.go | 29 +- pkg/types/telemetrytypes/field.go | 8 + pkg/types/telemetrytypes/json_access_plan.go | 48 +- tests/fixtures/logs.py | 28 + tests/fixtures/querier.py | 36 + tests/fixtures/traces.py | 94 + .../03_body_array_functions.py | 130 +- .../04_unknown_key_collisions.py | 301 +++ .../queriercommon/04_filter_operators.py | 15 +- .../tests/querierlogs/07_list_expectations.py | 50 +- .../querierlogs/09_json_body_functions.py | 114 +- .../tests/querierlogs/10_filter_operators.py | 133 ++ .../querierlogs/11_aggregation_functions.py | 169 ++ .../querierlogs/12_aggregation_timeseries.py | 206 ++ .../querierlogs/13_unknown_key_collisions.py | 228 ++ .../tests/querierlogs/14_select_fields.py | 212 ++ .../tests/queriermetrics/10_key_resolution.py | 210 ++ .../tests/querierscalar/05_reduce_to.py | 32 +- .../querierscalar/06_metrics_aggregations.py | 108 + .../tests/querierscalar/07_unknown_keys.py | 204 ++ .../tests/queriertraces/01_list.py | 1997 +++++++++++------ .../tests/queriertraces/02_aggregation.py | 725 +++--- .../tests/queriertraces/03_fill.py | 1080 ++------- .../tests/queriertraces/04_trace_id_filter.py | 342 ++- .../queriertraces/05_resource_evolution.py | 20 +- .../tests/queriertraces/06_trace_operator.py | 51 +- .../tests/queriertraces/07_filter_errors.py | 16 +- .../queriertraces/08_aggregation_min_max.py | 135 +- .../tests/queriertraces/09_unknown_keys.py | 10 +- .../queriertraces/10_filter_operators.py | 129 ++ .../queriertraces/11_aggregation_depth.py | 251 +++ 80 files changed, 6421 insertions(+), 3541 deletions(-) create mode 100644 pkg/querybuilder/exists_expr.go create mode 100644 pkg/telemetrylogs/groupby_unknown_key_test.go create mode 100644 pkg/telemetrylogs/json_planless_test.go create mode 100644 tests/integration/tests/querier_json_body/04_unknown_key_collisions.py create mode 100644 tests/integration/tests/querierlogs/10_filter_operators.py create mode 100644 tests/integration/tests/querierlogs/11_aggregation_functions.py create mode 100644 tests/integration/tests/querierlogs/12_aggregation_timeseries.py create mode 100644 tests/integration/tests/querierlogs/13_unknown_key_collisions.py create mode 100644 tests/integration/tests/querierlogs/14_select_fields.py create mode 100644 tests/integration/tests/queriermetrics/10_key_resolution.py create mode 100644 tests/integration/tests/querierscalar/06_metrics_aggregations.py create mode 100644 tests/integration/tests/querierscalar/07_unknown_keys.py create mode 100644 tests/integration/tests/queriertraces/10_filter_operators.py create mode 100644 tests/integration/tests/queriertraces/11_aggregation_depth.py diff --git a/pkg/modules/rawdataexport/implrawdataexport/handler.go b/pkg/modules/rawdataexport/implrawdataexport/handler.go index 74fd47068dc..e0d10485f6d 100644 --- a/pkg/modules/rawdataexport/implrawdataexport/handler.go +++ b/pkg/modules/rawdataexport/implrawdataexport/handler.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "reflect" "slices" "strconv" "time" @@ -285,6 +286,13 @@ func constructCSVRecordFromQueryResponse(data map[string]any, headerToIndexMappi for key, value := range data { if index, exists := headerToIndexMapping[key]; exists && value != nil { + if rv := reflect.ValueOf(value); rv.Kind() == reflect.Pointer { + if rv.IsNil() { + continue + } + value = rv.Elem().Interface() + } + var valueStr string switch v := value.(type) { case string: diff --git a/pkg/modules/rulestatehistory/implrulestatehistory/condition_builder.go b/pkg/modules/rulestatehistory/implrulestatehistory/condition_builder.go index 38ce8e81d9d..1e7b5498c2a 100644 --- a/pkg/modules/rulestatehistory/implrulestatehistory/condition_builder.go +++ b/pkg/modules/rulestatehistory/implrulestatehistory/condition_builder.go @@ -21,13 +21,15 @@ func newConditionBuilder(fm qbtypes.FieldMapper) qbtypes.ConditionBuilder { return &conditionBuilder{fm: fm} } +// Rule state history has no resource sub-query, so options are unused. func (c *conditionBuilder) ConditionFor( ctx context.Context, orgID valuer.UUID, startNs uint64, endNs uint64, key *telemetrytypes.TelemetryFieldKey, - fieldKeysForName []*telemetrytypes.TelemetryFieldKey, + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, + _ qbtypes.ConditionBuilderOptions, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder, @@ -38,7 +40,7 @@ func (c *conditionBuilder) ConditionFor( return nil, nil, err } - keys, warning := querybuilder.ResolveKeys(key, fieldKeysForName) + keys, warning := querybuilder.ResolveKeys(key, querybuilder.MatchingFieldKeys(key, fieldKeys)) var warnings []string if warning != "" { warnings = append(warnings, warning) diff --git a/pkg/modules/rulestatehistory/implrulestatehistory/field_mapper.go b/pkg/modules/rulestatehistory/implrulestatehistory/field_mapper.go index c69d149a979..29da8e134ad 100644 --- a/pkg/modules/rulestatehistory/implrulestatehistory/field_mapper.go +++ b/pkg/modules/rulestatehistory/implrulestatehistory/field_mapper.go @@ -64,7 +64,7 @@ func (m *fieldMapper) ColumnFor(ctx context.Context, _ valuer.UUID, _, _ uint64, return []*schema.Column{col}, nil } -func (m *fieldMapper) ColumnExpressionFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, field *telemetrytypes.TelemetryFieldKey, _ map[string][]*telemetrytypes.TelemetryFieldKey) (string, error) { +func (m *fieldMapper) ColumnExpressionFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, field *telemetrytypes.TelemetryFieldKey, _ telemetrytypes.FieldDataType, _ map[string][]*telemetrytypes.TelemetryFieldKey) (string, error) { colName, err := m.FieldFor(ctx, orgID, tsStart, tsEnd, field) if err != nil { return "", err diff --git a/pkg/querier/signozquerier/provider.go b/pkg/querier/signozquerier/provider.go index a8314545148..764d6596c77 100644 --- a/pkg/querier/signozquerier/provider.go +++ b/pkg/querier/signozquerier/provider.go @@ -79,7 +79,7 @@ func newProvider( traceFieldMapper := telemetrytraces.NewFieldMapper() traceConditionBuilder := telemetrytraces.NewConditionBuilder(traceFieldMapper) - traceAggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, traceFieldMapper, traceConditionBuilder, nil, flagger) + traceAggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, traceFieldMapper, traceConditionBuilder, flagger) traceStmtBuilder := telemetrytraces.NewTraceQueryStatementBuilder( settings, telemetryMetadataStore, @@ -111,7 +111,6 @@ func newProvider( telemetrylogs.DefaultFullTextColumn, logFieldMapper, logConditionBuilder, - telemetrylogs.GetBodyJSONKey, flagger, ) logStmtBuilder := telemetrylogs.NewLogQueryStatementBuilder( @@ -121,7 +120,6 @@ func newProvider( logConditionBuilder, logAggExprRewriter, telemetrylogs.DefaultFullTextColumn, - telemetrylogs.GetBodyJSONKey, flagger, telemetryStore, cfg.SkipResourceFingerprint.Enabled, @@ -136,7 +134,6 @@ func newProvider( telemetryaudit.DefaultFullTextColumn, auditFieldMapper, auditConditionBuilder, - nil, flagger, ) auditStmtBuilder := telemetryaudit.NewAuditQueryStatementBuilder( @@ -146,7 +143,6 @@ func newProvider( auditConditionBuilder, auditAggExprRewriter, telemetryaudit.DefaultFullTextColumn, - nil, flagger, ) diff --git a/pkg/query-service/rules/setups_test.go b/pkg/query-service/rules/setups_test.go index ee5d04eb762..4797f66ad6f 100644 --- a/pkg/query-service/rules/setups_test.go +++ b/pkg/query-service/rules/setups_test.go @@ -80,7 +80,6 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry telemetrylogs.DefaultFullTextColumn, logFieldMapper, logConditionBuilder, - telemetrylogs.GetBodyJSONKey, fl, ) logStmtBuilder := telemetrylogs.NewLogQueryStatementBuilder( @@ -90,7 +89,6 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry logConditionBuilder, logAggExprRewriter, telemetrylogs.DefaultFullTextColumn, - telemetrylogs.GetBodyJSONKey, fl, nil, false, @@ -133,7 +131,7 @@ func prepareQuerierForTraces(t *testing.T, telemetryStore telemetrystore.Telemet traceConditionBuilder := telemetrytraces.NewConditionBuilder(traceFieldMapper) fl := flaggertest.New(t) - traceAggExprRewriter := querybuilder.NewAggExprRewriter(providerSettings, nil, traceFieldMapper, traceConditionBuilder, nil, fl) + traceAggExprRewriter := querybuilder.NewAggExprRewriter(providerSettings, nil, traceFieldMapper, traceConditionBuilder, fl) traceStmtBuilder := telemetrytraces.NewTraceQueryStatementBuilder( providerSettings, metadataStore, diff --git a/pkg/query-service/rules/threshold_rule_test.go b/pkg/query-service/rules/threshold_rule_test.go index 5a1747c9572..850bd5a2054 100644 --- a/pkg/query-service/rules/threshold_rule_test.go +++ b/pkg/query-service/rules/threshold_rule_test.go @@ -839,7 +839,7 @@ func TestThresholdRuleTracesLink(t *testing.T) { queryString := "SELECT any" telemetryStore.Mock(). ExpectQuery(queryString). - WithArgs(nil, nil, nil, nil, nil, nil, nil). + WithArgs(nil, nil, nil, nil, nil). WillReturnRows(rows) querier := prepareQuerierForTraces(t, telemetryStore, keysMap) @@ -957,7 +957,7 @@ func TestThresholdRuleLogsLink(t *testing.T) { queryString := "SELECT any" telemetryStore.Mock(). ExpectQuery(queryString). - WithArgs(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil). + WithArgs(nil, nil, nil, nil, nil, nil, nil, nil, nil). WillReturnRows(rows) querier := prepareQuerierForLogs(t, telemetryStore, keysMap) diff --git a/pkg/querybuilder/agg_rewrite.go b/pkg/querybuilder/agg_rewrite.go index 80825deabbb..c4c26f183c5 100644 --- a/pkg/querybuilder/agg_rewrite.go +++ b/pkg/querybuilder/agg_rewrite.go @@ -10,7 +10,6 @@ import ( "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/factory" "github.com/SigNoz/signoz/pkg/flagger" - "github.com/SigNoz/signoz/pkg/types/featuretypes" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/valuer" @@ -22,7 +21,6 @@ type aggExprRewriter struct { fullTextColumn *telemetrytypes.TelemetryFieldKey fieldMapper qbtypes.FieldMapper conditionBuilder qbtypes.ConditionBuilder - jsonKeyToKey qbtypes.JsonKeyToFieldFunc flagger flagger.Flagger } @@ -33,7 +31,6 @@ func NewAggExprRewriter( fullTextColumn *telemetrytypes.TelemetryFieldKey, fieldMapper qbtypes.FieldMapper, conditionBuilder qbtypes.ConditionBuilder, - jsonKeyToKey qbtypes.JsonKeyToFieldFunc, fl flagger.Flagger, ) *aggExprRewriter { set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/querybuilder/agg_rewrite") @@ -43,7 +40,6 @@ func NewAggExprRewriter( fullTextColumn: fullTextColumn, fieldMapper: fieldMapper, conditionBuilder: conditionBuilder, - jsonKeyToKey: jsonKeyToKey, flagger: fl, } } @@ -92,7 +88,6 @@ func (r *aggExprRewriter) Rewrite( r.fullTextColumn, r.fieldMapper, r.conditionBuilder, - r.jsonKeyToKey, r.flagger, ) // Rewrite the first select item (our expression) @@ -147,7 +142,6 @@ type exprVisitor struct { fullTextColumn *telemetrytypes.TelemetryFieldKey fieldMapper qbtypes.FieldMapper conditionBuilder qbtypes.ConditionBuilder - jsonKeyToKey qbtypes.JsonKeyToFieldFunc flagger flagger.Flagger Modified bool chArgs []any @@ -164,7 +158,6 @@ func newExprVisitor( fullTextColumn *telemetrytypes.TelemetryFieldKey, fieldMapper qbtypes.FieldMapper, conditionBuilder qbtypes.ConditionBuilder, - jsonKeyToKey qbtypes.JsonKeyToFieldFunc, fl flagger.Flagger, ) *exprVisitor { return &exprVisitor{ @@ -177,7 +170,6 @@ func newExprVisitor( fullTextColumn: fullTextColumn, fieldMapper: fieldMapper, conditionBuilder: conditionBuilder, - jsonKeyToKey: jsonKeyToKey, flagger: fl, } } @@ -212,8 +204,6 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error { dataType = telemetrytypes.FieldDataTypeFloat64 } - bodyJSONEnabled := v.flagger.BooleanOrEmpty(v.ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(v.orgID)) - // Handle *If functions with predicate + values if aggFunc.FuncCombinator { // Map the predicate (last argument) @@ -254,12 +244,11 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error { for i := 0; i < len(args)-1; i++ { origVal := args[i].String() fieldKey := telemetrytypes.GetFieldKeyFromKeyText(origVal) - expr, exprArgs, err := CollisionHandledFinalExpr(v.ctx, v.orgID, v.startNs, v.endNs, &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonKeyToKey, bodyJSONEnabled) + expr, err := v.fieldMapper.ColumnExpressionFor(v.ctx, v.orgID, v.startNs, v.endNs, &fieldKey, dataType, v.fieldKeys) if err != nil { return errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to get table field name for %q", origVal) } - v.chArgs = append(v.chArgs, exprArgs...) - newVal := expr + newVal := sqlbuilder.Escape(expr) parsedVal, err := parseFragment(newVal) if err != nil { return err @@ -272,12 +261,11 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error { for i, arg := range args { orig := arg.String() fieldKey := telemetrytypes.GetFieldKeyFromKeyText(orig) - expr, exprArgs, err := CollisionHandledFinalExpr(v.ctx, v.orgID, v.startNs, v.endNs, &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonKeyToKey, bodyJSONEnabled) + expr, err := v.fieldMapper.ColumnExpressionFor(v.ctx, v.orgID, v.startNs, v.endNs, &fieldKey, dataType, v.fieldKeys) if err != nil { return err } - v.chArgs = append(v.chArgs, exprArgs...) - newCol := expr + newCol := sqlbuilder.Escape(expr) parsed, err := parseFragment(newCol) if err != nil { return err diff --git a/pkg/querybuilder/exists_expr.go b/pkg/querybuilder/exists_expr.go new file mode 100644 index 00000000000..63dcd0644d6 --- /dev/null +++ b/pkg/querybuilder/exists_expr.go @@ -0,0 +1,100 @@ +package querybuilder + +import ( + "fmt" + + schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" + "github.com/SigNoz/signoz/pkg/errors" + qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" + "github.com/SigNoz/signoz/pkg/types/telemetrytypes" +) + +// ExistsExpression renders the existence predicate for a key resolved to the given +// columns (negated when exists is false). Comparisons are against constants rendered +// as literals, so the expression carries no bind args and can guard column expressions +// directly. Signal-specific presence checks (body JSON paths, label maps) are handled +// by the field mappers before falling through to this. +func ExistsExpression(columns []*schema.Column, key *telemetrytypes.TelemetryFieldKey, tsStart, tsEnd uint64, fieldExpression string, exists bool) (string, error) { + newColumns, evolutionsEntries, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd) + if err != nil { + return "", err + } + if len(newColumns) == 0 { + return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "no valid evolution found for field %s in the given time range", key.Name) + } + + comparison := func(operator string, value string) string { + return fmt.Sprintf("%s %s %s", fieldExpression, operator, value) + } + + if len(newColumns) > 1 { + if exists { + return fieldExpression + " IS NOT NULL", nil + } + return fieldExpression + " IS NULL", nil + } + + column := newColumns[0] + switch column.Type.GetType() { + case schema.ColumnTypeEnumJSON: + // the ::String cast in the value expression folds NULL to '', so the + // presence check must address the raw JSON path + columnName := column.Name + if len(evolutionsEntries) > 0 && evolutionsEntries[0] != nil { + columnName = evolutionsEntries[0].ColumnName + } + rawPath := fmt.Sprintf("%s.`%s`", columnName, key.Name) + if exists { + return rawPath + " IS NOT NULL", nil + } + return rawPath + " IS NULL", nil + case schema.ColumnTypeEnumString, + schema.ColumnTypeEnumFixedString, + schema.ColumnTypeEnumDateTime64: + if exists { + return comparison("<>", "''"), nil + } + return comparison("=", "''"), nil + case schema.ColumnTypeEnumLowCardinality: + switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() { + case schema.ColumnTypeEnumString: + if exists { + return comparison("<>", "''"), nil + } + return comparison("=", "''"), nil + default: + return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for low cardinality column type %s", elementType) + } + case schema.ColumnTypeEnumUInt64, + schema.ColumnTypeEnumUInt32, + schema.ColumnTypeEnumUInt8, + schema.ColumnTypeEnumInt8, + schema.ColumnTypeEnumInt16, + schema.ColumnTypeEnumBool: + if exists { + return comparison("<>", "0"), nil + } + return comparison("=", "0"), nil + case schema.ColumnTypeEnumMap: + keyType := column.Type.(schema.MapColumnType).KeyType + if _, ok := keyType.(schema.LowCardinalityColumnType); !ok { + return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type) + } + + switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() { + case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64: + leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name) + if key.Materialized { + leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key) + } + if exists { + return leftOperand, nil + } + return "NOT " + leftOperand, nil + default: + return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType) + } + default: + return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", column.Type) + } +} diff --git a/pkg/querybuilder/fallback_expr.go b/pkg/querybuilder/fallback_expr.go index 1275a268ac8..768d6d7cea9 100644 --- a/pkg/querybuilder/fallback_expr.go +++ b/pkg/querybuilder/fallback_expr.go @@ -1,128 +1,17 @@ package querybuilder import ( - "context" "encoding/json" "fmt" "math" "reflect" "regexp" "strconv" - "strings" - "github.com/SigNoz/signoz/pkg/errors" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" - "github.com/SigNoz/signoz/pkg/valuer" - "github.com/huandu/go-sqlbuilder" - "golang.org/x/exp/maps" ) -func CollisionHandledFinalExpr( - ctx context.Context, - orgID valuer.UUID, - startNs uint64, - endNs uint64, - field *telemetrytypes.TelemetryFieldKey, - fm qbtypes.FieldMapper, - cb qbtypes.ConditionBuilder, - keys map[string][]*telemetrytypes.TelemetryFieldKey, - requiredDataType telemetrytypes.FieldDataType, - jsonKeyToKey qbtypes.JsonKeyToFieldFunc, - bodyJSONEnabled bool, -) (string, []any, error) { - - if requiredDataType != telemetrytypes.FieldDataTypeString && - requiredDataType != telemetrytypes.FieldDataTypeFloat64 { - return "", nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported data type %s", requiredDataType) - } - - var dummyValue any - if requiredDataType == telemetrytypes.FieldDataTypeFloat64 { - dummyValue = 0.0 - } else { - dummyValue = "" - } - - var stmts []string - var allArgs []any - - addCondition := func(key *telemetrytypes.TelemetryFieldKey) error { - sb := sqlbuilder.NewSelectBuilder() - conds, _, err := cb.ConditionFor(ctx, orgID, startNs, endNs, key, []*telemetrytypes.TelemetryFieldKey{key}, qbtypes.FilterOperatorExists, nil, sb) - if err != nil { - return err - } - sb.Where(conds[0]) - - expr, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse) - expr = strings.TrimPrefix(expr, "WHERE ") - stmts = append(stmts, expr) - allArgs = append(allArgs, args...) - return nil - } - - fieldExpression, fieldForErr := fm.FieldFor(ctx, orgID, startNs, endNs, field) - if errors.Is(fieldForErr, qbtypes.ErrColumnNotFound) { - // the key didn't have the right context to be added to the query - // we try to use the context we know of - keysForField := keys[field.Name] - - if len(keysForField) == 0 { - // check if the key exists with {fieldContext}.{key} - // because the context could be legitimate prefix in user data, example `metric.max` - keyWithContext := fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), field.Name) - if len(keys[keyWithContext]) > 0 { - keysForField = keys[keyWithContext] - } - } - - if len(keysForField) == 0 { - // No metadata match: let the field mapper synthesize type-variant keys. - // keys were already checked above, so pass nil here. - keysForField = fm.CandidateKeys(ctx, orgID, field, nil, nil) - } - if len(keysForField) == 0 { - // The mapper can't synthesize (e.g. metrics): fall back to the typo suggestion. - wrappedErr := errors.WithSuggestiveAdditionalf(fieldForErr, errors.NewSuggestionsOnLevenshteinDistance(field.Name, errors.NounKeys, maps.Keys(keys)), "field `%s` not found", field.Name) - return "", nil, wrappedErr - } - for _, key := range keysForField { - err := addCondition(key) - if err != nil { - return "", nil, err - } - fieldExpression, _ = fm.FieldFor(ctx, orgID, startNs, endNs, key) - fieldExpression, _ = DataTypeCollisionHandledFieldName(key, dummyValue, fieldExpression, qbtypes.FilterOperatorUnknown) - stmts = append(stmts, fieldExpression) - } - } else { - err := addCondition(field) - if err != nil { - return "", nil, err - } - - // first if condition covers the older tests and second if condition covers the array conditions - if !bodyJSONEnabled && field.FieldContext == telemetrytypes.FieldContextBody && jsonKeyToKey != nil { - return "", nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "Group by/Aggregation isn't available for the body column") - } else if strings.Contains(field.Name, telemetrytypes.ArraySep) || strings.Contains(field.Name, telemetrytypes.ArrayAnyIndex) { - return "", nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "Group by/Aggregation isn't available for the Array Paths: %s", field.Name) - } else { - fieldExpression, _ = DataTypeCollisionHandledFieldName(field, dummyValue, fieldExpression, qbtypes.FilterOperatorUnknown) - } - - stmts = append(stmts, fieldExpression) - } - - for idx := range stmts { - stmts[idx] = sqlbuilder.Escape(stmts[idx]) - } - - multiIfStmt := fmt.Sprintf("multiIf(%s, NULL)", strings.Join(stmts, ", ")) - - return multiIfStmt, allArgs, nil -} - func GroupByKeys(keys []qbtypes.GroupByKey) []string { k := []string{} for _, key := range keys { @@ -264,6 +153,10 @@ func DataTypeCollisionHandledFieldName(key *telemetrytypes.TelemetryFieldKey, va } else if hasString(v) { tblFieldName, value = castString(tblFieldName), toStrings(v) } + case bool: + // a bool can't equal a number; compare as strings (type-safe, matches + // nothing) instead of erroring with "Bad get: ... Float64" (CH 170). + tblFieldName, value = castString(tblFieldName), fmt.Sprintf("%t", v) } case telemetrytypes.FieldDataTypeBool, diff --git a/pkg/querybuilder/key_resolution.go b/pkg/querybuilder/key_resolution.go index 287b6409bf7..a96959f7d51 100644 --- a/pkg/querybuilder/key_resolution.go +++ b/pkg/querybuilder/key_resolution.go @@ -79,28 +79,26 @@ func NewKeyNotFoundWarning(name string) string { // key is honored as-is; a bare key defaults to attribute context with the data type // inferred from the operand, or fanned out across string/number/bool without one. func SynthesizeKeys(field *telemetrytypes.TelemetryFieldKey, value any) []*telemetrytypes.TelemetryFieldKey { - base := *field - if base.FieldContext == telemetrytypes.FieldContextUnspecified { - base.FieldContext = telemetrytypes.FieldContextAttribute + fieldContext := field.FieldContext + if fieldContext == telemetrytypes.FieldContextUnspecified { + fieldContext = telemetrytypes.FieldContextAttribute } + fieldDataType := field.FieldDataType // Resource values are strings; pin the type so operand coercion applies. - if base.FieldContext == telemetrytypes.FieldContextResource && - base.FieldDataType == telemetrytypes.FieldDataTypeUnspecified { - base.FieldDataType = telemetrytypes.FieldDataTypeString + if fieldContext == telemetrytypes.FieldContextResource && + fieldDataType == telemetrytypes.FieldDataTypeUnspecified { + fieldDataType = telemetrytypes.FieldDataTypeString } // A set data type needs only one synthesized key. - if base.FieldDataType != telemetrytypes.FieldDataTypeUnspecified { - clone := base - return []*telemetrytypes.TelemetryFieldKey{&clone} + if fieldDataType != telemetrytypes.FieldDataTypeUnspecified { + return []*telemetrytypes.TelemetryFieldKey{telemetrytypes.NewTelemetryFieldKey(field.Name, fieldContext, fieldDataType)} } dataTypes := inferDataTypesFromOperand(value) keys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(dataTypes)) for _, dt := range dataTypes { - clone := base - clone.FieldDataType = dt - keys = append(keys, &clone) + keys = append(keys, telemetrytypes.NewTelemetryFieldKey(field.Name, fieldContext, dt)) } return keys } diff --git a/pkg/querybuilder/where_clause_visitor.go b/pkg/querybuilder/where_clause_visitor.go index a43ec79429d..f3dda70c249 100644 --- a/pkg/querybuilder/where_clause_visitor.go +++ b/pkg/querybuilder/where_clause_visitor.go @@ -371,24 +371,6 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext key := v.Visit(ctx.Key()).(*telemetrytypes.TelemetryFieldKey) matching := MatchingFieldKeys(key, v.fieldKeys) - // Skip resource filtering on the main table when a sub-query covers it; a resolving - // condition builder applies this itself in ConditionForKeys. - if _, resolvesKeys := v.conditionBuilder.(qbtypes.ResolvingConditionBuilder); v.skipResourceFilter && len(matching) > 0 && !resolvesKeys { - resolved, warning := ResolveKeys(key, matching) - // emit the ambiguity warning even when the term is skipped below - v.addWarnings([]string{warning}, len(matching) > 1) - filtered := []*telemetrytypes.TelemetryFieldKey{} - for _, k := range resolved { - if k.FieldContext != telemetrytypes.FieldContextResource { - filtered = append(filtered, k) - } - } - if len(filtered) == 0 { - return SkipConditionLiteral - } - matching = filtered - } - // Handle EXISTS specially if ctx.EXISTS() != nil { op := qbtypes.FilterOperatorExists @@ -858,18 +840,7 @@ func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any { // buildConditions invokes the condition builder for a filter term, folding its // warnings/errors into visitor state; returns false if an error was recorded. func (v *filterExpressionVisitor) buildConditions(key *telemetrytypes.TelemetryFieldKey, matching []*telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) ([]string, bool) { - var ( - conds []string - warns []string - err error - ) - // A resolving condition builder owns key resolution, so hand it the raw key + full map; - // other signals use the pre-matched ConditionFor path. - if rcb, ok := v.conditionBuilder.(qbtypes.ResolvingConditionBuilder); ok { - conds, warns, err = rcb.ConditionForKeys(v.context, v.orgID, v.startNs, v.endNs, key, v.fieldKeys, qbtypes.ConditionBuilderOptions{SkipResourceFilter: v.skipResourceFilter}, op, value, v.builder) - } else { - conds, warns, err = v.conditionBuilder.ConditionFor(v.context, v.orgID, v.startNs, v.endNs, key, matching, op, value, v.builder) - } + conds, warns, err := v.conditionBuilder.ConditionFor(v.context, v.orgID, v.startNs, v.endNs, key, v.fieldKeys, qbtypes.ConditionBuilderOptions{SkipResourceFilter: v.skipResourceFilter}, op, value, v.builder) if err != nil { _, _, _, _, errURL, _ := errors.Unwrapb(err) assignIfEmpty(&v.mainErrorURL, errURL) diff --git a/pkg/querybuilder/where_clause_visitor_test.go b/pkg/querybuilder/where_clause_visitor_test.go index ed73304023b..cee56b252cd 100644 --- a/pkg/querybuilder/where_clause_visitor_test.go +++ b/pkg/querybuilder/where_clause_visitor_test.go @@ -741,6 +741,9 @@ var visitTestKeys = map[string][]*telemetrytypes.TelemetryFieldKey{ {Name: "by", FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString}}, "cz": {{Name: "cz", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeNumber}, {Name: "cz", FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString}}, + // full-text column: a real intrinsic (log context, never resource) so it resolves via the + // map and is not dropped under SkipResourceFilter — mirrors logs' DefaultFullTextColumn. + "body": {{Name: "body", FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString}}, } type resourceConditionBuilder struct{} @@ -751,7 +754,8 @@ func (b *resourceConditionBuilder) ConditionFor( _ uint64, _ uint64, key *telemetrytypes.TelemetryFieldKey, - fieldKeysForName []*telemetrytypes.TelemetryFieldKey, + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, + _ qbtypes.ConditionBuilderOptions, operator qbtypes.FilterOperator, _ any, _ *sqlbuilder.SelectBuilder, @@ -762,7 +766,7 @@ func (b *resourceConditionBuilder) ConditionFor( return nil, nil, nil } - keys, warning := ResolveKeys(key, fieldKeysForName) + keys, warning := ResolveKeys(key, MatchingFieldKeys(key, fieldKeys)) var warnings []string if warning != "" { warnings = append(warnings, warning) @@ -787,7 +791,8 @@ func (b *conditionBuilder) ConditionFor( _ uint64, _ uint64, key *telemetrytypes.TelemetryFieldKey, - fieldKeysForName []*telemetrytypes.TelemetryFieldKey, + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, + options qbtypes.ConditionBuilderOptions, operator qbtypes.FilterOperator, _ any, _ *sqlbuilder.SelectBuilder, @@ -803,7 +808,7 @@ func (b *conditionBuilder) ConditionFor( return []string{fmt.Sprintf("%s_cond", key.Name)}, nil, nil } - keys, warning := ResolveKeys(key, fieldKeysForName) + keys, warning := ResolveKeys(key, MatchingFieldKeys(key, fieldKeys)) var warnings []string if warning != "" { warnings = append(warnings, warning) @@ -813,6 +818,20 @@ func (b *conditionBuilder) ConditionFor( return nil, warnings, NewKeyNotFoundError(key.Name) } + // A resource sub-query already covers the term; drop resource keys from the main query. + if options.SkipResourceFilter { + filtered := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys)) + for _, k := range keys { + if k.FieldContext != telemetrytypes.FieldContextResource { + filtered = append(filtered, k) + } + } + if len(filtered) == 0 { + return nil, warnings, nil + } + keys = filtered + } + conds := make([]string, 0, len(keys)) for _, k := range keys { conds = append(conds, fmt.Sprintf("%s_cond", k.Name)) @@ -850,7 +869,7 @@ func visitComparisonOpts(t *testing.T) (rsbOpts, sbOpts FilterExprVisitorOpts) { // bodyCol is the full-text column; conditionBuilder returns "body_cond" for it. bodyCol := &telemetrytypes.TelemetryFieldKey{ Name: "body", - FieldContext: telemetrytypes.FieldContextResource, + FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString, } rsbOpts = FilterExprVisitorOpts{ diff --git a/pkg/telemetryaudit/condition_builder.go b/pkg/telemetryaudit/condition_builder.go index 56c5ed753ea..cf34c69026f 100644 --- a/pkg/telemetryaudit/condition_builder.go +++ b/pkg/telemetryaudit/condition_builder.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/querybuilder" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" @@ -30,11 +29,6 @@ func (c *conditionBuilder) conditionFor( value any, sb *sqlbuilder.SelectBuilder, ) (string, error) { - columns, err := c.fm.ColumnFor(ctx, orgID, startNs, endNs, key) - if err != nil { - return "", err - } - if operator.IsStringSearchOperator() { value = querybuilder.FormatValueForContains(value) } @@ -114,60 +108,15 @@ func (c *conditionBuilder) conditionFor( } return sb.And(conditions...), nil case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists: - var value any - column := columns[0] - - switch column.Type.GetType() { - case schema.ColumnTypeEnumJSON: - if operator == qbtypes.FilterOperatorExists { - return sb.IsNotNull(fieldExpression), nil - } - return sb.IsNull(fieldExpression), nil - case schema.ColumnTypeEnumLowCardinality: - switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() { - case schema.ColumnTypeEnumString: - value = "" - if operator == qbtypes.FilterOperatorExists { - return sb.NE(fieldExpression, value), nil - } - return sb.E(fieldExpression, value), nil - default: - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for low cardinality column type %s", elementType) - } - case schema.ColumnTypeEnumString: - value = "" - if operator == qbtypes.FilterOperatorExists { - return sb.NE(fieldExpression, value), nil - } - return sb.E(fieldExpression, value), nil - case schema.ColumnTypeEnumUInt64, schema.ColumnTypeEnumUInt32, schema.ColumnTypeEnumUInt8: - value = 0 - if operator == qbtypes.FilterOperatorExists { - return sb.NE(fieldExpression, value), nil - } - return sb.E(fieldExpression, value), nil - case schema.ColumnTypeEnumMap: - keyType := column.Type.(schema.MapColumnType).KeyType - if _, ok := keyType.(schema.LowCardinalityColumnType); !ok { - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type) - } - - switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() { - case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64: - leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name) - if key.Materialized { - leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key) - } - if operator == qbtypes.FilterOperatorExists { - return sb.E(leftOperand, true), nil - } - return sb.NE(leftOperand, true), nil - default: - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType) - } - default: - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", column.Type) + columns, err := c.fm.ColumnFor(ctx, orgID, startNs, endNs, key) + if err != nil { + return "", err + } + pred, err := querybuilder.ExistsExpression(columns, key, startNs, endNs, fieldExpression, operator == qbtypes.FilterOperatorExists) + if err != nil { + return "", err } + return sqlbuilder.Escape(pred), nil } return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator: %v", operator) } @@ -178,7 +127,8 @@ func (c *conditionBuilder) ConditionFor( startNs uint64, endNs uint64, key *telemetrytypes.TelemetryFieldKey, - fieldKeysForName []*telemetrytypes.TelemetryFieldKey, + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, + options qbtypes.ConditionBuilderOptions, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder, @@ -189,7 +139,7 @@ func (c *conditionBuilder) ConditionFor( return nil, nil, err } - keys, warning := querybuilder.ResolveKeys(key, fieldKeysForName) + keys, warning := querybuilder.ResolveKeys(key, querybuilder.MatchingFieldKeys(key, fieldKeys)) var warnings []string if warning != "" { warnings = append(warnings, warning) @@ -198,6 +148,20 @@ func (c *conditionBuilder) ConditionFor( return nil, warnings, querybuilder.NewKeyNotFoundError(key.Name) } + // Drop resource keys the sub-query already covers; if none remain, skip the term (not an error). + if options.SkipResourceFilter { + filtered := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys)) + for _, k := range keys { + if k.FieldContext != telemetrytypes.FieldContextResource { + filtered = append(filtered, k) + } + } + if len(filtered) == 0 { + return nil, warnings, nil + } + keys = filtered + } + conds := make([]string, 0, len(keys)) for _, k := range keys { cond, err := c.conditionForKey(ctx, orgID, startNs, endNs, k, operator, value, sb) diff --git a/pkg/telemetryaudit/field_mapper.go b/pkg/telemetryaudit/field_mapper.go index c3df68c6240..f093ec319dc 100644 --- a/pkg/telemetryaudit/field_mapper.go +++ b/pkg/telemetryaudit/field_mapper.go @@ -6,6 +6,7 @@ import ( schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/querybuilder" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/valuer" @@ -101,8 +102,10 @@ func (m *fieldMapper) ColumnExpressionFor( orgID valuer.UUID, tsStart, tsEnd uint64, field *telemetrytypes.TelemetryFieldKey, + requiredDataType telemetrytypes.FieldDataType, keys map[string][]*telemetrytypes.TelemetryFieldKey, ) (string, error) { + resolved := field fieldExpression, err := m.FieldFor(ctx, orgID, tsStart, tsEnd, field) if errors.Is(err, qbtypes.ErrColumnNotFound) { keysForField := keys[field.Name] @@ -115,10 +118,31 @@ func (m *fieldMapper) ColumnExpressionFor( return "", wrappedErr } } else { + resolved = keysForField[0] fieldExpression, _ = m.FieldFor(ctx, orgID, tsStart, tsEnd, keysForField[0]) } } + // Group-by/order (String) and aggregation (String/Float64): exists-guarded and coerced + // to requiredDataType, returned bare (the caller adds any alias). Raw select + // (Unspecified) returns the aliased column expression. + if requiredDataType != telemetrytypes.FieldDataTypeUnspecified { + var dummyValue any = "" + if requiredDataType == telemetrytypes.FieldDataTypeFloat64 { + dummyValue = 0.0 + } + columns, err := m.getColumn(ctx, resolved) + if err != nil { + return "", err + } + guard, err := querybuilder.ExistsExpression(columns, resolved, tsStart, tsEnd, fieldExpression, true) + if err != nil { + return "", err + } + coerced, _ := querybuilder.DataTypeCollisionHandledFieldName(resolved, dummyValue, fieldExpression, qbtypes.FilterOperatorUnknown) + return fmt.Sprintf("multiIf(%s, %s, NULL)", guard, coerced), nil + } + return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil } diff --git a/pkg/telemetryaudit/statement_builder.go b/pkg/telemetryaudit/statement_builder.go index c754016a1c7..6e229f1f0a6 100644 --- a/pkg/telemetryaudit/statement_builder.go +++ b/pkg/telemetryaudit/statement_builder.go @@ -25,7 +25,6 @@ type auditQueryStatementBuilder struct { resourceFilterStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation] aggExprRewriter qbtypes.AggExprRewriter fullTextColumn *telemetrytypes.TelemetryFieldKey - jsonKeyToKey qbtypes.JsonKeyToFieldFunc } var _ qbtypes.StatementBuilder[qbtypes.LogAggregation] = (*auditQueryStatementBuilder)(nil) @@ -37,7 +36,6 @@ func NewAuditQueryStatementBuilder( conditionBuilder qbtypes.ConditionBuilder, aggExprRewriter qbtypes.AggExprRewriter, fullTextColumn *telemetrytypes.TelemetryFieldKey, - jsonKeyToKey qbtypes.JsonKeyToFieldFunc, flagger flagger.Flagger, ) *auditQueryStatementBuilder { auditSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryaudit") @@ -61,7 +59,6 @@ func NewAuditQueryStatementBuilder( resourceFilterStmtBuilder: resourceFilterStmtBuilder, aggExprRewriter: aggExprRewriter, fullTextColumn: fullTextColumn, - jsonKeyToKey: jsonKeyToKey, } } @@ -245,7 +242,7 @@ func (b *auditQueryStatementBuilder) buildListQuery( continue } - colExpr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &query.SelectFields[index], keys) + colExpr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &query.SelectFields[index], telemetrytypes.FieldDataTypeUnspecified, keys) if err != nil { return nil, err } @@ -261,7 +258,7 @@ func (b *auditQueryStatementBuilder) buildListQuery( } for _, orderBy := range query.Order { - colExpr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &orderBy.Key.TelemetryFieldKey, keys) + colExpr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &orderBy.Key.TelemetryFieldKey, telemetrytypes.FieldDataTypeUnspecified, keys) if err != nil { return nil, err } @@ -323,13 +320,12 @@ func (b *auditQueryStatementBuilder) buildTimeSeriesQuery( fieldNames := make([]string, 0, len(query.GroupBy)) for _, gb := range query.GroupBy { - expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, orgID, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey, false) + expr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &gb.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return nil, err } - colExpr := fmt.Sprintf("toString(%s) AS `%s`", expr, gb.Name) - allGroupByArgs = append(allGroupByArgs, args...) + colExpr := fmt.Sprintf("toString(%s) AS `%s`", sqlbuilder.Escape(expr), gb.Name) sb.SelectMore(colExpr) fieldNames = append(fieldNames, fmt.Sprintf("`%s`", gb.Name)) } @@ -459,13 +455,12 @@ func (b *auditQueryStatementBuilder) buildScalarQuery( var allGroupByArgs []any for _, gb := range query.GroupBy { - expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, orgID, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey, false) + expr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &gb.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return nil, err } - colExpr := fmt.Sprintf("toString(%s) AS `%s`", expr, gb.Name) - allGroupByArgs = append(allGroupByArgs, args...) + colExpr := fmt.Sprintf("toString(%s) AS `%s`", sqlbuilder.Escape(expr), gb.Name) sb.SelectMore(colExpr) } diff --git a/pkg/telemetryaudit/statement_builder_test.go b/pkg/telemetryaudit/statement_builder_test.go index 02721a47919..e260f4cb581 100644 --- a/pkg/telemetryaudit/statement_builder_test.go +++ b/pkg/telemetryaudit/statement_builder_test.go @@ -56,7 +56,7 @@ func newTestAuditStatementBuilder(t *testing.T) *auditQueryStatementBuilder { fm := NewFieldMapper() cb := NewConditionBuilder(fm) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) return NewAuditQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -65,7 +65,6 @@ func newTestAuditStatementBuilder(t *testing.T) *auditQueryStatementBuilder { cb, aggExprRewriter, DefaultFullTextColumn, - nil, fl, ) } @@ -94,8 +93,8 @@ func TestStatementBuilder(t *testing.T) { Limit: 100, }, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, event_name, attributes_string, attributes_number, attributes_bool, resource, scope_string FROM signoz_audit.distributed_logs WHERE (`attribute_string_signoz$$audit$$principal$$id` = ? AND `attribute_string_signoz$$audit$$principal$$id_exists` = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{"019a-1234-abcd-5678", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 100}, + Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, event_name, attributes_string, attributes_number, attributes_bool, resource, scope_string FROM signoz_audit.distributed_logs WHERE (`attribute_string_signoz$$audit$$principal$$id` = ? AND `attribute_string_signoz$$audit$$principal$$id_exists`) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{"019a-1234-abcd-5678", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 100}, }, }, // List: all failed actions (materialized outcome filter) @@ -111,8 +110,8 @@ func TestStatementBuilder(t *testing.T) { Limit: 100, }, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, event_name, attributes_string, attributes_number, attributes_bool, resource, scope_string FROM signoz_audit.distributed_logs WHERE (`attribute_string_signoz$$audit$$outcome` = ? AND `attribute_string_signoz$$audit$$outcome_exists` = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{"failure", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 100}, + Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, event_name, attributes_string, attributes_number, attributes_bool, resource, scope_string FROM signoz_audit.distributed_logs WHERE (`attribute_string_signoz$$audit$$outcome` = ? AND `attribute_string_signoz$$audit$$outcome_exists`) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{"failure", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 100}, }, }, // List: change history of a specific dashboard (two materialized column AND) @@ -145,8 +144,8 @@ func TestStatementBuilder(t *testing.T) { Limit: 100, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_audit.distributed_logs_resource WHERE (simpleJSONExtractString(labels, 'signoz.audit.resource.kind') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, event_name, attributes_string, attributes_number, attributes_bool, resource, scope_string FROM signoz_audit.distributed_logs WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (`attribute_string_signoz$$audit$$action` = ? AND `attribute_string_signoz$$audit$$action_exists` = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{"dashboard", "%signoz.audit.resource.kind%", "%signoz.audit.resource.kind\":\"dashboard%", uint64(1747945619), uint64(1747983448), "delete", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 100}, + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_audit.distributed_logs_resource WHERE (simpleJSONExtractString(labels, 'signoz.audit.resource.kind') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, event_name, attributes_string, attributes_number, attributes_bool, resource, scope_string FROM signoz_audit.distributed_logs WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (`attribute_string_signoz$$audit$$action` = ? AND `attribute_string_signoz$$audit$$action_exists`) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{"dashboard", "%signoz.audit.resource.kind%", "%signoz.audit.resource.kind\":\"dashboard%", uint64(1747945619), uint64(1747983448), "delete", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 100}, }, }, // List: all actions by service accounts (materialized principal.type) @@ -162,8 +161,8 @@ func TestStatementBuilder(t *testing.T) { Limit: 100, }, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, event_name, attributes_string, attributes_number, attributes_bool, resource, scope_string FROM signoz_audit.distributed_logs WHERE (`attribute_string_signoz$$audit$$principal$$type` = ? AND `attribute_string_signoz$$audit$$principal$$type_exists` = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{"service_account", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 100}, + Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, event_name, attributes_string, attributes_number, attributes_bool, resource, scope_string FROM signoz_audit.distributed_logs WHERE (`attribute_string_signoz$$audit$$principal$$type` = ? AND `attribute_string_signoz$$audit$$principal$$type_exists`) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{"service_account", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 100}, }, }, // Scalar: alert — count forbidden errors (outcome + action AND) @@ -182,8 +181,8 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "SELECT count() AS __result_0 FROM signoz_audit.distributed_logs WHERE ((`attribute_string_signoz$$audit$$outcome` = ? AND `attribute_string_signoz$$audit$$outcome_exists` = ?) AND (`attribute_string_signoz$$audit$$action` = ? AND `attribute_string_signoz$$audit$$action_exists` = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY __result_0 DESC", - Args: []any{"failure", true, "update", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)}, + Query: "SELECT count() AS __result_0 FROM signoz_audit.distributed_logs WHERE ((`attribute_string_signoz$$audit$$outcome` = ? AND `attribute_string_signoz$$audit$$outcome_exists`) AND (`attribute_string_signoz$$audit$$action` = ? AND `attribute_string_signoz$$audit$$action_exists`)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY __result_0 DESC", + Args: []any{"failure", "update", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)}, }, }, // TimeSeries: failures grouped by principal email with top-N limit @@ -206,8 +205,8 @@ func TestStatementBuilder(t *testing.T) { Limit: 5, }, expected: qbtypes.Statement{ - Query: "WITH __limit_cte AS (SELECT toString(multiIf(`attribute_string_signoz$$audit$$principal$$email_exists` = ?, `attribute_string_signoz$$audit$$principal$$email`, NULL)) AS `signoz.audit.principal.email`, count() AS __result_0 FROM signoz_audit.distributed_logs WHERE (`attribute_string_signoz$$audit$$outcome` = ? AND `attribute_string_signoz$$audit$$outcome_exists` = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `signoz.audit.principal.email` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toString(multiIf(`attribute_string_signoz$$audit$$principal$$email_exists` = ?, `attribute_string_signoz$$audit$$principal$$email`, NULL)) AS `signoz.audit.principal.email`, count() AS __result_0 FROM signoz_audit.distributed_logs WHERE (`attribute_string_signoz$$audit$$outcome` = ? AND `attribute_string_signoz$$audit$$outcome_exists` = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`signoz.audit.principal.email`) GLOBAL IN (SELECT `signoz.audit.principal.email` FROM __limit_cte) GROUP BY ts, `signoz.audit.principal.email`", - Args: []any{true, "failure", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 5, true, "failure", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)}, + Query: "WITH __limit_cte AS (SELECT toString(multiIf(`attribute_string_signoz$$audit$$principal$$email_exists`, `attribute_string_signoz$$audit$$principal$$email`, NULL)) AS `signoz.audit.principal.email`, count() AS __result_0 FROM signoz_audit.distributed_logs WHERE (`attribute_string_signoz$$audit$$outcome` = ? AND `attribute_string_signoz$$audit$$outcome_exists`) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `signoz.audit.principal.email` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toString(multiIf(`attribute_string_signoz$$audit$$principal$$email_exists`, `attribute_string_signoz$$audit$$principal$$email`, NULL)) AS `signoz.audit.principal.email`, count() AS __result_0 FROM signoz_audit.distributed_logs WHERE (`attribute_string_signoz$$audit$$outcome` = ? AND `attribute_string_signoz$$audit$$outcome_exists`) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`signoz.audit.principal.email`) GLOBAL IN (SELECT `signoz.audit.principal.email` FROM __limit_cte) GROUP BY ts, `signoz.audit.principal.email`", + Args: []any{"failure", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 5, "failure", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)}, }, }, } diff --git a/pkg/telemetrylogs/condition_builder.go b/pkg/telemetrylogs/condition_builder.go index 57ad3ebacee..045629057ee 100644 --- a/pkg/telemetrylogs/condition_builder.go +++ b/pkg/telemetrylogs/condition_builder.go @@ -21,6 +21,8 @@ type conditionBuilder struct { fl flagger.Flagger } +var _ qbtypes.ConditionBuilder = (*conditionBuilder)(nil) + func NewConditionBuilder(fm qbtypes.FieldMapper, fl flagger.Flagger) *conditionBuilder { return &conditionBuilder{fm: fm, fl: fl} } @@ -64,6 +66,16 @@ func (c *conditionBuilder) conditionForArrayFunction( if c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) { // JSON access plan: data-type collision handling, nested array paths. valueType, needle := InferDataType(needle, operator, key) + // A not-found (synthesized) body path carries no metadata plan; build an exhaustive + // one so the query runs against the underlying data (with the not-found warning) + // instead of erroring, matching the regular-operator path. + if len(key.JSONPlan) == 0 { + keyCopy := telemetrytypes.NewTelemetryFieldKey(key.Name, key.FieldContext, key.FieldDataType) + if err := keyCopy.SetExhaustiveJSONAccessPlan(telemetrytypes.JSONColumnMetadata{BaseColumn: LogsV2BodyV2Column}, valueType); err != nil { + return "", err + } + key = keyCopy + } return NewJSONConditionBuilder(key, valueType).buildArrayFunctionCondition(operator, needle, sb) } @@ -142,13 +154,22 @@ func (c *conditionBuilder) conditionForHasToken( return fmt.Sprintf("hasToken(LOWER(%s), LOWER(%s))", bodyMessageField, sb.Var(needle)), nil } if key.FieldContext == telemetrytypes.FieldContextBody { + // A not-found (synthesized) body path carries no metadata plan; build an exhaustive + // one so hasToken runs against the underlying data instead of erroring. + if len(key.JSONPlan) == 0 { + keyCopy := telemetrytypes.NewTelemetryFieldKey(key.Name, key.FieldContext, key.FieldDataType) + if err := keyCopy.SetExhaustiveJSONAccessPlan(telemetrytypes.JSONColumnMetadata{BaseColumn: LogsV2BodyV2Column}, telemetrytypes.FieldDataTypeString); err != nil { + return "", err + } + key = keyCopy + } return NewJSONConditionBuilder(key, telemetrytypes.FieldDataTypeString).buildTokenFunctionCondition(needle, sb) } return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "function `hasToken` only supports the body field or a body JSON string field as first parameter").WithUrl(hasTokenFunctionDocURL) } -func (c *conditionBuilder) conditionFor( +func (c *conditionBuilder) conditionForResolvedKey( ctx context.Context, orgID valuer.UUID, startNs, endNs uint64, @@ -163,6 +184,10 @@ func (c *conditionBuilder) conditionFor( } columns, err := c.fm.ColumnFor(ctx, orgID, startNs, endNs, key) + if errors.Is(err, qbtypes.ErrColumnNotFound) && key.FieldContext == telemetrytypes.FieldContextUnspecified { + key = telemetrytypes.NewTelemetryFieldKey(key.Name, telemetrytypes.FieldContextBody, key.FieldDataType) + columns, err = c.fm.ColumnFor(ctx, orgID, startNs, endNs, key) + } if err != nil { return "", err } @@ -176,6 +201,15 @@ func (c *conditionBuilder) conditionFor( for _, column := range columns { if column.Type.GetType() == schema.ColumnTypeEnumJSON && isBodyJSONSearch(key, columns) && c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) && key.Name != messageSubField { valueType, value := InferDataType(value, operator, key) + if len(key.JSONPlan) == 0 { + keyCopy := telemetrytypes.NewTelemetryFieldKey(key.Name, key.FieldContext, key.FieldDataType) + if err := keyCopy.SetExhaustiveJSONAccessPlan( + telemetrytypes.JSONColumnMetadata{BaseColumn: LogsV2BodyV2Column}, valueType, + ); err != nil { + return "", err + } + key = keyCopy + } cond, err := NewJSONConditionBuilder(key, valueType).buildJSONCondition(operator, value, sb) if err != nil { return "", err @@ -244,6 +278,19 @@ func (c *conditionBuilder) conditionFor( case qbtypes.FilterOperatorNotILike: return sb.NotILike(fieldExpression, value), nil + case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists: + if isBodyJSONSearch(key, columns) && !c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) { + if operator == qbtypes.FilterOperatorExists { + return GetBodyJSONKeyForExists(ctx, key, operator, value), nil + } + return "NOT " + GetBodyJSONKeyForExists(ctx, key, operator, value), nil + } + pred, err := querybuilder.ExistsExpression(columns, key, startNs, endNs, fieldExpression, operator == qbtypes.FilterOperatorExists) + if err != nil { + return "", err + } + return sqlbuilder.Escape(pred), nil + case qbtypes.FilterOperatorContains: return sb.ILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil case qbtypes.FilterOperatorNotContains: @@ -301,133 +348,87 @@ func (c *conditionBuilder) conditionFor( } return sb.And(conditions...), nil - // exists and not exists - // in the UI based query builder, `exists` and `not exists` are used for - // key membership checks, so depending on the column type, the condition changes - case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists: - if isBodyJSONSearch(key, columns) && !c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) { - if operator == qbtypes.FilterOperatorExists { - return GetBodyJSONKeyForExists(ctx, key, operator, value), nil - } else { - return "NOT " + GetBodyJSONKeyForExists(ctx, key, operator, value), nil - } - } - - var value any - column := columns[0] - if len(key.Evolutions) > 0 { - // we will use the corresponding column and its evolution entry for the query - newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs) - if err != nil { - return "", err - } - - if len(newColumns) == 0 { - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "no valid evolution found for field %s in the given time range", key.Name) - } - - // This mean tblFieldName is with multiIf, we just need to do a null check. - if len(newColumns) > 1 { - if operator == qbtypes.FilterOperatorExists { - return sb.IsNotNull(fieldExpression), nil - } else { - return sb.IsNull(fieldExpression), nil - } - } - - // otherwise we have to find the correct exist operator based on the column type - column = newColumns[0] - } - - switch column.Type.GetType() { - case schema.ColumnTypeEnumJSON: - if operator == qbtypes.FilterOperatorExists { - return sb.IsNotNull(fieldExpression), nil - } - return sb.IsNull(fieldExpression), nil - case schema.ColumnTypeEnumLowCardinality: - switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() { - case schema.ColumnTypeEnumString: - value = "" - if operator == qbtypes.FilterOperatorExists { - return sb.NE(fieldExpression, value), nil - } - return sb.E(fieldExpression, value), nil - default: - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for low cardinality column type %s", elementType) - } - case schema.ColumnTypeEnumString: - value = "" - if operator == qbtypes.FilterOperatorExists { - return sb.NE(fieldExpression, value), nil - } else { - return sb.E(fieldExpression, value), nil - } - case schema.ColumnTypeEnumUInt64, schema.ColumnTypeEnumUInt32, schema.ColumnTypeEnumUInt8: - value = 0 - if operator == qbtypes.FilterOperatorExists { - return sb.NE(fieldExpression, value), nil - } else { - return sb.E(fieldExpression, value), nil - } - case schema.ColumnTypeEnumMap: - keyType := column.Type.(schema.MapColumnType).KeyType - if _, ok := keyType.(schema.LowCardinalityColumnType); !ok { - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type) - } - - switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() { - case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64: - leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name) - if key.Materialized { - leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key) - } - if operator == qbtypes.FilterOperatorExists { - return sb.E(leftOperand, true), nil - } else { - return sb.NE(leftOperand, true), nil - } - default: - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType) - } - default: - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", column.Type) - - } } return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator: %v", operator) } +// candidateLookupKeys returns the metadata map only for fold-contexts, where CandidateKeys +// would otherwise fold the prefix into the key name. Handing it the map lets a same-named +// key under another context resolve first (as ColumnExpressionFor does). Strict contexts +// (resource/attribute/scope) get nil so their explicit context is always honored. +func candidateLookupKeys(key *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) map[string][]*telemetrytypes.TelemetryFieldKey { + if key.FieldContext == telemetrytypes.FieldContextLog { + return fieldKeys + } + return nil +} + func (c *conditionBuilder) ConditionFor( ctx context.Context, orgID valuer.UUID, startNs uint64, endNs uint64, key *telemetrytypes.TelemetryFieldKey, - fieldKeysForName []*telemetrytypes.TelemetryFieldKey, + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, + options qbtypes.ConditionBuilderOptions, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder, ) ([]string, []string, error) { + matches := querybuilder.MatchingFieldKeys(key, fieldKeys) + skipResourceFilter := options.SkipResourceFilter - keys, warning := querybuilder.ResolveKeys(key, fieldKeysForName) + keys, warning := querybuilder.ResolveKeys(key, matches) var warnings []string if warning != "" { warnings = append(warnings, warning) } + synthesized := false if len(keys) == 0 { - // No known field key matched. Legacy string-body mode still searches unknown - // Body-context keys as body JSON paths; JSON-body mode requires a metadata match. + _, isIntrinsicColumn := logsV2Columns[key.Name] switch { case key.FieldContext == telemetrytypes.FieldContextBody && key.Name == "": return nil, warnings, errors.NewInvalidInputf(errors.CodeInvalidInput, "missing key for body json search - expected key of the form `body.key` (ex: `body.status`)") - case key.FieldContext == telemetrytypes.FieldContextBody && - !c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)): + case key.FieldContext == telemetrytypes.FieldContextLog && isIntrinsicColumn: keys = []*telemetrytypes.TelemetryFieldKey{key} default: - return nil, warnings, querybuilder.NewKeyNotFoundError(key.Name) + // Fold-contexts get the metadata map so a same-named key under another context + // wins before the prefix folds into the key name (matching ColumnExpressionFor); + // strict contexts pass nil and stay honored as-is. + keys = c.fm.CandidateKeys(ctx, orgID, key, value, candidateLookupKeys(key, fieldKeys)) + if operator.IsFunctionOperator() { + if key.FieldContext != telemetrytypes.FieldContextBody { + // has/hasAny/hasAll/hasToken are body-JSON only + return nil, warnings, querybuilder.NewFunctionUnsupportedError(operator) + } + bodyKeys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys)) + for _, k := range keys { + if k.FieldContext == telemetrytypes.FieldContextBody { + bodyKeys = append(bodyKeys, k) + } + } + keys = bodyKeys + } + if len(keys) == 0 { + return nil, warnings, querybuilder.NewKeyNotFoundError(key.Name) + } + synthesized = true + warnings = append(warnings, querybuilder.NewKeyNotFoundWarning(key.Name)) + } + } + + if skipResourceFilter && !synthesized { + filtered := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys)) + for _, k := range keys { + if k.FieldContext != telemetrytypes.FieldContextResource { + filtered = append(filtered, k) + } + } + if len(filtered) == 0 { + return nil, warnings, nil } + keys = filtered } conds := make([]string, 0, len(keys)) @@ -468,7 +469,7 @@ func (c *conditionBuilder) conditionForKey( sb *sqlbuilder.SelectBuilder, ) (string, error) { - condition, err := c.conditionFor(ctx, orgID, startNs, endNs, key, operator, value, sb) + condition, err := c.conditionForResolvedKey(ctx, orgID, startNs, endNs, key, operator, value, sb) if err != nil { return "", err } @@ -491,7 +492,7 @@ func (c *conditionBuilder) conditionForKey( } if buildExistCondition { - existsCondition, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorExists, nil, sb) + existsCondition, err := c.conditionForResolvedKey(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorExists, nil, sb) if err != nil { return "", err } diff --git a/pkg/telemetrylogs/condition_builder_test.go b/pkg/telemetrylogs/condition_builder_test.go index 5e54c8c00a6..7d5dfdbe19b 100644 --- a/pkg/telemetrylogs/condition_builder_test.go +++ b/pkg/telemetrylogs/condition_builder_test.go @@ -55,7 +55,7 @@ func TestExistsConditionForWithEvolutions(t *testing.T) { }, operator: qbtypes.FilterOperatorExists, value: nil, - expectedSQL: "WHERE resource.`service.name`::String IS NOT NULL", + expectedSQL: "WHERE resource.`service.name` IS NOT NULL", expectedError: nil, }, { @@ -87,8 +87,8 @@ func TestExistsConditionForWithEvolutions(t *testing.T) { }, operator: qbtypes.FilterOperatorExists, value: nil, - expectedSQL: "WHERE mapContains(resources_string, 'service.name') = ?", - expectedArgs: []any{true}, + expectedSQL: "WHERE mapContains(resources_string, 'service.name')", + expectedArgs: nil, expectedError: nil, }, { @@ -132,7 +132,7 @@ func TestExistsConditionForWithEvolutions(t *testing.T) { for _, tc := range testCases { sb := sqlbuilder.NewSelectBuilder() t.Run(tc.name, func(t *testing.T) { - cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, tc.startTs, tc.endTs, &tc.key, []*telemetrytypes.TelemetryFieldKey{&tc.key}, tc.operator, tc.value, sb) + cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, tc.startTs, tc.endTs, &tc.key, map[string][]*telemetrytypes.TelemetryFieldKey{tc.key.Name: {&tc.key}}, qbtypes.ConditionBuilderOptions{}, tc.operator, tc.value, sb) sb.Where(cond...) if tc.expectedError != nil { @@ -182,8 +182,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorGreaterThan, value: float64(100), - expectedSQL: "(toFloat64(attributes_number['request.duration']) > ? AND mapContains(attributes_number, 'request.duration') = ?)", - expectedArgs: []any{float64(100), true}, + expectedSQL: "(toFloat64(attributes_number['request.duration']) > ? AND mapContains(attributes_number, 'request.duration'))", + expectedArgs: []any{float64(100)}, expectedError: nil, }, { @@ -195,8 +195,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorLessThan, value: float64(1024), - expectedSQL: "(toFloat64(attributes_number['request.size']) < ? AND mapContains(attributes_number, 'request.size') = ?)", - expectedArgs: []any{float64(1024), true}, + expectedSQL: "(toFloat64(attributes_number['request.size']) < ? AND mapContains(attributes_number, 'request.size'))", + expectedArgs: []any{float64(1024)}, expectedError: nil, }, { @@ -232,8 +232,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorILike, value: "%admin%", - expectedSQL: "(LOWER(attributes_string['user.id']) LIKE LOWER(?) AND mapContains(attributes_string, 'user.id') = ?)", - expectedArgs: []any{"%admin%", true}, + expectedSQL: "(LOWER(attributes_string['user.id']) LIKE LOWER(?) AND mapContains(attributes_string, 'user.id'))", + expectedArgs: []any{"%admin%"}, expectedError: nil, }, { @@ -259,7 +259,7 @@ func TestConditionFor(t *testing.T) { operator: qbtypes.FilterOperatorContains, value: 521509198310, expectedSQL: "LOWER(attributes_string['user.id']) LIKE LOWER(?)", - expectedArgs: []any{"%521509198310%", true}, + expectedArgs: []any{"%521509198310%"}, expectedError: nil, }, { @@ -283,8 +283,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorContains, value: "admin", - expectedSQL: "(LOWER(attributes_string['user.id']) LIKE LOWER(?) AND mapContains(attributes_string, 'user.id') = ?)", - expectedArgs: []any{"%admin%", true}, + expectedSQL: "(LOWER(attributes_string['user.id']) LIKE LOWER(?) AND mapContains(attributes_string, 'user.id'))", + expectedArgs: []any{"%admin%"}, expectedError: nil, }, { @@ -330,8 +330,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorExists, value: nil, - expectedSQL: "WHERE body <> ?", - expectedArgs: []any{""}, + expectedSQL: "WHERE body <> ''", + expectedArgs: nil, expectedError: nil, }, { @@ -342,8 +342,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorNotExists, value: nil, - expectedSQL: "WHERE body = ?", - expectedArgs: []any{""}, + expectedSQL: "WHERE body = ''", + expectedArgs: nil, expectedError: nil, }, { @@ -355,8 +355,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorExists, value: nil, - expectedSQL: "mapContains(attributes_string, 'user.id') = ?", - expectedArgs: []any{true}, + expectedSQL: "mapContains(attributes_string, 'user.id')", + expectedArgs: nil, expectedError: nil, }, { @@ -368,8 +368,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorNotExists, value: nil, - expectedSQL: "mapContains(attributes_string, 'user.id') <> ?", - expectedArgs: []any{true}, + expectedSQL: "NOT mapContains(attributes_string, 'user.id')", + expectedArgs: nil, expectedError: nil, }, { @@ -382,8 +382,8 @@ func TestConditionFor(t *testing.T) { evolutions: mockEvolution, operator: qbtypes.FilterOperatorExists, value: nil, - expectedSQL: "mapContains(resources_string, 'service.name') = ?", - expectedArgs: []any{true}, + expectedSQL: "mapContains(resources_string, 'service.name')", + expectedArgs: nil, expectedError: nil, }, { @@ -396,8 +396,8 @@ func TestConditionFor(t *testing.T) { evolutions: mockEvolution, operator: qbtypes.FilterOperatorNotExists, value: nil, - expectedSQL: "mapContains(resources_string, 'service.name') <> ?", - expectedArgs: []any{true}, + expectedSQL: "NOT mapContains(resources_string, 'service.name')", + expectedArgs: nil, expectedError: nil, }, { @@ -433,8 +433,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorRegexp, value: "^https://.*\\.example\\.com.*$", - expectedSQL: "(match(attributes_string['http.url'], ?) AND mapContains(attributes_string, 'http.url') = ?)", - expectedArgs: []any{"^https://.*\\.example\\.com.*$", true}, + expectedSQL: "(match(attributes_string['http.url'], ?) AND mapContains(attributes_string, 'http.url'))", + expectedArgs: []any{"^https://.*\\.example\\.com.*$"}, expectedError: nil, }, { @@ -461,8 +461,8 @@ func TestConditionFor(t *testing.T) { evolutions: mockEvolution, operator: qbtypes.FilterOperatorRegexp, value: "frontend-.*", - expectedSQL: "WHERE (match(`resource_string_service$$name`, ?) AND `resource_string_service$$name_exists` = ?)", - expectedArgs: []any{"frontend-.*", true}, + expectedSQL: "WHERE (match(`resource_string_service$$name`, ?) AND `resource_string_service$$name_exists`)", + expectedArgs: []any{"frontend-.*"}, expectedError: nil, }, { @@ -523,7 +523,7 @@ func TestConditionFor(t *testing.T) { sb := sqlbuilder.NewSelectBuilder() t.Run(tc.name, func(t *testing.T) { tc.key.Evolutions = tc.evolutions - cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, 0, 0, &tc.key, []*telemetrytypes.TelemetryFieldKey{&tc.key}, tc.operator, tc.value, sb) + cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, 0, 0, &tc.key, map[string][]*telemetrytypes.TelemetryFieldKey{tc.key.Name: {&tc.key}}, qbtypes.ConditionBuilderOptions{}, tc.operator, tc.value, sb) sb.Where(cond...) if tc.expectedError != nil { @@ -538,6 +538,61 @@ func TestConditionFor(t *testing.T) { } } +// A prefixed key absent from metadata synthesizes both spellings: the stripped +// interpretation first, the literal `{context}.{name}` spelling second. +func TestConditionForSynthesizedPrefixedKeys(t *testing.T) { + ctx := context.Background() + fl := flaggertest.New(t) + cb := NewConditionBuilder(NewFieldMapper(fl), fl) + + t.Run("bare intrinsic column resolves to the column, not synthesized attributes", func(t *testing.T) { + sb := sqlbuilder.NewSelectBuilder() + key := telemetrytypes.TelemetryFieldKey{Name: "severity_text"} + conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "ERROR", sb) + require.NoError(t, err) + require.Len(t, conds, 1) + sb.Where(conds...) + sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + assert.Contains(t, sql, "severity_text = ?") + assert.NotContains(t, sql, "mapContains") + }) + + t.Run("attribute context", func(t *testing.T) { + sb := sqlbuilder.NewSelectBuilder() + key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldContext: telemetrytypes.FieldContextAttribute} + conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb) + require.NoError(t, err) + assert.NotEmpty(t, warnings) + require.Len(t, conds, 2) + assert.Contains(t, conds[0], "mapContains(attributes_string, 'custom.key')") + assert.Contains(t, conds[1], "mapContains(attributes_string, 'attribute.custom.key')") + }) + + t.Run("resource context", func(t *testing.T) { + sb := sqlbuilder.NewSelectBuilder() + key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldContext: telemetrytypes.FieldContextResource} + conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb) + require.NoError(t, err) + assert.NotEmpty(t, warnings) + require.Len(t, conds, 2) + assert.Contains(t, conds[0], "mapContains(resources_string, 'custom.key')") + assert.Contains(t, conds[1], "mapContains(resources_string, 'resource.custom.key')") + }) + + t.Run("log context folds to attributes then body", func(t *testing.T) { + sb := sqlbuilder.NewSelectBuilder() + key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldContext: telemetrytypes.FieldContextLog} + conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb) + require.NoError(t, err) + assert.NotEmpty(t, warnings) + require.Len(t, conds, 4) + assert.Contains(t, conds[0], "mapContains(attributes_string, 'custom.key')") + assert.Contains(t, conds[1], "mapContains(attributes_string, 'log.custom.key')") + assert.Contains(t, conds[2], `"custom"."key"`) + assert.Contains(t, conds[3], `"log"."custom"."key"`) + }) +} + func TestConditionForMultipleKeys(t *testing.T) { ctx := context.Background() @@ -579,7 +634,7 @@ func TestConditionForMultipleKeys(t *testing.T) { t.Run(tc.name, func(t *testing.T) { var err error for _, key := range tc.keys { - cond, err := conditionBuilder.conditionFor(ctx, valuer.UUID{}, 0, 0, &key, tc.operator, tc.value, sb) + cond, err := conditionBuilder.conditionForResolvedKey(ctx, valuer.UUID{}, 0, 0, &key, tc.operator, tc.value, sb) sb.Where(cond) if err != nil { t.Fatalf("Error getting condition for key %s: %v", key.Name, err) @@ -837,7 +892,7 @@ func TestConditionForJSONBodySearch(t *testing.T) { for _, tc := range testCases { sb := sqlbuilder.NewSelectBuilder() t.Run(tc.name, func(t *testing.T) { - cond, err := conditionBuilder.conditionFor(ctx, valuer.UUID{}, 0, 0, &tc.key, tc.operator, tc.value, sb) + cond, err := conditionBuilder.conditionForResolvedKey(ctx, valuer.UUID{}, 0, 0, &tc.key, tc.operator, tc.value, sb) sb.Where(cond) if tc.expectedError != nil { diff --git a/pkg/telemetrylogs/field_mapper.go b/pkg/telemetrylogs/field_mapper.go index 327bf159d00..586583b1181 100644 --- a/pkg/telemetrylogs/field_mapper.go +++ b/pkg/telemetrylogs/field_mapper.go @@ -9,13 +9,12 @@ import ( "github.com/SigNoz/signoz-otel-collector/utils" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/flagger" + "github.com/SigNoz/signoz/pkg/querybuilder" "github.com/SigNoz/signoz/pkg/types/featuretypes" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/valuer" "github.com/huandu/go-sqlbuilder" - - "golang.org/x/exp/maps" ) var ( @@ -75,7 +74,7 @@ func NewFieldMapper(fl flagger.Flagger) qbtypes.FieldMapper { func (m *fieldMapper) getColumn(ctx context.Context, orgID valuer.UUID, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) { switch key.FieldContext { case telemetrytypes.FieldContextResource: - columns := []*schema.Column{logsV2Columns["resources_string"], logsV2Columns["resource"]} + columns := []*schema.Column{logsV2Columns["resource"], logsV2Columns["resources_string"]} return columns, nil case telemetrytypes.FieldContextScope: switch key.Name { @@ -104,24 +103,12 @@ func (m *fieldMapper) getColumn(ctx context.Context, orgID valuer.UUID, key *tel } // Fall back to legacy body column return []*schema.Column{logsV2Columns["body"]}, nil - case telemetrytypes.FieldContextLog, telemetrytypes.FieldContextUnspecified: + case telemetrytypes.FieldContextLog: if key.Name == LogsV2BodyColumn && m.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) { return []*schema.Column{logsV2Columns[messageSubColumn]}, nil } col, ok := logsV2Columns[key.Name] if !ok { - // check if the key has body JSON search - if strings.HasPrefix(key.Name, telemetrytypes.BodyJSONStringSearchPrefix) { - // Use body_v2 if feature flag is enabled and we have a body condition builder - if m.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) { - // TODO(Piyush): Update this to support multiple JSON columns based on evolutions - // i.e return both the body json and body json promoted and let the evolutions decide which one to use - // based on the query range time. - return []*schema.Column{logsV2Columns[LogsV2BodyV2Column]}, nil - } - // Fall back to legacy body column - return []*schema.Column{logsV2Columns["body"]}, nil - } return nil, qbtypes.ErrColumnNotFound } return []*schema.Column{col}, nil @@ -136,16 +123,9 @@ func (m *fieldMapper) FieldFor(ctx context.Context, orgID valuer.UUID, tsStart, return "", err } - var newColumns []*schema.Column - var evolutionsEntries []*telemetrytypes.EvolutionEntry - if len(key.Evolutions) > 0 { - // we will use the corresponding column and its evolution entry for the query - newColumns, evolutionsEntries, err = qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd) - if err != nil { - return "", err - } - } else { - newColumns = columns + newColumns, evolutionsEntries, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd) + if err != nil { + return "", err } exprs := []string{} @@ -204,7 +184,7 @@ func (m *fieldMapper) FieldFor(ctx context.Context, orgID valuer.UUID, tsStart, // a key could have been materialized, if so return the materialized column name if key.Materialized { exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key)) - existExpr = append(existExpr, fmt.Sprintf("%s==true", telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key))) + existExpr = append(existExpr, telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)) } else { exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name)) existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name)) @@ -242,50 +222,177 @@ func (m *fieldMapper) ColumnExpressionFor( orgID valuer.UUID, tsStart, tsEnd uint64, field *telemetrytypes.TelemetryFieldKey, + requiredDataType telemetrytypes.FieldDataType, keys map[string][]*telemetrytypes.TelemetryFieldKey, ) (string, error) { - fieldExpression, err := m.FieldFor(ctx, orgID, tsStart, tsEnd, field) - if errors.Is(err, qbtypes.ErrColumnNotFound) { - // the key didn't have the right context to be added to the query - // we try to use the context we know of - keysForField := keys[field.Name] - if len(keysForField) == 0 { - // is it a static field? - if _, ok := logsV2Columns[field.Name]; ok { - // if it is, attach the column name directly - field.FieldContext = telemetrytypes.FieldContextLog - fieldExpression, _ = m.FieldFor(ctx, orgID, tsStart, tsEnd, field) + + bodyJSONEnabled := m.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) + + var candidates []*telemetrytypes.TelemetryFieldKey + switch _, err := m.FieldFor(ctx, orgID, tsStart, tsEnd, field); { + case err == nil: + if field.FieldContext == telemetrytypes.FieldContextBody && !bodyJSONEnabled { + return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "Operation isn't available for the body column") + } + candidates = []*telemetrytypes.TelemetryFieldKey{field} + case errors.Is(err, qbtypes.ErrColumnNotFound): + if _, ok := logsV2Columns[field.Name]; ok { + field.FieldContext = telemetrytypes.FieldContextLog + candidates = []*telemetrytypes.TelemetryFieldKey{field} + break + } + candidates = keys[field.Name] + if len(candidates) == 0 { + candidates = keys[fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), field.Name)] + } + if len(candidates) == 0 { + // synthesized attribute candidates first, body path last; legacy body + // doesn't support group by/select, so bare keys keep attributes only + for _, key := range m.CandidateKeys(ctx, orgID, field, nil, nil) { + if !bodyJSONEnabled && field.FieldContext == telemetrytypes.FieldContextUnspecified && + key.FieldContext == telemetrytypes.FieldContextBody { + continue + } + candidates = append(candidates, key) + } + } + if len(candidates) == 0 { + return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "field `%s` not found", field.Name) + } + default: + return "", err + } + + // Group-by/order (String) and aggregation (String/Float64): every candidate is + // exists-guarded and coerced to requiredDataType, in a single multiIf. Raw select + // (Unspecified) keeps the lighter native shape below. + if requiredDataType != telemetrytypes.FieldDataTypeUnspecified { + // arrays cannot sit inside Nullable/multiIf, so a lone array candidate stays bare + if len(candidates) == 1 && (strings.Contains(candidates[0].Name, telemetrytypes.ArraySep) || + strings.Contains(candidates[0].Name, telemetrytypes.ArrayAnyIndex) || + candidates[0].FieldDataType.IsArray()) { + return m.FieldFor(ctx, orgID, tsStart, tsEnd, candidates[0]) + } + var dummyValue any = "" + if requiredDataType == telemetrytypes.FieldDataTypeFloat64 { + dummyValue = 0.0 + } + var stmts []string + for _, key := range candidates { + guard, err := m.existsExpressionFor(ctx, orgID, tsStart, tsEnd, key, true) + if err != nil { + return "", err + } + var fieldExpression string + if key.FieldContext == telemetrytypes.FieldContextBody && !bodyJSONEnabled { + fieldExpression, _ = GetBodyJSONKey(ctx, key, qbtypes.FilterOperatorUnknown, dummyValue) } else { - // - the context is not provided - // - there are not keys for the field - // - it is not a static field - // - the next best thing to do is see if there is a typo - // and suggest a correction - wrappedErr := errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field `%s` not found", field.Name).WithSuggestions(errors.NewSuggestionsOnLevenshteinDistance(field.Name, errors.NounKeys, maps.Keys(keys))...) - return "", wrappedErr + fieldExpression, err = m.FieldFor(ctx, orgID, tsStart, tsEnd, key) + if err != nil { + return "", err + } + fieldExpression, _ = querybuilder.DataTypeCollisionHandledFieldName(key, dummyValue, fieldExpression, qbtypes.FilterOperatorUnknown) } - } else if len(keysForField) == 1 { - // we have a single key for the field, use it - fieldExpression, _ = m.FieldFor(ctx, orgID, tsStart, tsEnd, keysForField[0]) + stmts = append(stmts, guard, fieldExpression) + } + return fmt.Sprintf("multiIf(%s, NULL)", strings.Join(stmts, ", ")), nil + } + + if len(candidates) == 1 { + // arrays cannot sit inside Nullable, so array candidates stay bare + if strings.Contains(candidates[0].Name, telemetrytypes.ArraySep) || + strings.Contains(candidates[0].Name, telemetrytypes.ArrayAnyIndex) || + candidates[0].FieldDataType.IsArray() { + return m.FieldFor(ctx, orgID, tsStart, tsEnd, candidates[0]) + } + if !m.membershipGuarded(ctx, orgID, tsStart, tsEnd, candidates[0]) { + return m.FieldFor(ctx, orgID, tsStart, tsEnd, candidates[0]) + } + guard, err := m.existsExpressionFor(ctx, orgID, tsStart, tsEnd, candidates[0], true) + if err != nil { + return "", err + } + var fieldExpression string + if candidates[0].FieldContext == telemetrytypes.FieldContextBody && !bodyJSONEnabled { + fieldExpression, _ = GetBodyJSONKey(ctx, candidates[0], qbtypes.FilterOperatorUnknown, "") } else { - // select any non-empty value from the keys - args := []string{} - for _, key := range keysForField { - fieldExpression, _ = m.FieldFor(ctx, orgID, tsStart, tsEnd, key) - args = append(args, fmt.Sprintf("toString(%s) != '', toString(%s)", fieldExpression, fieldExpression)) + fieldExpression, err = m.FieldFor(ctx, orgID, tsStart, tsEnd, candidates[0]) + if err != nil { + return "", err } - fieldExpression = fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", ")) } - } else if err != nil { - return "", err + return fmt.Sprintf("multiIf(%s, %s, NULL)", guard, fieldExpression), nil } - return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil + var stmts []string + for _, key := range candidates { + guard, err := m.existsExpressionFor(ctx, orgID, tsStart, tsEnd, key, true) + if err != nil { + return "", err + } + stmts = append(stmts, guard) + + var fieldExpression string + if key.FieldContext == telemetrytypes.FieldContextBody && !bodyJSONEnabled { + fieldExpression, _ = GetBodyJSONKey(ctx, key, qbtypes.FilterOperatorUnknown, "") + } else { + fieldExpression, err = m.FieldFor(ctx, orgID, tsStart, tsEnd, key) + if err != nil { + return "", err + } + fieldExpression, _ = querybuilder.DataTypeCollisionHandledFieldName(key, "", fieldExpression, qbtypes.FilterOperatorUnknown) + } + stmts = append(stmts, fieldExpression) + } + + return fmt.Sprintf("multiIf(%s, NULL)", strings.Join(stmts, ", ")), nil } -// CandidateKeys returns nil: logs has no synthesize-on-unknown-key fallback, so an -// unknown key stays unresolved and the caller errors. -func (m *fieldMapper) CandidateKeys(_ context.Context, _ valuer.UUID, _ *telemetrytypes.TelemetryFieldKey, _ any, _ map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey { +func (m *fieldMapper) CandidateKeys(_ context.Context, _ valuer.UUID, field *telemetrytypes.TelemetryFieldKey, value any, keys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey { + if matches := keys[field.Name]; len(matches) > 0 { + return matches + } + if matches := keys[fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), field.Name)]; len(matches) > 0 { + return matches + } + + switch field.FieldContext { + case telemetrytypes.FieldContextBody: + if field.Name == "" { + return nil + } + return []*telemetrytypes.TelemetryFieldKey{telemetrytypes.NewTelemetryFieldKey(field.Name, field.FieldContext, field.FieldDataType)} + case telemetrytypes.FieldContextUnspecified: + // a real column wins before synthesis (so adjustKeys is not needed to resolve these) + if _, ok := logsV2Columns[field.Name]; ok { + return []*telemetrytypes.TelemetryFieldKey{{Name: field.Name, FieldContext: telemetrytypes.FieldContextLog}} + } + bodyKey := telemetrytypes.NewTelemetryFieldKey(field.Name, telemetrytypes.FieldContextBody, field.FieldDataType) + if value == nil && bodyKey.FieldDataType == telemetrytypes.FieldDataTypeUnspecified { + bodyKey.FieldDataType = telemetrytypes.FieldDataTypeString + } + return append(querybuilder.SynthesizeKeys(field, value), bodyKey) + case telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource: + // stripped interpretation first, the literal `{context}.{name}` spelling second + literal := telemetrytypes.NewTelemetryFieldKey(field.FieldContext.StringValue()+"."+field.Name, field.FieldContext, field.FieldDataType) + return append(querybuilder.SynthesizeKeys(field, value), querybuilder.SynthesizeKeys(literal, value)...) + case telemetrytypes.FieldContextLog: + if _, ok := logsV2Columns[field.Name]; ok { + return []*telemetrytypes.TelemetryFieldKey{telemetrytypes.NewTelemetryFieldKey(field.Name, field.FieldContext, field.FieldDataType)} + } + stripped := telemetrytypes.NewTelemetryFieldKey(field.Name, telemetrytypes.FieldContextUnspecified, field.FieldDataType) + literal := telemetrytypes.NewTelemetryFieldKey(field.FieldContext.StringValue()+"."+field.Name, telemetrytypes.FieldContextUnspecified, field.FieldDataType) + // attribute candidates first (stripped, then literal), body paths last + candidates := append(querybuilder.SynthesizeKeys(stripped, value), querybuilder.SynthesizeKeys(literal, value)...) + for _, key := range []*telemetrytypes.TelemetryFieldKey{stripped, literal} { + bodyKey := telemetrytypes.NewTelemetryFieldKey(key.Name, telemetrytypes.FieldContextBody, key.FieldDataType) + if value == nil && bodyKey.FieldDataType == telemetrytypes.FieldDataTypeUnspecified { + bodyKey.FieldDataType = telemetrytypes.FieldDataTypeString + } + candidates = append(candidates, bodyKey) + } + return candidates + } return nil } @@ -293,8 +400,23 @@ func (m *fieldMapper) CandidateKeys(_ context.Context, _ valuer.UUID, _ *telemet func (m *fieldMapper) buildFieldForJSON(key *telemetrytypes.TelemetryFieldKey) (string, error) { plan := key.JSONPlan if len(plan) == 0 { - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, - "Could not find any valid paths for: %s", key.Name) + if key.KeyNameContainsArray() { + keyCopy := telemetrytypes.NewTelemetryFieldKey(key.Name, key.FieldContext, key.FieldDataType) + if err := keyCopy.SetExhaustiveJSONAccessPlan( + telemetrytypes.JSONColumnMetadata{BaseColumn: LogsV2BodyV2Column}, key.FieldDataType, + ); err != nil { + return "", err + } + return m.buildArrayConcat(keyCopy.JSONPlan) + } + + elemType := key.GetJSONDataType() + if elemType.StringValue() == "" { + elemType = telemetrytypes.String + } + + fieldPath := fmt.Sprintf("%s.`%s`", LogsV2BodyV2Column, key.Name) + return fmt.Sprintf("dynamicElement(%s, '%s')", fieldPath, elemType.StringValue()), nil } if plan[0].IsTerminal { @@ -430,3 +552,86 @@ func (m *fieldMapper) buildArrayMap(currentNode *telemetrytypes.JSONAccessNode, return fmt.Sprintf("arrayMap(%s->%s, %s)", currentNode.Alias(), nestedExpr, arrayExpr), nil } + +func (m *fieldMapper) membershipGuarded(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) bool { + if key.FieldContext == telemetrytypes.FieldContextBody { + return true + } + columns, err := m.getColumn(ctx, orgID, key) + if err != nil { + return false + } + newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd) + if err != nil || len(newColumns) != 1 { + return false + } + columnType := newColumns[0].Type.GetType() + return columnType == schema.ColumnTypeEnumMap || columnType == schema.ColumnTypeEnumJSON +} + +func (m *fieldMapper) existsExpressionFor( + ctx context.Context, + orgID valuer.UUID, + tsStart, tsEnd uint64, + key *telemetrytypes.TelemetryFieldKey, + exists bool, +) (string, error) { + columns, err := m.getColumn(ctx, orgID, key) + if errors.Is(err, qbtypes.ErrColumnNotFound) && key.FieldContext == telemetrytypes.FieldContextUnspecified { + bodyKey := telemetrytypes.NewTelemetryFieldKey(key.Name, telemetrytypes.FieldContextBody, key.FieldDataType) + columns, err = m.getColumn(ctx, orgID, bodyKey) + } + if err != nil { + return "", err + } + + operator := qbtypes.FilterOperatorExists + if !exists { + operator = qbtypes.FilterOperatorNotExists + } + + bodyJSONEnabled := m.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) + + for _, column := range columns { + if column.Type.GetType() == schema.ColumnTypeEnumJSON && isBodyJSONSearch(key, columns) && bodyJSONEnabled && key.Name != messageSubField { + valueType, value := InferDataType(nil, operator, key) + if len(key.JSONPlan) == 0 { + keyCopy := telemetrytypes.NewTelemetryFieldKey(key.Name, key.FieldContext, key.FieldDataType) + if err := keyCopy.SetExhaustiveJSONAccessPlan( + telemetrytypes.JSONColumnMetadata{BaseColumn: LogsV2BodyV2Column}, valueType, + ); err != nil { + return "", err + } + key = keyCopy + } + sb := sqlbuilder.NewSelectBuilder() + cond, err := NewJSONConditionBuilder(key, valueType).buildJSONCondition(operator, value, sb) + if err != nil { + return "", err + } + sb.Where(cond) + expr, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + expr = strings.TrimPrefix(expr, "WHERE ") + if len(args) > 0 { + expr, err = sqlbuilder.ClickHouse.Interpolate(expr, args) + if err != nil { + return "", err + } + } + return expr, nil + } + } + + if isBodyJSONSearch(key, columns) && !bodyJSONEnabled { + if exists { + return GetBodyJSONKeyForExists(ctx, key, operator, nil), nil + } + return "NOT " + GetBodyJSONKeyForExists(ctx, key, operator, nil), nil + } + + fieldExpression, err := m.FieldFor(ctx, orgID, tsStart, tsEnd, key) + if err != nil { + return "", err + } + return querybuilder.ExistsExpression(columns, key, tsStart, tsEnd, fieldExpression, exists) +} diff --git a/pkg/telemetrylogs/field_mapper_test.go b/pkg/telemetrylogs/field_mapper_test.go index 3024e4ac968..12b2bbff728 100644 --- a/pkg/telemetrylogs/field_mapper_test.go +++ b/pkg/telemetrylogs/field_mapper_test.go @@ -29,7 +29,7 @@ func TestGetColumn(t *testing.T) { Name: "service.name", FieldContext: telemetrytypes.FieldContextResource, }, - expectedCol: []*schema.Column{logsV2Columns["resources_string"], logsV2Columns["resource"]}, + expectedCol: []*schema.Column{logsV2Columns["resource"], logsV2Columns["resources_string"]}, expectedError: nil, }, { @@ -580,7 +580,7 @@ func TestFieldForWithMaterialized(t *testing.T) { name: "Multi evolution - both columns (JSON + materialized)", start: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC), end: time.Date(2024, 4, 2, 0, 0, 0, 0, time.UTC), - expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, `resource_string_service$$name_exists`==true, `resource_string_service$$name`, NULL)", + expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, `resource_string_service$$name_exists`, `resource_string_service$$name`, NULL)", }, } diff --git a/pkg/telemetrylogs/filter_expr_logs_test.go b/pkg/telemetrylogs/filter_expr_logs_test.go index 78eefa394ba..f56fdd7b1cf 100644 --- a/pkg/telemetrylogs/filter_expr_logs_test.go +++ b/pkg/telemetrylogs/filter_expr_logs_test.go @@ -148,32 +148,32 @@ func TestFilterExprLogs(t *testing.T) { category: "Key with curly brace", query: `{UserId} = "U101"`, shouldPass: true, - expectedQuery: "WHERE (attributes_string['{UserId}'] = ? AND mapContains(attributes_string, '{UserId}') = ?)", - expectedArgs: []any{"U101", true}, + expectedQuery: "WHERE (attributes_string['{UserId}'] = ? AND mapContains(attributes_string, '{UserId}'))", + expectedArgs: []any{"U101"}, expectedErrorContains: "", }, { category: "Key with @symbol", query: `user@email = "u@example.com"`, shouldPass: true, - expectedQuery: "WHERE (attributes_string['user@email'] = ? AND mapContains(attributes_string, 'user@email') = ?)", - expectedArgs: []any{"u@example.com", true}, + expectedQuery: "WHERE (attributes_string['user@email'] = ? AND mapContains(attributes_string, 'user@email'))", + expectedArgs: []any{"u@example.com"}, expectedErrorContains: "", }, { category: "Key with @symbol", query: `#user_name = "anon42069"`, shouldPass: true, - expectedQuery: "WHERE (attributes_string['#user_name'] = ? AND mapContains(attributes_string, '#user_name') = ?)", - expectedArgs: []any{"anon42069", true}, + expectedQuery: "WHERE (attributes_string['#user_name'] = ? AND mapContains(attributes_string, '#user_name'))", + expectedArgs: []any{"anon42069"}, expectedErrorContains: "", }, { category: "Key with @symbol", query: `gen_ai.completion.0.content = "जब तक इस देश में सिनेमा है"`, shouldPass: true, - expectedQuery: "WHERE (attributes_string['gen_ai.completion.0.content'] = ? AND mapContains(attributes_string, 'gen_ai.completion.0.content') = ?)", - expectedArgs: []any{"जब तक इस देश में सिनेमा है", true}, + expectedQuery: "WHERE (attributes_string['gen_ai.completion.0.content'] = ? AND mapContains(attributes_string, 'gen_ai.completion.0.content'))", + expectedArgs: []any{"जब तक इस देश में सिनेमा है"}, expectedErrorContains: "", }, // Searches with special characters @@ -443,8 +443,8 @@ func TestFilterExprLogs(t *testing.T) { category: "FREETEXT with conditions", query: "critical NOT resolved status=open", shouldPass: true, - expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND NOT (match(LOWER(body), LOWER(?))) AND (toString(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?))", - expectedArgs: []any{"critical", "resolved", "open", true}, + expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND NOT (match(LOWER(body), LOWER(?))) AND (toString(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')))", + expectedArgs: []any{"critical", "resolved", "open"}, expectedErrorContains: "", }, { @@ -453,32 +453,32 @@ func TestFilterExprLogs(t *testing.T) { category: "FREETEXT with conditions", query: "critical NOT resolved status > open", shouldPass: true, - expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND NOT (match(LOWER(body), LOWER(?))) AND (toString(attributes_number['status']) > ? AND mapContains(attributes_number, 'status') = ?))", - expectedArgs: []any{"critical", "resolved", "open", true}, + expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND NOT (match(LOWER(body), LOWER(?))) AND (toString(attributes_number['status']) > ? AND mapContains(attributes_number, 'status')))", + expectedArgs: []any{"critical", "resolved", "open"}, expectedErrorContains: "", }, { category: "FREETEXT with conditions", query: "database error type=mysql", shouldPass: true, - expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND match(LOWER(body), LOWER(?)) AND (attributes_string['type'] = ? AND mapContains(attributes_string, 'type') = ?))", - expectedArgs: []any{"database", "error", "mysql", true}, + expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND match(LOWER(body), LOWER(?)) AND (attributes_string['type'] = ? AND mapContains(attributes_string, 'type')))", + expectedArgs: []any{"database", "error", "mysql"}, expectedErrorContains: "", }, { category: "FREETEXT with conditions", query: "\"connection timeout\" duration>30", shouldPass: true, - expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND (toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration') = ?))", - expectedArgs: []any{"connection timeout", float64(30), true}, + expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND (toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration')))", + expectedArgs: []any{"connection timeout", float64(30)}, expectedErrorContains: "", }, { category: "FREETEXT with conditions", query: "warning level=critical", shouldPass: true, - expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?))", - expectedArgs: []any{"warning", "critical", true}, + expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level')))", + expectedArgs: []any{"warning", "critical"}, expectedErrorContains: "", }, { @@ -495,32 +495,32 @@ func TestFilterExprLogs(t *testing.T) { category: "FREETEXT with parentheses", query: "error (status.code=500 OR status.code=503)", shouldPass: true, - expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND (((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?))))", - expectedArgs: []any{"error", float64(500), true, float64(503), true}, + expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND (((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')))))", + expectedArgs: []any{"error", float64(500), float64(503)}, expectedErrorContains: "", }, { category: "FREETEXT with parentheses", query: "(status.code=500 OR status.code=503) error", shouldPass: true, - expectedQuery: "WHERE ((((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?))) AND match(LOWER(body), LOWER(?)))", - expectedArgs: []any{float64(500), true, float64(503), true, "error"}, + expectedQuery: "WHERE ((((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')))) AND match(LOWER(body), LOWER(?)))", + expectedArgs: []any{float64(500), float64(503), "error"}, expectedErrorContains: "", }, { category: "FREETEXT with parentheses", query: "error AND (status.code=500 OR status.code=503)", shouldPass: true, - expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND (((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?))))", - expectedArgs: []any{"error", float64(500), true, float64(503), true}, + expectedQuery: "WHERE (match(LOWER(body), LOWER(?)) AND (((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')))))", + expectedArgs: []any{"error", float64(500), float64(503)}, expectedErrorContains: "", }, { category: "FREETEXT with parentheses", query: "(status.code=500 OR status.code=503) AND error", shouldPass: true, - expectedQuery: "WHERE ((((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code') = ?))) AND match(LOWER(body), LOWER(?)))", - expectedArgs: []any{float64(500), true, float64(503), true, "error"}, + expectedQuery: "WHERE ((((toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')) OR (toFloat64(attributes_number['status.code']) = ? AND mapContains(attributes_number, 'status.code')))) AND match(LOWER(body), LOWER(?)))", + expectedArgs: []any{float64(500), float64(503), "error"}, expectedErrorContains: "", }, @@ -632,7 +632,7 @@ func TestFilterExprLogs(t *testing.T) { query: "and", shouldPass: false, expectedQuery: "", - expectedArgs: []any{}, + expectedArgs: nil, expectedErrorContains: "expecting one of {(, ), FREETEXT, NOT, boolean, has(), hasAll(), hasAny(), hasToken(), number, quoted text} but got 'and'", }, { @@ -640,7 +640,7 @@ func TestFilterExprLogs(t *testing.T) { query: "or", shouldPass: false, expectedQuery: "", - expectedArgs: []any{}, + expectedArgs: nil, expectedErrorContains: "expecting one of {(, ), AND, FREETEXT, NOT, boolean, has(), hasAll(), hasAny(), hasToken(), number, quoted text} but got 'or'", }, { @@ -648,7 +648,7 @@ func TestFilterExprLogs(t *testing.T) { query: "not", shouldPass: false, expectedQuery: "", - expectedArgs: []any{}, + expectedArgs: nil, expectedErrorContains: "expecting one of {(, ), FREETEXT, boolean, has(), hasAll(), hasAny(), hasToken(), number, quoted text} but got EOF", }, { @@ -656,7 +656,7 @@ func TestFilterExprLogs(t *testing.T) { query: "like", shouldPass: false, expectedQuery: "", - expectedArgs: []any{}, + expectedArgs: nil, expectedErrorContains: "expecting one of {(, ), AND, FREETEXT, NOT, boolean, has(), hasAll(), hasAny(), hasToken(), number, quoted text} but got 'like'", }, { @@ -664,7 +664,7 @@ func TestFilterExprLogs(t *testing.T) { query: "between", shouldPass: false, expectedQuery: "", - expectedArgs: []any{}, + expectedArgs: nil, expectedErrorContains: "expecting one of {(, ), AND, FREETEXT, NOT, boolean, has(), hasAll(), hasAny(), hasToken(), number, quoted text} but got 'between'", }, { @@ -672,7 +672,7 @@ func TestFilterExprLogs(t *testing.T) { query: "in", shouldPass: false, expectedQuery: "", - expectedArgs: []any{}, + expectedArgs: nil, expectedErrorContains: "expecting one of {(, ), AND, FREETEXT, NOT, boolean, has(), hasAll(), hasAny(), hasToken(), number, quoted text} but got 'in'", }, { @@ -680,7 +680,7 @@ func TestFilterExprLogs(t *testing.T) { query: "exists", shouldPass: false, expectedQuery: "", - expectedArgs: []any{}, + expectedArgs: nil, expectedErrorContains: "expecting one of {(, ), AND, FREETEXT, NOT, boolean, has(), hasAll(), hasAny(), hasToken(), number, quoted text} but got 'exists'", }, { @@ -688,7 +688,7 @@ func TestFilterExprLogs(t *testing.T) { query: "regexp", shouldPass: false, expectedQuery: "", - expectedArgs: []any{}, + expectedArgs: nil, expectedErrorContains: "expecting one of {(, ), AND, FREETEXT, NOT, boolean, has(), hasAll(), hasAny(), hasToken(), number, quoted text} but got 'regexp'", }, { @@ -696,7 +696,7 @@ func TestFilterExprLogs(t *testing.T) { query: "contains", shouldPass: false, expectedQuery: "", - expectedArgs: []any{}, + expectedArgs: nil, expectedErrorContains: "expecting one of {(, ), AND, FREETEXT, NOT, boolean, has(), hasAll(), hasAny(), hasToken(), number, quoted text} but got 'contains'", }, { @@ -704,7 +704,7 @@ func TestFilterExprLogs(t *testing.T) { query: "has", shouldPass: false, expectedQuery: "", - expectedArgs: []any{}, + expectedArgs: nil, expectedErrorContains: "expecting one of {(, )} but got EOF", }, { @@ -712,7 +712,7 @@ func TestFilterExprLogs(t *testing.T) { query: "hasany", shouldPass: false, expectedQuery: "", - expectedArgs: []any{}, + expectedArgs: nil, expectedErrorContains: "expecting one of {(, )} but got EOF", }, { @@ -720,7 +720,7 @@ func TestFilterExprLogs(t *testing.T) { query: "hasall", shouldPass: false, expectedQuery: "", - expectedArgs: []any{}, + expectedArgs: nil, expectedErrorContains: "expecting one of {(, )} but got EOF", }, @@ -736,10 +736,10 @@ func TestFilterExprLogs(t *testing.T) { { category: "Key-operator-value boundary", query: "greater>than", - shouldPass: false, - expectedQuery: "", - expectedArgs: []any{}, - expectedErrorContains: "key `greater` not found", + shouldPass: true, + expectedQuery: `WHERE ((attributes_string['greater'] > ? AND mapContains(attributes_string, 'greater')) OR (JSON_VALUE(body, '$."greater"') > ? AND JSON_EXISTS(body, '$."greater"')))`, + expectedArgs: []any{"than", "than"}, + expectedErrorContains: "", }, { category: "Key-operator-value boundary", @@ -752,10 +752,10 @@ func TestFilterExprLogs(t *testing.T) { { category: "Key-operator-value boundary", query: "less10", shouldPass: true, - expectedQuery: "WHERE (toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?)", - expectedArgs: []any{float64(10), true}, + expectedQuery: "WHERE (toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count'))", + expectedArgs: []any{float64(10)}, expectedErrorContains: "", }, { category: "Greater than", query: "duration>1000", shouldPass: true, - expectedQuery: "WHERE (toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration') = ?)", - expectedArgs: []any{float64(1000), true}, + expectedQuery: "WHERE (toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration'))", + expectedArgs: []any{float64(1000)}, expectedErrorContains: "", }, @@ -981,16 +981,16 @@ func TestFilterExprLogs(t *testing.T) { category: "Greater than or equal", query: "count>=10", shouldPass: true, - expectedQuery: "WHERE (toFloat64(attributes_number['count']) >= ? AND mapContains(attributes_number, 'count') = ?)", - expectedArgs: []any{float64(10), true}, + expectedQuery: "WHERE (toFloat64(attributes_number['count']) >= ? AND mapContains(attributes_number, 'count'))", + expectedArgs: []any{float64(10)}, expectedErrorContains: "", }, { category: "Greater than or equal", query: "duration>=1000", shouldPass: true, - expectedQuery: "WHERE (toFloat64(attributes_number['duration']) >= ? AND mapContains(attributes_number, 'duration') = ?)", - expectedArgs: []any{float64(1000), true}, + expectedQuery: "WHERE (toFloat64(attributes_number['duration']) >= ? AND mapContains(attributes_number, 'duration'))", + expectedArgs: []any{float64(1000)}, expectedErrorContains: "", }, @@ -999,32 +999,32 @@ func TestFilterExprLogs(t *testing.T) { category: "LIKE operator", query: "message LIKE \"%error%\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['message'] LIKE ? AND mapContains(attributes_string, 'message') = ?)", - expectedArgs: []any{"%error%", true}, + expectedQuery: "WHERE (attributes_string['message'] LIKE ? AND mapContains(attributes_string, 'message'))", + expectedArgs: []any{"%error%"}, expectedErrorContains: "", }, { category: "LIKE operator", query: "path LIKE \"/api/%\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['path'] LIKE ? AND mapContains(attributes_string, 'path') = ?)", - expectedArgs: []any{"/api/%", true}, + expectedQuery: "WHERE (attributes_string['path'] LIKE ? AND mapContains(attributes_string, 'path'))", + expectedArgs: []any{"/api/%"}, expectedErrorContains: "", }, { category: "LIKE operator", query: "email LIKE \"%@example.com\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['email'] LIKE ? AND mapContains(attributes_string, 'email') = ?)", - expectedArgs: []any{"%@example.com", true}, + expectedQuery: "WHERE (attributes_string['email'] LIKE ? AND mapContains(attributes_string, 'email'))", + expectedArgs: []any{"%@example.com"}, expectedErrorContains: "", }, { category: "LIKE operator", query: "filename LIKE \"%.pdf\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['filename'] LIKE ? AND mapContains(attributes_string, 'filename') = ?)", - expectedArgs: []any{"%.pdf", true}, + expectedQuery: "WHERE (attributes_string['filename'] LIKE ? AND mapContains(attributes_string, 'filename'))", + expectedArgs: []any{"%.pdf"}, expectedErrorContains: "", }, @@ -1033,32 +1033,32 @@ func TestFilterExprLogs(t *testing.T) { category: "ILIKE operator", query: "message ILIKE \"%error%\"", shouldPass: true, - expectedQuery: "WHERE (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?)", - expectedArgs: []any{"%error%", true}, + expectedQuery: "WHERE (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message'))", + expectedArgs: []any{"%error%"}, expectedErrorContains: "", }, { category: "ILIKE operator", query: "path ILIKE \"/api/%\"", shouldPass: true, - expectedQuery: "WHERE (LOWER(attributes_string['path']) LIKE LOWER(?) AND mapContains(attributes_string, 'path') = ?)", - expectedArgs: []any{"/api/%", true}, + expectedQuery: "WHERE (LOWER(attributes_string['path']) LIKE LOWER(?) AND mapContains(attributes_string, 'path'))", + expectedArgs: []any{"/api/%"}, expectedErrorContains: "", }, { category: "ILIKE operator", query: "email ILIKE \"%@EXAMPLE.com\"", shouldPass: true, - expectedQuery: "WHERE (LOWER(attributes_string['email']) LIKE LOWER(?) AND mapContains(attributes_string, 'email') = ?)", - expectedArgs: []any{"%@EXAMPLE.com", true}, + expectedQuery: "WHERE (LOWER(attributes_string['email']) LIKE LOWER(?) AND mapContains(attributes_string, 'email'))", + expectedArgs: []any{"%@EXAMPLE.com"}, expectedErrorContains: "", }, { category: "ILIKE operator", query: "filename ILIKE \"%.PDF\"", shouldPass: true, - expectedQuery: "WHERE (LOWER(attributes_string['filename']) LIKE LOWER(?) AND mapContains(attributes_string, 'filename') = ?)", - expectedArgs: []any{"%.PDF", true}, + expectedQuery: "WHERE (LOWER(attributes_string['filename']) LIKE LOWER(?) AND mapContains(attributes_string, 'filename'))", + expectedArgs: []any{"%.PDF"}, expectedErrorContains: "", }, @@ -1135,24 +1135,24 @@ func TestFilterExprLogs(t *testing.T) { category: "BETWEEN operator", query: "count BETWEEN 1 AND 10", shouldPass: true, - expectedQuery: "WHERE (toFloat64(attributes_number['count']) BETWEEN ? AND ? AND mapContains(attributes_number, 'count') = ?)", - expectedArgs: []any{float64(1), float64(10), true}, + expectedQuery: "WHERE (toFloat64(attributes_number['count']) BETWEEN ? AND ? AND mapContains(attributes_number, 'count'))", + expectedArgs: []any{float64(1), float64(10)}, expectedErrorContains: "", }, { category: "BETWEEN operator", query: "duration BETWEEN 100 AND 1000", shouldPass: true, - expectedQuery: "WHERE (toFloat64(attributes_number['duration']) BETWEEN ? AND ? AND mapContains(attributes_number, 'duration') = ?)", - expectedArgs: []any{float64(100), float64(1000), true}, + expectedQuery: "WHERE (toFloat64(attributes_number['duration']) BETWEEN ? AND ? AND mapContains(attributes_number, 'duration'))", + expectedArgs: []any{float64(100), float64(1000)}, expectedErrorContains: "", }, { category: "BETWEEN operator", query: "amount BETWEEN 0.1 AND 9.9", shouldPass: true, - expectedQuery: "WHERE (toFloat64(attributes_number['amount']) BETWEEN ? AND ? AND mapContains(attributes_number, 'amount') = ?)", - expectedArgs: []any{0.1, 9.9, true}, + expectedQuery: "WHERE (toFloat64(attributes_number['amount']) BETWEEN ? AND ? AND mapContains(attributes_number, 'amount'))", + expectedArgs: []any{0.1, 9.9}, expectedErrorContains: "", }, @@ -1187,16 +1187,16 @@ func TestFilterExprLogs(t *testing.T) { category: "IN operator (parentheses)", query: "status IN (200, 201, 202)", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ?) AND mapContains(attributes_number, 'status') = ?)", - expectedArgs: []any{float64(200), float64(201), float64(202), true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ?) AND mapContains(attributes_number, 'status'))", + expectedArgs: []any{float64(200), float64(201), float64(202)}, expectedErrorContains: "", }, { category: "IN operator (parentheses)", query: "error.code IN (404, 500, 503)", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ?) AND mapContains(attributes_number, 'error.code') = ?)", - expectedArgs: []any{float64(404), float64(500), float64(503), true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ?) AND mapContains(attributes_number, 'error.code'))", + expectedArgs: []any{float64(404), float64(500), float64(503)}, expectedErrorContains: "", }, { @@ -1221,16 +1221,16 @@ func TestFilterExprLogs(t *testing.T) { category: "IN operator (brackets)", query: "status IN [200, 201, 202]", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ?) AND mapContains(attributes_number, 'status') = ?)", - expectedArgs: []any{float64(200), float64(201), float64(202), true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ? OR toFloat64(attributes_number['status']) = ?) AND mapContains(attributes_number, 'status'))", + expectedArgs: []any{float64(200), float64(201), float64(202)}, expectedErrorContains: "", }, { category: "IN operator (brackets)", query: "error.code IN [404, 500, 503]", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ?) AND mapContains(attributes_number, 'error.code') = ?)", - expectedArgs: []any{float64(404), float64(500), float64(503), true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ? OR toFloat64(attributes_number['error.code']) = ?) AND mapContains(attributes_number, 'error.code'))", + expectedArgs: []any{float64(404), float64(500), float64(503)}, expectedErrorContains: "", }, { @@ -1323,32 +1323,32 @@ func TestFilterExprLogs(t *testing.T) { category: "EXISTS operator", query: "user.id EXISTS", shouldPass: true, - expectedQuery: "WHERE mapContains(attributes_string, 'user.id') = ?", - expectedArgs: []any{true}, + expectedQuery: "WHERE mapContains(attributes_string, 'user.id')", + expectedArgs: nil, expectedErrorContains: "", }, { category: "EXISTS operator", query: "metadata.version EXISTS", shouldPass: true, - expectedQuery: "WHERE mapContains(attributes_string, 'metadata.version') = ?", - expectedArgs: []any{true}, + expectedQuery: "WHERE mapContains(attributes_string, 'metadata.version')", + expectedArgs: nil, expectedErrorContains: "", }, { category: "EXISTS operator", query: "request.headers.authorization EXISTS", shouldPass: true, - expectedQuery: "WHERE mapContains(attributes_string, 'request.headers.authorization') = ?", - expectedArgs: []any{true}, + expectedQuery: "WHERE mapContains(attributes_string, 'request.headers.authorization')", + expectedArgs: nil, expectedErrorContains: "", }, { category: "EXISTS operator", query: "response.body.data EXISTS", shouldPass: true, - expectedQuery: "WHERE mapContains(attributes_string, 'response.body.data') = ?", - expectedArgs: []any{true}, + expectedQuery: "WHERE mapContains(attributes_string, 'response.body.data')", + expectedArgs: nil, expectedErrorContains: "", }, { @@ -1364,40 +1364,40 @@ func TestFilterExprLogs(t *testing.T) { category: "NOT EXISTS operator", query: "user.id NOT EXISTS", shouldPass: true, - expectedQuery: "WHERE mapContains(attributes_string, 'user.id') <> ?", - expectedArgs: []any{true}, + expectedQuery: "WHERE NOT mapContains(attributes_string, 'user.id')", + expectedArgs: nil, expectedErrorContains: "", }, { category: "NOT EXISTS operator", query: "user.id not exists", shouldPass: true, - expectedQuery: "WHERE mapContains(attributes_string, 'user.id') <> ?", - expectedArgs: []any{true}, + expectedQuery: "WHERE NOT mapContains(attributes_string, 'user.id')", + expectedArgs: nil, expectedErrorContains: "", }, { category: "NOT EXISTS operator", query: "metadata.version NOT EXISTS", shouldPass: true, - expectedQuery: "WHERE mapContains(attributes_string, 'metadata.version') <> ?", - expectedArgs: []any{true}, + expectedQuery: "WHERE NOT mapContains(attributes_string, 'metadata.version')", + expectedArgs: nil, expectedErrorContains: "", }, { category: "NOT EXISTS operator", query: "request.headers.authorization NOT EXISTS", shouldPass: true, - expectedQuery: "WHERE mapContains(attributes_string, 'request.headers.authorization') <> ?", - expectedArgs: []any{true}, + expectedQuery: "WHERE NOT mapContains(attributes_string, 'request.headers.authorization')", + expectedArgs: nil, expectedErrorContains: "", }, { category: "NOT EXISTS operator", query: "response.body.data NOT EXISTS", shouldPass: true, - expectedQuery: "WHERE mapContains(attributes_string, 'response.body.data') <> ?", - expectedArgs: []any{true}, + expectedQuery: "WHERE NOT mapContains(attributes_string, 'response.body.data')", + expectedArgs: nil, expectedErrorContains: "", }, { @@ -1413,32 +1413,32 @@ func TestFilterExprLogs(t *testing.T) { category: "REGEXP operator", query: "message REGEXP \"^ERROR:\"", shouldPass: true, - expectedQuery: "WHERE (match(attributes_string['message'], ?) AND mapContains(attributes_string, 'message') = ?)", - expectedArgs: []any{"^ERROR:", true}, + expectedQuery: "WHERE (match(attributes_string['message'], ?) AND mapContains(attributes_string, 'message'))", + expectedArgs: []any{"^ERROR:"}, expectedErrorContains: "", }, { category: "REGEXP operator", query: "email REGEXP \"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\\\.[a-zA-Z]{2,}$\"", shouldPass: true, - expectedQuery: "WHERE (match(attributes_string['email'], ?) AND mapContains(attributes_string, 'email') = ?)", - expectedArgs: []any{"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$", true}, + expectedQuery: "WHERE (match(attributes_string['email'], ?) AND mapContains(attributes_string, 'email'))", + expectedArgs: []any{"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$"}, expectedErrorContains: "", }, { category: "REGEXP operator", query: "version REGEXP \"^v\\\\d+\\\\.\\\\d+\\\\.\\\\d+$\"", shouldPass: true, - expectedQuery: "WHERE (match(attributes_string['version'], ?) AND mapContains(attributes_string, 'version') = ?)", - expectedArgs: []any{"^v\\d+\\.\\d+\\.\\d+$", true}, + expectedQuery: "WHERE (match(attributes_string['version'], ?) AND mapContains(attributes_string, 'version'))", + expectedArgs: []any{"^v\\d+\\.\\d+\\.\\d+$"}, expectedErrorContains: "", }, { category: "REGEXP operator", query: "path REGEXP \"^/api/v\\\\d+/users/\\\\d+$\"", shouldPass: true, - expectedQuery: "WHERE (match(attributes_string['path'], ?) AND mapContains(attributes_string, 'path') = ?)", - expectedArgs: []any{"^/api/v\\d+/users/\\d+$", true}, + expectedQuery: "WHERE (match(attributes_string['path'], ?) AND mapContains(attributes_string, 'path'))", + expectedArgs: []any{"^/api/v\\d+/users/\\d+$"}, expectedErrorContains: "", }, { @@ -1489,8 +1489,8 @@ func TestFilterExprLogs(t *testing.T) { category: "CONTAINS operator", query: "message CONTAINS \"error\"", shouldPass: true, - expectedQuery: "WHERE (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?)", - expectedArgs: []any{"%error%", true}, + expectedQuery: "WHERE (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message'))", + expectedArgs: []any{"%error%"}, expectedErrorContains: "", }, { @@ -1505,16 +1505,16 @@ func TestFilterExprLogs(t *testing.T) { category: "CONTAINS operator", query: "level CONTAINS \"critical\"", shouldPass: true, - expectedQuery: "WHERE (LOWER(attributes_string['level']) LIKE LOWER(?) AND mapContains(attributes_string, 'level') = ?)", - expectedArgs: []any{"%critical%", true}, + expectedQuery: "WHERE (LOWER(attributes_string['level']) LIKE LOWER(?) AND mapContains(attributes_string, 'level'))", + expectedArgs: []any{"%critical%"}, expectedErrorContains: "", }, { category: "CONTAINS operator", query: "path CONTAINS \"api\"", shouldPass: true, - expectedQuery: "WHERE (LOWER(attributes_string['path']) LIKE LOWER(?) AND mapContains(attributes_string, 'path') = ?)", - expectedArgs: []any{"%api%", true}, + expectedQuery: "WHERE (LOWER(attributes_string['path']) LIKE LOWER(?) AND mapContains(attributes_string, 'path'))", + expectedArgs: []any{"%api%"}, expectedErrorContains: "", }, @@ -1586,8 +1586,8 @@ func TestFilterExprLogs(t *testing.T) { category: "Materialized key", query: "materialized.key.name=\"test\"", shouldPass: true, - expectedQuery: "WHERE (`attribute_string_materialized$$key$$name` = ? AND `attribute_string_materialized$$key$$name_exists` = ?)", - expectedArgs: []any{"test", true}, + expectedQuery: "WHERE (`attribute_string_materialized$$key$$name` = ? AND `attribute_string_materialized$$key$$name_exists`)", + expectedArgs: []any{"test"}, expectedErrorContains: "", }, @@ -1596,24 +1596,24 @@ func TestFilterExprLogs(t *testing.T) { category: "Explicit AND", query: "status=200 AND service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))", - expectedArgs: []any{float64(200), true, "api"}, + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))", + expectedArgs: []any{float64(200), "api"}, expectedErrorContains: "", }, { category: "Explicit AND", query: "count>0 AND duration<1000", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?))", - expectedArgs: []any{float64(0), true, float64(1000), true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count')) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration')))", + expectedArgs: []any{float64(0), float64(1000)}, expectedErrorContains: "", }, { category: "Explicit AND", query: "message CONTAINS \"error\" AND level=\"ERROR\"", shouldPass: true, - expectedQuery: "WHERE ((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?))", - expectedArgs: []any{"%error%", true, "ERROR", true}, + expectedQuery: "WHERE ((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message')) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level')))", + expectedArgs: []any{"%error%", "ERROR"}, expectedErrorContains: "", }, @@ -1622,8 +1622,8 @@ func TestFilterExprLogs(t *testing.T) { category: "Explicit OR", query: "status=200 OR status=201", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) OR (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?))", - expectedArgs: []any{float64(200), true, float64(201), true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) OR (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')))", + expectedArgs: []any{float64(200), float64(201)}, expectedErrorContains: "", }, { @@ -1638,8 +1638,8 @@ func TestFilterExprLogs(t *testing.T) { category: "Explicit OR", query: "count<10 OR count>100", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['count']) < ? AND mapContains(attributes_number, 'count') = ?) OR (toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?))", - expectedArgs: []any{float64(10), true, float64(100), true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['count']) < ? AND mapContains(attributes_number, 'count')) OR (toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count')))", + expectedArgs: []any{float64(10), float64(100)}, expectedErrorContains: "", }, @@ -1648,8 +1648,8 @@ func TestFilterExprLogs(t *testing.T) { category: "NOT with expressions", query: "NOT status=200", shouldPass: true, - expectedQuery: "WHERE NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?))", - expectedArgs: []any{float64(200), true}, + expectedQuery: "WHERE NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')))", + expectedArgs: []any{float64(200)}, expectedErrorContains: "", }, { @@ -1664,8 +1664,8 @@ func TestFilterExprLogs(t *testing.T) { category: "NOT with expressions", query: "NOT count>10", shouldPass: true, - expectedQuery: "WHERE NOT ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?))", - expectedArgs: []any{float64(10), true}, + expectedQuery: "WHERE NOT ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count')))", + expectedArgs: []any{float64(10)}, expectedErrorContains: "", }, @@ -1674,24 +1674,24 @@ func TestFilterExprLogs(t *testing.T) { category: "AND + OR combinations", query: "status=200 AND (service.name=\"api\" OR service.name=\"web\")", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))))", - expectedArgs: []any{float64(200), true, "api", "web"}, + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))))", + expectedArgs: []any{float64(200), "api", "web"}, expectedErrorContains: "", }, { category: "AND + OR combinations", query: "(count>10 AND count<100) OR (duration>1000 AND duration<5000)", shouldPass: true, - expectedQuery: "WHERE ((((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND (toFloat64(attributes_number['count']) < ? AND mapContains(attributes_number, 'count') = ?))) OR (((toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration') = ?) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?))))", - expectedArgs: []any{float64(10), true, float64(100), true, float64(1000), true, float64(5000), true}, + expectedQuery: "WHERE ((((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count')) AND (toFloat64(attributes_number['count']) < ? AND mapContains(attributes_number, 'count')))) OR (((toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration')) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration')))))", + expectedArgs: []any{float64(10), float64(100), float64(1000), float64(5000)}, expectedErrorContains: "", }, { category: "AND + OR combinations", query: "level=\"ERROR\" OR (level=\"WARN\" AND message CONTAINS \"timeout\")", shouldPass: true, - expectedQuery: "WHERE ((attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?) OR (((attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?) AND (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?))))", - expectedArgs: []any{"ERROR", true, "WARN", true, "%timeout%", true}, + expectedQuery: "WHERE ((attributes_string['level'] = ? AND mapContains(attributes_string, 'level')) OR (((attributes_string['level'] = ? AND mapContains(attributes_string, 'level')) AND (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message')))))", + expectedArgs: []any{"ERROR", "WARN", "%timeout%"}, expectedErrorContains: "", }, @@ -1700,16 +1700,16 @@ func TestFilterExprLogs(t *testing.T) { category: "AND + NOT combinations", query: "status=200 AND NOT service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND NOT ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))", - expectedArgs: []any{float64(200), true, "api"}, + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND NOT ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))", + expectedArgs: []any{float64(200), "api"}, expectedErrorContains: "", }, { category: "AND + NOT combinations", query: "count>0 AND NOT error.code EXISTS", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND NOT (mapContains(attributes_number, 'error.code') = ?))", - expectedArgs: []any{float64(0), true, true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count')) AND NOT (mapContains(attributes_number, 'error.code')))", + expectedArgs: []any{float64(0)}, expectedErrorContains: "", }, @@ -1718,16 +1718,16 @@ func TestFilterExprLogs(t *testing.T) { category: "OR + NOT combinations", query: "NOT status=200 OR NOT service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?)) OR NOT ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))", - expectedArgs: []any{float64(200), true, "api"}, + expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status'))) OR NOT ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))", + expectedArgs: []any{float64(200), "api"}, expectedErrorContains: "", }, { category: "OR + NOT combinations", query: "NOT count>0 OR NOT error.code EXISTS", shouldPass: true, - expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?)) OR NOT (mapContains(attributes_number, 'error.code') = ?))", - expectedArgs: []any{float64(0), true, true}, + expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count'))) OR NOT (mapContains(attributes_number, 'error.code')))", + expectedArgs: []any{float64(0)}, expectedErrorContains: "", }, @@ -1736,24 +1736,24 @@ func TestFilterExprLogs(t *testing.T) { category: "AND + OR + NOT combinations", query: "status=200 AND (service.name=\"api\" OR NOT duration>1000)", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR NOT ((toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration') = ?)))))", - expectedArgs: []any{float64(200), true, "api", float64(1000), true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR NOT ((toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration'))))))", + expectedArgs: []any{float64(200), "api", float64(1000)}, expectedErrorContains: "", }, { category: "AND + OR + NOT combinations", query: "(level=\"ERROR\" OR level=\"FATAL\") AND NOT message CONTAINS \"expected\"", shouldPass: true, - expectedQuery: "WHERE ((((attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?) OR (attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?))) AND NOT ((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?)))", - expectedArgs: []any{"ERROR", true, "FATAL", true, "%expected%", true}, + expectedQuery: "WHERE ((((attributes_string['level'] = ? AND mapContains(attributes_string, 'level')) OR (attributes_string['level'] = ? AND mapContains(attributes_string, 'level')))) AND NOT ((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message'))))", + expectedArgs: []any{"ERROR", "FATAL", "%expected%"}, expectedErrorContains: "", }, { category: "AND + OR + NOT combinations", query: "NOT (status=200 AND service.name=\"api\") OR count>0", shouldPass: true, - expectedQuery: "WHERE (NOT ((((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))) OR (toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?))", - expectedArgs: []any{float64(200), true, "api", float64(0), true}, + expectedQuery: "WHERE (NOT ((((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))) OR (toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count')))", + expectedArgs: []any{float64(200), "api", float64(0)}, expectedErrorContains: "", }, @@ -1762,24 +1762,24 @@ func TestFilterExprLogs(t *testing.T) { category: "Implicit AND", query: "status=200 service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))", - expectedArgs: []any{float64(200), true, "api"}, + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))", + expectedArgs: []any{float64(200), "api"}, expectedErrorContains: "", }, { category: "Implicit AND", query: "count>0 duration<1000", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?))", - expectedArgs: []any{float64(0), true, float64(1000), true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count')) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration')))", + expectedArgs: []any{float64(0), float64(1000)}, expectedErrorContains: "", }, { category: "Implicit AND", query: "message CONTAINS \"error\" level=\"ERROR\"", shouldPass: true, - expectedQuery: "WHERE ((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?))", - expectedArgs: []any{"%error%", true, "ERROR", true}, + expectedQuery: "WHERE ((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message')) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level')))", + expectedArgs: []any{"%error%", "ERROR"}, expectedErrorContains: "", }, @@ -1788,16 +1788,16 @@ func TestFilterExprLogs(t *testing.T) { category: "Mixed implicit/explicit AND", query: "status=200 AND service.name=\"api\" duration<1000", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?))", - expectedArgs: []any{float64(200), true, "api", float64(1000), true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration')))", + expectedArgs: []any{float64(200), "api", float64(1000)}, expectedErrorContains: "", }, { category: "Mixed implicit/explicit AND", query: "count>0 level=\"ERROR\" AND message CONTAINS \"error\"", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?) AND (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?))", - expectedArgs: []any{float64(0), true, "ERROR", true, "%error%", true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count')) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level')) AND (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message')))", + expectedArgs: []any{float64(0), "ERROR", "%error%"}, expectedErrorContains: "", }, @@ -1806,8 +1806,8 @@ func TestFilterExprLogs(t *testing.T) { category: "Simple grouping", query: "(status=200)", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?))", - expectedArgs: []any{float64(200), true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')))", + expectedArgs: []any{float64(200)}, expectedErrorContains: "", }, { @@ -1822,8 +1822,8 @@ func TestFilterExprLogs(t *testing.T) { category: "Simple grouping", query: "(count>0)", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?))", - expectedArgs: []any{float64(0), true}, + expectedQuery: "WHERE ((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count')))", + expectedArgs: []any{float64(0)}, expectedErrorContains: "", }, @@ -1832,8 +1832,8 @@ func TestFilterExprLogs(t *testing.T) { category: "Nested grouping", query: "((status=200))", shouldPass: true, - expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?)))", - expectedArgs: []any{float64(200), true}, + expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status'))))", + expectedArgs: []any{float64(200)}, expectedErrorContains: "", }, { @@ -1848,8 +1848,8 @@ func TestFilterExprLogs(t *testing.T) { category: "Nested grouping", query: "((count>0) AND (duration<1000))", shouldPass: true, - expectedQuery: "WHERE ((((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?)) AND ((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?))))", - expectedArgs: []any{float64(0), true, float64(1000), true}, + expectedQuery: "WHERE ((((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count'))) AND ((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration')))))", + expectedArgs: []any{float64(0), float64(1000)}, expectedErrorContains: "", }, @@ -1858,24 +1858,24 @@ func TestFilterExprLogs(t *testing.T) { category: "Complex nested grouping", query: "(status=200 AND (service.name=\"api\" OR service.name=\"web\"))", shouldPass: true, - expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))))", - expectedArgs: []any{float64(200), true, "api", "web"}, + expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))))", + expectedArgs: []any{float64(200), "api", "web"}, expectedErrorContains: "", }, { category: "Complex nested grouping", query: "((count>0 AND count<100) OR (duration>1000 AND duration<5000))", shouldPass: true, - expectedQuery: "WHERE (((((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND (toFloat64(attributes_number['count']) < ? AND mapContains(attributes_number, 'count') = ?))) OR (((toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration') = ?) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?)))))", - expectedArgs: []any{float64(0), true, float64(100), true, float64(1000), true, float64(5000), true}, + expectedQuery: "WHERE (((((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count')) AND (toFloat64(attributes_number['count']) < ? AND mapContains(attributes_number, 'count')))) OR (((toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration')) AND (toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration'))))))", + expectedArgs: []any{float64(0), float64(100), float64(1000), float64(5000)}, expectedErrorContains: "", }, { category: "Complex nested grouping", query: "(level=\"ERROR\" OR (level=\"WARN\" AND (message CONTAINS \"timeout\" OR message CONTAINS \"connection\")))", shouldPass: true, - expectedQuery: "WHERE (((attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?) OR (((attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?) AND (((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?) OR (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?)))))))", - expectedArgs: []any{"ERROR", true, "WARN", true, "%timeout%", true, "%connection%", true}, + expectedQuery: "WHERE (((attributes_string['level'] = ? AND mapContains(attributes_string, 'level')) OR (((attributes_string['level'] = ? AND mapContains(attributes_string, 'level')) AND (((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message')) OR (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message'))))))))", + expectedArgs: []any{"ERROR", "WARN", "%timeout%", "%connection%"}, expectedErrorContains: "", }, @@ -1884,16 +1884,16 @@ func TestFilterExprLogs(t *testing.T) { category: "Deep nesting", query: "(((status=200 OR status=201) AND service.name=\"api\") OR ((status=202 OR status=203) AND service.name=\"web\"))", shouldPass: true, - expectedQuery: "WHERE (((((((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) OR (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?))) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))) OR (((((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) OR (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?))) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))))", - expectedArgs: []any{float64(200), true, float64(201), true, "api", float64(202), true, float64(203), true, "web"}, + expectedQuery: "WHERE (((((((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) OR (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')))) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))) OR (((((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) OR (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')))) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))))", + expectedArgs: []any{float64(200), float64(201), "api", float64(202), float64(203), "web"}, expectedErrorContains: "", }, { category: "Deep nesting", query: "(count>0 AND ((duration<1000 AND service.name=\"api\") OR (duration<500 AND service.name=\"web\")))", shouldPass: true, - expectedQuery: "WHERE (((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count') = ?) AND (((((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))) OR (((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))))))", - expectedArgs: []any{float64(0), true, float64(1000), true, "api", float64(500), true, "web"}, + expectedQuery: "WHERE (((toFloat64(attributes_number['count']) > ? AND mapContains(attributes_number, 'count')) AND (((((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))) OR (((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))))))", + expectedArgs: []any{float64(0), float64(1000), "api", float64(500), "web"}, expectedErrorContains: "", }, @@ -1918,16 +1918,16 @@ func TestFilterExprLogs(t *testing.T) { category: "String quote styles", query: "message=\"This is a \\\"quoted\\\" message\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['message'] = ? AND mapContains(attributes_string, 'message') = ?)", - expectedArgs: []any{"This is a \\\"quoted\\\" message", true}, + expectedQuery: "WHERE (attributes_string['message'] = ? AND mapContains(attributes_string, 'message'))", + expectedArgs: []any{"This is a \\\"quoted\\\" message"}, expectedErrorContains: "", }, { category: "String quote styles", query: "message='This is a \\'quoted\\' message'", shouldPass: true, - expectedQuery: "WHERE (attributes_string['message'] = ? AND mapContains(attributes_string, 'message') = ?)", - expectedArgs: []any{"This is a 'quoted' message", true}, + expectedQuery: "WHERE (attributes_string['message'] = ? AND mapContains(attributes_string, 'message'))", + expectedArgs: []any{"This is a 'quoted' message"}, expectedErrorContains: "", }, @@ -1936,32 +1936,32 @@ func TestFilterExprLogs(t *testing.T) { category: "Numeric values", query: "status=200", shouldPass: true, - expectedQuery: "WHERE (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?)", - expectedArgs: []any{float64(200), true}, + expectedQuery: "WHERE (toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status'))", + expectedArgs: []any{float64(200)}, expectedErrorContains: "", }, { category: "Numeric values", query: "count=0", shouldPass: true, - expectedQuery: "WHERE (toFloat64(attributes_number['count']) = ? AND mapContains(attributes_number, 'count') = ?)", - expectedArgs: []any{float64(0), true}, + expectedQuery: "WHERE (toFloat64(attributes_number['count']) = ? AND mapContains(attributes_number, 'count'))", + expectedArgs: []any{float64(0)}, expectedErrorContains: "", }, { category: "Numeric values", query: "duration=1000.5", shouldPass: true, - expectedQuery: "WHERE (toFloat64(attributes_number['duration']) = ? AND mapContains(attributes_number, 'duration') = ?)", - expectedArgs: []any{float64(1000.5), true}, + expectedQuery: "WHERE (toFloat64(attributes_number['duration']) = ? AND mapContains(attributes_number, 'duration'))", + expectedArgs: []any{float64(1000.5)}, expectedErrorContains: "", }, { category: "Numeric values", query: "amount=-10.25", shouldPass: true, - expectedQuery: "WHERE (toFloat64(attributes_number['amount']) = ? AND mapContains(attributes_number, 'amount') = ?)", - expectedArgs: []any{float64(-10.25), true}, + expectedQuery: "WHERE (toFloat64(attributes_number['amount']) = ? AND mapContains(attributes_number, 'amount'))", + expectedArgs: []any{float64(-10.25)}, expectedErrorContains: "", }, @@ -1970,32 +1970,32 @@ func TestFilterExprLogs(t *testing.T) { category: "Boolean values", query: "isEnabled=true", shouldPass: true, - expectedQuery: "WHERE (attributes_bool['isEnabled'] = ? AND mapContains(attributes_bool, 'isEnabled') = ?)", - expectedArgs: []any{true, true}, + expectedQuery: "WHERE (attributes_bool['isEnabled'] = ? AND mapContains(attributes_bool, 'isEnabled'))", + expectedArgs: []any{true}, expectedErrorContains: "", }, { category: "Boolean values", query: "isDisabled=false", shouldPass: true, - expectedQuery: "WHERE (attributes_bool['isDisabled'] = ? AND mapContains(attributes_bool, 'isDisabled') = ?)", - expectedArgs: []any{false, true}, + expectedQuery: "WHERE (attributes_bool['isDisabled'] = ? AND mapContains(attributes_bool, 'isDisabled'))", + expectedArgs: []any{false}, expectedErrorContains: "", }, { category: "Boolean values", query: "is_valid=TRUE", shouldPass: true, - expectedQuery: "WHERE (attributes_bool['is_valid'] = ? AND mapContains(attributes_bool, 'is_valid') = ?)", - expectedArgs: []any{true, true}, + expectedQuery: "WHERE (attributes_bool['is_valid'] = ? AND mapContains(attributes_bool, 'is_valid'))", + expectedArgs: []any{true}, expectedErrorContains: "", }, { category: "Boolean values", query: "is_invalid=FALSE", shouldPass: true, - expectedQuery: "WHERE (attributes_bool['is_invalid'] = ? AND mapContains(attributes_bool, 'is_invalid') = ?)", - expectedArgs: []any{false, true}, + expectedQuery: "WHERE (attributes_bool['is_invalid'] = ? AND mapContains(attributes_bool, 'is_invalid'))", + expectedArgs: []any{false}, expectedErrorContains: "", }, @@ -2005,16 +2005,16 @@ func TestFilterExprLogs(t *testing.T) { category: "Null values", query: "value=null", shouldPass: true, - expectedQuery: "WHERE (attributes_string['value'] = ? AND mapContains(attributes_string, 'value') = ?)", - expectedArgs: []any{"null", true}, + expectedQuery: "WHERE (attributes_string['value'] = ? AND mapContains(attributes_string, 'value'))", + expectedArgs: []any{"null"}, expectedErrorContains: "", }, { category: "Null values", query: "value=undefined", shouldPass: true, - expectedQuery: "WHERE (attributes_string['value'] = ? AND mapContains(attributes_string, 'value') = ?)", - expectedArgs: []any{"undefined", true}, + expectedQuery: "WHERE (attributes_string['value'] = ? AND mapContains(attributes_string, 'value'))", + expectedArgs: []any{"undefined"}, expectedErrorContains: "", }, @@ -2023,32 +2023,32 @@ func TestFilterExprLogs(t *testing.T) { category: "Nested object paths", query: "user.profile.name=\"John\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['user.profile.name'] = ? AND mapContains(attributes_string, 'user.profile.name') = ?)", - expectedArgs: []any{"John", true}, + expectedQuery: "WHERE (attributes_string['user.profile.name'] = ? AND mapContains(attributes_string, 'user.profile.name'))", + expectedArgs: []any{"John"}, expectedErrorContains: "", }, { category: "Nested object paths", query: "request.headers.authorization EXISTS", shouldPass: true, - expectedQuery: "WHERE mapContains(attributes_string, 'request.headers.authorization') = ?", - expectedArgs: []any{true}, + expectedQuery: "WHERE mapContains(attributes_string, 'request.headers.authorization')", + expectedArgs: nil, expectedErrorContains: "", }, { category: "Nested object paths", query: "response.body.data.items[].id=123", - shouldPass: false, - expectedQuery: "", - expectedArgs: nil, - expectedErrorContains: "key `response.body.data.items[].id` not found", + shouldPass: true, + expectedQuery: `WHERE ((toFloat64(attributes_number['response.body.data.items[].id']) = ? AND mapContains(attributes_number, 'response.body.data.items[].id')) OR (JSONExtract(JSON_VALUE(body, '$."response"."body"."data"."items"[*]."id"'), 'Float64') = ? AND JSON_EXISTS(body, '$."response"."body"."data"."items"[*]."id"')))`, + expectedArgs: []any{float64(123), float64(123)}, + expectedErrorContains: "", }, { category: "Nested object paths", query: "metadata.dimensions.width>1000", shouldPass: true, - expectedQuery: "WHERE (toFloat64(attributes_number['metadata.dimensions.width']) > ? AND mapContains(attributes_number, 'metadata.dimensions.width') = ?)", - expectedArgs: []any{float64(1000), true}, + expectedQuery: "WHERE (toFloat64(attributes_number['metadata.dimensions.width']) > ? AND mapContains(attributes_number, 'metadata.dimensions.width'))", + expectedArgs: []any{float64(1000)}, expectedErrorContains: "", }, @@ -2070,29 +2070,29 @@ func TestFilterExprLogs(t *testing.T) { category: "Operator precedence", query: "NOT status=200 AND service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?)) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))", - expectedArgs: []any{float64(200), true, "api"}, // Should be (NOT status=200) AND service.name="api" + expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status'))) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))", + expectedArgs: []any{float64(200), "api"}, // Should be (NOT status=200) AND service.name="api" }, { category: "Operator precedence", query: "status=200 AND service.name=\"api\" OR service.name=\"web\"", shouldPass: true, - expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))", - expectedArgs: []any{float64(200), true, "api", "web"}, // Should be (status=200 AND service.name="api") OR service.name="web" + expectedQuery: "WHERE (((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))", + expectedArgs: []any{float64(200), "api", "web"}, // Should be (status=200 AND service.name="api") OR service.name="web" }, { category: "Operator precedence", query: "NOT status=200 OR NOT service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?)) OR NOT ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))", - expectedArgs: []any{float64(200), true, "api"}, // Should be (NOT status=200) OR (NOT service.name="api") + expectedQuery: "WHERE (NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status'))) OR NOT ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)))", + expectedArgs: []any{float64(200), "api"}, // Should be (NOT status=200) OR (NOT service.name="api") }, { category: "Operator precedence", query: "status=200 OR service.name=\"api\" AND level=\"ERROR\"", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) OR ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level') = ?)))", - expectedArgs: []any{float64(200), true, "api", "ERROR", true}, // Should be status=200 OR (service.name="api" AND level="ERROR") + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) OR ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) AND (attributes_string['level'] = ? AND mapContains(attributes_string, 'level'))))", + expectedArgs: []any{float64(200), "api", "ERROR"}, // Should be status=200 OR (service.name="api" AND level="ERROR") }, // Different whitespace patterns @@ -2116,8 +2116,8 @@ func TestFilterExprLogs(t *testing.T) { category: "Whitespace patterns", query: "status=200 AND service.name=\"api\"", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))", - expectedArgs: []any{float64(200), true, "api"}, // Multiple spaces + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))", + expectedArgs: []any{float64(200), "api"}, // Multiple spaces }, // More Unicode characters @@ -2125,29 +2125,29 @@ func TestFilterExprLogs(t *testing.T) { category: "Unicode", query: "message=\"café\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['message'] = ? AND mapContains(attributes_string, 'message') = ?)", - expectedArgs: []any{"café", true}, + expectedQuery: "WHERE (attributes_string['message'] = ? AND mapContains(attributes_string, 'message'))", + expectedArgs: []any{"café"}, }, { category: "Unicode", query: "user.name=\"Jörg Müller\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['user.name'] = ? AND mapContains(attributes_string, 'user.name') = ?)", - expectedArgs: []any{"Jörg Müller", true}, + expectedQuery: "WHERE (attributes_string['user.name'] = ? AND mapContains(attributes_string, 'user.name'))", + expectedArgs: []any{"Jörg Müller"}, }, { category: "Unicode", query: "location=\"東京\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['location'] = ? AND mapContains(attributes_string, 'location') = ?)", - expectedArgs: []any{"東京", true}, + expectedQuery: "WHERE (attributes_string['location'] = ? AND mapContains(attributes_string, 'location'))", + expectedArgs: []any{"東京"}, }, { category: "Unicode", query: "description=\"Это тестовое сообщение\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['description'] = ? AND mapContains(attributes_string, 'description') = ?)", - expectedArgs: []any{"Это тестовое сообщение", true}, + expectedQuery: "WHERE (attributes_string['description'] = ? AND mapContains(attributes_string, 'description'))", + expectedArgs: []any{"Это тестовое сообщение"}, }, // Special characters @@ -2155,15 +2155,15 @@ func TestFilterExprLogs(t *testing.T) { category: "Special characters", query: "path=\"/special/!@#$%^&*()\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['path'] = ? AND mapContains(attributes_string, 'path') = ?)", - expectedArgs: []any{"/special/!@#$%^&*()", true}, + expectedQuery: "WHERE (attributes_string['path'] = ? AND mapContains(attributes_string, 'path'))", + expectedArgs: []any{"/special/!@#$%^&*()"}, }, { category: "Special characters", query: "query=\"?param1=value1¶m2=value2\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['query'] = ? AND mapContains(attributes_string, 'query') = ?)", - expectedArgs: []any{"?param1=value1¶m2=value2", true}, + expectedQuery: "WHERE (attributes_string['query'] = ? AND mapContains(attributes_string, 'query'))", + expectedArgs: []any{"?param1=value1¶m2=value2"}, }, // Special characters in keys @@ -2171,22 +2171,22 @@ func TestFilterExprLogs(t *testing.T) { category: "Special characters in keys", query: "special-key=\"value\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['special-key'] = ? AND mapContains(attributes_string, 'special-key') = ?)", - expectedArgs: []any{"value", true}, // With hyphen + expectedQuery: "WHERE (attributes_string['special-key'] = ? AND mapContains(attributes_string, 'special-key'))", + expectedArgs: []any{"value"}, // With hyphen }, { category: "Special characters in keys", query: "special.key=\"value\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['special.key'] = ? AND mapContains(attributes_string, 'special.key') = ?)", - expectedArgs: []any{"value", true}, // With dot + expectedQuery: "WHERE (attributes_string['special.key'] = ? AND mapContains(attributes_string, 'special.key'))", + expectedArgs: []any{"value"}, // With dot }, { category: "Special characters in keys", query: "special_key=\"value\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['special_key'] = ? AND mapContains(attributes_string, 'special_key') = ?)", - expectedArgs: []any{"value", true}, // With underscore + expectedQuery: "WHERE (attributes_string['special_key'] = ? AND mapContains(attributes_string, 'special_key'))", + expectedArgs: []any{"value"}, // With underscore }, // Using operator keywords as keys @@ -2262,15 +2262,15 @@ func TestFilterExprLogs(t *testing.T) { category: "Common filters", query: "status IN (\"pending\", \"processing\", \"completed\") AND NOT is_deleted=true", shouldPass: true, - expectedQuery: "WHERE (((toString(attributes_number['status']) = ? OR toString(attributes_number['status']) = ? OR toString(attributes_number['status']) = ?) AND mapContains(attributes_number, 'status') = ?) AND NOT ((attributes_bool['is_deleted'] = ? AND mapContains(attributes_bool, 'is_deleted') = ?)))", - expectedArgs: []any{"pending", "processing", "completed", true, true, true}, + expectedQuery: "WHERE (((toString(attributes_number['status']) = ? OR toString(attributes_number['status']) = ? OR toString(attributes_number['status']) = ?) AND mapContains(attributes_number, 'status')) AND NOT ((attributes_bool['is_deleted'] = ? AND mapContains(attributes_bool, 'is_deleted'))))", + expectedArgs: []any{"pending", "processing", "completed", true}, }, { category: "Common filters", query: "(first_name LIKE \"John%\" OR last_name LIKE \"Smith%\") AND age>=18", shouldPass: true, - expectedQuery: "WHERE ((((attributes_string['first_name'] LIKE ? AND mapContains(attributes_string, 'first_name') = ?) OR (attributes_string['last_name'] LIKE ? AND mapContains(attributes_string, 'last_name') = ?))) AND (toFloat64(attributes_number['age']) >= ? AND mapContains(attributes_number, 'age') = ?))", - expectedArgs: []any{"John%", true, "Smith%", true, float64(18), true}, + expectedQuery: "WHERE ((((attributes_string['first_name'] LIKE ? AND mapContains(attributes_string, 'first_name')) OR (attributes_string['last_name'] LIKE ? AND mapContains(attributes_string, 'last_name')))) AND (toFloat64(attributes_number['age']) >= ? AND mapContains(attributes_number, 'age')))", + expectedArgs: []any{"John%", "Smith%", float64(18)}, }, { category: "Common filters", @@ -2278,7 +2278,7 @@ func TestFilterExprLogs(t *testing.T) { shouldPass: false, expectedQuery: "", expectedArgs: nil, - expectedErrorContains: "key `user_id` not found", + expectedErrorContains: "function `has` supports only body JSON search", }, // More common filter patterns @@ -2286,8 +2286,8 @@ func TestFilterExprLogs(t *testing.T) { category: "More common filters", query: "service.name=\"api\" AND (status>=500 OR duration>1000) AND NOT message CONTAINS \"expected\"", shouldPass: true, - expectedQuery: "WHERE ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) AND (((toFloat64(attributes_number['status']) >= ? AND mapContains(attributes_number, 'status') = ?) OR (toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration') = ?))) AND NOT ((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?)))", - expectedArgs: []any{"api", float64(500), true, float64(1000), true, "%expected%", true}, + expectedQuery: "WHERE ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) AND (((toFloat64(attributes_number['status']) >= ? AND mapContains(attributes_number, 'status')) OR (toFloat64(attributes_number['duration']) > ? AND mapContains(attributes_number, 'duration')))) AND NOT ((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message'))))", + expectedArgs: []any{"api", float64(500), float64(1000), "%expected%"}, }, // Edge cases @@ -2329,22 +2329,22 @@ func TestFilterExprLogs(t *testing.T) { category: "Escaped values", query: "message=\"Line 1\\nLine 2\\tTabbed\\rCarriage return\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['message'] = ? AND mapContains(attributes_string, 'message') = ?)", - expectedArgs: []any{"Line 1\\nLine 2\\tTabbed\\rCarriage return", true}, + expectedQuery: "WHERE (attributes_string['message'] = ? AND mapContains(attributes_string, 'message'))", + expectedArgs: []any{"Line 1\\nLine 2\\tTabbed\\rCarriage return"}, }, { category: "Escaped values", query: "path=\"C:\\\\Program Files\\\\Application\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['path'] = ? AND mapContains(attributes_string, 'path') = ?)", - expectedArgs: []any{"C:\\Program Files\\Application", true}, + expectedQuery: "WHERE (attributes_string['path'] = ? AND mapContains(attributes_string, 'path'))", + expectedArgs: []any{"C:\\Program Files\\Application"}, }, { category: "Escaped values", query: "path=\"^prefix\\\\.suffix$\\\\d+\\\\w+\"", shouldPass: true, - expectedQuery: "WHERE (attributes_string['path'] = ? AND mapContains(attributes_string, 'path') = ?)", - expectedArgs: []any{"^prefix\\.suffix$\\d+\\w+", true}, + expectedQuery: "WHERE (attributes_string['path'] = ? AND mapContains(attributes_string, 'path'))", + expectedArgs: []any{"^prefix\\.suffix$\\d+\\w+"}, }, // Inconsistent/unusual whitespace @@ -2352,8 +2352,8 @@ func TestFilterExprLogs(t *testing.T) { category: "Unusual whitespace", query: "status = 200 AND service.name = \"api\"", shouldPass: true, - expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))", - expectedArgs: []any{float64(200), true, "api"}, + expectedQuery: "WHERE ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status')) AND (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL))", + expectedArgs: []any{float64(200), "api"}, }, { category: "Unusual whitespace", @@ -2413,15 +2413,15 @@ func TestFilterExprLogs(t *testing.T) { ) `, shouldPass: true, - expectedQuery: "WHERE ((((((((toFloat64(attributes_number['status']) >= ? AND mapContains(attributes_number, 'status') = ?) AND (toFloat64(attributes_number['status']) < ? AND mapContains(attributes_number, 'status') = ?))) OR (((toFloat64(attributes_number['status']) >= ? AND mapContains(attributes_number, 'status') = ?) AND (toFloat64(attributes_number['status']) < ? AND mapContains(attributes_number, 'status') = ?) AND NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status') = ?)))))) AND ((((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? OR multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? OR multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ?) AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (((multiIf(resource.`service.type` IS NOT NULL, resource.`service.type`::String, mapContains(resources_string, 'service.type'), resources_string['service.type'], NULL) = ? AND multiIf(resource.`service.type` IS NOT NULL, resource.`service.type`::String, mapContains(resources_string, 'service.type'), resources_string['service.type'], NULL) IS NOT NULL) AND NOT ((multiIf(resource.`service.deprecated` IS NOT NULL, resource.`service.deprecated`::String, mapContains(resources_string, 'service.deprecated'), resources_string['service.deprecated'], NULL) = ? AND multiIf(resource.`service.deprecated` IS NOT NULL, resource.`service.deprecated`::String, mapContains(resources_string, 'service.deprecated'), resources_string['service.deprecated'], NULL) IS NOT NULL)))))))) AND (((((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration') = ?) OR ((toFloat64(attributes_number['duration']) BETWEEN ? AND ? AND mapContains(attributes_number, 'duration') = ?)))) AND ((multiIf(resource.`environment` IS NOT NULL, resource.`environment`::String, mapContains(resources_string, 'environment'), resources_string['environment'], NULL) <> ? OR (((multiIf(resource.`environment` IS NOT NULL, resource.`environment`::String, mapContains(resources_string, 'environment'), resources_string['environment'], NULL) = ? AND multiIf(resource.`environment` IS NOT NULL, resource.`environment`::String, mapContains(resources_string, 'environment'), resources_string['environment'], NULL) IS NOT NULL) AND (attributes_bool['is_automated_test'] = ? AND mapContains(attributes_bool, 'is_automated_test') = ?))))))) AND NOT ((((((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?) OR (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message') = ?))) AND (attributes_string['severity'] = ? AND mapContains(attributes_string, 'severity') = ?)))))", + expectedQuery: "WHERE ((((((((toFloat64(attributes_number['status']) >= ? AND mapContains(attributes_number, 'status')) AND (toFloat64(attributes_number['status']) < ? AND mapContains(attributes_number, 'status')))) OR (((toFloat64(attributes_number['status']) >= ? AND mapContains(attributes_number, 'status')) AND (toFloat64(attributes_number['status']) < ? AND mapContains(attributes_number, 'status')) AND NOT ((toFloat64(attributes_number['status']) = ? AND mapContains(attributes_number, 'status'))))))) AND ((((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? OR multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? OR multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ?) AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (((multiIf(resource.`service.type` IS NOT NULL, resource.`service.type`::String, mapContains(resources_string, 'service.type'), resources_string['service.type'], NULL) = ? AND multiIf(resource.`service.type` IS NOT NULL, resource.`service.type`::String, mapContains(resources_string, 'service.type'), resources_string['service.type'], NULL) IS NOT NULL) AND NOT ((multiIf(resource.`service.deprecated` IS NOT NULL, resource.`service.deprecated`::String, mapContains(resources_string, 'service.deprecated'), resources_string['service.deprecated'], NULL) = ? AND multiIf(resource.`service.deprecated` IS NOT NULL, resource.`service.deprecated`::String, mapContains(resources_string, 'service.deprecated'), resources_string['service.deprecated'], NULL) IS NOT NULL)))))))) AND (((((toFloat64(attributes_number['duration']) < ? AND mapContains(attributes_number, 'duration')) OR ((toFloat64(attributes_number['duration']) BETWEEN ? AND ? AND mapContains(attributes_number, 'duration'))))) AND ((multiIf(resource.`environment` IS NOT NULL, resource.`environment`::String, mapContains(resources_string, 'environment'), resources_string['environment'], NULL) <> ? OR (((multiIf(resource.`environment` IS NOT NULL, resource.`environment`::String, mapContains(resources_string, 'environment'), resources_string['environment'], NULL) = ? AND multiIf(resource.`environment` IS NOT NULL, resource.`environment`::String, mapContains(resources_string, 'environment'), resources_string['environment'], NULL) IS NOT NULL) AND (attributes_bool['is_automated_test'] = ? AND mapContains(attributes_bool, 'is_automated_test')))))))) AND NOT ((((((LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message')) OR (LOWER(attributes_string['message']) LIKE LOWER(?) AND mapContains(attributes_string, 'message')))) AND (attributes_string['severity'] = ? AND mapContains(attributes_string, 'severity'))))))", expectedArgs: []any{ - float64(200), true, float64(300), true, float64(400), true, float64(500), true, float64(404), true, + float64(200), float64(300), float64(400), float64(500), float64(404), "api", "web", "auth", "internal", true, - float64(1000), true, float64(1000), float64(5000), true, - "test", "test", true, true, - "%warning%", true, "%deprecated%", true, - "low", true, + float64(1000), float64(1000), float64(5000), + "test", "test", true, + "%warning%", "%deprecated%", + "low", }, }, } @@ -2532,8 +2532,8 @@ func TestFilterExprLogsConflictNegation(t *testing.T) { category: "exists", query: "body EXISTS", shouldPass: true, - expectedQuery: "WHERE (body <> ? OR mapContains(attributes_string, 'body') = ?)", - expectedArgs: []any{"", true}, + expectedQuery: "WHERE (body <> '' OR mapContains(attributes_string, 'body'))", + expectedArgs: nil, expectedErrorContains: "", }, } diff --git a/pkg/telemetrylogs/groupby_unknown_key_test.go b/pkg/telemetrylogs/groupby_unknown_key_test.go new file mode 100644 index 00000000000..6b2b523d143 --- /dev/null +++ b/pkg/telemetrylogs/groupby_unknown_key_test.go @@ -0,0 +1,72 @@ +package telemetrylogs + +import ( + "context" + "testing" + "time" + + "github.com/SigNoz/signoz/pkg/flagger/flaggertest" + "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" + "github.com/SigNoz/signoz/pkg/querybuilder" + qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" + "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest" + "github.com/SigNoz/signoz/pkg/valuer" + "github.com/stretchr/testify/require" +) + +// A bare key absent from metadata groups by a multiIf over the synthesized +// attribute maps first (string, number, bool) and the body path last; legacy +// body doesn't support group by, so with the flag off only attributes remain. +func TestStatementBuilderGroupByUnknownKey(t *testing.T) { + releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC) + releaseTimeNano := uint64(releaseTime.UnixNano()) + + cases := []struct { + name string + useJSONBody bool + expected string + }{ + { + name: "legacy body", + useJSONBody: false, + expected: "SELECT toString(multiIf(mapContains(attributes_string, 'totally.unknown.key'), attributes_string['totally.unknown.key'], mapContains(attributes_number, 'totally.unknown.key'), toString(attributes_number['totally.unknown.key']), mapContains(attributes_bool, 'totally.unknown.key'), toString(attributes_bool['totally.unknown.key']), NULL)) AS `__GROUP_BY_KEY_0_totally.unknown.key`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_totally.unknown.key` ORDER BY __result_0 DESC", + }, + { + name: "json body", + useJSONBody: true, + expected: "SELECT toString(multiIf(mapContains(attributes_string, 'totally.unknown.key'), attributes_string['totally.unknown.key'], mapContains(attributes_number, 'totally.unknown.key'), toString(attributes_number['totally.unknown.key']), mapContains(attributes_bool, 'totally.unknown.key'), toString(attributes_bool['totally.unknown.key']), (dynamicElement(body_v2.`totally.unknown.key`, 'String') IS NOT NULL), dynamicElement(body_v2.`totally.unknown.key`, 'String'), NULL)) AS `__GROUP_BY_KEY_0_totally.unknown.key`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_totally.unknown.key` ORDER BY __result_0 DESC", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + fl := flaggertest.WithUseJSONBody(t, c.useJSONBody) + mockMetadataStore := telemetrytypestest.NewMockMetadataStore() + mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime) + fm := NewFieldMapper(fl) + cb := NewConditionBuilder(fm, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) + statementBuilder := NewLogQueryStatementBuilder( + instrumentationtest.New().ToProviderSettings(), + mockMetadataStore, fm, cb, aggExprRewriter, + DefaultFullTextColumn, fl, nil, false, 100000, + ) + + query := qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{ + Signal: telemetrytypes.SignalLogs, + StepInterval: qbtypes.Step{Duration: 30 * time.Second}, + Aggregations: []qbtypes.LogAggregation{{Expression: "count()"}}, + GroupBy: []qbtypes.GroupByKey{ + {TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "totally.unknown.key"}}, + }, + } + q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, + releaseTimeNano+uint64(24*time.Hour.Nanoseconds()), + releaseTimeNano+uint64(48*time.Hour.Nanoseconds()), + qbtypes.RequestTypeScalar, query, nil) + require.NoError(t, err) + require.Equal(t, c.expected, q.Query) + }) + } +} diff --git a/pkg/telemetrylogs/json_planless_test.go b/pkg/telemetrylogs/json_planless_test.go new file mode 100644 index 00000000000..e63bc267fac --- /dev/null +++ b/pkg/telemetrylogs/json_planless_test.go @@ -0,0 +1,86 @@ +package telemetrylogs + +import ( + "strings" + "testing" + + qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" + "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/huandu/go-sqlbuilder" + "github.com/stretchr/testify/require" +) + +func TestExhaustiveJSONPlan_ConditionBuilder(t *testing.T) { + cases := []struct { + name string + path string + operator qbtypes.FilterOperator + value any + valueType telemetrytypes.FieldDataType + expectedSQL string + expectedArgs []any + }{ + { + name: "single hop array, both containers", + path: "education[].name", + operator: qbtypes.FilterOperatorEqual, + value: "IIT", + valueType: telemetrytypes.FieldDataTypeString, + expectedSQL: "(((arrayExists(`body_v2.education`-> dynamicElement(`body_v2.education`.`name`, 'String') = ?, dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education`-> dynamicElement(`body_v2.education`.`name`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(body_v2.`education`, 'Array(Dynamic)')))))) AND has(JSONAllPaths(body_v2), 'education'))", + expectedArgs: []any{"IIT", "IIT"}, + }, + { + name: "double hop array, both containers each hop", + path: "education[].awards[].name", + operator: qbtypes.FilterOperatorEqual, + value: "Iron Award", + valueType: telemetrytypes.FieldDataTypeString, + expectedSQL: "(((arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')) OR arrayExists(`body_v2.education`-> (arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')) OR arrayExists(`body_v2.education[].awards`-> dynamicElement(`body_v2.education[].awards`.`name`, 'String') = ?, arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(body_v2.`education`, 'Array(Dynamic)')))))) AND has(JSONAllPaths(body_v2), 'education'))", + expectedArgs: []any{"Iron Award", "Iron Award", "Iron Award", "Iron Award"}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + key := &telemetrytypes.TelemetryFieldKey{ + Name: c.path, + Signal: telemetrytypes.SignalLogs, + FieldContext: telemetrytypes.FieldContextBody, + } + require.Empty(t, key.JSONPlan) + require.NoError(t, key.SetExhaustiveJSONAccessPlan( + telemetrytypes.JSONColumnMetadata{BaseColumn: LogsV2BodyV2Column}, c.valueType)) + + sb := sqlbuilder.NewSelectBuilder() + cond, err := NewJSONConditionBuilder(key, c.valueType).buildJSONCondition(c.operator, c.value, sb) + require.NoError(t, err) + + sb.Select("1").From("t").Where(cond) + sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + where := sql[strings.Index(sql, "WHERE ")+len("WHERE "):] + t.Logf("WHERE: %s", where) + + require.Equal(t, c.expectedArgs, args) + if c.expectedSQL != "" { + require.Equal(t, c.expectedSQL, where) + } + }) + } +} + +func TestExhaustiveJSONPlan_FieldMapper(t *testing.T) { + m := &fieldMapper{} + + key := &telemetrytypes.TelemetryFieldKey{ + Name: "education[].name", + Signal: telemetrytypes.SignalLogs, + FieldContext: telemetrytypes.FieldContextBody, + FieldDataType: telemetrytypes.FieldDataTypeString, + } + + expr, err := m.buildFieldForJSON(key) + require.NoError(t, err) + require.Equal(t, + "arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->dynamicElement(`body_v2.education`.`name`, 'String'), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')), arrayMap(`body_v2.education`->dynamicElement(`body_v2.education`.`name`, 'String'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(body_v2.`education`, 'Array(Dynamic)'))))))", + expr) +} diff --git a/pkg/telemetrylogs/json_stmt_builder_test.go b/pkg/telemetrylogs/json_stmt_builder_test.go index 60fd89a65e9..ce94cb5d1b8 100644 --- a/pkg/telemetrylogs/json_stmt_builder_test.go +++ b/pkg/telemetrylogs/json_stmt_builder_test.go @@ -64,7 +64,7 @@ func TestJSONStmtBuilder_TimeSeries(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __limit_cte AS (SELECT toString(multiIf((dynamicElement(body_v2.`user.age`, 'Int64') IS NOT NULL), toString(dynamicElement(body_v2.`user.age`, 'Int64')), (dynamicElement(body_v2.`user.age`, 'String') IS NOT NULL), dynamicElement(body_v2.`user.age`, 'String'), NULL)) AS `user.age`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `user.age` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf((dynamicElement(body_v2.`user.age`, 'Int64') IS NOT NULL), toString(dynamicElement(body_v2.`user.age`, 'Int64')), (dynamicElement(body_v2.`user.age`, 'String') IS NOT NULL), dynamicElement(body_v2.`user.age`, 'String'), NULL)) AS `user.age`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`user.age`) GLOBAL IN (SELECT `user.age` FROM __limit_cte) GROUP BY ts, `user.age`", + Query: "WITH __limit_cte AS (SELECT toString(multiIf((dynamicElement(body_v2.`user.age`, 'Int64') IS NOT NULL), toString(dynamicElement(body_v2.`user.age`, 'Int64')), (dynamicElement(body_v2.`user.age`, 'String') IS NOT NULL), dynamicElement(body_v2.`user.age`, 'String'), NULL)) AS `__GROUP_BY_KEY_0_user.age`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_user.age` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf((dynamicElement(body_v2.`user.age`, 'Int64') IS NOT NULL), toString(dynamicElement(body_v2.`user.age`, 'Int64')), (dynamicElement(body_v2.`user.age`, 'String') IS NOT NULL), dynamicElement(body_v2.`user.age`, 'String'), NULL)) AS `__GROUP_BY_KEY_0_user.age`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_user.age`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_user.age` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_user.age`", Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)}, }, }, @@ -88,7 +88,10 @@ func TestJSONStmtBuilder_TimeSeries(t *testing.T) { }, }, }, - expectedErrContains: "Group by/Aggregation isn't available for the Array Paths", + expected: qbtypes.Statement{ + Query: "WITH __limit_cte AS (SELECT toString(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))))) AS `__GROUP_BY_KEY_0_education[].awards[].type`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_education[].awards[].type` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'))))) AS `__GROUP_BY_KEY_0_education[].awards[].type`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_education[].awards[].type`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_education[].awards[].type` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_education[].awards[].type`", + Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)}, + }, }, } @@ -184,8 +187,8 @@ func TestJSONStmtBuilder_PrimitivePaths(t *testing.T) { name: "message Exists", filter: "message Exists", expected: TestExpected{ - WhereClause: "body_v2.message <> ?", - Args: []any{"", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + WhereClause: "body_v2.message <> ''", + Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, }, { @@ -1081,7 +1084,7 @@ func TestJSONStmtBuilder_SelectField(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AS `education[].awards[].participated[].members` FROM signoz_logs.distributed_logs_v2 WHERE (dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "SELECT timestamp, id, arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AS `__SELECT_KEY_0_education[].awards[].participated[].members` FROM signoz_logs.distributed_logs_v2 WHERE (dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, }, @@ -1102,7 +1105,7 @@ func TestJSONStmtBuilder_SelectField(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AS `education[].awards[].type` FROM signoz_logs.distributed_logs_v2 WHERE (dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "SELECT timestamp, id, arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->dynamicElement(`body_v2.education[].awards`.`type`, 'String'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AS `__SELECT_KEY_0_education[].awards[].type` FROM signoz_logs.distributed_logs_v2 WHERE (dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, }, @@ -1120,7 +1123,7 @@ func TestJSONStmtBuilder_SelectField(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, dynamicElement(body_v2.`user.name`, 'String') AS `user.name` FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Query: "SELECT timestamp, id, multiIf((dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL), dynamicElement(body_v2.`user.name`, 'String'), NULL) AS `__SELECT_KEY_0_user.name` FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, }, @@ -1174,7 +1177,7 @@ func TestJSONStmtBuilder_OrderBy(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) AS `education[].awards[].participated[].members` asc LIMIT ?", + Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))) asc LIMIT ?", Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, }, @@ -1197,7 +1200,7 @@ func TestJSONStmtBuilder_OrderBy(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY dynamicElement(body_v2.`user.name`, 'String') AS `user.name` asc LIMIT ?", + Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY multiIf((dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL), dynamicElement(body_v2.`user.name`, 'String'), NULL) asc LIMIT ?", Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, }, @@ -1259,8 +1262,8 @@ func TestResourceAggrAndGroupBy_WithJSONEnabled(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`region`::String IS NOT NULL, resource.`region`::String, NULL)) AS `region`, countDistinct(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE ((dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) OR mapContains(attributes_string, 'user.name') = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY ts, `region`", - Args: []any{true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)}, + Query: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`region` IS NOT NULL, resource.`region`::String, NULL)) AS `__GROUP_BY_KEY_0_region`, countDistinct(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE ((dynamicElement(body_v2.`user.name`, 'String') IS NOT NULL) OR mapContains(attributes_string, 'user.name')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY ts, `__GROUP_BY_KEY_0_region`", + Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)}, Warnings: []string{"Key `user.name` is ambiguous, found 2 different combinations of field context / data type: [name=user.name,context=body,datatype=string name=user.name,context=attribute,datatype=string]."}, }, }, @@ -1331,7 +1334,7 @@ func buildJSONTestStatementBuilder(t *testing.T, addIndexes bool) (*logQueryStat fm := NewFieldMapper(fl) cb := NewConditionBuilder(fm, fl) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewLogQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -1340,7 +1343,6 @@ func buildJSONTestStatementBuilder(t *testing.T, addIndexes bool) (*logQueryStat cb, aggExprRewriter, DefaultFullTextColumn, - GetBodyJSONKey, fl, nil, false, diff --git a/pkg/telemetrylogs/json_string.go b/pkg/telemetrylogs/json_string.go index d9e595d3439..2fd37f76b48 100644 --- a/pkg/telemetrylogs/json_string.go +++ b/pkg/telemetrylogs/json_string.go @@ -123,7 +123,15 @@ func GetBodyJSONKey(_ context.Context, key *telemetrytypes.TelemetryFieldKey, op // for all types except strings, we need to extract the value from the JSON_VALUE return fmt.Sprintf("JSONExtract(JSON_VALUE(body, '$.%s'), '%s')", getBodyJSONPath(key), dataType.CHDataType()), value } - // for string types, we should compare with the JSON_VALUE + // JSON_VALUE returns a String; stringify list operands so a numeric element in a mixed + // set (e.g. IN ['alpha', 42]) doesn't hit a String-vs-number supertype error (CH 386). + if list, ok := value.([]any); ok { + strs := make([]any, len(list)) + for i, e := range list { + strs[i] = fmt.Sprintf("%v", e) + } + value = strs + } return fmt.Sprintf("JSON_VALUE(body, '$.%s')", getBodyJSONPath(key)), value } @@ -198,11 +206,12 @@ func legacyCoerceNeedle(v any, dt telemetrytypes.FieldDataType) any { // getBodyJSONArrayKey extracts the leaf as Array(Nullable(
)) — Nullable so a value of a // different JSON type maps to NULL instead of corrupting (e.g. a non-numeric string → 0). func getBodyJSONArrayKey(key *telemetrytypes.TelemetryFieldKey, dt telemetrytypes.FieldDataType) string { - arrKey := *key - if !strings.HasSuffix(arrKey.Name, "[*]") && !strings.HasSuffix(arrKey.Name, "[]") { - arrKey.Name += "[*]" + name := key.Name + if !strings.HasSuffix(name, "[*]") && !strings.HasSuffix(name, "[]") { + name += "[*]" } - return fmt.Sprintf("JSONExtract(JSON_QUERY(body, '$.%s'), 'Array(Nullable(%s))')", getBodyJSONPath(&arrKey), dt.CHDataType()) + arrKey := telemetrytypes.NewTelemetryFieldKey(name, key.FieldContext, key.FieldDataType) + return fmt.Sprintf("JSONExtract(JSON_QUERY(body, '$.%s'), 'Array(Nullable(%s))')", getBodyJSONPath(arrKey), dt.CHDataType()) } // getBodyJSONScalarKey builds the single-element-set fallback for a scalar body value: the leaf @@ -215,9 +224,8 @@ func getBodyJSONScalarKey(key *telemetrytypes.TelemetryFieldKey, dt telemetrytyp if strings.Contains(name, "[") { return "", "", false } - scalarKey := *key - scalarKey.Name = name - path := getBodyJSONPath(&scalarKey) + scalarKey := telemetrytypes.NewTelemetryFieldKey(name, key.FieldContext, key.FieldDataType) + path := getBodyJSONPath(scalarKey) if dt == telemetrytypes.FieldDataTypeString { expr = fmt.Sprintf("JSON_VALUE(body, '$.%s')", path) } else { diff --git a/pkg/telemetrylogs/statement_builder.go b/pkg/telemetrylogs/statement_builder.go index d43d1b72eaa..ac07fff8cf8 100644 --- a/pkg/telemetrylogs/statement_builder.go +++ b/pkg/telemetrylogs/statement_builder.go @@ -30,7 +30,6 @@ type logQueryStatementBuilder struct { skipResourceFingerprintEnabled bool fullTextColumn *telemetrytypes.TelemetryFieldKey - jsonKeyToKey qbtypes.JsonKeyToFieldFunc } var _ qbtypes.StatementBuilder[qbtypes.LogAggregation] = (*logQueryStatementBuilder)(nil) @@ -42,7 +41,6 @@ func NewLogQueryStatementBuilder( conditionBuilder qbtypes.ConditionBuilder, aggExprRewriter qbtypes.AggExprRewriter, fullTextColumn *telemetrytypes.TelemetryFieldKey, - jsonKeyToKey qbtypes.JsonKeyToFieldFunc, fl flagger.Flagger, telemetryStore telemetrystore.TelemetryStore, skipResourceFingerprintEnable bool, @@ -73,7 +71,6 @@ func NewLogQueryStatementBuilder( fl: fl, skipResourceFingerprintEnabled: skipResourceFingerprintEnable, fullTextColumn: fullTextColumn, - jsonKeyToKey: jsonKeyToKey, } } @@ -273,9 +270,8 @@ func (b *logQueryStatementBuilder) buildListQuery( ) (*qbtypes.Statement, error) { var ( - cteFragments []string - cteArgs [][]any - bodyJSONEnabled = b.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) + cteFragments []string + cteArgs [][]any ) frag, args, skipResourceFilter, err := b.maybeAttachResourceFilter(ctx, orgID, sb, query, start, end, variables) @@ -299,7 +295,7 @@ func (b *logQueryStatementBuilder) buildListQuery( sb.SelectMore(LogsV2SeverityNumberColumn) sb.SelectMore(LogsV2ScopeNameColumn) sb.SelectMore(LogsV2ScopeVersionColumn) - sb.SelectMore(bodyAliasExpression(bodyJSONEnabled)) + sb.SelectMore(bodyAliasExpression(b.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)))) sb.SelectMore(LogsV2AttributesStringColumn) sb.SelectMore(LogsV2AttributesNumberColumn) sb.SelectMore(LogsV2AttributesBoolColumn) @@ -314,11 +310,11 @@ func (b *logQueryStatementBuilder) buildListQuery( } // get column expression for the field - use array index directly to avoid pointer to loop variable - colExpr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &query.SelectFields[index], keys) + colExpr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &query.SelectFields[index], telemetrytypes.FieldDataTypeUnspecified, keys) if err != nil { return nil, err } - sb.SelectMore(colExpr) + sb.SelectMore(fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colExpr), selectColumnAlias(index, query.SelectFields[index].Name))) } } @@ -333,11 +329,11 @@ func (b *logQueryStatementBuilder) buildListQuery( // Add order by for _, orderBy := range query.Order { - colExpr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &orderBy.Key.TelemetryFieldKey, keys) + colExpr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &orderBy.Key.TelemetryFieldKey, telemetrytypes.FieldDataTypeUnspecified, keys) if err != nil { return nil, err } - sb.OrderBy(fmt.Sprintf("%s %s", colExpr, orderBy.Direction.StringValue())) + sb.OrderBy(fmt.Sprintf("%s %s", sqlbuilder.Escape(colExpr), orderBy.Direction.StringValue())) } // Add limit and offset @@ -377,9 +373,8 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery( ) (*qbtypes.Statement, error) { var ( - cteFragments []string - cteArgs [][]any - bodyJSONEnabled = b.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) + cteFragments []string + cteArgs [][]any ) frag, args, skipResourceFilter, err := b.maybeAttachResourceFilter(ctx, orgID, sb, query, start, end, variables) @@ -396,20 +391,21 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery( int64(query.StepInterval.Seconds()), )) - var allGroupByArgs []any - // Keep original column expressions so we can build the tuple + bodyJSONEnabled := b.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) fieldNames := make([]string, 0, len(query.GroupBy)) - for _, gb := range query.GroupBy { - expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, orgID, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey, bodyJSONEnabled) + for i, gb := range query.GroupBy { + if !bodyJSONEnabled && (strings.Contains(gb.Name, telemetrytypes.ArraySep) || strings.Contains(gb.Name, telemetrytypes.ArrayAnyIndex)) { + return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "Group by/Aggregation isn't available for the Array Paths: %s", gb.Name) + } + expr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &gb.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return nil, err } - colExpr := fmt.Sprintf("toString(%s) AS `%s`", expr, gb.Name) - allGroupByArgs = append(allGroupByArgs, args...) - sb.SelectMore(colExpr) - fieldNames = append(fieldNames, fmt.Sprintf("`%s`", gb.Name)) + fieldAlias := groupByColumnAlias(i, gb.Name) + sb.SelectMore(fmt.Sprintf("toString(%s) AS `%s`", sqlbuilder.Escape(expr), fieldAlias)) + fieldNames = append(fieldNames, fmt.Sprintf("`%s`", fieldAlias)) } // Aggregations @@ -456,7 +452,7 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery( // Group by all dimensions sb.GroupBy("ts") - sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...) + sb.GroupBy(fieldNames...) if query.Having != nil && query.Having.Expression != "" { // Rewrite having expression to use SQL column names rewriter := querybuilder.NewHavingExpressionRewriter() @@ -471,13 +467,17 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery( for _, orderBy := range query.Order { _, ok := aggOrderBy(orderBy, query) if !ok { - sb.OrderBy(fmt.Sprintf("`%s` %s", orderBy.Key.Name, orderBy.Direction.StringValue())) + orderCol := orderBy.Key.Name + if alias, ok := groupByOrderAlias(orderBy.Key.Name, query.GroupBy); ok { + orderCol = alias + } + sb.OrderBy(fmt.Sprintf("`%s` %s", orderCol, orderBy.Direction.StringValue())) } } sb.OrderBy("ts desc") } - combinedArgs := append(allGroupByArgs, allAggChArgs...) + combinedArgs := allAggChArgs mainSQL, mainArgs := sb.BuildWithFlavor(sqlbuilder.ClickHouse, combinedArgs...) // Stitch it all together: WITH … SELECT … @@ -486,7 +486,7 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery( } else { sb.GroupBy("ts") - sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...) + sb.GroupBy(fieldNames...) if query.Having != nil && query.Having.Expression != "" { rewriter := querybuilder.NewHavingExpressionRewriter() rewrittenExpr, err := rewriter.RewriteForLogs(query.Having.Expression, query.Aggregations) @@ -500,13 +500,17 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery( for _, orderBy := range query.Order { _, ok := aggOrderBy(orderBy, query) if !ok { - sb.OrderBy(fmt.Sprintf("`%s` %s", orderBy.Key.Name, orderBy.Direction.StringValue())) + orderCol := orderBy.Key.Name + if alias, ok := groupByOrderAlias(orderBy.Key.Name, query.GroupBy); ok { + orderCol = alias + } + sb.OrderBy(fmt.Sprintf("`%s` %s", orderCol, orderBy.Direction.StringValue())) } } sb.OrderBy("ts desc") } - combinedArgs := append(allGroupByArgs, allAggChArgs...) + combinedArgs := allAggChArgs mainSQL, mainArgs := sb.BuildWithFlavor(sqlbuilder.ClickHouse, combinedArgs...) // Stitch it all together: WITH … SELECT … @@ -537,9 +541,8 @@ func (b *logQueryStatementBuilder) buildScalarQuery( ) (*qbtypes.Statement, error) { var ( - cteFragments []string - cteArgs [][]any - bodyJSONEnabled = b.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) + cteFragments []string + cteArgs [][]any ) frag, args, skipResourceFilter, err := b.maybeAttachResourceFilter(ctx, orgID, sb, query, start, end, variables) @@ -553,17 +556,20 @@ func (b *logQueryStatementBuilder) buildScalarQuery( allAggChArgs := []any{} - var allGroupByArgs []any - - for _, gb := range query.GroupBy { - expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, orgID, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey, bodyJSONEnabled) + bodyJSONEnabled := b.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) + fieldNames := make([]string, 0, len(query.GroupBy)) + for i, gb := range query.GroupBy { + if !bodyJSONEnabled && (strings.Contains(gb.Name, telemetrytypes.ArraySep) || strings.Contains(gb.Name, telemetrytypes.ArrayAnyIndex)) { + return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "Group by/Aggregation isn't available for the Array Paths: %s", gb.Name) + } + expr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &gb.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return nil, err } - colExpr := fmt.Sprintf("toString(%s) AS `%s`", expr, gb.Name) - allGroupByArgs = append(allGroupByArgs, args...) - sb.SelectMore(colExpr) + fieldAlias := groupByColumnAlias(i, gb.Name) + sb.SelectMore(fmt.Sprintf("toString(%s) AS `%s`", sqlbuilder.Escape(expr), fieldAlias)) + fieldNames = append(fieldNames, fmt.Sprintf("`%s`", fieldAlias)) } // for scalar queries, the rate would be end-start @@ -596,7 +602,7 @@ func (b *logQueryStatementBuilder) buildScalarQuery( } // Group by dimensions - sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...) + sb.GroupBy(fieldNames...) // Add having clause if needed if query.Having != nil && query.Having.Expression != "" { @@ -614,7 +620,11 @@ func (b *logQueryStatementBuilder) buildScalarQuery( if ok { sb.OrderBy(fmt.Sprintf("__result_%d %s", idx, orderBy.Direction.StringValue())) } else { - sb.OrderBy(fmt.Sprintf("`%s` %s", orderBy.Key.Name, orderBy.Direction.StringValue())) + orderCol := orderBy.Key.Name + if alias, ok := groupByOrderAlias(orderBy.Key.Name, query.GroupBy); ok { + orderCol = alias + } + sb.OrderBy(fmt.Sprintf("`%s` %s", orderCol, orderBy.Direction.StringValue())) } } @@ -628,7 +638,7 @@ func (b *logQueryStatementBuilder) buildScalarQuery( sb.Limit(query.Limit) } - combinedArgs := append(allGroupByArgs, allAggChArgs...) + combinedArgs := allAggChArgs mainSQL, mainArgs := sb.BuildWithFlavor(sqlbuilder.ClickHouse, combinedArgs...) @@ -713,6 +723,23 @@ func aggOrderBy(k qbtypes.OrderBy, q qbtypes.QueryBuilderQuery[qbtypes.LogAggreg return 0, false } +func groupByColumnAlias(i int, name string) string { + return fmt.Sprintf("__GROUP_BY_KEY_%d_%s", i, name) +} + +func selectColumnAlias(i int, name string) string { + return fmt.Sprintf("__SELECT_KEY_%d_%s", i, name) +} + +func groupByOrderAlias(orderKey string, groupBy []qbtypes.GroupByKey) (string, bool) { + for i := range groupBy { + if groupBy[i].Name == orderKey { + return groupByColumnAlias(i, groupBy[i].Name), true + } + } + return "", false +} + func (b *logQueryStatementBuilder) maybeAttachResourceFilter( ctx context.Context, orgID valuer.UUID, diff --git a/pkg/telemetrylogs/stmt_builder_test.go b/pkg/telemetrylogs/stmt_builder_test.go index 123fcff9bb1..23a03c8008c 100644 --- a/pkg/telemetrylogs/stmt_builder_test.go +++ b/pkg/telemetrylogs/stmt_builder_test.go @@ -73,7 +73,7 @@ func TestStatementBuilderTimeSeries(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, countDistinct(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, countDistinct(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS `__GROUP_BY_KEY_0_service.name`, countDistinct(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS `__GROUP_BY_KEY_0_service.name`, countDistinct(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`", Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1705397400), uint64(1705485600), "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600), 10, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)}, }, expectedErr: nil, @@ -104,8 +104,8 @@ func TestStatementBuilderTimeSeries(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, countDistinct(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.method'] = ? AND mapContains(attributes_string, 'http.method') = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, countDistinct(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.method'] = ? AND mapContains(attributes_string, 'http.method') = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`", - Args: []any{"redis-manual", "GET", true, "1705226400000000000", uint64(1705224600), "1705485600000000000", uint64(1705485600), 10, "redis-manual", "GET", true, "1705226400000000000", uint64(1705224600), "1705485600000000000", uint64(1705485600)}, + Query: "WITH __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, countDistinct(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.method'] = ? AND mapContains(attributes_string, 'http.method'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, countDistinct(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.method'] = ? AND mapContains(attributes_string, 'http.method'))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`", + Args: []any{"redis-manual", "GET", "1705226400000000000", uint64(1705224600), "1705485600000000000", uint64(1705485600), 10, "redis-manual", "GET", "1705226400000000000", uint64(1705224600), "1705485600000000000", uint64(1705485600)}, }, expectedErr: nil, }, @@ -145,7 +145,7 @@ func TestStatementBuilderTimeSeries(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY `service.name` desc LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name` ORDER BY `service.name` desc, ts desc", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY `__GROUP_BY_KEY_0_service.name` desc LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name` ORDER BY `__GROUP_BY_KEY_0_service.name` desc, ts desc", Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1705397400), uint64(1705485600), "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600), 10, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)}, }, expectedErr: nil, @@ -178,8 +178,8 @@ func TestStatementBuilderTimeSeries(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(`attribute_string_materialized$$key$$name_exists` = ?, `attribute_string_materialized$$key$$name`, NULL)) AS `materialized.key.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `materialized.key.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(`attribute_string_materialized$$key$$name_exists` = ?, `attribute_string_materialized$$key$$name`, NULL)) AS `materialized.key.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`materialized.key.name`) GLOBAL IN (SELECT `materialized.key.name` FROM __limit_cte) GROUP BY ts, `materialized.key.name`", - Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1705397400), uint64(1705485600), true, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600), 10, true, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)}, + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(`attribute_string_materialized$$key$$name_exists`, `attribute_string_materialized$$key$$name`, NULL)) AS `__GROUP_BY_KEY_0_materialized.key.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_materialized.key.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(`attribute_string_materialized$$key$$name_exists`, `attribute_string_materialized$$key$$name`, NULL)) AS `__GROUP_BY_KEY_0_materialized.key.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_materialized.key.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_materialized.key.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_materialized.key.name`", + Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1705397400), uint64(1705485600), "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600), 10, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)}, }, }, { @@ -201,8 +201,8 @@ func TestStatementBuilderTimeSeries(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE ((match(`attribute_string_materialized$$key$$name`, ?) AND `attribute_string_materialized$$key$$name_exists` = ?) OR (`attribute_string_materialized$$key$$name` = ? AND `attribute_string_materialized$$key$$name_exists` = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY ts", - Args: []any{"redis.*", true, "memcached", true, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)}, + Query: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE ((match(`attribute_string_materialized$$key$$name`, ?) AND `attribute_string_materialized$$key$$name_exists`) OR (`attribute_string_materialized$$key$$name` = ? AND `attribute_string_materialized$$key$$name_exists`)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY ts", + Args: []any{"redis.*", "memcached", "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)}, }, expectedErr: nil, }, @@ -219,7 +219,7 @@ func TestStatementBuilderTimeSeries(t *testing.T) { fm := NewFieldMapper(fl) cb := NewConditionBuilder(fm, fl) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewLogQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -228,7 +228,6 @@ func TestStatementBuilderTimeSeries(t *testing.T) { cb, aggExprRewriter, DefaultFullTextColumn, - GetBodyJSONKey, fl, nil, false, @@ -302,7 +301,7 @@ func TestStatementBuilderListQuery(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY `attribute_string_materialized$$key$$name` AS `materialized.key.name` desc LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY multiIf(`attribute_string_materialized$$key$$name_exists`, `attribute_string_materialized$$key$$name`, NULL) desc LIMIT ?", Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -330,8 +329,8 @@ func TestStatementBuilderListQuery(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((match(`attribute_string_materialized$$key$$name`, ?) AND `attribute_string_materialized$$key$$name_exists` = ?) OR (`attribute_string_materialized$$key$$name` = ? AND `attribute_string_materialized$$key$$name_exists` = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY `attribute_string_materialized$$key$$name` AS `materialized.key.name` desc LIMIT ?", - Args: []any{"redis.*", true, "memcached", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((match(`attribute_string_materialized$$key$$name`, ?) AND `attribute_string_materialized$$key$$name_exists`) OR (`attribute_string_materialized$$key$$name` = ? AND `attribute_string_materialized$$key$$name_exists`)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY multiIf(`attribute_string_materialized$$key$$name_exists`, `attribute_string_materialized$$key$$name`, NULL) desc LIMIT ?", + Args: []any{"redis.*", "memcached", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, }, @@ -363,7 +362,7 @@ func TestStatementBuilderListQuery(t *testing.T) { mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime) cb := NewConditionBuilder(fm, fl) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewLogQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -372,7 +371,6 @@ func TestStatementBuilderListQuery(t *testing.T) { cb, aggExprRewriter, DefaultFullTextColumn, - GetBodyJSONKey, fl, nil, false, @@ -448,7 +446,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND match(LOWER(body), LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY `attribute_string_materialized$$key$$name` AS `materialized.key.name` desc LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND match(LOWER(body), LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY multiIf(`attribute_string_materialized$$key$$name_exists`, `attribute_string_materialized$$key$$name`, NULL) desc LIMIT ?", Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), "hello", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -464,8 +462,9 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (JSON_VALUE(body, '$.\"status\"') = ? AND JSON_EXISTS(body, '$.\"status\"')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{"success", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (JSON_VALUE(body, '$.\"status\"') = ? AND JSON_EXISTS(body, '$.\"status\"')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{"success", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Warnings: []string{querybuilder.NewKeyNotFoundWarning("status")}, }, expectedErr: nil, }, @@ -480,8 +479,9 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((JSONExtract(JSON_QUERY(body, '$.\"user_names\"[*]'), 'Array(String)') = ?) AND JSON_EXISTS(body, '$.\"user_names\"[*]')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{"john_doe", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((JSONExtract(JSON_QUERY(body, '$.\"user_names\"[*]'), 'Array(String)') = ?) AND JSON_EXISTS(body, '$.\"user_names\"[*]')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{"john_doe", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Warnings: []string{querybuilder.NewKeyNotFoundWarning("user_names[*]")}, }, expectedErr: nil, }, @@ -496,8 +496,9 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (has(JSONExtract(JSON_QUERY(body, '$.\"user_names\"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$.\"user_names\"') = ? AND JSONType(body, 'user_names') NOT IN ('Array', 'Object', 'Null')), false)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{"john_doe", "john_doe", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (has(JSONExtract(JSON_QUERY(body, '$.\"user_names\"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$.\"user_names\"') = ? AND JSONType(body, 'user_names') NOT IN ('Array', 'Object', 'Null')), false)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{"john_doe", "john_doe", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Warnings: []string{querybuilder.NewKeyNotFoundWarning("user_names[*]")}, }, expectedErr: nil, }, @@ -512,7 +513,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) { mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime) cb := NewConditionBuilder(fm, fl) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewLogQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -521,7 +522,6 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) { cb, aggExprRewriter, DefaultFullTextColumn, - GetBodyJSONKey, fl, nil, false, @@ -578,7 +578,7 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) { }, }, }, - expectedErrContains: "Group by/Aggregation isn't available for the body column", + expectedErrContains: "Operation isn't available for the body column", }, } @@ -591,7 +591,7 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) { mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime) cb := NewConditionBuilder(fm, fl) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewLogQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -600,7 +600,6 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) { cb, aggExprRewriter, DefaultFullTextColumn, - GetBodyJSONKey, fl, nil, false, @@ -674,7 +673,7 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND LOWER(body) LIKE LOWER(?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY `attribute_string_materialized$$key$$name` AS `materialized.key.name` desc LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND LOWER(body) LIKE LOWER(?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY multiIf(`attribute_string_materialized$$key$$name_exists`, `attribute_string_materialized$$key$$name`, NULL) desc LIMIT ?", Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), "%error%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, @@ -689,7 +688,7 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) { mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision() cb := NewConditionBuilder(fm, fl) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewLogQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -698,7 +697,6 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) { cb, aggExprRewriter, DefaultFullTextColumn, - GetBodyJSONKey, fl, nil, false, @@ -916,7 +914,7 @@ func TestAdjustKey(t *testing.T) { mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision() cb := NewConditionBuilder(fm, fl) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewLogQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -925,7 +923,6 @@ func TestAdjustKey(t *testing.T) { cb, aggExprRewriter, DefaultFullTextColumn, - GetBodyJSONKey, fl, nil, false, @@ -969,8 +966,8 @@ func TestStmtBuilderBodyField(t *testing.T) { }, enableUseJSONBody: true, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE body_v2.message <> ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{"", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE body_v2.message <> '' AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, Warnings: []string{bodySearchDefaultWarning}, }, expectedErr: nil, @@ -985,8 +982,8 @@ func TestStmtBuilderBodyField(t *testing.T) { }, enableUseJSONBody: false, expected: qbtypes.Statement{ - Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE body <> ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", - Args: []any{"", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, + Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE body <> '' AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?", + Args: []any{"1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10}, }, expectedErr: nil, }, @@ -1065,7 +1062,7 @@ func TestStmtBuilderBodyField(t *testing.T) { f := field mockMetadataStore.KeysMap[field.Name] = append(mockMetadataStore.KeysMap[field.Name], &f) } - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewLogQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), mockMetadataStore, @@ -1073,7 +1070,6 @@ func TestStmtBuilderBodyField(t *testing.T) { cb, aggExprRewriter, DefaultFullTextColumn, - GetBodyJSONKey, fl, nil, false, @@ -1167,7 +1163,7 @@ func TestStmtBuilderBodyFullTextSearch(t *testing.T) { f := field mockMetadataStore.KeysMap[field.Name] = append(mockMetadataStore.KeysMap[field.Name], &f) } - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewLogQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), mockMetadataStore, @@ -1175,7 +1171,6 @@ func TestStmtBuilderBodyFullTextSearch(t *testing.T) { cb, aggExprRewriter, DefaultFullTextColumn, - GetBodyJSONKey, fl, nil, false, @@ -1288,7 +1283,6 @@ func newSkipResourceFingerprintLogsBuilder( DefaultFullTextColumn, fm, cb, - GetBodyJSONKey, fl, ) @@ -1299,7 +1293,6 @@ func newSkipResourceFingerprintLogsBuilder( cb, aggExprRewriter, DefaultFullTextColumn, - GetBodyJSONKey, fl, telemetryStore, skipEnable, diff --git a/pkg/telemetrymetadata/condition_builder.go b/pkg/telemetrymetadata/condition_builder.go index 30d7e6aed9e..aa18e5d3254 100644 --- a/pkg/telemetrymetadata/condition_builder.go +++ b/pkg/telemetrymetadata/condition_builder.go @@ -20,12 +20,14 @@ func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder { return &conditionBuilder{fm: fm} } +// Metadata has no resource sub-query, so options are unused. func (c *conditionBuilder) ConditionFor( ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey, - fieldKeysForName []*telemetrytypes.TelemetryFieldKey, + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, + _ qbtypes.ConditionBuilderOptions, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder, @@ -36,9 +38,8 @@ func (c *conditionBuilder) ConditionFor( return nil, nil, err } - // metadata builds best-effort filters for related-values lookups; an unknown key - // simply yields no condition rather than an error. - keys, warning := querybuilder.ResolveKeys(key, fieldKeysForName) + // an unknown key simply yields no condition rather than an error. + keys, warning := querybuilder.ResolveKeys(key, querybuilder.MatchingFieldKeys(key, fieldKeys)) var warnings []string if warning != "" { warnings = append(warnings, warning) diff --git a/pkg/telemetrymetadata/condition_builder_test.go b/pkg/telemetrymetadata/condition_builder_test.go index bdb0361ecb6..17591bf18a0 100644 --- a/pkg/telemetrymetadata/condition_builder_test.go +++ b/pkg/telemetrymetadata/condition_builder_test.go @@ -54,7 +54,7 @@ func TestConditionFor(t *testing.T) { for _, tc := range testCases { sb := sqlbuilder.NewSelectBuilder() t.Run(tc.name, func(t *testing.T) { - cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, 0, 0, &tc.key, []*telemetrytypes.TelemetryFieldKey{&tc.key}, tc.operator, tc.value, sb) + cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, 0, 0, &tc.key, map[string][]*telemetrytypes.TelemetryFieldKey{tc.key.Name: {&tc.key}}, qbtypes.ConditionBuilderOptions{}, tc.operator, tc.value, sb) sb.Where(cond...) if tc.expectedError != nil { diff --git a/pkg/telemetrymetadata/field_mapper.go b/pkg/telemetrymetadata/field_mapper.go index 125e012ddd4..02faf7ec406 100644 --- a/pkg/telemetrymetadata/field_mapper.go +++ b/pkg/telemetrymetadata/field_mapper.go @@ -79,6 +79,7 @@ func (m *fieldMapper) ColumnExpressionFor( orgID valuer.UUID, startNs, endNs uint64, field *telemetrytypes.TelemetryFieldKey, + _ telemetrytypes.FieldDataType, keys map[string][]*telemetrytypes.TelemetryFieldKey, ) (string, error) { diff --git a/pkg/telemetrymetadata/metadata.go b/pkg/telemetrymetadata/metadata.go index b2340c8badc..73b4719f678 100644 --- a/pkg/telemetrymetadata/metadata.go +++ b/pkg/telemetrymetadata/metadata.go @@ -1444,20 +1444,20 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, orgID valuer. // search on attributes key.FieldContext = telemetrytypes.FieldContextAttribute - attrConds, _, err := t.conditionBuilder.ConditionFor(ctx, orgID, 0, 0, key, []*telemetrytypes.TelemetryFieldKey{key}, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb) + attrConds, _, err := t.conditionBuilder.ConditionFor(ctx, orgID, 0, 0, key, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}}, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb) if err == nil { conds = append(conds, attrConds...) } // search on resource key.FieldContext = telemetrytypes.FieldContextResource - resourceConds, _, err := t.conditionBuilder.ConditionFor(ctx, orgID, 0, 0, key, []*telemetrytypes.TelemetryFieldKey{key}, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb) + resourceConds, _, err := t.conditionBuilder.ConditionFor(ctx, orgID, 0, 0, key, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}}, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb) if err == nil { conds = append(conds, resourceConds...) } key.FieldContext = origContext } else { - keyConds, _, err := t.conditionBuilder.ConditionFor(ctx, orgID, 0, 0, key, []*telemetrytypes.TelemetryFieldKey{key}, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb) + keyConds, _, err := t.conditionBuilder.ConditionFor(ctx, orgID, 0, 0, key, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}}, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb) if err == nil { conds = append(conds, keyConds...) } diff --git a/pkg/telemetrymeter/statement_builder.go b/pkg/telemetrymeter/statement_builder.go index e5e4b85c1bc..4d9bd23ae68 100644 --- a/pkg/telemetrymeter/statement_builder.go +++ b/pkg/telemetrymeter/statement_builder.go @@ -126,7 +126,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath( stepSec, )) for _, g := range query.GroupBy { - col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, keys) + col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return "", nil, err } @@ -215,7 +215,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta( )) for _, g := range query.GroupBy { - col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, keys) + col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return "", nil, err } @@ -293,7 +293,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified( stepSec, )) for _, g := range query.GroupBy { - col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, keys) + col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return "", nil, err } diff --git a/pkg/telemetrymetrics/condition_builder.go b/pkg/telemetrymetrics/condition_builder.go index f9e893c78ca..2681b905689 100644 --- a/pkg/telemetrymetrics/condition_builder.go +++ b/pkg/telemetrymetrics/condition_builder.go @@ -143,13 +143,15 @@ func (c *conditionBuilder) conditionFor( return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator: %v", operator) } +// Metrics has no resource sub-query, so options are unused. func (c *conditionBuilder) ConditionFor( ctx context.Context, orgID valuer.UUID, startNs uint64, endNs uint64, key *telemetrytypes.TelemetryFieldKey, - fieldKeysForName []*telemetrytypes.TelemetryFieldKey, + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, + _ qbtypes.ConditionBuilderOptions, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder, @@ -160,7 +162,7 @@ func (c *conditionBuilder) ConditionFor( return nil, nil, err } - keys, warning := querybuilder.ResolveKeys(key, fieldKeysForName) + keys, warning := querybuilder.ResolveKeys(key, querybuilder.MatchingFieldKeys(key, fieldKeys)) var warnings []string if warning != "" { warnings = append(warnings, warning) diff --git a/pkg/telemetrymetrics/condition_builder_test.go b/pkg/telemetrymetrics/condition_builder_test.go index a85afd6f13a..9cdad13dcae 100644 --- a/pkg/telemetrymetrics/condition_builder_test.go +++ b/pkg/telemetrymetrics/condition_builder_test.go @@ -235,7 +235,7 @@ func TestConditionFor(t *testing.T) { for _, tc := range testCases { sb := sqlbuilder.NewSelectBuilder() t.Run(tc.name, func(t *testing.T) { - cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, 0, 0, &tc.key, []*telemetrytypes.TelemetryFieldKey{&tc.key}, tc.operator, tc.value, sb) + cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, 0, 0, &tc.key, map[string][]*telemetrytypes.TelemetryFieldKey{tc.key.Name: {&tc.key}}, qbtypes.ConditionBuilderOptions{}, tc.operator, tc.value, sb) sb.Where(cond...) if tc.expectedError != nil { @@ -290,7 +290,7 @@ func TestConditionForMultipleKeys(t *testing.T) { t.Run(tc.name, func(t *testing.T) { var err error for _, key := range tc.keys { - cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, []*telemetrytypes.TelemetryFieldKey{&key}, tc.operator, tc.value, sb) + cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, qbtypes.ConditionBuilderOptions{}, tc.operator, tc.value, sb) sb.Where(cond...) if err != nil { t.Fatalf("Error getting condition for key %s: %v", key.Name, err) diff --git a/pkg/telemetrymetrics/field_mapper.go b/pkg/telemetrymetrics/field_mapper.go index fea15d8e78c..09615ba3b21 100644 --- a/pkg/telemetrymetrics/field_mapper.go +++ b/pkg/telemetrymetrics/field_mapper.go @@ -102,6 +102,7 @@ func (m *fieldMapper) ColumnExpressionFor( orgID valuer.UUID, startNs, endNs uint64, field *telemetrytypes.TelemetryFieldKey, + _ telemetrytypes.FieldDataType, keys map[string][]*telemetrytypes.TelemetryFieldKey, ) (string, error) { diff --git a/pkg/telemetrymetrics/statement_builder.go b/pkg/telemetrymetrics/statement_builder.go index 9a24be1876e..103fb9375e2 100644 --- a/pkg/telemetrymetrics/statement_builder.go +++ b/pkg/telemetrymetrics/statement_builder.go @@ -315,7 +315,7 @@ func (b *MetricQueryStatementBuilder) buildReducedTimeSeriesCTE( sb.From(fmt.Sprintf("%s.%s", DBName, TimeseriesV4ReducedLocalTableName)) sb.Select("fingerprint") for _, g := range query.GroupBy { - col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, keys) + col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return "", nil, err } @@ -533,7 +533,7 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE( sb.Select("fingerprint") for _, g := range query.GroupBy { - col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, keys) + col, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &g.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return "", nil, err } diff --git a/pkg/telemetryresourcefilter/condition_builder.go b/pkg/telemetryresourcefilter/condition_builder.go index 6aaf4c2152a..3ae65a733de 100644 --- a/pkg/telemetryresourcefilter/condition_builder.go +++ b/pkg/telemetryresourcefilter/condition_builder.go @@ -16,10 +16,7 @@ type defaultConditionBuilder struct { fm qbtypes.FieldMapper } -var ( - _ qbtypes.ConditionBuilder = (*defaultConditionBuilder)(nil) - _ qbtypes.ResolvingConditionBuilder = (*defaultConditionBuilder)(nil) -) +var _ qbtypes.ConditionBuilder = (*defaultConditionBuilder)(nil) func NewConditionBuilder(fm qbtypes.FieldMapper) *defaultConditionBuilder { return &defaultConditionBuilder{fm: fm} @@ -47,50 +44,20 @@ func keyIndexFilter(key *telemetrytypes.TelemetryFieldKey) any { return fmt.Sprintf(`%%%s%%`, key.Name) } -// ConditionFor builds resource-fingerprint conditions from the caller's pre-matched -// fieldKeysForName. Resolution and the build live in conditionsForKeys. +// SkipResourceFilter is not applicable here: the fingerprint table only stores resource attributes. func (b *defaultConditionBuilder) ConditionFor( ctx context.Context, _ valuer.UUID, startNs uint64, endNs uint64, key *telemetrytypes.TelemetryFieldKey, - fieldKeysForName []*telemetrytypes.TelemetryFieldKey, - op qbtypes.FilterOperator, - value any, - sb *sqlbuilder.SelectBuilder, -) ([]string, []string, error) { - return b.conditionsForKeys(ctx, startNs, endNs, key, fieldKeysForName, op, value, sb) -} - -// ConditionForKeys owns key resolution from the full metadata map so the where-clause -// visitor need not pre-resolve; SkipResourceFilter is not applicable here (the fingerprint -// table only stores resource attributes). See qbtypes.ResolvingConditionBuilder. -func (b *defaultConditionBuilder) ConditionForKeys( - ctx context.Context, - _ valuer.UUID, - startNs uint64, - endNs uint64, - key *telemetrytypes.TelemetryFieldKey, - keys map[string][]*telemetrytypes.TelemetryFieldKey, + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, _ qbtypes.ConditionBuilderOptions, op qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder, ) ([]string, []string, error) { - return b.conditionsForKeys(ctx, startNs, endNs, key, querybuilder.MatchingFieldKeys(key, keys), op, value, sb) -} - -func (b *defaultConditionBuilder) conditionsForKeys( - ctx context.Context, - startNs uint64, - endNs uint64, - key *telemetrytypes.TelemetryFieldKey, - matches []*telemetrytypes.TelemetryFieldKey, - op qbtypes.FilterOperator, - value any, - sb *sqlbuilder.SelectBuilder, -) ([]string, []string, error) { + matches := querybuilder.MatchingFieldKeys(key, fieldKeys) // has/hasAny/hasAll/hasToken are logs-body-only functions; they never apply to the // resource fingerprint table, so skip them (the main query still evaluates them). diff --git a/pkg/telemetryresourcefilter/condition_builder_test.go b/pkg/telemetryresourcefilter/condition_builder_test.go index 50f88044fbf..8ef97cff00d 100644 --- a/pkg/telemetryresourcefilter/condition_builder_test.go +++ b/pkg/telemetryresourcefilter/condition_builder_test.go @@ -206,7 +206,7 @@ func TestConditionBuilder(t *testing.T) { for _, tc := range testCases { sb := sqlbuilder.NewSelectBuilder() t.Run(tc.name, func(t *testing.T) { - cond, _, err := conditionBuilder.ConditionFor(context.Background(), valuer.UUID{}, 0, 0, tc.key, []*telemetrytypes.TelemetryFieldKey{tc.key}, tc.op, tc.value, sb) + cond, _, err := conditionBuilder.ConditionFor(context.Background(), valuer.UUID{}, 0, 0, tc.key, map[string][]*telemetrytypes.TelemetryFieldKey{tc.key.Name: {tc.key}}, qbtypes.ConditionBuilderOptions{}, tc.op, tc.value, sb) sb.Where(cond...) if tc.expectedErr != nil { diff --git a/pkg/telemetryresourcefilter/field_mapper.go b/pkg/telemetryresourcefilter/field_mapper.go index e3fff4ee8d8..c7c874c0a72 100644 --- a/pkg/telemetryresourcefilter/field_mapper.go +++ b/pkg/telemetryresourcefilter/field_mapper.go @@ -76,6 +76,7 @@ func (m *defaultFieldMapper) ColumnExpressionFor( orgID valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey, + _ telemetrytypes.FieldDataType, _ map[string][]*telemetrytypes.TelemetryFieldKey, ) (string, error) { fieldExpression, err := m.FieldFor(ctx, orgID, tsStart, tsEnd, key) diff --git a/pkg/telemetrytraces/condition_builder.go b/pkg/telemetrytraces/condition_builder.go index 31685393bd7..82355c89dc8 100644 --- a/pkg/telemetrytraces/condition_builder.go +++ b/pkg/telemetrytraces/condition_builder.go @@ -8,7 +8,6 @@ import ( "strings" "time" - schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/querybuilder" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" @@ -22,10 +21,7 @@ type conditionBuilder struct { fm qbtypes.FieldMapper } -var ( - _ qbtypes.ConditionBuilder = (*conditionBuilder)(nil) - _ qbtypes.ResolvingConditionBuilder = (*conditionBuilder)(nil) -) +var _ qbtypes.ConditionBuilder = (*conditionBuilder)(nil) func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder { return &conditionBuilder{fm: fm} @@ -46,13 +42,6 @@ func (c *conditionBuilder) conditionFor( value = querybuilder.FormatValueForContains(value) } - // first, locate the raw column type (so we can choose the right EXISTS logic) - columns, err := c.fm.ColumnFor(ctx, orgID, startNs, endNs, key) - if err != nil { - return "", err - } - - // then ask the mapper for the actual SQL reference fieldExpression, err := c.fm.FieldFor(ctx, orgID, startNs, endNs, key) if err != nil { return "", err @@ -164,144 +153,52 @@ func (c *conditionBuilder) conditionFor( // in the query builder, `exists` and `not exists` are used for // key membership checks, so depending on the column type, the condition changes case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists: - - var value any - column := columns[0] - if len(key.Evolutions) > 0 { - // we will use the corresponding column and its evolution entry for the query - newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs) - if err != nil { - return "", err - } - - if len(newColumns) == 0 { - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "no valid evolution found for field %s in the given time range", key.Name) - } - - // Multiple columns means fieldExpression is a multiIf returning NULL when none match, - // so a simple null check is sufficient. - if len(newColumns) > 1 { - if operator == qbtypes.FilterOperatorExists { - return sb.IsNotNull(fieldExpression), nil - } else { - return sb.IsNull(fieldExpression), nil - } - } - - // otherwise we have to find the correct exist operator based on the column type - column = newColumns[0] + columns, err := c.fm.ColumnFor(ctx, orgID, startNs, endNs, key) + if err != nil { + return "", err } - - switch column.Type.GetType() { - case schema.ColumnTypeEnumJSON: - if operator == qbtypes.FilterOperatorExists { - return sb.IsNotNull(fieldExpression), nil - } else { - return sb.IsNull(fieldExpression), nil - } - case schema.ColumnTypeEnumString, - schema.ColumnTypeEnumFixedString, - schema.ColumnTypeEnumDateTime64: - value = "" - if operator == qbtypes.FilterOperatorExists { - return sb.NE(fieldExpression, value), nil - } else { - return sb.E(fieldExpression, value), nil - } - case schema.ColumnTypeEnumLowCardinality: - switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() { - case schema.ColumnTypeEnumString: - value = "" - if operator == qbtypes.FilterOperatorExists { - return sb.NE(fieldExpression, value), nil - } - return sb.E(fieldExpression, value), nil - default: - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for low cardinality column type %s", elementType) - } - - case schema.ColumnTypeEnumUInt64, - schema.ColumnTypeEnumUInt32, - schema.ColumnTypeEnumUInt8, - schema.ColumnTypeEnumInt8, - schema.ColumnTypeEnumInt16, - schema.ColumnTypeEnumBool: - value = 0 - if operator == qbtypes.FilterOperatorExists { - return sb.NE(fieldExpression, value), nil - } else { - return sb.E(fieldExpression, value), nil - } - case schema.ColumnTypeEnumMap: - keyType := column.Type.(schema.MapColumnType).KeyType - if _, ok := keyType.(schema.LowCardinalityColumnType); !ok { - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type) - } - - switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() { - case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64: - leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name) - if key.Materialized { - leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key) - } - if operator == qbtypes.FilterOperatorExists { - return sb.E(leftOperand, true), nil - } else { - return sb.NE(leftOperand, true), nil - } - default: - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType) - } - default: - return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", column.Type) + pred, err := querybuilder.ExistsExpression(columns, key, startNs, endNs, fieldExpression, operator == qbtypes.FilterOperatorExists) + if err != nil { + return "", err } + return sqlbuilder.Escape(pred), nil } return "", nil } -// ConditionFor builds the conditions for a filter term from the caller's pre-matched -// fieldKeysForName; skip-resource-filter is the caller's responsibility on this path. -func (c *conditionBuilder) ConditionFor( - ctx context.Context, - orgID valuer.UUID, - startNs uint64, - endNs uint64, - key *telemetrytypes.TelemetryFieldKey, - fieldKeysForName []*telemetrytypes.TelemetryFieldKey, - operator qbtypes.FilterOperator, - value any, - sb *sqlbuilder.SelectBuilder, -) ([]string, []string, error) { - return c.conditionsForKeys(ctx, orgID, startNs, endNs, key, fieldKeysForName, false, operator, value, sb) +// isFoldContext reports whether the context is one CandidateKeys would fold the prefix into +// the key name for (span/trace). These behave like a default context that also addresses +// columns and attributes, unlike strict resource/attribute/scope contexts. +func isFoldContext(fc telemetrytypes.FieldContext) bool { + switch fc { + case telemetrytypes.FieldContextSpan, telemetrytypes.FieldContextTrace: + return true + } + return false } -// ConditionForKeys owns key resolution from the full metadata map so the where-clause -// visitor hands over the raw key + map. See qbtypes.ResolvingConditionBuilder. -func (c *conditionBuilder) ConditionForKeys( - ctx context.Context, - orgID valuer.UUID, - startNs uint64, - endNs uint64, - key *telemetrytypes.TelemetryFieldKey, - keys map[string][]*telemetrytypes.TelemetryFieldKey, - options qbtypes.ConditionBuilderOptions, - operator qbtypes.FilterOperator, - value any, - sb *sqlbuilder.SelectBuilder, -) ([]string, []string, error) { - return c.conditionsForKeys(ctx, orgID, startNs, endNs, key, querybuilder.MatchingFieldKeys(key, keys), options.SkipResourceFilter, operator, value, sb) +// candidateLookupKeys returns the metadata map only for fold-contexts, where CandidateKeys +// would otherwise fold the prefix into the key name. Handing it the map lets a same-named +// key under another context resolve first (as ColumnExpressionFor does). Strict contexts +// (resource/attribute/scope) get nil so their explicit context is always honored. +func candidateLookupKeys(key *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) map[string][]*telemetrytypes.TelemetryFieldKey { + if isFoldContext(key.FieldContext) { + return fieldKeys + } + return nil } -// conditionsForKeys resolves matches to the key(s) to filter on (ResolveKeys, else -// synthesized keys with a warning) and builds one condition per resolved key. -func (c *conditionBuilder) conditionsForKeys( +// ConditionFor resolves the referenced key to the key(s) to filter on (ResolveKeys, else +// synthesized keys with a warning) and builds one condition per resolved key. fieldKeys is +// the full metadata map; the builder owns key resolution. +func (c *conditionBuilder) ConditionFor( ctx context.Context, orgID valuer.UUID, startNs uint64, endNs uint64, key *telemetrytypes.TelemetryFieldKey, - matches []*telemetrytypes.TelemetryFieldKey, - skipResourceFilter bool, + fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, + options qbtypes.ConditionBuilderOptions, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder, @@ -312,16 +209,46 @@ func (c *conditionBuilder) conditionsForKeys( return nil, nil, err } + matches := querybuilder.MatchingFieldKeys(key, fieldKeys) + skipResourceFilter := options.SkipResourceFilter + keys, warning := querybuilder.ResolveKeys(key, matches) var warnings []string if warning != "" { warnings = append(warnings, warning) } + // A bare key that names a real column filters on the column too — first. When metadata + // only knows the name under other contexts, prepend the column and keep metadata matches + // only where their type is consistent with it (a corrupt entry can't degrade the column). + if key.FieldContext == telemetrytypes.FieldContextUnspecified && len(keys) > 0 { + hasColumn := false + for _, k := range keys { + if k.FieldContext == telemetrytypes.FieldContextSpan { + hasColumn = true + break + } + } + if !hasColumn { + probe := telemetrytypes.NewTelemetryFieldKey(key.Name, telemetrytypes.FieldContextSpan, key.FieldDataType) + if cols, colErr := c.fm.ColumnFor(ctx, orgID, startNs, endNs, probe); colErr == nil && len(cols) > 0 { + combined := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys)+1) + combined = append(combined, probe) + for _, k := range keys { + if columnMatchesDataType(cols[0], k.FieldDataType) { + combined = append(combined, k) + } + } + keys = combined + } + } + } + synthesized := false if len(keys) == 0 { - // Synthesize key(s) to query anyway; a mapper without synthesis returns nil and - // the unknown key stays a hard "not found" error. - keys = c.fm.CandidateKeys(ctx, orgID, key, value, nil) + // Not in metadata. CandidateKeys resolves it: fold contexts (span/trace) get the + // metadata map so it can honor a real column, correct to a stripped-name metadata + // match, or synthesize; strict contexts pass nil and keep their synthesize path. + keys = c.fm.CandidateKeys(ctx, orgID, key, value, candidateLookupKeys(key, fieldKeys)) if len(keys) == 0 { return nil, warnings, querybuilder.NewKeyNotFoundError(key.Name) } diff --git a/pkg/telemetrytraces/condition_builder_test.go b/pkg/telemetrytraces/condition_builder_test.go index 0cbb73ca09e..d018b7b15c5 100644 --- a/pkg/telemetrytraces/condition_builder_test.go +++ b/pkg/telemetrytraces/condition_builder_test.go @@ -47,8 +47,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorGreaterThan, value: float64(100), - expectedSQL: "(toFloat64(attributes_number['request.duration']) > ? AND mapContains(attributes_number, 'request.duration') = ?)", - expectedArgs: []any{float64(100), true}, + expectedSQL: "(toFloat64(attributes_number['request.duration']) > ? AND mapContains(attributes_number, 'request.duration'))", + expectedArgs: []any{float64(100)}, expectedError: nil, }, { @@ -60,8 +60,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorLessThan, value: float64(1024), - expectedSQL: "(toFloat64(attributes_number['request.size']) < ? AND mapContains(attributes_number, 'request.size') = ?)", - expectedArgs: []any{float64(1024), true}, + expectedSQL: "(toFloat64(attributes_number['request.size']) < ? AND mapContains(attributes_number, 'request.size'))", + expectedArgs: []any{float64(1024)}, expectedError: nil, }, { @@ -97,8 +97,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorILike, value: "%admin%", - expectedSQL: "(LOWER(attributes_string['user.id']) LIKE LOWER(?) AND mapContains(attributes_string, 'user.id') = ?)", - expectedArgs: []any{"%admin%", true}, + expectedSQL: "(LOWER(attributes_string['user.id']) LIKE LOWER(?) AND mapContains(attributes_string, 'user.id'))", + expectedArgs: []any{"%admin%"}, expectedError: nil, }, { @@ -124,7 +124,7 @@ func TestConditionFor(t *testing.T) { operator: qbtypes.FilterOperatorContains, value: 521509198310, expectedSQL: "LOWER(attributes_string['user.id']) LIKE LOWER(?)", - expectedArgs: []any{"%521509198310%", true}, + expectedArgs: []any{"%521509198310%"}, expectedError: nil, }, { @@ -195,7 +195,7 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorExists, value: nil, - expectedSQL: "mapContains(attributes_string, 'user.id') = ?", + expectedSQL: "mapContains(attributes_string, 'user.id')", expectedError: nil, }, { @@ -207,7 +207,7 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorNotExists, value: nil, - expectedSQL: "mapContains(attributes_string, 'user.id') <> ?", + expectedSQL: "NOT mapContains(attributes_string, 'user.id')", expectedError: nil, }, { @@ -245,8 +245,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorContains, value: "admin", - expectedSQL: "(LOWER(attributes_string['user.id']) LIKE LOWER(?) AND mapContains(attributes_string, 'user.id') = ?)", - expectedArgs: []any{"%admin%", true}, + expectedSQL: "(LOWER(attributes_string['user.id']) LIKE LOWER(?) AND mapContains(attributes_string, 'user.id'))", + expectedArgs: []any{"%admin%"}, expectedError: nil, }, { @@ -258,8 +258,8 @@ func TestConditionFor(t *testing.T) { }, operator: qbtypes.FilterOperatorIn, value: []any{"admin", "user"}, - expectedSQL: "((attributes_string['user.id'] = ? OR attributes_string['user.id'] = ?) AND mapContains(attributes_string, 'user.id') = ?)", - expectedArgs: []any{"admin", "user", true}, + expectedSQL: "((attributes_string['user.id'] = ? OR attributes_string['user.id'] = ?) AND mapContains(attributes_string, 'user.id'))", + expectedArgs: []any{"admin", "user"}, expectedError: nil, }, { @@ -294,7 +294,7 @@ func TestConditionFor(t *testing.T) { for _, tc := range testCases { sb := sqlbuilder.NewSelectBuilder() t.Run(tc.name, func(t *testing.T) { - cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, 1761437108000000000, 1761458708000000000, &tc.key, []*telemetrytypes.TelemetryFieldKey{&tc.key}, tc.operator, tc.value, sb) + cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, 1761437108000000000, 1761458708000000000, &tc.key, map[string][]*telemetrytypes.TelemetryFieldKey{tc.key.Name: {&tc.key}}, qbtypes.ConditionBuilderOptions{}, tc.operator, tc.value, sb) sb.Where(cond...) if tc.expectedError != nil { @@ -332,7 +332,7 @@ func TestConditionForResourceWithEvolution(t *testing.T) { operator: qbtypes.FilterOperatorExists, tsStart: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()), tsEnd: uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()), - expectedSQL: "WHERE resource.`service.name`::String IS NOT NULL", + expectedSQL: "WHERE resource.`service.name` IS NOT NULL", }, { name: "NotExists - window after release - JSON only", @@ -345,7 +345,7 @@ func TestConditionForResourceWithEvolution(t *testing.T) { operator: qbtypes.FilterOperatorNotExists, tsStart: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()), tsEnd: uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()), - expectedSQL: "WHERE resource.`service.name`::String IS NULL", + expectedSQL: "WHERE resource.`service.name` IS NULL", }, { name: "Exists - window before release - map only", @@ -358,7 +358,7 @@ func TestConditionForResourceWithEvolution(t *testing.T) { operator: qbtypes.FilterOperatorExists, tsStart: uint64(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC).UnixNano()), tsEnd: uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()), - expectedSQL: "WHERE mapContains(resources_string, 'service.name') = ?", + expectedSQL: "WHERE mapContains(resources_string, 'service.name')", }, { name: "Exists - window straddles release - multiIf null check", @@ -381,7 +381,7 @@ func TestConditionForResourceWithEvolution(t *testing.T) { for _, tc := range testCases { sb := sqlbuilder.NewSelectBuilder() t.Run(tc.name, func(t *testing.T) { - cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, tc.tsStart, tc.tsEnd, &tc.key, []*telemetrytypes.TelemetryFieldKey{&tc.key}, tc.operator, nil, sb) + cond, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, tc.tsStart, tc.tsEnd, &tc.key, map[string][]*telemetrytypes.TelemetryFieldKey{tc.key.Name: {&tc.key}}, qbtypes.ConditionBuilderOptions{}, tc.operator, nil, sb) require.NoError(t, err) sb.Where(cond...) sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) @@ -399,12 +399,12 @@ func TestConditionForSynthesizedKeys(t *testing.T) { cb := NewConditionBuilder(fm) // no metadata matches -> the builder must synthesize from user input - var noMatches []*telemetrytypes.TelemetryFieldKey + var noMatches map[string][]*telemetrytypes.TelemetryFieldKey t.Run("bare key with string operand -> attribute string", func(t *testing.T) { sb := sqlbuilder.NewSelectBuilder() key := telemetrytypes.TelemetryFieldKey{Name: "error.type"} - conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.FilterOperatorEqual, "timeout", sb) + conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "timeout", sb) assert.NoError(t, err) assert.NotEmpty(t, warnings, "a not-found warning should be emitted") sb.Where(conds...) @@ -416,7 +416,7 @@ func TestConditionForSynthesizedKeys(t *testing.T) { t.Run("bare key with number operand -> attribute number", func(t *testing.T) { sb := sqlbuilder.NewSelectBuilder() key := telemetrytypes.TelemetryFieldKey{Name: "http.status"} - conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.FilterOperatorGreaterThan, float64(5), sb) + conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorGreaterThan, float64(5), sb) assert.NoError(t, err) assert.NotEmpty(t, warnings) sb.Where(conds...) @@ -427,7 +427,7 @@ func TestConditionForSynthesizedKeys(t *testing.T) { t.Run("bare key with bool operand -> attribute bool", func(t *testing.T) { sb := sqlbuilder.NewSelectBuilder() key := telemetrytypes.TelemetryFieldKey{Name: "sampled"} - conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.FilterOperatorEqual, true, sb) + conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, true, sb) assert.NoError(t, err) sb.Where(conds...) sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) @@ -437,7 +437,7 @@ func TestConditionForSynthesizedKeys(t *testing.T) { t.Run("exists with no operand fans out across type variants", func(t *testing.T) { sb := sqlbuilder.NewSelectBuilder() key := telemetrytypes.TelemetryFieldKey{Name: "exception.type"} - conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.FilterOperatorExists, nil, sb) + conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorExists, nil, sb) assert.NoError(t, err) assert.NotEmpty(t, warnings) assert.Len(t, conds, 3, "exists should fan out to string/number/bool") @@ -451,7 +451,7 @@ func TestConditionForSynthesizedKeys(t *testing.T) { t.Run("qualified data type honored without fanout", func(t *testing.T) { sb := sqlbuilder.NewSelectBuilder() key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldDataType: telemetrytypes.FieldDataTypeString} - conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.FilterOperatorEqual, "v", sb) + conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb) assert.NoError(t, err) assert.Len(t, conds, 1) sb.Where(conds...) @@ -459,28 +459,96 @@ func TestConditionForSynthesizedKeys(t *testing.T) { assert.Contains(t, sql, "attributes_string['custom.key']") }) - t.Run("qualified resource context honored", func(t *testing.T) { + t.Run("bare intrinsic column resolves to the column, not synthesized attributes", func(t *testing.T) { + sb := sqlbuilder.NewSelectBuilder() + key := telemetrytypes.TelemetryFieldKey{Name: "duration_nano"} + conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorGreaterThan, float64(100), sb) + require.NoError(t, err) + require.Len(t, conds, 1) + sb.Where(conds...) + sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + assert.Contains(t, sql, "duration_nano > ?") + assert.NotContains(t, sql, "attributes_string") + assert.NotContains(t, sql, "attributes_number") + }) + + t.Run("bare deprecated alias resolves to the calculated column", func(t *testing.T) { + sb := sqlbuilder.NewSelectBuilder() + key := telemetrytypes.TelemetryFieldKey{Name: "httpMethod"} + conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "GET", sb) + require.NoError(t, err) + require.Len(t, conds, 1) + sb.Where(conds...) + sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + assert.Contains(t, sql, "http_method = ?") + assert.NotContains(t, sql, "attributes_string") + }) + + t.Run("span-context key that is a real column stays the column", func(t *testing.T) { + sb := sqlbuilder.NewSelectBuilder() + key := telemetrytypes.TelemetryFieldKey{Name: "duration_nano", FieldContext: telemetrytypes.FieldContextSpan} + conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorGreaterThan, float64(100), sb) + require.NoError(t, err) + require.Len(t, conds, 1) + sb.Where(conds...) + sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + assert.Contains(t, sql, "duration_nano > ?") + assert.NotContains(t, sql, "attributes_") + }) + + t.Run("span-context key not a column is corrected to a stripped-name metadata match", func(t *testing.T) { + sb := sqlbuilder.NewSelectBuilder() + key := telemetrytypes.TelemetryFieldKey{Name: "http.method", FieldContext: telemetrytypes.FieldContextSpan} + keysMap := map[string][]*telemetrytypes.TelemetryFieldKey{ + "http.method": {{Name: "http.method", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString}}, + } + conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, keysMap, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "GET", sb) + require.NoError(t, err) + assert.NotEmpty(t, warnings) + require.Len(t, conds, 1) + sb.Where(conds...) + sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + assert.Contains(t, sql, "attributes_string['http.method']") + assert.NotContains(t, sql, "attributes_string['span.http.method']") + }) + + t.Run("span-context key absent from metadata synthesizes the stripped name", func(t *testing.T) { + sb := sqlbuilder.NewSelectBuilder() + key := telemetrytypes.TelemetryFieldKey{Name: "custom.attr", FieldContext: telemetrytypes.FieldContextSpan} + conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "v", sb) + require.NoError(t, err) + assert.NotEmpty(t, warnings) + require.Len(t, conds, 1) + sb.Where(conds...) + sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + assert.Contains(t, sql, "attributes_string['custom.attr']") + assert.NotContains(t, sql, "span.custom.attr") + }) + + t.Run("qualified resource context honored with literal spelling second", func(t *testing.T) { sb := sqlbuilder.NewSelectBuilder() key := telemetrytypes.TelemetryFieldKey{Name: "k8s.cluster.name", FieldContext: telemetrytypes.FieldContextResource} - conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.FilterOperatorEqual, "prod", sb) + conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, "prod", sb) assert.NoError(t, err) - assert.Len(t, conds, 1) + assert.Len(t, conds, 2, "stripped interpretation first, literal `resource.` spelling second") sb.Where(conds...) sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) - assert.Contains(t, sql, "resources_string") + assert.Contains(t, sql, "resources_string['k8s.cluster.name']") + assert.Contains(t, sql, "resources_string['resource.k8s.cluster.name']") }) t.Run("synthesized resource key survives skip-resource-filter", func(t *testing.T) { // the resource sub-query never covered a key absent from metadata sb := sqlbuilder.NewSelectBuilder() key := telemetrytypes.TelemetryFieldKey{Name: "deployment.environment", FieldContext: telemetrytypes.FieldContextResource} - conds, warnings, err := cb.ConditionForKeys(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{SkipResourceFilter: true}, qbtypes.FilterOperatorEqual, "prod", sb) + conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, nil, qbtypes.ConditionBuilderOptions{SkipResourceFilter: true}, qbtypes.FilterOperatorEqual, "prod", sb) assert.NoError(t, err) assert.NotEmpty(t, warnings) - assert.Len(t, conds, 1, "the synthesized resource condition must not be dropped") + assert.Len(t, conds, 2, "the synthesized resource conditions must not be dropped") sb.Where(conds...) sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) assert.Contains(t, sql, "resources_string['deployment.environment']") + assert.Contains(t, sql, "resources_string['resource.deployment.environment']") }) t.Run("metadata-backed resource key still dropped under skip-resource-filter", func(t *testing.T) { @@ -489,7 +557,7 @@ func TestConditionForSynthesizedKeys(t *testing.T) { keysMap := map[string][]*telemetrytypes.TelemetryFieldKey{ "service.name": {{Name: "service.name", FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString}}, } - conds, _, err := cb.ConditionForKeys(ctx, valuer.UUID{}, 0, 0, &key, keysMap, qbtypes.ConditionBuilderOptions{SkipResourceFilter: true}, qbtypes.FilterOperatorEqual, "redis", sb) + conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, keysMap, qbtypes.ConditionBuilderOptions{SkipResourceFilter: true}, qbtypes.FilterOperatorEqual, "redis", sb) assert.NoError(t, err) assert.Empty(t, conds, "the resource CTE covers metadata-backed keys") }) @@ -497,7 +565,7 @@ func TestConditionForSynthesizedKeys(t *testing.T) { t.Run("synthesized resource key coerces numeric operand", func(t *testing.T) { sb := sqlbuilder.NewSelectBuilder() key := telemetrytypes.TelemetryFieldKey{Name: "replica.count", FieldContext: telemetrytypes.FieldContextResource} - conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.FilterOperatorEqual, float64(3), sb) + conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorEqual, float64(3), sb) assert.NoError(t, err) sb.Where(conds...) sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) @@ -508,7 +576,7 @@ func TestConditionForSynthesizedKeys(t *testing.T) { t.Run("negative operator builds without exists guard (matches-everything semantics)", func(t *testing.T) { sb := sqlbuilder.NewSelectBuilder() key := telemetrytypes.TelemetryFieldKey{Name: "error.type"} - conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.FilterOperatorNotEqual, "fatal", sb) + conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{},qbtypes.FilterOperatorNotEqual, "fatal", sb) assert.NoError(t, err) sb.Where(conds...) sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse) diff --git a/pkg/telemetrytraces/field_mapper.go b/pkg/telemetrytraces/field_mapper.go index d2cb2bbd2be..40705b6da8c 100644 --- a/pkg/telemetrytraces/field_mapper.go +++ b/pkg/telemetrytraces/field_mapper.go @@ -174,7 +174,7 @@ func (m *defaultFieldMapper) getColumn( ) ([]*schema.Column, error) { switch key.FieldContext { case telemetrytypes.FieldContextResource: - return []*schema.Column{indexV3Columns["resources_string"], indexV3Columns["resource"]}, nil + return []*schema.Column{indexV3Columns["resource"], indexV3Columns["resources_string"]}, nil case telemetrytypes.FieldContextScope: return []*schema.Column{}, qbtypes.ErrColumnNotFound case telemetrytypes.FieldContextAttribute: @@ -188,26 +188,12 @@ func (m *defaultFieldMapper) getColumn( case telemetrytypes.FieldDataTypeBool: return []*schema.Column{indexV3Columns["attributes_bool"]}, nil } - case telemetrytypes.FieldContextSpan, telemetrytypes.FieldContextUnspecified: - /* - TODO: This is incorrect, we cannot assume all unspecified context fields are span context. - User could be referring to attributes, but we cannot fix this until we fix where_clause vistior - https://github.com/SigNoz/signoz/pull/10102 - */ + case telemetrytypes.FieldContextSpan: // Check if this is a span scope field if strings.ToLower(key.Name) == SpanSearchScopeRoot || strings.ToLower(key.Name) == SpanSearchScopeEntryPoint { // The actual SQL will be generated in the condition builder return []*schema.Column{{Name: key.Name, Type: schema.ColumnTypeBool}}, nil } - - // TODO(srikanthccv): remove this when it's safe to remove - // issue with CH aliasing - - /* - NOTE: There are fields which are deprecated for only to not show up as user suggestion and is possible that - they don't have a mapping in oldToNew map. So we need to look up in indexV3Columns directly for those fields. - For example: kind, timestamp etc. - */ if _, ok := CalculatedFieldsDeprecated[key.Name]; ok { // Check if we have a mapping for the deprecated calculated field if col, ok := indexV3Columns[oldToNew[key.Name]]; ok { @@ -287,16 +273,9 @@ func (m *defaultFieldMapper) resolveColumnExprs( return nil, nil, nil, err } - var newColumns []*schema.Column - var evolutionsEntries []*telemetrytypes.EvolutionEntry - if len(key.Evolutions) > 0 { - // we will use the corresponding column and its evolution entry for the query - newColumns, evolutionsEntries, err = qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs) - if err != nil { - return nil, nil, nil, err - } - } else { - newColumns = columns + newColumns, evolutionsEntries, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs) + if err != nil { + return nil, nil, nil, err } for i, column := range newColumns { @@ -343,7 +322,7 @@ func (m *defaultFieldMapper) resolveColumnExprs( // a key could have been materialized, if so return the materialized column name if key.Materialized { exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key)) - existExprs = append(existExprs, fmt.Sprintf("%s==true", telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key))) + existExprs = append(existExprs, telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)) } else { exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name)) existExprs = append(existExprs, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name)) @@ -365,6 +344,7 @@ func (m *defaultFieldMapper) ColumnExpressionFor( orgID valuer.UUID, startNs, endNs uint64, field *telemetrytypes.TelemetryFieldKey, + requiredDataType telemetrytypes.FieldDataType, keys map[string][]*telemetrytypes.TelemetryFieldKey, ) (string, error) { @@ -374,23 +354,40 @@ func (m *defaultFieldMapper) ColumnExpressionFor( case err == nil: candidates = []*telemetrytypes.TelemetryFieldKey{field} case errors.Is(err, qbtypes.ErrColumnNotFound): - // is it a static field? attach the column directly. - if _, ok := indexV3Columns[field.Name]; ok { - field.FieldContext = telemetrytypes.FieldContextSpan - candidates = []*telemetrytypes.TelemetryFieldKey{field} - } else { - // metadata matches (incl. the collision set) or synthesized type-variant keys. - candidates = m.CandidateKeys(ctx, orgID, field, nil, keys) - if len(candidates) == 0 { - return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field `%s` not found", field.Name).WithSuggestions(errors.NewSuggestionsOnLevenshteinDistance(field.Name, errors.NounKeys, maps.Keys(keys))...) - } + // column (when the bare name is one) plus metadata matches, else synthesized + // type-variant keys. + candidates = m.CandidateKeys(ctx, orgID, field, nil, keys) + if len(candidates) == 0 { + return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field `%s` not found", field.Name).WithSuggestions(errors.NewSuggestionsOnLevenshteinDistance(field.Name, errors.NounKeys, maps.Keys(keys))...) } default: return "", err } - // Single candidate: exists-guard wrap so an absent key yields NULL; multi-column and - // physical columns stay bare. + // Group-by/order (String) and aggregation (String/Float64): every candidate is + // exists-guarded and coerced to requiredDataType, in a single multiIf. Raw select + // (Unspecified) keeps the lighter native shape below. + if requiredDataType != telemetrytypes.FieldDataTypeUnspecified { + var dummyValue any = "" + if requiredDataType == telemetrytypes.FieldDataTypeFloat64 { + dummyValue = 0.0 + } + stmts := make([]string, 0, len(candidates)*2) + for _, key := range candidates { + value, err := m.FieldFor(ctx, orgID, startNs, endNs, key) + if err != nil { + return "", err + } + guard, err := m.existsExpressionFor(ctx, orgID, startNs, endNs, key, true) + if err != nil { + return "", err + } + coerced, _ := querybuilder.DataTypeCollisionHandledFieldName(key, dummyValue, value, qbtypes.FilterOperatorUnknown) + stmts = append(stmts, guard, coerced) + } + return fmt.Sprintf("multiIf(%s, NULL)", strings.Join(stmts, ", ")), nil + } + if len(candidates) == 1 { value, err := m.FieldFor(ctx, orgID, startNs, endNs, candidates[0]) if err != nil { @@ -398,7 +395,11 @@ func (m *defaultFieldMapper) ColumnExpressionFor( } exprs, existExprs, _, _ := m.resolveColumnExprs(ctx, startNs, endNs, candidates[0]) if len(exprs) == 1 && len(existExprs) == 1 { - return fmt.Sprintf("multiIf(%s, %s, NULL)", existExprs[0], value), nil + guard, err := m.existsExpressionFor(ctx, orgID, startNs, endNs, candidates[0], true) + if err != nil { + return "", err + } + return fmt.Sprintf("multiIf(%s, %s, NULL)", guard, value), nil } return value, nil } @@ -411,41 +412,108 @@ func (m *defaultFieldMapper) ColumnExpressionFor( if err != nil { return "", err } - args = append(args, fmt.Sprintf("%s, toString(%s)", m.existsGuardFor(ctx, startNs, endNs, key, value), value)) + guard, err := m.existsExpressionFor(ctx, orgID, startNs, endNs, key, true) + if err != nil { + return "", err + } + args = append(args, fmt.Sprintf("%s, toString(%s)", guard, value)) } return fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", ")), nil } -// membershipGuard returns the key's existence guard (mapContains / IS NOT NULL / _exists), -// OR-combined across its columns, or "" when the column type has none (physical column). -func (m *defaultFieldMapper) membershipGuard(ctx context.Context, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) string { - _, existExprs, _, err := m.resolveColumnExprs(ctx, startNs, endNs, key) - if err != nil || len(existExprs) == 0 { - return "" +// columnMatchesDataType reports whether a metadata field's data type is consistent with a +// column's ClickHouse type. A bare key's column is only unioned with same-named metadata +// keys that could be the same field; a string attribute named `timestamp` is corrupt +// metadata against the DateTime column and must not degrade the intrinsic. +func columnMatchesDataType(col *schema.Column, dt telemetrytypes.FieldDataType) bool { + if dt == telemetrytypes.FieldDataTypeUnspecified { + return true } - if len(existExprs) == 1 { - return existExprs[0] + switch col.Type.GetType() { + case schema.ColumnTypeEnumBool: + return dt == telemetrytypes.FieldDataTypeBool + case schema.ColumnTypeEnumUInt64, schema.ColumnTypeEnumUInt32, + schema.ColumnTypeEnumInt8, schema.ColumnTypeEnumInt16, + schema.ColumnTypeEnumFloat64, schema.ColumnTypeEnumDateTime64: + return dt == telemetrytypes.FieldDataTypeInt64 || + dt == telemetrytypes.FieldDataTypeFloat64 || + dt == telemetrytypes.FieldDataTypeNumber + default: // String, FixedString, LowCardinality(String), … + return dt == telemetrytypes.FieldDataTypeString } - return "(" + strings.Join(existExprs, " OR ") + ")" } -// existsGuardFor is membershipGuard with a non-empty fallback, for use as a multiIf branch -// condition where a bare guard-less column still needs a discriminator. -func (m *defaultFieldMapper) existsGuardFor(ctx context.Context, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey, value string) string { - if g := m.membershipGuard(ctx, startNs, endNs, key); g != "" { - return g +// CandidateKeys resolves a referenced field to the key(s) to query when it isn't already a +// resolved column. A bare key unions the real column with type-consistent same-named +// metadata keys; a forgiving span/trace context is honored as-is, correcting to a +// metadata match for the stripped name when present; strict attribute/resource contexts +// are honored as-is. Falls back to synthesized type-variant keys. +func (m *defaultFieldMapper) CandidateKeys(ctx context.Context, _ valuer.UUID, field *telemetrytypes.TelemetryFieldKey, value any, keys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey { + // A real column is considered before metadata for bare and forgiving contexts, so a + // same-named corrupt attribute can't shadow the intrinsic/calculated column. + switch field.FieldContext { + case telemetrytypes.FieldContextUnspecified: + // bare key: the column comes first, alongside same-named metadata keys under other + // contexts whose type is consistent with the column (corrupt entries dropped). + probe := telemetrytypes.NewTelemetryFieldKey(field.Name, telemetrytypes.FieldContextSpan, field.FieldDataType) + if cols, err := m.getColumn(ctx, 0, 0, probe); err == nil && len(cols) > 0 { + candidates := []*telemetrytypes.TelemetryFieldKey{probe} + for _, match := range keys[field.Name] { + if match.FieldContext != telemetrytypes.FieldContextSpan && + columnMatchesDataType(cols[0], match.FieldDataType) { + candidates = append(candidates, match) + } + } + return candidates + } + case telemetrytypes.FieldContextSpan, telemetrytypes.FieldContextTrace: + // forgiving context honored as-is: a real column wins (span.duration_nano -> col). + if _, err := m.getColumn(ctx, 0, 0, field); err == nil { + return []*telemetrytypes.TelemetryFieldKey{telemetrytypes.NewTelemetryFieldKey(field.Name, field.FieldContext, field.FieldDataType)} + } } - return fmt.Sprintf("toString(%s) != ''", value) -} -// CandidateKeys prefers metadata matches for the name (or `{context}.{name}`); when none -// match it synthesizes type-variant keys across the span attribute maps. -func (m *defaultFieldMapper) CandidateKeys(_ context.Context, _ valuer.UUID, field *telemetrytypes.TelemetryFieldKey, value any, keys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey { + // Metadata match by name, then the literal `{context}.{name}` spelling (a context can be + // a legitimate prefix in user data, e.g. `metric.max_count`). For a forgiving context + // this is the correction step (span.http.method -> attribute http.method). if matches := keys[field.Name]; len(matches) > 0 { return matches } if matches := keys[fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), field.Name)]; len(matches) > 0 { return matches } - return querybuilder.SynthesizeKeys(field, value) + + // No metadata: synthesize per context. + switch field.FieldContext { + case telemetrytypes.FieldContextUnspecified: + return querybuilder.SynthesizeKeys(field, value) + case telemetrytypes.FieldContextSpan, telemetrytypes.FieldContextTrace: + // honored as-is: the stripped name lives in the attribute maps + stripped := telemetrytypes.NewTelemetryFieldKey(field.Name, telemetrytypes.FieldContextUnspecified, field.FieldDataType) + return querybuilder.SynthesizeKeys(stripped, value) + case telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource: + // strict context honored as-is: stripped interpretation first, literal spelling second + literal := telemetrytypes.NewTelemetryFieldKey(field.FieldContext.StringValue()+"."+field.Name, field.FieldContext, field.FieldDataType) + return append(querybuilder.SynthesizeKeys(field, value), querybuilder.SynthesizeKeys(literal, value)...) + } + // contexts that don't exist on spans (log, body, scope, …) have nothing to synthesize + return nil +} + +func (m *defaultFieldMapper) existsExpressionFor( + ctx context.Context, + orgID valuer.UUID, + tsStart, tsEnd uint64, + key *telemetrytypes.TelemetryFieldKey, + exists bool, +) (string, error) { + columns, err := m.getColumn(ctx, tsStart, tsEnd, key) + if err != nil { + return "", err + } + fieldExpression, err := m.FieldFor(ctx, orgID, tsStart, tsEnd, key) + if err != nil { + return "", err + } + return querybuilder.ExistsExpression(columns, key, tsStart, tsEnd, fieldExpression, exists) } diff --git a/pkg/telemetrytraces/field_mapper_test.go b/pkg/telemetrytraces/field_mapper_test.go index 450b8a5f521..8f278d99b75 100644 --- a/pkg/telemetrytraces/field_mapper_test.go +++ b/pkg/telemetrytraces/field_mapper_test.go @@ -80,7 +80,7 @@ func TestGetFieldKeyName(t *testing.T) { Materialized: true, Evolutions: mockEvolution, }, - expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`==true, `resource_string_deployment$$environment`, NULL)", + expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`, `resource_string_deployment$$environment`, NULL)", expectedError: nil, }, { @@ -189,7 +189,7 @@ func TestFieldForResourceWithEvolution(t *testing.T) { }, tsStart: uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()), tsEnd: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()), - expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`==true, `resource_string_deployment$$environment`, NULL)", + expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`, `resource_string_deployment$$environment`, NULL)", }, } diff --git a/pkg/telemetrytraces/statement_builder.go b/pkg/telemetrytraces/statement_builder.go index 89256195cf6..898270e0f72 100644 --- a/pkg/telemetrytraces/statement_builder.go +++ b/pkg/telemetrytraces/statement_builder.go @@ -296,7 +296,7 @@ func (b *traceQueryStatementBuilder) buildListQuery( } for i, field := range query.SelectFields { - expr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &field, keys) + expr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &field, telemetrytypes.FieldDataTypeUnspecified, keys) if err != nil { return nil, err } @@ -320,7 +320,7 @@ func (b *traceQueryStatementBuilder) buildListQuery( // Add order by for _, orderBy := range query.Order { - expr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &orderBy.Key.TelemetryFieldKey, keys) + expr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &orderBy.Key.TelemetryFieldKey, telemetrytypes.FieldDataTypeUnspecified, keys) if err != nil { return nil, err } @@ -504,7 +504,7 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery( // Keep original column expressions so we can build the tuple fieldNames := make([]string, 0, len(query.GroupBy)) for i, gb := range query.GroupBy { - expr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &gb.TelemetryFieldKey, keys) + expr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &gb.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return nil, err } @@ -660,7 +660,7 @@ func (b *traceQueryStatementBuilder) buildScalarQuery( fieldNames := make([]string, 0, len(query.GroupBy)) for i, gb := range query.GroupBy { - expr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &gb.TelemetryFieldKey, keys) + expr, err := b.fm.ColumnExpressionFor(ctx, orgID, start, end, &gb.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return nil, err } diff --git a/pkg/telemetrytraces/stmt_builder_test.go b/pkg/telemetrytraces/stmt_builder_test.go index e55740e0b59..a5231da4415 100644 --- a/pkg/telemetrytraces/stmt_builder_test.go +++ b/pkg/telemetrytraces/stmt_builder_test.go @@ -69,7 +69,7 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`", Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, @@ -98,8 +98,8 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __limit_cte AS (SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.request.method'] = ? AND mapContains(attributes_string, 'http.request.method') = ?)) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.request.method'] = ? AND mapContains(attributes_string, 'http.request.method') = ?)) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`", - Args: []any{"redis-manual", "GET", true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "redis-manual", "GET", true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, + Query: "WITH __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.request.method'] = ? AND mapContains(attributes_string, 'http.request.method'))) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.request.method'] = ? AND mapContains(attributes_string, 'http.request.method'))) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`", + Args: []any{"redis-manual", "GET", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "redis-manual", "GET", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, }, @@ -127,8 +127,8 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __limit_cte AS (SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE ((match(`attribute_string_materialized$$key$$name`, ?) AND `attribute_string_materialized$$key$$name_exists` = ?) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE ((match(`attribute_string_materialized$$key$$name`, ?) AND `attribute_string_materialized$$key$$name_exists` = ?) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`", - Args: []any{"redis-manual", true, "redis-manual", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "redis-manual", true, "redis-manual", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, + Query: "WITH __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE ((match(`attribute_string_materialized$$key$$name`, ?) AND `attribute_string_materialized$$key$$name_exists`) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE ((match(`attribute_string_materialized$$key$$name`, ?) AND `attribute_string_materialized$$key$$name_exists`) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`", + Args: []any{"redis-manual", "redis-manual", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "redis-manual", "redis-manual", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, }, @@ -158,7 +158,7 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(attribute_string_http$$route) AS `__GROUP_BY_KEY_0_httpRoute`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_httpRoute` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(attribute_string_http$$route) AS `__GROUP_BY_KEY_0_httpRoute`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_httpRoute`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_httpRoute` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_httpRoute`", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(attribute_string_http$$route <> '', attribute_string_http$$route, NULL)) AS `__GROUP_BY_KEY_0_httpRoute`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_httpRoute` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(attribute_string_http$$route <> '', attribute_string_http$$route, NULL)) AS `__GROUP_BY_KEY_0_httpRoute`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_httpRoute`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_httpRoute` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_httpRoute`", Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, @@ -196,8 +196,8 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __limit_cte AS (SELECT toString(attribute_string_http$$route) AS `__GROUP_BY_KEY_0_httpRoute`, toString(http_method) AS `__GROUP_BY_KEY_1_httpMethod`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE ((resource_string_service$$name = ? AND resource_string_service$$name <> ?) AND http_method <> ? AND kind_string = ?) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_httpRoute`, `__GROUP_BY_KEY_1_httpMethod` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(attribute_string_http$$route) AS `__GROUP_BY_KEY_0_httpRoute`, toString(http_method) AS `__GROUP_BY_KEY_1_httpMethod`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE ((resource_string_service$$name = ? AND resource_string_service$$name <> ?) AND http_method <> ? AND kind_string = ?) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_httpRoute`, `__GROUP_BY_KEY_1_httpMethod`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_httpRoute`, `__GROUP_BY_KEY_1_httpMethod` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_httpRoute`, `__GROUP_BY_KEY_1_httpMethod`", - Args: []any{"redis-manual", "", "", "Server", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "redis-manual", "", "", "Server", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, + Query: "WITH __limit_cte AS (SELECT toString(multiIf(attribute_string_http$$route <> '', attribute_string_http$$route, NULL)) AS `__GROUP_BY_KEY_0_httpRoute`, toString(multiIf(http_method <> '', http_method, NULL)) AS `__GROUP_BY_KEY_1_httpMethod`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE ((resource_string_service$$name = ? AND resource_string_service$$name <> '') AND http_method <> '' AND kind_string = ?) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_httpRoute`, `__GROUP_BY_KEY_1_httpMethod` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(attribute_string_http$$route <> '', attribute_string_http$$route, NULL)) AS `__GROUP_BY_KEY_0_httpRoute`, toString(multiIf(http_method <> '', http_method, NULL)) AS `__GROUP_BY_KEY_1_httpMethod`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE ((resource_string_service$$name = ? AND resource_string_service$$name <> '') AND http_method <> '' AND kind_string = ?) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_httpRoute`, `__GROUP_BY_KEY_1_httpMethod`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_httpRoute`, `__GROUP_BY_KEY_1_httpMethod` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_httpRoute`, `__GROUP_BY_KEY_1_httpMethod`", + Args: []any{"redis-manual", "Server", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "redis-manual", "Server", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, }, @@ -237,8 +237,8 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, sum(multiIf(mapContains(attributes_number, 'metric.max_count') = ?, toFloat64(attributes_number['metric.max_count']), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 desc LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, sum(multiIf(mapContains(attributes_number, 'metric.max_count') = ?, toFloat64(attributes_number['metric.max_count']), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name` ORDER BY ts desc", - Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, sum(multiIf(mapContains(attributes_number, 'metric.max_count'), toFloat64(attributes_number['metric.max_count']), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 desc LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, sum(multiIf(mapContains(attributes_number, 'metric.max_count'), toFloat64(attributes_number['metric.max_count']), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name` ORDER BY ts desc", + Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, }, @@ -266,8 +266,8 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, sum(multiIf(`attribute_number_cart$$items_count_exists` = ?, toFloat64(`attribute_number_cart$$items_count`), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, sum(multiIf(`attribute_number_cart$$items_count_exists` = ?, toFloat64(`attribute_number_cart$$items_count`), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`", - Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, sum(multiIf(`attribute_number_cart$$items_count_exists`, toFloat64(`attribute_number_cart$$items_count`), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, sum(multiIf(`attribute_number_cart$$items_count_exists`, toFloat64(`attribute_number_cart$$items_count`), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`", + Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, }, @@ -305,8 +305,8 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, sum(multiIf(`attribute_number_cart$$items_count_exists` = ?, toFloat64(`attribute_number_cart$$items_count`), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY `__GROUP_BY_KEY_0_service.name` desc LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, sum(multiIf(`attribute_number_cart$$items_count_exists` = ?, toFloat64(`attribute_number_cart$$items_count`), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name` ORDER BY `__GROUP_BY_KEY_0_service.name` desc, ts desc", - Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, sum(multiIf(`attribute_number_cart$$items_count_exists`, toFloat64(`attribute_number_cart$$items_count`), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY `__GROUP_BY_KEY_0_service.name` desc LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, sum(multiIf(`attribute_number_cart$$items_count_exists`, toFloat64(`attribute_number_cart$$items_count`), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name` ORDER BY `__GROUP_BY_KEY_0_service.name` desc, ts desc", + Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, }, @@ -336,7 +336,7 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(response_status_code) AS `__GROUP_BY_KEY_0_responseStatusCode`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_responseStatusCode` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(response_status_code) AS `__GROUP_BY_KEY_0_responseStatusCode`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_responseStatusCode`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_responseStatusCode` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_responseStatusCode`", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(response_status_code <> '', response_status_code, NULL)) AS `__GROUP_BY_KEY_0_responseStatusCode`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_responseStatusCode` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(response_status_code <> '', response_status_code, NULL)) AS `__GROUP_BY_KEY_0_responseStatusCode`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_responseStatusCode`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_responseStatusCode` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_responseStatusCode`", Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, @@ -367,8 +367,8 @@ func TestStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(response_status_code) AS `__GROUP_BY_KEY_0_responseStatusCode`, quantile(0.90)(multiIf(duration_nano <> ?, duration_nano, NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_responseStatusCode` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(response_status_code) AS `__GROUP_BY_KEY_0_responseStatusCode`, quantile(0.90)(multiIf(duration_nano <> ?, duration_nano, NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_responseStatusCode`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_responseStatusCode` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_responseStatusCode`", - Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), 0, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, 0, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(response_status_code <> '', response_status_code, NULL)) AS `__GROUP_BY_KEY_0_responseStatusCode`, quantile(0.90)(multiIf(duration_nano <> 0, duration_nano, NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_responseStatusCode` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(response_status_code <> '', response_status_code, NULL)) AS `__GROUP_BY_KEY_0_responseStatusCode`, quantile(0.90)(multiIf(duration_nano <> 0, duration_nano, NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_responseStatusCode`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_responseStatusCode` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_responseStatusCode`", + Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, }, @@ -379,7 +379,7 @@ func TestStatementBuilder(t *testing.T) { cb := NewConditionBuilder(fm) mockMetadataStore := telemetrytypestest.NewMockMetadataStore() mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewTraceQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -464,7 +464,7 @@ func TestStatementBuilderListQuery(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp AS `__SELECT_KEY_0_timestamp`, trace_id AS `__SELECT_KEY_1_trace_id`, span_id AS `__SELECT_KEY_2_span_id`, name AS `__SELECT_KEY_3_name`, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) AS `__SELECT_KEY_4_service.name`, duration_nano AS `__SELECT_KEY_5_duration_nano`, multiIf(`attribute_number_cart$$items_count_exists`==true, `attribute_number_cart$$items_count`, NULL) AS `__SELECT_KEY_6_cart.items_count` FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp AS `__SELECT_KEY_0_timestamp`, trace_id AS `__SELECT_KEY_1_trace_id`, span_id AS `__SELECT_KEY_2_span_id`, name AS `__SELECT_KEY_3_name`, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) AS `__SELECT_KEY_4_service.name`, duration_nano AS `__SELECT_KEY_5_duration_nano`, multiIf(`attribute_number_cart$$items_count_exists`, `attribute_number_cart$$items_count`, NULL) AS `__SELECT_KEY_6_cart.items_count` FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, }, expectedErr: nil, @@ -581,7 +581,7 @@ func TestStatementBuilderListQuery(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp AS `__SELECT_KEY_0_timestamp`, trace_id AS `__SELECT_KEY_1_trace_id`, span_id AS `__SELECT_KEY_2_span_id`, name AS `__SELECT_KEY_3_name`, resource_string_service$$name AS `__SELECT_KEY_4_serviceName`, duration_nano AS `__SELECT_KEY_5_durationNano`, http_method AS `__SELECT_KEY_6_httpMethod`, multiIf(`attribute_string_mixed$$materialization$$key_exists`==true, toString(`attribute_string_mixed$$materialization$$key`), (resource.`mixed.materialization.key` IS NOT NULL OR mapContains(resources_string, 'mixed.materialization.key')), toString(multiIf(resource.`mixed.materialization.key` IS NOT NULL, resource.`mixed.materialization.key`::String, mapContains(resources_string, 'mixed.materialization.key'), resources_string['mixed.materialization.key'], NULL)), NULL) AS `__SELECT_KEY_7_mixed.materialization.key` FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp AS `__SELECT_KEY_0_timestamp`, trace_id AS `__SELECT_KEY_1_trace_id`, span_id AS `__SELECT_KEY_2_span_id`, name AS `__SELECT_KEY_3_name`, resource_string_service$$name AS `__SELECT_KEY_4_serviceName`, duration_nano AS `__SELECT_KEY_5_durationNano`, http_method AS `__SELECT_KEY_6_httpMethod`, multiIf(`attribute_string_mixed$$materialization$$key_exists`, toString(`attribute_string_mixed$$materialization$$key`), multiIf(resource.`mixed.materialization.key` IS NOT NULL, resource.`mixed.materialization.key`::String, mapContains(resources_string, 'mixed.materialization.key'), resources_string['mixed.materialization.key'], NULL) IS NOT NULL, toString(multiIf(resource.`mixed.materialization.key` IS NOT NULL, resource.`mixed.materialization.key`::String, mapContains(resources_string, 'mixed.materialization.key'), resources_string['mixed.materialization.key'], NULL)), NULL) AS `__SELECT_KEY_7_mixed.materialization.key` FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, }, expectedErr: nil, @@ -626,7 +626,7 @@ func TestStatementBuilderListQuery(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp AS `__SELECT_KEY_0_timestamp`, trace_id AS `__SELECT_KEY_1_trace_id`, span_id AS `__SELECT_KEY_2_span_id`, name AS `__SELECT_KEY_3_name`, resource_string_service$$name AS `__SELECT_KEY_4_serviceName`, duration_nano AS `__SELECT_KEY_5_durationNano`, http_method AS `__SELECT_KEY_6_httpMethod`, multiIf(`attribute_string_mixed$$materialization$$key_exists`==true, `attribute_string_mixed$$materialization$$key`, NULL) AS `__SELECT_KEY_7_mixed.materialization.key` FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?", + Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp AS `__SELECT_KEY_0_timestamp`, trace_id AS `__SELECT_KEY_1_trace_id`, span_id AS `__SELECT_KEY_2_span_id`, name AS `__SELECT_KEY_3_name`, resource_string_service$$name AS `__SELECT_KEY_4_serviceName`, duration_nano AS `__SELECT_KEY_5_durationNano`, http_method AS `__SELECT_KEY_6_httpMethod`, multiIf(`attribute_string_mixed$$materialization$$key_exists`, `attribute_string_mixed$$materialization$$key`, NULL) AS `__SELECT_KEY_7_mixed.materialization.key` FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?", Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, }, expectedErr: nil, @@ -680,7 +680,7 @@ func TestStatementBuilderListQuery(t *testing.T) { cb := NewConditionBuilder(fm) mockMetadataStore := telemetrytypestest.NewMockMetadataStore() mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewTraceQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -810,7 +810,7 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) { if mockMetadataStore.KeysMap == nil { mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime) } - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewTraceQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -853,7 +853,7 @@ func TestStatementBuilderGroupByResourceEvolution(t *testing.T) { startMs: 1747947419000, // 2025-05-22 21:56:59 UTC, ~3m before release endMs: 1747983448000, // 2025-05-23 07:57:28 UTC, ~10h after release expected: qbtypes.Statement{ - Query: "WITH __limit_cte AS (SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`", + Query: "WITH __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`", Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, }, @@ -882,7 +882,7 @@ func TestStatementBuilderGroupByResourceEvolution(t *testing.T) { cb := NewConditionBuilder(fm) mockMetadataStore := telemetrytypestest.NewMockMetadataStore() mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewTraceQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -959,8 +959,8 @@ func TestStatementBuilderTraceQuery(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __toe AS (SELECT trace_id FROM signoz_traces.distributed_signoz_index_v3 WHERE (`attribute_string_materialized$$key$$name` = ? AND `attribute_string_materialized$$key$$name_exists` = ?) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __toe_duration_sorted AS (SELECT trace_id, duration_nano, resource_string_service$$name as `service.name`, name FROM signoz_traces.distributed_signoz_index_v3 WHERE parent_span_id = '' AND trace_id GLOBAL IN __toe AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? ORDER BY duration_nano DESC LIMIT 1 BY trace_id) SELECT __toe_duration_sorted.`service.name` AS `service.name`, __toe_duration_sorted.name AS `name`, count() AS span_count, __toe_duration_sorted.duration_nano AS `duration_nano`, __toe_duration_sorted.trace_id AS `trace_id` FROM __toe INNER JOIN __toe_duration_sorted ON __toe.trace_id = __toe_duration_sorted.trace_id GROUP BY trace_id, duration_nano, name, `service.name` ORDER BY duration_nano DESC LIMIT 1 BY trace_id LIMIT ? SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000", - Args: []any{"redis-manual", true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, + Query: "WITH __toe AS (SELECT trace_id FROM signoz_traces.distributed_signoz_index_v3 WHERE (`attribute_string_materialized$$key$$name` = ? AND `attribute_string_materialized$$key$$name_exists`) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __toe_duration_sorted AS (SELECT trace_id, duration_nano, resource_string_service$$name as `service.name`, name FROM signoz_traces.distributed_signoz_index_v3 WHERE parent_span_id = '' AND trace_id GLOBAL IN __toe AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? ORDER BY duration_nano DESC LIMIT 1 BY trace_id) SELECT __toe_duration_sorted.`service.name` AS `service.name`, __toe_duration_sorted.name AS `name`, count() AS span_count, __toe_duration_sorted.duration_nano AS `duration_nano`, __toe_duration_sorted.trace_id AS `trace_id` FROM __toe INNER JOIN __toe_duration_sorted ON __toe.trace_id = __toe_duration_sorted.trace_id GROUP BY trace_id, duration_nano, name, `service.name` ORDER BY duration_nano DESC LIMIT 1 BY trace_id LIMIT ? SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000", + Args: []any{"redis-manual", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, }, expectedErr: nil, }, @@ -975,8 +975,8 @@ func TestStatementBuilderTraceQuery(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __toe AS (SELECT trace_id FROM signoz_traces.distributed_signoz_index_v3 WHERE ((match(`attribute_string_materialized$$key$$name`, ?) AND `attribute_string_materialized$$key$$name_exists` = ?) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __toe_duration_sorted AS (SELECT trace_id, duration_nano, resource_string_service$$name as `service.name`, name FROM signoz_traces.distributed_signoz_index_v3 WHERE parent_span_id = '' AND trace_id GLOBAL IN __toe AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? ORDER BY duration_nano DESC LIMIT 1 BY trace_id) SELECT __toe_duration_sorted.`service.name` AS `service.name`, __toe_duration_sorted.name AS `name`, count() AS span_count, __toe_duration_sorted.duration_nano AS `duration_nano`, __toe_duration_sorted.trace_id AS `trace_id` FROM __toe INNER JOIN __toe_duration_sorted ON __toe.trace_id = __toe_duration_sorted.trace_id GROUP BY trace_id, duration_nano, name, `service.name` ORDER BY duration_nano DESC LIMIT 1 BY trace_id LIMIT ? SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000", - Args: []any{"redis-manual", true, "redis-manual", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, + Query: "WITH __toe AS (SELECT trace_id FROM signoz_traces.distributed_signoz_index_v3 WHERE ((match(`attribute_string_materialized$$key$$name`, ?) AND `attribute_string_materialized$$key$$name_exists`) OR (multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL)) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __toe_duration_sorted AS (SELECT trace_id, duration_nano, resource_string_service$$name as `service.name`, name FROM signoz_traces.distributed_signoz_index_v3 WHERE parent_span_id = '' AND trace_id GLOBAL IN __toe AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? ORDER BY duration_nano DESC LIMIT 1 BY trace_id) SELECT __toe_duration_sorted.`service.name` AS `service.name`, __toe_duration_sorted.name AS `name`, count() AS span_count, __toe_duration_sorted.duration_nano AS `duration_nano`, __toe_duration_sorted.trace_id AS `trace_id` FROM __toe INNER JOIN __toe_duration_sorted ON __toe.trace_id = __toe_duration_sorted.trace_id GROUP BY trace_id, duration_nano, name, `service.name` ORDER BY duration_nano DESC LIMIT 1 BY trace_id LIMIT ? SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000", + Args: []any{"redis-manual", "redis-manual", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, }, expectedErr: nil, }, @@ -1021,8 +1021,8 @@ func TestStatementBuilderTraceQuery(t *testing.T) { Limit: 10, }, expected: qbtypes.Statement{ - Query: "WITH __toe AS (SELECT trace_id FROM signoz_traces.distributed_signoz_index_v3 WHERE (((name, resource_string_service$$name) GLOBAL IN (SELECT DISTINCT name, serviceName from signoz_traces.distributed_top_level_operations WHERE time >= toDateTime(1747947419))) AND parent_span_id != '' OR (`attribute_string_materialized$$key$$name` = ? AND `attribute_string_materialized$$key$$name_exists` = ?)) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __toe_duration_sorted AS (SELECT trace_id, duration_nano, resource_string_service$$name as `service.name`, name FROM signoz_traces.distributed_signoz_index_v3 WHERE parent_span_id = '' AND trace_id GLOBAL IN __toe AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? ORDER BY duration_nano DESC LIMIT 1 BY trace_id) SELECT __toe_duration_sorted.`service.name` AS `service.name`, __toe_duration_sorted.name AS `name`, count() AS span_count, __toe_duration_sorted.duration_nano AS `duration_nano`, __toe_duration_sorted.trace_id AS `trace_id` FROM __toe INNER JOIN __toe_duration_sorted ON __toe.trace_id = __toe_duration_sorted.trace_id GROUP BY trace_id, duration_nano, name, `service.name` ORDER BY duration_nano DESC LIMIT 1 BY trace_id LIMIT ? SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000", - Args: []any{"redis-manual", true, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, + Query: "WITH __toe AS (SELECT trace_id FROM signoz_traces.distributed_signoz_index_v3 WHERE (((name, resource_string_service$$name) GLOBAL IN (SELECT DISTINCT name, serviceName from signoz_traces.distributed_top_level_operations WHERE time >= toDateTime(1747947419))) AND parent_span_id != '' OR (`attribute_string_materialized$$key$$name` = ? AND `attribute_string_materialized$$key$$name_exists`)) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __toe_duration_sorted AS (SELECT trace_id, duration_nano, resource_string_service$$name as `service.name`, name FROM signoz_traces.distributed_signoz_index_v3 WHERE parent_span_id = '' AND trace_id GLOBAL IN __toe AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? ORDER BY duration_nano DESC LIMIT 1 BY trace_id) SELECT __toe_duration_sorted.`service.name` AS `service.name`, __toe_duration_sorted.name AS `name`, count() AS span_count, __toe_duration_sorted.duration_nano AS `duration_nano`, __toe_duration_sorted.trace_id AS `trace_id` FROM __toe INNER JOIN __toe_duration_sorted ON __toe.trace_id = __toe_duration_sorted.trace_id GROUP BY trace_id, duration_nano, name, `service.name` ORDER BY duration_nano DESC LIMIT 1 BY trace_id LIMIT ? SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000", + Args: []any{"redis-manual", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10}, }, expectedErr: nil, }, @@ -1049,7 +1049,7 @@ func TestStatementBuilderTraceQuery(t *testing.T) { cb := NewConditionBuilder(fm) mockMetadataStore := telemetrytypestest.NewMockMetadataStore() mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewTraceQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -1690,7 +1690,7 @@ func newSkipResourceFingerprintBuilder( mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime) aggExprRewriter := querybuilder.NewAggExprRewriter( - instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl, + instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl, ) return NewTraceQueryStatementBuilder( @@ -1717,7 +1717,7 @@ func TestStatementBuilderGroupByUnseenKey(t *testing.T) { cb := NewConditionBuilder(fm) mockMetadataStore := telemetrytypestest.NewMockMetadataStore() mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewTraceQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), @@ -1759,7 +1759,7 @@ func TestStatementBuilderAggregationUnseenKey(t *testing.T) { cb := NewConditionBuilder(fm) mockMetadataStore := telemetrytypestest.NewMockMetadataStore() mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewTraceQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), diff --git a/pkg/telemetrytraces/trace_operator_cte_builder.go b/pkg/telemetrytraces/trace_operator_cte_builder.go index b47334cbd60..0bb01533ddb 100644 --- a/pkg/telemetrytraces/trace_operator_cte_builder.go +++ b/pkg/telemetrytraces/trace_operator_cte_builder.go @@ -491,7 +491,7 @@ func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFrom if selectedFields[field.Name] { continue } - expr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.orgID, b.start, b.end, &field, keys) + expr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.orgID, b.start, b.end, &field, telemetrytypes.FieldDataTypeUnspecified, keys) if err != nil { b.stmtBuilder.logger.WarnContext(ctx, "failed to map select field", slog.String("field", field.Name), errors.Attr(err)) @@ -512,7 +512,7 @@ func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFrom // Add order by support orderApplied := false for _, orderBy := range b.operator.Order { - expr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.orgID, b.start, b.end, &orderBy.Key.TelemetryFieldKey, keys) + expr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.orgID, b.start, b.end, &orderBy.Key.TelemetryFieldKey, telemetrytypes.FieldDataTypeUnspecified, keys) if err != nil { return nil, err } @@ -631,7 +631,7 @@ func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, sele )) for _, gb := range b.operator.GroupBy { - expr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.orgID, b.start, b.end, &gb.TelemetryFieldKey, keys) + expr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.orgID, b.start, b.end, &gb.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return nil, errors.NewInvalidInputf( errors.CodeInvalidInput, @@ -727,7 +727,7 @@ func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFro sb := sqlbuilder.NewSelectBuilder() for _, gb := range b.operator.GroupBy { - expr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.orgID, b.start, b.end, &gb.TelemetryFieldKey, keys) + expr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.orgID, b.start, b.end, &gb.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return nil, errors.NewInvalidInputf( errors.CodeInvalidInput, @@ -853,7 +853,7 @@ func (b *traceOperatorCTEBuilder) buildScalarQuery(ctx context.Context, selectFr sb := sqlbuilder.NewSelectBuilder() for _, gb := range b.operator.GroupBy { - expr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.orgID, b.start, b.end, &gb.TelemetryFieldKey, keys) + expr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.orgID, b.start, b.end, &gb.TelemetryFieldKey, telemetrytypes.FieldDataTypeString, keys) if err != nil { return nil, errors.NewInvalidInputf( errors.CodeInvalidInput, diff --git a/pkg/telemetrytraces/trace_operator_cte_builder_test.go b/pkg/telemetrytraces/trace_operator_cte_builder_test.go index e5c06f86744..ba4b1effaf0 100644 --- a/pkg/telemetrytraces/trace_operator_cte_builder_test.go +++ b/pkg/telemetrytraces/trace_operator_cte_builder_test.go @@ -23,7 +23,7 @@ func newTestTraceOperatorStatementBuilder(t *testing.T) *traceOperatorStatementB cb := NewConditionBuilder(fm) mockMetadataStore := telemetrytypestest.NewMockMetadataStore() mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) traceStmtBuilder := NewTraceQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), mockMetadataStore, fm, cb, aggExprRewriter, nil, fl, false, 100000, @@ -284,7 +284,7 @@ func TestTraceOperatorStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH toDateTime64(1747947419000000000, 9) AS t_from, toDateTime64(1747983448000000000, 9) AS t_to, 1747945619 AS bucket_from, 1747983448 AS bucket_to, all_spans AS (SELECT *, resource_string_service$$name AS `service.name` FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __resource_filter_A AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), A AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter_A) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __resource_filter_B AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), B AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter_B) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), A_DIR_DESC_B AS (SELECT p.* FROM A AS p INNER JOIN B AS c ON p.trace_id = c.trace_id AND p.span_id = c.parent_span_id) SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM A_DIR_DESC_B GROUP BY ts, `service.name` ORDER BY ts desc SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000", + Query: "WITH toDateTime64(1747947419000000000, 9) AS t_from, toDateTime64(1747983448000000000, 9) AS t_to, 1747945619 AS bucket_from, 1747983448 AS bucket_to, all_spans AS (SELECT *, resource_string_service$$name AS `service.name` FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __resource_filter_A AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), A AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter_A) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __resource_filter_B AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), B AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter_B) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), A_DIR_DESC_B AS (SELECT p.* FROM A AS p INNER JOIN B AS c ON p.trace_id = c.trace_id AND p.span_id = c.parent_span_id) SELECT toStartOfInterval(timestamp, INTERVAL 60 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM A_DIR_DESC_B GROUP BY ts, `service.name` ORDER BY ts desc SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000", Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "frontend", "%service.name%", "%service.name\":\"frontend%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "backend", "%service.name%", "%service.name\":\"backend%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)}, }, expectedErr: nil, @@ -343,8 +343,8 @@ func TestTraceOperatorStatementBuilder(t *testing.T) { }, }, expected: qbtypes.Statement{ - Query: "WITH toDateTime64(1747947419000000000, 9) AS t_from, toDateTime64(1747983448000000000, 9) AS t_to, 1747945619 AS bucket_from, 1747983448 AS bucket_to, all_spans AS (SELECT *, resource_string_service$$name AS `service.name` FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __resource_filter_A AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), A AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter_A) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), B AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND toFloat64(response_status_code) < ?), A_AND_B AS (SELECT l.* FROM A AS l INNER JOIN B AS r ON l.trace_id = r.trace_id) SELECT toString(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)) AS `service.name`, avg(multiIf(duration_nano <> ?, duration_nano, NULL)) AS __result_0 FROM A_AND_B GROUP BY `service.name` ORDER BY __result_0 desc SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000", - Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "frontend", "%service.name%", "%service.name\":\"frontend%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), float64(400), 0}, + Query: "WITH toDateTime64(1747947419000000000, 9) AS t_from, toDateTime64(1747983448000000000, 9) AS t_to, 1747945619 AS bucket_from, 1747983448 AS bucket_to, all_spans AS (SELECT *, resource_string_service$$name AS `service.name` FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __resource_filter_A AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), A AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter_A) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), B AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND toFloat64(response_status_code) < ?), A_AND_B AS (SELECT l.* FROM A AS l INNER JOIN B AS r ON l.trace_id = r.trace_id) SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, avg(multiIf(duration_nano <> 0, duration_nano, mapContains(attributes_number, 'duration_nano'), toFloat64(attributes_number['duration_nano']), NULL)) AS __result_0 FROM A_AND_B GROUP BY `service.name` ORDER BY __result_0 desc SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000", + Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "frontend", "%service.name%", "%service.name\":\"frontend%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), float64(400)}, }, expectedErr: nil, }, diff --git a/pkg/telemetrytraces/trace_time_range_test.go b/pkg/telemetrytraces/trace_time_range_test.go index 4363bb88b01..12e40611207 100644 --- a/pkg/telemetrytraces/trace_time_range_test.go +++ b/pkg/telemetrytraces/trace_time_range_test.go @@ -39,7 +39,7 @@ func TestTraceTimeRangeOptimization(t *testing.T) { }} fl := flaggertest.New(t) - aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl) + aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl) statementBuilder := NewTraceQueryStatementBuilder( instrumentationtest.New().ToProviderSettings(), diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/evolution.go b/pkg/types/querybuildertypes/querybuildertypesv5/evolution.go index bb6a80c0ed7..2660c8d66f6 100644 --- a/pkg/types/querybuildertypes/querybuildertypesv5/evolution.go +++ b/pkg/types/querybuildertypes/querybuildertypesv5/evolution.go @@ -19,6 +19,9 @@ import ( // - For each column, includes its evolution if it's >= latest base evolution and <= tsEndTime // - Results are sorted by ReleaseTime descending (newest first) func SelectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetrytypes.EvolutionEntry, tsStart, tsEnd uint64) ([]*schema.Column, []*telemetrytypes.EvolutionEntry, error) { + if len(evolutions) == 0 { + return columns, nil, nil + } sortedEvolutions := make([]*telemetrytypes.EvolutionEntry, len(evolutions)) copy(sortedEvolutions, evolutions) diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/qb.go b/pkg/types/querybuildertypes/querybuildertypesv5/qb.go index 3f9e13bd394..352a53a5e41 100644 --- a/pkg/types/querybuildertypes/querybuildertypesv5/qb.go +++ b/pkg/types/querybuildertypes/querybuildertypesv5/qb.go @@ -18,39 +18,28 @@ var ( ErrUnsupportedOperator = errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator") ) -type JsonKeyToFieldFunc func(context.Context, *telemetrytypes.TelemetryFieldKey, FilterOperator, any) (string, any) - // FieldMapper maps the telemetry field key to the table field name. type FieldMapper interface { // FieldFor returns the field name for the given key. FieldFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) // ColumnFor returns the column for the given key. ColumnFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) - // ColumnExpressionFor returns the column expression for the given key. Most mappers - // return it aliased (`expr AS name`); the traces mapper returns a bare expression and - // callers add their own alias. - ColumnExpressionFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) (string, error) + // ColumnExpressionFor returns the column expression for the given key. The audit, + // metrics, metadata, and resource-filter mappers return it aliased (`expr AS name`); + // the logs and traces mappers return a bare expression and callers add their own alias. + // requiredDataType selects the mode: Unspecified builds a group-by/select expression; + // a concrete type (String/Float64) builds an aggregation-argument expression coerced to it. + ColumnExpressionFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey, requiredDataType telemetrytypes.FieldDataType, keys map[string][]*telemetrytypes.TelemetryFieldKey) (string, error) // CandidateKeys returns the key(s) to query for a referenced field: metadata matches for // the name (or `{context}.{name}`) first, else synthesized type-variant keys for sources // that support it, else nil (caller errors). value is the filter operand, nil otherwise. CandidateKeys(ctx context.Context, orgID valuer.UUID, field *telemetrytypes.TelemetryFieldKey, value any, keys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey } -// ConditionBuilder builds the conditions for the filter. +// ConditionBuilder builds the conditions for a filter term. The builder owns key resolution: +// the visitor hands it the raw key + full metadata map + options, not a pre-matched key list. type ConditionBuilder interface { - // ConditionFor returns the conditions and any advisory warnings for a filter - // term. key is the field key as parsed from the query text; fieldKeysForName is - // the set of known field keys matching it (may be empty). The builder owns the - // decision of what to do — resolve ambiguity, fall back to a body JSON search, - // emit a "not found" error, or skip — and which errors/warnings are apt. - ConditionFor(ctx context.Context, orgID valuer.UUID, startNs uint64, endNs uint64, key *telemetrytypes.TelemetryFieldKey, fieldKeysForName []*telemetrytypes.TelemetryFieldKey, operator FilterOperator, value any, sb *sqlbuilder.SelectBuilder) ([]string, []string, error) -} - -// ResolvingConditionBuilder is an optional ConditionBuilder that owns key resolution itself: -// the where-clause visitor hands over the raw key + full metadata map instead of pre-matching, -// so lookup, collision handling, resource-filter skip, and synthesis live in one place. -type ResolvingConditionBuilder interface { - ConditionForKeys(ctx context.Context, orgID valuer.UUID, startNs uint64, endNs uint64, key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey, options ConditionBuilderOptions, operator FilterOperator, value any, sb *sqlbuilder.SelectBuilder) ([]string, []string, error) + ConditionFor(ctx context.Context, orgID valuer.UUID, startNs uint64, endNs uint64, key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey, options ConditionBuilderOptions, operator FilterOperator, value any, sb *sqlbuilder.SelectBuilder) ([]string, []string, error) } type ConditionBuilderOptions struct { diff --git a/pkg/types/telemetrytypes/field.go b/pkg/types/telemetrytypes/field.go index c0d7d42f6a4..303e22113b6 100644 --- a/pkg/types/telemetrytypes/field.go +++ b/pkg/types/telemetrytypes/field.go @@ -356,6 +356,14 @@ func NewFieldValueSelectorFromPostableFieldValueParams(params PostableFieldValue return fieldValueSelector } +func NewTelemetryFieldKey(name string, fieldContext FieldContext, fieldDataType FieldDataType) *TelemetryFieldKey { + return &TelemetryFieldKey{ + Name: name, + FieldContext: fieldContext, + FieldDataType: fieldDataType, + } +} + type TelemetryFieldKeySkipIndex struct { Name string `json:"name"` // Name is TelemetryFieldKey.Name not IndexName from ClickHouse FieldContext FieldContext `json:"fieldContext,omitzero"` diff --git a/pkg/types/telemetrytypes/json_access_plan.go b/pkg/types/telemetrytypes/json_access_plan.go index 71655dbfcad..03b510c36da 100644 --- a/pkg/types/telemetrytypes/json_access_plan.go +++ b/pkg/types/telemetrytypes/json_access_plan.go @@ -104,6 +104,7 @@ type planBuilder struct { paths []string // cumulative paths for type cache lookups segments []string // individual path segments for node names isPromoted bool + exhaustive bool typeCache map[string][]FieldDataType } @@ -159,16 +160,18 @@ func (pb *planBuilder) buildPlan(index int, parent *JSONAccessNode, isDynArrChil } } else { var err error - // Use cached types from the batched metadata query - types, ok := pb.typeCache[pathSoFar] - if !ok { - return nil, errors.NewInternalf(errors.CodeInvalidInput, "types missing for path %s", pathSoFar) - } + hasJSON, hasDynamic := pb.exhaustive, pb.exhaustive + if !pb.exhaustive { + types, ok := pb.typeCache[pathSoFar] + if !ok { + return nil, errors.NewInternalf(errors.CodeInvalidInput, "types missing for path %s", pathSoFar) + } - hasJSON := slices.Contains(types, FieldDataTypeArrayJSON) - hasDynamic := slices.Contains(types, FieldDataTypeArrayDynamic) - if !hasJSON && !hasDynamic { - return nil, errors.NewInternalf(CodePlanFieldDataTypeMissing, "array data type missing for path %s", pathSoFar) + hasJSON = slices.Contains(types, FieldDataTypeArrayJSON) + hasDynamic = slices.Contains(types, FieldDataTypeArrayDynamic) + if !hasJSON && !hasDynamic { + return nil, errors.NewInternalf(CodePlanFieldDataTypeMissing, "array data type missing for path %s", pathSoFar) + } } if hasJSON { @@ -229,3 +232,30 @@ func (key *TelemetryFieldKey) SetJSONAccessPlan(columnInfo JSONColumnMetadata, t return nil } + +func (key *TelemetryFieldKey) SetExhaustiveJSONAccessPlan(columnInfo JSONColumnMetadata, terminalType FieldDataType) error { + if key.Name == "" { + return errors.NewInvalidInputf(errors.CodeInvalidInput, "path is empty") + } + + if terminalType == FieldDataTypeUnspecified { + terminalType = FieldDataTypeString + } + + keyForPlan := NewTelemetryFieldKey(key.Name, key.FieldContext, terminalType) + + pb := &planBuilder{ + key: keyForPlan, + paths: key.ArrayParentPaths(), + segments: key.ArrayPathSegments(), + exhaustive: true, + } + + node, err := pb.buildPlan(0, NewRootJSONAccessNode(columnInfo.BaseColumn, 32, 0), false) + if err != nil { + return err + } + key.JSONPlan = append(key.JSONPlan, node) + + return nil +} diff --git a/tests/fixtures/logs.py b/tests/fixtures/logs.py index f0fc93d59aa..cff6f55d372 100644 --- a/tests/fixtures/logs.py +++ b/tests/fixtures/logs.py @@ -542,6 +542,34 @@ def _insert_logs(logs: list[Logs]) -> None: ) +@pytest.fixture(name="insert_logs_with_unknown_keys", scope="function") +def insert_logs_with_unknown_keys( + insert_logs: Callable[[list[Logs]], None], +) -> Callable[[list[Logs], set[str]], None]: + """Insert logs while treating the named ATTRIBUTE keys as absent from field-key + metadata. + + Each named key keeps its value in the row's attribute maps but has its attribute + metadata dropped (attribute_keys -> distributed_logs_attribute_keys and the tag + entry in distributed_tag_attributes_v2), so the querier can't resolve it and falls + back to the unknown-key synthesize path. This is the only way to drive the + synthesized attribute-map branches with real data — a normally-seeded attribute is + always registered in metadata and so never counts as "unknown". + + Only attribute-side metadata is stripped: resource keys always keep their metadata + (resource metadata is assumed never missing), so passing a resource key here has no + effect. Teardown/truncation is inherited from the underlying insert_logs fixture. + """ + + def _insert(logs: list[Logs], unknown_keys: set[str]) -> None: + for log in logs: + log.attribute_keys = [k for k in log.attribute_keys if k.name not in unknown_keys] + log.tag_attributes = [t for t in log.tag_attributes if not (t.tag_key in unknown_keys and t.tag_type == "tag")] + insert_logs(logs) + + return _insert + + @pytest.fixture(name="materialize_log_field", scope="function") def materialize_log_field( signoz: types.SigNoz, diff --git a/tests/fixtures/querier.py b/tests/fixtures/querier.py index 95a80738358..138efbfb238 100644 --- a/tests/fixtures/querier.py +++ b/tests/fixtures/querier.py @@ -79,6 +79,7 @@ class BuilderQuery: name: str = "A" source: str | None = None limit: int | None = None + offset: int | None = None filter_expression: str | None = None select_fields: list[TelemetryFieldKey] | None = None order: list[OrderBy] | None = None @@ -94,6 +95,8 @@ def to_dict(self) -> dict: spec["source"] = self.source if self.limit is not None: spec["limit"] = self.limit + if self.offset is not None: + spec["offset"] = self.offset if self.filter_expression: spec["filter"] = {"expression": self.filter_expression} if self.select_fields: @@ -570,6 +573,7 @@ def build_scalar_query( having_expression: str | None = None, step_interval: int = DEFAULT_STEP_INTERVAL, disabled: bool = False, + functions: list[dict] | None = None, ) -> dict: spec: dict[str, Any] = { "name": name, @@ -597,9 +601,41 @@ def build_scalar_query( if having_expression: spec["having"] = {"expression": having_expression} + if functions: + spec["functions"] = functions + return {"type": "builder_query", "spec": spec} +def build_traces_scalar_query( + aggregations: list[dict], + *, + name: str = "A", + group_by: list[dict] | None = None, + order: list[dict] | None = None, + limit: int | None = None, + filter_expression: str | None = None, + having_expression: str | None = None, + step_interval: int = DEFAULT_STEP_INTERVAL, + disabled: bool = False, + functions: list[dict] | None = None, +) -> dict: + """build_scalar_query pinned to the traces signal, with name defaulting to 'A'.""" + return build_scalar_query( + name=name, + signal="traces", + aggregations=aggregations, + group_by=group_by, + order=order, + limit=limit, + filter_expression=filter_expression, + having_expression=having_expression, + step_interval=step_interval, + disabled=disabled, + functions=functions, + ) + + def build_raw_query( name: str, signal: str, diff --git a/tests/fixtures/traces.py b/tests/fixtures/traces.py index ddc4ce7422c..34bece5f62a 100644 --- a/tests/fixtures/traces.py +++ b/tests/fixtures/traces.py @@ -836,9 +836,25 @@ def insert_traces_to_clickhouse(conn, traces: list[Traces]) -> None: "tag_attributes_v2", "span_attributes_keys", "signoz_error_index_v2", + "top_level_operations", ] +def insert_top_level_operations_to_clickhouse(conn, operations: list[tuple[str, str]]) -> None: + """Seed distributed_top_level_operations with (name, serviceName) rows so the + isEntryPoint span-scope filter has entries to match against. The `time` + column defaults to now(), which satisfies the filter's `time >= start` + guard for any recent query window.""" + if not operations: + return + conn.insert( + database="signoz_traces", + table="distributed_top_level_operations", + column_names=["name", "serviceName"], + data=[[name, service_name] for name, service_name in operations], + ) + + def truncate_traces_tables(conn, cluster: str) -> None: """Truncate all traces tables. Used by the pytest fixture teardown and by the seeder's DELETE /telemetry/traces endpoint.""" @@ -861,6 +877,18 @@ def _insert_traces(traces: list[Traces]) -> None: ) +@pytest.fixture(name="insert_top_level_operations", scope="function") +def insert_top_level_operations( + clickhouse: types.TestContainerClickhouse, +) -> Generator[Callable[[list[tuple[str, str]]], None], Any]: + def _insert(operations: list[tuple[str, str]]) -> None: + insert_top_level_operations_to_clickhouse(clickhouse.conn, operations) + + yield _insert + + clickhouse.conn.query(f"TRUNCATE TABLE signoz_traces.top_level_operations ON CLUSTER '{clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' SYNC") + + @pytest.fixture(name="remove_traces_ttl_and_storage_settings", scope="function") def remove_traces_ttl_and_storage_settings(signoz: types.SigNoz): """ @@ -883,3 +911,69 @@ def remove_traces_ttl_and_storage_settings(signoz: types.SigNoz): signoz.telemetrystore.conn.query(f"ALTER TABLE signoz_traces.{table} ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' RESET SETTING storage_policy;") except Exception as e: # pylint: disable=broad-exception-caught print(f"ttl and storage policy reset failed for {table}: {e}") + + +# ============================================================================ +# Clean / corrupt factor (shared across the traces querier tests) +# ============================================================================ +# Colliding/corrupt metadata mixed into every seeded span in the "corrupt" +# variant of the list/aggregation tests. It injects: +# - intrinsic column names as span attributes (timestamp, duration_nano, ...) +# - calculated column names as span attributes (http_method, db_name, ...) +# - a key (service.name) present in both attributes and resources +# Values deliberately mix Python types (str / int / float / bool) so the +# collision also spans the attributes_string / attributes_number / +# attributes_bool type-variant columns — e.g. a string attribute named +# duration_nano vs the numeric intrinsic, a numeric attribute named +# response_status_code vs the string calculated column, a bool attribute named +# db_name vs the string calculated column. +# Field-key collision resolution must keep the real intrinsic/calculated/resource +# columns winning regardless of the attribute's type, so every assertion holds +# identically for clean and corrupt spans. The corrupt values only ever surface +# inside the raw `attributes` map. +CORRUPT_ATTRIBUTES: dict[str, Any] = { + # intrinsic names (real column type in comment) + "timestamp": "corrupt_data", # string vs number + "trace_id": 12345, # number vs string + "span_id": True, # bool vs string + "parent_span_id": "corrupt_data", # string vs string + "trace_state": 99, # number vs string + "flags": "corrupt_data", # string vs number + "name": 42, # number vs string + "kind": "corrupt_data", # string vs number + "kind_string": 7, # number vs string + "duration_nano": "corrupt_data", # string vs number + "status_code": True, # bool vs number + "status_message": 500, # number vs string + "status_code_string": False, # bool vs string + # calculated names (real column type in comment) + "response_status_code": 999, # number vs string + "external_http_url": 8080, # number vs string + "http_url": True, # bool vs string + "external_http_method": 1, # number vs string + "http_method": False, # bool vs string + "http_host": 12, # number vs string + "db_name": True, # bool vs string + "db_operation": 3.5, # number vs string + "has_error": "corrupt_data", # string vs bool + "is_remote": 1, # number vs string + # collides with the resource key of the same name + "service.name": "collision-attr-value", +} +# Resources are always stored as strings, so the resource-side collision only +# varies the key names. +CORRUPT_RESOURCES: dict[str, Any] = { + "timestamp": "corrupt_data", + "duration_nano": "corrupt_data", + "http_method": "corrupt_data", +} + + +def trace_noise(variant: str) -> tuple[dict[str, Any], dict[str, Any]]: + """(extra_attributes, extra_resources) to merge into every span a traces test seeds, keyed by + variant. "clean" adds nothing; "corrupt" injects colliding intrinsic/calculated field names and + an attribute/resource collision so a test parametrized over both doubles as a + collision-robustness check. Returns fresh dicts so callers can mutate them safely.""" + if variant == "clean": + return {}, {} + return dict(CORRUPT_ATTRIBUTES), dict(CORRUPT_RESOURCES) diff --git a/tests/integration/tests/querier_json_body/03_body_array_functions.py b/tests/integration/tests/querier_json_body/03_body_array_functions.py index 742c3eb3505..ad5ff4e6302 100644 --- a/tests/integration/tests/querier_json_body/03_body_array_functions.py +++ b/tests/integration/tests/querier_json_body/03_body_array_functions.py @@ -7,11 +7,13 @@ from fixtures import types from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.jsontypes import JSONPathType from fixtures.logs import Logs from fixtures.querier import ( RequestType, build_order_by, build_raw_query, + get_all_warnings, get_rows, make_query_request, ) @@ -253,11 +255,13 @@ def test_logs_json_body_function_scalar_path( @pytest.mark.parametrize( - "expression", + "expression, expected_error", [ - pytest.param('has(code.function, "main")', id="has_non_body_key"), - pytest.param('hasToken(code.function, "main")', id="hastoken_non_body_key"), - pytest.param("hasToken(body, 123)", id="hastoken_non_string_value"), + # A has-family function on a non-body key is unsupported (same message as flag-off); + # a not-found *body* path instead warns and queries (see test_logs_json_body_unregistered_path_warns). + pytest.param('has(code.function, "main")', "function `has` supports only body JSON search", id="has_non_body_key"), + pytest.param('hasToken(code.function, "main")', "function `hasToken` only supports body field as first parameter", id="hastoken_non_body_key"), + pytest.param("hasToken(body, 123)", "expects value parameter to be a string", id="hastoken_non_string_value"), ], ) def test_logs_json_body_function_errors( @@ -265,8 +269,10 @@ def test_logs_json_body_function_errors( create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], expression: str, + expected_error: str, ) -> None: - """has/hasToken misuse (non-body key, non-string token) is rejected (400).""" + """has/hasToken misuse (non-body key, non-string token) is rejected (400) with a message that + names the body-only restriction rather than a generic "key not found".""" now = datetime.now(tz=UTC) token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) response = make_query_request( @@ -277,7 +283,8 @@ def test_logs_json_body_function_errors( request_type=RequestType.RAW, queries=[build_raw_query("A", "logs", order=[build_order_by("timestamp")], limit=100, filter_expression=expression)], ) - assert response.status_code == HTTPStatus.BAD_REQUEST + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert expected_error in response.text, f"{expression}: expected {expected_error!r} in {response.text}" @pytest.mark.parametrize( @@ -914,8 +921,9 @@ def test_logs_json_body_array_hop_syntax( insert_logs: Callable[[list[Logs]], None], export_json_types: Callable[[list[Logs]], None], ) -> None: - """In JSON mode the array-hop is written body.edu[].name; the legacy body.edu[*].name form is - rejected (400) during field resolution.""" + """In JSON mode the array-hop is written body.edu[].name (resolved from metadata). The legacy + body.edu[*].name form is no longer a hard error: it is an unregistered body path, so it warns + and queries the underlying data (200) like any other not-found body path.""" now = datetime.now(tz=UTC) logs = [ Logs(timestamp=now - timedelta(seconds=2), resources={"service.name": "app-service"}, body_v2=json.dumps({"edu": [{"name": "IIT"}, {"name": "MIT"}]}), body_promoted="", severity_text="INFO"), @@ -939,7 +947,8 @@ def test_logs_json_body_array_hop_syntax( assert response.status_code == HTTPStatus.OK, response.text assert len(get_rows(response)) == 1 - # [*] is not accepted in JSON mode. + # [*] is not the JSON-mode hop syntax; it is treated as an unregistered body path and + # warns + queries (200) rather than erroring. response = make_query_request( signoz, token, @@ -948,7 +957,7 @@ def test_logs_json_body_array_hop_syntax( request_type=RequestType.RAW, queries=[build_raw_query("A", "logs", order=[build_order_by("timestamp")], limit=100, filter_expression="has(body.edu[*].name, 'MIT')")], ) - assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert response.status_code == HTTPStatus.OK, response.text def test_logs_json_body_nested_family( @@ -1048,8 +1057,10 @@ def test_logs_json_body_degenerate_targets( insert_logs: Callable[[list[Logs]], None], export_json_types: Callable[[list[Logs]], None], ) -> None: - """has on an unsupported target — an object leaf, the bare body, or an array-of-arrays — is - rejected (400) during field resolution, never a ClickHouse 500.""" + """has on a degenerate target, never a ClickHouse 500. An object leaf or an array-of-arrays has + no primitive leaf registered in metadata, so it is a not-found body path — it warns and queries + (200, no match), the same as any other unregistered body path. Only the bare body (a full-text + target, not a JSON path) stays a 400.""" now = datetime.now(tz=UTC) logs = [ Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": "app-service"}, body_v2=json.dumps({"obj": {"b": "x"}, "matrix": [[1, 2], [3]]}), body_promoted="", severity_text="INFO"), @@ -1060,7 +1071,8 @@ def test_logs_json_body_degenerate_targets( start_ms = int((now - timedelta(seconds=10)).timestamp() * 1000) end_ms = int(now.timestamp() * 1000) - for expression in ["has(body.obj, 'x')", "has(body, 'x')", "has(body.matrix, 1)"]: + # An object leaf / array-of-arrays: no primitive leaf in metadata -> warn + query (200, no match). + for expression in ["has(body.obj, 'x')", "has(body.matrix, 1)"]: response = make_query_request( signoz, token, @@ -1069,7 +1081,20 @@ def test_logs_json_body_degenerate_targets( request_type=RequestType.RAW, queries=[build_raw_query("A", "logs", order=[build_order_by("timestamp")], limit=100, filter_expression=expression)], ) - assert response.status_code == HTTPStatus.BAD_REQUEST, f"{expression}: expected 400, got {response.status_code}: {response.text}" + assert response.status_code == HTTPStatus.OK, f"{expression}: expected 200, got {response.status_code}: {response.text}" + assert get_rows(response) == [], f"{expression}: degenerate target should match nothing" + + # The bare body is not a JSON path -> has-family is unsupported on it (400). + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[build_raw_query("A", "logs", order=[build_order_by("timestamp")], limit=100, filter_expression="has(body, 'x')")], + ) + assert response.status_code == HTTPStatus.BAD_REQUEST, f"has(body): expected 400, got {response.status_code}: {response.text}" + assert "function `has` supports only body JSON search" in response.text, response.text def test_logs_json_body_genuine_empty_and_zero_elements( @@ -1134,3 +1159,80 @@ def test_logs_json_body_quote_unicode_needles( ) assert response.status_code == HTTPStatus.OK, f"{expression}: {response.text}" assert len(get_rows(response)) == 1, f"{expression}: expected 1 row" + + +@pytest.mark.parametrize( + "needle", + [ + pytest.param("192.168.1.1", id="dotted_ip"), + pytest.param("user_id", id="underscore"), + pytest.param("error-code", id="hyphen"), + pytest.param("production node", id="whitespace"), + ], +) +def test_logs_json_body_hastoken_separator_needle_errors( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], + export_json_types: Callable[[list[Logs]], None], + needle: str, +) -> None: + """KNOWN BUG (currently 500) — same defect as the flag-off path. In JSON mode hasToken searches + body.message via ClickHouse hasToken(LOWER(body.message), LOWER(?)), which rejects a needle + containing whitespace or separator characters (`.`/`_`/`-`/space) with error code 36. The needle + is passed straight through, so the query fails at execution and the raw error surfaces as a 500 + instead of a 400 / graceful fallback. Flip to 400 (or a fallback match) once the needle is + validated.""" + now = datetime.now(tz=UTC) + logs = [ + Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": "app-service"}, body_v2=json.dumps({"message": "client 192.168.1.1 user_id error-code production node"}), body_promoted="", severity_text="INFO"), + ] + export_json_types(logs) + insert_logs(logs) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(seconds=10)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type=RequestType.RAW, + queries=[build_raw_query("A", "logs", order=[build_order_by("timestamp")], limit=100, filter_expression=f"hasToken(body, '{needle}')")], + ) + assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR, f"{needle}: expected 500, got {response.status_code}: {response.text}" + assert "Needle must not contain whitespace or separator characters" in response.text, response.text + + +def test_logs_json_body_unregistered_path_warns( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], + export_json_types: Callable[[list[JSONPathType]], None], +) -> None: + """A has-family call on a body path that is absent from metadata is NOT a 400: it emits the + key-not-found warning and queries the underlying data (200), the same as a regular filter + operator on an unknown key. Here only `known` is registered; `unknown` is not. + + (Note: like a regular filter, a not-found *flat* body path in JSON mode currently resolves to + no match even when the value is present — an operator-wide JSON limitation — so we assert the + 200 + warning contract rather than a match count.)""" + now = datetime.now(tz=UTC) + logs = [ + Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": "app-service"}, body_v2=json.dumps({"known": ["a"], "unknown": ["a", "b"]}), body_promoted="", severity_text="INFO"), + ] + export_json_types([JSONPathType("known", "[]string")]) # register only `known` + insert_logs(logs) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + for expression in ["has(body.unknown, 'a')", "hasAny(body.unknown, ['a'])", "hasAll(body.unknown, ['a', 'b'])", "hasToken(body.unknown, 'a')"]: + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(seconds=10)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type=RequestType.RAW, + queries=[build_raw_query("A", "logs", order=[build_order_by("timestamp")], limit=100, filter_expression=expression)], + ) + assert response.status_code == HTTPStatus.OK, f"{expression}: expected 200, got {response.status_code}: {response.text}" + warnings = [w.get("message", "") for w in get_all_warnings(response.json())] + assert any("not found in metadata" in w for w in warnings), f"{expression}: expected a key-not-found warning, got {warnings}" diff --git a/tests/integration/tests/querier_json_body/04_unknown_key_collisions.py b/tests/integration/tests/querier_json_body/04_unknown_key_collisions.py new file mode 100644 index 00000000000..a0944d053ce --- /dev/null +++ b/tests/integration/tests/querier_json_body/04_unknown_key_collisions.py @@ -0,0 +1,301 @@ +import json +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http import HTTPStatus + +import pytest + +from fixtures import types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.jsontypes import JSONPathType +from fixtures.logs import Logs +from fixtures.querier import ( + RequestType, + build_aggregation, + build_group_by_field, + build_raw_query, + build_scalar_query, + get_rows, + get_scalar_table_data, + make_query_request, + make_scalar_query_request, +) + +# Unknown-key collision with use_json_body ON. The synthesized fallback for a bare +# unknown key is a multiIf over attributes_string -> attributes_number -> +# attributes_bool -> the body_v2 JSON path. This suite drives the FULL chain: one +# unknown key whose value lives in a different physical place per record — a string / +# number / bool attribute, or the body JSON — and asserts the querier finds and buckets +# every record. It also checks that the with-metadata vs without-metadata parity +# (13_unknown_key_collisions.py, flag off) still holds with the trailing body branch +# present. On a stack without the synthesize change the unknown-key half is a 400. + +SERVICE = "json-collision-svc" + + +def test_json_body_unknown_key_collision_groupby( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs_with_unknown_keys: Callable[[list[Logs], set[str]], None], +) -> None: + """ + Setup: + 6 logs carrying `field` in different physical places: 2 as a string attribute + ("sa"), 1 as a number attribute (10), 1 as a bool attribute (True), 2 in the body + JSON ("bs"). `field` is stripped from attribute metadata, so it is unknown + everywhere and takes the synthesize path. + + Tests: + Grouping by the unknown `field` walks the whole multiIf (attribute maps then body) + and buckets every record by its value, no matter which place holds it. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + logs = [ + Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": SERVICE}, attributes={"field": "sa"}, body="attr-str-0"), + Logs(timestamp=now - timedelta(seconds=2), resources={"service.name": SERVICE}, attributes={"field": "sa"}, body="attr-str-1"), + Logs(timestamp=now - timedelta(seconds=3), resources={"service.name": SERVICE}, attributes={"field": 10}, body="attr-num"), + Logs(timestamp=now - timedelta(seconds=4), resources={"service.name": SERVICE}, attributes={"field": True}, body="attr-bool"), + Logs(timestamp=now - timedelta(seconds=5), resources={"service.name": SERVICE}, attributes={"http.method": "GET"}, body=json.dumps({"field": "bs"})), + Logs(timestamp=now - timedelta(seconds=6), resources={"service.name": SERVICE}, attributes={"http.method": "GET"}, body=json.dumps({"field": "bs"})), + ] + insert_logs_with_unknown_keys(logs, {"field"}) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_scalar_query_request( + signoz, + token, + now, + [ + build_scalar_query( + "A", + "logs", + [build_aggregation("count()")], + group_by=[build_group_by_field("field", field_data_type="", field_context="")], + filter_expression=f'service.name = "{SERVICE}"', + ) + ], + ) + assert response.status_code == HTTPStatus.OK, response.text + + counts = {row[0]: row[-1] for row in get_scalar_table_data(response.json())} + assert counts == {"sa": 2, "10": 1, "true": 1, "bs": 2}, counts + + +@pytest.mark.parametrize( + "predicate,expected_rids", + [ + pytest.param('field = "sa"', {"r0", "r1"}, id="string_attr"), + pytest.param("field = 10", {"r2"}, id="number_attr"), + pytest.param("field = true", {"r3"}, id="bool_attr"), + pytest.param('field = "bs"', {"r4", "r5"}, id="body_value"), + pytest.param("field exists", {"r0", "r1", "r2", "r3", "r4", "r5"}, id="exists_all"), + ], +) +def test_json_body_unknown_key_collision_filter( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs_with_unknown_keys: Callable[[list[Logs], set[str]], None], + predicate: str, + expected_rids: set[str], +) -> None: + """ + Setup: + The attribute/body collision dataset, each record tagged with a unique `rid` + resource so the filter result is checked by exact record identity, not just count. + + Tests: + Filtering the unknown `field` resolves attributes OR the body JSON path and selects + exactly the expected records: a typed operand hits its attribute map, a body value + is matched via the body branch, and `exists` finds all six records. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + logs = [ + Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": SERVICE, "rid": "r0"}, attributes={"field": "sa"}, body="attr-str-0"), + Logs(timestamp=now - timedelta(seconds=2), resources={"service.name": SERVICE, "rid": "r1"}, attributes={"field": "sa"}, body="attr-str-1"), + Logs(timestamp=now - timedelta(seconds=3), resources={"service.name": SERVICE, "rid": "r2"}, attributes={"field": 10}, body="attr-num"), + Logs(timestamp=now - timedelta(seconds=4), resources={"service.name": SERVICE, "rid": "r3"}, attributes={"field": True}, body="attr-bool"), + Logs(timestamp=now - timedelta(seconds=5), resources={"service.name": SERVICE, "rid": "r4"}, attributes={"http.method": "GET"}, body=json.dumps({"field": "bs"})), + Logs(timestamp=now - timedelta(seconds=6), resources={"service.name": SERVICE, "rid": "r5"}, attributes={"http.method": "GET"}, body=json.dumps({"field": "bs"})), + ] + insert_logs_with_unknown_keys(logs, {"field"}) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type=RequestType.RAW, + queries=[build_raw_query("A", "logs", limit=100, filter_expression=f'service.name = "{SERVICE}" AND ({predicate})')], + ) + assert response.status_code == HTTPStatus.OK, response.text + matched = {row["data"]["resources_string"]["rid"] for row in get_rows(response)} + assert matched == expected_rids, f"{predicate}: matched {matched} != {expected_rids}" + + +def test_json_body_unknown_key_attribute_parity( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs_with_unknown_keys: Callable[[list[Logs], set[str]], None], +) -> None: + """ + Setup: + 4 logs carrying the same value under known_key (in metadata) and unknown_key + (stripped), string / number / bool across records — no body values this time. + + Tests: + Even with use_json_body on (the synthesized multiIf carries a trailing body + branch), grouping by the unknown attribute key still matches grouping by the known + one: the extra body branch does not perturb attribute-map resolution. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + specs = [("log-a", "alpha"), ("log-b", 42), ("log-c", 42), ("log-d", True)] + logs = [ + Logs( + timestamp=now - timedelta(seconds=i + 1), + resources={"service.name": SERVICE}, + attributes={"known_key": value, "unknown_key": value}, + body=body, + ) + for i, (body, value) in enumerate(specs) + ] + insert_logs_with_unknown_keys(logs, {"unknown_key"}) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + counts: dict[str, dict[str, int]] = {} + for key in ("known_key", "unknown_key"): + response = make_scalar_query_request( + signoz, + token, + now, + [ + build_scalar_query( + "A", + "logs", + [build_aggregation("count()")], + group_by=[build_group_by_field(key, field_data_type="", field_context="")], + filter_expression=f'service.name = "{SERVICE}"', + ) + ], + ) + assert response.status_code == HTTPStatus.OK, f"{key}: {response.text}" + counts[key] = {row[0]: row[-1] for row in get_scalar_table_data(response.json())} + + assert counts["known_key"] == counts["unknown_key"], f"with-metadata {counts['known_key']} != without-metadata {counts['unknown_key']}" + assert counts["known_key"] == {"alpha": 1, "42": 2, "true": 1}, counts["known_key"] + + +def test_json_body_unknown_key_aggregation_over_body( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs_with_unknown_keys: Callable[[list[Logs], set[str]], None], +) -> None: + """ + Setup: + The same attribute/body collision dataset — `field` carried as a string / number / + bool attribute and in the body JSON, one place per record. + + Tests: + Aggregating over the unknown `field` walks the whole multiIf including the body + branch: count() sees every record (all 6) and count_distinct() counts the distinct + stringified values across attribute maps AND the body ("sa", "10", "true", "bs"). + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + logs = [ + Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": SERVICE}, attributes={"field": "sa"}, body="attr-str-0"), + Logs(timestamp=now - timedelta(seconds=2), resources={"service.name": SERVICE}, attributes={"field": "sa"}, body="attr-str-1"), + Logs(timestamp=now - timedelta(seconds=3), resources={"service.name": SERVICE}, attributes={"field": 10}, body="attr-num"), + Logs(timestamp=now - timedelta(seconds=4), resources={"service.name": SERVICE}, attributes={"field": True}, body="attr-bool"), + Logs(timestamp=now - timedelta(seconds=5), resources={"service.name": SERVICE}, attributes={"http.method": "GET"}, body=json.dumps({"field": "bs"})), + Logs(timestamp=now - timedelta(seconds=6), resources={"service.name": SERVICE}, attributes={"http.method": "GET"}, body=json.dumps({"field": "bs"})), + ] + insert_logs_with_unknown_keys(logs, {"field"}) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_scalar_query_request( + signoz, + token, + now, + [ + build_scalar_query( + "A", + "logs", + [build_aggregation("count(field)", "cnt"), build_aggregation("count_distinct(field)", "distinct")], + filter_expression=f'service.name = "{SERVICE}"', + ) + ], + ) + assert response.status_code == HTTPStatus.OK, response.text + data = get_scalar_table_data(response.json()) + assert len(data) == 1, f"expected a single ungrouped row, got {data}" + # count() over all 6 records; count_distinct() over {sa, 10, true, bs} incl. the body value. + assert tuple(data[0]) == (6, 4), data[0] + + +def test_json_body_unknown_key_body_path_parity( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], + export_json_types: Callable[[list[JSONPathType]], None], +) -> None: + """ + Setup: + Logs whose body carries the same value under two paths — known_field and + unknown_field — but only known_field is exported to the body path metadata + (distributed_field_keys); unknown_field is left unregistered. + + Tests: + Grouping/filtering by the registered body path (known_field, metadata-driven) and + by the unregistered one (unknown_field, synthesized body branch) return identical + results — the synthesize fallback reproduces exported-metadata resolution for body + paths, the body counterpart of the attribute parity in 13_unknown_key_collisions.py. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + logs = [] + i = 0 + for value, count in [("alpha", 2), ("beta", 1)]: + for _ in range(count): + logs.append( + Logs( + timestamp=now - timedelta(seconds=i + 1), + resources={"service.name": SERVICE}, + body=json.dumps({"known_field": value, "unknown_field": value}), + ) + ) + i += 1 + insert_logs(logs) + # Register only known_field in the body path metadata; unknown_field stays unknown. + export_json_types([JSONPathType("known_field", "string")]) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + counts: dict[str, dict[str, int]] = {} + for key in ("known_field", "unknown_field"): + response = make_scalar_query_request( + signoz, + token, + now, + [ + build_scalar_query( + "A", + "logs", + [build_aggregation("count()")], + group_by=[build_group_by_field(key, field_data_type="", field_context="")], + filter_expression=f'service.name = "{SERVICE}"', + ) + ], + ) + assert response.status_code == HTTPStatus.OK, f"{key}: {response.text}" + counts[key] = {row[0]: row[-1] for row in get_scalar_table_data(response.json())} + + assert counts["known_field"] == counts["unknown_field"], f"registered {counts['known_field']} != synthesized {counts['unknown_field']}" + assert counts["known_field"] == {"alpha": 2, "beta": 1}, counts["known_field"] diff --git a/tests/integration/tests/queriercommon/04_filter_operators.py b/tests/integration/tests/queriercommon/04_filter_operators.py index b6e266f2679..669a835ce1b 100644 --- a/tests/integration/tests/queriercommon/04_filter_operators.py +++ b/tests/integration/tests/queriercommon/04_filter_operators.py @@ -7,12 +7,12 @@ from fixtures import types from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD from fixtures.logs import Logs -from fixtures.querier import get_column_data_from_response, make_query_request +from fixtures.querier import get_all_warnings, get_column_data_from_response, make_query_request # Filter-operator coverage that 01_filter_expression.py (NOT semantics) and # 06_json_body.py (CONTAINS) leave out: ILIKE / NOT LIKE / NOT CONTAINS, the # `key:number` data-type-suffix disambiguator on an ambiguous key, and a -# truly-unknown key (rejected as a bad request on this HEAD). +# truly-unknown key (synthesized, query succeeds with a not-found warning). # # Data model mirrors 01_filter_expression.py::test_not_filter_expression: # alpha-log: resources region="us-east"; attributes status_code=200 (number) @@ -97,11 +97,8 @@ def test_logs_filter_key_not_found( get_token: Callable[[str, str], str], insert_logs: Callable[[list[Logs]], None], ) -> None: - """A filter on a key that exists in no context is rejected (400). - - NOTE: reflects current HEAD behavior. The parked `convert-not-found-to-warning` - change will turn this into a 200 with a warning — update this assertion when it lands. - """ + """A filter on a key that exists in no context is synthesized: the query succeeds + (200) with a key-not-found warning, so typos still surface.""" now = datetime.now(tz=UTC) insert_logs( [ @@ -138,4 +135,6 @@ def test_logs_filter_key_not_found( } ], ) - assert response.status_code == HTTPStatus.BAD_REQUEST + assert response.status_code == HTTPStatus.OK, response.text + messages = [w.get("message", "") for w in get_all_warnings(response.json())] + assert any("totally.unknown.key" in m and "not found" in m for m in messages), f"expected a key-not-found warning, got warnings={messages}" diff --git a/tests/integration/tests/querierlogs/07_list_expectations.py b/tests/integration/tests/querierlogs/07_list_expectations.py index 2d674d200bf..a516591dd02 100644 --- a/tests/integration/tests/querierlogs/07_list_expectations.py +++ b/tests/integration/tests/querierlogs/07_list_expectations.py @@ -259,11 +259,12 @@ limit=1, order=[OrderBy(TelemetryFieldKey("attribute.timestamp"), "desc")], ), - lambda x: [], + lambda x: _flatten_log(x[0]), id="no-select-order-attr-timestamp-desc", - # Behaviour: [BUG] - # AdjustKeys logic adjusts key "attribute.timestamp" to "attribute.timestamp:string" - # Because of aliasing bug, result is empty + # Behaviour: + # Order by attribute.timestamp resolves to attributes_string['timestamp']. + # x[0] and x[3] share "corrupt_data" (x[1]/x[2] are unset -> ""); desc puts the + # "corrupt_data" rows first, x[0] returned by storage order. ), pytest.param( BuilderQuery( @@ -304,12 +305,12 @@ order=[OrderBy(TelemetryFieldKey("attribute.timestamp"), "desc")], limit=1, ), - lambda x: [], # Because of aliasing bug, this returns no data + lambda x: [x[0].id, x[0].timestamp], id="select-attr-timestamp-order-attr-timestamp-desc", - # Behaviour [BUG - user didn't get what they expected]: - # AdjustKeys logic adjusts key "attribute.timestamp" to "attribute.timestamp:string" - # Logs stmt builder by default adds timestamp field to select fields and ignores user input timestamp field - # Because of Logs stmt builder behaviour, we ran into aliasing bug, result is empty + # Behaviour: + # Order by attribute.timestamp uses attributes_string['timestamp']; x[0]/x[3] tie + # on "corrupt_data" and x[0] is returned. The attribute.timestamp select collapses + # to the intrinsic timestamp, so the row projects {id, timestamp}. ), pytest.param( BuilderQuery( @@ -540,17 +541,16 @@ def test_logs_list_query_timestamp_expectations( order=[OrderBy(TelemetryFieldKey("attribute.trace_id"), "desc")], ), lambda x: [ - [*_flatten_log(x[1])[:14], x[1].attributes_string.get("trace_id", "")], - [*_flatten_log(x[2])[:14], x[2].attributes_string.get("trace_id", "")], - [*_flatten_log(x[0])[:14], x[0].attributes_string.get("trace_id", "")], - [*_flatten_log(x[3])[:14], x[3].attributes_string.get("trace_id", "")], + _flatten_log(x[1]), + _flatten_log(x[2]), + _flatten_log(x[0]), + _flatten_log(x[3]), ], id="no-select-attribute-trace-id-order", # Justification (expected values and row order): - # attribute.trace_id values: x[0]="", x[1]="2", x[2]="", x[3]="" - # Behaviour: [BUG - user didn't get what they expected] - # AdjustKeys adjusts "attribute.trace_id" to "attribute.trace_id:string" - # Order by attribute.trace_id maps to attributes_string['trace_id'] + # Order by attribute.trace_id -> attributes_string['trace_id']: x[1]="2" first, + # then x[2]/x[0]/x[3] (unset) in storage order. The projected 'trace_id' field is + # the intrinsic column (x[1]=""); the attribute "2" lives in attributes_string. ), pytest.param( BuilderQuery( @@ -580,19 +580,17 @@ def test_logs_list_query_timestamp_expectations( order=[OrderBy(TelemetryFieldKey("attribute.trace_id"), "desc")], ), lambda x: [ - [x[1].id, x[1].timestamp, x[1].attributes_string.get("trace_id", "")], - [x[2].id, x[2].timestamp, x[2].attributes_string.get("trace_id", "")], - [x[0].id, x[0].timestamp, x[0].attributes_string.get("trace_id", "")], - [x[3].id, x[3].timestamp, x[3].attributes_string.get("trace_id", "")], + [x[1].id, x[1].timestamp, x[1].attributes_string.get("trace_id", None)], + [x[2].id, x[2].timestamp, x[2].attributes_string.get("trace_id", None)], + [x[0].id, x[0].timestamp, x[0].attributes_string.get("trace_id", None)], + [x[3].id, x[3].timestamp, x[3].attributes_string.get("trace_id", None)], ], id="select-attribute-trace-id-order-attribute-trace-id-desc", # Justification (expected values and row order): # AdjustKeys: no-op for both select and order, "attribute.trace_id" is a valid attribute key - # Field mapping: "attribute.trace_id" → attributes_string["trace_id"] - # Values: x[0]="", x[1]="2", x[2]="", x[3]="" (only x[1] has attribute.trace_id set) - # Order: attribute.trace_id DESC → x[1]("2") first, then x[2](""), x[0](""), x[3]("") in storage order - # Behaviour: - # AdjustKeys no-op + # Field mapping: attribute.trace_id → multiIf(mapContains(attributes_string,'trace_id'), attributes_string['trace_id'], NULL) + # Values: x[1]="2"; x[0]/x[2]/x[3] are absent → None (map-select exists-guard distinguishes absent from "") + # Order: attribute.trace_id DESC → x[1]("2") first, then x[2], x[0], x[3] (absent) in storage order ), pytest.param( BuilderQuery( diff --git a/tests/integration/tests/querierlogs/09_json_body_functions.py b/tests/integration/tests/querierlogs/09_json_body_functions.py index e2ed902fe5b..faea6fe8dd5 100644 --- a/tests/integration/tests/querierlogs/09_json_body_functions.py +++ b/tests/integration/tests/querierlogs/09_json_body_functions.py @@ -362,11 +362,13 @@ def test_logs_json_body_function_scalar_path( @pytest.mark.parametrize( - "expression", + "expression, expected_error", [ - pytest.param('has(code.function, "main")', id="has_non_body_key"), - pytest.param('hasToken(code.function, "main")', id="hastoken_non_body_key"), - pytest.param("hasToken(body, 123)", id="hastoken_non_string_value"), + # A has-family function on a non-body key is unsupported (not merely "key not found"): + # the error names the body-only restriction so the user knows to prefix with `body.`. + pytest.param('has(code.function, "main")', "function `has` supports only body JSON search", id="has_non_body_key"), + pytest.param('hasToken(code.function, "main")', "function `hasToken` only supports body field as first parameter", id="hastoken_non_body_key"), + pytest.param("hasToken(body, 123)", "expects value parameter to be a string", id="hastoken_non_string_value"), ], ) def test_logs_json_body_function_errors( @@ -374,8 +376,10 @@ def test_logs_json_body_function_errors( create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], expression: str, + expected_error: str, ) -> None: - """has/hasToken support only the body JSON field; misuse is rejected (400).""" + """has/hasToken support only the body JSON field; misuse is rejected (400) with a message that + names the body-only restriction rather than a generic "key not found".""" now = datetime.now(tz=UTC) token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) response = make_query_request( @@ -386,7 +390,8 @@ def test_logs_json_body_function_errors( request_type=RequestType.RAW, queries=[build_raw_query("A", "logs", order=[build_order_by("timestamp")], limit=100, filter_expression=expression)], ) - assert response.status_code == HTTPStatus.BAD_REQUEST + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + assert expected_error in response.text, f"{expression}: expected {expected_error!r} in {response.text}" @pytest.mark.parametrize( @@ -1174,3 +1179,100 @@ def test_logs_json_body_quote_unicode_needles( ) assert response.status_code == HTTPStatus.OK, f"{expression}: {response.text}" assert len(get_rows(response)) == 1, f"{expression}: expected 1 row" + + +@pytest.mark.parametrize( + "needle", + [ + pytest.param("192.168.1.1", id="dotted_ip"), + pytest.param("user_id", id="underscore"), + pytest.param("error-code", id="hyphen"), + pytest.param("production node", id="whitespace"), + ], +) +def test_logs_json_body_hastoken_separator_needle_errors( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], + needle: str, +) -> None: + """KNOWN BUG (currently 500). hasToken maps to ClickHouse hasToken(LOWER(body), LOWER(?)), + which rejects a needle containing whitespace or separator characters (`.`/`_`/`-`/space) with + error code 36 ("Needle must not contain whitespace or separator characters"). The querier + passes the user needle straight through, so the query fails during execution and the raw CH + error surfaces as a 500 rather than a 400 / graceful fallback. Common needles like an IP, + `user_id` or a UUID hit this. Flip this to 400 (or a fallback match) once the needle is + validated.""" + now = datetime.now(tz=UTC) + insert_logs( + [ + Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": "app-service"}, body=json.dumps({"message": "client 192.168.1.1 user_id error-code production node"}), severity_text="INFO"), + ] + ) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(seconds=10)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type=RequestType.RAW, + queries=[build_raw_query("A", "logs", order=[build_order_by("timestamp")], limit=100, filter_expression=f"hasToken(body, '{needle}')")], + ) + assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR, f"{needle}: expected 500, got {response.status_code}: {response.text}" + assert "Needle must not contain whitespace or separator characters" in response.text, response.text + + +@pytest.mark.parametrize( + "func", + [ + pytest.param("hasAny", id="hasany"), + pytest.param("hasAll", id="hasall"), + ], +) +def test_logs_json_body_hasany_hasall_large_quoted_int_errors( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], + func: str, +) -> None: + """KNOWN BUG (currently 500). A quoted integer needle >= 2^32 in hasAny/hasAll fails with + ClickHouse code 386 ("no supertype for types UInt64, Int64") -> 500. The body array is + extracted as Array(Nullable(Int64)), but the needle array is bound as Array(UInt64) for values + that don't fit UInt32, and CH has no Int64/UInt64 supertype at the array-vs-array type level. + Single has() is unaffected because scalar-vs-array coercion is value-level (asserted below as a + contrast). When fixed (bind the needle array as Int64) this should return 200 with an exact + match like has().""" + now = datetime.now(tz=UTC) + insert_logs( + [ + Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": "app-service"}, body=json.dumps({"id": [9007199254740993]}), severity_text="INFO"), + ] + ) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms = int((now - timedelta(seconds=10)).timestamp() * 1000) + end_ms = int(now.timestamp() * 1000) + + # hasAny/hasAll with a quoted big int (>= 2^32) -> 500 (no supertype). + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[build_raw_query("A", "logs", order=[build_order_by("timestamp")], limit=100, filter_expression=f"{func}(body.id, ['9007199254740993'])")], + ) + assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR, f"{func}: expected 500, got {response.status_code}: {response.text}" + + # Contrast: single has() with the same quoted big int works (exact Int64 match, 1 row). + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[build_raw_query("A", "logs", order=[build_order_by("timestamp")], limit=100, filter_expression="has(body.id, '9007199254740993')")], + ) + assert response.status_code == HTTPStatus.OK, response.text + assert len(get_rows(response)) == 1, "single has() with a quoted big int should match exactly one log" diff --git a/tests/integration/tests/querierlogs/10_filter_operators.py b/tests/integration/tests/querierlogs/10_filter_operators.py new file mode 100644 index 00000000000..7672382cdfc --- /dev/null +++ b/tests/integration/tests/querierlogs/10_filter_operators.py @@ -0,0 +1,133 @@ +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http import HTTPStatus + +import pytest + +from fixtures import types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.logs import Logs +from fixtures.querier import get_column_data_from_response, make_query_request + +# Filter-operator coverage for logs, modelled on the shapes that dominate real +# dashboard/alert traffic. The logs counterpart of queriertraces/10_filter_operators.py: +# `=` + `AND` + `IN` compound filters, CONTAINS/NOT CONTAINS, LIKE/ILIKE/NOT LIKE, +# REGEXP, `!=` (incl. the `!= ''` non-empty idiom), numeric comparisons, nested OR, +# and EXISTS/NOT EXISTS. Logs-specific: the text operators run against the `body` +# column (full-text) rather than an intrinsic name column, and there is no calculated +# response_status_code, so numeric comparisons use plain number attributes. +# +# Unknown-key rejection is out of scope: logs has no synthesize-on-unknown-key +# fallback (unlike traces), so a bad key is a 400 — already covered by +# queriercommon/04_filter_operators.py::test_logs_filter_key_not_found. + + +@pytest.mark.parametrize( + "expression,expected_bodies", + [ + # ── IN / NOT IN (the #1 production pattern by traffic) ────────────────── + pytest.param("resource.service.name IN ['checkout', 'cart']", {"checkout-handler", "cart-handler"}, id="in_bracket_list"), + pytest.param("resource.service.name IN ('checkout', 'cart')", {"checkout-handler", "cart-handler"}, id="in_paren_list"), + pytest.param("resource.service.name NOT IN ['checkout', 'cart']", {"payment-processor", "search-service", "notify-worker"}, id="not_in"), + # ── Compound AND + IN, the dominant real-world shape ──────────────────── + pytest.param("resource.deployment.environment = 'prod' AND resource.service.name IN ['checkout', 'search']", {"checkout-handler", "search-service"}, id="and_eq_in"), + # ── Nested OR within AND ──────────────────────────────────────────────── + pytest.param("resource.deployment.environment = 'prod' AND (resource.service.name = 'cart' OR resource.service.name = 'search')", {"cart-handler", "search-service"}, id="and_nested_or"), + # ── Inequality, incl. the `!= ''` non-empty idiom ─────────────────────── + pytest.param("http.request.method != 'GET'", {"cart-handler", "search-service"}, id="not_equal"), + pytest.param("error.type != ''", {"payment-processor"}, id="not_equal_empty"), + # ── CONTAINS / NOT CONTAINS (full-text over body) ─────────────────────── + pytest.param("body CONTAINS 'handler'", {"checkout-handler", "cart-handler"}, id="contains"), + pytest.param("body NOT CONTAINS 'handler'", {"payment-processor", "search-service", "notify-worker"}, id="not_contains"), + # ── LIKE / ILIKE / NOT LIKE ───────────────────────────────────────────── + pytest.param("body LIKE '%processor'", {"payment-processor"}, id="like"), + pytest.param("body ILIKE '%HANDLER'", {"checkout-handler", "cart-handler"}, id="ilike"), + pytest.param("body NOT LIKE '%handler'", {"payment-processor", "search-service", "notify-worker"}, id="not_like"), + # ── REGEXP / NOT REGEXP ───────────────────────────────────────────────── + pytest.param("body REGEXP '^cart'", {"cart-handler"}, id="regexp"), + pytest.param("body NOT REGEXP 'handler'", {"payment-processor", "search-service", "notify-worker"}, id="not_regexp"), + # ── Numeric comparisons over number attributes ────────────────────────── + pytest.param("resp_code >= 400", {"cart-handler", "payment-processor"}, id="gte_number_attr"), + pytest.param("resp_code < 300", {"checkout-handler", "search-service", "notify-worker"}, id="lt_number_attr"), + pytest.param("latency_ms > 400", {"payment-processor", "notify-worker"}, id="gt_number_attr"), + pytest.param("latency_ms <= 100", {"checkout-handler", "search-service"}, id="lte_number_attr"), + # ── EXISTS / NOT EXISTS ───────────────────────────────────────────────── + pytest.param("error.type exists", {"payment-processor"}, id="exists"), + pytest.param("error.type not exists", {"checkout-handler", "cart-handler", "search-service", "notify-worker"}, id="not_exists"), + # ── Exact body equality (logs-specific) ───────────────────────────────── + pytest.param("body = 'checkout-handler'", {"checkout-handler"}, id="body_equals"), + ], +) +def test_logs_filter_operators( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], + expression: str, + expected_bodies: set[str], +) -> None: + """ + Setup: + Five logs across services / environments / methods, each body naming itself, + with numeric `resp_code` / `latency_ms` attributes and an `error.type` + attribute present on one. + + Tests: + Each production-shaped filter operator selects exactly the expected logs. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + # (body, service, deployment.environment, http.request.method, resp_code, latency_ms, has_error_type) + specs = [ + ("checkout-handler", "checkout", "prod", "GET", 200, 100, False), + ("cart-handler", "cart", "prod", "POST", 404, 250, False), + ("payment-processor", "payment", "staging", "GET", 500, 500, True), + ("search-service", "search", "prod", "DELETE", 201, 50, False), + ("notify-worker", "notify", "dev", "GET", 200, 1000, False), + ] + logs = [] + for i, (body, service, env, method, resp_code, latency, has_error_type) in enumerate(specs): + attributes = {"http.request.method": method, "resp_code": resp_code, "latency_ms": latency} + if has_error_type: + attributes["error.type"] = "timeout" + logs.append( + Logs( + timestamp=now - timedelta(seconds=i + 1), + resources={"service.name": service, "deployment.environment": env}, + attributes=attributes, + body=body, + ) + ) + insert_logs(logs) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type="raw", + queries=[ + { + "type": "builder_query", + "spec": { + "name": "A", + "signal": "logs", + "disabled": False, + "limit": 100, + "offset": 0, + "filter": {"expression": expression}, + "order": [ + {"key": {"name": "timestamp"}, "direction": "desc"}, + {"key": {"name": "id"}, "direction": "desc"}, + ], + "having": {"expression": ""}, + "aggregations": [{"expression": "count()"}], + }, + } + ], + ) + + assert response.status_code == HTTPStatus.OK, response.text + assert response.json()["status"] == "success" + assert set(get_column_data_from_response(response.json(), "body")) == expected_bodies diff --git a/tests/integration/tests/querierlogs/11_aggregation_functions.py b/tests/integration/tests/querierlogs/11_aggregation_functions.py new file mode 100644 index 00000000000..783ad175479 --- /dev/null +++ b/tests/integration/tests/querierlogs/11_aggregation_functions.py @@ -0,0 +1,169 @@ +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http import HTTPStatus + +import numpy as np +import pytest + +from fixtures import types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.logs import Logs +from fixtures.querier import ( + build_aggregation, + build_group_by_field, + build_order_by, + build_scalar_query, + get_scalar_table_data, + make_scalar_query_request, +) + +# Numeric aggregation-function coverage for logs — the logs counterpart of +# queriertraces/02_aggregation.py::test_traces_aggregate_functions. Logs querier +# tests elsewhere only ever use count()/count_distinct(); here a grouped scalar +# query computes sum / avg / min / max / p50 / p90 / p95 / p99 / countIf / +# count_distinct over a numeric log attribute and asserts every value. +# +# Log number attributes are stored as Float64, so aggregates come back as floats; +# percentiles are matched with pytest.approx (ClickHouse quantile() and +# numpy.percentile both linear-interpolate, but avoid ULP-level exact equality). + + +def test_logs_aggregate_functions( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], +) -> None: + """ + Setup: + svc-a: 3 logs (latency_ms 10/20/30, endpoints /x /x /y); svc-b: 1 log + (latency_ms 40, endpoint /z). + + Tests: + A grouped scalar query computes count / sum / avg / min / max / p50 / p90 / + p95 / p99 over latency_ms, countIf over a numeric threshold, and + count_distinct over a string attribute — all matching values derived from the + inserted logs, ordered by count() desc. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + # (service, latency_ms, endpoint) + specs = [ + ("svc-a", 10, "/x"), + ("svc-a", 20, "/x"), + ("svc-a", 30, "/y"), + ("svc-b", 40, "/z"), + ] + logs = [ + Logs( + timestamp=now - timedelta(seconds=i + 1), + resources={"service.name": service}, + attributes={"latency_ms": latency, "endpoint": endpoint}, + body=f"{service} log {i}", + ) + for i, (service, latency, endpoint) in enumerate(specs) + ] + insert_logs(logs) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + query = build_scalar_query( + name="A", + signal="logs", + aggregations=[ + build_aggregation("count()", "cnt"), + build_aggregation("sum(latency_ms)", "sum_l"), + build_aggregation("avg(latency_ms)", "avg_l"), + build_aggregation("min(latency_ms)", "min_l"), + build_aggregation("max(latency_ms)", "max_l"), + build_aggregation("p50(latency_ms)", "p50_l"), + build_aggregation("p90(latency_ms)", "p90_l"), + build_aggregation("p95(latency_ms)", "p95_l"), + build_aggregation("p99(latency_ms)", "p99_l"), + build_aggregation("countIf(latency_ms >= 25)", "slow"), + build_aggregation("count_distinct(endpoint)", "endpoints"), + ], + group_by=[build_group_by_field("service.name", "string", "resource")], + order=[build_order_by("count()", "desc")], + ) + response = make_scalar_query_request(signoz, token, now, [query]) + + assert response.status_code == HTTPStatus.OK, response.text + data = get_scalar_table_data(response.json()) + + # Ordered by count() desc: svc-a (3) then svc-b (1). + assert [row[0] for row in data] == ["svc-a", "svc-b"] + by_service = {row[0]: row[1:] for row in data} + + for service, latencies, endpoints in [ + ("svc-a", [10, 20, 30], ["/x", "/x", "/y"]), + ("svc-b", [40], ["/z"]), + ]: + expected = [ + len(latencies), # count() + sum(latencies), # sum(latency_ms) + sum(latencies) / len(latencies), # avg(latency_ms) + min(latencies), # min(latency_ms) + max(latencies), # max(latency_ms) + float(np.percentile(latencies, 50)), # p50(latency_ms) + float(np.percentile(latencies, 90)), # p90(latency_ms) + float(np.percentile(latencies, 95)), # p95(latency_ms) + float(np.percentile(latencies, 99)), # p99(latency_ms) + sum(1 for latency in latencies if latency >= 25), # countIf(latency_ms >= 25) + len(set(endpoints)), # count_distinct(endpoint) + ] + assert by_service[service] == pytest.approx(expected), f"{service}: {by_service[service]} != {expected}" + + +@pytest.mark.parametrize( + "having_expression,aggregations,expected_services", + [ + pytest.param("count() > 2", ["count()"], {"svc-a"}, id="count_gt_2"), + pytest.param("count() < 2", ["count()"], {"svc-b"}, id="count_lt_2"), + pytest.param("sum(latency_ms) >= 50", ["count()", "sum(latency_ms)"], {"svc-a"}, id="sum_gte_50"), + ], +) +def test_logs_aggregate_having( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], + having_expression: str, + aggregations: list[str], + expected_services: set[str], +) -> None: + """ + Setup: + svc-a: 3 logs (latency_ms 10/20/30, sum 60); svc-b: 1 log (latency_ms 40). + + Tests: + A grouped scalar query with `having` keeps only the qualifying groups, for a + count() predicate (both directions) and a sum() predicate — extending logs + HAVING coverage beyond the single count() case in querierscalar/04_having.py. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + specs = [("svc-a", 10), ("svc-a", 20), ("svc-a", 30), ("svc-b", 40)] + logs = [ + Logs( + timestamp=now - timedelta(seconds=i + 1), + resources={"service.name": service}, + attributes={"latency_ms": latency}, + body=f"{service} log {i}", + ) + for i, (service, latency) in enumerate(specs) + ] + insert_logs(logs) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + query = build_scalar_query( + name="A", + signal="logs", + aggregations=[build_aggregation(expr) for expr in aggregations], + group_by=[build_group_by_field("service.name", "string", "resource")], + having_expression=having_expression, + ) + response = make_scalar_query_request(signoz, token, now, [query]) + + assert response.status_code == HTTPStatus.OK, response.text + data = get_scalar_table_data(response.json()) + assert {row[0] for row in data} == expected_services diff --git a/tests/integration/tests/querierlogs/12_aggregation_timeseries.py b/tests/integration/tests/querierlogs/12_aggregation_timeseries.py new file mode 100644 index 00000000000..195a0ac493a --- /dev/null +++ b/tests/integration/tests/querierlogs/12_aggregation_timeseries.py @@ -0,0 +1,206 @@ +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http import HTTPStatus + +import pytest + +from fixtures import types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.logs import Logs +from fixtures.querier import ( + RequestType, + build_aggregation, + build_group_by_field, + build_order_by, + build_scalar_query, + get_all_series, + get_scalar_table_data, + index_series_by_label, + make_query_request, + make_scalar_query_request, +) + +# Time-series aggregation shapes for logs — the logs counterpart of +# queriertraces/08_aggregation_min_max.py and the time_series limit in +# queriertraces/02_aggregation.py, plus a multi-key group-by (a shape neither +# signal covered before). Logs querier tests elsewhere only exercise single-key +# grouped counts, so this file adds min/max over a numeric attribute, top-N / +# bottom-N series limiting, and a two-key group-by cross product. + + +def test_logs_aggregate_min_max( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], +) -> None: + """ + Setup: + svc-a: 3 logs (latency_ms 10/20/30); svc-b: 1 log (latency_ms 40). + + Tests: + max(latency_ms) / min(latency_ms) as two aggregations in one time_series query + grouped by service.name return each service's latency extremes. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + specs = [("svc-a", 10), ("svc-a", 20), ("svc-a", 30), ("svc-b", 40)] + logs = [ + Logs( + timestamp=now - timedelta(seconds=i + 1), + resources={"service.name": service}, + attributes={"latency_ms": latency}, + body=f"{service} log {i}", + ) + for i, (service, latency) in enumerate(specs) + ] + insert_logs(logs) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int((now + timedelta(seconds=5)).timestamp() * 1000), + request_type=RequestType.TIME_SERIES, + queries=[ + build_scalar_query( + name="A", + signal="logs", + aggregations=[ + build_aggregation("max(latency_ms)", "maxLatency"), + build_aggregation("min(latency_ms)", "minLatency"), + ], + group_by=[build_group_by_field("service.name", "string", "resource")], + ) + ], + ) + + assert response.status_code == HTTPStatus.OK, response.text + aggregations = response.json()["data"]["data"]["results"][0]["aggregations"] + max_by_svc = index_series_by_label(aggregations[0]["series"], "service.name") + min_by_svc = index_series_by_label(aggregations[1]["series"], "service.name") + + latencies_by_service = {"svc-a": [10, 20, 30], "svc-b": [40]} + for service, latencies in latencies_by_service.items(): + assert max_by_svc[service]["values"][0]["value"] == max(latencies) + assert min_by_svc[service]["values"][0]["value"] == min(latencies) + + +@pytest.mark.parametrize( + "limit,direction", + [ + pytest.param(2, "desc", id="top_2_desc"), + pytest.param(3, "desc", id="top_3_desc"), + pytest.param(2, "asc", id="bottom_2_asc"), + pytest.param(10, "desc", id="limit_exceeds_group_count"), + ], +) +def test_logs_aggregate_time_series_limit( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], + limit: int, + direction: str, +) -> None: + """ + Setup: + Four services with distinct log counts (a=5, b=3, c=7, d=1). + + Tests: + A time_series group-by with a limit returns only the N series that the ordered + aggregation keeps — top-N for desc, bottom-N for asc — each series summing to + that service's log count. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + counts = {"svc-a": 5, "svc-b": 3, "svc-c": 7, "svc-d": 1} + logs = [ + Logs( + timestamp=now - timedelta(seconds=i + 1), + resources={"service.name": service}, + body=f"{service} log {i}", + ) + for service, count in counts.items() + for i in range(count) + ] + insert_logs(logs) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type=RequestType.TIME_SERIES, + queries=[ + build_scalar_query( + name="A", + signal="logs", + aggregations=[build_aggregation("count()")], + group_by=[build_group_by_field("service.name", "string", "resource")], + order=[build_order_by("count()", direction)], + limit=limit, + ) + ], + ) + + assert response.status_code == HTTPStatus.OK, response.text + + ordered = sorted(counts.items(), key=lambda kv: kv[1], reverse=(direction == "desc")) + expected = dict(ordered[:limit]) + + by_service = index_series_by_label(get_all_series(response.json(), "A"), "service.name") + assert set(by_service) == set(expected) + for service, series in by_service.items(): + assert sum(point["value"] for point in series["values"]) == expected[service] + + +def test_logs_aggregate_multi_group_by( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], +) -> None: + """ + Setup: + Logs across (service.name, deployment.environment) pairs: + (svc-a, prod)=2, (svc-a, dev)=1, (svc-b, prod)=3. + + Tests: + A scalar query grouped by two resource keys returns one row per present pair + (the cross product, not the full Cartesian) with the correct per-pair count. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + pair_counts = {("svc-a", "prod"): 2, ("svc-a", "dev"): 1, ("svc-b", "prod"): 3} + logs = [ + Logs( + timestamp=now - timedelta(seconds=i + 1), + resources={"service.name": service, "deployment.environment": env}, + body=f"{service}-{env} log {i}", + ) + for (service, env), count in pair_counts.items() + for i in range(count) + ] + insert_logs(logs) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + query = build_scalar_query( + name="A", + signal="logs", + aggregations=[build_aggregation("count()")], + group_by=[ + build_group_by_field("service.name", "string", "resource"), + build_group_by_field("deployment.environment", "string", "resource"), + ], + ) + response = make_scalar_query_request(signoz, token, now, [query]) + + assert response.status_code == HTTPStatus.OK, response.text + data = get_scalar_table_data(response.json()) + + actual = {(row[0], row[1], row[2]) for row in data} + expected = {(service, env, count) for (service, env), count in pair_counts.items()} + assert actual == expected diff --git a/tests/integration/tests/querierlogs/13_unknown_key_collisions.py b/tests/integration/tests/querierlogs/13_unknown_key_collisions.py new file mode 100644 index 00000000000..c04b3a802c0 --- /dev/null +++ b/tests/integration/tests/querierlogs/13_unknown_key_collisions.py @@ -0,0 +1,228 @@ +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http import HTTPStatus + +import pytest + +from fixtures import types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.logs import Logs +from fixtures.querier import ( + RequestType, + build_aggregation, + build_group_by_field, + build_raw_query, + build_scalar_query, + get_rows, + get_scalar_table_data, + make_query_request, + make_scalar_query_request, +) + +# With-metadata vs without-metadata parity for the logs unknown-key (synthesize) +# fallback, focused on the storage-column collision. +# +# Every record carries the SAME value under two attribute names: +# - known_key : registered in field-key metadata -> normal resolution path +# - unknown_key : the metadata rows are stripped after construction, so the querier +# can't resolve it and falls back to the synthesized +# multiIf(attributes_string -> attributes_number -> attributes_bool) +# The value's Python type varies per record, so across the dataset the shared value +# lives in attributes_string / attributes_number / attributes_bool — but in exactly +# one map per record (a record holds a given key in only one physical place). +# +# Querying known_key vs unknown_key must return identical results: the synthesized +# multiIf has to reproduce exactly what metadata-driven resolution produces, no matter +# which type-map a record's value sits in. +# +# Scope: resources / top-level columns are out of scope for this parity — a bare +# synthesized key resolves the attribute maps (+ body) only, so a resource-qualified +# unknown key filters to 0 (covered in 10_unknown_keys.py), which is intentionally NOT +# equal to the metadata-driven resource path. On a stack without the synthesize change +# the unknown_key half is a 400, so these tests are red until that change lands. + +# (body, value) — the value's type decides which attribute map holds it, one per record. +COLLISION_SPECS: list[tuple[str, object]] = [ + ("log-a", "alpha"), # attributes_string + ("log-b", "alpha"), # attributes_string + ("log-c", "beta"), # attributes_string + ("log-d", 42), # attributes_number + ("log-e", 42), # attributes_number + ("log-f", 7), # attributes_number + ("log-g", True), # attributes_bool + ("log-h", False), # attributes_bool +] +SERVICE = "collision-svc" + + +def test_logs_unknown_key_collision_groupby_parity( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs_with_unknown_keys: Callable[[list[Logs], set[str]], None], +) -> None: + """ + Setup: + 8 logs sharing a value under known_key (in metadata) and unknown_key (stripped + from metadata); the value is a string / number / bool depending on the record, so + the shared value spans attributes_string / attributes_number / attributes_bool. + + Tests: + Grouping by unknown_key (synthesized multiIf) yields the same buckets and counts + as grouping by known_key (metadata-driven), across all three attribute maps. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + logs = [ + Logs( + timestamp=now - timedelta(seconds=i + 1), + resources={"service.name": SERVICE}, + attributes={"known_key": value, "unknown_key": value}, + body=body, + ) + for i, (body, value) in enumerate(COLLISION_SPECS) + ] + insert_logs_with_unknown_keys(logs, {"unknown_key"}) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + counts: dict[str, dict[str, int]] = {} + for key in ("known_key", "unknown_key"): + response = make_scalar_query_request( + signoz, + token, + now, + [ + build_scalar_query( + "A", + "logs", + [build_aggregation("count()")], + group_by=[build_group_by_field(key, field_data_type="", field_context="")], + filter_expression=f'service.name = "{SERVICE}"', + ) + ], + ) + assert response.status_code == HTTPStatus.OK, f"{key}: {response.text}" + counts[key] = {row[0]: row[-1] for row in get_scalar_table_data(response.json())} + + assert counts["known_key"] == counts["unknown_key"], f"with-metadata {counts['known_key']} != without-metadata {counts['unknown_key']}" + # Sanity: the collision actually spanned all three attribute maps (string/number/bool). + assert counts["known_key"] == {"alpha": 2, "beta": 1, "42": 2, "7": 1, "true": 1, "false": 1}, counts["known_key"] + + +ALL_BODIES = {body for body, _ in COLLISION_SPECS} + + +@pytest.mark.parametrize( + "predicate,expected_bodies", + [ + pytest.param('{key} = "alpha"', {"log-a", "log-b"}, id="eq_string"), + pytest.param("{key} = 42", {"log-d", "log-e"}, id="eq_number"), + pytest.param("{key} = true", {"log-g"}, id="eq_bool"), + pytest.param("{key} exists", ALL_BODIES, id="exists"), + pytest.param("{key} IN ['alpha', 42]", {"log-a", "log-b", "log-d", "log-e"}, id="in_mixed"), + ], +) +def test_logs_unknown_key_collision_filter_parity( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs_with_unknown_keys: Callable[[list[Logs], set[str]], None], + predicate: str, + expected_bodies: set[str], +) -> None: + """ + Setup: + The same collision dataset as the group-by parity test. + + Tests: + Filtering unknown_key (synthesized) selects the same records as filtering known_key + (metadata-driven) across operators: typed equality (the operand type picks the + matching attribute map), `exists` (fans out across all three maps), and a + mixed-type `IN` list (string + number operands union their maps). + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + logs = [ + Logs( + timestamp=now - timedelta(seconds=i + 1), + resources={"service.name": SERVICE}, + attributes={"known_key": value, "unknown_key": value}, + body=body, + ) + for i, (body, value) in enumerate(COLLISION_SPECS) + ] + insert_logs_with_unknown_keys(logs, {"unknown_key"}) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000) + end_ms = int(now.timestamp() * 1000) + + bodies: dict[str, set[str]] = {} + for key in ("known_key", "unknown_key"): + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[build_raw_query("A", "logs", limit=100, filter_expression=f'service.name = "{SERVICE}" AND ({predicate.format(key=key)})')], + ) + assert response.status_code == HTTPStatus.OK, f"{key}: {response.text}" + bodies[key] = {row["data"]["body"] for row in get_rows(response)} + + assert bodies["known_key"] == bodies["unknown_key"], f"with-metadata {bodies['known_key']} != without-metadata {bodies['unknown_key']}" + # Sanity: metadata-driven resolution selected exactly the expected records. + assert bodies["known_key"] == expected_bodies, bodies["known_key"] + + +def test_logs_unknown_key_collision_aggregation_parity( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs_with_unknown_keys: Callable[[list[Logs], set[str]], None], +) -> None: + """ + Setup: + The same collision dataset as the group-by parity test. + + Tests: + Aggregating over unknown_key (synthesized) matches aggregating over known_key + (metadata-driven): count() sees every record (all 8 carry the key across the three + attribute maps) and count_distinct() sees the 6 distinct stringified values. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + logs = [ + Logs( + timestamp=now - timedelta(seconds=i + 1), + resources={"service.name": SERVICE}, + attributes={"known_key": value, "unknown_key": value}, + body=body, + ) + for i, (body, value) in enumerate(COLLISION_SPECS) + ] + insert_logs_with_unknown_keys(logs, {"unknown_key"}) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + aggregated: dict[str, tuple] = {} + for key in ("known_key", "unknown_key"): + response = make_scalar_query_request( + signoz, + token, + now, + [ + build_scalar_query( + "A", + "logs", + [build_aggregation(f"count({key})", "cnt"), build_aggregation(f"count_distinct({key})", "distinct")], + filter_expression=f'service.name = "{SERVICE}"', + ) + ], + ) + assert response.status_code == HTTPStatus.OK, f"{key}: {response.text}" + data = get_scalar_table_data(response.json()) + assert len(data) == 1, f"{key}: expected a single ungrouped row, got {data}" + aggregated[key] = tuple(data[0]) + + assert aggregated["known_key"] == aggregated["unknown_key"], f"with-metadata {aggregated['known_key']} != without-metadata {aggregated['unknown_key']}" + # Sanity: count() over every record, count_distinct() over the 6 distinct values. + assert aggregated["known_key"] == (8, 6), aggregated["known_key"] diff --git a/tests/integration/tests/querierlogs/14_select_fields.py b/tests/integration/tests/querierlogs/14_select_fields.py new file mode 100644 index 00000000000..5ee767b6505 --- /dev/null +++ b/tests/integration/tests/querierlogs/14_select_fields.py @@ -0,0 +1,212 @@ +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http import HTTPStatus +from typing import Any + +import pytest + +from fixtures import types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.logs import Logs +from fixtures.querier import BuilderQuery, OrderBy, TelemetryFieldKey, get_all_warnings, get_rows, make_query_request + +# Select-field projection for logs — the logs counterpart of the traces +# select-field tests (queriertraces/01_list.py). A raw logs query with no +# selectFields returns the whole row (body + the resources_string / attributes_* +# maps); projecting specific fields instead returns each one flattened to a +# top-level key by its short name, with its value type preserved. + +SERVICE = "select-svc" + + +@pytest.mark.parametrize( + "select_name,key,expected,expected_type", + [ + pytest.param("string_attr", "string_attr", "hello", str, id="string_attr"), + pytest.param("number_attr", "number_attr", 42.5, float, id="number_attr"), + pytest.param("bool_attr", "bool_attr", True, bool, id="bool_attr"), + pytest.param("severity_text", "severity_text", "ERROR", str, id="intrinsic_severity_text"), + ], +) +def test_logs_list_select_field_types( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], + select_name: str, + key: str, + expected: Any, + expected_type: type, +) -> None: + """ + Setup: + One log carrying typed attributes (string / number / bool) and severity ERROR. + + Tests: + Projecting a string / number / bool attribute or the severity_text intrinsic + returns the value with the right Python type under its short-name key. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + insert_logs( + [ + Logs( + timestamp=now - timedelta(seconds=1), + resources={"service.name": SERVICE}, + attributes={"string_attr": "hello", "number_attr": 42.5, "bool_attr": True}, + body="select test", + severity_text="ERROR", + ) + ] + ) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type="raw", + queries=[ + BuilderQuery( + signal="logs", + name="A", + limit=10, + filter_expression=f'service.name = "{SERVICE}"', + select_fields=[TelemetryFieldKey(select_name)], + order=[OrderBy(TelemetryFieldKey("timestamp"), "desc")], + ).to_dict() + ], + ) + + assert response.status_code == HTTPStatus.OK, response.text + rows = get_rows(response) + assert len(rows) == 1 + data = rows[0]["data"] + assert data[key] == expected + assert isinstance(data[key], expected_type) + + +def test_logs_list_select_projects_columns( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], +) -> None: + """ + Setup: + One log with a string attribute plus body and resources. + + Tests: + Projecting a single field returns it as a top-level key and drops the default + row payload — the resources_string / attributes_* maps and body are no longer + present when a selectFields list is given. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + insert_logs( + [ + Logs( + timestamp=now - timedelta(seconds=1), + resources={"service.name": SERVICE}, + attributes={"string_attr": "hello"}, + body="select test", + ) + ] + ) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type="raw", + queries=[ + BuilderQuery( + signal="logs", + name="A", + limit=10, + filter_expression=f'service.name = "{SERVICE}"', + select_fields=[TelemetryFieldKey("string_attr")], + order=[OrderBy(TelemetryFieldKey("timestamp"), "desc")], + ).to_dict() + ], + ) + + assert response.status_code == HTTPStatus.OK, response.text + rows = get_rows(response) + assert len(rows) == 1 + data = rows[0]["data"] + assert data["string_attr"] == "hello" + # Projection drops the default full-row payload. + assert "attributes_string" not in data + assert "resources_string" not in data + + +@pytest.mark.parametrize("surface", ["filter", "select", "order"]) +def test_logs_list_unknown_log_context_synthesizes( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], + surface: str, +) -> None: + """ + Setup: + One log. + + Tests: + The intrinsic `log.` context is forgiving, like bare / `attribute.` / + `resource.` keys (and the traces `span.` context): an unknown key there + synthesizes against the log attribute maps (and the body path) rather than + erroring, because existence is a property of the data, not of metadata. A + filter reference matches nothing and surfaces a not-found warning (under the + stripped name); a select projects a null column; an order clause succeeds + and returns the log. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + service = "log-forgiving-ctx-svc" + insert_logs([Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": service}, body="forgiving ctx")]) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000) + end_ms = int(now.timestamp() * 1000) + key = "log.does_not_exist" + svc = f'service.name = "{service}"' + + if surface == "filter": + query = BuilderQuery(signal="logs", name="A", limit=10, filter_expression=f"{svc} AND {key} = 'nope'") + elif surface == "select": + query = BuilderQuery( + signal="logs", + name="A", + limit=10, + filter_expression=svc, + select_fields=[TelemetryFieldKey(key)], + order=[OrderBy(TelemetryFieldKey("timestamp"), "desc")], + ) + else: # order + query = BuilderQuery(signal="logs", name="A", limit=10, filter_expression=svc, order=[OrderBy(TelemetryFieldKey(key), "desc")]) + + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type="raw", + queries=[query.to_dict()], + ) + + assert response.status_code == HTTPStatus.OK, response.text + rows = get_rows(response) + + if surface == "filter": + assert rows == [] + messages = [w.get("message", "") for w in get_all_warnings(response.json())] + assert any("does_not_exist" in m and "not found" in m for m in messages), messages + elif surface == "select": + assert len(rows) == 1 + assert rows[0]["data"]["does_not_exist"] is None + else: # order + assert len(rows) == 1 diff --git a/tests/integration/tests/queriermetrics/10_key_resolution.py b/tests/integration/tests/queriermetrics/10_key_resolution.py new file mode 100644 index 00000000000..db7f24f5e80 --- /dev/null +++ b/tests/integration/tests/queriermetrics/10_key_resolution.py @@ -0,0 +1,210 @@ +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http import HTTPStatus + +from fixtures import querier, types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.metrics import Metrics + +# Metrics key resolution differs from logs/traces: every label lives in the `labels` +# JSON, so attribute./resource./scope. and a bare key all resolve to +# JSONExtractString(labels, '') — there is no per-context storage and no +# attribute-map synthesize fallback. These tests pin that model down. +METRIC = "test.metric.keyres" + + +def test_metrics_group_by_context_collapse( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_metrics: Callable[[list[Metrics]], None], +) -> None: + """Grouping by `region` as attribute / resource / scope / unspecified all resolve to + JSONExtractString(labels,'region') and produce identical buckets — label context is a no-op.""" + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + insert_metrics( + [ + Metrics( + metric_name=METRIC, + labels={"region": region}, + timestamp=now - timedelta(seconds=1), + temporality="Unspecified", + type_="Gauge", + is_monotonic=False, + value=value, + ) + for region, value in [("us", 30.0), ("eu", 10.0)] + ] + ) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + results: dict[str, dict] = {} + for ctx in ("attribute", "resource", "scope", ""): + response = querier.make_scalar_query_request( + signoz, + token, + now, + [ + querier.build_scalar_query( + name="A", + signal="metrics", + aggregations=[querier.build_metrics_aggregation(METRIC, "latest", "sum", "unspecified")], + group_by=[querier.build_group_by_field("region", "string", ctx)], + ) + ], + ) + assert response.status_code == HTTPStatus.OK, f"ctx={ctx!r}: {response.text}" + results[ctx] = {row[0]: row[-1] for row in querier.get_scalar_table_data(response.json())} + + assert results[""] == {"us": 30.0, "eu": 10.0}, results[""] + assert all(r == results[""] for r in results.values()), results + + +def test_metrics_filter_label_context( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_metrics: Callable[[list[Metrics]], None], +) -> None: + """Unlike group-by (which collapses every context to labels), a label *filter* resolves via + metadata under the label's registered (attribute) context: bare `region` and `attribute.region` + are equivalent, but an explicit mismatched context (`resource.region`) is not found (400).""" + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + insert_metrics( + [ + Metrics( + metric_name=METRIC, + labels={"region": region}, + timestamp=now - timedelta(seconds=1), + temporality="Unspecified", + type_="Gauge", + is_monotonic=False, + value=value, + ) + for region, value in [("us", 30.0), ("eu", 10.0)] + ] + ) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + # bare and attribute. both resolve to the attribute-registered label and select only `us`. + for expr in ('region = "us"', 'attribute.region = "us"'): + response = querier.make_scalar_query_request( + signoz, + token, + now, + [ + querier.build_scalar_query( + name="A", + signal="metrics", + aggregations=[querier.build_metrics_aggregation(METRIC, "latest", "sum", "unspecified")], + group_by=[querier.build_group_by_field("region", "string", "")], + filter_expression=expr, + ) + ], + ) + assert response.status_code == HTTPStatus.OK, f"{expr}: {response.text}" + data = {row[0]: row[-1] for row in querier.get_scalar_table_data(response.json())} + assert data == {"us": 30.0}, f"{expr}: {data}" + + # resource. is a context the label is not registered under -> hard "not found". + response = querier.make_scalar_query_request( + signoz, + token, + now, + [ + querier.build_scalar_query( + name="A", + signal="metrics", + aggregations=[querier.build_metrics_aggregation(METRIC, "latest", "sum", "unspecified")], + filter_expression='resource.region = "us"', + ) + ], + ) + assert response.status_code == HTTPStatus.BAD_REQUEST, response.text + + +def test_metrics_group_by_unknown_label( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_metrics: Callable[[list[Metrics]], None], +) -> None: + """Grouping by a label no metric carries resolves to JSONExtractString(labels,'') + = '' for every series, so all series collapse into one bucket rather than the query failing.""" + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + insert_metrics( + [ + Metrics( + metric_name=METRIC, + labels={"region": region}, + timestamp=now - timedelta(seconds=1), + temporality="Unspecified", + type_="Gauge", + is_monotonic=False, + value=value, + ) + for region, value in [("us", 30.0), ("eu", 10.0)] + ] + ) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = querier.make_scalar_query_request( + signoz, + token, + now, + [ + querier.build_scalar_query( + name="A", + signal="metrics", + aggregations=[querier.build_metrics_aggregation(METRIC, "latest", "sum", "unspecified")], + group_by=[querier.build_group_by_field("does_not_exist_label", "string", "attribute")], + ) + ], + ) + assert response.status_code == HTTPStatus.OK, response.text + data = querier.get_scalar_table_data(response.json()) + assert len(data) == 1, data + assert data[0][-1] == 40.0, data # 30 + 10 summed into the single (empty) bucket + + +def test_metrics_filter_unknown_label_matches_nothing( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_metrics: Callable[[list[Metrics]], None], +) -> None: + """A filter on a label no metric carries resolves to JSONExtractString(labels,'') + = '' and matches nothing: metrics returns 200 with an empty result and — unlike the + logs/traces synthesize path — emits no key-not-found warning.""" + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + insert_metrics( + [ + Metrics( + metric_name=METRIC, + labels={"region": "us"}, + timestamp=now - timedelta(seconds=1), + temporality="Unspecified", + type_="Gauge", + is_monotonic=False, + value=30.0, + ) + ] + ) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = querier.make_scalar_query_request( + signoz, + token, + now, + [ + querier.build_scalar_query( + name="A", + signal="metrics", + aggregations=[querier.build_metrics_aggregation(METRIC, "latest", "sum", "unspecified")], + filter_expression='does_not_exist_label = "x"', + ) + ], + ) + assert response.status_code == HTTPStatus.OK, response.text + assert querier.get_scalar_table_data(response.json()) == [] + assert querier.get_all_warnings(response.json()) == [] diff --git a/tests/integration/tests/querierscalar/05_reduce_to.py b/tests/integration/tests/querierscalar/05_reduce_to.py index 73ba94353aa..2f119423958 100644 --- a/tests/integration/tests/querierscalar/05_reduce_to.py +++ b/tests/integration/tests/querierscalar/05_reduce_to.py @@ -2,6 +2,8 @@ from datetime import UTC, datetime, timedelta from http import HTTPStatus +import pytest + from fixtures import types from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD from fixtures.metrics import Metrics @@ -15,17 +17,33 @@ ) # Metric scalar `reduceTo`: collapse a time series to a single value for a value/table -# panel. Metrics scalar group-by is covered by 03_metrics.py, but reduceTo (last/sum/ -# avg/min/max) is not exercised anywhere. +# panel. Metrics scalar group-by is covered by 03_metrics.py; this file exercises the +# reduceTo modes (last / sum / avg / min / max / count / median), of which only sum was +# previously tested. -def test_metrics_scalar_reduce_to_sum( +@pytest.mark.parametrize( + "reduce_to,expected", + [ + pytest.param("last", 30.0, id="last"), + pytest.param("sum", 60.0, id="sum"), + pytest.param("avg", 20.0, id="avg"), + pytest.param("min", 10.0, id="min"), + pytest.param("max", 30.0, id="max"), + pytest.param("count", 3.0, id="count"), + pytest.param("median", 20.0, id="median"), + ], +) +def test_metrics_scalar_reduce_to_modes( signoz: types.SigNoz, create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], insert_metrics: Callable[[list[Metrics]], None], + reduce_to: str, + expected: float, ) -> None: - """reduceTo=sum collapses a series' per-step points (10, 20, 30) to 60.""" + """reduceTo collapses a series' per-step points (10, 20, 30) to a single value: + last=30, sum=60, avg=20, min=10, max=30, count=3, median=20.""" now = datetime.now(tz=UTC).replace(second=0, microsecond=0) # same metric_name + labels => one fingerprint/series; three samples in three # distinct 60s step buckets so latest-per-step yields 10, 20, 30. @@ -39,7 +57,7 @@ def test_metrics_scalar_reduce_to_sum( token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) aggregation = build_metrics_aggregation("test.reduce.metric", "latest", "sum", "unspecified") - aggregation["reduceTo"] = "sum" + aggregation["reduceTo"] = reduce_to query = build_scalar_query( name="A", signal="metrics", @@ -48,7 +66,7 @@ def test_metrics_scalar_reduce_to_sum( ) response = make_scalar_query_request(signoz, token, now, [query]) - assert response.status_code == HTTPStatus.OK + assert response.status_code == HTTPStatus.OK, response.text assert response.json()["status"] == "success" data = get_scalar_table_data(response.json()) - assert_scalar_result_order(data, [("service-a", 60.0)], "metrics reduceTo=sum") + assert_scalar_result_order(data, [("service-a", expected)], f"metrics reduceTo={reduce_to}") diff --git a/tests/integration/tests/querierscalar/06_metrics_aggregations.py b/tests/integration/tests/querierscalar/06_metrics_aggregations.py new file mode 100644 index 00000000000..e5badfa5d96 --- /dev/null +++ b/tests/integration/tests/querierscalar/06_metrics_aggregations.py @@ -0,0 +1,108 @@ +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http import HTTPStatus + +import pytest + +from fixtures import types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.metrics import Metrics +from fixtures.querier import ( + build_group_by_field, + build_metrics_aggregation, + build_scalar_query, + get_scalar_table_data, + make_scalar_query_request, +) + +# Metrics scalar breadth beyond 03_metrics.py (which only uses space_aggregation=sum) +# and 04_having.py (logs only): the other space aggregations (avg / min / max / count) +# and HAVING on a grouped metrics query. + +METRIC_VALUES = {"service-a": 50.0, "service-b": 30.0, "service-c": 70.0, "service-d": 10.0} + + +@pytest.mark.parametrize( + "space_aggregation,expected", + [ + pytest.param("sum", 160.0, id="sum"), + pytest.param("avg", 40.0, id="avg"), + pytest.param("min", 10.0, id="min"), + pytest.param("max", 70.0, id="max"), + pytest.param("count", 4.0, id="count"), + ], +) +def test_metrics_scalar_space_aggregations( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_metrics: Callable[[list[Metrics]], None], + space_aggregation: str, + expected: float, +) -> None: + """ + Setup: + Four gauge series with latest values 50 / 30 / 70 / 10. + + Tests: + An ungrouped scalar query combines the four series with the given space + aggregation: sum=160, avg=40, min=10, max=70, count=4. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + insert_metrics([Metrics(metric_name="test.metric", labels={"service.name": service}, timestamp=now - timedelta(seconds=1), temporality="Unspecified", type_="Gauge", is_monotonic=False, value=value) for service, value in METRIC_VALUES.items()]) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + query = build_scalar_query( + name="A", + signal="metrics", + aggregations=[build_metrics_aggregation("test.metric", "latest", space_aggregation, "unspecified")], + ) + response = make_scalar_query_request(signoz, token, now, [query]) + + assert response.status_code == HTTPStatus.OK, response.text + assert response.json()["status"] == "success" + data = get_scalar_table_data(response.json()) + assert len(data) == 1, f"expected a single ungrouped row, got {data}" + assert data[0][0] == pytest.approx(expected), f"{space_aggregation}: {data[0][0]} != {expected}" + + +@pytest.mark.parametrize( + "having_expression,expected_services", + [ + pytest.param("sum(test.metric) > 40", {"service-a", "service-c"}, id="gt"), + pytest.param("sum(test.metric) < 40", {"service-b", "service-d"}, id="lt"), + ], +) +def test_metrics_scalar_having( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_metrics: Callable[[list[Metrics]], None], + having_expression: str, + expected_services: set[str], +) -> None: + """ + Setup: + Four gauge series (per service.name) with latest values 50 / 30 / 70 / 10. + + Tests: + A grouped metrics scalar query with `having` on the space-aggregated value keeps + only the qualifying services — HAVING was previously logs-only. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + insert_metrics([Metrics(metric_name="test.metric", labels={"service.name": service}, timestamp=now - timedelta(seconds=1), temporality="Unspecified", type_="Gauge", is_monotonic=False, value=value) for service, value in METRIC_VALUES.items()]) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + query = build_scalar_query( + name="A", + signal="metrics", + aggregations=[build_metrics_aggregation("test.metric", "latest", "sum", "unspecified")], + group_by=[build_group_by_field("service.name", "string", "attribute")], + having_expression=having_expression, + ) + response = make_scalar_query_request(signoz, token, now, [query]) + + assert response.status_code == HTTPStatus.OK, response.text + assert response.json()["status"] == "success" + data = get_scalar_table_data(response.json()) + assert {row[0] for row in data} == expected_services diff --git a/tests/integration/tests/querierscalar/07_unknown_keys.py b/tests/integration/tests/querierscalar/07_unknown_keys.py new file mode 100644 index 00000000000..2794f5c2a89 --- /dev/null +++ b/tests/integration/tests/querierscalar/07_unknown_keys.py @@ -0,0 +1,204 @@ +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http import HTTPStatus + +from fixtures import querier, types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.logs import Logs +from fixtures.querier import get_all_warnings, get_scalar_table_data, make_scalar_query_request +from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode + +# Unknown-key resolution in the scalar shape (group-by -> one value per bucket). Logs/traces +# synthesize an unknown key (attribute maps + body). Observed here: the *filter* path emits a +# key-not-found warning and matches nothing (count 0); the *group-by* path silently buckets +# every record into a single NULL bucket and emits no warning. +UNKNOWN = "totally.unknown.key" + + +def test_logs_scalar_filter_unknown_key_synthesizes( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], +) -> None: + """A scalar count() filtered on an unknown bare key synthesizes columns: 200 with a + key-not-found warning, matching nothing (count 0).""" + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + insert_logs([Logs(timestamp=now - timedelta(seconds=i + 1), resources={"service.name": "svc"}, body=f"log {i}") for i in range(4)]) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_scalar_query_request( + signoz, + token, + now, + [ + querier.build_scalar_query( + name="A", + signal="logs", + aggregations=[querier.build_aggregation("count()")], + filter_expression=f'{UNKNOWN} = "x"', + ) + ], + ) + assert response.status_code == HTTPStatus.OK, response.text + messages = [w.get("message", "") for w in get_all_warnings(response.json())] + assert any(UNKNOWN in m and "not found" in m for m in messages), messages + assert get_scalar_table_data(response.json()) == [[0]] + + +def test_logs_scalar_group_by_over_empty_filter_returns_no_rows( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], +) -> None: + """The contrast to the ungrouped filter case (which returns [[0]]): the *same* empty-match + filter with a group-by returns no rows at all — zero matching records means zero groups. + A regression toward [[0]] (or the ungrouped case toward []) breaks this invariant.""" + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + insert_logs([Logs(timestamp=now - timedelta(seconds=i + 1), resources={"service.name": "svc"}, body=f"log {i}") for i in range(4)]) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_scalar_query_request( + signoz, + token, + now, + [ + querier.build_scalar_query( + name="A", + signal="logs", + aggregations=[querier.build_aggregation("count()")], + group_by=[querier.build_group_by_field("service.name", "string", "resource")], + filter_expression=f'{UNKNOWN} = "x"', + ) + ], + ) + assert response.status_code == HTTPStatus.OK, response.text + assert get_scalar_table_data(response.json()) == [] + + +def test_logs_scalar_group_by_unknown_key_null_bucket( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_logs: Callable[[list[Logs]], None], +) -> None: + """Grouping a scalar count() by an unknown bare key synthesizes a NULL column: all records + collapse into one bucket (200), and — unlike the filter path — no key-not-found warning.""" + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + service = "scalar-unknown-gb-logs" + insert_logs([Logs(timestamp=now - timedelta(seconds=i + 1), resources={"service.name": service}, body=f"log {i}") for i in range(4)]) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_scalar_query_request( + signoz, + token, + now, + [ + querier.build_scalar_query( + name="A", + signal="logs", + aggregations=[querier.build_aggregation("count()")], + group_by=[querier.build_group_by_field(UNKNOWN, "string", "")], + filter_expression=f'service.name = "{service}"', + ) + ], + ) + assert response.status_code == HTTPStatus.OK, response.text + messages = [w.get("message", "") for w in get_all_warnings(response.json())] + assert not any(UNKNOWN in m for m in messages), messages + data = get_scalar_table_data(response.json()) + assert len(data) == 1, data + assert data[0][-1] == 4, data + + +def test_traces_scalar_filter_unknown_key_synthesizes( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], +) -> None: + """A scalar count() filtered on an unknown bare key synthesizes columns: 200 with a + key-not-found warning, matching nothing (count 0).""" + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + insert_traces( + [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + resources={"service.name": "svc"}, + name=f"span {i}", + ) + for i in range(4) + ] + ) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_scalar_query_request( + signoz, + token, + now, + [ + querier.build_scalar_query( + name="A", + signal="traces", + aggregations=[querier.build_aggregation("count()")], + filter_expression=f'{UNKNOWN} = "x"', + ) + ], + ) + assert response.status_code == HTTPStatus.OK, response.text + messages = [w.get("message", "") for w in get_all_warnings(response.json())] + assert any(UNKNOWN in m and "not found" in m for m in messages), messages + assert get_scalar_table_data(response.json()) == [[0]] + + +def test_traces_scalar_group_by_unknown_key_null_bucket( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], +) -> None: + """Grouping a scalar count() by an unknown bare key synthesizes a NULL column: all spans + collapse into one bucket (200), and — unlike the filter path — no key-not-found warning.""" + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + service = "scalar-unknown-gb-traces" + insert_traces( + [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + resources={"service.name": service}, + name=f"span {i}", + ) + for i in range(4) + ] + ) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_scalar_query_request( + signoz, + token, + now, + [ + querier.build_scalar_query( + name="A", + signal="traces", + aggregations=[querier.build_aggregation("count()")], + group_by=[querier.build_group_by_field(UNKNOWN, "string", "")], + filter_expression=f'service.name = "{service}"', + ) + ], + ) + assert response.status_code == HTTPStatus.OK, response.text + messages = [w.get("message", "") for w in get_all_warnings(response.json())] + assert not any(UNKNOWN in m for m in messages), messages + data = get_scalar_table_data(response.json()) + assert len(data) == 1, data + assert data[0][-1] == 4, data diff --git a/tests/integration/tests/queriertraces/01_list.py b/tests/integration/tests/queriertraces/01_list.py index 67d8b42fdc3..416c201c036 100644 --- a/tests/integration/tests/queriertraces/01_list.py +++ b/tests/integration/tests/queriertraces/01_list.py @@ -4,14 +4,19 @@ from typing import Any import pytest -import requests from fixtures import types from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD from fixtures.querier import ( - assert_identical_query_response, + Aggregation, + BuilderQuery, + OrderBy, + RequestType, + TelemetryFieldKey, format_timestamp, generate_traces_with_corrupt_metadata, + get_all_warnings, + get_rows, make_query_request, ) from fixtures.traces import ( @@ -23,27 +28,90 @@ TracesLink, TracesRefType, TracesStatusCode, + trace_noise, ) +# The clean/corrupt `trace_noise` factor (fixtures/traces.py) is applied per test via +# @pytest.mark.parametrize("noise", ["clean", "corrupt"]) so each test doubles as a +# collision-robustness check. + +def _query_window(now: datetime) -> tuple[int, int]: + """[now-1min, now+1s) in epoch millis — the default window for list tests + that seed spans a few seconds in the past.""" + return ( + int((now - timedelta(minutes=1)).timestamp() * 1000), + int((now + timedelta(seconds=1)).timestamp() * 1000), + ) + + +def _expected_list_row(span: Traces) -> dict[str, Any]: + """The full empty-selectFields list row the API returns for `span`: every + intrinsic + calculated column (mirroring the fixture's OTel-derived + computation), the merged `attributes` map (the union of the typed attribute + dicts) and the `resource` map. Deriving it from the span object lets list + responses be asserted as an exact round-trip of the inserted span — which + holds for clean and corrupt spans alike, since the corrupt values live only + inside the `attributes` map.""" + return { + "timestamp": format_timestamp(span.timestamp), + "trace_id": span.trace_id, + "span_id": span.span_id, + "trace_state": span.trace_state, + "parent_span_id": span.parent_span_id, + "flags": int(span.flags), + "name": span.name, + "kind": int(span.kind), + "kind_string": span.kind_string, + "duration_nano": int(span.duration_nano), + "status_code": int(span.status_code), + "status_message": span.status_message, + "status_code_string": span.status_code_string, + "events": span.events, + "links": span.links, + "response_status_code": span.response_status_code, + "external_http_url": span.external_http_url, + "http_url": span.http_url, + "external_http_method": span.external_http_method, + "http_method": span.http_method, + "http_host": span.http_host, + "db_name": span.db_name, + "db_operation": span.db_operation, + "has_error": span.has_error, + "is_remote": span.is_remote, + "attributes": {**span.attribute_string, **span.attributes_number, **span.attributes_bool}, + "resource": span.resources_string, + } + + +# ============================================================================ +# Basic list & row shape +# ============================================================================ + + +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) def test_traces_list( signoz: types.SigNoz, create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], insert_traces: Callable[[list[Traces]], None], + noise: str, ) -> None: """ Setup: - Insert 4 traces with different attributes. + Insert 4 spans across two services. http-service: POST /integration -> SELECT, HTTP PATCH topic-service: topic publish Tests: - 1. Query traces for the last 5 minutes and check if the spans are returned in the correct order - 2. Query root spans for the last 5 minutes and check if the spans are returned in the correct order - 3. Query values of http.request.method attribute from the autocomplete API - 4. Query values of http.request.method attribute from the fields API + 1. A raw list ordered by timestamp desc returns the spans in order with the + projected intrinsic / calculated / resource columns (field context and + datatype specified inline in the key name). + 2. isRoot = 'true' returns only the two root spans. + Under the corrupt variant the projected columns still resolve to the real + intrinsic/calculated/resource values (never the colliding attributes). """ + extra_attrs, extra_resources = trace_noise(noise) http_service_trace_id = TraceIdGenerator.trace_id() http_service_span_id = TraceIdGenerator.span_id() http_service_db_span_id = TraceIdGenerator.span_id() @@ -53,703 +121,134 @@ def test_traces_list( now = datetime.now(tz=UTC).replace(second=0, microsecond=0) - insert_traces( - [ - Traces( - timestamp=now - timedelta(seconds=4), - duration=timedelta(seconds=3), - trace_id=http_service_trace_id, - span_id=http_service_span_id, - parent_span_id="", - name="POST /integration", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={ - "deployment.environment": "production", - "service.name": "http-service", - "os.type": "linux", - "host.name": "linux-000", - "cloud.provider": "integration", - "cloud.account.id": "000", - }, - attributes={ - "net.transport": "IP.TCP", - "http.scheme": "http", - "http.user_agent": "Integration Test", - "http.request.method": "POST", - "http.response.status_code": "200", - }, - ), - Traces( - timestamp=now - timedelta(seconds=3.5), - duration=timedelta(seconds=0.5), - trace_id=http_service_trace_id, - span_id=http_service_db_span_id, - parent_span_id=http_service_span_id, - name="SELECT", - kind=TracesKind.SPAN_KIND_CLIENT, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={ - "deployment.environment": "production", - "service.name": "http-service", - "os.type": "linux", - "host.name": "linux-000", - "cloud.provider": "integration", - "cloud.account.id": "000", - }, - attributes={ - "db.name": "integration", - "db.operation": "SELECT", - "db.statement": "SELECT * FROM integration", - }, - ), - Traces( - timestamp=now - timedelta(seconds=3), - duration=timedelta(seconds=1), - trace_id=http_service_trace_id, - span_id=http_service_patch_span_id, - parent_span_id=http_service_span_id, - name="HTTP PATCH", - kind=TracesKind.SPAN_KIND_CLIENT, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={ - "deployment.environment": "production", - "service.name": "http-service", - "os.type": "linux", - "host.name": "linux-000", - "cloud.provider": "integration", - "cloud.account.id": "000", - }, - attributes={ - "http.request.method": "PATCH", - "http.status_code": "404", - }, - ), - Traces( - timestamp=now - timedelta(seconds=1), - duration=timedelta(seconds=4), - trace_id=topic_service_trace_id, - span_id=topic_service_span_id, - parent_span_id="", - name="topic publish", - kind=TracesKind.SPAN_KIND_PRODUCER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={ - "deployment.environment": "production", - "service.name": "topic-service", - "os.type": "linux", - "host.name": "linux-001", - "cloud.provider": "integration", - "cloud.account.id": "001", - }, - attributes={ - "message.type": "SENT", - "messaging.operation": "publish", - "messaging.message.id": "001", - }, - ), - ] - ) + spans = [ + Traces( + timestamp=now - timedelta(seconds=4), + duration=timedelta(seconds=3), + trace_id=http_service_trace_id, + span_id=http_service_span_id, + parent_span_id="", + name="POST /integration", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"deployment.environment": "production", "service.name": "http-service", "os.type": "linux", "host.name": "linux-000", "cloud.provider": "integration", "cloud.account.id": "000", **extra_resources}, + attributes={"net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", **extra_attrs}, + ), + Traces( + timestamp=now - timedelta(seconds=3.5), + duration=timedelta(seconds=0.5), + trace_id=http_service_trace_id, + span_id=http_service_db_span_id, + parent_span_id=http_service_span_id, + name="SELECT", + kind=TracesKind.SPAN_KIND_CLIENT, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"deployment.environment": "production", "service.name": "http-service", "os.type": "linux", "host.name": "linux-000", "cloud.provider": "integration", "cloud.account.id": "000", **extra_resources}, + attributes={"db.name": "integration", "db.operation": "SELECT", "db.statement": "SELECT * FROM integration", **extra_attrs}, + ), + Traces( + timestamp=now - timedelta(seconds=3), + duration=timedelta(seconds=1), + trace_id=http_service_trace_id, + span_id=http_service_patch_span_id, + parent_span_id=http_service_span_id, + name="HTTP PATCH", + kind=TracesKind.SPAN_KIND_CLIENT, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"deployment.environment": "production", "service.name": "http-service", "os.type": "linux", "host.name": "linux-000", "cloud.provider": "integration", "cloud.account.id": "000", **extra_resources}, + attributes={"http.request.method": "PATCH", "http.status_code": "404", **extra_attrs}, + ), + Traces( + timestamp=now - timedelta(seconds=1), + duration=timedelta(seconds=4), + trace_id=topic_service_trace_id, + span_id=topic_service_span_id, + parent_span_id="", + name="topic publish", + kind=TracesKind.SPAN_KIND_PRODUCER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"deployment.environment": "production", "service.name": "topic-service", "os.type": "linux", "host.name": "linux-001", "cloud.provider": "integration", "cloud.account.id": "001", **extra_resources}, + attributes={"message.type": "SENT", "messaging.operation": "publish", "messaging.message.id": "001", **extra_attrs}, + ), + ] + post_span, select_span, patch_span, topic_span = spans + insert_traces(spans) token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000) + end_ms = int((now + timedelta(seconds=1)).timestamp() * 1000) - # Query all traces for the past 5 minutes - response = requests.post( - signoz.self.host_configs["8080"].get("/api/v5/query_range"), - timeout=2, - headers={ - "authorization": f"Bearer {token}", - }, - json={ - "schemaVersion": "v1", - "start": int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000), - "end": int(datetime.now(tz=UTC).timestamp() * 1000), - "requestType": "raw", - "compositeQuery": { - "queries": [ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "disabled": False, - "limit": 10, - "offset": 0, - "order": [ - {"key": {"name": "timestamp"}, "direction": "desc"}, - ], - "selectFields": [ - { - "name": "service.name", - "fieldDataType": "string", - "fieldContext": "resource", - "signal": "traces", - }, - { - "name": "name", - "fieldDataType": "string", - "fieldContext": "span", - "signal": "traces", - }, - { - "name": "duration_nano", - "fieldDataType": "", - "fieldContext": "span", - "signal": "traces", - }, - { - "name": "http_method", - "fieldDataType": "", - "fieldContext": "span", - "signal": "traces", - }, - { - "name": "response_status_code", - "fieldDataType": "", - "fieldContext": "span", - "signal": "traces", - }, - ], - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - } - ] - }, - "formatOptions": {"formatTableResultForUI": False, "fillGaps": False}, - }, - ) - - # Query results with context appended to key names - response_with_inline_context = make_query_request( + # 1. Raw list ordered by timestamp desc, projecting keys with context and + # datatype specified inline in the name. + response = make_query_request( signoz, token, - start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000), - end_ms=int(datetime.now(tz=UTC).timestamp() * 1000), - request_type="raw", + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, queries=[ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "disabled": False, - "limit": 10, - "offset": 0, - "order": [ - {"key": {"name": "timestamp"}, "direction": "desc"}, - ], - "selectFields": [ - { - "name": "resource.service.name", - "fieldDataType": "string", - "signal": "traces", - }, - { - "name": "span.name:string", - "signal": "traces", - }, - { - "name": "span.duration_nano", - "signal": "traces", - }, - { - "name": "span.http_method", - "signal": "traces", - }, - { - "name": "span.response_status_code", - "signal": "traces", - }, - ], - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - } + BuilderQuery( + signal="traces", + name="A", + limit=10, + order=[OrderBy(TelemetryFieldKey("timestamp"), "desc")], + select_fields=[ + TelemetryFieldKey("resource.service.name"), + TelemetryFieldKey("span.name:string"), + TelemetryFieldKey("span.duration_nano"), + TelemetryFieldKey("span.http_method"), + TelemetryFieldKey("span.response_status_code"), + ], + aggregations=[Aggregation("count()")], + ).to_dict() ], ) - assert_identical_query_response(response, response_with_inline_context) assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - - results = response.json()["data"]["data"]["results"] - assert len(results) == 1 - - rows = results[0]["rows"] + rows = get_rows(response) assert len(rows) == 4 - # Care about the order of the rows - row_0 = dict(rows[0]["data"]) - assert row_0.pop("timestamp") is not None - assert row_0 == { - "duration_nano": 4 * 1e9, - "http_method": "", - "name": "topic publish", - "response_status_code": "", - "service.name": "topic-service", - "span_id": topic_service_span_id, - "trace_id": topic_service_trace_id, - } - - row_2 = dict(rows[1]["data"]) - assert row_2.pop("timestamp") is not None - assert row_2 == { - "duration_nano": 1 * 1e9, - "http_method": "PATCH", - "name": "HTTP PATCH", - "response_status_code": "404", - "service.name": "http-service", - "span_id": http_service_patch_span_id, - "trace_id": http_service_trace_id, - } - - row_3 = dict(rows[2]["data"]) - assert row_3.pop("timestamp") is not None - assert row_3 == { - "duration_nano": 0.5 * 1e9, - "http_method": "", - "name": "SELECT", - "response_status_code": "", - "service.name": "http-service", - "span_id": http_service_db_span_id, - "trace_id": http_service_trace_id, - } - - row_1 = dict(rows[3]["data"]) - assert row_1.pop("timestamp") is not None - assert row_1 == { - "duration_nano": 3 * 1e9, - "http_method": "POST", - "name": "POST /integration", - "response_status_code": "200", - "service.name": "http-service", - "span_id": http_service_span_id, - "trace_id": http_service_trace_id, - } - - # Query root spans for the last 5 minutes and check if the spans are returned in the correct order - response = requests.post( - signoz.self.host_configs["8080"].get("/api/v5/query_range"), - timeout=2, - headers={ - "authorization": f"Bearer {token}", - }, - json={ - "schemaVersion": "v1", - "start": int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000), - "end": int(datetime.now(tz=UTC).timestamp() * 1000), - "requestType": "raw", - "compositeQuery": { - "queries": [ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "disabled": False, - "limit": 10, - "offset": 0, - "filter": {"expression": "isRoot = 'true'"}, - "order": [ - {"key": {"name": "timestamp"}, "direction": "desc"}, - ], - "selectFields": [ - { - "name": "service.name", - "fieldDataType": "string", - "fieldContext": "resource", - } - ], - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - } - ] - }, - "formatOptions": {"formatTableResultForUI": False, "fillGaps": False}, - }, - ) - - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - - results = response.json()["data"]["data"]["results"] - assert len(results) == 1 - - rows = results[0]["rows"] - assert len(rows) == 2 - - assert rows[0]["data"]["service.name"] == "topic-service" - assert rows[1]["data"]["service.name"] == "http-service" - - # Query values of http.request.method attribute from the autocomplete API - response = requests.get( - signoz.self.host_configs["8080"].get("/api/v3/autocomplete/attribute_values"), - timeout=2, - headers={ - "authorization": f"Bearer {token}", - }, - params={ - "aggregateOperator": "noop", - "dataSource": "traces", - "aggregateAttribute": "", - "attributeKey": "http.request.method", - "searchText": "", - "filterAttributeKeyDataType": "string", - "tagType": "tag", - }, - ) - - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - - values = response.json()["data"]["stringAttributeValues"] - assert len(values) == 2 - - assert set(values) == set(["POST", "PATCH"]) - - # Query values of http.request.method attribute from the fields API - response = requests.get( - signoz.self.host_configs["8080"].get("/api/v1/fields/values"), - timeout=2, - headers={ - "authorization": f"Bearer {token}", - }, - params={ - "signal": "traces", - "name": "http.request.method", - "searchText": "", - }, - ) - - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - - values = response.json()["data"]["values"]["stringValues"] - assert len(values) == 2 - - assert set(values) == set(["POST", "PATCH"]) - - # Query keys from the fields API with context specified in the key - response = requests.get( - signoz.self.host_configs["8080"].get("/api/v1/fields/keys"), - timeout=2, - headers={ - "authorization": f"Bearer {token}", - }, - params={ - "signal": "traces", - "searchText": "resource.servic", - }, - ) - - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - - keys = response.json()["data"]["keys"] - assert "service.name" in keys - assert any(k["fieldContext"] == "resource" for k in keys["service.name"]) - - # Query values of service.name resource attribute using context-prefixed key - response = requests.get( - signoz.self.host_configs["8080"].get("/api/v1/fields/values"), - timeout=2, - headers={ - "authorization": f"Bearer {token}", - }, - params={ - "signal": "traces", - "name": "resource.service.name", - "searchText": "", - }, - ) - - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - - values = response.json()["data"]["values"]["stringValues"] - assert set(values) == set(["topic-service", "http-service"]) - - -@pytest.mark.parametrize( - "payload,status_code,results", - [ - # Case 1: order by timestamp; empty selectFields returns the full - # response shape (all intrinsic + calculated columns plus the merged - # `attributes` and `resource` maps). x[3] (topic-service) is latest. - pytest.param( - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "disabled": False, - "order": [{"key": {"name": "timestamp"}, "direction": "desc"}], - "limit": 1, - }, - }, - HTTPStatus.OK, - lambda x: [ - { - **x[3].attribute_string, - **x[3].attributes_number, - **x[3].attributes_bool, - }, # attributes - x[3].db_name, - x[3].db_operation, - int(x[3].duration_nano), - x[3].events, - x[3].external_http_method, - x[3].external_http_url, - int(x[3].flags), - x[3].has_error, - x[3].http_host, - x[3].http_method, - x[3].http_url, - x[3].is_remote, - int(x[3].kind), - x[3].kind_string, - x[3].links, - x[3].name, - x[3].parent_span_id, - x[3].resources_string, - x[3].response_status_code, - x[3].span_id, - int(x[3].status_code), - x[3].status_code_string, - x[3].status_message, - format_timestamp(x[3].timestamp), - x[3].trace_id, - x[3].trace_state, - ], # type: Callable[[List[Traces]], List[Any]] - ), - # Case 2: order by attribute.timestamp. The key resolves to the - # intrinsic span.timestamp column, so the latest span (x[3]) is - # returned with the same full response shape as Case 1. - pytest.param( - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "disabled": False, - "order": [{"key": {"name": "attribute.timestamp"}, "direction": "desc"}], - "limit": 1, - }, - }, - HTTPStatus.OK, - lambda x: [ - { - **x[3].attribute_string, - **x[3].attributes_number, - **x[3].attributes_bool, - }, # attributes - x[3].db_name, - x[3].db_operation, - int(x[3].duration_nano), - x[3].events, - x[3].external_http_method, - x[3].external_http_url, - int(x[3].flags), - x[3].has_error, - x[3].http_host, - x[3].http_method, - x[3].http_url, - x[3].is_remote, - int(x[3].kind), - x[3].kind_string, - x[3].links, - x[3].name, - x[3].parent_span_id, - x[3].resources_string, - x[3].response_status_code, - x[3].span_id, - int(x[3].status_code), - x[3].status_code_string, - x[3].status_message, - format_timestamp(x[3].timestamp), - x[3].trace_id, - x[3].trace_state, - ], # type: Callable[[List[Traces]], List[Any]] - ), - # Case 3: select timestamp with empty order by - pytest.param( - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "disabled": False, - "selectFields": [{"name": "timestamp"}], - "limit": 1, - }, - }, - HTTPStatus.OK, - lambda x: [ - x[2].span_id, - format_timestamp(x[2].timestamp), - x[2].trace_id, - ], # type: Callable[[List[Traces]], List[Any]] - ), - # Case 4: select attribute.timestamp with empty order by - # This returns the one span which has attribute.timestamp - pytest.param( - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "filter": {"expression": "attribute.timestamp exists"}, - "disabled": False, - "selectFields": [{"name": "attribute.timestamp"}], - "limit": 1, - }, - }, - HTTPStatus.OK, - lambda x: [ - x[0].span_id, - format_timestamp(x[0].timestamp), - x[0].trace_id, - ], # type: Callable[[List[Traces]], List[Any]] - ), - # Case 5: select timestamp with timestamp order by - pytest.param( - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "disabled": False, - "selectFields": [{"name": "timestamp"}], - "limit": 1, - "order": [{"key": {"name": "timestamp"}, "direction": "asc"}], - }, - }, - HTTPStatus.OK, - lambda x: [ - x[0].span_id, - format_timestamp(x[0].timestamp), - x[0].trace_id, - ], # type: Callable[[List[Traces]], List[Any]] - ), - # Case 6: select duration_nano with duration order by - pytest.param( - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "disabled": False, - "selectFields": [{"name": "duration_nano"}], - "limit": 1, - "order": [{"key": {"name": "duration_nano"}, "direction": "desc"}], - }, - }, - HTTPStatus.OK, - lambda x: [ - x[1].duration_nano, - x[1].span_id, - format_timestamp(x[1].timestamp), - x[1].trace_id, - ], # type: Callable[[List[Traces]], List[Any]] - ), - # Case 7: select attribute.duration_nano with attribute.duration_nano order by - pytest.param( - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "disabled": False, - "selectFields": [{"name": "attribute.duration_nano"}], - "filter": {"expression": "attribute.duration_nano exists"}, - "limit": 1, - "order": [ - { - "key": {"name": "attribute.duration_nano"}, - "direction": "desc", - } - ], - }, - }, - HTTPStatus.OK, - lambda x: [ - "corrupt_data", - x[3].span_id, - format_timestamp(x[3].timestamp), - x[3].trace_id, - ], # type: Callable[[List[Traces]], List[Any]] - ), - # Case 8: select attribute.duration_nano with duration order by - pytest.param( - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "disabled": False, - "selectFields": [{"name": "attribute.duration_nano"}], - "limit": 1, - "order": [{"key": {"name": "duration_nano"}, "direction": "desc"}], - }, - }, - HTTPStatus.OK, - lambda x: [ - x[1].duration_nano, - x[1].span_id, - format_timestamp(x[1].timestamp), - x[1].trace_id, - ], # type: Callable[[List[Traces]], List[Any]] - ), - ], -) -def test_traces_list_with_corrupt_data( - signoz: types.SigNoz, - create_user_admin: None, # pylint: disable=unused-argument - get_token: Callable[[str, str], str], - insert_traces: Callable[[list[Traces]], None], - payload: dict[str, Any], - status_code: HTTPStatus, - results: Callable[[list[Traces]], list[Any]], -) -> None: - """ - Setup: - Insert 4 traces with corrupt attributes. - Tests: - """ - - traces = generate_traces_with_corrupt_metadata() - insert_traces(traces) - # 4 Traces with corrupt metadata inserted - # traces[i] occured before traces[j] where i < j - - token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + # Ordered by timestamp desc: topic publish, HTTP PATCH, SELECT, POST. Every + # expected value is derived from the inserted span objects, so the corrupt + # variant (colliding attributes) must not change any projected column. + for row, span in zip(rows, [topic_span, patch_span, select_span, post_span], strict=True): + data = dict(row["data"]) + assert data.pop("timestamp") is not None + assert data == { + "duration_nano": int(span.duration_nano), + "http_method": span.http_method, + "name": span.name, + "response_status_code": span.response_status_code, + "service.name": span.service_name, + "span_id": span.span_id, + "trace_id": span.trace_id, + } + # 2. Root spans only. response = make_query_request( signoz, token, - start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000), - end_ms=int(datetime.now(tz=UTC).timestamp() * 1000), - request_type="raw", - queries=[payload], + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[ + BuilderQuery( + signal="traces", + name="A", + limit=10, + filter_expression="isRoot = 'true'", + order=[OrderBy(TelemetryFieldKey("timestamp"), "desc")], + select_fields=[TelemetryFieldKey("resource.service.name")], + aggregations=[Aggregation("count()")], + ).to_dict() + ], ) - assert response.status_code == status_code - - if response.status_code == HTTPStatus.OK: - if not results(traces): - # No results expected - assert response.json()["data"]["data"]["results"][0]["rows"] is None - else: - data = response.json()["data"]["data"]["results"][0]["rows"][0]["data"] - # Cannot compare values as they are randomly generated - for key, value in zip(list(data.keys()), results(traces)): - assert data[key] == value + assert response.status_code == HTTPStatus.OK + rows = get_rows(response) + assert len(rows) == 2 + assert rows[0]["data"]["service.name"] == topic_span.service_name + assert rows[1]["data"]["service.name"] == post_span.service_name def _verify_events_links_full(rows: list[dict], traces: list[Traces]) -> None: @@ -770,31 +269,25 @@ def _verify_events_links_skip(rows: list[dict], traces: list[Traces]) -> None: @pytest.mark.parametrize( - "select_fields,status_code,expected_keys,verify_values", + "select_fields,expected_keys,verify_values", [ + pytest.param([], ALL_SELECT_FIELDS, _verify_events_links_full, id="empty_returns_all_fields"), pytest.param( - [], - HTTPStatus.OK, - ALL_SELECT_FIELDS, - _verify_events_links_full, - ), - pytest.param( - [ - {"name": "service.name"}, - ], - HTTPStatus.OK, + [TelemetryFieldKey("service.name")], ["timestamp", "trace_id", "span_id", "service.name"], _verify_events_links_skip, + id="projected_subset", ), ], ) -def test_traces_list_with_select_fields( +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_list_select_fields( signoz: types.SigNoz, create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], insert_traces: Callable[[list[Traces]], None], - select_fields: list[dict], - status_code: HTTPStatus, + noise: str, + select_fields: list[TelemetryFieldKey], expected_keys: list[str], verify_values: Callable[[list[dict], list[Traces]], None], ) -> None: @@ -804,13 +297,14 @@ def test_traces_list_with_select_fields( events and one user-supplied link. Tests: - 1. Empty select fields should return all the fields, and the `events` / - `links` columns should arrive parsed into structured objects (events - carry `attributes`, links carry only `traceId`/`spanId` — refType is - dropped at the consume layer). - 2. Non-empty select field should return the select field along with - timestamp, trace_id and span_id. + 1. Empty selectFields returns all fields, and the `events` / `links` columns + arrive parsed into structured objects (events carry `attributes`, links + carry only `traceId`/`spanId` — refType is dropped at the consume layer). + 2. A non-empty selectField returns that field along with timestamp, trace_id + and span_id. + The returned key set is invariant to the corrupt-variant collisions. """ + extra_attrs, extra_resources = trace_noise(noise) now = datetime.now(tz=UTC).replace(second=0, microsecond=0) parent_trace_id = TraceIdGenerator.trace_id() @@ -836,7 +330,6 @@ def test_traces_list_with_select_fields( ) traces = [ - # Root span: no events, no links. Verifies the empty-case parsed shape. Traces( timestamp=now - timedelta(seconds=4), duration=timedelta(seconds=3), @@ -846,12 +339,9 @@ def test_traces_list_with_select_fields( name="root span", kind=TracesKind.SPAN_KIND_SERVER, status_code=TracesStatusCode.STATUS_CODE_OK, - resources={"service.name": "events-links-service"}, - attributes={"http.request.method": "GET"}, + resources={"service.name": "events-links-service", **extra_resources}, + attributes={"http.request.method": "GET", **extra_attrs}, ), - # Child span: two events + one user-supplied link. The fixture - # auto-inserts a CHILD_OF link for the parent, so the parsed response - # contains two links total — the auto-inserted one first. Traces( timestamp=now - timedelta(seconds=3), duration=timedelta(seconds=1), @@ -861,8 +351,8 @@ def test_traces_list_with_select_fields( name="child span", kind=TracesKind.SPAN_KIND_INTERNAL, status_code=TracesStatusCode.STATUS_CODE_OK, - resources={"service.name": "events-links-service"}, - attributes={"http.request.method": "GET"}, + resources={"service.name": "events-links-service", **extra_resources}, + attributes={"http.request.method": "GET", **extra_attrs}, events=[event_one, event_two], links=[user_link], ), @@ -872,34 +362,1071 @@ def test_traces_list_with_select_fields( token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - payload = { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "filter": {"expression": "resource.service.name = 'events-links-service'"}, - "selectFields": select_fields, - "order": [{"key": {"name": "timestamp"}, "direction": "asc"}], - "limit": 10, - }, - } - response = make_query_request( signoz, token, - start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000), - end_ms=int(datetime.now(tz=UTC).timestamp() * 1000), - request_type="raw", - queries=[payload], + start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int((now + timedelta(seconds=1)).timestamp() * 1000), + request_type=RequestType.RAW, + queries=[ + BuilderQuery( + signal="traces", + name="A", + limit=10, + filter_expression="resource.service.name = 'events-links-service'", + select_fields=select_fields, + order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")], + ).to_dict() + ], ) - assert response.status_code == status_code - - if response.status_code != HTTPStatus.OK: - return + assert response.status_code == HTTPStatus.OK - rows = response.json()["data"]["data"]["results"][0]["rows"] + rows = get_rows(response) assert len(rows) == 2 for row in rows: assert set(row["data"].keys()) == set(expected_keys) verify_values(rows, traces) + + +@pytest.mark.parametrize( + "select_field,key,expected,expected_type", + [ + pytest.param(TelemetryFieldKey("span.string_attr"), "string_attr", lambda s: s.attribute_string["string_attr"], str, id="string_attr"), + pytest.param(TelemetryFieldKey("span.number_attr"), "number_attr", lambda s: s.attributes_number["number_attr"], float, id="number_attr"), + pytest.param(TelemetryFieldKey("span.bool_attr"), "bool_attr", lambda s: s.attributes_bool["bool_attr"], bool, id="bool_attr"), + pytest.param(TelemetryFieldKey("db_name"), "db_name", lambda s: s.db_name, str, id="calculated_db_name"), + pytest.param(TelemetryFieldKey("duration_nano"), "duration_nano", lambda s: int(s.duration_nano), int, id="intrinsic_duration_nano"), + pytest.param(TelemetryFieldKey("durationNano"), "durationNano", lambda s: int(s.duration_nano), int, id="deprecated_alias_durationNano"), + pytest.param(TelemetryFieldKey("does_not_exist"), "does_not_exist", lambda s: None, type(None), id="nonexistent_attr_bare"), + ], +) +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_list_select_field_types( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, + select_field: TelemetryFieldKey, + key: str, + expected: Callable[[Traces], Any], + expected_type: type, +) -> None: + """ + Setup: + Insert one span carrying typed attributes and db attributes. + + Tests: + Projecting a string / number / bool span attribute, a calculated column, an + intrinsic column and its deprecated alias returns the value (derived from + the inserted span) with the right Python type; a bare non-existent attribute + projects as null. Under the corrupt variant the same-named colliding + attributes must not shadow the intrinsic/calculated columns. + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(microsecond=0) + span = Traces( + timestamp=now - timedelta(seconds=1), + duration=timedelta(seconds=3), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name="typed-span", + kind=TracesKind.SPAN_KIND_CLIENT, + resources={"service.name": "typed-service", **extra_resources}, + attributes={"string_attr": "hello", "number_attr": 42.5, "bool_attr": True, "db.name": "orders_db", "db.operation": "SELECT", **extra_attrs}, + ) + insert_traces([span]) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms, end_ms = _query_window(now) + + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[BuilderQuery(signal="traces", name="A", select_fields=[select_field]).to_dict()], + ) + + assert response.status_code == HTTPStatus.OK + rows = get_rows(response) + assert len(rows) == 1 + data = rows[0]["data"] + assert key in data + expected_value = expected(span) + if expected_value is None: + assert data[key] is None + else: + assert data[key] == expected_value + assert isinstance(data[key], expected_type) + + +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_list_select_field_dedup( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, +) -> None: + """ + Setup: + Insert one span. + + Tests: + Selecting the same field twice collapses to a single key in the response. + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(microsecond=0) + span = Traces( + timestamp=now - timedelta(seconds=1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name="dedup-span", + resources={"service.name": "dedup-service", **extra_resources}, + attributes={"dup_attr": "value", **extra_attrs}, + ) + insert_traces([span]) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms, end_ms = _query_window(now) + + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[BuilderQuery(signal="traces", name="A", select_fields=[TelemetryFieldKey("span.dup_attr"), TelemetryFieldKey("span.dup_attr")]).to_dict()], + ) + + assert response.status_code == HTTPStatus.OK + rows = get_rows(response) + assert len(rows) == 1 + keys = list(rows[0]["data"].keys()) + assert keys.count("dup_attr") == 1 + assert rows[0]["data"]["dup_attr"] == span.attribute_string["dup_attr"] + + +@pytest.mark.parametrize( + "select_field,projected_key,expected", + [ + pytest.param(TelemetryFieldKey("db_name"), "db_name", lambda s: s.db_name, id="db_name_bare_calculated"), + pytest.param(TelemetryFieldKey("span.db_name"), "db_name", lambda s: s.db_name, id="db_name_span_calculated"), + pytest.param(TelemetryFieldKey("attribute.db_name"), "db_name", lambda s: s.attribute_string["db_name"], id="db_name_attribute"), + pytest.param(TelemetryFieldKey("http_method"), "http_method", lambda s: s.http_method, id="http_method_bare_calculated"), + pytest.param(TelemetryFieldKey("span.http_method"), "http_method", lambda s: s.http_method, id="http_method_span_calculated"), + pytest.param(TelemetryFieldKey("attribute.http_method"), "http_method", lambda s: s.attribute_string["http_method"], id="http_method_attribute"), + ], +) +def test_traces_list_select_calculated_field_collision( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + select_field: TelemetryFieldKey, + projected_key: str, + expected: Callable[[Traces], Any], +) -> None: + """ + Setup: + Insert one span whose derived calculated columns (db_name from db.name, + http_method from http.request.method) collide with literal attributes of the + same name. + + Tests: + Selecting a calculated field by its bare name or `span.` context returns the + derived column value (the same-named attribute is shadowed); the explicit + `attribute.` prefix reaches the colliding attribute value instead. Every form + projects under the calculated field's name. + """ + now = datetime.now(tz=UTC).replace(microsecond=0) + span = Traces( + timestamp=now - timedelta(seconds=1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name="calc-collision", + kind=TracesKind.SPAN_KIND_CLIENT, + resources={"service.name": "calc-collision-service"}, + attributes={"db.name": "realdb", "db_name": "attrval", "http.request.method": "GET", "http_method": "attrmethod"}, + ) + insert_traces([span]) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms, end_ms = _query_window(now) + + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[BuilderQuery(signal="traces", name="A", select_fields=[select_field]).to_dict()], + ) + + assert response.status_code == HTTPStatus.OK + rows = get_rows(response) + assert len(rows) == 1 + assert rows[0]["data"][projected_key] == expected(span) + + +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_list_response_shape_values( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, +) -> None: + """ + Setup: + Insert a curated set of spans exercising every derived/intrinsic response + field: an error span, a client span with HTTP url/method, a db span, a + host span, and remote/kind variants. + + Tests: + With empty selectFields every intrinsic/calculated column (has_error, + is_remote, external_http_url, db_name, http_host, status_code_string, + kind_string, ...) round-trips with its populated value, and the merged + `attributes` map preserves string / number / bool types. Under the corrupt + variant these columns keep winning over the same-named colliding attributes. + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(microsecond=0) + trace_id = TraceIdGenerator.trace_id() + + def span(marker: str, *, kind: TracesKind, status_code: TracesStatusCode = TracesStatusCode.STATUS_CODE_UNSET, status_message: str = "", flags: int = 0, attributes: dict[str, Any]) -> Traces: + return Traces( + timestamp=now - timedelta(seconds=1), + duration=timedelta(seconds=1), + trace_id=trace_id, + span_id=TraceIdGenerator.span_id(), + name=f"shape-{marker}", + kind=kind, + status_code=status_code, + status_message=status_message, + flags=flags, + resources={"service.name": "shape-service", **extra_resources}, + attributes={"marker": marker, **attributes, **extra_attrs}, + ) + + spans = [ + span("error", kind=TracesKind.SPAN_KIND_SERVER, status_code=TracesStatusCode.STATUS_CODE_ERROR, status_message="boom", attributes={"str_a": "x", "num_a": 7.5, "bool_a": True}), + span("client", kind=TracesKind.SPAN_KIND_CLIENT, attributes={"http.url": "https://api.example.com/v1/orders", "http.request.method": "GET"}), + span("db", kind=TracesKind.SPAN_KIND_CLIENT, attributes={"db.name": "orders_db", "db.operation": "SELECT"}), + span("host", kind=TracesKind.SPAN_KIND_PRODUCER, attributes={"server.address": "payments.internal:8080"}), + span("remote_yes", kind=TracesKind.SPAN_KIND_INTERNAL, flags=0x300, attributes={}), + span("remote_no", kind=TracesKind.SPAN_KIND_CONSUMER, flags=0x100, attributes={}), + span("remote_unknown", kind=TracesKind.SPAN_KIND_UNSPECIFIED, flags=0, attributes={}), + ] + insert_traces(spans) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms, end_ms = _query_window(now) + + # Each span's full empty-selectFields row is derived from the span object: + # intrinsic + calculated columns come from the fixture's own OTel-mirroring + # computation, and the merged `attributes` map is exactly the union of the + # typed attribute dicts. This holds identically for clean and corrupt spans. + for span_obj in spans: + marker = span_obj.attribute_string["marker"] + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[BuilderQuery(signal="traces", name="A", filter_expression=f"marker = '{marker}'").to_dict()], + ) + assert response.status_code == HTTPStatus.OK, f"marker={marker}: {response.text}" + rows = get_rows(response) + assert len(rows) == 1 + data = rows[0]["data"] + assert data == _expected_list_row(span_obj) + # `==` treats True/1 and 7.5/7.5 as equal regardless of type, so pin the + # string/number/bool type preservation for the error span explicitly. + if marker == "error": + assert isinstance(data["attributes"]["str_a"], str) + assert isinstance(data["attributes"]["num_a"], float) + assert isinstance(data["attributes"]["bool_a"], bool) + + +# ============================================================================ +# Ordering +# ============================================================================ + + +@pytest.mark.parametrize("direction", ["asc", "desc"]) +@pytest.mark.parametrize( + "order_key,sort_value", + [ + pytest.param(TelemetryFieldKey("timestamp"), lambda m: -m["ts_ago"], id="timestamp"), + pytest.param(TelemetryFieldKey("duration_nano"), lambda m: m["duration"], id="duration_nano_intrinsic"), + pytest.param(TelemetryFieldKey("name"), lambda m: m["name"], id="name_intrinsic"), + pytest.param(TelemetryFieldKey("span.kind:number"), lambda m: m["kind"].value, id="kind_intrinsic"), + pytest.param(TelemetryFieldKey("response_status_code"), lambda m: m["status"], id="response_status_code_calculated"), + pytest.param(TelemetryFieldKey("resource.service.name"), lambda m: m["service"], id="resource_attribute"), + pytest.param(TelemetryFieldKey("span.priority:number"), lambda m: m["priority"], id="number_span_attribute"), + ], +) +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_list_ordering( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, + order_key: TelemetryFieldKey, + sort_value: Callable[[dict[str, Any]], Any], + direction: str, +) -> None: + """ + Setup: + Insert 4 spans whose fields are permuted so that each order key produces a + distinct ordering. + + Tests: + Ordering by an intrinsic (timestamp, duration_nano, name, kind), a + calculated column (response_status_code), a resource attribute and a number + span attribute all sort correctly in both directions — including when + same-named corrupt attributes collide with those columns. + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(microsecond=0) + trace_id = TraceIdGenerator.trace_id() + metas: list[dict[str, Any]] = [ + {"marker": "A", "ts_ago": 4, "duration": 2, "name": "span-d", "kind": TracesKind.SPAN_KIND_CONSUMER, "status": "203", "service": "svc-c", "priority": 30}, + {"marker": "B", "ts_ago": 3, "duration": 4, "name": "span-c", "kind": TracesKind.SPAN_KIND_SERVER, "status": "201", "service": "svc-a", "priority": 10}, + {"marker": "C", "ts_ago": 2, "duration": 1, "name": "span-b", "kind": TracesKind.SPAN_KIND_PRODUCER, "status": "204", "service": "svc-d", "priority": 40}, + {"marker": "D", "ts_ago": 1, "duration": 3, "name": "span-a", "kind": TracesKind.SPAN_KIND_CLIENT, "status": "202", "service": "svc-b", "priority": 20}, + ] + insert_traces( + [ + Traces( + timestamp=now - timedelta(seconds=m["ts_ago"]), + duration=timedelta(seconds=m["duration"]), + trace_id=trace_id, + span_id=TraceIdGenerator.span_id(), + name=m["name"], + kind=m["kind"], + resources={"service.name": m["service"], **extra_resources}, + attributes={"marker": m["marker"], "priority": m["priority"], "http.response.status_code": m["status"], **extra_attrs}, + ) + for m in metas + ] + ) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms, end_ms = _query_window(now) + + expected = [m["marker"] for m in sorted(metas, key=sort_value, reverse=(direction == "desc"))] + + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[ + BuilderQuery( + signal="traces", + name="A", + select_fields=[TelemetryFieldKey("span.marker")], + order=[OrderBy(order_key, direction)], + ).to_dict() + ], + ) + + assert response.status_code == HTTPStatus.OK + rows = get_rows(response) + assert [row["data"]["marker"] for row in rows] == expected + + +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_list_order_multi_key_tiebreak( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, +) -> None: + """ + Setup: + Insert 3 spans sharing the same timestamp with distinct durations. + + Tests: + A secondary order key (duration_nano) breaks the timestamp tie + deterministically, and flipping its direction flips the tie-break. + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(microsecond=0) + trace_id = TraceIdGenerator.trace_id() + shared_ts = now - timedelta(seconds=5) + spans = [ + Traces( + timestamp=shared_ts, + duration=timedelta(seconds=seconds), + trace_id=trace_id, + span_id=TraceIdGenerator.span_id(), + name=f"tie-{seconds}", + resources={"service.name": "tiebreak-service", **extra_resources}, + attributes=dict(extra_attrs), + ) + for seconds in (1, 2, 3) + ] + insert_traces(spans) + by_duration_secs = {1: spans[0].span_id, 2: spans[1].span_id, 3: spans[2].span_id} + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms, end_ms = _query_window(now) + + def query(duration_direction: str) -> list[str]: + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[ + BuilderQuery( + signal="traces", + name="A", + order=[OrderBy(TelemetryFieldKey("timestamp"), "desc"), OrderBy(TelemetryFieldKey("duration_nano"), duration_direction)], + ).to_dict() + ], + ) + assert response.status_code == HTTPStatus.OK + return [row["data"]["span_id"] for row in get_rows(response)] + + assert query("desc") == [by_duration_secs[3], by_duration_secs[2], by_duration_secs[1]] + assert query("asc") == [by_duration_secs[1], by_duration_secs[2], by_duration_secs[3]] + + +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_list_no_default_order( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, +) -> None: + """ + Setup: + Insert 4 spans. + + Tests: + A raw list query with no order clause succeeds and returns every matching + span (the builder adds no default ORDER BY, so only membership is pinned). + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(microsecond=0) + trace_id = TraceIdGenerator.trace_id() + spans = [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + trace_id=trace_id, + span_id=TraceIdGenerator.span_id(), + name=f"unordered-{i}", + resources={"service.name": "no-order-service", **extra_resources}, + attributes=dict(extra_attrs), + ) + for i in range(4) + ] + insert_traces(spans) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms, end_ms = _query_window(now) + + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[BuilderQuery(signal="traces", name="A", filter_expression="resource.service.name = 'no-order-service'").to_dict()], + ) + + assert response.status_code == HTTPStatus.OK + rows = get_rows(response) + assert {row["data"]["span_id"] for row in rows} == {span.span_id for span in spans} + + +# ============================================================================ +# Pagination +# ============================================================================ + + +@pytest.mark.parametrize( + "limit,offset", + [ + pytest.param(2, 0, id="first_page"), + pytest.param(2, 2, id="second_page"), + pytest.param(2, 4, id="last_partial_page"), + pytest.param(10, 0, id="limit_exceeds_count"), + pytest.param(None, 0, id="default_limit_omitted"), + pytest.param(2, 10, id="offset_past_end"), + ], +) +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_list_pagination( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, + limit: int | None, + offset: int, +) -> None: + """ + Setup: + Insert 5 spans with strictly increasing timestamps. + + Tests: + Ordered by timestamp desc, limit caps the row count and offset pages + through the result; an offset past the end returns no rows. + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(microsecond=0) + trace_id = TraceIdGenerator.trace_id() + spans = [ + Traces( + timestamp=now - timedelta(seconds=10 - i), + trace_id=trace_id, + span_id=TraceIdGenerator.span_id(), + name=f"span-{i}", + resources={"service.name": "pagination-service", **extra_resources}, + attributes=dict(extra_attrs), + ) + for i in range(5) + ] + insert_traces(spans) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms, end_ms = _query_window(now) + + # timestamp desc => newest span (i=4) first + ordered_span_ids = [span.span_id for span in reversed(spans)] + effective_limit = 100 if limit is None else limit + expected_ids = ordered_span_ids[offset : offset + effective_limit] + + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[ + BuilderQuery( + signal="traces", + name="A", + limit=limit, + offset=offset, + order=[OrderBy(TelemetryFieldKey("timestamp"), "desc")], + ).to_dict() + ], + ) + + assert response.status_code == HTTPStatus.OK + rows = get_rows(response) + assert [row["data"]["span_id"] for row in rows] == expected_ids + + +# ============================================================================ +# Time window +# ============================================================================ + + +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_list_time_boundaries( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, +) -> None: + """ + Setup: + Insert spans exactly at the query start, in the middle, and exactly at the + query end (all second-aligned so the ms->ns conversion lands exactly). + + Tests: + The query window is [start, end): the span at start is included and the + span at end is excluded. + """ + extra_attrs, extra_resources = trace_noise(noise) + start_dt = datetime.now(tz=UTC).replace(microsecond=0) - timedelta(seconds=30) + end_dt = start_dt + timedelta(seconds=10) + mid_dt = start_dt + timedelta(seconds=5) + + trace_id = TraceIdGenerator.trace_id() + resources = {"service.name": "boundary-service", **extra_resources} + start_span = Traces(timestamp=start_dt, trace_id=trace_id, span_id=TraceIdGenerator.span_id(), name="at-start", resources=resources, attributes=dict(extra_attrs)) + mid_span = Traces(timestamp=mid_dt, trace_id=trace_id, span_id=TraceIdGenerator.span_id(), name="in-middle", resources=resources, attributes=dict(extra_attrs)) + end_span = Traces(timestamp=end_dt, trace_id=trace_id, span_id=TraceIdGenerator.span_id(), name="at-end", resources=resources, attributes=dict(extra_attrs)) + insert_traces([start_span, mid_span, end_span]) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_query_request( + signoz, + token, + start_ms=int(start_dt.timestamp() * 1000), + end_ms=int(end_dt.timestamp() * 1000), + request_type=RequestType.RAW, + queries=[BuilderQuery(signal="traces", name="A").to_dict()], + ) + + assert response.status_code == HTTPStatus.OK + rows = get_rows(response) + returned = {row["data"]["span_id"] for row in rows} + assert start_span.span_id in returned + assert mid_span.span_id in returned + assert end_span.span_id not in returned + + +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_list_empty_result( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, +) -> None: + """ + Setup: + Insert one span. + + Tests: + A filter matching no span, and a time window that excludes the span, both + return zero rows (rows is null, surfaced as an empty list). + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(microsecond=0) + insert_traces( + [ + Traces( + timestamp=now - timedelta(seconds=1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name="lonely-span", + resources={"service.name": "empty-service", **extra_resources}, + attributes=dict(extra_attrs), + ) + ] + ) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms, end_ms = _query_window(now) + + # Filter matches nothing. + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[BuilderQuery(signal="traces", name="A", filter_expression="resource.service.name = 'does-not-exist'").to_dict()], + ) + assert response.status_code == HTTPStatus.OK + assert get_rows(response) == [] + + # Time window excludes the span. + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(hours=2)).timestamp() * 1000), + end_ms=int((now - timedelta(hours=1)).timestamp() * 1000), + request_type=RequestType.RAW, + queries=[BuilderQuery(signal="traces", name="A").to_dict()], + ) + assert response.status_code == HTTPStatus.OK + assert get_rows(response) == [] + + +# ============================================================================ +# Span scope +# ============================================================================ + + +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_list_span_scope( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + insert_top_level_operations: Callable[[list[tuple[str, str]]], None], + noise: str, +) -> None: + """ + Setup: + Insert a root span, a non-root entry-point span (its (name, service) is + seeded into top_level_operations) and a non-root internal span. + + Tests: + 1. isRoot = 'true' returns only the root span. + 2. isEntryPoint = 'true' returns only the non-root span whose operation is + registered in top_level_operations. + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(microsecond=0) + trace_id = TraceIdGenerator.trace_id() + root_span_id = TraceIdGenerator.span_id() + resources = {"service.name": "scope-svc", **extra_resources} + + root_span = Traces( + timestamp=now - timedelta(seconds=3), + trace_id=trace_id, + span_id=root_span_id, + parent_span_id="", + name="op-root", + kind=TracesKind.SPAN_KIND_SERVER, + resources=resources, + attributes=dict(extra_attrs), + ) + entry_span = Traces( + timestamp=now - timedelta(seconds=2), + trace_id=trace_id, + span_id=TraceIdGenerator.span_id(), + parent_span_id=root_span_id, + name="op-entry", + kind=TracesKind.SPAN_KIND_SERVER, + resources=resources, + attributes=dict(extra_attrs), + ) + internal_span = Traces( + timestamp=now - timedelta(seconds=1), + trace_id=trace_id, + span_id=TraceIdGenerator.span_id(), + parent_span_id=root_span_id, + name="op-internal", + kind=TracesKind.SPAN_KIND_INTERNAL, + resources=resources, + attributes=dict(extra_attrs), + ) + insert_traces([root_span, entry_span, internal_span]) + insert_top_level_operations([("op-entry", "scope-svc")]) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms, end_ms = _query_window(now) + + def query_scope(expression: str) -> list[str]: + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[BuilderQuery(signal="traces", name="A", filter_expression=expression).to_dict()], + ) + assert response.status_code == HTTPStatus.OK, response.text + return [row["data"]["span_id"] for row in get_rows(response)] + + assert query_scope("isRoot = 'true'") == [root_span.span_id] + assert query_scope("isEntryPoint = 'true'") == [entry_span.span_id] + + +# ============================================================================ +# Field-key resolution over corrupt metadata +# ============================================================================ + + +@pytest.mark.parametrize( + "payload,status_code,expected", + [ + # Case 1: order by timestamp; empty selectFields returns the full + # response shape (all intrinsic + calculated columns plus the merged + # `attributes` and `resource` maps). x[3] (topic-service) is latest. + pytest.param( + BuilderQuery(signal="traces", name="A", order=[OrderBy(TelemetryFieldKey("timestamp"), "desc")], limit=1), + HTTPStatus.OK, + lambda x: _expected_list_row(x[3]), + id="order_timestamp_full_shape", + ), + # Case 2: order by attribute.timestamp. The key resolves to the intrinsic + # span.timestamp column, so the latest span (x[3]) is returned with the + # same full response shape as Case 1. + pytest.param( + BuilderQuery(signal="traces", name="A", order=[OrderBy(TelemetryFieldKey("attribute.timestamp"), "desc")], limit=1), + HTTPStatus.OK, + lambda x: _expected_list_row(x[3]), + id="order_attribute_timestamp_full_shape", + ), + # Case 3: select timestamp with empty order by. + pytest.param( + BuilderQuery(signal="traces", name="A", select_fields=[TelemetryFieldKey("timestamp")], limit=1), + HTTPStatus.OK, + lambda x: {"span_id": x[2].span_id, "timestamp": format_timestamp(x[2].timestamp), "trace_id": x[2].trace_id}, + id="select_timestamp", + ), + # Case 4: select attribute.timestamp with empty order by. The key resolves + # to the intrinsic timestamp column, projecting the default id columns. + pytest.param( + BuilderQuery(signal="traces", name="A", filter_expression="attribute.timestamp exists", select_fields=[TelemetryFieldKey("attribute.timestamp")], limit=1), + HTTPStatus.OK, + lambda x: {"span_id": x[0].span_id, "timestamp": format_timestamp(x[0].timestamp), "trace_id": x[0].trace_id}, + id="select_attribute_timestamp", + ), + # Case 5: select timestamp with timestamp order by. + pytest.param( + BuilderQuery(signal="traces", name="A", select_fields=[TelemetryFieldKey("timestamp")], limit=1, order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")]), + HTTPStatus.OK, + lambda x: {"span_id": x[0].span_id, "timestamp": format_timestamp(x[0].timestamp), "trace_id": x[0].trace_id}, + id="select_timestamp_order_timestamp", + ), + # Case 6: select duration_nano with duration order by (longest span x[1]). + pytest.param( + BuilderQuery(signal="traces", name="A", select_fields=[TelemetryFieldKey("duration_nano")], limit=1, order=[OrderBy(TelemetryFieldKey("duration_nano"), "desc")]), + HTTPStatus.OK, + lambda x: {"duration_nano": int(x[1].duration_nano), "span_id": x[1].span_id, "timestamp": format_timestamp(x[1].timestamp), "trace_id": x[1].trace_id}, + id="select_duration", + ), + # Case 7: select attribute.duration_nano with attribute.duration_nano order + # by — the explicit `attribute.` prefix reaches the attribute value. + pytest.param( + BuilderQuery(signal="traces", name="A", select_fields=[TelemetryFieldKey("attribute.duration_nano")], filter_expression="attribute.duration_nano exists", limit=1, order=[OrderBy(TelemetryFieldKey("attribute.duration_nano"), "desc")]), + HTTPStatus.OK, + lambda x: {"duration_nano": "corrupt_data", "span_id": x[3].span_id, "timestamp": format_timestamp(x[3].timestamp), "trace_id": x[3].trace_id}, + id="select_attribute_duration", + ), + # Case 8: select attribute.duration_nano with intrinsic duration order by — + # the longest span x[1] has no such attribute, so it falls back to the + # intrinsic duration_nano value. + pytest.param( + BuilderQuery(signal="traces", name="A", select_fields=[TelemetryFieldKey("attribute.duration_nano")], limit=1, order=[OrderBy(TelemetryFieldKey("duration_nano"), "desc")]), + HTTPStatus.OK, + lambda x: {"duration_nano": int(x[1].duration_nano), "span_id": x[1].span_id, "timestamp": format_timestamp(x[1].timestamp), "trace_id": x[1].trace_id}, + id="select_attribute_duration_order_intrinsic", + ), + ], +) +def test_traces_list_with_corrupt_data( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + payload: BuilderQuery, + status_code: HTTPStatus, + expected: Callable[[list[Traces]], dict[str, Any]], +) -> None: + """ + Setup: + Insert 4 spans whose metadata carries intrinsic/calculated field names as + attributes (e.g. attribute.timestamp, attribute.duration_nano). + + Tests: + Ordering / selecting a name that exists both as an intrinsic column and as + an attribute resolves to the intrinsic column, while the explicit + `attribute.` prefix reaches the attribute value. Each expected row is derived + from the inserted span objects. + """ + traces = generate_traces_with_corrupt_metadata() + insert_traces(traces) + # traces[i] occurred before traces[j] where i < j. + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_query_request( + signoz, + token, + start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int(datetime.now(tz=UTC).timestamp() * 1000), + request_type=RequestType.RAW, + queries=[payload.to_dict()], + ) + + assert response.status_code == status_code + + if response.status_code == HTTPStatus.OK: + assert get_rows(response)[0]["data"] == expected(traces) + + +@pytest.mark.parametrize("surface", ["filter", "select", "order"]) +def test_traces_list_unknown_span_context_synthesizes( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + surface: str, +) -> None: + """ + Setup: + Insert one span. + + Tests: + The intrinsic `span.` context is forgiving, like bare / `attribute.` / + `resource.` keys (and the logs `log.` context): an unknown key there + synthesizes against the span attribute maps rather than erroring, because + existence is a property of the data, not of metadata. A filter reference + matches nothing and surfaces a not-found warning (under the stripped name); + a select projects a null column; an order clause succeeds and returns the + span. (See test_traces_list_unknown_other_context_synthesizes for the + `attribute.`/`resource.` contexts.) + """ + now = datetime.now(tz=UTC).replace(microsecond=0) + span = Traces( + timestamp=now - timedelta(seconds=1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name="forgiving-ctx-span", + resources={"service.name": "forgiving-ctx-service"}, + ) + insert_traces([span]) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms, end_ms = _query_window(now) + key = "span.does_not_exist" + svc = "resource.service.name = 'forgiving-ctx-service'" + + if surface == "filter": + query = BuilderQuery(signal="traces", name="A", filter_expression=f"{svc} AND {key} = 'nope'") + elif surface == "select": + query = BuilderQuery(signal="traces", name="A", filter_expression=svc, select_fields=[TelemetryFieldKey(key)]) + else: # order + query = BuilderQuery(signal="traces", name="A", filter_expression=svc, order=[OrderBy(TelemetryFieldKey(key), "desc")]) + + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[query.to_dict()], + ) + + assert response.status_code == HTTPStatus.OK, response.text + rows = get_rows(response) + + if surface == "filter": + assert rows == [] + messages = [w.get("message", "") for w in get_all_warnings(response.json())] + assert any("does_not_exist" in m and "not found" in m for m in messages), messages + elif surface == "select": + assert {span.span_id} == {row["data"]["span_id"] for row in rows} + for row in rows: + assert row["data"]["does_not_exist"] is None + else: # order + assert {span.span_id} == {row["data"]["span_id"] for row in rows} + + +@pytest.mark.parametrize("context", ["attribute", "resource"]) +@pytest.mark.parametrize("surface", ["select", "order"]) +def test_traces_list_unknown_other_context_synthesizes( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + context: str, + surface: str, +) -> None: + """ + Setup: + Insert 3 spans. + + Tests: + Like the intrinsic `span.` context (see + test_traces_list_unknown_span_context_synthesizes), the `attribute.` + and `resource.` contexts synthesize an unknown key (existence is a property + of the data, not metadata): a filter reference matches nothing, a select + projects a null column, and an order clause succeeds and returns every span. + """ + now = datetime.now(tz=UTC).replace(microsecond=0) + trace_id = TraceIdGenerator.trace_id() + spans = [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + trace_id=trace_id, + span_id=TraceIdGenerator.span_id(), + name=f"synth-{i}", + resources={"service.name": "synth-service"}, + ) + for i in range(3) + ] + insert_traces(spans) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms, end_ms = _query_window(now) + key = f"{context}.does_not_exist" + svc = "resource.service.name = 'synth-service'" + + if surface == "filter": + query = BuilderQuery(signal="traces", name="A", filter_expression=f"{svc} AND {key} = 'nope'") + elif surface == "select": + query = BuilderQuery(signal="traces", name="A", filter_expression=svc, select_fields=[TelemetryFieldKey(key)]) + else: # order + query = BuilderQuery(signal="traces", name="A", filter_expression=svc, order=[OrderBy(TelemetryFieldKey(key), "desc")]) + + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[query.to_dict()], + ) + + assert response.status_code == HTTPStatus.OK, response.text + rows = get_rows(response) + + if surface == "filter": + assert rows == [] + elif surface == "select": + assert {span.span_id for span in spans} == {row["data"]["span_id"] for row in rows} + for row in rows: + assert row["data"]["does_not_exist"] is None + else: # order + assert {span.span_id for span in spans} == {row["data"]["span_id"] for row in rows} + + +def test_traces_list_order_unknown_key_synthesizes( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], +) -> None: + """ + Setup: + Insert 3 spans. + + Tests: + Ordering by a bare (context-less) unknown key synthesizes a null column and + succeeds, returning every matching span — the ordering counterpart of the + bare-select-null case and the unknown-filter synthesis in 09_unknown_keys. + """ + now = datetime.now(tz=UTC).replace(microsecond=0) + trace_id = TraceIdGenerator.trace_id() + spans = [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + trace_id=trace_id, + span_id=TraceIdGenerator.span_id(), + name=f"order-unknown-{i}", + resources={"service.name": "order-unknown-service"}, + ) + for i in range(3) + ] + insert_traces(spans) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + start_ms, end_ms = _query_window(now) + + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[ + BuilderQuery( + signal="traces", + name="A", + filter_expression="resource.service.name = 'order-unknown-service'", + order=[OrderBy(TelemetryFieldKey("does_not_exist"), "desc")], + ).to_dict() + ], + ) + + assert response.status_code == HTTPStatus.OK + rows = get_rows(response) + assert {row["data"]["span_id"] for row in rows} == {span.span_id for span in spans} diff --git a/tests/integration/tests/queriertraces/02_aggregation.py b/tests/integration/tests/queriertraces/02_aggregation.py index a36ba82e3d3..9d3d9d6ece1 100644 --- a/tests/integration/tests/queriertraces/02_aggregation.py +++ b/tests/integration/tests/queriertraces/02_aggregation.py @@ -1,72 +1,70 @@ from collections.abc import Callable from datetime import UTC, datetime, timedelta from http import HTTPStatus +from typing import Any +import numpy as np import pytest from fixtures import types from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD from fixtures.querier import ( + Aggregation, + BuilderQuery, + OrderBy, + RequestType, + TelemetryFieldKey, + assert_scalar_result_order, + build_aggregation, + build_group_by_field, + build_order_by, + build_traces_scalar_query, + get_all_series, + get_scalar_table_data, + index_series_by_label, make_query_request, + make_scalar_query_request, ) -from fixtures.traces import ( - TraceIdGenerator, - Traces, - TracesKind, - TracesStatusCode, -) +from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode, trace_noise + +# The clean/corrupt `trace_noise` factor (fixtures/traces.py) is applied per test via +# @pytest.mark.parametrize("noise", …). Aggregating duration_nano, counting on the calculated +# response_status_code and grouping by service.name must all keep resolving to +# the real intrinsic/calculated/resource columns even when same-named colliding +# attributes are present, so the corrupt variant yields identical values. + + +# ============================================================================ +# Order-by referencing an aggregation +# ============================================================================ @pytest.mark.parametrize( "order_by,aggregation_alias,expected_status", [ - # Case 1a: count by count() - pytest.param({"name": "count()"}, "count_", HTTPStatus.OK), - # Case 1b: count by count() with alias span.count_ - pytest.param({"name": "count()"}, "span.count_", HTTPStatus.OK), - # Case 2a: count by count() with context specified in the key - pytest.param({"name": "count()", "fieldContext": "span"}, "count_", HTTPStatus.OK), - # Case 2b: count by count() with context specified in the key with alias span.count_ - pytest.param({"name": "count()", "fieldContext": "span"}, "span.count_", HTTPStatus.OK), - # Case 3a: count by span.count() and context specified in the key [BAD REQUEST] - pytest.param( - {"name": "span.count()", "fieldContext": "span"}, - "count_", - HTTPStatus.BAD_REQUEST, - ), - # Case 3b: count by span.count() and context specified in the key with alias span.count_ [BAD REQUEST] - pytest.param( - {"name": "span.count()", "fieldContext": "span"}, - "span.count_", - HTTPStatus.BAD_REQUEST, - ), - # Case 4a: count by span.count() and context specified in the key - pytest.param({"name": "span.count()", "fieldContext": ""}, "count_", HTTPStatus.OK), - # Case 4b: count by span.count() and context specified in the key with alias span.count_ - pytest.param({"name": "span.count()", "fieldContext": ""}, "span.count_", HTTPStatus.OK), - # Case 5a: count by count_ - pytest.param({"name": "count_"}, "count_", HTTPStatus.OK), - # Case 5b: count by count_ with alias span.count_ - pytest.param({"name": "count_"}, "count_", HTTPStatus.OK), - # Case 6a: count by span.count_ - pytest.param({"name": "span.count_"}, "count_", HTTPStatus.OK), - # Case 6b: count by span.count_ with alias span.count_ - pytest.param({"name": "span.count_"}, "span.count_", HTTPStatus.OK), - # Case 7a: count by span.count_ and context specified in the key [BAD REQUEST] - pytest.param( - {"name": "span.count_", "fieldContext": "span"}, - "count_", - HTTPStatus.BAD_REQUEST, - ), - # Case 7b: count by span.count_ and context specified in the key with alias span.count_ - pytest.param( - {"name": "span.count_", "fieldContext": "span"}, - "span.count_", - HTTPStatus.OK, - ), + # Case 1: count by count() + pytest.param({"name": "count()"}, "count_", HTTPStatus.OK, id="count()_alias_count_"), + pytest.param({"name": "count()"}, "span.count_", HTTPStatus.OK, id="count()_alias_span.count_"), + # Case 2: count() with context specified in the key + pytest.param({"name": "count()", "fieldContext": "span"}, "count_", HTTPStatus.OK, id="span-ctx_count()_alias_count_"), + pytest.param({"name": "count()", "fieldContext": "span"}, "span.count_", HTTPStatus.OK, id="span-ctx_count()_alias_span.count_"), + # Case 3: span.count() and context specified in the key [BAD REQUEST] + pytest.param({"name": "span.count()", "fieldContext": "span"}, "count_", HTTPStatus.BAD_REQUEST, id="span.count()_span-ctx_bad"), + pytest.param({"name": "span.count()", "fieldContext": "span"}, "span.count_", HTTPStatus.BAD_REQUEST, id="span.count()_span-ctx_alias_bad"), + # Case 4: span.count() with empty context + pytest.param({"name": "span.count()", "fieldContext": ""}, "count_", HTTPStatus.OK, id="span.count()_no-ctx_alias_count_"), + pytest.param({"name": "span.count()", "fieldContext": ""}, "span.count_", HTTPStatus.OK, id="span.count()_no-ctx_alias_span.count_"), + # Case 5: count_ (the alias) + pytest.param({"name": "count_"}, "count_", HTTPStatus.OK, id="count_alias_bare"), + # Case 6: span.count_ + pytest.param({"name": "span.count_"}, "count_", HTTPStatus.OK, id="span.count_alias_count_"), + pytest.param({"name": "span.count_"}, "span.count_", HTTPStatus.OK, id="span.count_alias_span.count_"), + # Case 7: span.count_ with context specified in the key + pytest.param({"name": "span.count_", "fieldContext": "span"}, "count_", HTTPStatus.BAD_REQUEST, id="span.count_span-ctx_bad"), + pytest.param({"name": "span.count_", "fieldContext": "span"}, "span.count_", HTTPStatus.OK, id="span.count_span-ctx_alias_span.count_"), ], ) -def test_traces_aggergate_order_by_count( +def test_traces_aggregate_order_by_count( signoz: types.SigNoz, create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], @@ -77,165 +75,157 @@ def test_traces_aggergate_order_by_count( ) -> None: """ Setup: - Insert 4 traces with different attributes. - http-service: POST /integration -> SELECT, HTTP PATCH - topic-service: topic publish + Insert 4 spans of a single service. Tests: - 1. Query traces count for spans grouped by service.name and host.name + An aggregation can be referenced in `order by` by its expression (count()), + its `span.`-prefixed expression, or its alias — with or without an explicit + fieldContext. Only the combinations that double-specify the context are a bad + request; the rest resolve and return the span count. """ - http_service_trace_id = TraceIdGenerator.trace_id() - http_service_span_id = TraceIdGenerator.span_id() - http_service_db_span_id = TraceIdGenerator.span_id() - http_service_patch_span_id = TraceIdGenerator.span_id() - topic_service_trace_id = TraceIdGenerator.trace_id() - topic_service_span_id = TraceIdGenerator.span_id() - now = datetime.now(tz=UTC).replace(second=0, microsecond=0) - - insert_traces( - [ - Traces( - timestamp=now - timedelta(seconds=4), - duration=timedelta(seconds=3), - trace_id=http_service_trace_id, - span_id=http_service_span_id, - parent_span_id="", - name="POST /integration", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={ - "deployment.environment": "production", - "service.name": "http-service", - "os.type": "linux", - "host.name": "linux-000", - "cloud.provider": "integration", - "cloud.account.id": "000", - }, - attributes={ - "net.transport": "IP.TCP", - "http.scheme": "http", - "http.user_agent": "Integration Test", - "http.request.method": "POST", - "http.response.status_code": "200", - }, - ), - Traces( - timestamp=now - timedelta(seconds=3.5), - duration=timedelta(seconds=0.5), - trace_id=http_service_trace_id, - span_id=http_service_db_span_id, - parent_span_id=http_service_span_id, - name="SELECT", - kind=TracesKind.SPAN_KIND_CLIENT, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={ - "deployment.environment": "production", - "service.name": "http-service", - "os.type": "linux", - "host.name": "linux-000", - "cloud.provider": "integration", - "cloud.account.id": "000", - }, - attributes={ - "db.name": "integration", - "db.operation": "SELECT", - "db.statement": "SELECT * FROM integration", - }, - ), - Traces( - timestamp=now - timedelta(seconds=3), - duration=timedelta(seconds=1), - trace_id=http_service_trace_id, - span_id=http_service_patch_span_id, - parent_span_id=http_service_span_id, - name="HTTP PATCH", - kind=TracesKind.SPAN_KIND_CLIENT, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={ - "deployment.environment": "production", - "service.name": "http-service", - "os.type": "linux", - "host.name": "linux-000", - "cloud.provider": "integration", - "cloud.account.id": "000", - }, - attributes={ - "http.request.method": "PATCH", - "http.status_code": "404", - }, - ), - Traces( - timestamp=now - timedelta(seconds=1), - duration=timedelta(seconds=4), - trace_id=topic_service_trace_id, - span_id=topic_service_span_id, - parent_span_id="", - name="topic publish", - kind=TracesKind.SPAN_KIND_PRODUCER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={ - "deployment.environment": "production", - "service.name": "topic-service", - "os.type": "linux", - "host.name": "linux-001", - "cloud.provider": "integration", - "cloud.account.id": "001", - }, - attributes={ - "message.type": "SENT", - "messaging.operation": "publish", - "messaging.message.id": "001", - }, - ), - ] - ) + trace_id = TraceIdGenerator.trace_id() + spans = [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + trace_id=trace_id, + span_id=TraceIdGenerator.span_id(), + name=f"op-{i}", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": "agg-order-service"}, + ) + for i in range(4) + ] + insert_traces(spans) token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - query = { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "disabled": False, - "order": [{"key": {"name": "count()"}, "direction": "desc"}], - "aggregations": [{"expression": "count()", "alias": "count_"}], - }, - } - - # Query traces count for spans - - query["spec"]["order"][0]["key"] = order_by - query["spec"]["aggregations"][0]["alias"] = aggregation_alias response = make_query_request( signoz, token, start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000), end_ms=int(datetime.now(tz=UTC).timestamp() * 1000), - request_type="time_series", - queries=[query], + request_type=RequestType.TIME_SERIES, + queries=[ + BuilderQuery( + signal="traces", + name="A", + order=[OrderBy(TelemetryFieldKey(order_by["name"], field_context=order_by.get("fieldContext")), "desc")], + aggregations=[Aggregation("count()", aggregation_alias)], + ).to_dict() + ], ) - assert response.status_code == expected_status + assert response.status_code == expected_status, response.text if expected_status != HTTPStatus.OK: return - assert response.json()["status"] == "success" results = response.json()["data"]["data"]["results"] assert len(results) == 1 - aggregations = results[0]["aggregations"] - assert len(aggregations) == 1 - series = aggregations[0]["series"] + series = results[0]["aggregations"][0]["series"] assert len(series) == 1 - assert series[0]["values"][0]["value"] == 4 + assert series[0]["values"][0]["value"] == len(spans) + + +# ============================================================================ +# Aggregation function value correctness (scalar, grouped) +# ============================================================================ + + +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_aggregate_functions( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, +) -> None: + """ + Setup: + svc-a: 3 spans (durations 1s/2s/3s; one ERROR; one 4XX via + http.response.status_code); svc-b: 1 span (duration 4s). A number attribute + `latency_ms` rides along on every span. + + Tests: + A grouped scalar query computes count / sum / avg / min / max / p50 / p90 over + duration_nano, countIf over the intrinsic status_code and the calculated + response_status_code, and avg over a numeric attribute — all matching values + derived from the inserted spans. Under the corrupt variant the same-named + colliding attributes must not change any of these. + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + + def mk(service: str, dur_s: float, status: TracesStatusCode, rsc: str, latency: float) -> Traces: + return Traces( + timestamp=now - timedelta(seconds=dur_s), + duration=timedelta(seconds=dur_s), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + kind=TracesKind.SPAN_KIND_SERVER, + status_code=status, + resources={"service.name": service, **extra_resources}, + name=f"{service}-{dur_s}", + attributes={"http.response.status_code": rsc, "latency_ms": latency, **extra_attrs}, + ) + + spans = [ + mk("svc-a", 1, TracesStatusCode.STATUS_CODE_OK, "200", 10), + mk("svc-a", 2, TracesStatusCode.STATUS_CODE_ERROR, "503", 20), + mk("svc-a", 3, TracesStatusCode.STATUS_CODE_OK, "404", 30), + mk("svc-b", 4, TracesStatusCode.STATUS_CODE_OK, "200", 40), + ] + insert_traces(spans) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + aggregations = [ + build_aggregation("count()", "cnt"), + build_aggregation("sum(duration_nano)", "sum_d"), + build_aggregation("avg(duration_nano)", "avg_d"), + build_aggregation("min(duration_nano)", "min_d"), + build_aggregation("max(duration_nano)", "max_d"), + build_aggregation("p50(duration_nano)", "p50_d"), + build_aggregation("p90(duration_nano)", "p90_d"), + build_aggregation("countIf(status_code = 2)", "errs"), + build_aggregation("avg(latency_ms)", "avg_lat"), + ] + query = build_traces_scalar_query( + aggregations=aggregations, + group_by=[build_group_by_field("service.name", "string", "resource")], + order=[build_order_by("count()", "desc")], + ) + response = make_scalar_query_request(signoz, token, now, [query]) + + assert response.status_code == HTTPStatus.OK, response.text + data = get_scalar_table_data(response.json()) + + def expected(group: list[Traces]) -> tuple: + durations = [int(s.duration_nano) for s in group] + latencies = [float(s.attributes_number["latency_ms"]) for s in group] + return ( + group[0].service_name, + len(group), # count() + sum(durations), # sum(duration_nano) + sum(durations) / len(durations), # avg(duration_nano) + min(durations), # min(duration_nano) + max(durations), # max(duration_nano) + float(np.percentile(durations, 50)), # p50(duration_nano) + float(np.percentile(durations, 90)), # p90(duration_nano) + sum(1 for s in group if int(s.status_code) == 2), # countIf(status_code = 2) + sum(latencies) / len(latencies), # avg(latency_ms) + ) + + # Ordered by count() desc: svc-a (3) then svc-b (1). + assert_scalar_result_order( + data, + [expected(spans[:3]), expected(spans[3:])], + "traces aggregate functions", + ) -def test_traces_aggregate_with_mixed_field_selectors( +def test_traces_aggregate_calculated_range_countif( signoz: types.SigNoz, create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], @@ -243,166 +233,233 @@ def test_traces_aggregate_with_mixed_field_selectors( ) -> None: """ Setup: - Insert 4 traces with different attributes. - http-service: POST /integration -> SELECT, HTTP PATCH - topic-service: topic publish + 3 spans whose derived response_status_code is 200 / 503 / 404. Tests: - 1. Query traces count for spans grouped by service.name + countIf(response_status_code >= 400 AND response_status_code < 500) counts the + single 4XX span (404). + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + spans = [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name=f"rsc-{rsc}", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": "rsc-service"}, + attributes={"http.response.status_code": rsc}, + ) + for i, rsc in enumerate(["200", "503", "404"]) + ] + insert_traces(spans) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + query = build_traces_scalar_query( + aggregations=[build_aggregation("countIf(response_status_code >= 400 AND response_status_code < 500)")], + ) + response = make_scalar_query_request(signoz, token, now, [query]) + + assert response.status_code == HTTPStatus.OK, response.text + data = get_scalar_table_data(response.json()) + expected = sum(1 for s in spans if s.response_status_code.isdigit() and 400 <= int(s.response_status_code) < 500) + assert data[0][0] == expected + + +# ============================================================================ +# HAVING (post-aggregation filter) +# ============================================================================ + + +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_aggregate_having( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, +) -> None: """ - http_service_trace_id = TraceIdGenerator.trace_id() - http_service_span_id = TraceIdGenerator.span_id() - http_service_db_span_id = TraceIdGenerator.span_id() - http_service_patch_span_id = TraceIdGenerator.span_id() - topic_service_trace_id = TraceIdGenerator.trace_id() - topic_service_span_id = TraceIdGenerator.span_id() + Setup: + svc-a: 3 spans, svc-b: 1 span. + Tests: + A grouped scalar query with `having count() > 2` returns only the groups + whose count qualifies (svc-a), derived from the inserted spans. + """ + extra_attrs, extra_resources = trace_noise(noise) now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + counts = {"svc-a": 3, "svc-b": 1} + spans = [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name=f"{service}-{i}", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": service, **extra_resources}, + attributes=dict(extra_attrs), + ) + for service, count in counts.items() + for i in range(count) + ] + insert_traces(spans) - insert_traces( - [ - Traces( - timestamp=now - timedelta(seconds=4), - duration=timedelta(seconds=3), - trace_id=http_service_trace_id, - span_id=http_service_span_id, - parent_span_id="", - name="POST /integration", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={ - "deployment.environment": "production", - "service.name": "http-service", - "os.type": "linux", - "host.name": "linux-000", - "cloud.provider": "integration", - "cloud.account.id": "000", - }, - attributes={ - "net.transport": "IP.TCP", - "http.scheme": "http", - "http.user_agent": "Integration Test", - "http.request.method": "POST", - "http.response.status_code": "200", - }, - ), - Traces( - timestamp=now - timedelta(seconds=3.5), - duration=timedelta(seconds=0.5), - trace_id=http_service_trace_id, - span_id=http_service_db_span_id, - parent_span_id=http_service_span_id, - name="SELECT", - kind=TracesKind.SPAN_KIND_CLIENT, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={ - "deployment.environment": "production", - "service.name": "http-service", - "os.type": "linux", - "host.name": "linux-000", - "cloud.provider": "integration", - "cloud.account.id": "000", - }, - attributes={ - "db.name": "integration", - "db.operation": "SELECT", - "db.statement": "SELECT * FROM integration", - }, - ), - Traces( - timestamp=now - timedelta(seconds=3), - duration=timedelta(seconds=1), - trace_id=http_service_trace_id, - span_id=http_service_patch_span_id, - parent_span_id=http_service_span_id, - name="HTTP PATCH", - kind=TracesKind.SPAN_KIND_CLIENT, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={ - "deployment.environment": "production", - "service.name": "http-service", - "os.type": "linux", - "host.name": "linux-000", - "cloud.provider": "integration", - "cloud.account.id": "000", - }, - attributes={ - "http.request.method": "PATCH", - "http.status_code": "404", - }, - ), - Traces( - timestamp=now - timedelta(seconds=1), - duration=timedelta(seconds=4), - trace_id=topic_service_trace_id, - span_id=topic_service_span_id, - parent_span_id="", - name="topic publish", - kind=TracesKind.SPAN_KIND_PRODUCER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={ - "deployment.environment": "production", - "service.name": "topic-service", - "os.type": "linux", - "host.name": "linux-001", - "cloud.provider": "integration", - "cloud.account.id": "001", - }, - attributes={ - "message.type": "SENT", - "messaging.operation": "publish", - "messaging.message.id": "001", - }, - ), - ] + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + query = build_traces_scalar_query( + aggregations=[build_aggregation("count()")], + group_by=[build_group_by_field("service.name", "string", "resource")], + order=[build_order_by("count()", "desc")], + having_expression="count() > 2", ) + response = make_scalar_query_request(signoz, token, now, [query]) + + assert response.status_code == HTTPStatus.OK, response.text + data = get_scalar_table_data(response.json()) + + expected = [(service, count) for service, count in sorted(counts.items(), key=lambda kv: kv[1], reverse=True) if count > 2] + assert_scalar_result_order(data, expected, "traces having count() > 2") + + +# ============================================================================ +# Limit (top-N series) — the time_series counterpart of the scalar top-N groups +# already covered in querierscalar/02_traces.py. +# ============================================================================ + + +@pytest.mark.parametrize( + "limit,direction", + [ + pytest.param(2, "desc", id="top_2_desc"), + pytest.param(3, "desc", id="top_3_desc"), + pytest.param(2, "asc", id="bottom_2_asc"), + pytest.param(10, "desc", id="limit_exceeds_group_count"), + ], +) +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_aggregate_time_series_limit( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, + limit: int, + direction: str, +) -> None: + """ + Setup: + Four services with distinct span counts (a=5, b=3, c=7, d=1). + + Tests: + A time_series group-by with a limit returns only the N series that the + ordered aggregation keeps — top-N for desc, bottom-N for asc — each series + summing to that service's span count. Under the corrupt variant the grouping + (resource.service.name) and count are unaffected by the colliding attributes. + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + counts = {"svc-a": 5, "svc-b": 3, "svc-c": 7, "svc-d": 1} + spans = [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name=f"{service}-{i}", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": service, **extra_resources}, + attributes=dict(extra_attrs), + ) + for service, count in counts.items() + for i in range(count) + ] + insert_traces(spans) token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - query = { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "groupBy": [ - { - "name": "service.name", - "fieldContext": "resource", - "fieldDataType": "string", - } - ], - "aggregations": [ - {"expression": "p99(duration_nano)", "alias": "p99"}, - {"expression": "avg(duration_nano)", "alias": "avgDuration"}, - {"expression": "count()", "alias": "numCalls"}, - {"expression": "countIf(status_code = 2)", "alias": "numErrors"}, - { - "expression": "countIf(response_status_code >= 400 AND response_status_code < 500)", - "alias": "num4XX", - }, - ], - "order": [{"key": {"name": "count()"}, "direction": "desc"}], - }, - } - - # Query traces count for spans + query = build_traces_scalar_query( + aggregations=[build_aggregation("count()")], + group_by=[build_group_by_field("service.name", "string", "resource")], + order=[build_order_by("count()", direction)], + limit=limit, + ) response = make_query_request( signoz, token, - start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000), - end_ms=int(datetime.now(tz=UTC).timestamp() * 1000), - request_type="time_series", + start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type=RequestType.TIME_SERIES, queries=[query], ) - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - results = response.json()["data"]["data"]["results"] - assert len(results) == 1 - aggregations = results[0]["aggregations"] + assert response.status_code == HTTPStatus.OK, response.text + + ordered = sorted(counts.items(), key=lambda kv: kv[1], reverse=(direction == "desc")) + expected = dict(ordered[:limit]) + + by_service = index_series_by_label(get_all_series(response.json(), "A"), "service.name") + assert set(by_service) == set(expected) + for service, series in by_service.items(): + assert sum(point["value"] for point in series["values"]) == expected[service] + + +# ============================================================================ +# Aggregating an unknown key +# ============================================================================ + + +@pytest.mark.parametrize( + "expression,expected_value", + [ + pytest.param("count(does_not_exist)", 0, id="count_unknown_is_zero"), + pytest.param("sum(does_not_exist)", None, id="sum_unknown_is_null"), + pytest.param("avg(does_not_exist)", None, id="avg_unknown_is_null"), + ], +) +def test_traces_aggregate_unknown_key( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + expression: str, + expected_value: Any, +) -> None: + """ + Setup: + Insert 3 spans. + + Tests: + Aggregating a bare unknown key synthesizes a null column: count() of it is 0 + (no non-null values) and sum()/avg() are null — the query never errors. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + spans = [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name=f"unknown-agg-{i}", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": "unknown-agg-service"}, + ) + for i in range(3) + ] + insert_traces(spans) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + query = build_traces_scalar_query(aggregations=[build_aggregation(expression)]) + response = make_scalar_query_request(signoz, token, now, [query]) - assert aggregations[0]["series"][0]["values"][0]["value"] >= 2.5 * 1e9 # p99 for http-service + assert response.status_code == HTTPStatus.OK, response.text + data = get_scalar_table_data(response.json()) + assert len(data) == 1 + if expected_value is None: + assert data[0][0] is None + else: + assert data[0][0] == expected_value diff --git a/tests/integration/tests/queriertraces/03_fill.py b/tests/integration/tests/queriertraces/03_fill.py index 64033794028..b0404f3d0e5 100644 --- a/tests/integration/tests/queriertraces/03_fill.py +++ b/tests/integration/tests/queriertraces/03_fill.py @@ -2,934 +2,306 @@ from datetime import UTC, datetime, timedelta from http import HTTPStatus -import requests +import pytest from fixtures import types from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD from fixtures.querier import ( + RequestType, assert_minutely_bucket_values, + build_aggregation, + build_formula_query, + build_group_by_field, + build_traces_scalar_query, find_named_result, + get_all_series, index_series_by_label, + make_query_request, ) -from fixtures.traces import ( - TraceIdGenerator, - Traces, - TracesKind, - TracesStatusCode, -) - - -def test_traces_fill_gaps( - signoz: types.SigNoz, - create_user_admin: None, # pylint: disable=unused-argument - get_token: Callable[[str, str], str], - insert_traces: Callable[[list[Traces]], None], -) -> None: - """ - Test fillGaps for traces without groupBy. - """ - now = datetime.now(tz=UTC).replace(second=0, microsecond=0) - trace_id = TraceIdGenerator.trace_id() - - traces: list[Traces] = [ - Traces( - timestamp=now - timedelta(minutes=3), - duration=timedelta(seconds=1), - trace_id=trace_id, - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="test-span", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "test-service"}, - attributes={"http.method": "GET"}, - ), - ] - insert_traces(traces) - - token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - - start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000) - end_ms = int(now.timestamp() * 1000) - - response = requests.post( - signoz.self.host_configs["8080"].get("/api/v5/query_range"), - timeout=5, - headers={"authorization": f"Bearer {token}"}, - json={ - "schemaVersion": "v1", - "start": start_ms, - "end": end_ms, - "requestType": "time_series", - "compositeQuery": { - "queries": [ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "stepInterval": 60, - "disabled": False, - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - } - ] - }, - "formatOptions": {"formatTableResultForUI": False, "fillGaps": True}, - }, - ) - - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" +from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode, trace_noise - results = response.json()["data"]["data"]["results"] - assert len(results) == 1 +# Each fill test runs under two mechanisms that both 0-fill empty time buckets: +# - "fillGaps": the `fillGaps` format option (fills the query-window timeline) +# - "fillZero": the `fillZero` function (applied to the query / formula series) +# and under the shared clean/corrupt `trace_noise` factor (fixtures/traces.py): count() +# and group-by(resource.service.name) buckets must be identical whether or not +# same-named colliding attributes are present. +FILL_MODES = ["fillGaps", "fillZero"] - aggregations = results[0]["aggregations"] - assert len(aggregations) == 1 - - series = aggregations[0]["series"] - assert len(series) >= 1 - values = series[0]["values"] - ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000) - assert_minutely_bucket_values( - values, - now, - expected_by_ts={ts_min_3: 1}, - context="traces/fillGaps", - ) +def _bucket_ms(span: Traces) -> int: + """The minutely bucket (epoch millis) a seeded span lands in.""" + return int(span.timestamp.timestamp() * 1000) -def test_traces_fill_gaps_with_group_by( +@pytest.mark.parametrize("fill_mode", FILL_MODES) +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_fill_plain( signoz: types.SigNoz, create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], insert_traces: Callable[[list[Traces]], None], + noise: str, + fill_mode: str, ) -> None: """ - Test fillGaps for traces with groupBy. - """ - now = datetime.now(tz=UTC).replace(second=0, microsecond=0) - - traces: list[Traces] = [ - Traces( - timestamp=now - timedelta(minutes=3), - duration=timedelta(seconds=1), - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="span-a", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "service-a"}, - attributes={"http.method": "GET"}, - ), - Traces( - timestamp=now - timedelta(minutes=2), - duration=timedelta(seconds=1), - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="span-b", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "service-b"}, - attributes={"http.method": "POST"}, - ), - ] - insert_traces(traces) - - token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - - start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000) - end_ms = int(now.timestamp() * 1000) - - response = requests.post( - signoz.self.host_configs["8080"].get("/api/v5/query_range"), - timeout=5, - headers={"authorization": f"Bearer {token}"}, - json={ - "schemaVersion": "v1", - "start": start_ms, - "end": end_ms, - "requestType": "time_series", - "compositeQuery": { - "queries": [ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "stepInterval": 60, - "disabled": False, - "groupBy": [ - { - "name": "service.name", - "fieldDataType": "string", - "fieldContext": "resource", - } - ], - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - } - ] - }, - "formatOptions": {"formatTableResultForUI": False, "fillGaps": True}, - }, - ) - - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - - results = response.json()["data"]["data"]["results"] - assert len(results) == 1 - - aggregations = results[0]["aggregations"] - assert len(aggregations) == 1 + Setup: + One span 3 minutes ago. - series = aggregations[0]["series"] - assert len(series) == 2, "Expected 2 series for 2 service groups" - - ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000) - ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000) - - series_by_service = index_series_by_label(series, "service.name") - assert set(series_by_service.keys()) == {"service-a", "service-b"} - - expectations: dict[str, dict[int, float]] = { - "service-a": {ts_min_3: 1}, - "service-b": {ts_min_2: 1}, - } - - for service_name, s in series_by_service.items(): - assert_minutely_bucket_values( - s["values"], - now, - expected_by_ts=expectations[service_name], - context=f"traces/fillGaps/{service_name}", - ) - - -def test_traces_fill_gaps_formula( - signoz: types.SigNoz, - create_user_admin: None, # pylint: disable=unused-argument - get_token: Callable[[str, str], str], - insert_traces: Callable[[list[Traces]], None], -) -> None: - """ - Test fillGaps for traces with formula. + Tests: + fillGaps / fillZero over count() with no group by leave the single populated + bucket at the span count and every other bucket in the window at 0. """ + extra_attrs, extra_resources = trace_noise(noise) now = datetime.now(tz=UTC).replace(second=0, microsecond=0) - - traces: list[Traces] = [ - Traces( - timestamp=now - timedelta(minutes=3), - duration=timedelta(seconds=1), - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="test-span", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "test"}, - attributes={"http.method": "GET"}, - ), - Traces( - timestamp=now - timedelta(minutes=2), - duration=timedelta(seconds=1), - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="another-test-span", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "another-test"}, - attributes={"http.method": "POST"}, - ), - ] - insert_traces(traces) + span = Traces( + timestamp=now - timedelta(minutes=3), + duration=timedelta(seconds=1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name="test-span", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": "fill-service", **extra_resources}, + attributes={"http.method": "GET", **extra_attrs}, + ) + insert_traces([span]) token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - - start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000) - end_ms = int(now.timestamp() * 1000) - - response = requests.post( - signoz.self.host_configs["8080"].get("/api/v5/query_range"), - timeout=5, - headers={"authorization": f"Bearer {token}"}, - json={ - "schemaVersion": "v1", - "start": start_ms, - "end": end_ms, - "requestType": "time_series", - "compositeQuery": { - "queries": [ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "stepInterval": 60, - "disabled": True, - "filter": {"expression": "service.name = 'test'"}, - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - }, - { - "type": "builder_query", - "spec": { - "name": "B", - "signal": "traces", - "stepInterval": 60, - "disabled": True, - "filter": {"expression": "service.name = 'another-test'"}, - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - }, - { - "type": "builder_formula", - "spec": { - "name": "F1", - "expression": "A + B", - "disabled": False, - }, - }, - ] - }, - "formatOptions": {"formatTableResultForUI": False, "fillGaps": True}, - }, + functions = [{"name": "fillZero"}] if fill_mode == "fillZero" else None + format_options = {"formatTableResultForUI": False, "fillGaps": fill_mode == "fillGaps"} + + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type=RequestType.TIME_SERIES, + queries=[build_traces_scalar_query(aggregations=[build_aggregation("count()")], functions=functions)], + format_options=format_options, ) - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - - results = response.json()["data"]["data"]["results"] - assert len(results) == 1 - - ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000) - ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000) - - f1 = find_named_result(results, "F1") - assert f1 is not None, "Expected formula result named F1" - - aggregations = f1.get("aggregations") or [] - assert len(aggregations) == 1 - series = aggregations[0]["series"] + assert response.status_code == HTTPStatus.OK, response.text + series = get_all_series(response.json(), "A") assert len(series) >= 1 - - assert_minutely_bucket_values( - series[0]["values"], - now, - expected_by_ts={ts_min_3: 1, ts_min_2: 1}, - context="traces/fillGaps/F1", - ) + assert_minutely_bucket_values(series[0]["values"], now, expected_by_ts={_bucket_ms(span): 1}, context=f"traces/{fill_mode}") -def test_traces_fill_gaps_formula_with_group_by( +@pytest.mark.parametrize("fill_mode", FILL_MODES) +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_fill_group_by( signoz: types.SigNoz, create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], insert_traces: Callable[[list[Traces]], None], + noise: str, + fill_mode: str, ) -> None: """ - Test fillGaps for traces with formula and groupBy. + Setup: + service-a span 3 minutes ago, service-b span 2 minutes ago. + + Tests: + fillGaps / fillZero over count() grouped by service.name yield one series per + service, each with its single populated bucket and zeros elsewhere. """ + extra_attrs, extra_resources = trace_noise(noise) now = datetime.now(tz=UTC).replace(second=0, microsecond=0) - - traces: list[Traces] = [ - Traces( - timestamp=now - timedelta(minutes=3), - duration=timedelta(seconds=1), - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="span-group1", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "group1"}, - attributes={"http.method": "GET"}, - ), - Traces( - timestamp=now - timedelta(minutes=2), - duration=timedelta(seconds=1), - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="span-group2", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "group2"}, - attributes={"http.method": "POST"}, - ), - ] - insert_traces(traces) + span_a = Traces( + timestamp=now - timedelta(minutes=3), + duration=timedelta(seconds=1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name="span-a", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": "service-a", **extra_resources}, + attributes={"http.method": "GET", **extra_attrs}, + ) + span_b = Traces( + timestamp=now - timedelta(minutes=2), + duration=timedelta(seconds=1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name="span-b", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": "service-b", **extra_resources}, + attributes={"http.method": "POST", **extra_attrs}, + ) + insert_traces([span_a, span_b]) token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - - start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000) - end_ms = int(now.timestamp() * 1000) - - response = requests.post( - signoz.self.host_configs["8080"].get("/api/v5/query_range"), - timeout=5, - headers={"authorization": f"Bearer {token}"}, - json={ - "schemaVersion": "v1", - "start": start_ms, - "end": end_ms, - "requestType": "time_series", - "compositeQuery": { - "queries": [ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "stepInterval": 60, - "disabled": True, - "groupBy": [ - { - "name": "service.name", - "fieldDataType": "string", - "fieldContext": "resource", - } - ], - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - }, - { - "type": "builder_query", - "spec": { - "name": "B", - "signal": "traces", - "stepInterval": 60, - "disabled": True, - "groupBy": [ - { - "name": "service.name", - "fieldDataType": "string", - "fieldContext": "resource", - } - ], - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - }, - { - "type": "builder_formula", - "spec": { - "name": "F1", - "expression": "A + B", - "disabled": False, - }, - }, - ] - }, - "formatOptions": {"formatTableResultForUI": False, "fillGaps": True}, - }, + functions = [{"name": "fillZero"}] if fill_mode == "fillZero" else None + format_options = {"formatTableResultForUI": False, "fillGaps": fill_mode == "fillGaps"} + + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type=RequestType.TIME_SERIES, + queries=[ + build_traces_scalar_query( + aggregations=[build_aggregation("count()")], + group_by=[build_group_by_field("service.name", "string", "resource")], + functions=functions, + ) + ], + format_options=format_options, ) - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - - results = response.json()["data"]["data"]["results"] - assert len(results) == 1 - - ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000) - ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000) - - f1 = find_named_result(results, "F1") - assert f1 is not None, "Expected formula result named F1" - - aggregations = f1.get("aggregations") or [] - assert len(aggregations) == 1 - series = aggregations[0]["series"] + assert response.status_code == HTTPStatus.OK, response.text + series = get_all_series(response.json(), "A") assert len(series) == 2 - series_by_service = index_series_by_label(series, "service.name") - assert set(series_by_service.keys()) == {"group1", "group2"} + by_service = index_series_by_label(series, "service.name") + assert set(by_service) == {"service-a", "service-b"} + assert_minutely_bucket_values(by_service["service-a"]["values"], now, expected_by_ts={_bucket_ms(span_a): 1}, context=f"traces/{fill_mode}/service-a") + assert_minutely_bucket_values(by_service["service-b"]["values"], now, expected_by_ts={_bucket_ms(span_b): 1}, context=f"traces/{fill_mode}/service-b") - expectations: dict[str, dict[int, float]] = { - "group1": {ts_min_3: 2}, - "group2": {ts_min_2: 2}, - } - for service_name, s in series_by_service.items(): - assert_minutely_bucket_values( - s["values"], - now, - expected_by_ts=expectations[service_name], - context=f"traces/fillGaps/F1/{service_name}", - ) - - -def test_traces_fill_zero( +@pytest.mark.parametrize("fill_mode", FILL_MODES) +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_fill_formula( signoz: types.SigNoz, create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], insert_traces: Callable[[list[Traces]], None], + noise: str, + fill_mode: str, ) -> None: """ - Test fillZero function for traces without groupBy. + Setup: + 'test' service span 3 minutes ago, 'another-test' service span 2 minutes ago. + + Tests: + A formula F1 = A + B over two disabled per-service count() queries fills empty + buckets to 0, so F1 carries a 1 in each service's populated bucket. """ + extra_attrs, extra_resources = trace_noise(noise) now = datetime.now(tz=UTC).replace(second=0, microsecond=0) - - traces: list[Traces] = [ - Traces( - timestamp=now - timedelta(minutes=3), - duration=timedelta(seconds=1), - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="test-span", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "test"}, - attributes={"http.method": "GET"}, - ), - ] - insert_traces(traces) - - token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - - start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000) - end_ms = int(now.timestamp() * 1000) - - response = requests.post( - signoz.self.host_configs["8080"].get("/api/v5/query_range"), - timeout=5, - headers={"authorization": f"Bearer {token}"}, - json={ - "schemaVersion": "v1", - "start": start_ms, - "end": end_ms, - "requestType": "time_series", - "compositeQuery": { - "queries": [ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "stepInterval": 60, - "disabled": False, - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - "functions": [{"name": "fillZero"}], - }, - } - ] - }, - "formatOptions": {"formatTableResultForUI": False, "fillGaps": False}, - }, + span_test = Traces( + timestamp=now - timedelta(minutes=3), + duration=timedelta(seconds=1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name="test-span", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": "test", **extra_resources}, + attributes={"http.method": "GET", **extra_attrs}, ) - - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - - results = response.json()["data"]["data"]["results"] - assert len(results) == 1 - - aggregations = results[0].get("aggregations") or [] - assert len(aggregations) == 1 - series = aggregations[0]["series"] - assert len(series) >= 1 - values = series[0]["values"] - - ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000) - assert_minutely_bucket_values( - values, - now, - expected_by_ts={ts_min_3: 1}, - context="traces/fillZero", + span_other = Traces( + timestamp=now - timedelta(minutes=2), + duration=timedelta(seconds=1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name="another-test-span", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": "another-test", **extra_resources}, + attributes={"http.method": "POST", **extra_attrs}, ) - - -def test_traces_fill_zero_with_group_by( - signoz: types.SigNoz, - create_user_admin: None, # pylint: disable=unused-argument - get_token: Callable[[str, str], str], - insert_traces: Callable[[list[Traces]], None], -) -> None: - """ - Test fillZero function for traces with groupBy. - """ - now = datetime.now(tz=UTC).replace(second=0, microsecond=0) - - traces: list[Traces] = [ - Traces( - timestamp=now - timedelta(minutes=3), - duration=timedelta(seconds=1), - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="span-a", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "service-a"}, - attributes={"http.method": "GET"}, - ), - Traces( - timestamp=now - timedelta(minutes=2), - duration=timedelta(seconds=1), - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="span-b", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "service-b"}, - attributes={"http.method": "POST"}, - ), - ] - insert_traces(traces) + insert_traces([span_test, span_other]) token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - - start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000) - end_ms = int(now.timestamp() * 1000) - - response = requests.post( - signoz.self.host_configs["8080"].get("/api/v5/query_range"), - timeout=5, - headers={"authorization": f"Bearer {token}"}, - json={ - "schemaVersion": "v1", - "start": start_ms, - "end": end_ms, - "requestType": "time_series", - "compositeQuery": { - "queries": [ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "stepInterval": 60, - "disabled": False, - "groupBy": [ - { - "name": "service.name", - "fieldDataType": "string", - "fieldContext": "resource", - } - ], - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - "functions": [{"name": "fillZero"}], - }, - } - ] - }, - "formatOptions": {"formatTableResultForUI": False, "fillGaps": False}, - }, + functions = [{"name": "fillZero"}] if fill_mode == "fillZero" else None + format_options = {"formatTableResultForUI": False, "fillGaps": fill_mode == "fillGaps"} + + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type=RequestType.TIME_SERIES, + queries=[ + build_traces_scalar_query(aggregations=[build_aggregation("count()")], filter_expression="resource.service.name = 'test'", disabled=True), + build_traces_scalar_query(name="B", aggregations=[build_aggregation("count()")], filter_expression="resource.service.name = 'another-test'", disabled=True), + build_formula_query("F1", "A + B", functions=functions), + ], + format_options=format_options, ) - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - - results = response.json()["data"]["data"]["results"] - assert len(results) == 1 - - aggregations = results[0]["aggregations"] - assert len(aggregations) == 1 - - series = aggregations[0]["series"] - assert len(series) == 2, "Expected 2 series for 2 service groups" - - ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000) - ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000) - - series_by_service = index_series_by_label(series, "service.name") - assert set(series_by_service.keys()) == {"service-a", "service-b"} - - expectations: dict[str, dict[int, float]] = { - "service-a": {ts_min_3: 1}, - "service-b": {ts_min_2: 1}, - } - - for service_name, s in series_by_service.items(): - assert_minutely_bucket_values( - s["values"], - now, - expected_by_ts=expectations[service_name], - context=f"traces/fillZero/{service_name}", - ) + assert response.status_code == HTTPStatus.OK, response.text + f1 = find_named_result(response.json()["data"]["data"]["results"], "F1") + assert f1 is not None + series = f1["aggregations"][0]["series"] + assert len(series) >= 1 + assert_minutely_bucket_values(series[0]["values"], now, expected_by_ts={_bucket_ms(span_test): 1, _bucket_ms(span_other): 1}, context=f"traces/{fill_mode}/F1") -def test_traces_fill_zero_formula( +@pytest.mark.parametrize("fill_mode", FILL_MODES) +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_fill_formula_group_by( signoz: types.SigNoz, create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], insert_traces: Callable[[list[Traces]], None], + noise: str, + fill_mode: str, ) -> None: """ - Test fillZero function for traces with formula. + Setup: + group1 span 3 minutes ago, group2 span 2 minutes ago. + + Tests: + A formula F1 = A + B over two disabled group-by count() queries yields one + series per service, each carrying A+B (=2) in its populated bucket and zeros + elsewhere. """ + extra_attrs, extra_resources = trace_noise(noise) now = datetime.now(tz=UTC).replace(second=0, microsecond=0) - - traces: list[Traces] = [ - Traces( - timestamp=now - timedelta(minutes=3), - duration=timedelta(seconds=1), - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="test-span", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "test"}, - attributes={"http.method": "GET"}, - ), - Traces( - timestamp=now - timedelta(minutes=2), - duration=timedelta(seconds=1), - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="another-test-span", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "another-test"}, - attributes={"http.method": "POST"}, - ), - ] - insert_traces(traces) - - token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - - start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000) - end_ms = int(now.timestamp() * 1000) - - response = requests.post( - signoz.self.host_configs["8080"].get("/api/v5/query_range"), - timeout=5, - headers={"authorization": f"Bearer {token}"}, - json={ - "schemaVersion": "v1", - "start": start_ms, - "end": end_ms, - "requestType": "time_series", - "compositeQuery": { - "queries": [ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "stepInterval": 60, - "disabled": True, - "filter": {"expression": "service.name = 'test'"}, - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - }, - { - "type": "builder_query", - "spec": { - "name": "B", - "signal": "traces", - "stepInterval": 60, - "disabled": True, - "filter": {"expression": "service.name = 'another-test'"}, - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - }, - { - "type": "builder_formula", - "spec": { - "name": "F1", - "expression": "A + B", - "disabled": False, - "functions": [{"name": "fillZero"}], - }, - }, - ] - }, - "formatOptions": {"formatTableResultForUI": False, "fillGaps": False}, - }, + span_g1 = Traces( + timestamp=now - timedelta(minutes=3), + duration=timedelta(seconds=1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name="span-group1", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": "group1", **extra_resources}, + attributes={"http.method": "GET", **extra_attrs}, ) - - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - - results = response.json()["data"]["data"]["results"] - assert len(results) == 1 - - ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000) - ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000) - - f1 = find_named_result(results, "F1") - assert f1 is not None, "Expected formula result named F1" - aggregations = f1.get("aggregations") or [] - assert len(aggregations) == 1 - series = aggregations[0]["series"] - assert len(series) >= 1 - - assert_minutely_bucket_values( - series[0]["values"], - now, - expected_by_ts={ts_min_3: 1, ts_min_2: 1}, - context="traces/fillZero/F1", + span_g2 = Traces( + timestamp=now - timedelta(minutes=2), + duration=timedelta(seconds=1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name="span-group2", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": "group2", **extra_resources}, + attributes={"http.method": "POST", **extra_attrs}, ) - - -def test_traces_fill_zero_formula_with_group_by( - signoz: types.SigNoz, - create_user_admin: None, # pylint: disable=unused-argument - get_token: Callable[[str, str], str], - insert_traces: Callable[[list[Traces]], None], -) -> None: - """ - Test fillZero function for traces with formula and groupBy. - """ - now = datetime.now(tz=UTC).replace(second=0, microsecond=0) - - traces: list[Traces] = [ - Traces( - timestamp=now - timedelta(minutes=3), - duration=timedelta(seconds=1), - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="span-group1", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "group1"}, - attributes={"http.method": "GET"}, - ), - Traces( - timestamp=now - timedelta(minutes=2), - duration=timedelta(seconds=1), - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - parent_span_id="", - name="span-group2", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources={"service.name": "group2"}, - attributes={"http.method": "POST"}, - ), - ] - insert_traces(traces) + insert_traces([span_g1, span_g2]) token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - - start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000) - end_ms = int(now.timestamp() * 1000) - - response = requests.post( - signoz.self.host_configs["8080"].get("/api/v5/query_range"), - timeout=5, - headers={"authorization": f"Bearer {token}"}, - json={ - "schemaVersion": "v1", - "start": start_ms, - "end": end_ms, - "requestType": "time_series", - "compositeQuery": { - "queries": [ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "stepInterval": 60, - "disabled": True, - "groupBy": [ - { - "name": "service.name", - "fieldDataType": "string", - "fieldContext": "resource", - } - ], - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - }, - { - "type": "builder_query", - "spec": { - "name": "B", - "signal": "traces", - "stepInterval": 60, - "disabled": True, - "groupBy": [ - { - "name": "service.name", - "fieldDataType": "string", - "fieldContext": "resource", - } - ], - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - }, - { - "type": "builder_formula", - "spec": { - "name": "F1", - "expression": "A + B", - "disabled": False, - "functions": [{"name": "fillZero"}], - }, - }, - ] - }, - "formatOptions": {"formatTableResultForUI": False, "fillGaps": False}, - }, + functions = [{"name": "fillZero"}] if fill_mode == "fillZero" else None + format_options = {"formatTableResultForUI": False, "fillGaps": fill_mode == "fillGaps"} + + group_by = [build_group_by_field("service.name", "string", "resource")] + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type=RequestType.TIME_SERIES, + queries=[ + build_traces_scalar_query(aggregations=[build_aggregation("count()")], group_by=group_by, disabled=True), + build_traces_scalar_query(name="B", aggregations=[build_aggregation("count()")], group_by=group_by, disabled=True), + build_formula_query("F1", "A + B", functions=functions), + ], + format_options=format_options, ) - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - - results = response.json()["data"]["data"]["results"] - assert len(results) == 1 - - ts_min_2 = int((now - timedelta(minutes=2)).timestamp() * 1000) - ts_min_3 = int((now - timedelta(minutes=3)).timestamp() * 1000) - - f1 = find_named_result(results, "F1") - assert f1 is not None, "Expected formula result named F1" - aggregations = f1.get("aggregations") or [] - assert len(aggregations) == 1 - series = aggregations[0]["series"] + assert response.status_code == HTTPStatus.OK, response.text + f1 = find_named_result(response.json()["data"]["data"]["results"], "F1") + assert f1 is not None + series = f1["aggregations"][0]["series"] assert len(series) == 2 - series_by_service = index_series_by_label(series, "service.name") - assert set(series_by_service.keys()) == {"group1", "group2"} - - expectations: dict[str, dict[int, float]] = { - "group1": {ts_min_3: 2}, - "group2": {ts_min_2: 2}, - } - - for service_name, s in series_by_service.items(): - assert_minutely_bucket_values( - s["values"], - now, - expected_by_ts=expectations[service_name], - context=f"traces/fillZero/F1/{service_name}", - ) + by_service = index_series_by_label(series, "service.name") + assert set(by_service) == {"group1", "group2"} + assert_minutely_bucket_values(by_service["group1"]["values"], now, expected_by_ts={_bucket_ms(span_g1): 2}, context=f"traces/{fill_mode}/F1/group1") + assert_minutely_bucket_values(by_service["group2"]["values"], now, expected_by_ts={_bucket_ms(span_g2): 2}, context=f"traces/{fill_mode}/F1/group2") diff --git a/tests/integration/tests/queriertraces/04_trace_id_filter.py b/tests/integration/tests/queriertraces/04_trace_id_filter.py index 570df2d1bd2..d7c0f262974 100644 --- a/tests/integration/tests/queriertraces/04_trace_id_filter.py +++ b/tests/integration/tests/queriertraces/04_trace_id_filter.py @@ -2,266 +2,220 @@ from datetime import UTC, datetime, timedelta from http import HTTPStatus +import pytest + from fixtures import types from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD from fixtures.querier import ( + Aggregation, + BuilderQuery, + OrderBy, + RequestType, + TelemetryFieldKey, make_query_request, ) -from fixtures.traces import ( - TraceIdGenerator, - Traces, - TracesKind, - TracesStatusCode, -) +from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode, trace_noise + +# The trace_id filter drives a trace_summary-backed time-range optimization +# (short-circuit outside the trace's window, no duplicate spans across the +# exponential buckets of a wide window). Under the shared clean/corrupt factor a +# colliding numeric `trace_id` attribute must not divert the equality filter off +# the intrinsic trace_id column. +OUTSIDE_RANGE_MSG = "lies outside the selected time range" + +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) def test_traces_list_filter_by_trace_id( signoz: types.SigNoz, create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], insert_traces: Callable[[list[Traces]], None], + noise: str, ) -> None: """ - Tests that filtering by trace_id: - 1. Returns the matching span (narrow window, single bucket). - 2. Does not return duplicate spans when the query window spans multiple - exponential buckets (>1 h) - 3. Returns no results when the query window does not contain the trace. + Setup: + A target trace (one root span) and an unrelated trace. + + Tests filtering a raw list by trace_id: + 1. A narrow window (single bucket) returns just the target span. + 2. A wide window (>1 h, multiple exponential buckets) returns it exactly once + (no duplicate-span regression). + 3. A window that does not contain the trace returns nothing and warns that it + lies outside the selected time range. """ - target_trace_id = TraceIdGenerator.trace_id() - other_trace_id = TraceIdGenerator.trace_id() - span_id_root = TraceIdGenerator.span_id() - other_span_id = TraceIdGenerator.span_id() - + extra_attrs, extra_resources = trace_noise(noise) now = datetime.now(tz=UTC).replace(second=0, microsecond=0) - - common_resources = { - "deployment.environment": "production", - "service.name": "trace-filter-service", - "cloud.provider": "integration", - } - - insert_traces( - [ - Traces( - timestamp=now - timedelta(seconds=10), - duration=timedelta(seconds=5), - trace_id=target_trace_id, - span_id=span_id_root, - parent_span_id="", - name="root-span", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources=common_resources, - attributes={"http.request.method": "GET"}, - ), - # span from a different trace — must not appear in results - Traces( - timestamp=now - timedelta(seconds=5), - duration=timedelta(seconds=1), - trace_id=other_trace_id, - span_id=other_span_id, - parent_span_id="", - name="other-root-span", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources=common_resources, - attributes={}, - ), - ] + resources = {"deployment.environment": "production", "service.name": "trace-filter-service", "cloud.provider": "integration", **extra_resources} + + target_span = Traces( + timestamp=now - timedelta(seconds=10), + duration=timedelta(seconds=5), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + parent_span_id="", + name="root-span", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources=resources, + attributes={"http.request.method": "GET", **extra_attrs}, + ) + other_span = Traces( + timestamp=now - timedelta(seconds=5), + duration=timedelta(seconds=1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + parent_span_id="", + name="other-root-span", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources=resources, + attributes=dict(extra_attrs), ) + insert_traces([target_span, other_span]) token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - trace_filter = f"trace_id = '{target_trace_id}'" - - def _query(start_ms: int, end_ms: int) -> tuple[list, list[str]]: + def query(start_ms: int, end_ms: int) -> tuple[list, list[str]]: response = make_query_request( signoz, token, start_ms=start_ms, end_ms=end_ms, - request_type="raw", + request_type=RequestType.RAW, queries=[ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "disabled": False, - "limit": 100, - "offset": 0, - "filter": {"expression": trace_filter}, - "order": [{"key": {"name": "timestamp"}, "direction": "desc"}], - "selectFields": [ - { - "name": "name", - "fieldDataType": "string", - "fieldContext": "span", - "signal": "traces", - } - ], - "having": {"expression": ""}, - "aggregations": [{"expression": "count()"}], - }, - } + BuilderQuery( + signal="traces", + name="A", + limit=100, + filter_expression=f"trace_id = '{target_span.trace_id}'", + order=[OrderBy(TelemetryFieldKey("timestamp"), "desc")], + select_fields=[TelemetryFieldKey("span.name")], + ).to_dict() ], ) - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" + assert response.status_code == HTTPStatus.OK, response.text rows = response.json()["data"]["data"]["results"][0]["rows"] or [] warning = (response.json().get("data") or {}).get("warning") or {} messages = [w.get("message", "") for w in (warning.get("warnings") or [])] return rows, messages - outside_range_msg = "lies outside the selected time range" - now_ms = int(now.timestamp() * 1000) - # --- Test 1: narrow window (single bucket, <1 h) --- - narrow_start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000) - narrow_rows, narrow_warnings = _query(narrow_start_ms, now_ms) + # 1. Narrow window (single bucket). + narrow_rows, narrow_warnings = query(int((now - timedelta(minutes=5)).timestamp() * 1000), now_ms) + assert len(narrow_rows) == 1 + assert narrow_rows[0]["data"]["span_id"] == target_span.span_id + assert narrow_rows[0]["data"]["trace_id"] == target_span.trace_id + assert not any(OUTSIDE_RANGE_MSG in m for m in narrow_warnings) - assert len(narrow_rows) == 1, f"Expected 1 span for trace_id filter (narrow window), got {len(narrow_rows)}" - assert narrow_rows[0]["data"]["span_id"] == span_id_root - assert narrow_rows[0]["data"]["trace_id"] == target_trace_id - assert not any(outside_range_msg in m for m in narrow_warnings), f"Did not expect outside-range warning, got {narrow_warnings}" + # 2. Wide window (>1 h, multi-bucket) — still exactly one span. + wide_rows, wide_warnings = query(int((now - timedelta(hours=12)).timestamp() * 1000), now_ms) + assert len(wide_rows) == 1, "possible duplicate-span regression across exponential buckets" + assert wide_rows[0]["data"]["span_id"] == target_span.span_id + assert not any(OUTSIDE_RANGE_MSG in m for m in wide_warnings) - # --- Test 2: wide window (>1 h, triggers multiple exponential buckets) --- - # should just return 1 span, not duplicate - wide_start_ms = int((now - timedelta(hours=12)).timestamp() * 1000) - wide_rows, wide_warnings = _query(wide_start_ms, now_ms) - - assert len(wide_rows) == 1, f"Expected 1 span for trace_id filter (wide window, multi-bucket), got {len(wide_rows)} — possible duplicate-span regression" - assert wide_rows[0]["data"]["span_id"] == span_id_root - assert wide_rows[0]["data"]["trace_id"] == target_trace_id - assert not any(outside_range_msg in m for m in wide_warnings), f"Did not expect outside-range warning, got {wide_warnings}" - - # --- Test 3: window that does not contain the trace returns no results + warning --- - past_start_ms = int((now - timedelta(hours=6)).timestamp() * 1000) - past_end_ms = int((now - timedelta(hours=2)).timestamp() * 1000) - past_rows, past_warnings = _query(past_start_ms, past_end_ms) - - assert len(past_rows) == 0, f"Expected 0 spans for trace_id filter outside time window, got {len(past_rows)}" - assert any(outside_range_msg in m for m in past_warnings), f"Expected outside-range warning, got warnings={past_warnings}" + # 3. Window that does not contain the trace — empty + outside-range warning. + past_rows, past_warnings = query(int((now - timedelta(hours=6)).timestamp() * 1000), int((now - timedelta(hours=2)).timestamp() * 1000)) + assert past_rows == [] + assert any(OUTSIDE_RANGE_MSG in m for m in past_warnings) +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) def test_traces_aggregation_filter_by_trace_id( signoz: types.SigNoz, create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], insert_traces: Callable[[list[Traces]], None], + noise: str, ) -> None: """ - Tests that the trace_id time-range optimization also applies to - non-window-list (time_series / aggregation) traces queries: - 1. Wide query window containing the trace returns the correct count. - 2. Query window outside the trace's time range short-circuits to empty. - 3. Filter referencing a trace_id with no row in trace_summary - short-circuits to empty (trace_summary is authoritative for traces). + Setup: + A target trace with two spans (root + child). + + Tests the trace_id time-range optimization for aggregation (time_series): + 1. A wide window containing the trace returns the correct count (both spans). + 2. A window outside the trace short-circuits to empty and warns. + 3. A trace_id with no row in trace_summary short-circuits to empty with no + outside-range warning (trace_summary is authoritative). """ - target_trace_id = TraceIdGenerator.trace_id() - target_root_span_id = TraceIdGenerator.span_id() - target_child_span_id = TraceIdGenerator.span_id() - missing_trace_id = TraceIdGenerator.trace_id() - + extra_attrs, extra_resources = trace_noise(noise) now = datetime.now(tz=UTC).replace(second=0, microsecond=0) - - common_resources = { - "deployment.environment": "production", - "service.name": "traces-agg-filter-service", - "cloud.provider": "integration", - } - - insert_traces( - [ - Traces( - timestamp=now - timedelta(seconds=10), - duration=timedelta(seconds=5), - trace_id=target_trace_id, - span_id=target_root_span_id, - parent_span_id="", - name="root-span", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources=common_resources, - attributes={"http.request.method": "GET"}, - ), - Traces( - timestamp=now - timedelta(seconds=9), - duration=timedelta(seconds=1), - trace_id=target_trace_id, - span_id=target_child_span_id, - parent_span_id=target_root_span_id, - name="child-span", - kind=TracesKind.SPAN_KIND_CLIENT, - status_code=TracesStatusCode.STATUS_CODE_OK, - status_message="", - resources=common_resources, - attributes={}, - ), - ] - ) + resources = {"deployment.environment": "production", "service.name": "traces-agg-filter-service", "cloud.provider": "integration", **extra_resources} + + trace_id = TraceIdGenerator.trace_id() + root_span_id = TraceIdGenerator.span_id() + target_spans = [ + Traces( + timestamp=now - timedelta(seconds=10), + duration=timedelta(seconds=5), + trace_id=trace_id, + span_id=root_span_id, + parent_span_id="", + name="root-span", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources=resources, + attributes={"http.request.method": "GET", **extra_attrs}, + ), + Traces( + timestamp=now - timedelta(seconds=9), + duration=timedelta(seconds=1), + trace_id=trace_id, + span_id=TraceIdGenerator.span_id(), + parent_span_id=root_span_id, + name="child-span", + kind=TracesKind.SPAN_KIND_CLIENT, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources=resources, + attributes=dict(extra_attrs), + ), + ] + missing_trace_id = TraceIdGenerator.trace_id() + insert_traces(target_spans) token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - def _count(start_ms: int, end_ms: int, trace_id: str) -> tuple[float, list[str]]: + def count(start_ms: int, end_ms: int, filter_trace_id: str) -> tuple[float, list[str]]: response = make_query_request( signoz, token, start_ms=start_ms, end_ms=end_ms, - request_type="time_series", + request_type=RequestType.TIME_SERIES, queries=[ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "stepInterval": 60, - "disabled": False, - "filter": {"expression": f"trace_id = '{trace_id}'"}, - "aggregations": [{"expression": "count()"}], - }, - } + BuilderQuery( + signal="traces", + name="A", + step_interval=60, + filter_expression=f"trace_id = '{filter_trace_id}'", + aggregations=[Aggregation("count()")], + ).to_dict() ], ) - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" + assert response.status_code == HTTPStatus.OK, response.text results = response.json()["data"]["data"]["results"] assert len(results) == 1 warning = (response.json().get("data") or {}).get("warning") or {} messages = [w.get("message", "") for w in (warning.get("warnings") or [])] aggregations = results[0].get("aggregations") or [] - if not aggregations: - return 0, messages - series = aggregations[0].get("series") or [] - if not series: - return 0, messages - return sum(v["value"] for v in series[0]["values"]), messages - - outside_range_msg = "lies outside the selected time range" + series = aggregations[0].get("series") if aggregations else None + total = sum(v["value"] for v in series[0]["values"]) if series else 0 + return total, messages now_ms = int(now.timestamp() * 1000) - # --- Test 1: wide window (>1 h) containing the trace returns both spans --- - wide_start_ms = int((now - timedelta(hours=12)).timestamp() * 1000) - wide_count, wide_warnings = _count(wide_start_ms, now_ms, target_trace_id) - assert wide_count == 2, f"Expected count=2 for trace_id aggregation (wide window), got {wide_count}" - assert not any(outside_range_msg in m for m in wide_warnings), f"Did not expect outside-range warning, got {wide_warnings}" + # 1. Wide window containing the trace returns both spans. + wide_count, wide_warnings = count(int((now - timedelta(hours=12)).timestamp() * 1000), now_ms, trace_id) + assert wide_count == len(target_spans) + assert not any(OUTSIDE_RANGE_MSG in m for m in wide_warnings) - # --- Test 2: window outside the trace short-circuits to empty + warning --- - past_start_ms = int((now - timedelta(hours=6)).timestamp() * 1000) - past_end_ms = int((now - timedelta(hours=2)).timestamp() * 1000) - past_count, past_warnings = _count(past_start_ms, past_end_ms, target_trace_id) - assert past_count == 0, f"Expected count=0 for trace_id aggregation outside time window, got {past_count}" - assert any(outside_range_msg in m for m in past_warnings), f"Expected outside-range warning, got warnings={past_warnings}" + # 2. Window outside the trace short-circuits to empty + warning. + past_count, past_warnings = count(int((now - timedelta(hours=6)).timestamp() * 1000), int((now - timedelta(hours=2)).timestamp() * 1000), trace_id) + assert past_count == 0 + assert any(OUTSIDE_RANGE_MSG in m for m in past_warnings) - # --- Test 3: trace_id with no entry in trace_summary short-circuits (no warning) --- - missing_start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000) - missing_count, missing_warnings = _count(missing_start_ms, now_ms, missing_trace_id) - assert missing_count == 0, f"Expected count=0 for trace_id absent from trace_summary, got {missing_count}" - assert not any(outside_range_msg in m for m in missing_warnings), f"Did not expect outside-range warning for missing trace_id, got {missing_warnings}" + # 3. trace_id absent from trace_summary short-circuits (no outside-range warning). + missing_count, missing_warnings = count(int((now - timedelta(minutes=5)).timestamp() * 1000), now_ms, missing_trace_id) + assert missing_count == 0 + assert not any(OUTSIDE_RANGE_MSG in m for m in missing_warnings) diff --git a/tests/integration/tests/queriertraces/05_resource_evolution.py b/tests/integration/tests/queriertraces/05_resource_evolution.py index c2ca4f41146..81d8c0ee83b 100644 --- a/tests/integration/tests/queriertraces/05_resource_evolution.py +++ b/tests/integration/tests/queriertraces/05_resource_evolution.py @@ -5,9 +5,11 @@ from fixtures import types from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD from fixtures.querier import ( + RequestType, assert_grouped_series, build_aggregation, build_group_by_field, + build_traces_scalar_query, get_resource_evolution_time, index_series_by_label, make_query_request, @@ -50,20 +52,12 @@ def _query_grouped_trace_series( token, start_ms=int(start.timestamp() * 1000), end_ms=int(end.timestamp() * 1000), - request_type="time_series", + request_type=RequestType.TIME_SERIES, queries=[ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "stepInterval": 60, - "disabled": False, - "groupBy": [build_group_by_field(group_by)], - "having": {"expression": ""}, - "aggregations": [build_aggregation(aggregation)], - }, - } + build_traces_scalar_query( + aggregations=[build_aggregation(aggregation)], + group_by=[build_group_by_field(group_by)], + ) ], ) diff --git a/tests/integration/tests/queriertraces/06_trace_operator.py b/tests/integration/tests/queriertraces/06_trace_operator.py index 788da60289c..c81c9de37a9 100644 --- a/tests/integration/tests/queriertraces/06_trace_operator.py +++ b/tests/integration/tests/queriertraces/06_trace_operator.py @@ -33,6 +33,7 @@ from fixtures import types from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD from fixtures.querier import ( + RequestType, assert_grouped_scalar, assert_raw_row_subset, assert_scalar_value, @@ -426,30 +427,23 @@ def test_trace_operator( if case.get("order"): spec["order"] = case["order"] - response = requests.post( - signoz.self.host_configs["8080"].get("/api/v5/query_range"), - timeout=5, - headers={"authorization": f"Bearer {token}"}, - json={ - "schemaVersion": "v1", - "start": start_ms, - "end": end_ms, - "requestType": "raw", - "compositeQuery": { - "queries": [ - { - "type": "builder_query", - "spec": {"name": "A", "signal": "traces", "filter": {"expression": case["filter_a"]}, "limit": 100}, - }, - { - "type": "builder_query", - "spec": {"name": "B", "signal": "traces", "filter": {"expression": case["filter_b"]}, "limit": 100}, - }, - {"type": "builder_trace_operator", "spec": spec}, - ] + response = make_query_request( + signoz, + token, + start_ms=start_ms, + end_ms=end_ms, + request_type=RequestType.RAW, + queries=[ + { + "type": "builder_query", + "spec": {"name": "A", "signal": "traces", "filter": {"expression": case["filter_a"]}, "limit": 100}, }, - "formatOptions": {"formatTableResultForUI": False, "fillGaps": False}, - }, + { + "type": "builder_query", + "spec": {"name": "B", "signal": "traces", "filter": {"expression": case["filter_b"]}, "limit": 100}, + }, + {"type": "builder_trace_operator", "spec": spec}, + ], ) assert response.status_code == HTTPStatus.OK, f"HTTP {response.status_code}: {response.text}" assert case["validate"](response), f"validation failed: {response.json()}" @@ -497,7 +491,7 @@ def _expected_trace_subset(trace: Traces) -> dict[str, Any]: }, }, ], - "raw", + RequestType.RAW, lambda response, traces: assert_raw_row_subset(response, "C", _expected_trace_subset(traces[0])), id="deprecated-intrinsic-filter", ), @@ -529,7 +523,7 @@ def _expected_trace_subset(trace: Traces) -> dict[str, Any]: }, }, ], - "raw", + RequestType.RAW, lambda response, traces: assert_raw_row_subset(response, "C", _expected_trace_subset(traces[0])), id="deprecated-calculated-filter", ), @@ -555,7 +549,7 @@ def _expected_trace_subset(trace: Traces) -> dict[str, Any]: }, }, ], - "scalar", + RequestType.SCALAR, lambda response, traces: assert_scalar_value(response, "C", len(traces)), id="context-prefixed-aggregation-alias-order", ), @@ -585,7 +579,7 @@ def _expected_trace_subset(trace: Traces) -> dict[str, Any]: }, }, ], - "scalar", + RequestType.SCALAR, lambda response, traces: assert_grouped_scalar(response, "C", expected_groups=1, expected_columns=2, last_col_value=len(traces)), id="duplicate-group-by-deduplicated", ), @@ -788,7 +782,7 @@ def test_trace_operator_select_fields( token, start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000), end_ms=int(datetime.now(tz=UTC).timestamp() * 1000), - request_type="raw", + request_type=RequestType.RAW, queries=queries, ) @@ -796,5 +790,6 @@ def test_trace_operator_select_fields( results = response.json()["data"]["data"]["results"] trace_operator_result = find_named_result(results, "C") + assert trace_operator_result is not None, "trace_operator result C not found" rows = trace_operator_result["rows"] verify_values(rows, parent_trace) diff --git a/tests/integration/tests/queriertraces/07_filter_errors.py b/tests/integration/tests/queriertraces/07_filter_errors.py index 58f264dece3..db4a2cef616 100644 --- a/tests/integration/tests/queriertraces/07_filter_errors.py +++ b/tests/integration/tests/queriertraces/07_filter_errors.py @@ -6,7 +6,7 @@ from fixtures import types from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD -from fixtures.querier import make_query_request +from fixtures.querier import Aggregation, BuilderQuery, RequestType, make_query_request # Traces filter validation: the body-JSON functions has()/hasToken() are not valid on the # traces signal. These are pre-query validation errors (400) independent of ingested data. @@ -33,17 +33,7 @@ def test_traces_filter_validation_errors( token, start_ms=int((now - timedelta(seconds=30)).timestamp() * 1000), end_ms=int(now.timestamp() * 1000), - request_type="scalar", - queries=[ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "filter": {"expression": expression}, - "aggregations": [{"expression": "count()"}], - }, - } - ], + request_type=RequestType.SCALAR, + queries=[BuilderQuery(signal="traces", name="A", filter_expression=expression, aggregations=[Aggregation("count()")]).to_dict()], ) assert response.status_code == HTTPStatus.BAD_REQUEST diff --git a/tests/integration/tests/queriertraces/08_aggregation_min_max.py b/tests/integration/tests/queriertraces/08_aggregation_min_max.py index 9229ec1546d..7892bbdee9d 100644 --- a/tests/integration/tests/queriertraces/08_aggregation_min_max.py +++ b/tests/integration/tests/queriertraces/08_aggregation_min_max.py @@ -2,67 +2,66 @@ from datetime import UTC, datetime, timedelta from http import HTTPStatus +import pytest + from fixtures import types from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD -from fixtures.querier import index_series_by_label, make_query_request -from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode +from fixtures.querier import ( + RequestType, + build_aggregation, + build_group_by_field, + build_traces_scalar_query, + index_series_by_label, + make_query_request, +) +from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode, trace_noise -# min()/max() aggregation expressions over a span field. The traces aggregation -# suite (02_aggregation.py) covers count/countIf/avg/p99 but not min/max. +# min()/max() aggregation expressions over duration_nano, grouped by service.name +# (the time_series counterpart of the scalar min/max in 02_aggregation.py). Under +# the shared clean/corrupt factor a same-named string attribute duration_nano must +# not divert the aggregation off the numeric intrinsic column. +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) def test_traces_aggregate_min_max( signoz: types.SigNoz, create_user_admin: None, # pylint: disable=unused-argument get_token: Callable[[str, str], str], insert_traces: Callable[[list[Traces]], None], + noise: str, ) -> None: - """max(duration_nano)/min(duration_nano) grouped by service.name return the extremes.""" + """ + Setup: + http-service: 3 spans (3s / 0.5s / 1s); topic-service: 1 span (4s). + + Tests: + max(duration_nano) / min(duration_nano) grouped by service.name return each + service's duration extremes, derived from the inserted spans. + """ + extra_attrs, extra_resources = trace_noise(noise) now = datetime.now(tz=UTC).replace(second=0, microsecond=0) - insert_traces( - [ - Traces( - timestamp=now - timedelta(seconds=4), - duration=timedelta(seconds=3), # 3e9 - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - name="POST /x", - kind=TracesKind.SPAN_KIND_SERVER, - status_code=TracesStatusCode.STATUS_CODE_OK, - resources={"service.name": "http-service"}, - ), - Traces( - timestamp=now - timedelta(seconds=3), - duration=timedelta(milliseconds=500), # 0.5e9 (min) - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - name="SELECT", - kind=TracesKind.SPAN_KIND_CLIENT, - status_code=TracesStatusCode.STATUS_CODE_OK, - resources={"service.name": "http-service"}, - ), - Traces( - timestamp=now - timedelta(seconds=2), - duration=timedelta(seconds=1), # 1e9 - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - name="PATCH", - kind=TracesKind.SPAN_KIND_CLIENT, - status_code=TracesStatusCode.STATUS_CODE_OK, - resources={"service.name": "http-service"}, - ), - Traces( - timestamp=now - timedelta(seconds=1), - duration=timedelta(seconds=4), # 4e9 - trace_id=TraceIdGenerator.trace_id(), - span_id=TraceIdGenerator.span_id(), - name="topic publish", - kind=TracesKind.SPAN_KIND_PRODUCER, - status_code=TracesStatusCode.STATUS_CODE_OK, - resources={"service.name": "topic-service"}, - ), - ] - ) + specs = [ + ("http-service", 3, "POST /x", TracesKind.SPAN_KIND_SERVER), + ("http-service", 0.5, "SELECT", TracesKind.SPAN_KIND_CLIENT), + ("http-service", 1, "PATCH", TracesKind.SPAN_KIND_CLIENT), + ("topic-service", 4, "topic publish", TracesKind.SPAN_KIND_PRODUCER), + ] + spans = [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + duration=timedelta(seconds=dur_s), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name=name, + kind=kind, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": service, **extra_resources}, + attributes=dict(extra_attrs), + ) + for i, (service, dur_s, name, kind) in enumerate(specs) + ] + insert_traces(spans) + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) response = make_query_request( @@ -70,29 +69,27 @@ def test_traces_aggregate_min_max( token, start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), end_ms=int((now + timedelta(seconds=5)).timestamp() * 1000), - request_type="time_series", + request_type=RequestType.TIME_SERIES, queries=[ - { - "type": "builder_query", - "spec": { - "name": "A", - "signal": "traces", - "groupBy": [{"name": "service.name", "fieldContext": "resource", "fieldDataType": "string"}], - "aggregations": [ - {"expression": "max(duration_nano)", "alias": "maxDuration"}, - {"expression": "min(duration_nano)", "alias": "minDuration"}, - ], - }, - } + build_traces_scalar_query( + aggregations=[ + build_aggregation("max(duration_nano)", "maxDuration"), + build_aggregation("min(duration_nano)", "minDuration"), + ], + group_by=[build_group_by_field("service.name", "string", "resource")], + ) ], ) - assert response.status_code == HTTPStatus.OK - assert response.json()["status"] == "success" - aggregations = response.json()["data"]["data"]["results"][0]["aggregations"] + assert response.status_code == HTTPStatus.OK, response.text + aggregations = response.json()["data"]["data"]["results"][0]["aggregations"] max_by_svc = index_series_by_label(aggregations[0]["series"], "service.name") min_by_svc = index_series_by_label(aggregations[1]["series"], "service.name") - assert max_by_svc["http-service"]["values"][0]["value"] == 3_000_000_000.0 - assert min_by_svc["http-service"]["values"][0]["value"] == 500_000_000.0 - assert max_by_svc["topic-service"]["values"][0]["value"] == 4_000_000_000.0 - assert min_by_svc["topic-service"]["values"][0]["value"] == 4_000_000_000.0 + + durations_by_service: dict[str, list[int]] = {} + for span, (service, *_rest) in zip(spans, specs, strict=True): + durations_by_service.setdefault(service, []).append(int(span.duration_nano)) + + for service, durations in durations_by_service.items(): + assert max_by_svc[service]["values"][0]["value"] == max(durations) + assert min_by_svc[service]["values"][0]["value"] == min(durations) diff --git a/tests/integration/tests/queriertraces/09_unknown_keys.py b/tests/integration/tests/queriertraces/09_unknown_keys.py index b7e3d49eb84..3992c55eb03 100644 --- a/tests/integration/tests/queriertraces/09_unknown_keys.py +++ b/tests/integration/tests/queriertraces/09_unknown_keys.py @@ -9,7 +9,7 @@ build_aggregation, build_group_by_field, build_raw_query, - build_scalar_query, + build_traces_scalar_query, get_rows, get_scalar_table_data, make_query_request, @@ -37,9 +37,7 @@ def test_traces_filter_unknown_key_synthesizes( end_ms=int(now.timestamp() * 1000), request_type=RequestType.SCALAR, queries=[ - build_scalar_query( - "A", - "traces", + build_traces_scalar_query( [build_aggregation("count()")], filter_expression='totally.unknown.key = "x"', ) @@ -137,9 +135,7 @@ def test_traces_group_by_unknown_key_null_bucket( end_ms=int(now.timestamp() * 1000), request_type=RequestType.SCALAR, queries=[ - build_scalar_query( - "A", - "traces", + build_traces_scalar_query( [build_aggregation("count()")], group_by=[build_group_by_field("totally.unknown.key", field_data_type="", field_context="")], filter_expression='service.name = "groupby-unknown-svc"', diff --git a/tests/integration/tests/queriertraces/10_filter_operators.py b/tests/integration/tests/queriertraces/10_filter_operators.py new file mode 100644 index 00000000000..b99fcd475bd --- /dev/null +++ b/tests/integration/tests/queriertraces/10_filter_operators.py @@ -0,0 +1,129 @@ +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http import HTTPStatus + +import pytest + +from fixtures import types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.querier import BuilderQuery, OrderBy, RequestType, TelemetryFieldKey, get_rows, make_query_request +from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode + +# Filter-operator coverage for traces, modelled on the shapes that dominate real +# dashboard/alert traffic (from a masked production sample of ~35k filter +# expressions). By far the most common are `=` + `AND` + `IN` (compound +# multi-clause filters); also common are CONTAINS/NOT CONTAINS, EXISTS/NOT EXISTS, +# LIKE/ILIKE/NOT LIKE, REGEXP, `!=` (incl. the `!= ''` non-empty idiom), numeric +# comparisons, nested OR, and deprecated intrinsic/calculated field names +# (responseStatusCode, ...). has()/hasToken() never appear (logs-body only) and +# BETWEEN is negligible, so neither is exercised here. +# +# No clean/corrupt factor: this file targets operator/parser resolution, which is +# orthogonal to field-collision (covered in 01_list.py / 02_aggregation.py). A +# single-term `response_status_code >= 400` would additionally re-hit the known +# calculated/numeric-attribute collision bug xfail'd in 02_aggregation.py. + + +@pytest.mark.parametrize( + "expression,expected_names", + [ + # ── IN / NOT IN (the #1 production pattern by traffic) ────────────────── + pytest.param("resource.service.name IN ['checkout', 'cart']", {"checkout-handler", "cart-handler"}, id="in_bracket_list"), + pytest.param("resource.service.name IN ('checkout', 'cart')", {"checkout-handler", "cart-handler"}, id="in_paren_list"), + pytest.param("resource.service.name NOT IN ['checkout', 'cart']", {"payment-processor", "search-service", "notify-worker"}, id="not_in"), + # ── Compound AND + IN, the dominant real-world shape ──────────────────── + pytest.param("resource.deployment.environment = 'prod' AND resource.service.name IN ['checkout', 'search']", {"checkout-handler", "search-service"}, id="and_eq_in"), + # ── Nested OR within AND ──────────────────────────────────────────────── + pytest.param("resource.deployment.environment = 'prod' AND (resource.service.name = 'cart' OR resource.service.name = 'search')", {"cart-handler", "search-service"}, id="and_nested_or"), + # ── Inequality, incl. the `!= ''` non-empty idiom ─────────────────────── + pytest.param("http.request.method != 'GET'", {"cart-handler", "search-service"}, id="not_equal"), + pytest.param("error.type != ''", {"payment-processor"}, id="not_equal_empty"), + # ── CONTAINS / NOT CONTAINS ───────────────────────────────────────────── + pytest.param("name CONTAINS 'handler'", {"checkout-handler", "cart-handler"}, id="contains"), + pytest.param("name NOT CONTAINS 'handler'", {"payment-processor", "search-service", "notify-worker"}, id="not_contains"), + # ── LIKE / ILIKE / NOT LIKE ───────────────────────────────────────────── + pytest.param("name LIKE '%processor'", {"payment-processor"}, id="like"), + pytest.param("name ILIKE '%HANDLER'", {"checkout-handler", "cart-handler"}, id="ilike"), + pytest.param("name NOT LIKE '%handler'", {"payment-processor", "search-service", "notify-worker"}, id="not_like"), + # ── REGEXP / NOT REGEXP ───────────────────────────────────────────────── + pytest.param("name REGEXP '^cart'", {"cart-handler"}, id="regexp"), + pytest.param("name NOT REGEXP 'handler'", {"payment-processor", "search-service", "notify-worker"}, id="not_regexp"), + # ── Numeric comparisons (calculated response_status_code + number attr) ─ + pytest.param("response_status_code >= 400", {"cart-handler", "payment-processor"}, id="gte_response_status_code"), + pytest.param("response_status_code < 300", {"checkout-handler", "search-service", "notify-worker"}, id="lt_response_status_code"), + pytest.param("latency_ms > 400", {"payment-processor", "notify-worker"}, id="gt_number_attr"), + pytest.param("latency_ms <= 100", {"checkout-handler", "search-service"}, id="lte_number_attr"), + # ── EXISTS / NOT EXISTS ───────────────────────────────────────────────── + pytest.param("error.type exists", {"payment-processor"}, id="exists"), + pytest.param("error.type not exists", {"checkout-handler", "cart-handler", "search-service", "notify-worker"}, id="not_exists"), + # ── Deprecated calculated field name (used in real production filters) ─── + pytest.param("responseStatusCode >= 400", {"cart-handler", "payment-processor"}, id="deprecated_responseStatusCode"), + ], +) +def test_traces_filter_operators( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + expression: str, + expected_names: set[str], +) -> None: + """ + Setup: + Five spans across services / environments / methods / statuses, with a + numeric `latency_ms` attribute and an `error.type` attribute present on one. + + Tests: + Each production-shaped filter operator selects exactly the expected spans. + """ + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + # (name, service, deployment.environment, http.request.method, http.response.status_code, latency_ms, has_error_type) + specs = [ + ("checkout-handler", "checkout", "prod", "GET", "200", 100, False), + ("cart-handler", "cart", "prod", "POST", "404", 250, False), + ("payment-processor", "payment", "staging", "GET", "500", 500, True), + ("search-service", "search", "prod", "DELETE", "201", 50, False), + ("notify-worker", "notify", "dev", "GET", "200", 1000, False), + ] + spans = [] + for i, (name, service, env, method, rsc, latency, has_error_type) in enumerate(specs): + attributes = {"http.request.method": method, "http.response.status_code": rsc, "latency_ms": latency} + if has_error_type: + attributes["error.type"] = "timeout" + spans.append( + Traces( + timestamp=now - timedelta(seconds=i + 1), + duration=timedelta(seconds=1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name=name, + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": service, "deployment.environment": env}, + attributes=attributes, + ) + ) + insert_traces(spans) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + response = make_query_request( + signoz, + token, + start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000), + end_ms=int(now.timestamp() * 1000), + request_type=RequestType.RAW, + queries=[ + BuilderQuery( + signal="traces", + name="A", + limit=100, + filter_expression=expression, + select_fields=[TelemetryFieldKey("span.name")], + order=[OrderBy(TelemetryFieldKey("timestamp"), "desc")], + ).to_dict() + ], + ) + + assert response.status_code == HTTPStatus.OK, response.text + assert {row["data"]["name"] for row in get_rows(response)} == expected_names diff --git a/tests/integration/tests/queriertraces/11_aggregation_depth.py b/tests/integration/tests/queriertraces/11_aggregation_depth.py new file mode 100644 index 00000000000..11388f1364f --- /dev/null +++ b/tests/integration/tests/queriertraces/11_aggregation_depth.py @@ -0,0 +1,251 @@ +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from http import HTTPStatus + +import numpy as np +import pytest + +from fixtures import types +from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD +from fixtures.querier import ( + build_aggregation, + build_group_by_field, + build_order_by, + build_traces_scalar_query, + get_scalar_table_data, + make_scalar_query_request, +) +from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode, trace_noise + +# Aggregation depth for traces, beyond 02_aggregation.py: the higher percentiles +# (p25/p95/p99, vs only p50/p90 there), grouping by a non-resource key (an intrinsic +# column and a numeric span attribute, vs only resource.service.name), a multi-key +# group-by cross product, and HAVING on non-count aggregations plus a compound +# predicate. Every test runs under the shared clean/corrupt trace_noise factor, so a +# same-named colliding attribute must not divert grouping or aggregation off the real +# intrinsic column. + + +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_aggregate_percentiles( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, +) -> None: + """ + Setup: + 5 spans with durations 1s..5s. + + Tests: + p25 / p50 / p90 / p95 / p99 over duration_nano match numpy's linear-interpolated + percentiles (ClickHouse quantile() uses the same interpolation for small inputs). + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + durations_s = [1, 2, 3, 4, 5] + spans = [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + duration=timedelta(seconds=dur_s), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name=f"pctl-{dur_s}", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": "pctl-svc", **extra_resources}, + attributes=dict(extra_attrs), + ) + for i, dur_s in enumerate(durations_s) + ] + insert_traces(spans) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + percentiles = [25, 50, 90, 95, 99] + query = build_traces_scalar_query( + aggregations=[build_aggregation(f"p{p}(duration_nano)", f"p{p}") for p in percentiles], + ) + response = make_scalar_query_request(signoz, token, now, [query]) + + assert response.status_code == HTTPStatus.OK, response.text + data = get_scalar_table_data(response.json()) + assert len(data) == 1 + + durations_nano = [int(s.duration_nano) for s in spans] + expected = [float(np.percentile(durations_nano, p)) for p in percentiles] + assert data[0] == pytest.approx(expected), f"{data[0]} != {expected}" + + +@pytest.mark.parametrize( + "group_key,expected", + [ + pytest.param("name", {"op-a": 2, "op-b": 1}, id="intrinsic_name"), + pytest.param("endpoint", {"/a": 2, "/b": 1}, id="string_span_attr"), + ], +) +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_aggregate_group_by_non_resource( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, + group_key: str, + expected: dict[str, int], +) -> None: + """ + Setup: + 3 spans: (name op-a, endpoint /a) x2 and (name op-b, endpoint /b) x1. + + Tests: + Grouping by an intrinsic column (name) or a string span attribute (endpoint) — + not just a resource key — buckets by the real value; under the corrupt variant a + same-named colliding attribute must not divert the grouping. + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + specs = [("op-a", "/a"), ("op-a", "/a"), ("op-b", "/b")] + spans = [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name=name, + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": "gb-svc", **extra_resources}, + attributes={"endpoint": endpoint, **extra_attrs}, + ) + for i, (name, endpoint) in enumerate(specs) + ] + insert_traces(spans) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + query = build_traces_scalar_query( + aggregations=[build_aggregation("count()")], + group_by=[build_group_by_field(group_key, field_data_type="", field_context="")], + filter_expression="resource.service.name = 'gb-svc'", + ) + response = make_scalar_query_request(signoz, token, now, [query]) + + assert response.status_code == HTTPStatus.OK, response.text + data = get_scalar_table_data(response.json()) + assert {row[0]: row[-1] for row in data} == expected + + +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_aggregate_multi_group_by( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, +) -> None: + """ + Setup: + Spans across (service.name, name) pairs: (svc-1, op-a) x2, (svc-1, op-b) x1, + (svc-2, op-a) x1. + + Tests: + Grouping by two keys (a resource key and an intrinsic column) returns one row per + present pair with the correct per-pair count. + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + pair_counts = {("svc-1", "op-a"): 2, ("svc-1", "op-b"): 1, ("svc-2", "op-a"): 1} + spans = [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name=name, + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": service, **extra_resources}, + attributes=dict(extra_attrs), + ) + for i, ((service, name), count) in enumerate(pair_counts.items()) + for _ in range(count) + ] + insert_traces(spans) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + query = build_traces_scalar_query( + aggregations=[build_aggregation("count()")], + group_by=[ + build_group_by_field("service.name", "string", "resource"), + build_group_by_field("name", field_data_type="", field_context=""), + ], + ) + response = make_scalar_query_request(signoz, token, now, [query]) + + assert response.status_code == HTTPStatus.OK, response.text + data = get_scalar_table_data(response.json()) + actual = {(row[0], row[1], row[2]) for row in data} + expected = {(service, name, count) for (service, name), count in pair_counts.items()} + assert actual == expected + + +@pytest.mark.parametrize( + "having_expression,expected_services", + [ + pytest.param("sum(duration_nano) > 4000000000", {"svc-b"}, id="sum_gt"), + pytest.param("count() > 2", {"svc-a"}, id="count_gt"), + pytest.param("count() >= 1 AND sum(duration_nano) < 4000000000", {"svc-a"}, id="compound_and"), + ], +) +@pytest.mark.parametrize("noise", ["clean", "corrupt"]) +def test_traces_aggregate_having_breadth( + signoz: types.SigNoz, + create_user_admin: None, # pylint: disable=unused-argument + get_token: Callable[[str, str], str], + insert_traces: Callable[[list[Traces]], None], + noise: str, + having_expression: str, + expected_services: set[str], +) -> None: + """ + Setup: + svc-a: 3 spans of 1s (sum 3s, count 3); svc-b: 1 span of 5s (sum 5s, count 1). + + Tests: + HAVING on a sum() aggregation, on count(), and a compound count()+sum() predicate + each keep only the qualifying groups — extending the count()-only HAVING in + 02_aggregation.py. + """ + extra_attrs, extra_resources = trace_noise(noise) + now = datetime.now(tz=UTC).replace(second=0, microsecond=0) + specs = [("svc-a", 1), ("svc-a", 1), ("svc-a", 1), ("svc-b", 5)] + spans = [ + Traces( + timestamp=now - timedelta(seconds=i + 1), + duration=timedelta(seconds=dur_s), + trace_id=TraceIdGenerator.trace_id(), + span_id=TraceIdGenerator.span_id(), + name=f"{service}-{i}", + kind=TracesKind.SPAN_KIND_SERVER, + status_code=TracesStatusCode.STATUS_CODE_OK, + resources={"service.name": service, **extra_resources}, + attributes=dict(extra_attrs), + ) + for i, (service, dur_s) in enumerate(specs) + ] + insert_traces(spans) + + token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) + + query = build_traces_scalar_query( + aggregations=[build_aggregation("count()"), build_aggregation("sum(duration_nano)")], + group_by=[build_group_by_field("service.name", "string", "resource")], + order=[build_order_by("count()", "desc")], + having_expression=having_expression, + ) + response = make_scalar_query_request(signoz, token, now, [query]) + + assert response.status_code == HTTPStatus.OK, response.text + data = get_scalar_table_data(response.json()) + assert {row[0] for row in data} == expected_services From 7be340ce69cca4188fa320b5d7aa7aa587aa0e37 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Mon, 20 Jul 2026 00:57:13 +0530 Subject: [PATCH 3/5] fix(clickhouseprometheus): include hour-floored registration rows at the query window end (#12153) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The series lookup bounded registration rows with unix_milli < end while the samples query uses <= end. The exporter writes registration rows hour-floored, so a series first registered in the hour starting exactly at end was invisible even though its samples were fetchable — queries ending exactly on an hour boundary missed newly-registered series. Co-authored-by: Claude Fable 5 --- pkg/prometheus/clickhouseprometheus/client.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/prometheus/clickhouseprometheus/client.go b/pkg/prometheus/clickhouseprometheus/client.go index 821509e792e..53a922b1647 100644 --- a/pkg/prometheus/clickhouseprometheus/client.go +++ b/pkg/prometheus/clickhouseprometheus/client.go @@ -141,7 +141,11 @@ func (client *client) queryToClickhouseQuery(_ context.Context, query *prompb.Qu var args []any conditions = append(conditions, fmt.Sprintf("metric_name = $%d", argCount+1)) conditions = append(conditions, "temporality IN ['Cumulative', 'Unspecified']") - conditions = append(conditions, fmt.Sprintf("unix_milli >= %d AND unix_milli < %d", start, end)) + // Inclusive upper bound: registration rows are hour-floored by the + // exporter, so a series first registered in the hour starting exactly at + // `end` would otherwise be invisible while its samples (<= end) are in + // range. + conditions = append(conditions, fmt.Sprintf("unix_milli >= %d AND unix_milli <= %d", start, end)) args = append(args, metricName) for _, m := range query.Matchers { @@ -202,6 +206,15 @@ func (client *client) getFingerprintsFromClickhouseQuery(ctx context.Context, qu // buildSamplesQuery renders the samples SQL (and args) that fetches data // points for the series selected by subQuery. +// +// Time bounds are inclusive on both ends because that is Prometheus's +// storage contract: Select(mint, maxt) returns [start, end] and the engine +// itself trims each evaluation window to left-open (T-window, T], so the +// sample at exactly `end` belongs to the last point. This deliberately +// differs from the query builder's `unix_milli < end`, which is correct for +// its own model — toStartOfInterval buckets covering [t, t+step), where a +// sample at `end` falls in an unrendered bucket and end-exclusive ranges +// tile exactly across cached time slices. func buildSamplesQuery(start int64, end int64, metricName string, subQuery string, args []any) (string, []any) { argCount := len(args) From 8c40d8c2cac2ab10a93930ee08188332535569a8 Mon Sep 17 00:00:00 2001 From: Pandey Date: Mon, 20 Jul 2026 01:22:51 +0530 Subject: [PATCH 4/5] fix(querybuildertypesv5): allow empty source, temporality and timeAggregation in the metric query spec (#12164) * fix(querybuildertypesv5): omit unset metric enum fields on the wire A metric builder query serialized empty strings for its enum fields because omitempty has no effect on struct-backed valuer types: "source":"", "aggregations":[{"temporality":"","timeAggregation":"","spaceAggregation":""}] Those "" values are not members of the corresponding OpenAPI enums (source=[meter], temporality=[delta,cumulative,unspecified], etc.), so a typed client reading a rule back rejected it (create -> GET round-trip drift; terraform-provider-signoz generate-config failed schema validation). Tag Source/Temporality/TimeAggregation/SpaceAggregation with ,omitzero so an unset value is dropped instead of emitted as an invalid "", matching the existing convention (dashboardtypes Sort, telemetrytypes field keys). Valid values still serialize. The OpenAPI spec regenerates byte-identical, which confirms the enums were already correct. * test(querybuildertypesv5): cover client-sent empty enum values A client (e.g. terraform) may send explicit source:"" / temporality:"" for an unset enum. Assert unmarshaling accepts them, normalizes to the zero value, and re-marshaling drops them so the round-trip never echoes an invalid "" back. * fix(querybuildertypesv5): allow empty metric enum values in the spec The server accepts and echoes back an unset source, temporality, and timeAggregation for a metric query (a create -> GET returns "" for them), but their OpenAPI enums omitted "". A typed client (terraform-provider-signoz generate-config) therefore rejected the config generated for an imported rule. Add "" as a valid member of the Source, Temporality, and TimeAggregation enums so the spec matches what the server actually accepts and returns. spaceAggregation is left unchanged: an empty value is rejected with 400 at creation (IsValid), so it is never stored or echoed and "" must stay out of its enum. Drop the earlier ,omitzero tags: these fields already always-serialize, so an accepted "" round-trips faithfully instead of being silently dropped (silent mutation is itself drift). source loses its no-op omitempty for the same reason. Regenerate the OpenAPI spec and frontend client (both git-diff gated). * test(querybuildertypesv5): assert accepted empty enums round-trip Empty source/temporality/timeAggregation are echoed back (not dropped) and are stable across marshal -> unmarshal -> marshal; spaceAggregation carries a valid value since an empty one is 400'd at creation. * test(querybuildertypesv5): merge and rename metric enum round-trip test Fold the unmarshal-echo case into the table-driven marshal round-trip test (its marshal -> unmarshal -> marshal check already covers the client-sends-"" path) and rename to TestQueryBuilderQuery_MetricAggregation_MarshalJSONEnumRoundTrip. * style(metrictypes): drop explanatory comments on enum changes Remove the comments added to Temporality/TimeAggregation Enum() and the metric enum round-trip test case. * style(telemetrytypes): drop explanatory comment on Source enum change Remove the comment added to Source.Enum(), keeping the pre-existing doc/TODO. --- docs/api/openapi.yml | 3 + .../api/generated/services/sigNoz.schemas.ts | 3 + pkg/types/metrictypes/metrictypes.go | 2 + .../querybuildertypesv5/builder_query.go | 2 +- .../querybuildertypesv5/builder_query_test.go | 58 +++++++++++++++++++ pkg/types/telemetrytypes/source.go | 1 + 6 files changed, 68 insertions(+), 1 deletion(-) diff --git a/docs/api/openapi.yml b/docs/api/openapi.yml index a3328c260cd..ac2e4c231fe 100644 --- a/docs/api/openapi.yml +++ b/docs/api/openapi.yml @@ -6334,6 +6334,7 @@ components: - delta - cumulative - unspecified + - "" type: string MetrictypesTimeAggregation: enum: @@ -6346,6 +6347,7 @@ components: - count_distinct - rate - increase + - "" type: string MetrictypesType: enum: @@ -8596,6 +8598,7 @@ components: TelemetrytypesSource: enum: - meter + - "" type: string TelemetrytypesTelemetryFieldKey: properties: diff --git a/frontend/src/api/generated/services/sigNoz.schemas.ts b/frontend/src/api/generated/services/sigNoz.schemas.ts index 6bc5b790ecc..f08e93b4395 100644 --- a/frontend/src/api/generated/services/sigNoz.schemas.ts +++ b/frontend/src/api/generated/services/sigNoz.schemas.ts @@ -3631,6 +3631,7 @@ export enum Querybuildertypesv5QueryBuilderQueryGithubComSigNozSignozPkgTypesQue } export enum TelemetrytypesSourceDTO { meter = 'meter', + '' = '', } export interface Querybuildertypesv5QueryBuilderQueryGithubComSigNozSignozPkgTypesQuerybuildertypesQuerybuildertypesv5LogAggregationDTO { /** @@ -3730,6 +3731,7 @@ export enum MetrictypesTemporalityDTO { delta = 'delta', cumulative = 'cumulative', unspecified = 'unspecified', + '' = '', } export enum MetrictypesTimeAggregationDTO { latest = 'latest', @@ -3741,6 +3743,7 @@ export enum MetrictypesTimeAggregationDTO { count_distinct = 'count_distinct', rate = 'rate', increase = 'increase', + '' = '', } export interface Querybuildertypesv5MetricAggregationDTO { comparisonSpaceAggregationParam?: MetrictypesComparisonSpaceAggregationParamDTO; diff --git a/pkg/types/metrictypes/metrictypes.go b/pkg/types/metrictypes/metrictypes.go index 4f8605b7bdc..2e7d9aa35de 100644 --- a/pkg/types/metrictypes/metrictypes.go +++ b/pkg/types/metrictypes/metrictypes.go @@ -74,6 +74,7 @@ func (Temporality) Enum() []any { Delta, Cumulative, Unspecified, + Unknown, } } @@ -187,6 +188,7 @@ func (TimeAggregation) Enum() []any { TimeAggregationCountDistinct, TimeAggregationRate, TimeAggregationIncrease, + TimeAggregationUnspecified, } } diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/builder_query.go b/pkg/types/querybuildertypes/querybuildertypesv5/builder_query.go index c49e91c10e2..68eba977ef1 100644 --- a/pkg/types/querybuildertypes/querybuildertypesv5/builder_query.go +++ b/pkg/types/querybuildertypes/querybuildertypesv5/builder_query.go @@ -22,7 +22,7 @@ type QueryBuilderQuery[T any] struct { Signal telemetrytypes.Signal `json:"signal,omitempty"` // source for query - Source telemetrytypes.Source `json:"source,omitempty"` + Source telemetrytypes.Source `json:"source"` // we want to support multiple aggregations // currently supported: []Aggregation, []MetricAggregation diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/builder_query_test.go b/pkg/types/querybuildertypes/querybuildertypesv5/builder_query_test.go index 2cdf4a43784..b4f4bf72055 100644 --- a/pkg/types/querybuildertypes/querybuildertypesv5/builder_query_test.go +++ b/pkg/types/querybuildertypes/querybuildertypesv5/builder_query_test.go @@ -651,3 +651,61 @@ func TestQueryBuilderQuery_UnmarshalJSON(t *testing.T) { assert.Equal(t, telemetrytypes.FieldContextSpan, query.Order[0].Key.FieldContext) }) } + +func TestQueryBuilderQuery_MetricAggregation_MarshalJSONEnumRoundTrip(t *testing.T) { + testCases := []struct { + name string + query QueryBuilderQuery[MetricAggregation] + present []string + }{ + { + name: "AcceptedEmptyEnumsAreEchoed", + query: QueryBuilderQuery[MetricAggregation]{ + Name: "A", + Signal: telemetrytypes.SignalMetrics, + Source: telemetrytypes.SourceUnspecified, + Aggregations: []MetricAggregation{{ + MetricName: "system.cpu.usage", + Temporality: metrictypes.Unknown, + TimeAggregation: metrictypes.TimeAggregationUnspecified, + SpaceAggregation: metrictypes.SpaceAggregationSum, + }}, + }, + present: []string{`"source":""`, `"temporality":""`, `"timeAggregation":""`, `"spaceAggregation":"sum"`}, + }, + { + name: "SetEnumsAreSerialized", + query: QueryBuilderQuery[MetricAggregation]{ + Name: "A", + Signal: telemetrytypes.SignalMetrics, + Source: telemetrytypes.SourceMeter, + Aggregations: []MetricAggregation{{ + MetricName: "system.cpu.usage", + Temporality: metrictypes.Cumulative, + TimeAggregation: metrictypes.TimeAggregationRate, + SpaceAggregation: metrictypes.SpaceAggregationSum, + }}, + }, + present: []string{`"source":"meter"`, `"temporality":"cumulative"`, `"timeAggregation":"rate"`, `"spaceAggregation":"sum"`}, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + expected, err := json.Marshal(testCase.query) + require.NoError(t, err) + + for _, fragment := range testCase.present { + assert.Contains(t, string(expected), fragment) + } + + // Marshal -> unmarshal -> marshal must be stable so a create -> GET + // round-trip preserves exactly what the client sent. + var decoded QueryBuilderQuery[MetricAggregation] + require.NoError(t, json.Unmarshal(expected, &decoded)) + actual, err := json.Marshal(decoded) + require.NoError(t, err) + assert.JSONEq(t, string(expected), string(actual)) + }) + } +} diff --git a/pkg/types/telemetrytypes/source.go b/pkg/types/telemetrytypes/source.go index bf6263e1791..aa32e20049d 100644 --- a/pkg/types/telemetrytypes/source.go +++ b/pkg/types/telemetrytypes/source.go @@ -17,5 +17,6 @@ var ( func (Source) Enum() []any { return []any{ SourceMeter, + SourceUnspecified, } } From 88bd7377581b393abbeb5138d9f156890a188cb6 Mon Sep 17 00:00:00 2001 From: Pandey Date: Mon, 20 Jul 2026 02:12:27 +0530 Subject: [PATCH 5/5] fix(dashboards-v2): round-trip all zero-valued spec fields (#12162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dashboardtypes): round-trip zero-valued variable/display fields omitempty dropped explicit zero values from the create -> GET response, so a typed client (Terraform/SDK) that sent them read back null and reported drift. Remove the tag so these always serialize: - Display.Description ("" round-trips; applies to dashboard/panel/variable displays) - TextVariableSpec.Constant (constant: false, like the disabled fix) - ListVariableSpec.CustomAllValue / CapturingRegexp ("" round-trips) Scalars carry no nullability, so the OpenAPI spec and generated client are unchanged. Sort stays omitzero: its "no sort" value is "none", not "", so omitzero only omits the invalid unset state. * fix(dashboardtypes): round-trip panel and dashboard links `links` used omitempty (dropped an explicit []) and its element type was the imported perses dashboard.Link, whose own fields tag name/tooltip/ renderVariables/targetBlank omitempty — so a link's false/"" were dropped too, and a typed client read them back as null. - Replicate dashboard.Link as a SigNoz Link type (same pattern as ListVariableSpec/TextVariableSpec) with every field always serialized. - Use ,omitzero on PanelSpec.Links and DashboardSpec.Links so an explicit [] round-trips while an unset list stays omitted (never null). Regenerate the OpenAPI spec and frontend client: the element schema is now DashboardtypesLink (was the perses DashboardLink) and links is nullable. Update the frontend consumers to the renamed type and coalesce the now type-nullable spec.links (never null on the wire) at its two boundaries. * test(dashboard): cover variable/display/link round-trip cases Extend the v2 dashboard round-trip test with the spec-wide zero values this PR fixes: a display description "", a text variable's constant false, a list variable's customAllValue/capturingRegexp "", an explicit [] of panel links that round-trips, a link whose own zero-valued fields (name/tooltip "", renderVariables/targetBlank false) echo back, and a linkless panel whose links stay omitted (never null). * test(dashboard): accept null-or-absent for unset panel links A panel with no links round-trips as "links": null rather than being omitted (the panel serialization path differs from the query slices, which omit). Both mean "no links" and neither drifts for a typed client, so assert the value is None (null or absent) instead of strictly absent. The explicit [] case still asserts a verbatim round-trip, which is the guarantee the fix provides. * fix(dashboardtypes): round-trip remaining zero-valued spec fields Complete the dashboards-v2 create -> GET round-trip audit: - DashboardSpec.Datasources: ,omitempty -> ,omitzero so an explicit {} round-trips (omitempty dropped it) while an unset map stays omitted. - DashboardV2 Image, DashboardSpec.Duration/RefreshInterval: drop ,omitempty so an explicit "" round-trips (same class as Display.Description). The server accepts "": DurationString.validate() returns nil for len 0, and Image/Duration/RefreshInterval have no create-time validation, so a GET-then-PUT of "" is not rejected. Scalars carry no nullability (no spec change); the datasources map is now nullable: true in the regenerated OpenAPI spec and client. * test(dashboard): cover datasources/image/duration/refreshInterval round-trip Extend the round-trip test with the spec-wide zero values just fixed: a dashboard-level image "", spec duration/refreshInterval "", and an explicit empty datasources {} that must echo back as {} (omitzero) rather than being dropped. --- docs/api/openapi.yml | 33 +++++----- .../api/generated/services/sigNoz.schemas.ts | 64 ++++++++++--------- .../ConfigPane/sectionRegistry.tsx | 4 +- .../ContextLinksSection/ContextLinkDialog.tsx | 6 +- .../ContextLinkListItem.tsx | 4 +- .../ContextLinksSection.tsx | 4 +- .../__tests__/ContextLinksSection.test.tsx | 6 +- .../Panels/types/sections.ts | 4 +- .../drilldown/resolvePanelContextLinks.ts | 4 +- .../DrilldownMenu/DrilldownAggregateMenu.tsx | 4 +- .../Panel/hooks/useDrilldown.tsx | 2 +- pkg/types/dashboardtypes/perses_dashboard.go | 2 +- .../dashboardtypes/perses_dashboard_data.go | 8 +-- pkg/types/dashboardtypes/perses_replicas.go | 32 +++++++--- .../tests/dashboard/03_v2_dashboard.py | 60 ++++++++++++++++- 15 files changed, 158 insertions(+), 79 deletions(-) diff --git a/docs/api/openapi.yml b/docs/api/openapi.yml index ac2e4c231fe..03b6c021494 100644 --- a/docs/api/openapi.yml +++ b/docs/api/openapi.yml @@ -2596,19 +2596,6 @@ components: repeatVariable: type: string type: object - DashboardLink: - properties: - name: - type: string - renderVariables: - type: boolean - targetBlank: - type: boolean - tooltip: - type: string - url: - type: string - type: object DashboardtypesAxes: properties: isLogScale: @@ -2747,6 +2734,7 @@ components: datasources: additionalProperties: $ref: '#/components/schemas/DashboardtypesDatasourceSpec' + nullable: true type: object display: $ref: '#/components/schemas/DashboardtypesDisplay' @@ -2758,7 +2746,8 @@ components: type: array links: items: - $ref: '#/components/schemas/DashboardLink' + $ref: '#/components/schemas/DashboardtypesLink' + nullable: true type: array panels: additionalProperties: @@ -3029,6 +3018,19 @@ components: - solid - dashed type: string + DashboardtypesLink: + properties: + name: + type: string + renderVariables: + type: boolean + targetBlank: + type: boolean + tooltip: + type: string + url: + type: string + type: object DashboardtypesListOrder: enum: - asc @@ -3389,7 +3391,8 @@ components: $ref: '#/components/schemas/DashboardtypesDisplay' links: items: - $ref: '#/components/schemas/DashboardLink' + $ref: '#/components/schemas/DashboardtypesLink' + nullable: true type: array plugin: $ref: '#/components/schemas/DashboardtypesPanelPlugin' diff --git a/frontend/src/api/generated/services/sigNoz.schemas.ts b/frontend/src/api/generated/services/sigNoz.schemas.ts index f08e93b4395..c60b0bd1906 100644 --- a/frontend/src/api/generated/services/sigNoz.schemas.ts +++ b/frontend/src/api/generated/services/sigNoz.schemas.ts @@ -3300,29 +3300,6 @@ export interface DashboardGridLayoutSpecDTO { repeatVariable?: string; } -export interface DashboardLinkDTO { - /** - * @type string - */ - name?: string; - /** - * @type boolean - */ - renderVariables?: boolean; - /** - * @type boolean - */ - targetBlank?: boolean; - /** - * @type string - */ - tooltip?: string; - /** - * @type string - */ - url?: string; -} - export interface DashboardtypesAxesDTO { /** * @type boolean @@ -4037,10 +4014,16 @@ export interface DashboardtypesDatasourceSpecDTO { plugin?: DashboardtypesDatasourcePluginDTO; } -export type DashboardtypesDashboardSpecDTODatasources = { +export type DashboardtypesDashboardSpecDTODatasourcesAnyOf = { [key: string]: DashboardtypesDatasourceSpecDTO; }; +/** + * @nullable + */ +export type DashboardtypesDashboardSpecDTODatasources = + DashboardtypesDashboardSpecDTODatasourcesAnyOf | null; + export enum DashboardtypesPanelKindDTO { Panel = 'Panel', } @@ -4055,6 +4038,29 @@ export interface DashboardtypesDisplayDTO { name: string; } +export interface DashboardtypesLinkDTO { + /** + * @type string + */ + name?: string; + /** + * @type boolean + */ + renderVariables?: boolean; + /** + * @type boolean + */ + targetBlank?: boolean; + /** + * @type string + */ + tooltip?: string; + /** + * @type string + */ + url?: string; +} + export enum DashboardtypesPanelPluginVariantGithubComSigNozSignozPkgTypesDashboardtypesTimeSeriesPanelSpecDTOKind { 'signoz/TimeSeriesPanel' = 'signoz/TimeSeriesPanel', } @@ -4616,9 +4622,9 @@ export interface DashboardtypesQueryDTO { export interface DashboardtypesPanelSpecDTO { display: DashboardtypesDisplayDTO; /** - * @type array + * @type array,null */ - links?: DashboardLinkDTO[]; + links?: DashboardtypesLinkDTO[] | null; plugin: DashboardtypesPanelPluginDTO; /** * @type array @@ -4792,7 +4798,7 @@ export type DashboardtypesVariableDTO = export interface DashboardtypesDashboardSpecDTO { /** - * @type object + * @type object,null */ datasources?: DashboardtypesDashboardSpecDTODatasources; display: DashboardtypesDisplayDTO; @@ -4805,9 +4811,9 @@ export interface DashboardtypesDashboardSpecDTO { */ layouts: DashboardtypesLayoutDTO[]; /** - * @type array + * @type array,null */ - links?: DashboardLinkDTO[]; + links?: DashboardtypesLinkDTO[] | null; /** * @type object */ diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sectionRegistry.tsx b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sectionRegistry.tsx index 43ece3171f9..78d5072b7e3 100644 --- a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sectionRegistry.tsx +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sectionRegistry.tsx @@ -1,6 +1,6 @@ import type { ComponentType } from 'react'; import type { - DashboardLinkDTO, + DashboardtypesLinkDTO, DashboardtypesAxesDTO, DashboardtypesBarChartVisualizationDTO, DashboardtypesHistogramBucketsDTO, @@ -120,7 +120,7 @@ export const SECTION_REGISTRY: { [SectionKind.ContextLinks]: { Component: ContextLinksSection, // Panel-level slice (spec.links), not under the plugin spec — no cast needed. - get: (spec): DashboardLinkDTO[] | undefined => spec.links, + get: (spec): DashboardtypesLinkDTO[] | undefined => spec.links ?? undefined, update: (spec, links): PanelSpec => ({ ...spec, links }), }, // One editor for every threshold variant (label / comparison / table); the kind's diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/ContextLinkDialog.tsx b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/ContextLinkDialog.tsx index 284e584e21d..e576caae737 100644 --- a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/ContextLinkDialog.tsx +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/ContextLinkDialog.tsx @@ -5,7 +5,7 @@ import { DialogWrapper } from '@signozhq/ui/dialog'; import { Switch } from '@signozhq/ui/switch'; import { Typography } from '@signozhq/ui/typography'; import { Input } from 'antd'; -import type { DashboardLinkDTO } from 'api/generated/services/sigNoz.schemas'; +import type { DashboardtypesLinkDTO } from 'api/generated/services/sigNoz.schemas'; import type { UrlParam, VariableItem } from './types'; import { @@ -21,10 +21,10 @@ import styles from './ContextLinksSection.module.scss'; interface ContextLinkDialogProps { open: boolean; /** The link being edited, or null when adding a new one. */ - initialLink: DashboardLinkDTO | null; + initialLink: DashboardtypesLinkDTO | null; variables: VariableItem[]; onOpenChange: (open: boolean) => void; - onSave: (link: DashboardLinkDTO) => void; + onSave: (link: DashboardtypesLinkDTO) => void; } const URL_ERROR = 'URL must start with http(s)://, /, or {{variable}}/'; diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/ContextLinkListItem.tsx b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/ContextLinkListItem.tsx index 9da61938335..a8c2334b34d 100644 --- a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/ContextLinkListItem.tsx +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/ContextLinkListItem.tsx @@ -1,12 +1,12 @@ import { Pencil, Trash2 } from '@signozhq/icons'; import { Button } from '@signozhq/ui/button'; import { Typography } from '@signozhq/ui/typography'; -import type { DashboardLinkDTO } from 'api/generated/services/sigNoz.schemas'; +import type { DashboardtypesLinkDTO } from 'api/generated/services/sigNoz.schemas'; import styles from './ContextLinksSection.module.scss'; interface ContextLinkListItemProps { - link: DashboardLinkDTO; + link: DashboardtypesLinkDTO; index: number; onEdit: () => void; onRemove: () => void; diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/ContextLinksSection.tsx b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/ContextLinksSection.tsx index d5d0d2745d1..f829cbfc990 100644 --- a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/ContextLinksSection.tsx +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/ContextLinksSection.tsx @@ -1,7 +1,7 @@ import { useState } from 'react'; import { Plus } from '@signozhq/icons'; import { Button } from '@signozhq/ui/button'; -import type { DashboardLinkDTO } from 'api/generated/services/sigNoz.schemas'; +import type { DashboardtypesLinkDTO } from 'api/generated/services/sigNoz.schemas'; import type { SectionEditorProps, SectionKind, @@ -34,7 +34,7 @@ function ContextLinksSection({ const removeAt = (index: number): void => onChange(links.filter((_, i) => i !== index)); - const handleSave = (link: DashboardLinkDTO): void => { + const handleSave = (link: DashboardtypesLinkDTO): void => { onChange( dialog.index === null ? [...links, link] diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/__tests__/ContextLinksSection.test.tsx b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/__tests__/ContextLinksSection.test.tsx index 9a1ffb2bacb..62392a3ecfc 100644 --- a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/__tests__/ContextLinksSection.test.tsx +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelEditor/ConfigPane/sections/ContextLinksSection/__tests__/ContextLinksSection.test.tsx @@ -1,5 +1,5 @@ import { fireEvent, render, screen } from '@testing-library/react'; -import type { DashboardLinkDTO } from 'api/generated/services/sigNoz.schemas'; +import type { DashboardtypesLinkDTO } from 'api/generated/services/sigNoz.schemas'; import ContextLinksSection from '../ContextLinksSection'; @@ -12,11 +12,11 @@ jest.mock('../useContextLinkVariables', () => ({ ], })); -const LINKS: DashboardLinkDTO[] = [ +const LINKS: DashboardtypesLinkDTO[] = [ { name: 'Docs', url: 'https://signoz.io', targetBlank: true }, ]; -const lastCall = (fn: jest.Mock): DashboardLinkDTO[] => +const lastCall = (fn: jest.Mock): DashboardtypesLinkDTO[] => fn.mock.calls[fn.mock.calls.length - 1][0]; describe('ContextLinksSection', () => { diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/types/sections.ts b/frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/types/sections.ts index ed12b65c6e8..80225c32374 100644 --- a/frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/types/sections.ts +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/types/sections.ts @@ -1,5 +1,5 @@ import type { - DashboardLinkDTO, + DashboardtypesLinkDTO, DashboardtypesAxesDTO, DashboardtypesBarChartVisualizationDTO, DashboardtypesComparisonThresholdDTO, @@ -89,7 +89,7 @@ export interface SectionSpecMap { // the `controls` bag gates which fields each kind writes. [SectionKind.Visualization]: DashboardtypesBarChartVisualizationDTO; [SectionKind.Thresholds]: AnyThreshold[]; // spec.plugin.spec.thresholds (variant picks the editor) - [SectionKind.ContextLinks]: DashboardLinkDTO[]; // spec.links (PANEL-level) + [SectionKind.ContextLinks]: DashboardtypesLinkDTO[]; // spec.links (PANEL-level) [SectionKind.Columns]: TelemetrytypesTelemetryFieldKeyDTO[]; // spec.plugin.spec.selectFields (List) } diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/utils/drilldown/resolvePanelContextLinks.ts b/frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/utils/drilldown/resolvePanelContextLinks.ts index fb480c7be6e..58843eaa165 100644 --- a/frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/utils/drilldown/resolvePanelContextLinks.ts +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/utils/drilldown/resolvePanelContextLinks.ts @@ -1,4 +1,4 @@ -import type { DashboardLinkDTO } from 'api/generated/services/sigNoz.schemas'; +import type { DashboardtypesLinkDTO } from 'api/generated/services/sigNoz.schemas'; import { resolveTexts } from 'hooks/dashboard/useContextVariables'; import { resolveContextLinkUrl } from './resolveContextLinkUrl'; @@ -15,7 +15,7 @@ export interface ResolvedDrilldownLink { * label + URL, drops links without a URL, and skips substitution when `renderVariables === false`. */ export function resolvePanelContextLinks( - links: DashboardLinkDTO[] | undefined, + links: DashboardtypesLinkDTO[] | undefined, processedVariables: Record, ): ResolvedDrilldownLink[] { const usable = (links ?? []).filter((link) => !!link.url); diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelsAndSectionsLayout/Panel/DrilldownMenu/DrilldownAggregateMenu.tsx b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelsAndSectionsLayout/Panel/DrilldownMenu/DrilldownAggregateMenu.tsx index 75bf7346f74..137f448b38c 100644 --- a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelsAndSectionsLayout/Panel/DrilldownMenu/DrilldownAggregateMenu.tsx +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelsAndSectionsLayout/Panel/DrilldownMenu/DrilldownAggregateMenu.tsx @@ -7,7 +7,7 @@ import { Loader, ScrollText, } from '@signozhq/icons'; -import type { DashboardLinkDTO } from 'api/generated/services/sigNoz.schemas'; +import type { DashboardtypesLinkDTO } from 'api/generated/services/sigNoz.schemas'; import { getAggregateColumnHeader } from 'container/QueryTable/Drilldown/drilldownUtils'; import ContextMenu from 'periscope/components/ContextMenu'; import type { DrilldownContext } from 'pages/DashboardPageV2/DashboardContainer/Panels/types/drilldown'; @@ -27,7 +27,7 @@ interface DrilldownAggregateMenuProps { /** While dashboard variables resolve, the actions show a spinner and are disabled. */ isResolving?: boolean; /** Panel's context links; resolved against the clicked point + variables here. */ - links: DashboardLinkDTO[] | undefined; + links: DashboardtypesLinkDTO[] | undefined; /** Whether the clicked point exposes group-by fields to bind to dashboard variables. */ canSetDashboardVariables: boolean; onViewLogs: () => void; diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelsAndSectionsLayout/Panel/hooks/useDrilldown.tsx b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelsAndSectionsLayout/Panel/hooks/useDrilldown.tsx index 6d379a95595..f0ec7bad3e6 100644 --- a/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelsAndSectionsLayout/Panel/hooks/useDrilldown.tsx +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/PanelsAndSectionsLayout/Panel/hooks/useDrilldown.tsx @@ -218,7 +218,7 @@ export function useDrilldown( context={context} query={v1Query} isResolving={isResolving} - links={panel.spec.links} + links={panel.spec.links ?? undefined} canSetDashboardVariables={dashboardVariables.hasFieldVariables} onViewLogs={(): void => navigate('view_logs')} onViewTraces={(): void => navigate('view_traces')} diff --git a/pkg/types/dashboardtypes/perses_dashboard.go b/pkg/types/dashboardtypes/perses_dashboard.go index c69b63b16f4..c2b5564fc8f 100644 --- a/pkg/types/dashboardtypes/perses_dashboard.go +++ b/pkg/types/dashboardtypes/perses_dashboard.go @@ -177,7 +177,7 @@ func nextCloneDisplayName(name string) string { type DashboardV2MetadataBase struct { SchemaVersion string `json:"schemaVersion" required:"true"` - Image string `json:"image,omitempty"` + Image string `json:"image"` } // ════════════════════════════════════════════════════════════════════════ diff --git a/pkg/types/dashboardtypes/perses_dashboard_data.go b/pkg/types/dashboardtypes/perses_dashboard_data.go index 489e001d75a..589d884e661 100644 --- a/pkg/types/dashboardtypes/perses_dashboard_data.go +++ b/pkg/types/dashboardtypes/perses_dashboard_data.go @@ -20,13 +20,13 @@ import ( // per-site discriminated oneOf. type DashboardSpec struct { Display Display `json:"display" required:"true"` - Datasources map[string]*DatasourceSpec `json:"datasources,omitempty"` + Datasources map[string]*DatasourceSpec `json:"datasources,omitzero"` Variables []Variable `json:"variables" required:"true" nullable:"false"` Panels map[string]*Panel `json:"panels" required:"true" nullable:"false"` Layouts []Layout `json:"layouts" required:"true" nullable:"false"` - Duration common.DurationString `json:"duration,omitempty"` - RefreshInterval common.DurationString `json:"refreshInterval,omitempty"` - Links []dashboard.Link `json:"links,omitempty"` + Duration common.DurationString `json:"duration"` + RefreshInterval common.DurationString `json:"refreshInterval"` + Links []Link `json:"links,omitzero"` } // ══════════════════════════════════════════════ diff --git a/pkg/types/dashboardtypes/perses_replicas.go b/pkg/types/dashboardtypes/perses_replicas.go index ca50d5ee320..2a6c92e0851 100644 --- a/pkg/types/dashboardtypes/perses_replicas.go +++ b/pkg/types/dashboardtypes/perses_replicas.go @@ -21,8 +21,10 @@ import ( const MaxDisplayNameLen = 128 type Display struct { - Name string `json:"name" required:"true"` - Description string `json:"description,omitempty"` + Name string `json:"name" required:"true"` + // Description always serializes ("" included) so a create -> GET round-trip + // preserves what a typed client sent; omitempty would drop an explicit "". + Description string `json:"description"` } func (d Display) Validate(label, path string) error { @@ -73,10 +75,22 @@ func (k *PanelKind) UnmarshalJSON(data []byte) error { } type PanelSpec struct { - Display Display `json:"display" required:"true"` - Plugin PanelPlugin `json:"plugin" required:"true"` - Queries []Query `json:"queries" required:"true" nullable:"false"` - Links []dashboard.Link `json:"links,omitempty"` + Display Display `json:"display" required:"true"` + Plugin PanelPlugin `json:"plugin" required:"true"` + Queries []Query `json:"queries" required:"true" nullable:"false"` + Links []Link `json:"links,omitzero"` +} + +// Link replicates dashboard.Link (Perses) so its zero-valued fields survive the +// create -> GET round-trip. Perses tags name/tooltip/renderVariables/targetBlank +// omitempty, which drops "" and false; here every field always serializes so a +// typed client reads back exactly what it sent. +type Link struct { + Name string `json:"name"` + URL string `json:"url"` + Tooltip string `json:"tooltip"` + RenderVariables bool `json:"renderVariables"` + TargetBlank bool `json:"targetBlank"` } // ══════════════════════════════════════════════ @@ -161,8 +175,8 @@ type ListVariableSpec struct { DefaultValue *VariableDefaultValue `json:"defaultValue,omitempty"` AllowAllValue bool `json:"allowAllValue"` AllowMultiple bool `json:"allowMultiple"` - CustomAllValue string `json:"customAllValue,omitempty"` - CapturingRegexp string `json:"capturingRegexp,omitempty"` + CustomAllValue string `json:"customAllValue"` + CapturingRegexp string `json:"capturingRegexp"` Sort ListVariableSpecSort `json:"sort,omitzero"` Plugin VariablePlugin `json:"plugin"` Name string `json:"name" required:"true" minLength:"1"` @@ -278,7 +292,7 @@ func (s *ListVariableSpecSort) UnmarshalJSON(data []byte) error { type TextVariableSpec struct { Display Display `json:"display" required:"true"` Value string `json:"value" required:"true"` - Constant bool `json:"constant,omitempty"` + Constant bool `json:"constant"` Name string `json:"name" required:"true" minLength:"1"` } diff --git a/tests/integration/tests/dashboard/03_v2_dashboard.py b/tests/integration/tests/dashboard/03_v2_dashboard.py index 1bd94f161df..888775f7b13 100644 --- a/tests/integration/tests/dashboard/03_v2_dashboard.py +++ b/tests/integration/tests/dashboard/03_v2_dashboard.py @@ -1570,6 +1570,11 @@ def make_dashboard(aggregations: list[dict]) -> dict: # the required-tag validation used to reject on create), builder slices set to an # explicit [] that must echo back as [] (yet stay absent, never null, when unset), # and scalars disabled/legend that must always echo false/"". +# +# It also covers the spec-wide zero values: a display description "", a text +# variable's constant false, a list variable's customAllValue/capturingRegexp "", +# an explicit [] of panel links, and a link whose own zero-valued fields +# (name/tooltip "", renderVariables/targetBlank false) must echo back. def test_dashboard_v2_roundtrip_preserves_zero_values( @@ -1582,10 +1587,36 @@ def test_dashboard_v2_roundtrip_preserves_zero_values( dashboard = { "schemaVersion": "v6", + # image (dashboard-level) and spec duration/refreshInterval/datasources + # each round-trip their zero value ("" / {}) rather than being dropped. + "image": "", "name": "roundtrip-zero-values", "tags": [], "spec": { - "display": {"name": "Roundtrip Zero Values"}, + "display": {"name": "Roundtrip Zero Values", "description": ""}, + "duration": "", + "refreshInterval": "", + "datasources": {}, + "variables": [ + # TextVariable: constant false must echo back (not be dropped). + {"kind": "TextVariable", "spec": {"display": {"name": "tv"}, "value": "x", "constant": False, "name": "tv"}}, + # ListVariable: customAllValue / capturingRegexp "" must echo back. + { + "kind": "ListVariable", + "spec": { + "display": {"name": "lv"}, + "allowAllValue": False, + "allowMultiple": False, + "customAllValue": "", + "capturingRegexp": "", + "plugin": {"kind": "signoz/DynamicVariable", "spec": {"name": "service.name", "signal": "metrics"}}, + "name": "lv", + }, + }, + ], + # Dashboard-level link with zero-valued inner fields: the replicated Link + # type must echo name/tooltip "" and renderVariables/targetBlank false. + "links": [{"url": "https://example.com", "name": "", "tooltip": "", "renderVariables": False, "targetBlank": False}], "panels": { # NumberPanel: ComparisonThreshold value 0 + a builder query whose # zero-valued slices/scalars are all set explicitly. @@ -1593,6 +1624,8 @@ def test_dashboard_v2_roundtrip_preserves_zero_values( "kind": "Panel", "spec": { "display": {"name": "number"}, + # Explicit empty panel links must round-trip as []. + "links": [], "plugin": { "kind": "signoz/NumberPanel", "spec": {"thresholds": [{"value": 0, "operator": "above_or_equal", "color": "#c2780b", "format": "background"}]}, @@ -1702,7 +1735,11 @@ def test_dashboard_v2_roundtrip_preserves_zero_values( timeout=5, ) assert response.status_code == HTTPStatus.OK, response.text - panels = response.json()["data"]["spec"]["panels"] + result_data = response.json()["data"] + result_spec = result_data["spec"] + panels = result_spec["panels"] + variables = {v["kind"]: v["spec"] for v in result_spec["variables"]} + link = result_spec["links"][0] def query_spec(panel_id: str) -> dict: return panels[panel_id]["spec"]["queries"][0]["spec"]["plugin"]["spec"] @@ -1731,6 +1768,19 @@ def threshold(panel_id: str) -> dict: ("promql legend empty", promql["legend"], ""), ("clickhouse disabled false", clickhouse["disabled"], False), ("clickhouse legend empty", clickhouse["legend"], ""), + ("dashboard display description empty", result_spec["display"]["description"], ""), + ("text variable constant false", variables["TextVariable"]["constant"], False), + ("list variable customAllValue empty", variables["ListVariable"]["customAllValue"], ""), + ("list variable capturingRegexp empty", variables["ListVariable"]["capturingRegexp"], ""), + ("panel empty links round-trip", panels["number"]["spec"]["links"], []), + ("dashboard link name empty", link["name"], ""), + ("dashboard link tooltip empty", link["tooltip"], ""), + ("dashboard link renderVariables false", link["renderVariables"], False), + ("dashboard link targetBlank false", link["targetBlank"], False), + ("dashboard image empty", result_data["image"], ""), + ("spec duration empty", result_spec["duration"], ""), + ("spec refreshInterval empty", result_spec["refreshInterval"], ""), + ("spec empty datasources round-trip", result_spec["datasources"], {}), ] for description, actual, expected in roundtrip_cases: assert actual == expected, description @@ -1744,6 +1794,12 @@ def threshold(panel_id: str) -> dict: ] for description, spec, key in absent_cases: assert key not in spec, description + + # A panel with no links comes back with no links value (null or absent); + # either is fine for a typed client (an unset optional attribute stays + # unset), so this is not a drift. The explicit [] case above is what the + # fix guarantees round-trips. + assert panels["timeseries"]["spec"].get("links") is None, "unset panel links stays unset" finally: requests.delete( signoz.self.host_configs["8080"].get(f"{BASE_URL}/{dashboard_id}"),