From 62737e10018bac07624d63260c9bcbfaec95b341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aybars=20Mete=20Kele=C5=9F?= Date: Tue, 4 Aug 2026 16:55:58 +0300 Subject: [PATCH] test: compare generated queries against SQLite --- internal/difftest/differential_test.go | 103 +++++++++++++++++ internal/difftest/engine.go | 34 ++++++ internal/difftest/generator.go | 147 ++++++++++++++++++------- internal/difftest/generator_test.go | 51 +++++++++ 4 files changed, 295 insertions(+), 40 deletions(-) create mode 100644 internal/difftest/differential_test.go create mode 100644 internal/difftest/engine.go diff --git a/internal/difftest/differential_test.go b/internal/difftest/differential_test.go new file mode 100644 index 0000000..7336bbf --- /dev/null +++ b/internal/difftest/differential_test.go @@ -0,0 +1,103 @@ +package difftest + +import ( + "flag" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/aybavs/sql-query-engine/internal/catalog" + "github.com/aybavs/sql-query-engine/internal/value" +) + +var ( + seed = flag.Int64("seed", 20260804, "seed for the query generator") + queries = flag.Int("queries", 300, "number of generated queries to compare") +) + +// diffFixture is deliberately small but full of NULLs, duplicate values, and +// ties, since those are where two engines are most likely to disagree. +func diffFixture(t *testing.T) (*catalog.Catalog, string) { + t.Helper() + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "users.csv"), []byte( + "1,alice,30,berlin\n"+ + "2,bob,,berlin\n"+ + "3,carol,40,\n"+ + "4,dan,25,london\n"+ + "5,erin,25,paris\n"+ + "6,frank,,\n", + ), 0o644) + os.WriteFile(filepath.Join(dir, "orders.csv"), []byte( + "10,1,100\n11,1,300\n12,2,50\n13,,75\n14,5,50\n", + ), 0o644) + + cat := catalog.New() + cat.Add(&catalog.Table{Name: "users", File: "users.csv", Columns: []catalog.Column{ + {Name: "id", Type: value.TInt}, + {Name: "name", Type: value.TText}, + {Name: "age", Type: value.TInt}, + {Name: "city", Type: value.TText}, + }}) + cat.Add(&catalog.Table{Name: "orders", File: "orders.csv", Columns: []catalog.Column{ + {Name: "id", Type: value.TInt}, + {Name: "user_id", Type: value.TInt}, + {Name: "total", Type: value.TInt}, + }}) + return cat, dir +} + +// TestDifferentialAgainstSQLite runs generated queries through this engine and +// through SQLite over identical data, and requires the results to agree. +func TestDifferentialAgainstSQLite(t *testing.T) { + cat, dir := diffFixture(t) + + oracle, err := NewOracle(cat, dir) + if err != nil { + t.Fatalf("oracle: %v", err) + } + defer oracle.Close() + + g := NewGenerator(*seed, cat) + compared, joined := 0, 0 + for i := 0; i < *queries; i++ { + q := g.Query() + if strings.Contains(q, " JOIN ") { + joined++ + } + + engineRows, engErr := runEngine(cat, dir, q) + oracleRows, orcErr := oracle.Query(q) + + // Both rejecting a query is uninteresting; disagreeing about whether a + // query is even valid is a finding worth reporting. + switch { + case engErr != nil && orcErr != nil: + continue + case engErr != nil: + t.Fatalf("seed %d query %d: engine rejected a query SQLite accepted\n %s\n %v", *seed, i, q, engErr) + case orcErr != nil: + t.Fatalf("seed %d query %d: SQLite rejected a query the engine accepted\n %s\n %v", *seed, i, q, orcErr) + } + + normalized := make([][]string, len(oracleRows)) + for j, r := range oracleRows { + normalized[j] = normalizeOracleRow(r) + } + if err := Compare(engineRows, normalized); err != nil { + t.Fatalf("seed %d query %d disagrees:\n %s\n%v", *seed, i, q, err) + } + compared++ + } + + if compared < *queries/2 { + t.Fatalf("only %d/%d generated queries were actually compared; the generator is producing too many invalid queries", compared, *queries) + } + // The hash join is the engine's most intricate operator, so it has to be a + // real share of what the oracle checks, not an occasional accident. + if joined < *queries/20 { + t.Fatalf("only %d/%d generated queries joined; the hash join is barely covered", joined, *queries) + } + t.Logf("compared %d generated queries against SQLite (%d of them joins, seed %d)", compared, joined, *seed) +} diff --git a/internal/difftest/engine.go b/internal/difftest/engine.go new file mode 100644 index 0000000..aa3d9aa --- /dev/null +++ b/internal/difftest/engine.go @@ -0,0 +1,34 @@ +package difftest + +import ( + "github.com/aybavs/sql-query-engine/internal/catalog" + "github.com/aybavs/sql-query-engine/internal/lexer" + "github.com/aybavs/sql-query-engine/internal/parser" + "github.com/aybavs/sql-query-engine/internal/plan" +) + +// runEngine executes a query through this engine — lexer, parser, planner, then +// the operator tree — and returns rows normalized for comparison. +func runEngine(cat *catalog.Catalog, dataDir, query string) ([][]string, error) { + toks, err := lexer.Lex(query) + if err != nil { + return nil, err + } + st, err := parser.New(toks).ParseSelect() + if err != nil { + return nil, err + } + op, _, err := plan.Build(st, cat, dataDir) + if err != nil { + return nil, err + } + + var out [][]string + for { + row, ok := op.Next() + if !ok { + return out, nil + } + out = append(out, normalizeEngineRow(row)) + } +} diff --git a/internal/difftest/generator.go b/internal/difftest/generator.go index 4392227..6949ba5 100644 --- a/internal/difftest/generator.go +++ b/internal/difftest/generator.go @@ -25,26 +25,84 @@ func NewGenerator(seed int64, cat *catalog.Catalog) *Generator { } } +// qualified is a column together with the table it came from, so a query over +// several tables can always write an unambiguous reference. +type qualified struct { + table *catalog.Table + column catalog.Column +} + +func (q qualified) ref() string { return q.table.Name + "." + q.column.Name } + func (g *Generator) Query() string { - t := g.tables[g.rnd.Intn(len(g.tables))] + // A join exercises the hash join, which is the most intricate operator in + // the engine and therefore the one most worth checking against an oracle. + if len(g.tables) >= 2 && g.chance(0.3) { + if q, ok := g.joinQuery(); ok { + return q + } + } + + scope := []*catalog.Table{g.tables[g.rnd.Intn(len(g.tables))]} if g.chance(0.35) { - return g.aggregateQuery(t) + return g.aggregateQuery(scope, scope[0].Name) + } + return g.selectQuery(scope, scope[0].Name) +} + +// joinQuery joins two tables on a pair of same-type columns. The pairing need +// not be semantically meaningful — an arbitrary equi-join still exercises +// build/probe, duplicate keys, and NULL keys, which is what is being checked. +func (g *Generator) joinQuery() (string, bool) { + i := g.rnd.Intn(len(g.tables)) + j := g.rnd.Intn(len(g.tables) - 1) + if j >= i { + j++ + } + left, right := g.tables[i], g.tables[j] + + lc, rc, ok := g.joinKeys(left, right) + if !ok { + return "", false + } + + scope := []*catalog.Table{left, right} + from := fmt.Sprintf("%s JOIN %s ON %s = %s", left.Name, right.Name, lc.ref(), rc.ref()) + + if g.chance(0.3) { + return g.aggregateQuery(scope, from), true } - return g.selectQuery(t) + return g.selectQuery(scope, from), true } -func (g *Generator) selectQuery(t *catalog.Table) string { +// joinKeys picks one column from each table that share a type. +func (g *Generator) joinKeys(left, right *catalog.Table) (qualified, qualified, bool) { + var pairs [][2]qualified + for _, lc := range g.usableColumns(left) { + for _, rc := range g.usableColumns(right) { + if lc.Type == rc.Type { + pairs = append(pairs, [2]qualified{{left, lc}, {right, rc}}) + } + } + } + if len(pairs) == 0 { + return qualified{}, qualified{}, false + } + p := pairs[g.rnd.Intn(len(pairs))] + return p[0], p[1], true +} + +func (g *Generator) selectQuery(scope []*catalog.Table, from string) string { var b strings.Builder b.WriteString("SELECT ") - b.WriteString(strings.Join(g.projections(t), ", ")) - fmt.Fprintf(&b, " FROM %s", t.Name) + b.WriteString(strings.Join(g.projections(scope), ", ")) + fmt.Fprintf(&b, " FROM %s", from) if g.chance(0.7) { - fmt.Fprintf(&b, " WHERE %s", g.predicate(t)) + fmt.Fprintf(&b, " WHERE %s", g.predicate(scope)) } if g.chance(0.5) { - c := g.column(t) - fmt.Fprintf(&b, " ORDER BY %s.%s", t.Name, c.Name) + fmt.Fprintf(&b, " ORDER BY %s", g.column(scope).ref()) if g.chance(0.5) { b.WriteString(" DESC") } @@ -57,54 +115,52 @@ func (g *Generator) selectQuery(t *catalog.Table) string { // aggregateQuery always groups by the column it projects, so the result is // well-defined rather than relying on SQLite's bare-column extension. -func (g *Generator) aggregateQuery(t *catalog.Table) string { - group := g.column(t) - agg := g.aggregate(t) +func (g *Generator) aggregateQuery(scope []*catalog.Table, from string) string { + group := g.column(scope) var b strings.Builder - fmt.Fprintf(&b, "SELECT %s.%s, %s FROM %s GROUP BY %s.%s", - t.Name, group.Name, agg, t.Name, t.Name, group.Name) + fmt.Fprintf(&b, "SELECT %s, %s FROM %s GROUP BY %s", + group.ref(), g.aggregate(scope), from, group.ref()) if g.chance(0.3) { fmt.Fprintf(&b, " HAVING COUNT(*) > %d", g.rnd.Intn(2)) } return b.String() } -func (g *Generator) aggregate(t *catalog.Table) string { +func (g *Generator) aggregate(scope []*catalog.Table) string { if g.chance(0.25) { return "COUNT(*)" } - numeric := g.numericColumns(t) + numeric := g.numericColumns(scope) if len(numeric) == 0 { return "COUNT(*)" } c := numeric[g.rnd.Intn(len(numeric))] fn := []string{"COUNT", "SUM", "AVG", "MIN", "MAX"}[g.rnd.Intn(5)] - return fmt.Sprintf("%s(%s.%s)", fn, t.Name, c.Name) + return fmt.Sprintf("%s(%s)", fn, c.ref()) } -func (g *Generator) projections(t *catalog.Table) []string { +func (g *Generator) projections(scope []*catalog.Table) []string { if g.chance(0.2) { return []string{"*"} } n := 1 + g.rnd.Intn(2) out := make([]string, n) for i := range out { - c := g.column(t) - out[i] = fmt.Sprintf("%s.%s", t.Name, c.Name) + out[i] = g.column(scope).ref() } return out } // predicate builds a boolean expression whose operands always share a type. -func (g *Generator) predicate(t *catalog.Table) string { - p := g.comparison(t) +func (g *Generator) predicate(scope []*catalog.Table) string { + p := g.comparison(scope) for g.chance(0.35) { op := "AND" if g.chance(0.5) { op = "OR" } - p = fmt.Sprintf("%s %s %s", p, op, g.comparison(t)) + p = fmt.Sprintf("%s %s %s", p, op, g.comparison(scope)) } if g.chance(0.15) { p = "NOT (" + p + ")" @@ -112,19 +168,18 @@ func (g *Generator) predicate(t *catalog.Table) string { return p } -func (g *Generator) comparison(t *catalog.Table) string { - c := g.column(t) - ref := fmt.Sprintf("%s.%s", t.Name, c.Name) +func (g *Generator) comparison(scope []*catalog.Table) string { + c := g.column(scope) if g.chance(0.2) { if g.chance(0.5) { - return ref + " IS NULL" + return c.ref() + " IS NULL" } - return ref + " IS NOT NULL" + return c.ref() + " IS NOT NULL" } op := []string{"=", "<>", "<", "<=", ">", ">="}[g.rnd.Intn(6)] - return fmt.Sprintf("%s %s %s", ref, op, g.literal(c)) + return fmt.Sprintf("%s %s %s", c.ref(), op, g.literal(c.column)) } // literal produces a value of the column's own type: comparing across types is @@ -141,26 +196,38 @@ func (g *Generator) literal(c catalog.Column) string { } } -// column returns a column of a type the generator supports. Booleans are -// excluded because SQLite stores them as integers. -func (g *Generator) column(t *catalog.Table) catalog.Column { - usable := make([]catalog.Column, 0, len(t.Columns)) - for _, c := range t.Columns { - if c.Type != value.TBool { - usable = append(usable, c) +// column picks a usable column from anywhere in scope. +func (g *Generator) column(scope []*catalog.Table) qualified { + var all []qualified + for _, t := range scope { + for _, c := range g.usableColumns(t) { + all = append(all, qualified{t, c}) } } - return usable[g.rnd.Intn(len(usable))] + return all[g.rnd.Intn(len(all))] } -func (g *Generator) numericColumns(t *catalog.Table) []catalog.Column { - var out []catalog.Column +// usableColumns excludes booleans, which SQLite stores as integers. +func (g *Generator) usableColumns(t *catalog.Table) []catalog.Column { + out := make([]catalog.Column, 0, len(t.Columns)) for _, c := range t.Columns { - if c.Type == value.TInt || c.Type == value.TFloat { + if c.Type != value.TBool { out = append(out, c) } } return out } +func (g *Generator) numericColumns(scope []*catalog.Table) []qualified { + var out []qualified + for _, t := range scope { + for _, c := range t.Columns { + if c.Type == value.TInt || c.Type == value.TFloat { + out = append(out, qualified{t, c}) + } + } + } + return out +} + func (g *Generator) chance(p float64) bool { return g.rnd.Float64() < p } diff --git a/internal/difftest/generator_test.go b/internal/difftest/generator_test.go index 434d077..9c71d82 100644 --- a/internal/difftest/generator_test.go +++ b/internal/difftest/generator_test.go @@ -4,8 +4,10 @@ import ( "strings" "testing" + "github.com/aybavs/sql-query-engine/internal/catalog" "github.com/aybavs/sql-query-engine/internal/lexer" "github.com/aybavs/sql-query-engine/internal/parser" + "github.com/aybavs/sql-query-engine/internal/value" ) func TestGeneratedQueriesParse(t *testing.T) { @@ -81,3 +83,52 @@ func TestGeneratorCoversTheGrammar(t *testing.T) { } } } + +// twoTableFixture mirrors the differential fixture's shape so join generation +// has something to work with. +func twoTableFixture(t *testing.T) *catalog.Catalog { + t.Helper() + cat := catalog.New() + cat.Add(&catalog.Table{Name: "users", File: "users.csv", Columns: []catalog.Column{ + {Name: "id", Type: value.TInt}, + {Name: "name", Type: value.TText}, + {Name: "age", Type: value.TInt}, + }}) + cat.Add(&catalog.Table{Name: "orders", File: "orders.csv", Columns: []catalog.Column{ + {Name: "id", Type: value.TInt}, + {Name: "user_id", Type: value.TInt}, + }}) + return cat +} + +// The hash join is the most intricate operator in the engine, so the generator +// has to reach it. +func TestGeneratorProducesJoins(t *testing.T) { + g := NewGenerator(11, twoTableFixture(t)) + joins := 0 + for i := 0; i < 300; i++ { + if strings.Contains(g.Query(), " JOIN ") { + joins++ + } + } + if joins < 20 { + t.Fatalf("only %d/300 queries joined; the hash join would be barely covered", joins) + } +} + +func TestGeneratedJoinQueriesParse(t *testing.T) { + g := NewGenerator(12, twoTableFixture(t)) + for i := 0; i < 300; i++ { + q := g.Query() + if !strings.Contains(q, " JOIN ") { + continue + } + toks, err := lexer.Lex(q) + if err != nil { + t.Fatalf("join query does not lex: %q: %v", q, err) + } + if _, err := parser.New(toks).ParseSelect(); err != nil { + t.Fatalf("join query does not parse: %q: %v", q, err) + } + } +}