test: compare generated queries against SQLite

internal/difftest/differential_test.go

@@ -0,0 +1,103 @@
+package difftest
+
+import (
+	"flag"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/aybavs/sql-query-engine/internal/catalog"
+	"github.com/aybavs/sql-query-engine/internal/value"
+)
+
+var (
+	seed    = flag.Int64("seed", 20260804, "seed for the query generator")
+	queries = flag.Int("queries", 300, "number of generated queries to compare")
+)
+
+// diffFixture is deliberately small but full of NULLs, duplicate values, and
+// ties, since those are where two engines are most likely to disagree.
+func diffFixture(t *testing.T) (*catalog.Catalog, string) {
+	t.Helper()
+	dir := t.TempDir()
+	os.WriteFile(filepath.Join(dir, "users.csv"), []byte(
+		"1,alice,30,berlin\n"+
+			"2,bob,,berlin\n"+
+			"3,carol,40,\n"+
+			"4,dan,25,london\n"+
+			"5,erin,25,paris\n"+
+			"6,frank,,\n",
+	), 0o644)
+	os.WriteFile(filepath.Join(dir, "orders.csv"), []byte(
+		"10,1,100\n11,1,300\n12,2,50\n13,,75\n14,5,50\n",
+	), 0o644)
+
+	cat := catalog.New()
+	cat.Add(&catalog.Table{Name: "users", File: "users.csv", Columns: []catalog.Column{
+		{Name: "id", Type: value.TInt},
+		{Name: "name", Type: value.TText},
+		{Name: "age", Type: value.TInt},
+		{Name: "city", Type: value.TText},
+	}})
+	cat.Add(&catalog.Table{Name: "orders", File: "orders.csv", Columns: []catalog.Column{
+		{Name: "id", Type: value.TInt},
+		{Name: "user_id", Type: value.TInt},
+		{Name: "total", Type: value.TInt},
+	}})
+	return cat, dir
+}
+
+// TestDifferentialAgainstSQLite runs generated queries through this engine and
+// through SQLite over identical data, and requires the results to agree.
+func TestDifferentialAgainstSQLite(t *testing.T) {
+	cat, dir := diffFixture(t)
+
+	oracle, err := NewOracle(cat, dir)
+	if err != nil {
+		t.Fatalf("oracle: %v", err)
+	}
+	defer oracle.Close()
+
+	g := NewGenerator(*seed, cat)
+	compared, joined := 0, 0
+	for i := 0; i < *queries; i++ {
+		q := g.Query()
+		if strings.Contains(q, " JOIN ") {
+			joined++
+		}
+
+		engineRows, engErr := runEngine(cat, dir, q)
+		oracleRows, orcErr := oracle.Query(q)
+
+		// Both rejecting a query is uninteresting; disagreeing about whether a
+		// query is even valid is a finding worth reporting.
+		switch {
+		case engErr != nil && orcErr != nil:
+			continue
+		case engErr != nil:
+			t.Fatalf("seed %d query %d: engine rejected a query SQLite accepted\n  %s\n  %v", *seed, i, q, engErr)
+		case orcErr != nil:
+			t.Fatalf("seed %d query %d: SQLite rejected a query the engine accepted\n  %s\n  %v", *seed, i, q, orcErr)
+		}
+
+		normalized := make([][]string, len(oracleRows))
+		for j, r := range oracleRows {
+			normalized[j] = normalizeOracleRow(r)
+		}
+		if err := Compare(engineRows, normalized); err != nil {
+			t.Fatalf("seed %d query %d disagrees:\n  %s\n%v", *seed, i, q, err)
+		}
+		compared++
+	}
+
+	if compared < *queries/2 {
+		t.Fatalf("only %d/%d generated queries were actually compared; the generator is producing too many invalid queries", compared, *queries)
+	}
+	// The hash join is the engine's most intricate operator, so it has to be a
+	// real share of what the oracle checks, not an occasional accident.
+	if joined < *queries/20 {
+		t.Fatalf("only %d/%d generated queries joined; the hash join is barely covered", joined, *queries)
+	}
+	t.Logf("compared %d generated queries against SQLite (%d of them joins, seed %d)", compared, joined, *seed)
+}

internal/difftest/engine.go

@@ -0,0 +1,34 @@
+package difftest
+
+import (
+	"github.com/aybavs/sql-query-engine/internal/catalog"
+	"github.com/aybavs/sql-query-engine/internal/lexer"
+	"github.com/aybavs/sql-query-engine/internal/parser"
+	"github.com/aybavs/sql-query-engine/internal/plan"
+)
+
+// runEngine executes a query through this engine — lexer, parser, planner, then
+// the operator tree — and returns rows normalized for comparison.
+func runEngine(cat *catalog.Catalog, dataDir, query string) ([][]string, error) {
+	toks, err := lexer.Lex(query)
+	if err != nil {
+		return nil, err
+	}
+	st, err := parser.New(toks).ParseSelect()
+	if err != nil {
+		return nil, err
+	}
+	op, _, err := plan.Build(st, cat, dataDir)
+	if err != nil {
+		return nil, err
+	}
+
+	var out [][]string
+	for {
+		row, ok := op.Next()
+		if !ok {
+			return out, nil
+		}
+		out = append(out, normalizeEngineRow(row))
+	}
+}

internal/difftest/generator.go

@@ -25,26 +25,84 @@ func NewGenerator(seed int64, cat *catalog.Catalog) *Generator {
 	}
 }
 
+// qualified is a column together with the table it came from, so a query over
+// several tables can always write an unambiguous reference.
+type qualified struct {
+	table  *catalog.Table
+	column catalog.Column
+}
+
+func (q qualified) ref() string { return q.table.Name + "." + q.column.Name }
+
 func (g *Generator) Query() string {
-	t := g.tables[g.rnd.Intn(len(g.tables))]
+	// A join exercises the hash join, which is the most intricate operator in
+	// the engine and therefore the one most worth checking against an oracle.
+	if len(g.tables) >= 2 && g.chance(0.3) {
+		if q, ok := g.joinQuery(); ok {
+			return q
+		}
+	}
+
+	scope := []*catalog.Table{g.tables[g.rnd.Intn(len(g.tables))]}
 	if g.chance(0.35) {
-		return g.aggregateQuery(t)
+		return g.aggregateQuery(scope, scope[0].Name)
+	}
+	return g.selectQuery(scope, scope[0].Name)
+}
+
+// joinQuery joins two tables on a pair of same-type columns. The pairing need
+// not be semantically meaningful — an arbitrary equi-join still exercises
+// build/probe, duplicate keys, and NULL keys, which is what is being checked.
+func (g *Generator) joinQuery() (string, bool) {
+	i := g.rnd.Intn(len(g.tables))
+	j := g.rnd.Intn(len(g.tables) - 1)
+	if j >= i {
+		j++
+	}
+	left, right := g.tables[i], g.tables[j]
+
+	lc, rc, ok := g.joinKeys(left, right)
+	if !ok {
+		return "", false
+	}
+
+	scope := []*catalog.Table{left, right}
+	from := fmt.Sprintf("%s JOIN %s ON %s = %s", left.Name, right.Name, lc.ref(), rc.ref())
+
+	if g.chance(0.3) {
+		return g.aggregateQuery(scope, from), true
 	}
-	return g.selectQuery(t)
+	return g.selectQuery(scope, from), true
 }
 
-func (g *Generator) selectQuery(t *catalog.Table) string {
+// joinKeys picks one column from each table that share a type.
+func (g *Generator) joinKeys(left, right *catalog.Table) (qualified, qualified, bool) {
+	var pairs [][2]qualified
+	for _, lc := range g.usableColumns(left) {
+		for _, rc := range g.usableColumns(right) {
+			if lc.Type == rc.Type {
+				pairs = append(pairs, [2]qualified{{left, lc}, {right, rc}})
+			}
+		}
+	}
+	if len(pairs) == 0 {
+		return qualified{}, qualified{}, false
+	}
+	p := pairs[g.rnd.Intn(len(pairs))]
+	return p[0], p[1], true
+}
+
+func (g *Generator) selectQuery(scope []*catalog.Table, from string) string {
 	var b strings.Builder
 	b.WriteString("SELECT ")
-	b.WriteString(strings.Join(g.projections(t), ", "))
-	fmt.Fprintf(&b, " FROM %s", t.Name)
+	b.WriteString(strings.Join(g.projections(scope), ", "))
+	fmt.Fprintf(&b, " FROM %s", from)
 
 	if g.chance(0.7) {
-		fmt.Fprintf(&b, " WHERE %s", g.predicate(t))
+		fmt.Fprintf(&b, " WHERE %s", g.predicate(scope))
 	}
 	if g.chance(0.5) {
-		c := g.column(t)
-		fmt.Fprintf(&b, " ORDER BY %s.%s", t.Name, c.Name)
+		fmt.Fprintf(&b, " ORDER BY %s", g.column(scope).ref())
 		if g.chance(0.5) {
 			b.WriteString(" DESC")
 		}
@@ -57,74 +115,71 @@ func (g *Generator) selectQuery(t *catalog.Table) string {
 
 // aggregateQuery always groups by the column it projects, so the result is
 // well-defined rather than relying on SQLite's bare-column extension.
-func (g *Generator) aggregateQuery(t *catalog.Table) string {
-	group := g.column(t)
-	agg := g.aggregate(t)
+func (g *Generator) aggregateQuery(scope []*catalog.Table, from string) string {
+	group := g.column(scope)
 
 	var b strings.Builder
-	fmt.Fprintf(&b, "SELECT %s.%s, %s FROM %s GROUP BY %s.%s",
-		t.Name, group.Name, agg, t.Name, t.Name, group.Name)
+	fmt.Fprintf(&b, "SELECT %s, %s FROM %s GROUP BY %s",
+		group.ref(), g.aggregate(scope), from, group.ref())
 	if g.chance(0.3) {
 		fmt.Fprintf(&b, " HAVING COUNT(*) > %d", g.rnd.Intn(2))
 	}
 	return b.String()
 }
 
-func (g *Generator) aggregate(t *catalog.Table) string {
+func (g *Generator) aggregate(scope []*catalog.Table) string {
 	if g.chance(0.25) {
 		return "COUNT(*)"
 	}
-	numeric := g.numericColumns(t)
+	numeric := g.numericColumns(scope)
 	if len(numeric) == 0 {
 		return "COUNT(*)"
 	}
 	c := numeric[g.rnd.Intn(len(numeric))]
 	fn := []string{"COUNT", "SUM", "AVG", "MIN", "MAX"}[g.rnd.Intn(5)]
-	return fmt.Sprintf("%s(%s.%s)", fn, t.Name, c.Name)
+	return fmt.Sprintf("%s(%s)", fn, c.ref())
 }
 
-func (g *Generator) projections(t *catalog.Table) []string {
+func (g *Generator) projections(scope []*catalog.Table) []string {
 	if g.chance(0.2) {
 		return []string{"*"}
 	}
 	n := 1 + g.rnd.Intn(2)
 	out := make([]string, n)
 	for i := range out {
-		c := g.column(t)
-		out[i] = fmt.Sprintf("%s.%s", t.Name, c.Name)
+		out[i] = g.column(scope).ref()
 	}
 	return out
 }
 
 // predicate builds a boolean expression whose operands always share a type.
-func (g *Generator) predicate(t *catalog.Table) string {
-	p := g.comparison(t)
+func (g *Generator) predicate(scope []*catalog.Table) string {
+	p := g.comparison(scope)
 	for g.chance(0.35) {
 		op := "AND"
 		if g.chance(0.5) {
 			op = "OR"
 		}
-		p = fmt.Sprintf("%s %s %s", p, op, g.comparison(t))
+		p = fmt.Sprintf("%s %s %s", p, op, g.comparison(scope))
 	}
 	if g.chance(0.15) {
 		p = "NOT (" + p + ")"
 	}
 	return p
 }
 
-func (g *Generator) comparison(t *catalog.Table) string {
-	c := g.column(t)
-	ref := fmt.Sprintf("%s.%s", t.Name, c.Name)
+func (g *Generator) comparison(scope []*catalog.Table) string {
+	c := g.column(scope)
 
 	if g.chance(0.2) {
 		if g.chance(0.5) {
-			return ref + " IS NULL"
+			return c.ref() + " IS NULL"
 		}
-		return ref + " IS NOT NULL"
+		return c.ref() + " IS NOT NULL"
 	}
 
 	op := []string{"=", "<>", "<", "<=", ">", ">="}[g.rnd.Intn(6)]
-	return fmt.Sprintf("%s %s %s", ref, op, g.literal(c))
+	return fmt.Sprintf("%s %s %s", c.ref(), op, g.literal(c.column))
 }
 
 // literal produces a value of the column's own type: comparing across types is
@@ -141,26 +196,38 @@ func (g *Generator) literal(c catalog.Column) string {
 	}
 }
 
-// column returns a column of a type the generator supports. Booleans are
-// excluded because SQLite stores them as integers.
-func (g *Generator) column(t *catalog.Table) catalog.Column {
-	usable := make([]catalog.Column, 0, len(t.Columns))
-	for _, c := range t.Columns {
-		if c.Type != value.TBool {
-			usable = append(usable, c)
+// column picks a usable column from anywhere in scope.
+func (g *Generator) column(scope []*catalog.Table) qualified {
+	var all []qualified
+	for _, t := range scope {
+		for _, c := range g.usableColumns(t) {
+			all = append(all, qualified{t, c})
 		}
 	}
-	return usable[g.rnd.Intn(len(usable))]
+	return all[g.rnd.Intn(len(all))]
 }
 
-func (g *Generator) numericColumns(t *catalog.Table) []catalog.Column {
-	var out []catalog.Column
+// usableColumns excludes booleans, which SQLite stores as integers.
+func (g *Generator) usableColumns(t *catalog.Table) []catalog.Column {
+	out := make([]catalog.Column, 0, len(t.Columns))
 	for _, c := range t.Columns {
-		if c.Type == value.TInt || c.Type == value.TFloat {
+		if c.Type != value.TBool {
 			out = append(out, c)
 		}
 	}
 	return out
 }
 
+func (g *Generator) numericColumns(scope []*catalog.Table) []qualified {
+	var out []qualified
+	for _, t := range scope {
+		for _, c := range t.Columns {
+			if c.Type == value.TInt || c.Type == value.TFloat {
+				out = append(out, qualified{t, c})
+			}
+		}
+	}
+	return out
+}
+
 func (g *Generator) chance(p float64) bool { return g.rnd.Float64() < p }