From 31a64fff65725e4152b5c6ab1358c13db2b538f4 Mon Sep 17 00:00:00 2001 From: Badr Ezzir Date: Tue, 7 Jul 2026 15:02:13 +0100 Subject: [PATCH 1/2] Deterministic code generation for platform - column discovery: keep the discovery SQL valid when stripping predicate-builder templates (Build("WHERE") -> WHERE 1=1, other builders dropped) and trim the wrapping ( ... ) of derived-table views before falsifying, so templated / aggregate views resolve their real column types from the DB instead of failing discovery and falling back to string/not-null. Applied in both the shape detector and discoverysql.PrepareDiscoverySQL (internal engine path). - gofmt generated Go (GenerateOutputCode + asset upload) so the EmbedFS method layout is stable across regenerations. - sort route entries by SourceURL in createPathFiles so paths.yaml is written in a deterministic order. Co-authored-by: Cursor --- internal/asset/files.go | 8 ++++++++ repository/codegen.go | 7 +++++++ repository/path/service.go | 6 ++++++ repository/shape/column/detector.go | 25 ++++++++++++++++++++++-- repository/shape/column/strip_test.go | 8 ++++---- repository/shape/discoverysql/prepare.go | 24 ++++++++++++++++++++++- 6 files changed, 71 insertions(+), 7 deletions(-) diff --git a/internal/asset/files.go b/internal/asset/files.go index e6f1e0982..e279786c0 100644 --- a/internal/asset/files.go +++ b/internal/asset/files.go @@ -5,6 +5,7 @@ import ( "github.com/viant/afs" "github.com/viant/afs/file" "github.com/viant/afs/url" + "go/format" "strings" ) @@ -36,5 +37,12 @@ func (f Files) uploadContent(ctx context.Context, fs afs.Service, URL string, co if ok, _ := fs.Exists(ctx, baseURL); !ok { _ = fs.Create(ctx, baseURL, file.DefaultDirOsMode, true) } + // gofmt generated Go files so their layout is stable across regenerations. + // Non-Go files and not-yet-valid Go are written through unchanged. + if strings.HasSuffix(URL, ".go") { + if formatted, err := format.Source([]byte(content)); err == nil { + content = string(formatted) + } + } return fs.Upload(ctx, URL, file.DefaultFileOsMode, strings.NewReader(content)) } diff --git a/repository/codegen.go b/repository/codegen.go index 137ac8c34..9bf574c74 100644 --- a/repository/codegen.go +++ b/repository/codegen.go @@ -13,6 +13,7 @@ import ( "github.com/viant/tagly/format/text" "github.com/viant/toolbox/data" "github.com/viant/xreflect" + "go/format" "path" "reflect" "strconv" @@ -198,6 +199,12 @@ func (c *Component) GenerateOutputCode(ctx context.Context, withDefineComponent, result := builder.String() result = c.View.Resource().ReverseSubstitutes(result) + // Normalize the generated source with gofmt so appended snippets (e.g. the + // EmbedFS method) are properly indented and the file ends with a newline. + // Fall back to the unformatted source if it is not yet valid Go. + if formatted, formatErr := format.Source([]byte(result)); formatErr == nil { + result = string(formatted) + } return result } diff --git a/repository/path/service.go b/repository/path/service.go index 44118dbbd..ae7bd5e8c 100644 --- a/repository/path/service.go +++ b/repository/path/service.go @@ -13,6 +13,7 @@ import ( "github.com/viant/datly/repository/version" "gopkg.in/yaml.v3" "path" + "sort" "strings" "sync" "time" @@ -151,6 +152,11 @@ func (s *Service) createPathFiles(ctx context.Context) error { if err != nil { return err } + // Sort candidates by URL so paths.yaml is written in a deterministic + // order regardless of the underlying filesystem listing order. + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].URL() < candidates[j].URL() + }) rootPath := url.Path(s.URL) for _, candidate := range candidates { if candidate.IsDir() { diff --git a/repository/shape/column/detector.go b/repository/shape/column/detector.go index 72ea292f3..891fc4266 100644 --- a/repository/shape/column/detector.go +++ b/repository/shape/column/detector.go @@ -6,6 +6,7 @@ import ( "reflect" "strings" + "github.com/viant/datly/shared" "github.com/viant/datly/view" viewcolumn "github.com/viant/datly/view/column" "github.com/viant/datly/view/state" @@ -110,6 +111,9 @@ func discoverySQL(aView *view.View, resource *view.Resource) string { } return cleaned } + // A view whose SQL is a derived table is wrapped in ( ... ); the balanced + // pair confuses the falsifier's parser, so trim it before falsifying. + cleaned = strings.TrimSpace(shared.TrimPair(cleaned, '(', ')')) if falsified, ok := falsifyQuery(cleaned); ok { return falsified } @@ -473,6 +477,21 @@ func needsDiscovery(aView *view.View) bool { return usesWildcard(aView) } +// discoveryReplacementFor keeps a discovery query valid once a ${...} template +// is removed. A predicate builder that opens a WHERE clause is replaced with a +// tautology so any trailing AND still binds; any other builder is dropped; a +// plain value expression becomes an empty string literal. +func discoveryReplacementFor(expr string) string { + if !strings.Contains(expr, ".Build(") { + return "''" + } + lower := strings.ToLower(expr) + if strings.Contains(lower, `build("where")`) || strings.Contains(lower, `build('where')`) { + return " WHERE 1 = 1 " + } + return "" +} + // stripTemplateVariables removes velocity/velty template constructs from SQL // so it can be parsed and executed for column discovery. // Handles: $variable, ${expression}, #if...#end, #foreach...#end, #set(...) @@ -508,8 +527,10 @@ func stripTemplateVariables(sql string) string { } j++ } - // Replace with empty string or placeholder - b.WriteString("''") + // Predicate builders emit SQL clauses (e.g. Build("WHERE")); replace + // them with a valid stand-in so the discovery query stays parseable. + // Plain value expressions fall back to an empty string literal. + b.WriteString(discoveryReplacementFor(sql[i+2 : j-1])) i = j continue } diff --git a/repository/shape/column/strip_test.go b/repository/shape/column/strip_test.go index df847559e..ebd03c0eb 100644 --- a/repository/shape/column/strip_test.go +++ b/repository/shape/column/strip_test.go @@ -44,7 +44,7 @@ func TestStripTemplateVariables(t *testing.T) { { name: "expression in braces", input: "SELECT * FROM VENDOR WHERE ${predicate.Build(\"AND\")}", - expect: "SELECT * FROM VENDOR WHERE ''", + expect: "SELECT * FROM VENDOR WHERE ", }, { name: "if directive", @@ -64,7 +64,7 @@ func TestStripTemplateVariables(t *testing.T) { { name: "mixed templates and SQL", input: "SELECT vendor.*, products.* FROM (SELECT * FROM VENDOR t) vendor JOIN (SELECT * FROM PRODUCT t WHERE 1=1 ${predicate.Builder().CombineOr($predicate.FilterGroup(0, \"AND\")).Build(\"AND\")}) products ON products.VENDOR_ID = vendor.ID", - expect: "SELECT vendor.*, products.* FROM (SELECT * FROM VENDOR t) vendor JOIN (SELECT * FROM PRODUCT t WHERE 1=1 '') products ON products.VENDOR_ID = vendor.ID", + expect: "SELECT vendor.*, products.* FROM (SELECT * FROM VENDOR t) vendor JOIN (SELECT * FROM PRODUCT t WHERE 1=1 ) products ON products.VENDOR_ID = vendor.ID", }, { name: "UNION ALL with templates", @@ -94,7 +94,7 @@ func TestStripTemplateVariables(t *testing.T) { { name: "complex predicate builder", input: "WHERE ${predicate.Builder().CombineOr($predicate.FilterGroup(0, \"AND\")).Build(\"AND\")}", - expect: "WHERE ''", + expect: "WHERE ", }, { name: "dollar at end", @@ -156,7 +156,7 @@ perf AS ( FROM fact_performance p JOIN params prm ON TRUE WHERE p.event_date BETWEEN prm.start_date AND prm.end_date - '' + GROUP BY 1 ) SELECT v.* FROM perf v ORDER BY v.agency_id`, diff --git a/repository/shape/discoverysql/prepare.go b/repository/shape/discoverysql/prepare.go index b89d6f97f..868e74eda 100644 --- a/repository/shape/discoverysql/prepare.go +++ b/repository/shape/discoverysql/prepare.go @@ -3,6 +3,7 @@ package discoverysql import ( "strings" + "github.com/viant/datly/shared" "github.com/viant/sqlparser" "github.com/viant/sqlparser/expr" "github.com/viant/sqlparser/node" @@ -22,12 +23,30 @@ func PrepareDiscoverySQL(sql string) (string, bool) { if cleaned == "" || !strings.Contains(strings.ToLower(cleaned), "select") { return cleaned, false } + // A view whose SQL is a derived table is wrapped in ( ... ); the balanced + // pair confuses the falsifier's parser, so trim it before falsifying. + cleaned = strings.TrimSpace(shared.TrimPair(cleaned, '(', ')')) if rewritten, ok := falsifyQuery(cleaned); ok { return rewritten, true } return falsifyQueryText(cleaned) } +// discoveryReplacementFor keeps a discovery query valid once a ${...} template +// is removed. A predicate builder that opens a WHERE clause is replaced with a +// tautology so any trailing AND still binds; any other builder is dropped; a +// plain value expression becomes an empty string literal. +func discoveryReplacementFor(expr string) string { + if !strings.Contains(expr, ".Build(") { + return "''" + } + lower := strings.ToLower(expr) + if strings.Contains(lower, `build("where")`) || strings.Contains(lower, `build('where')`) { + return " WHERE 1 = 1 " + } + return "" +} + func falsifyQuery(sql string) (string, bool) { sql = strings.TrimSpace(sql) if sql == "" { @@ -573,7 +592,10 @@ func stripTemplateVariables(sql string) string { } j++ } - b.WriteString("''") + // Predicate builders emit SQL clauses (e.g. Build("WHERE")); replace + // them with a valid stand-in so the discovery query stays parseable. + // Plain value expressions fall back to an empty string literal. + b.WriteString(discoveryReplacementFor(sql[i+2 : j-1])) i = j continue } From 3675fa4a4171b43b9c03d273d7f480a00648e7dc Mon Sep 17 00:00:00 2001 From: Badr Ezzir Date: Mon, 13 Jul 2026 12:16:15 +0100 Subject: [PATCH 2/2] ENG-00000: Fix: ensure valid discovery SQL for unterminated `${` expressions in shape detector and PrepareDiscoverySQL --- repository/shape/column/detector.go | 7 ++++++- repository/shape/discoverysql/prepare.go | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/repository/shape/column/detector.go b/repository/shape/column/detector.go index 891fc4266..351df5546 100644 --- a/repository/shape/column/detector.go +++ b/repository/shape/column/detector.go @@ -530,7 +530,12 @@ func stripTemplateVariables(sql string) string { // Predicate builders emit SQL clauses (e.g. Build("WHERE")); replace // them with a valid stand-in so the discovery query stays parseable. // Plain value expressions fall back to an empty string literal. - b.WriteString(discoveryReplacementFor(sql[i+2 : j-1])) + exprEnd := j - 1 + if exprEnd < i+2 { + // Unterminated "${" at end of input: no expression body. + exprEnd = i + 2 + } + b.WriteString(discoveryReplacementFor(sql[i+2 : exprEnd])) i = j continue } diff --git a/repository/shape/discoverysql/prepare.go b/repository/shape/discoverysql/prepare.go index 868e74eda..ac66f3415 100644 --- a/repository/shape/discoverysql/prepare.go +++ b/repository/shape/discoverysql/prepare.go @@ -595,7 +595,12 @@ func stripTemplateVariables(sql string) string { // Predicate builders emit SQL clauses (e.g. Build("WHERE")); replace // them with a valid stand-in so the discovery query stays parseable. // Plain value expressions fall back to an empty string literal. - b.WriteString(discoveryReplacementFor(sql[i+2 : j-1])) + exprEnd := j - 1 + if exprEnd < i+2 { + // Unterminated "${" at end of input: no expression body. + exprEnd = i + 2 + } + b.WriteString(discoveryReplacementFor(sql[i+2 : exprEnd])) i = j continue }