diff --git a/cmd/tfpfgen/blueprint_draft.go b/cmd/tfpfgen/blueprint_draft.go index bc3ebc67..1ad97b1f 100644 --- a/cmd/tfpfgen/blueprint_draft.go +++ b/cmd/tfpfgen/blueprint_draft.go @@ -18,7 +18,7 @@ import ( ) const usageBlueprintDraft = "blueprint draft [-openapi-dir DIR] [-snapshot NAME] [-tag TAG] " + - "[-sdk-dialect restyService|kiotaFluent] [-out DIR] [-dry-run]" + "[-sdk-dialect restyService|kiotaFluent] [-exclusions FILE] [-out DIR] [-dry-run]" func runBlueprintDraft(args []string) error { fs, _ := newFlagSet("blueprint draft", usageBlueprintDraft) @@ -44,6 +44,8 @@ func runBlueprintDraft(args []string) error { "binding shape to infer: restyService, or kiotaFluent for a kiota-generated SDK") sdkModels = fs.String("sdk-models-package", "", "import path of the kiota SDK's models package (required with -sdk-dialect kiotaFluent)") + exclusions = fs.String("exclusions", "", + "exclusions sidecar; defaults to /"+openapi.ExclusionsFileName+" when present") ) if err := parse(fs, args); err != nil { @@ -88,6 +90,15 @@ func runBlueprintDraft(args []string) error { log.Printf("specification: %s (%s %s)", path, doc.Title, doc.Version) + exclusionsPath := *exclusions + if exclusionsPath == "" { + exclusionsPath = filepath.Join(*openapiDir, openapi.ExclusionsFileName) + } + excluded, err := openapi.LoadExclusions(exclusionsPath) + if err != nil { + return err + } + candidates := filterCandidates(doc.Discover(), *tag, *includeUnusable) if len(candidates) == 0 { return fmt.Errorf("%w: no candidates matched", errNothingToDo) @@ -111,12 +122,13 @@ func runBlueprintDraft(args []string) error { SDKModelsImport: *sdkModels, } - return inferAll(doc, candidates, opts, *out, *scenarioDrafts) + return inferAll(doc, candidates, excluded, opts, *out, *scenarioDrafts) } func inferAll( doc *openapi.Document, candidates []openapi.Candidate, + excluded openapi.Exclusions, opts openapi.InferOptions, out string, planDrafts string, @@ -128,10 +140,39 @@ func inferAll( ) for _, c := range candidates { - if kind, why := c.Classify(); kind != openapi.CandidateKindResource { + // The sidecar speaks first: a curated exclusion is a decision already + // made, and the run repeats its reason as a named skip. + if reason, is := excluded.Match(c); is { + log.Printf("excluded %s: %s", c.Key, reason) + skipped++ + continue + } + + kind, why := c.Classify() + + if kind == openapi.CandidateKindDataSource { + ds, dsNotes, err := doc.InferDataSource(c, opts) + notes = append(notes, dsNotes...) + if err != nil { + log.Printf("skipped %s: %v", c.Key, err) + skipped++ + continue + } + bp := blueprint.Blueprint{FormatVersion: blueprint.FormatVersion, DataSources: []blueprint.DataSource{ds}} + path := filepath.Join(out, "datasources", ds.Key+blueprint.Ext) + if err := blueprint.Save(path, bp); err != nil { + return err + } + log.Printf("wrote %s (dataSource, %d attributes, %d selector(s))", + path, len(ds.Schema.Attributes), len(ds.Binding.Selectors)) + written++ + continue + } + + if kind != openapi.CandidateKindResource { // Said out loud rather than silently skipped: silence reads as agreement, - // and a data source or action the spec offers deserves at least a line - // saying inference does not reach it yet. + // and an action the spec offers deserves at least a line saying inference + // does not reach it yet. log.Printf("skipped %s: %s inference is not implemented (%s)", c.Key, kind, why) skipped++ continue diff --git a/docs/cli.md b/docs/cli.md index cdb28b6d..db1af94c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -214,7 +214,7 @@ Infers draft blueprints (and optionally scenario worksheets) from a pinned OpenAPI snapshot. ``` -tfpfgen blueprint draft [-openapi-dir DIR] [-snapshot NAME] [-tag TAG] [-sdk-dialect restyService|kiotaFluent] [-out DIR] [-dry-run] +tfpfgen blueprint draft [-openapi-dir DIR] [-snapshot NAME] [-tag TAG] [-sdk-dialect restyService|kiotaFluent] [-exclusions FILE] [-out DIR] [-dry-run] ``` | Flag | Default | Purpose | @@ -233,6 +233,7 @@ tfpfgen blueprint draft [-openapi-dir DIR] [-snapshot NAME] [-tag TAG] [-sdk-dia | `-scenario-drafts` | — | also scaffold a `KEY.scenario.draft.json` scenario worksheet per resource under this directory | | `-sdk-dialect` | `restyService` | binding shape to infer: `restyService`, or `kiotaFluent` for a kiota-generated SDK | | `-sdk-models-package` | — | import path of the kiota SDK's models package (required with `-sdk-dialect kiotaFluent`; the resty `-sdk-service-root`/`-sdk-accessor` knobs are refused under it) | +| `-exclusions` | `/draft-exclusions.json` | curated sidecar of families drafting must skip, each entry carrying its reason; the run repeats every exclusion as a named skip | `blueprint draft -dry-run` is the survey: it reports every candidate the document offers and why the ineligible ones are ineligible. The write path diff --git a/internal/blueprint/binding.go b/internal/blueprint/binding.go index d3b7d648..dd8e9815 100644 --- a/internal/blueprint/binding.go +++ b/internal/blueprint/binding.go @@ -33,14 +33,54 @@ type ResourceBinding struct { type DataSourceBinding struct { Service ServiceRef `json:"service"` - // Read is the only operation. It is a pointer for symmetry with ResourceBinding's + // Read is the direct fetch. It is a pointer for symmetry with ResourceBinding's // operations and so that "not yet authored" is distinguishable from an empty call, - // which is what an imported draft needs; Validate requires it. + // which is what an imported draft needs; Validate requires it unless the binding + // is list-resolved with no direct fetch at all. Read *Operation `json:"read,omitempty"` + // List is the resolver: when the practitioner selects by anything other than + // the direct identifier, the list is fetched and the selectors narrow it to + // exactly one element -- a lookup must be predictable, so zero matches and + // several matches are both errors, never a guess. With Read present the + // matched element only supplies the identifier and the direct fetch runs as + // usual, so state always maps from one response shape; without Read the + // matched element itself is that shape. + List *Operation `json:"list,omitempty"` + // CollectionField reaches the elements inside the list response, e.g. + // "GetTags()" under method access. + CollectionField string `json:"collectionField,omitempty"` + // ElementType is the list element's Go type, e.g. "models.Tags_API_Tagable". + ElementType string `json:"elementType,omitempty"` + + // Selectors are the configuration attributes a practitioner may look up by. + // Exactly one must be set per read, which the generated code enforces. + Selectors []Selector `json:"selectors,omitempty"` + + // ElementIDField reads the direct identifier off a matched element -- the + // accessor base, e.g. "Id" -- and ElementIDFlatten converts it into the + // identifier attribute, after which the direct Read runs exactly as if the + // practitioner had supplied the id. Required when both List and Read exist. + ElementIDField string `json:"elementIdField,omitempty"` + ElementIDFlatten *ConvertCall `json:"elementIdFlatten,omitempty"` + Response ResponseModel `json:"response"` } +// Selector is one attribute a data source can be looked up by. +type Selector struct { + // Attribute is the schema attribute acting as the selector. + Attribute string `json:"attribute"` + // GoField is the model field the attribute lives in. + GoField string `json:"goField"` + // SDKField is the accessor base on the list element the selector matches + // against, e.g. "TestName". Empty when ViaRead. + SDKField string `json:"sdkField,omitempty"` + // ViaRead marks the direct identifier: no list resolution, the value feeds + // the Read chain as-is. + ViaRead bool `json:"viaRead,omitempty"` +} + // EphemeralBinding wires an ephemeral resource to the SDK. // // Open is the one operation this toolkit renders: it runs when Terraform needs the value, diff --git a/internal/blueprint/validate.go b/internal/blueprint/validate.go index f2832b3c..9bc552a5 100644 --- a/internal/blueprint/validate.go +++ b/internal/blueprint/validate.go @@ -426,6 +426,25 @@ func (d DataSource) validate(at string, p *problems) { dup(p, seenNames, a.Name, aat+".name", "attribute name") dup(p, seenFields, a.GoField, aat+".goField", "model field") } + + // A selector is configuration: it must name a declared attribute the + // practitioner can actually set. + for i, s := range d.Binding.Selectors { + sat := fmt.Sprintf("%s.binding.selectors[%d]", at, i) + found := false + for _, a := range d.Schema.Attributes { + if a.Name != s.Attribute { + continue + } + found = true + if a.ComputedOptionalRequired == Computed { + p.add(sat+".attribute", "%q is computed, which a practitioner cannot set", s.Attribute) + } + } + if !found && s.Attribute != "" { + p.add(sat+".attribute", "names attribute %q, which the data source does not declare", s.Attribute) + } + } } func dup(p *problems, seen map[string]bool, value, path, what string) { @@ -1548,12 +1567,53 @@ func (b DataSourceBinding) validate(at string, p *problems) { p.add(at+".response.accessStyle", "%q is not a known access style", b.Response.AccessStyle) } - if b.Read == nil { + if b.Read == nil && b.List == nil { p.add(at+".read", "is required: a data source with no read operation has nothing to do") return } + if b.Read != nil { + b.Read.validate(at+".read", p) + } - b.Read.validate(at+".read", p) + // The list-resolver contract: selectors narrow the list to exactly one + // element, so a lookup stays predictable. Whatever needs the list must + // bring the list, and the pieces that reach inside it. + needsList := b.Read == nil + for _, s := range b.Selectors { + if !s.ViaRead { + needsList = true + } + } + if needsList && b.List == nil { + p.add(at+".list", "is required: a selector that is not viaRead resolves through the list") + } + if b.List != nil { + b.List.validate(at+".list", p) + required(p, at+".collectionField", b.CollectionField) + required(p, at+".elementType", b.ElementType) + if len(b.Selectors) == 0 && b.Read != nil { + p.add(at+".selectors", "a list with a direct read wants selectors; without any, the list is never consulted") + } + if b.Read != nil { + // A matched element only supplies the identifier; the direct read + // does the fetching. That handoff needs the id spelled out. + required(p, at+".elementIdField", b.ElementIDField) + if b.ElementIDFlatten == nil { + p.add(at+".elementIdFlatten", "is required with both list and read: it converts the matched element's identifier into the id attribute") + } + } + } + for i, s := range b.Selectors { + sat := fmt.Sprintf("%s.selectors[%d]", at, i) + required(p, sat+".attribute", s.Attribute) + required(p, sat+".goField", s.GoField) + if s.ViaRead && b.Read == nil { + p.add(sat+".viaRead", "names the direct read, which this binding does not declare") + } + if !s.ViaRead && s.SDKField == "" { + p.add(sat+".sdkField", "is required: a list-resolved selector names the element field it matches") + } + } } func (o Operation) validate(at string, p *problems) { diff --git a/internal/generate/datasource.go b/internal/generate/datasource.go index 65290e95..a26bcb06 100644 --- a/internal/generate/datasource.go +++ b/internal/generate/datasource.go @@ -2,6 +2,7 @@ package generate import ( "fmt" + "strings" "github.com/deploymenttheory/terraform-plugin-framework-codegen/internal/blueprint" ) @@ -45,6 +46,56 @@ type DataSourceView struct { // Read is the SDK call, and State is the flatten function it feeds. Read *OpView State StateView + + // Resolve is the selector machinery: the list call that narrows a lookup to + // exactly one element. Nil for a plain single-operation data source. + Resolve *ResolveView +} + +// ResolveView renders the list-then-match resolver. +// +// The contract it renders: exactly one selector must be set; anything other +// than the direct identifier fetches the list and filters it, and zero or +// several matches are both errors -- a lookup must be predictable, never a +// guess. With a direct read present the matched element only supplies the +// identifier; without one the matched element is the state source itself. +type ResolveView struct { + // IDGoField is the identifier attribute's model field, e.g. "ID". Empty in + // MapsElement mode, which has no direct read to feed. + IDGoField string + + // AllSelectorGoFields drive the exactly-one count; SelectorList names + // the attributes in the diagnostic, comma-joined. + AllSelectorGoFields []string + SelectorList string + + // Matchers are the non-identifier selectors: each compares a configured + // attribute against a getter on the list element. + Matchers []MatcherView + + // List is the list call, bound to the "listing" variable. + List *OpView + // CollectionField reaches the elements, e.g. "GetTags()". + CollectionField string + // ElementType is the element's Go type, for the matches slice. + ElementType string + + // ElementIDExpr converts the matched element's identifier into the id + // attribute, e.g. `convert.PtrStringToFramework(match.GetId())`. + ElementIDExpr string + // MapsElement is the no-direct-read mode: the matched element maps to + // state directly and the function returns inside the resolver. + MapsElement bool +} + +// MatcherView is one selector comparison. +type MatcherView struct { + // GoField is the configured attribute's model field. + GoField string + // Getter is the element access expression, e.g. `el.GetTestName()`. + Getter string + // AttrName names the attribute in diagnostics. + AttrName string } // DataSourceImports holds the rendered import block for each emitted file. @@ -164,21 +215,37 @@ func DataSource( impState.add(pkgTypes, "") } - if d.Binding.Read == nil { + if d.Binding.Read == nil && d.Binding.List == nil { return DataSourceView{}, &ErrUnsupported{ What: sc.what, Why: "a data source with no read operation has nothing to generate", } } - read, err := opView( - sc.what, d.Binding.Service.Accessor, - *d.Binding.Read, "crud.PhaseRead", "errors.OpRead", "ReadTimeout", bindsResult, - ) - if err != nil { - return DataSourceView{}, err + if d.Binding.Read != nil { + read, err := opView( + sc.what, d.Binding.Service.Accessor, + *d.Binding.Read, "crud.PhaseRead", "errors.OpRead", "ReadTimeout", bindsResult, + ) + if err != nil { + return DataSourceView{}, err + } + v.Read = read + } + + if d.Binding.List != nil { + resolve, err := resolveView(sc.what, d) + if err != nil { + return DataSourceView{}, err + } + v.Resolve = resolve + // The resolver's diagnostics count matches and quote selectors. + impRead.add("fmt", "") + if resolve.ElementIDExpr != "" { + impRead.add(sup.Convert.Path, sup.Convert.Alias) + } + impRead.add(d.Binding.Service.ImportPath, d.Binding.Service.Alias) } - v.Read = read org := bp.Provider.GoModule v.Imports = DataSourceImports{ @@ -190,6 +257,74 @@ func DataSource( return v, nil } +// resolveView builds the selector resolver from the binding. +func resolveView(what string, d blueprint.DataSource) (*ResolveView, error) { + b := d.Binding + + list, err := opView( + what, b.Service.Accessor, + *b.List, "crud.PhaseRead", "errors.OpRead", "ReadTimeout", bindsResult, + ) + if err != nil { + return nil, err + } + // The direct read owns "remote"; the resolver's fetch is the listing. + list.ResultVar = "listing" + list.Assign = "listing, err :=" + + v := &ResolveView{ + List: list, + CollectionField: b.CollectionField, + ElementType: b.ElementType, + MapsElement: b.Read == nil, + } + + var names []string + + goFieldOf := func(attr string) string { + for _, a := range d.Schema.Attributes { + if a.Name == attr { + return a.GoField + } + } + return "" + } + + for _, s := range b.Selectors { + goField := s.GoField + if goField == "" { + goField = goFieldOf(s.Attribute) + } + v.AllSelectorGoFields = append(v.AllSelectorGoFields, goField) + names = append(names, s.Attribute) + + if s.ViaRead { + v.IDGoField = goField + continue + } + v.Matchers = append(v.Matchers, MatcherView{ + GoField: goField, + Getter: readExpr(b.Response.AccessStyle, "el", s.SDKField), + AttrName: s.Attribute, + }) + } + + v.SelectorList = strings.Join(names, ", ") + + if !v.MapsElement { + if v.IDGoField == "" { + return nil, &ErrUnsupported{ + What: what, + Why: "a list-resolved data source with a direct read declares no viaRead selector, so the resolver has nowhere to put the identifier", + } + } + v.ElementIDExpr = convertExpr(*b.ElementIDFlatten, + readExpr(b.Response.AccessStyle, "match", b.ElementIDField)) + } + + return v, nil +} + // dataSourceInterfaces are the framework interfaces a generated data source asserts. // // Every data source implements both, so unlike the resource equivalent there is nothing diff --git a/internal/generate/datasource_resolver_test.go b/internal/generate/datasource_resolver_test.go new file mode 100644 index 00000000..bf479cd2 --- /dev/null +++ b/internal/generate/datasource_resolver_test.go @@ -0,0 +1,155 @@ +package generate + +import ( + "strings" + "testing" + + "github.com/deploymenttheory/terraform-plugin-framework-codegen/internal/blueprint" +) + +// selectorBlueprint is a fluent, selector-shaped data source: looked up by +// exactly one of id (direct read) or test_name (list-resolved). +func selectorBlueprint() (blueprint.Blueprint, blueprint.DataSource) { + ds := blueprint.DataSource{ + Key: "test", Name: "test", GoPackage: "test", + GoPackageAlias: "testv7", GoTypeName: "TestDataSource", + ModelTypeName: "TestDataSourceModel", + Binding: blueprint.DataSourceBinding{ + Service: blueprint.ServiceRef{ + ImportPath: "example.com/prov/internal/sdk/models", + Alias: "models", + Accessor: "d.client", + }, + Read: &blueprint.Operation{ + Style: blueprint.CallStyleFluent, + Chain: []blueprint.ChainSegment{ + {Method: "Tests"}, + {Method: "ByTestId", Args: []blueprint.Argument{{Kind: blueprint.ArgConfigField, Field: "ID"}}}, + {Method: "Get", Args: []blueprint.Argument{{Kind: blueprint.ArgContext}, {Kind: blueprint.ArgLiteral, Expr: "nil"}}}, + }, + Return: blueprint.ReturnResultError, ResultType: "models.Testable", + }, + List: &blueprint.Operation{ + Style: blueprint.CallStyleFluent, + Chain: []blueprint.ChainSegment{ + {Method: "Tests"}, + {Method: "Get", Args: []blueprint.Argument{{Kind: blueprint.ArgContext}, {Kind: blueprint.ArgLiteral, Expr: "nil"}}}, + }, + Return: blueprint.ReturnResultError, ResultType: "models.Testsable", + }, + CollectionField: "GetTests()", + ElementType: "models.SimpleTestable", + Selectors: []blueprint.Selector{ + {Attribute: "id", GoField: "ID", ViaRead: true}, + {Attribute: "test_name", GoField: "TestName", SDKField: "TestName"}, + }, + ElementIDField: "TestId", + ElementIDFlatten: &blueprint.ConvertCall{Func: "convert.PtrStringToFramework"}, + Response: blueprint.ResponseModel{ + Type: "models.Testable", AccessStyle: blueprint.AccessMethod, + }, + }, + } + idAttr := attr("id", blueprint.KindString, blueprint.ComputedOptional) + idAttr.GoField = "ID" + idAttr.Wire = blueprint.WireBinding{ + JSONPath: "testId", SDKField: "TestId", SDKGoType: "*string", + Flatten: &blueprint.ConvertCall{Func: "convert.PtrStringToFramework"}, + } + nameAttr := attr("test_name", blueprint.KindString, blueprint.ComputedOptional) + nameAttr.GoField = "TestName" + nameAttr.Wire = blueprint.WireBinding{ + JSONPath: "testName", SDKField: "TestName", SDKGoType: "*string", + Flatten: &blueprint.ConvertCall{Func: "convert.PtrStringToFramework"}, + } + ds.Schema.Attributes = []blueprint.Attribute{idAttr, nameAttr} + + bp := blueprint.Blueprint{ + FormatVersion: blueprint.FormatVersion, + Provider: blueprint.Provider{ + Name: "te", TypePrefix: "te", GoModule: "example.com/prov", + SDK: blueprint.SDKModule{ + ModulePath: "example.com/prov", ClientType: "*sdk.Client", + Dialect: blueprint.DialectKiotaFluent, + }, + }, + DataSources: []blueprint.DataSource{ds}, + } + return bp, ds +} + +// TestUnit_Generate_SelectorResolverView proves the resolver view carries the +// contract: exactly-one enforcement inputs, the matcher against the element, +// and the identifier handoff into the direct read. +func TestUnit_Generate_SelectorResolverView(t *testing.T) { + t.Parallel() + + bp, ds := selectorBlueprint() + + v, err := DataSource(bp, ds, Options{}) + if err != nil { + t.Fatalf("DataSource: %v", err) + } + if v.Resolve == nil { + t.Fatal("a binding with a list must produce a resolver") + } + + r := v.Resolve + if r.MapsElement { + t.Error("a binding with a direct read must not map the element") + } + if r.IDGoField != "ID" { + t.Errorf("IDGoField = %q", r.IDGoField) + } + if r.SelectorList != "id, test_name" { + t.Errorf("SelectorList = %q", r.SelectorList) + } + if len(r.Matchers) != 1 || r.Matchers[0].Getter != "el.GetTestName()" { + t.Errorf("matchers = %+v", r.Matchers) + } + if r.ElementIDExpr != "convert.PtrStringToFramework(match.GetTestId())" { + t.Errorf("ElementIDExpr = %q", r.ElementIDExpr) + } + if r.List.ResultVar != "listing" { + t.Errorf("the list call must bind listing, got %q", r.List.ResultVar) + } +} + +// TestUnit_Generate_SelectorResolverRenders proves the read template renders +// the whole contract: the exactly-one refusal, the list-and-match loop, the +// zero and many refusals, and the identifier handoff before the direct read. +func TestUnit_Generate_SelectorResolverRenders(t *testing.T) { + t.Parallel() + + bp, ds := selectorBlueprint() + + g, err := New() + if err != nil { + t.Fatalf("New: %v", err) + } + v, err := DataSource(bp, ds, Options{}) + if err != nil { + t.Fatalf("DataSource: %v", err) + } + out, err := g.renderFile("datasource_read.go.tmpl", v) + if err != nil { + t.Fatalf("render: %v", err) + } + text := string(out) + + for _, want := range []string{ + `Set exactly one of: id, test_name.`, + `if data.ID.IsNull() {`, + `listing, err := d.client.Tests().Get(ctx, nil)`, + `for _, el := range listing.GetTests() {`, + `el.GetTestName() == nil || *el.GetTestName() != data.TestName.ValueString()`, + `"Ambiguous match"`, + `"No match"`, + `data.ID = convert.PtrStringToFramework(match.GetTestId())`, + `remote, err := d.client.Tests().ByTestId(data.ID.ValueString()).Get(ctx, nil)`, + } { + if !strings.Contains(text, want) { + t.Errorf("the rendered read is missing %q:\n%s", want, text) + } + } +} diff --git a/internal/openapi/exclusions.go b/internal/openapi/exclusions.go new file mode 100644 index 00000000..bda7f4c6 --- /dev/null +++ b/internal/openapi/exclusions.go @@ -0,0 +1,72 @@ +package openapi + +import ( + "encoding/json" + "fmt" + "os" + "strings" +) + +// ExclusionsFileName is the conventional sidecar beside an openapi directory's +// snapshots, e.g. openapi/thousandeyes/draft-exclusions.json. +const ExclusionsFileName = "draft-exclusions.json" + +// Exclusion is one curated statement that a path family must not be drafted. +// +// The judgement lives in a committed sidecar rather than in inference, because +// the document does not say which of its readable surfaces are configuration +// and which are telemetry -- a time-windowed read is a strong hint, a plain +// one proves nothing -- and a heuristic that guessed would guess silently. +// Every entry names its reason, and the draft run repeats it as a named skip. +type Exclusion struct { + // PathPrefix excludes every family whose collection path starts with it. + PathPrefix string `json:"pathPrefix,omitempty"` + // Key excludes one family exactly. + Key string `json:"key,omitempty"` + // Reason is required: an unexplained exclusion is indistinguishable from a + // mistake. + Reason string `json:"reason"` +} + +// Exclusions is the parsed sidecar. +type Exclusions struct { + Exclusions []Exclusion `json:"exclusions"` +} + +// LoadExclusions reads a sidecar; a missing file is simply no exclusions. +func LoadExclusions(path string) (Exclusions, error) { + data, err := os.ReadFile(path) //nolint:gosec // operator-supplied path by design + if os.IsNotExist(err) { + return Exclusions{}, nil + } + if err != nil { + return Exclusions{}, err + } + + var e Exclusions + if err := json.Unmarshal(data, &e); err != nil { + return Exclusions{}, fmt.Errorf("%s is not a usable exclusions sidecar: %w", path, err) + } + for i, x := range e.Exclusions { + if x.Reason == "" { + return Exclusions{}, fmt.Errorf("%s: exclusions[%d] has no reason; an unexplained exclusion is indistinguishable from a mistake", path, i) + } + if x.PathPrefix == "" && x.Key == "" { + return Exclusions{}, fmt.Errorf("%s: exclusions[%d] names neither a pathPrefix nor a key", path, i) + } + } + return e, nil +} + +// Match reports whether a candidate is excluded, and why. +func (e Exclusions) Match(c Candidate) (string, bool) { + for _, x := range e.Exclusions { + if x.Key != "" && x.Key == c.Key { + return x.Reason, true + } + if x.PathPrefix != "" && strings.HasPrefix(c.CollectionPath, x.PathPrefix) { + return x.Reason, true + } + } + return "", false +} diff --git a/internal/openapi/exclusions_test.go b/internal/openapi/exclusions_test.go new file mode 100644 index 00000000..82d07a69 --- /dev/null +++ b/internal/openapi/exclusions_test.go @@ -0,0 +1,44 @@ +package openapi + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestUnit_Exclusions_MatchByPrefixAndKey(t *testing.T) { + t.Parallel() + + e := Exclusions{Exclusions: []Exclusion{ + {PathPrefix: "/test-results", Reason: "telemetry"}, + {Key: "event", Reason: "activity log"}, + }} + + if reason, ok := e.Match(Candidate{Key: "test_results_bgp", CollectionPath: "/test-results/{testId}/bgp"}); !ok || reason != "telemetry" { + t.Errorf("prefix match = %q, %v", reason, ok) + } + if reason, ok := e.Match(Candidate{Key: "event", CollectionPath: "/events"}); !ok || reason != "activity log" { + t.Errorf("key match = %q, %v", reason, ok) + } + if _, ok := e.Match(Candidate{Key: "tag", CollectionPath: "/tags"}); ok { + t.Error("an unexcluded candidate matched") + } +} + +func TestUnit_Exclusions_LoadRefusesUnreasonedEntries(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, ExclusionsFileName) + if err := os.WriteFile(path, []byte(`{"exclusions":[{"pathPrefix":"/x"}]}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadExclusions(path); err == nil || !strings.Contains(err.Error(), "reason") { + t.Errorf("an unexplained exclusion must refuse to load, got: %v", err) + } + + if _, err := LoadExclusions(filepath.Join(dir, "absent.json")); err != nil { + t.Errorf("a missing sidecar is simply no exclusions, got: %v", err) + } +} diff --git a/internal/openapi/infer_datasource.go b/internal/openapi/infer_datasource.go new file mode 100644 index 00000000..4ee0421c --- /dev/null +++ b/internal/openapi/infer_datasource.go @@ -0,0 +1,319 @@ +package openapi + +import ( + "fmt" + "sort" + "strings" + + "github.com/deploymenttheory/terraform-plugin-framework-codegen/internal/blueprint" + "github.com/deploymenttheory/terraform-plugin-framework-codegen/internal/naming" + + base "github.com/pb33f/libopenapi/datamodel/high/base" +) + +// ErrNotADataSource marks a candidate the data-source inference refuses. +var ErrNotADataSource = fmt.Errorf("not a data source candidate") + +// InferDataSource turns a readable-but-not-creatable candidate into a +// selector-shaped data source blueprint. +// +// The shape is the one a practitioner actually uses: one data source per +// family, looked up by exactly one of its selectors. The list operation is the +// resolver -- selectors narrow it to exactly one element, predictably, with +// zero and several matches both refused -- and where the family also offers a +// direct by-identifier read, the matched element only supplies the identifier +// and the direct read does the fetching, so state always maps from one +// response shape. +// +// Implemented for the kiotaFluent dialect only: the resty pilot's data +// sources were hand-authored, and no resty consumer is asking. +func (d *Document) InferDataSource(c Candidate, opts InferOptions) (blueprint.DataSource, []Caveat, error) { + kind, why := c.Classify() + if kind != CandidateKindDataSource { + return blueprint.DataSource{}, nil, fmt.Errorf("%w: %s: %s", ErrNotADataSource, c.Key, why) + } + if opts.SDKDialect != blueprint.DialectKiotaFluent { + return blueprint.DataSource{}, nil, fmt.Errorf( + "%w: %s: dataSource inference exists for the kiotaFluent dialect only", ErrNotADataSource, c.Key) + } + if c.List == nil { + return blueprint.DataSource{}, nil, fmt.Errorf( + "%w: %s: no list operation; a lookup with no resolver would be a bare item read, which curation can author directly", + ErrNotADataSource, c.Key) + } + + goType := namingOpts.GoTypeName(c.Key) + "DataSource" + + ds := blueprint.DataSource{ + Key: c.Key, + Name: naming.TerraformName(c.Key), + GoPackage: naming.SnakeDirName(c.Key), + GoPackageAlias: namingOpts.PackageAlias(opts.APIVersionDir, c.Key+"_ds"), + GoTypeName: goType, + ModelTypeName: goType + "Model", + ServiceGroup: naming.SnakeDirName(c.Tag), + APIVersionDir: opts.APIVersionDir, + } + if s := summaryOf(c); s != "" { + ds.Schema.MarkdownDescription = s + } + + ctxArg := blueprint.Argument{Kind: blueprint.ArgContext} + nilCfg := blueprint.Argument{Kind: blueprint.ArgLiteral, Expr: "nil"} + cfgID := blueprint.Argument{Kind: blueprint.ArgConfigField, Field: "ID"} + + // The list resolver: response type, the field holding the elements, and + // the element type itself. + listRespName := schemaNameOf(d.operationResponseProxy(c.List)) + collectionJSON, elementName := d.collectionOf(c.List) + if listRespName == "" || collectionJSON == "" || elementName == "" { + return blueprint.DataSource{}, nil, fmt.Errorf( + "%w: %s: the list response declares no element collection this can resolve through", + ErrNotADataSource, c.Key) + } + elementType := "models." + kiotaName(elementName) + "able" + + binding := blueprint.DataSourceBinding{ + Service: blueprint.ServiceRef{ + ImportPath: strings.TrimSuffix(opts.SDKModelsImport, "/"), + Alias: "models", + Accessor: "d.client", + }, + List: &blueprint.Operation{ + Style: blueprint.CallStyleFluent, + Chain: kiotaChain(c.List.Path, "Get", []blueprint.Argument{ctxArg, nilCfg}), + Return: blueprint.ReturnResultError, ResultType: "models." + kiotaName(listRespName) + "able", + HTTPMethod: c.List.Method, PathTemplate: c.List.Path, + }, + CollectionField: "Get" + kiotaAccessorBase(collectionJSON) + "()", + ElementType: elementType, + } + + // Attributes come from whichever schema state will map from: the direct + // read's response when one exists, the list element otherwise. + var stateSchema *base.Schema + if c.Read != nil { + stateSchema = responseSchema(d.operation(c.Read)) + respName := schemaNameOf(d.operationResponseProxy(c.Read)) + if respName == "" { + return blueprint.DataSource{}, nil, fmt.Errorf( + "%w: %s: the read declares no response schema", ErrNotADataSource, c.Key) + } + binding.Read = &blueprint.Operation{ + Style: blueprint.CallStyleFluent, + Chain: kiotaChainWith(c.Read.Path, "Get", cfgID, []blueprint.Argument{ctxArg, nilCfg}), + Return: blueprint.ReturnResultError, ResultType: "models." + kiotaName(respName) + "able", + HTTPMethod: c.Read.Method, PathTemplate: c.Read.Path, + } + binding.Response = blueprint.ResponseModel{ + Type: "models." + kiotaName(respName) + "able", AccessStyle: blueprint.AccessMethod, + } + } else { + stateSchema = d.elementSchemaOf(c.List) + binding.Response = blueprint.ResponseModel{ + Type: elementType, AccessStyle: blueprint.AccessMethod, + } + } + + ictx := newInferCtx(c.Key, "models", opts.SDKDialect) + + fields := Fields(stateSchema) + sort.Slice(fields, func(i, j int) bool { return fields[i].Name < fields[j].Name }) + + var attrs []blueprint.Attribute + for _, f := range fields { + if skip, why := skipField(f); skip { + ictx.note(f.Name, why) + continue + } + attr, why := ictx.attributeOf(f, f.Name, false) + if why != "" { + ictx.note(f.Name, why) + continue + } + attr.ComputedOptionalRequired = blueprint.Computed + stripExpand(&attr) + attrs = append(attrs, attr) + } + if len(attrs) == 0 { + return blueprint.DataSource{}, ictx.notes, fmt.Errorf( + "%w: %s: nothing usable in its schemas", ErrNotADataSource, c.Key) + } + + // Selectors: the direct identifier when a by-id read exists, plus every + // name-shaped string field on the element -- the fields a practitioner + // actually knows an object by. Each selector attribute is configuration as + // well as output, so it flips to optional-and-computed. + elementFields := Fields(d.elementSchemaOf(c.List)) + selectorSet := map[string]bool{} + for _, f := range elementFields { + if f.Kind != blueprint.KindString || f.Deprecated { + continue + } + lower := strings.ToLower(f.Name) + if lower == "name" || strings.HasSuffix(lower, "name") { + selectorSet[f.Name] = true + } + } + + var selectors []blueprint.Selector + if c.Read != nil { + idAttr, idJSON := d.identifierOf(c, attrs) + if idAttr == "" { + return blueprint.DataSource{}, ictx.notes, fmt.Errorf( + "%w: %s: no identifier attribute to feed the direct read", ErrNotADataSource, c.Key) + } + selectors = append(selectors, blueprint.Selector{ + Attribute: idAttr, GoField: goFieldOf(attrs, idAttr), ViaRead: true, + }) + markSelector(attrs, idAttr) + + binding.ElementIDField = kiotaAccessorBase(idJSON) + binding.ElementIDFlatten = &blueprint.ConvertCall{Func: "convert.PtrStringToFramework"} + } + + for _, f := range elementFields { + if !selectorSet[f.Name] { + continue + } + attrName := naming.TerraformName(f.Name) + if !hasAttribute(attrs, attrName) { + continue + } + selectors = append(selectors, blueprint.Selector{ + Attribute: attrName, + GoField: goFieldOf(attrs, attrName), + SDKField: kiotaAccessorBase(f.Name), + }) + markSelector(attrs, attrName) + } + + if len(selectors) == 0 { + return blueprint.DataSource{}, ictx.notes, fmt.Errorf( + "%w: %s: no identifier and no name-shaped field; there is nothing predictable to look this up by", + ErrNotADataSource, c.Key) + } + + binding.Selectors = selectors + ds.Binding = binding + ds.Schema.Attributes = attrs + + return ds, ictx.notes, nil +} + +// stripExpand removes the write direction from an attribute and its children: +// a data source sends no body, so only flatten survives. +func stripExpand(a *blueprint.Attribute) { + a.Wire.Expand = nil + a.Wire.UpdateExpand = nil + if a.Type.NestedObject != nil { + for i := range a.Type.NestedObject.Attributes { + stripExpand(&a.Type.NestedObject.Attributes[i]) + } + } +} + +// markSelector flips a selector attribute to optional-and-computed: settable +// as the lookup key, filled from the response either way. +func markSelector(attrs []blueprint.Attribute, name string) { + for i := range attrs { + if attrs[i].Name == name { + attrs[i].ComputedOptionalRequired = blueprint.ComputedOptional + } + } +} + +func goFieldOf(attrs []blueprint.Attribute, name string) string { + for _, a := range attrs { + if a.Name == name { + return a.GoField + } + } + return "" +} + +// identifierOf picks the attribute the direct read is keyed by: the item +// path's parameter, matched against the state schema's fields. +func (d *Document) identifierOf(c Candidate, attrs []blueprint.Attribute) (attrName, jsonName string) { + param := "" + if i := strings.LastIndex(c.ItemPath, "{"); i >= 0 { + param = strings.Trim(c.ItemPath[i:], "{}") + } + + for _, cand := range []string{param, "id"} { + if cand == "" { + continue + } + name := naming.TerraformName(cand) + if hasAttribute(attrs, name) { + return name, cand + } + } + // The API frequently names the parameter what the schema abbreviates: + // {testId} against a schema field "testId" is covered above; a schema that + // only carries "id" against a parameter "tagId" lands here. + if hasAttribute(attrs, "id") { + return "id", "id" + } + return "", "" +} + +// operationResponseProxy is responseProxy for one operation. +func (d *Document) operationResponseProxy(op *Operation) *base.SchemaProxy { + o := d.operation(op) + if o == nil || o.Responses == nil || o.Responses.Codes == nil { + return nil + } + for pair := o.Responses.Codes.First(); pair != nil; pair = pair.Next() { + if !strings.HasPrefix(pair.Key(), "2") || pair.Value() == nil || pair.Value().Content == nil { + continue + } + if p := proxyFromContent(pair.Value().Content); p != nil { + return p + } + } + return nil +} + +// collectionOf finds the array-of-objects property inside the list response: +// its JSON name reaches the elements, its item schema names their type. +func (d *Document) collectionOf(list *Operation) (jsonName, elementSchema string) { + s := responseSchema(d.operation(list)) + if s == nil || s.Properties == nil { + return "", "" + } + for pair := s.Properties.First(); pair != nil; pair = pair.Next() { + prop := pair.Value() + if prop == nil { + continue + } + ps := prop.Schema() + if ps == nil || len(ps.Type) == 0 || ps.Type[0] != "array" || ps.Items == nil || ps.Items.A == nil { + continue + } + if name := refTypeName(ps.Items.A); name != "" { + return pair.Key(), name + } + } + return "", "" +} + +// elementSchemaOf resolves the list element's schema. +func (d *Document) elementSchemaOf(list *Operation) *base.Schema { + s := responseSchema(d.operation(list)) + if s == nil || s.Properties == nil { + return nil + } + for pair := s.Properties.First(); pair != nil; pair = pair.Next() { + prop := pair.Value() + if prop == nil { + continue + } + ps := prop.Schema() + if ps == nil || len(ps.Type) == 0 || ps.Type[0] != "array" || ps.Items == nil || ps.Items.A == nil { + continue + } + return ps.Items.A.Schema() + } + return nil +} diff --git a/internal/openapi/kiota.go b/internal/openapi/kiota.go index 815e6cd1..6be6f591 100644 --- a/internal/openapi/kiota.go +++ b/internal/openapi/kiota.go @@ -71,9 +71,14 @@ func kiotaAccessorBase(jsonName string) string { // // /tags/{tagId} + GET -> Tags().ByTagId(id).Get(ctx, nil) func kiotaChain(pathTemplate, verb string, verbArgs []blueprint.Argument) []blueprint.ChainSegment { - var chain []blueprint.ChainSegment + return kiotaChainWith(pathTemplate, verb, + blueprint.Argument{Kind: blueprint.ArgStateField, Field: "ID"}, verbArgs) +} - idArg := blueprint.Argument{Kind: blueprint.ArgStateField, Field: "ID"} +// kiotaChainWith is kiotaChain with the identifier argument chosen by the +// caller: a resource reads it from state, a data source from configuration. +func kiotaChainWith(pathTemplate, verb string, idArg blueprint.Argument, verbArgs []blueprint.Argument) []blueprint.ChainSegment { + var chain []blueprint.ChainSegment for _, segment := range strings.Split(strings.Trim(pathTemplate, "/"), "/") { if segment == "" { diff --git a/internal/sdkbind/verify.go b/internal/sdkbind/verify.go index b81dff93..5bcda7b6 100644 --- a/internal/sdkbind/verify.go +++ b/internal/sdkbind/verify.go @@ -194,6 +194,46 @@ func verifyDataSource( verifyOperation(l, clientType, d.Key, svc, "read", *d.Binding.Read, r) } + // The selector resolver: the list operation, the accessor reaching its + // elements, and every field a selector matches against are all names the + // generated code will spell, so each is checked against the SDK here. + if d.Binding.List != nil { + verifyOperation(l, clientType, d.Key, svc, "list", *d.Binding.List, r) + + if verifyNamedType(l, d.Key, "binding.elementType", d.Binding.ElementType, svc, r) { + element := typeNameOf(d.Binding.ElementType) + + if d.Binding.List.ResultType != "" && + verifyNamedType(l, d.Key, "binding.list.resultType", d.Binding.List.ResultType, svc, r) { + field := strings.TrimSuffix(strings.TrimSuffix(d.Binding.CollectionField, "()"), "()") + field = strings.TrimPrefix(field, "Get") + verifyFieldOn( + l, d.Binding.Response.AccessStyle, d.Key, + "binding.collectionField", + typeNameOf(d.Binding.List.ResultType), field, svc, r, + ) + } + + for i, s := range d.Binding.Selectors { + if s.ViaRead || s.SDKField == "" { + continue + } + verifyFieldOn( + l, d.Binding.Response.AccessStyle, d.Key, + fmt.Sprintf("binding.selectors[%d].sdkField", i), + element, s.SDKField, svc, r, + ) + } + if d.Binding.ElementIDField != "" { + verifyFieldOn( + l, d.Binding.Response.AccessStyle, d.Key, + "binding.elementIdField", + element, d.Binding.ElementIDField, svc, r, + ) + } + } + } + if !responseOK { // The type is already reported once; checking fields against it would repeat that // one cause per attribute. diff --git a/internal/templates/datasource_read.go.tmpl b/internal/templates/datasource_read.go.tmpl index ac7ee999..743b1254 100644 --- a/internal/templates/datasource_read.go.tmpl +++ b/internal/templates/datasource_read.go.tmpl @@ -6,13 +6,12 @@ import ( {{ .Imports.Read }} ) -{{ with .Read }} -// Read fetches {{ $.DataSourceName }} from the API. +// Read fetches {{ .DataSourceName }} from the API. // // A data source reads its arguments from configuration rather than from prior state: // there is no prior state to read, because nothing here was created by Terraform. -func (d *{{ $.GoTypeName }}) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) { - var data {{ $.ModelTypeName }} +func (d *{{ .GoTypeName }}) Read(ctx context.Context, req datasource.ReadRequest, resp *datasource.ReadResponse) { + var data {{ .ModelTypeName }} resp.Diagnostics.Append(req.Config.Get(ctx, &data)...) if resp.Diagnostics.HasError() { @@ -26,7 +25,117 @@ func (d *{{ $.GoTypeName }}) Read(ctx context.Context, req datasource.ReadReques return } defer cancel() +{{- with .Resolve }} + + // A lookup must be predictable: exactly one selector chooses the object, + // and the resolver below refuses zero matches and several matches alike. + selectorsSet := 0 +{{- range .AllSelectorGoFields }} + if !data.{{ . }}.IsNull() { + selectorsSet++ + } +{{- end }} + if selectorsSet != 1 { + resp.Diagnostics.AddError( + "Exactly one selector is required", + "Set exactly one of: {{ .SelectorList }}.", + ) + return + } + + {{ if .MapsElement }}{{ .List.Assign }} {{ .List.Call }}{{ else }}if data.{{ .IDGoField }}.IsNull() { + {{ .List.Assign }} {{ .List.Call }}{{ end }} +{{- if .MapsElement }} + if err != nil { + errors.Handle(&resp.Diagnostics, TypeName, {{ .List.ErrorOp }}, err) + return + } +{{- if .List.NilResultGuard }} + if listing == nil { + errors.Handle(&resp.Diagnostics, TypeName, {{ .List.ErrorOp }}, errors.ErrEmptyResponse) + return + } +{{- end }} + var matches []{{ .ElementType }} + for _, el := range listing.{{ .CollectionField }} { +{{- range .Matchers }} + if !data.{{ .GoField }}.IsNull() && ({{ .Getter }} == nil || *{{ .Getter }} != data.{{ .GoField }}.ValueString()) { + continue + } +{{- end }} + matches = append(matches, el) + } + switch len(matches) { + case 1: + case 0: + resp.Diagnostics.AddError( + "No match", + "The configured selector matched nothing. The object may not exist, or may be invisible to this account group.", + ) + return + default: + resp.Diagnostics.AddError( + "Ambiguous match", + fmt.Sprintf("The configured selector matched %d objects; a lookup must resolve to exactly one. Narrow the selector.", len(matches)), + ) + return + } +{{- if $.State.NeedsDiagnostics }} + resp.Diagnostics.Append(mapRemoteStateToTerraform(ctx, &data, matches[0])...) + if resp.Diagnostics.HasError() { + return + } +{{- else }} + mapRemoteStateToTerraform(ctx, &data, matches[0]) +{{- end }} + + resp.Diagnostics.Append(resp.State.Set(ctx, &data)...) + return +{{- else }} + if err != nil { + errors.Handle(&resp.Diagnostics, TypeName, {{ .List.ErrorOp }}, err) + return + } +{{- if .List.NilResultGuard }} + if listing == nil { + errors.Handle(&resp.Diagnostics, TypeName, {{ .List.ErrorOp }}, errors.ErrEmptyResponse) + return + } +{{- end }} + var matches []{{ .ElementType }} + for _, el := range listing.{{ .CollectionField }} { +{{- range .Matchers }} + if !data.{{ .GoField }}.IsNull() && ({{ .Getter }} == nil || *{{ .Getter }} != data.{{ .GoField }}.ValueString()) { + continue + } +{{- end }} + matches = append(matches, el) + } + switch len(matches) { + case 1: + case 0: + resp.Diagnostics.AddError( + "No match", + "The configured selector matched nothing. The object may not exist, or may be invisible to this account group.", + ) + return + default: + resp.Diagnostics.AddError( + "Ambiguous match", + fmt.Sprintf("The configured selector matched %d objects; a lookup must resolve to exactly one. Narrow the selector.", len(matches)), + ) + return + } + + // The matched element only supplies the identifier; the direct read + // below does the fetching, so state always maps from one shape. + match := matches[0] + data.{{ .IDGoField }} = {{ .ElementIDExpr }} + } +{{- end }} +{{- end }} +{{ with .Read }} {{ .Assign }} {{ .Call }} if err != nil { // Unlike a resource read, a missing object is an error rather than a signal to @@ -56,5 +165,5 @@ func (d *{{ $.GoTypeName }}) Read(ctx context.Context, req datasource.ReadReques {{- end }} resp.Diagnostics.Append(resp.State.Set(ctx, &data)...) +{{- end }} } -{{ end }}