diff --git a/cmd/tfpfgen/main_test.go b/cmd/tfpfgen/main_test.go index 654368de..1a080eda 100644 --- a/cmd/tfpfgen/main_test.go +++ b/cmd/tfpfgen/main_test.go @@ -78,6 +78,9 @@ var builtCommands = map[string]bool{ // sdk generate shells out to kiota; the verb itself is built, and its // no-argument invocation fails on required flags, not errNotImplemented. "sdk generate": true, + // sdk push publishes an external-mode SDK tree the way provider push + // publishes a provider tree. + "sdk push": true, // blueprint draft infers resource blueprints from a pinned snapshot. Data // sources, actions and the provider block are still hand-authored, and every // skip is printed as a note rather than being silent. diff --git a/cmd/tfpfgen/push.go b/cmd/tfpfgen/push.go index ff8f2e27..6c7f7f11 100644 --- a/cmd/tfpfgen/push.go +++ b/cmd/tfpfgen/push.go @@ -205,7 +205,13 @@ func runProviderPush(args []string) error { return nil } - prURL, err := openPullRequest(target, token, branch, base, m, changed) + prURL, err := openPullRequest(target, token, branch, base, + "Regenerate provider from blueprints", + fmt.Sprintf( + "Generated by `tfpfgen %s` from %s — %d file(s) changed.\n\n"+ + "This branch is generator-owned and force-pushed on regeneration. "+ + "Review the diff here; to change the content, edit the blueprints and regenerate.", + m.ToolVersion, blueprintSources(m), changed)) if err != nil { // The push itself succeeded, and saying so matters more than the PR // call failing: the work is on the branch either way. @@ -421,14 +427,7 @@ func blueprintSources(m manifest.Manifest) string { // openPullRequest opens the PR, or finds the one already open for the branch. // A second push to the same branch must not fail over a PR that already says // exactly what this one would. -func openPullRequest(t repoTarget, token, branch, base string, m manifest.Manifest, changed int) (string, error) { - title := "Regenerate provider from blueprints" - body := fmt.Sprintf( - "Generated by `tfpfgen %s` from %s — %d file(s) changed.\n\n"+ - "This branch is generator-owned and force-pushed on regeneration. "+ - "Review the diff here; to change the content, edit the blueprints and regenerate.", - m.ToolVersion, blueprintSources(m), changed) - +func openPullRequest(t repoTarget, token, branch, base, title, body string) (string, error) { payload, err := json.Marshal(map[string]string{ "title": title, "body": body, diff --git a/cmd/tfpfgen/push_test.go b/cmd/tfpfgen/push_test.go index 3a75e252..cebe507c 100644 --- a/cmd/tfpfgen/push_test.go +++ b/cmd/tfpfgen/push_test.go @@ -276,8 +276,6 @@ func TestUnit_CLI_Push_DryRunPushesNothing(t *testing.T) { func TestUnit_CLI_Push_PullRequest(t *testing.T) { t.Parallel() - m := manifest.New("test", nil) - t.Run("created", func(t *testing.T) { t.Parallel() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -293,7 +291,7 @@ func TestUnit_CLI_Push_PullRequest(t *testing.T) { defer srv.Close() target := repoTarget{owner: "org", name: "prov", gitHub: true, apiBase: srv.URL} - url, err := openPullRequest(target, "tok", "tfpfgen/generate-abc", "main", m, 3) + url, err := openPullRequest(target, "tok", "tfpfgen/generate-abc", "main", "title", "body") if err != nil { t.Fatalf("openPullRequest: %v", err) } @@ -315,7 +313,7 @@ func TestUnit_CLI_Push_PullRequest(t *testing.T) { defer srv.Close() target := repoTarget{owner: "org", name: "prov", gitHub: true, apiBase: srv.URL} - url, err := openPullRequest(target, "tok", "tfpfgen/generate-abc", "main", m, 3) + url, err := openPullRequest(target, "tok", "tfpfgen/generate-abc", "main", "title", "body") if err != nil { t.Fatalf("openPullRequest: %v", err) } diff --git a/cmd/tfpfgen/sdk.go b/cmd/tfpfgen/sdk.go index d1249fd2..39b12e64 100644 --- a/cmd/tfpfgen/sdk.go +++ b/cmd/tfpfgen/sdk.go @@ -33,6 +33,12 @@ var sdkVerbs = []command{ usage: usageSDKGenerate, run: runSDKGenerate, }, + { + name: "push", + summary: "publish an external-mode SDK tree to a git repository, via branch and pull request", + usage: usageSDKPush, + run: runSDKPush, + }, } func runSDK(args []string) error { diff --git a/cmd/tfpfgen/sdk_push.go b/cmd/tfpfgen/sdk_push.go new file mode 100644 index 00000000..80f7ba3f --- /dev/null +++ b/cmd/tfpfgen/sdk_push.go @@ -0,0 +1,270 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + + "github.com/deploymenttheory/terraform-plugin-framework-codegen/internal/kiota" + "github.com/deploymenttheory/terraform-plugin-framework-codegen/internal/manifest" + "github.com/deploymenttheory/terraform-plugin-framework-codegen/internal/version" +) + +const usageSDKPush = "sdk push -out DIR -repo URL [-branch NAME] [-base NAME] [-dry-run]" + +// sdkPushBranchPrefix namespaces the branches sdk push creates, beside +// provider push's. The suffix is the digest of the lock file, so the same +// generation always names the same branch. +const sdkPushBranchPrefix = "tfpfgen/sdk-" + +func runSDKPush(args []string) error { + fs, _ := newFlagSet("sdk push", usageSDKPush) + + var o pushOptions + fs.StringVar(&o.out, "out", "", "SDK root to publish, as written by sdk generate -mode external (required)") + fs.StringVar(&o.repo, "repo", "", "target repository: a clone URL or GitHub owner/name (required)") + fs.StringVar(&o.branch, "branch", "", + "branch to push; defaults to "+sdkPushBranchPrefix+" derived from "+kiota.LockFileName) + fs.StringVar(&o.base, "base", "", + "branch to diff and open the pull request against; defaults to the repository's default branch") + fs.BoolVar(&o.dryRun, "dry-run", false, "clone and compare, but push nothing and open nothing") + + if err := parse(fs, args); err != nil { + return err + } + + if o.out == "" { + return usagef("-out is required: it names the SDK root to publish") + } + if o.repo == "" { + return usagef("-repo is required: it names the repository to publish into") + } + + // The lock is the SDK's provenance record, the way the manifest is the + // provider's: a tree without one is a tree this pipeline has not produced, + // and publishing it would put content of unknown origin under a commit + // message that claims otherwise. + lock, hadLock, err := kiota.ReadLock(o.out) + if err != nil { + return err + } + if !hadLock { + return usagef("%s has no %s; run sdk generate first -- push publishes generated output, not arbitrary trees", + o.out, kiota.LockFileName) + } + + // Only an external-mode tree can live in its own repository: an embedded + // SDK's import path is the provider module plus a directory, and moving the + // tree without that module context breaks every import in it. + if _, err := os.Stat(filepath.Join(o.out, "go.mod")); err != nil { + return usagef("%s has no go.mod, so it is an embedded SDK; only a tree from "+ + "sdk generate -mode external can be published to its own repository", o.out) + } + + target, err := parseRepo(o.repo) + if err != nil { + return err + } + + token := os.Getenv(pushTokenEnv) + if token == "" { + token = os.Getenv(pushTokenFallbackEnv) + } + if target.gitHub && token == "" && !o.dryRun { + return usagef("%s (or %s) must be set to push to %s", pushTokenEnv, pushTokenFallbackEnv, target.host) + } + + if _, err := exec.LookPath("git"); err != nil { + return fmt.Errorf("sdk push needs git on PATH: %w", err) + } + + work, err := os.MkdirTemp("", "tfpfgen-sdk-push-*") + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(work) }() + + g := gitRunner{dir: work, host: target.host, token: token} + + cloneArgs := []string{"clone", "--depth", "1"} + if o.base != "" { + cloneArgs = append(cloneArgs, "--branch", o.base) + } + cloneArgs = append(cloneArgs, target.cloneURL, work) + if _, err := g.in("").run(cloneArgs...); err != nil { + return fmt.Errorf("cloning %s: %w", target.cloneURL, err) + } + + base := o.base + if base == "" { + out, err := g.run("rev-parse", "--abbrev-ref", "HEAD") + if err != nil { + return fmt.Errorf("finding the default branch: %w", err) + } + base = strings.TrimSpace(out) + } + + // The SDK root carries no manifest of its own -- it is byte-compared + // against fresh kiota output, and a foreign file would fail that check -- + // so the inventory that makes pruning safe lives in the target repository + // instead, written by each push and read by the next. Files the target + // carries that no push produced -- its licence, its workflows -- are never + // touched, exactly as provider push behaves. + previous, hadManifest, err := manifest.Load(work) + if err != nil { + return err + } + + produced, err := syncTree(o.out, work) + if err != nil { + return err + } + + if err := writeSDKManifest(work, produced, lock); err != nil { + return err + } + produced[manifest.Name] = true + + if hadManifest { + orphans, err := previous.Orphans(work, produced) + if err != nil { + return err + } + for _, p := range orphans { + if err := os.Remove(filepath.Join(work, p)); err != nil { + return fmt.Errorf("pruning %s: %w", p, err) + } + log.Printf("pruned %s (generated last push, no longer produced)", p) + } + } + + if _, err := g.run("add", "--all"); err != nil { + return err + } + status, err := g.run("status", "--porcelain") + if err != nil { + return err + } + if strings.TrimSpace(status) == "" { + log.Printf("✅ %s already matches %s; nothing to push", target.cloneURL, o.out) + return nil + } + + changed := strings.Count(strings.TrimSpace(status), "\n") + 1 + + if o.dryRun { + fmt.Fprint(os.Stdout, status) + log.Printf("%d file(s) would change on %s; dry run, nothing was pushed", changed, base) + return nil + } + + branch := o.branch + if branch == "" { + digest, err := lockDigest(o.out) + if err != nil { + return err + } + branch = sdkPushBranchPrefix + digest + } + + if _, err := g.run("checkout", "-B", branch); err != nil { + return err + } + if _, err := g.run("commit", "-m", sdkPushCommitMessage(lock, changed)); err != nil { + return err + } + // Forced, and only ever onto the generator-owned branch namespace: the + // content is a pure function of (kiota version, pinned document, patches), + // so the newest generation is always the right thing for the branch to hold. + if _, err := g.run("push", "--force", "origin", "HEAD:refs/heads/"+branch); err != nil { + return fmt.Errorf("pushing %s: %w", branch, err) + } + + log.Printf("pushed %d file change(s) to %s on %s", changed, target.cloneURL, branch) + + if !target.gitHub { + log.Printf("note: %s is not a GitHub host, so no pull request was opened; merge %s where the repository lives", + target.host, branch) + return nil + } + + prURL, err := openPullRequest(target, token, branch, base, + "Regenerate SDK from the pinned OpenAPI document", + fmt.Sprintf( + "Generated by kiota %s via `tfpfgen %s` from the document with hash `%s` — %d file(s) changed.\n\n"+ + "This branch is generator-owned and force-pushed on regeneration. "+ + "Review the diff here; to change the content, refresh the snapshot or its patches and regenerate.", + lock.KiotaVersion, version.Version, shortHash(lock.DescriptionHash), changed)) + if err != nil { + // The push itself succeeded, and saying so matters more than the PR + // call failing: the work is on the branch either way. + return fmt.Errorf("the branch is pushed, but opening the pull request failed: %w", err) + } + + log.Printf("✅ pull request: %s", prURL) + return nil +} + +// writeSDKManifest records what this push produced, into the target work tree. +// Deterministic by construction -- sorted entries, no timestamp -- so a push of +// an unchanged generation writes an unchanged manifest and the no-op detection +// stays honest. +func writeSDKManifest(work string, produced map[string]bool, lock kiota.Lock) error { + paths := make([]string, 0, len(produced)) + for p := range produced { + paths = append(paths, p) + } + sort.Strings(paths) + + entries := make([]manifest.Entry, 0, len(paths)) + for _, p := range paths { + data, err := os.ReadFile(filepath.Join(work, filepath.FromSlash(p))) //nolint:gosec // paths this run wrote + if err != nil { + return err + } + sum := sha256.Sum256(data) + entries = append(entries, manifest.Entry{ + Path: p, + SHA256: hex.EncodeToString(sum[:]), + Blueprint: "openapi document " + shortHash(lock.DescriptionHash), + }) + } + + return manifest.Save(work, manifest.New(version.Version, entries)) +} + +// lockDigest derives the branch suffix from the lock file bytes, the same +// shape as provider push's manifest digest: one generation, one branch. +func lockDigest(root string) (string, error) { + data, err := os.ReadFile(filepath.Join(root, kiota.LockFileName)) //nolint:gosec // fixed name under the SDK root + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:])[:12], nil +} + +// sdkPushCommitMessage states provenance the way provider push does: what +// produced the tree, from what, and how much moved. +func sdkPushCommitMessage(lock kiota.Lock, changed int) string { + return fmt.Sprintf( + "Regenerate SDK from the pinned OpenAPI document\n\n"+ + "Produced by kiota %s via tfpfgen %s from the document with hash %s; %d file(s) changed.\n"+ + "This branch is generator-owned: refresh the snapshot or its patches, not these files.", + lock.KiotaVersion, version.Version, shortHash(lock.DescriptionHash), changed) +} + +// shortHash abbreviates the lock's 128-hex-digit description hash to a +// reviewable prefix. +func shortHash(h string) string { + if len(h) > 12 { + return h[:12] + } + return h +} diff --git a/cmd/tfpfgen/sdk_push_test.go b/cmd/tfpfgen/sdk_push_test.go new file mode 100644 index 00000000..90be7a29 --- /dev/null +++ b/cmd/tfpfgen/sdk_push_test.go @@ -0,0 +1,202 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/deploymenttheory/terraform-plugin-framework-codegen/internal/kiota" + "github.com/deploymenttheory/terraform-plugin-framework-codegen/internal/manifest" +) + +// generatedSDKTree writes an SDK root carrying a kiota lock and a go.mod, +// which is what makes it something sdk push will agree to publish. +func generatedSDKTree(t *testing.T, files map[string]string) string { + t.Helper() + + root := t.TempDir() + files[kiota.LockFileName] = `{ + "descriptionHash": "ABCDEF0123456789ABCDEF", + "descriptionLocation": "../openapi/test/api.yaml", + "kiotaVersion": "1.34.1" +} +` + if _, ok := files["go.mod"]; !ok { + files["go.mod"] = "module example.com/sdk\n" + } + for rel, content := range files { + p := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } + return root +} + +func TestUnit_CLI_SDKPush_RefusalsNameTheirCause(t *testing.T) { + quiet(t) + noTokens(t) + + if err := runSDKPush(nil); err == nil { + t.Error("expected -out to be required") + } + if err := runSDKPush([]string{"-out", t.TempDir()}); err == nil { + t.Error("expected -repo to be required") + } + + // A tree without a lock is a tree the pipeline has not produced. + err := runSDKPush([]string{"-out", t.TempDir(), "-repo", "owner/name"}) + if err == nil || !strings.Contains(err.Error(), "sdk generate first") { + t.Errorf("expected the lock refusal, got: %v", err) + } + + // A lock without a go.mod is an embedded SDK, whose import path only works + // inside the provider module. + embedded := t.TempDir() + if err := os.WriteFile(filepath.Join(embedded, kiota.LockFileName), []byte(`{"kiotaVersion":"1.34.1"}`), 0o600); err != nil { + t.Fatal(err) + } + err = runSDKPush([]string{"-out", embedded, "-repo", "owner/name"}) + if err == nil || !strings.Contains(err.Error(), "-mode external") { + t.Errorf("expected the embed refusal to name -mode external, got: %v", err) + } + + // GitHub without a token cannot push. + out := generatedSDKTree(t, map[string]string{"client.go": "package sdk\n"}) + err = runSDKPush([]string{"-out", out, "-repo", "owner/name"}) + if err == nil || !strings.Contains(err.Error(), pushTokenEnv) { + t.Errorf("expected the token refusal to name %s, got: %v", pushTokenEnv, err) + } +} + +// TestUnit_CLI_SDKPush_FirstPushOpensABranch is the main path: a target whose +// default branch holds an older generation (inventoried by the manifest the +// previous push wrote) plus files of its own, and a push that must sync, prune +// the orphan, leave the target's own files alone, and land on the +// generator-owned branch -- without a pull request, because file:// is not a +// GitHub host. +func TestUnit_CLI_SDKPush_FirstPushOpensABranch(t *testing.T) { + quiet(t) + needGit(t) + noTokens(t) + + oldManifest, err := manifest.Marshal(manifest.New("old", []manifest.Entry{ + {Path: "models/old_model.go", SHA256: "x", Blueprint: "openapi document abc"}, + })) + if err != nil { + t.Fatal(err) + } + + repo := seedTarget(t, map[string]string{ + ".github/workflows/ci.yml": "name: ci\n", + "LICENSE": "MIT\n", + "models/old_model.go": "package models\n", + manifest.Name: string(oldManifest), + }) + + out := generatedSDKTree(t, map[string]string{ + "models/new_model.go": "package models\n", + "client.go": "package sdk\n", + }) + + if err := runSDKPush([]string{"-out", out, "-repo", repo}); err != nil { + t.Fatalf("sdk push: %v", err) + } + + bare := strings.TrimPrefix(repo, "file://") + branches := gitIn(t, bare, "branch", "--list") + if !strings.Contains(branches, sdkPushBranchPrefix) { + t.Fatalf("no generator-owned branch was pushed; branches:\n%s", branches) + } + branch := "" + for _, b := range strings.Fields(branches) { + if strings.HasPrefix(b, sdkPushBranchPrefix) { + branch = b + } + } + + files := gitIn(t, bare, "ls-tree", "-r", "--name-only", branch) + for _, want := range []string{ + "models/new_model.go", "client.go", "go.mod", kiota.LockFileName, + ".github/workflows/ci.yml", "LICENSE", manifest.Name, + } { + if !strings.Contains(files, want) { + t.Errorf("the pushed branch is missing %s:\n%s", want, files) + } + } + if strings.Contains(files, "models/old_model.go") { + t.Error("the orphaned generated file was not pruned") + } + + msg := gitIn(t, bare, "log", "-1", "--format=%B", branch) + for _, want := range []string{"kiota 1.34.1", "ABCDEF012345", "generator-owned"} { + if !strings.Contains(msg, want) { + t.Errorf("the commit message omits %q:\n%s", want, msg) + } + } +} + +// TestUnit_CLI_SDKPush_SecondIdenticalPushIsANoOp proves the manifest sdk push +// writes is deterministic: after the first push's branch becomes the default +// branch, pushing the same tree again must find nothing to do. +func TestUnit_CLI_SDKPush_SecondIdenticalPushIsANoOp(t *testing.T) { + quiet(t) + needGit(t) + noTokens(t) + + repo := seedTarget(t, map[string]string{"README.md": "seed\n"}) + out := generatedSDKTree(t, map[string]string{"client.go": "package sdk\n"}) + + if err := runSDKPush([]string{"-out", out, "-repo", repo}); err != nil { + t.Fatalf("first push: %v", err) + } + + bare := strings.TrimPrefix(repo, "file://") + branches := gitIn(t, bare, "branch", "--list") + branch := "" + for _, b := range strings.Fields(branches) { + if strings.HasPrefix(b, sdkPushBranchPrefix) { + branch = b + } + } + if branch == "" { + t.Fatalf("no branch was pushed:\n%s", branches) + } + // Merge the generation into the default branch, as a human would. + gitIn(t, bare, "update-ref", "refs/heads/main", branch) + gitIn(t, bare, "branch", "-D", branch) + + if err := runSDKPush([]string{"-out", out, "-repo", repo}); err != nil { + t.Fatalf("second push: %v", err) + } + if branches := gitIn(t, bare, "branch", "--list"); strings.Contains(branches, sdkPushBranchPrefix) { + t.Errorf("an up-to-date push must not create a branch:\n%s", branches) + } +} + +func TestUnit_CLI_SDKPush_DryRunPushesNothing(t *testing.T) { + quiet(t) + needGit(t) + noTokens(t) + + repo := seedTarget(t, map[string]string{"README.md": "seed\n"}) + out := generatedSDKTree(t, map[string]string{"client.go": "package sdk\n"}) + + stdout := captureStdout(t, func() { + if err := runSDKPush([]string{"-out", out, "-repo", repo, "-dry-run"}); err != nil { + t.Errorf("dry run: %v", err) + } + }) + if !strings.Contains(stdout, "client.go") { + t.Errorf("the dry run should report what would change:\n%s", stdout) + } + + bare := strings.TrimPrefix(repo, "file://") + if branches := gitIn(t, bare, "branch", "--list"); strings.Contains(branches, sdkPushBranchPrefix) { + t.Errorf("a dry run must not push:\n%s", branches) + } +} diff --git a/docs/cli.md b/docs/cli.md index 9d7c83a9..cdb28b6d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -19,7 +19,7 @@ order `help` prints them. | Command | Purpose | Verbs | |---|---|---| | `openapi` | fetch and pin upstream OpenAPI documents | `fetch` | -| `sdk` | generate a Go SDK from a pinned OpenAPI snapshot | `generate` | +| `sdk` | generate a Go SDK from a pinned OpenAPI snapshot | `generate`, `push` | | `blueprint` | draft, merge, validate, diff or list blueprints | `draft`, `merge`; `validate`, `diff`, `list` planned | | `probe` | exercise a resource's lifecycle; record or replay cassettes | `record`, `replay`, `verify`, `sweep`, `list` | | `provider` | generate a terraform-plugin-framework provider from blueprints | `generate`, `push`; `scaffold` planned | @@ -175,6 +175,37 @@ document, an `add` that finds its value already present refuses as *stale* — the prompt to delete the patch. With no patches directory, the snapshot is read directly. +### `sdk push` + +Publishes an external-mode SDK tree to its own repository, as a branch and a +pull request — the SDK counterpart of `provider push`. + +``` +tfpfgen sdk push -out DIR -repo URL [-branch NAME] [-base NAME] [-dry-run] +``` + +| Flag | Default | Purpose | +|---|---|---| +| `-out` | — | SDK root to publish, as written by `sdk generate -mode external` (required) | +| `-repo` | — | target repository: a clone URL or GitHub `owner/name` (required) | +| `-branch` | `tfpfgen/sdk-` | branch to push; the digest is derived from `kiota-lock.json`, so the same generation always names the same branch | +| `-base` | the repository's default branch | branch to diff and open the pull request against | +| `-dry-run` | `false` | clone and compare, but push nothing and open nothing | + +The token doctrine, sync, prune and pull-request behaviour are `provider +push`'s exactly, with the SDK's own provenance records in the provider's +places. Push refuses a tree without a `kiota-lock.json` (nothing this pipeline +generated) and a tree without a `go.mod` (an *embedded* SDK, whose import path +only works inside the provider module — only `-mode external` output can live +in its own repository). Because the SDK root itself carries no manifest — it +is byte-compared against fresh kiota output, and a foreign file would fail +that check — the inventory that makes pruning safe is written into the +*target* repository as `.tfpfgen/manifest.json` by each push and read by the +next, so the target's own files (its licence, its workflows) are never +touched. Commits and pull requests land on the generator-owned +`tfpfgen/sdk-*` branch namespace, naming the kiota version and the pinned +document's hash. + ## `blueprint` ### `blueprint draft`