Add continue_prev invariant test against latest released CLI

acceptance/acceptance_test.go

@@ -27,6 +27,7 @@ import (
 	"unicode/utf8"
 
 	"github.com/google/uuid"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/databricks/cli/acceptance/internal"
 	"github.com/databricks/cli/internal/testutil"
@@ -311,9 +312,13 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 		} else if UseVersion != "" {
 			version := UseVersion
 			if version == "latest" {
-				version = resolveLatestVersion(t, buildDir)
+				v, err := resolveLatestVersion(buildDir)
+				require.NoError(t, err)
+				version = v
 			}
-			execPath = DownloadCLI(t, buildDir, version)
+			p, err := DownloadCLI(buildDir, version)
+			require.NoError(t, err)
+			execPath = p
 		} else {
 			execPath = BuildCLI(t, buildDir, coverDir, runtime.GOOS, runtime.GOARCH)
 		}
@@ -330,9 +335,36 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 	repls.SetPath(execPath, "[CLI]")
 
 	if !inprocessMode {
-		cli293Path := DownloadCLI(t, buildDir, "0.293.0")
+		// Download 0.293.0 and the latest released CLI concurrently: on a cold
+		// CI cache both downloads run in parallel; the file-based caches in
+		// DownloadCLI / resolveLatestVersion make warm runs a no-op either way.
+		var (
+			cli293Path, cliPrevPath, prevVersion string
+			g                                    errgroup.Group
+		)
+		g.Go(func() error {
+			p, err := DownloadCLI(buildDir, "0.293.0")
+			cli293Path = p
+			return err
+		})
+		g.Go(func() error {
+			v, err := resolveLatestVersion(buildDir)
+			if err != nil {
+				return err
+			}
+			prevVersion = v
+			p, err := DownloadCLI(buildDir, v)
+			cliPrevPath = p
+			return err
+		})
+		require.NoError(t, g.Wait())
+
 		t.Setenv("CLI_293", cli293Path)
 		repls.SetPath(cli293Path, "[CLI_293]")
+
+		t.Setenv("CLI_PREV", cliPrevPath)
+		repls.SetPath(cliPrevPath, "[CLI_PREV]")
+		repls.Set(prevVersion, "[CLI_PREV_VERSION]")
 	}
 
 	paths := []string{
@@ -1226,39 +1258,53 @@ func CreateReleaseArtifact(t *testing.T, cwd, releasesDir, coverDir, osName, arc
 
 // resolveLatestVersion returns the latest released CLI version (e.g. "0.293.0"),
 // using a file-based cache in buildDir valid for 1 hour.
-func resolveLatestVersion(t *testing.T, buildDir string) string {
+func resolveLatestVersion(buildDir string) (string, error) {
 	cachePath := filepath.Join(buildDir, "latest_version.txt")
 	if info, err := os.Stat(cachePath); err == nil && time.Since(info.ModTime()) < time.Hour {
 		data, err := os.ReadFile(cachePath)
-		require.NoError(t, err)
+		if err != nil {
+			return "", err
+		}
 		if version := strings.TrimSpace(string(data)); version != "" {
-			return version
+			return version, nil
 		}
 	}
 
 	const url = "https://api.github.com/repos/databricks/cli/releases/latest"
 	resp, err := http.Get(url)
-	require.NoError(t, err)
+	if err != nil {
+		return "", err
+	}
 	defer resp.Body.Close()
-	require.Equal(t, http.StatusOK, resp.StatusCode, "failed to fetch %s: %s", url, resp.Status)
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("failed to fetch %s: %s", url, resp.Status)
+	}
 
 	var release struct {
 		TagName string `json:"tag_name"`
 	}
-	require.NoError(t, json.NewDecoder(resp.Body).Decode(&release))
+	if err := json.NewDecoder(resp.Body).Decode(&release); err != nil {
+		return "", err
+	}
 	version := strings.TrimPrefix(release.TagName, "v")
-	require.NotEmpty(t, version, "empty tag_name in GitHub latest release response")
+	if version == "" {
+		return "", errors.New("empty tag_name in GitHub latest release response")
+	}
 
-	require.NoError(t, os.WriteFile(cachePath, []byte(version), 0o644))
-	return version
+	if err := os.WriteFile(cachePath, []byte(version), 0o644); err != nil {
+		return "", err
+	}
+	return version, nil
 }
 
 // DownloadCLI downloads a released CLI binary archive for the given version,
 // extracts the executable, and returns its path.
-func DownloadCLI(t *testing.T, buildDir, version string) string {
+func DownloadCLI(buildDir, version string) (string, error) {
 	// Prepare target directory for this version
 	versionDir := filepath.Join(buildDir, version)
-	require.NoError(t, os.MkdirAll(versionDir, 0o755))
+	if err := os.MkdirAll(versionDir, 0o755); err != nil {
+		return "", err
+	}
 
 	execName := "databricks"
 	if runtime.GOOS == "windows" {
@@ -1268,7 +1314,7 @@ func DownloadCLI(t *testing.T, buildDir, version string) string {
 
 	// If already downloaded, reuse
 	if _, err := os.Stat(execPath); err == nil {
-		return execPath
+		return execPath, nil
 	}
 
 	osName := runtime.GOOS
@@ -1279,55 +1325,85 @@ func DownloadCLI(t *testing.T, buildDir, version string) string {
 	url := fmt.Sprintf("https://github.com/databricks/cli/releases/download/v%s/%s", version, archiveName)
 	zipPath := filepath.Join(versionDir, archiveName)
 
-	downloadToFile(t, url, zipPath)
-	extractFileFromZip(t, zipPath, execName, versionDir)
+	if err := downloadToFile(url, zipPath); err != nil {
+		return "", err
+	}
+	if err := extractFileFromZip(zipPath, execName, versionDir); err != nil {
+		return "", err
+	}
 
-	return execPath
+	return execPath, nil
 }
 
 // downloadToFile downloads contents from url into the given destination path
-func downloadToFile(t *testing.T, url, destPath string) {
-	require.NoError(t, os.MkdirAll(filepath.Dir(destPath), 0o755))
+func downloadToFile(url, destPath string) error {
+	if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
+		return err
+	}
 	out, err := os.Create(destPath)
-	require.NoError(t, err)
+	if err != nil {
+		return err
+	}
 	defer out.Close()
 
 	resp, err := http.Get(url)
-	require.NoError(t, err)
+	if err != nil {
+		return err
+	}
 	defer resp.Body.Close()
-	require.Equal(t, http.StatusOK, resp.StatusCode, "failed to download %s: %s", url, resp.Status)
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to download %s: %s", url, resp.Status)
+	}
 
 	_, err = io.Copy(out, resp.Body)
-	require.NoError(t, err)
+	return err
 }
 
 // extractFileFromZip finds a file by name inside a zip archive and writes it
-// into destDir using the file's base name. Fails the test if not found.
-func extractFileFromZip(t *testing.T, zipPath, fileName, destDir string) {
+// into destDir using the file's base name. The destination is written atomically
+// (temp file + rename) so a failed extract can't leave a truncated binary that
+// DownloadCLI would later accept via os.Stat.
+func extractFileFromZip(zipPath, fileName, destDir string) error {
 	r, err := zip.OpenReader(zipPath)
-	require.NoError(t, err)
+	if err != nil {
+		return err
+	}
 	defer r.Close()
 
 	targetBase := filepath.Base(fileName)
 	destPath := filepath.Join(destDir, targetBase)
-	var found bool
 	for _, f := range r.File {
 		if filepath.Base(f.Name) != targetBase {
 			continue
 		}
 		rc, err := f.Open()
-		require.NoError(t, err)
-		defer rc.Close()
-
-		out, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o755)
-		require.NoError(t, err)
-		_, err = io.Copy(out, rc)
-		_ = out.Close()
-		require.NoError(t, err)
-		found = true
-		break
+		if err != nil {
+			return err
+		}
+		tmp, err := os.CreateTemp(destDir, targetBase+".*.tmp")
+		if err != nil {
+			rc.Close()
+			return err
+		}
+		tmpPath := tmp.Name()
+		_, copyErr := io.Copy(tmp, rc)
+		closeErr := tmp.Close()
+		rc.Close()
+		if copyErr != nil || closeErr != nil {
+			os.Remove(tmpPath)
+			return errors.Join(copyErr, closeErr)
+		}
+		if err := os.Chmod(tmpPath, 0o755); err != nil {
+			os.Remove(tmpPath)
+			return err
+		}
+		if err := os.Rename(tmpPath, destPath); err != nil {
+			os.Remove(tmpPath)
+			return err
+		}
+		return nil
 	}
-	require.True(t, found, "file %s not found in archive %s", targetBase, zipPath)
+	return fmt.Errorf("file %s not found in archive %s", targetBase, zipPath)
 }
 
 func copyFile(src, dst string) error {

acceptance/bundle/invariant/continue_prev/output.txt

@@ -0,0 +1,4 @@
+
+>>> [CLI_PREV] --version
+Databricks CLI v[CLI_PREV_VERSION]
+INPUT_CONFIG_OK

acceptance/bundle/invariant/continue_prev/script

@@ -0,0 +1,41 @@
+# Invariant to test: current CLI can deploy on top of state produced by the previous release
+
+cp -r "$TESTDIR/../data/." . &> LOG.cp
+
+INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh"
+if [ -f "$INIT_SCRIPT" ]; then
+    source "$INIT_SCRIPT" &> LOG.init
+fi
+
+envsubst < "$TESTDIR/../configs/$INPUT_CONFIG" > databricks.yml
+
+cleanup() {
+    $CLI bundle destroy --auto-approve &> LOG.destroy
+    cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null
+
+    CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh"
+    if [ -f "$CLEANUP_SCRIPT" ]; then
+        source "$CLEANUP_SCRIPT" &> LOG.cleanup
+    fi
+}
+
+trap cleanup EXIT
+
+# Deploy with previous released CLI to produce previous-version state
+trace $CLI_PREV --version
+$CLI_PREV bundle deploy &> LOG.deploy.prev
+cat LOG.deploy.prev | contains.py '!panic' '!internal error' > /dev/null
+
+echo INPUT_CONFIG_OK
+
+# Deploy with current CLI on top of previous state
+$CLI bundle deploy &> LOG.deploy
+cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null
+
+# Verify no drift after current CLI deploy
+$CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err
+cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null
+verify_no_drift.py LOG.planjson
+
+$CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan
+cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null

acceptance/bundle/invariant/continue_prev/test.toml

@@ -0,0 +1,13 @@
+# Computed volume_path was added after the previous release (#5550), so the
+# seed deploy with the previous CLI cannot resolve ${resources.volumes.*.volume_path}.
+# Covered by no_drift on direct.
+EnvMatrixExclude.no_volume_path_job_ref = ["INPUT_CONFIG=volume_path_job_ref.yml.tmpl"]
+
+# postgres_project.yml.tmpl sets enable_pg_native_login: false, which triggers a
+# state serialization bug on the previous CLI (fixed by #5782).
+# Broken on CLI_PREV=1.6.0; remove after the next release.
+EnvMatrixExclude.no_postgres_project = ["INPUT_CONFIG=postgres_project.yml.tmpl"]
+
+# The 1000-task scale case is covered by no_drift. Running it here adds ~1.5 min
+# per variant (two full deploys at 1000 tasks) without incremental coverage.
+EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"]