feat: track applied migrations as a list instead of a scalar schema_version

Makefile

@@ -1,6 +1,6 @@
 SHELL = /bin/sh
 
-VERSION=1.9.0
+VERSION=2.0.0
 BUILD=`git rev-parse HEAD`
 
 LDFLAGS=-ldflags "-w -s \

UPGRADING.md

@@ -0,0 +1,100 @@
+# Upgrading testtrack-cli
+
+## 1.x to 2.0.0
+
+2.0.0 changes the on-disk schema format (`testtrack/schema.{yml,json}`).
+
+### What changed
+
+The schema's applied-migration high-water mark used to be a single scalar
+field:
+
+```yaml
+serializer_version: 1
+schema_version: "2020011543200"
+```
+
+Every new migration rewrote that one line, so any two branches that added
+migrations conflicted on it. 2.0.0 replaces the scalar with a list of every
+applied migration version (`serializer_version` bumps to `2`):
+
+```yaml
+serializer_version: 2
+schema_versions:
+- "2020060151730"
+- "2020011543200"
+```
+
+The list is ordered by a hash of each version (not chronologically), so
+concurrently added migrations scatter through the file instead of clustering
+on one line. The ordering is otherwise meaningless — don't rely on it.
+
+### This is a breaking format change
+
+A 2.0.0 CLI **refuses to read** an old (`serializer_version: 1`) schema file.
+Any command that reads the schema errors and tells you to run `testtrack schema
+upgrade` — `create`, `migrate`, `sync`, `decide`, the `destroy` commands, and
+`schema load` (essentially everything except `assign`/`unassign`, which use a
+different read path). This is deliberate: the old format can't be losslessly
+read into the new struct, so the CLI makes you convert it explicitly rather than
+silently round-tripping a lossy upgrade.
+
+Conversely, a pre-2.0.0 CLI does **not** understand `schema_versions`. If it
+reads a v2 schema it ignores the field, and any command that rewrites the
+schema produces a hybrid: the file keeps `serializer_version: 2` (the old CLI
+round-trips whatever number it read) but is otherwise back in the v1 shape — a
+scalar `schema_version`, no `schema_versions` list. A 2.0.0 CLI detects the
+leftover `schema_version` field, refuses to read the hybrid, and tells you to
+run `testtrack schema upgrade`, which repairs it by rebuilding the version list
+from `testtrack/migrate`. So the whole team — and every CI/build environment —
+has to be on 2.0.0 before you commit a v2 schema, or the format will flip-flop
+between commits.
+
+Because a 2.0.0 CLI rejects a v1 schema and a 1.x CLI mangles a v2 schema, the
+switch is a flag day: the moment the v2 schema lands in the repo, every consumer
+of it must already be on 2.0.0. Order the rollout so the CLI bump and the schema
+commit line up, not in separate steps.
+
+1. **Make 2.0.0 available, but don't switch anything to it yet.** Cut the
+   `v2.0.0` release, update the brew formula, and make the pinned binary
+   available to CI — without yet changing the CI pin or asking devs to upgrade.
+
+2. **Land the schema conversion and the CI bump together, in one change.** In
+   your app root:
+
+   ```sh
+   testtrack schema upgrade
+   ```
+
+   This converts `testtrack/schema.{yml,json}` to the v2 format **in place**: it
+   keeps the materialized state already in the file (splits, decisions,
+   retirements, etc.) and only rebuilds the `schema_versions` list from the
+   migration filenames in `testtrack/migrate`. In the **same** PR, bump the
+   CI/build pin to `v2.0.0`. They must be atomic: if CI runs 2.0.0 against a
+   still-v1 schema, `testtrack migrate` errors; if the v2 schema lands while CI
+   still runs 1.x, the old binary rewrites it into the hybrid described above,
+   and every 2.0.0 user is blocked until someone re-runs `schema upgrade`.
+   Expect a large
+   one-time diff — the new `schema_versions` block lists every migration — but
+   the splits body is left untouched.
+
+3. **Developers upgrade as that change lands** (`brew upgrade testtrack-cli`). A
+   developer must be on 2.0.0 before running any testtrack command against the
+   upgraded schema — an older binary silently rewrites it into the hybrid,
+   which 2.0.0 CLIs then refuse to read until it's repaired with
+   `schema upgrade`.
+
+### Notes
+
+- **Use `schema upgrade`, not `schema generate`, to convert.** `generate`
+  rebuilds the schema by replaying every migration from scratch, which fails on
+  many long-lived apps — e.g. when `testtrack/migrate` predates some splits, or
+  contains a decision/retirement for a split that was created out of band in the
+  TestTrack admin (so there's no create migration to replay). `upgrade` doesn't
+  replay; it preserves the already-correct materialized state and just restamps
+  the format, so it works regardless.
+- `schema upgrade` reads the old file directly (it's the one command that
+  bypasses the version guard), so it's always the recovery path if you hit the
+  "older schema format" error.
+- No server-side change is required. The TestTrack server tracks applied
+  migrations itself; `schema_version`/`schema_versions` is a CLI-only artifact.

cmds/schema_upgrade.go

@@ -0,0 +1,40 @@
+package cmds
+
+import (
+	"github.com/Betterment/testtrack-cli/schema"
+	"github.com/spf13/cobra"
+)
+
+var schemaUpgradeDoc = `
+Upgrades testtrack/schema.{json,yml} to the schema format this CLI writes,
+in place, without replaying migrations.
+
+It keeps the materialized state already in the file (splits, decisions,
+retirements, feature completions, identifier types) and only rebuilds the
+applied-migration version list from the files in testtrack/migrate. This is the
+upgrade path to use when a newer CLI refuses to read an older-format schema.
+
+Unlike 'schema generate', upgrade does not rebuild the schema from scratch, so
+it works on apps whose migrations can't be replayed cleanly - e.g. when
+testtrack/migrate predates some splits, or references splits that were created
+out of band in the TestTrack admin.
+`
+
+func init() {
+	schemaCmd.AddCommand(schemaUpgradeCmd)
+}
+
+var schemaUpgradeCmd = &cobra.Command{
+	Use:   "upgrade",
+	Short: "Upgrade schema.{json,yml} to the current format in place",
+	Long:  schemaUpgradeDoc,
+	Args:  cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, _ []string) error {
+		return schemaUpgrade()
+	},
+}
+
+func schemaUpgrade() error {
+	_, err := schema.Upgrade()
+	return err
+}

fakeserver/server_test.go

@@ -20,8 +20,9 @@ import (
 )
 
 var testSchema = `
-serializer_version: 1
-schema_version: "2020011774023"
+serializer_version: 2
+schema_versions:
+- "2020011774023"
 splits:
 - name: test.test_experiment
   weights:
@@ -34,8 +35,8 @@ splits:
 `
 
 var otherTestSchema = `{
-  "serializer_version": 1,
-  "schema_version": "2020011774023",
+  "serializer_version": 2,
+  "schema_versions": ["2020011774023"],
   "splits": [
     {
       "name": "test.json_experiment",

featurecompletions/featurecompletions.go

@@ -67,7 +67,7 @@ func (f *FeatureCompletion) Filename() *string {
 // File returns a serializable MigrationFile for this migration
 func (f *FeatureCompletion) File() *serializers.MigrationFile {
 	return &serializers.MigrationFile{
-		SerializerVersion: serializers.SerializerVersion,
+		SerializerVersion: serializers.MigrationSerializerVersion,
 		FeatureCompletion: f.serializable(),
 	}
 }

identifiertypes/identifiertypes.go

@@ -54,7 +54,7 @@ func (i *IdentifierType) Filename() *string {
 // File returns a serializable MigrationFile for this migration
 func (i *IdentifierType) File() *serializers.MigrationFile {
 	return &serializers.MigrationFile{
-		SerializerVersion: serializers.SerializerVersion,
+		SerializerVersion: serializers.MigrationSerializerVersion,
 		IdentifierType:    i.serializable(),
 	}
 }

migrationloaders/migrationloaders.go

@@ -17,25 +17,65 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-// Load loads a set of migrations
-func Load() (migrations.Repository, error) {
+// Filenames lists the migration filenames in testtrack/migrate, skipping
+// hidden files and directories. It is the single source of truth for which
+// directory entries count as migrations — Load and `schema upgrade` must
+// agree on that, or the schema's recorded versions drift from what the
+// loader sees.
+func Filenames() ([]string, error) {
 	files, err := os.ReadDir("testtrack/migrate")
 	if err != nil {
 		return nil, err
 	}
-
-	migrationRepo := make(migrations.Repository)
+	filenames := make([]string, 0, len(files))
 	for _, file := range files {
-		if strings.HasPrefix(file.Name(), ".") {
-			continue // Skip hidden files
+		if file.IsDir() || strings.HasPrefix(file.Name(), ".") {
+			continue
+		}
+		filenames = append(filenames, file.Name())
+	}
+	return filenames, nil
+}
+
+// Versions returns the unique migration versions recorded in the filenames in
+// testtrack/migrate, without reading file contents — usable even on repos
+// whose migrations can't be parsed or replayed.
+func Versions() ([]string, error) {
+	filenames, err := Filenames()
+	if err != nil {
+		return nil, err
+	}
+	seen := make(map[string]bool, len(filenames))
+	versions := make([]string, 0, len(filenames))
+	for _, filename := range filenames {
+		version, err := migrations.ExtractVersionFromFilename(filename)
+		if err != nil {
+			return nil, fmt.Errorf("%w - delete or rename it if it isn't a testtrack migration", err)
+		}
+		if seen[version] {
+			continue
 		}
+		seen[version] = true
+		versions = append(versions, version)
+	}
+	return versions, nil
+}
 
-		migrationVersion, err := migrations.ExtractVersionFromFilename(file.Name())
+// Load loads a set of migrations
+func Load() (migrations.Repository, error) {
+	filenames, err := Filenames()
+	if err != nil {
+		return nil, err
+	}
+
+	migrationRepo := make(migrations.Repository)
+	for _, filename := range filenames {
+		migrationVersion, err := migrations.ExtractVersionFromFilename(filename)
 		if err != nil {
 			return nil, err
 		}
 
-		fileBytes, err := os.ReadFile(path.Join("testtrack/migrate", file.Name()))
+		fileBytes, err := os.ReadFile(path.Join("testtrack/migrate", filename))
 		if err != nil {
 			return nil, err
 		}
@@ -62,7 +102,7 @@ func Load() (migrations.Repository, error) {
 		} else if migrationFile.IdentifierType != nil {
 			migrationRepo[migrationVersion] = identifiertypes.FromFile(&migrationVersion, migrationFile.IdentifierType)
 		} else {
-			return nil, fmt.Errorf("testtrack/migrate/%s didn't match a known migration type", file.Name())
+			return nil, fmt.Errorf("testtrack/migrate/%s didn't match a known migration type", filename)
 		}
 	}
 	return migrationRepo, nil

migrationmanagers/migrationmanagers.go

@@ -94,8 +94,8 @@ func (m *MigrationManager) ApplyToSchema(migrationRepo migrations.Repository, id
 	}
 
 	appliedVersion := m.migration.MigrationVersion()
-	if appliedVersion != nil && m.schema.SchemaVersion < *appliedVersion {
-		m.schema.SchemaVersion = *appliedVersion
+	if appliedVersion != nil {
+		m.schema.AddVersion(*appliedVersion)
 	}
 	return nil
 }
@@ -160,10 +160,5 @@ func (m *MigrationManager) SyncVersion() error {
 		return fmt.Errorf("got %d status code", resp.StatusCode)
 	}
 
-	appliedVersion := m.migration.MigrationVersion()
-	if m.schema.SchemaVersion < *appliedVersion {
-		m.schema.SchemaVersion = *appliedVersion
-	}
-
 	return nil
 }

remotekills/remotekills.go

@@ -91,7 +91,7 @@ func (r *RemoteKill) Filename() *string {
 // File returns a serializable MigrationFile for this migration
 func (r *RemoteKill) File() *serializers.MigrationFile {
 	return &serializers.MigrationFile{
-		SerializerVersion: serializers.SerializerVersion,
+		SerializerVersion: serializers.MigrationSerializerVersion,
 		RemoteKill:        r.serializable(),
 	}
 }