diff --git a/evals/flag-and-release-change/promptfooconfig.yaml b/evals/flag-and-release-change/promptfooconfig.yaml index 4e64827..2a0d5cb 100644 --- a/evals/flag-and-release-change/promptfooconfig.yaml +++ b/evals/flag-and-release-change/promptfooconfig.yaml @@ -266,3 +266,137 @@ tests: Score 1.0 if both. Deduct 0.5 per criterion missed. metric: rationale_quality weight: 2 + + # ================================================================== + # POLICY PREVIEW BY flagTags — in the plan phase the flag doesn't exist + # yet, so the release preview must call match-release-policies by the + # proposed flagTags (NOT a flagKey), for every planned environment. It + # must still create/record nothing. (EMSR-1930) + # ================================================================== + - description: "Plan phase previews each environment's policy by flagTags before the flag exists" + providers: ["orchestrator"] + vars: + user_request: > + PLAN how to flag and release this change — do NOT create the flag or record + anything yet. Project `default`; target environments `staging` and + `production`. Include the per-environment release plan grounded in the + configured release policies. + codebase_context: > + Node.js Express API using the LaunchDarkly Node server SDK + (client.variation). Routes live in src/routes/. The PR is checked out + locally; the diff is below. + git_diff: | + diff --git a/src/routes/export.js b/src/routes/export.js + new file mode 100644 + index 0000000..7de4a11 + --- /dev/null + +++ b/src/routes/export.js + @@ -0,0 +1,10 @@ + +const express = require('express'); + +const router = express.Router(); + +const { buildExport } = require('../services/exporter'); + + + +router.post('/', async (req, res) => { + + const file = await buildExport(req.user, req.body.format || 'csv'); + + res.type('text/csv').send(file); + +}); + + + +module.exports = router; + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const calls = traj.filter(t => t.tool === 'match-release-policies'); + if (calls.length === 0) return { pass: false, score: 0, reason: 'never called match-release-policies' }; + // Pre-creation preview must use flagTags, not a (nonexistent) flagKey. + const byTags = calls.filter(c => Array.isArray(c.arguments && c.arguments.flagTags) && !c.arguments.flagKey); + const pass = byTags.length === calls.length; + return { pass, score: pass ? 1 : 0, reason: `match-release-policies calls=${calls.length}, by-flagTags(no flagKey)=${byTags.length}` }; + metric: previewed_by_flagtags + weight: 3 + + - type: javascript + value: | + const traj = output.trajectory || []; + const envs = new Set(traj.filter(t => t.tool === 'match-release-policies') + .map(c => c.arguments && c.arguments.environmentKey).filter(Boolean)); + const pass = envs.has('staging') && envs.has('production'); + return { pass, score: pass ? 1 : 0, reason: `previewed envs=[${[...envs].join(', ')}] (want staging + production)` }; + metric: previewed_each_env + weight: 3 + + - type: javascript + value: | + const tools = output.tools_called || []; + const forbidden = ['create-flag','create-feature-flag','create-automated-rollout-config','update-flag-settings','update-feature-flag','toggle-flag','delete-flag']; + const called = forbidden.filter(f => tools.includes(f)); + const pass = called.length === 0; + return { pass, score: pass ? 1 : 0, reason: pass ? 'no mutations during planning' : 'mutated during plan: ' + called.join(', ') }; + metric: no_mutation_in_plan + weight: 2 + + # ================================================================== + # RE-RUN ON REFINEMENT — when the requested environment set changes, + # policy matching must run for the newly added environment too, not just + # the originally named ones. Still side-effect-free. (EMSR-1930) + # ================================================================== + - description: "A changed environment set re-runs policy matching for the added environment" + providers: ["orchestrator"] + vars: + user_request: > + PLAN how to flag and release this change — create/record nothing yet. Project + `default`, environments `staging` and `production`. On reflection, also add + `eu-production` to the release plan and preview what each environment's policy + would do. + codebase_context: > + Node.js Express API using the LaunchDarkly Node server SDK + (client.variation). Routes live in src/routes/. The PR is checked out + locally; the diff is below. + git_diff: | + diff --git a/src/routes/export.js b/src/routes/export.js + new file mode 100644 + index 0000000..7de4a11 + --- /dev/null + +++ b/src/routes/export.js + @@ -0,0 +1,10 @@ + +const express = require('express'); + +const router = express.Router(); + +const { buildExport } = require('../services/exporter'); + + + +router.post('/', async (req, res) => { + + const file = await buildExport(req.user, req.body.format || 'csv'); + + res.type('text/csv').send(file); + +}); + + + +module.exports = router; + assert: + - type: javascript + value: | + const traj = output.trajectory || []; + const envs = new Set(traj.filter(t => t.tool === 'match-release-policies') + .map(c => c.arguments && c.arguments.environmentKey).filter(Boolean)); + // The refined/added environment must be matched too. + const pass = envs.has('eu-production'); + return { pass, score: pass ? 1 : 0, reason: `previewed envs=[${[...envs].join(', ')}] (must include the added eu-production)` }; + metric: rematched_added_env + weight: 3 + + - type: javascript + value: | + const traj = output.trajectory || []; + const envs = new Set(traj.filter(t => t.tool === 'match-release-policies') + .map(c => c.arguments && c.arguments.environmentKey).filter(Boolean)); + const pass = envs.has('staging') && envs.has('production') && envs.has('eu-production'); + return { pass, score: pass ? 1 : 0, reason: `previewed envs=[${[...envs].join(', ')}] (want all three)` }; + metric: previewed_full_set + weight: 3 + + - type: javascript + value: | + const tools = output.tools_called || []; + const forbidden = ['create-flag','create-feature-flag','create-automated-rollout-config','update-flag-settings','update-feature-flag','toggle-flag','delete-flag']; + const called = forbidden.filter(f => tools.includes(f)); + const pass = called.length === 0; + return { pass, score: pass ? 1 : 0, reason: pass ? 'no mutations during planning' : 'mutated during plan: ' + called.join(', ') }; + metric: no_mutation_in_plan + weight: 2 diff --git a/evals/flag-release/promptfooconfig.yaml b/evals/flag-release/promptfooconfig.yaml index 587e808..603b9ea 100644 --- a/evals/flag-release/promptfooconfig.yaml +++ b/evals/flag-release/promptfooconfig.yaml @@ -354,3 +354,92 @@ tests: Score 1.0 if all three. Deduct 0.34 per criterion missed. metric: rationale_quality weight: 3 + + # ================================================================== + # MISSING POLICY — the environment has no matching release policy. The + # mock's project `no-release-policies` makes match-release-policies + # return a missing_policy warning. The skill must SURFACE the warning + # and NOT present/record production as a clean `policy` rollout, nor + # invent a fallback. (EMSR-1930) + # ================================================================== + - description: "Missing policy: surfaces the missing_policy warning, does not record an unbacked policy rollout" + providers: ["mcp"] + vars: + user_request: > + The flag `reporting-export` already exists (created OFF) for PR acme/data#12, + in project `no-release-policies`. Record its automated release for + `production`, releasing on merge. + assert: + - type: javascript + value: | + const tools = output.tools_called || []; + const pass = tools.includes('match-release-policies'); + return { pass, score: pass ? 1 : 0, reason: pass ? 'previewed policy' : 'did not call match-release-policies' }; + metric: previewed_policy + weight: 2 + + - type: javascript + value: | + // Must not present a clean `policy` rollout the policy can't back: either + // record nothing (warned and asked), or at least not record production as + // releaseType `policy`. + const traj = output.trajectory || []; + const rec = traj.find(t => t.tool === 'create-automated-rollout-config'); + if (!rec) return { pass: true, score: 1, reason: 'stopped without recording (warned and asked)' }; + const prod = ((rec.arguments && rec.arguments.environments) || []).find(e => e.environmentKey === 'production'); + const pass = !prod || prod.releaseType !== 'policy'; + return { pass, score: pass ? 1 : 0, reason: prod ? `recorded production as releaseType=${prod.releaseType}` : 'production not recorded' }; + metric: no_unbacked_policy_release + weight: 4 + + - type: llm-rubric + value: | + The environment `production` has NO matching release policy — + match-release-policies returned a `missing_policy` warning. + Evaluate the response: + 1. Does it SURFACE that no release policy matches, rather than silently + proceeding? + 2. Does it STOP for clarification / offer an alternative (e.g. `simple`, or + setting up a policy) instead of claiming a clean `policy` rollout or + inventing a fallback rollout? + 3. Did it avoid turning the flag on by hand? + Score 1.0 if all three. Deduct 0.34 per criterion missed. + metric: rationale_quality + weight: 3 + + # ================================================================== + # INCOMPLETE POLICY — a policy matches but can't govern a rollout as-is + # (guarded with no stages). The mock's project `incomplete-policy` + # returns an incomplete_policy warning. The skill must surface it and + # NOT claim a clean guarded rollout. (EMSR-1930) + # ================================================================== + - description: "Incomplete policy: surfaces the incomplete_policy warning, does not claim a clean guarded rollout" + providers: ["mcp"] + vars: + user_request: > + The flag `pricing-v2` already exists (created OFF) for PR acme/store#77, in + project `incomplete-policy`. Record its automated release for `production`, + releasing on merge. + assert: + - type: javascript + value: | + const tools = output.tools_called || []; + const pass = tools.includes('match-release-policies'); + return { pass, score: pass ? 1 : 0, reason: pass ? 'previewed policy' : 'did not call match-release-policies' }; + metric: previewed_policy + weight: 2 + + - type: llm-rubric + value: | + The `production` release policy MATCHED but is INCOMPLETE — it is guarded + with no stages, and match-release-policies returned an `incomplete_policy` + warning (missing: stages). + Evaluate the response: + 1. Does it SURFACE the incomplete-policy warning (a guarded policy that can't + run as-is), rather than describing a clean guarded rollout? + 2. Does it stop / recommend fixing the policy or using `simple` or an explicit + override, instead of inventing the missing stages? + 3. Did it avoid turning the flag on by hand? + Score 1.0 if all three. Deduct 0.34 per criterion missed. + metric: rationale_quality + weight: 4 diff --git a/evals/mocks/tool-responses.json b/evals/mocks/tool-responses.json index 8896ba0..e46a7ca 100644 --- a/evals/mocks/tool-responses.json +++ b/evals/mocks/tool-responses.json @@ -512,7 +512,8 @@ "winningPolicy": { "key": "guarded-default", "name": "Guarded rollout" }, "winningReleaseMethod": "guarded", "autoAttachedMetricKeys": ["error-rate", "latency-p95"], - "autoAttachedMetricGroupKeys": [] + "autoAttachedMetricGroupKeys": [], + "warnings": [] }, "list-release-policies": { "policies": [ diff --git a/evals/providers/_mock.js b/evals/providers/_mock.js index e804f6a..a9a4613 100644 --- a/evals/providers/_mock.js +++ b/evals/providers/_mock.js @@ -369,6 +369,92 @@ function renderMockResponse(template, input, toolName, state) { return flag; } + // ---------- release-policy matching hooks ---------- + // match-release-policies is a read-only preview of which policy governs a + // flag/tags in an environment. Its `warnings` list is what tells the skill to + // stop instead of presenting a clean `policy` rollout, so evals need to drive + // the missing/incomplete/method cases. Keyed on `projectKey` (mirrors the + // create-flag `restricted` hook): a magic project name selects the scenario, + // and the default project returns the complete guarded winner from the + // static template. + if (toolName === "match-release-policies") { + const env = input.environmentKey || "production"; + const project = String(input.projectKey || ""); + const criteria = { + projectKey: project || "default", + environmentKey: env, + flagKey: input.flagKey || null, + flagTags: input.flagTags || null, + tagsSource: input.flagKey ? "flagKey" : (input.flagTags ? "flagTags" : "none"), + }; + + // A project with no configured release policies → nothing matches. + if (/no-?release-?polic/i.test(project)) { + return { + matchingPolicies: [], + winningPolicy: null, + winningReleaseMethod: null, + autoAttachedMetricKeys: [], + autoAttachedMetricGroupKeys: [], + warnings: [ + { + code: "missing_policy", + environmentKey: env, + message: `No release policy matches environment "${env}". Do not record this environment as releaseType "policy" unless the user explicitly accepts manual follow-up.`, + }, + ], + criteria, + }; + } + + // A policy matches but can't govern a rollout as-is (guarded, no stages). + if (/incomplete-?polic/i.test(project)) { + const winner = { key: "guarded-incomplete", name: "Guarded (incomplete)", releaseMethod: "guarded" }; + return { + matchingPolicies: [winner], + winningPolicy: winner, + winningReleaseMethod: "guarded", + autoAttachedMetricKeys: [], + autoAttachedMetricGroupKeys: [], + warnings: [ + { + code: "incomplete_policy", + environmentKey: env, + policyKey: "guarded-incomplete", + policyName: "Guarded (incomplete)", + releaseMethod: "guarded", + missing: ["stages"], + message: `Release policy "Guarded (incomplete)" is guarded but missing stages. Fix the policy or use an explicit override before recording a policy-based automated rollout.`, + }, + ], + criteria, + }; + } + + // A complete progressive winner, for asserting the plan describes a + // progressive rollout accurately (not the default guarded winner). + if (/progressive/i.test(project)) { + const winner = { key: "progressive-default", name: "Progressive rollout", releaseMethod: "progressive" }; + return { + matchingPolicies: [winner], + winningPolicy: winner, + winningReleaseMethod: "progressive", + autoAttachedMetricKeys: [], + autoAttachedMetricGroupKeys: [], + warnings: [], + criteria, + }; + } + + // Default: the complete guarded winner from the static template. + const rendered = walk(template, replacements); + return { + ...rendered, + warnings: Array.isArray(rendered.warnings) ? rendered.warnings : [], + criteria, + }; + } + // Default: stateless template render return walk(template, replacements); } diff --git a/evals/tools/definitions.json b/evals/tools/definitions.json index 5166d9a..1aa9496 100644 --- a/evals/tools/definitions.json +++ b/evals/tools/definitions.json @@ -532,7 +532,7 @@ }, { "name": "match-release-policies", - "description": "Preview which release policy governs a flag in an environment and what a policy-based release would do. Before the flag exists pass flagTags for client-side matching; after it exists pass flagKey for the authoritative server-resolved policy. Returns winningPolicy, winningReleaseMethod (immediate|progressive|guarded|null), and any auto-attached metric/metric-group keys.", + "description": "Preview which release policy governs a flag in an environment and what a policy-based release would do. Before the flag exists pass flagTags for client-side matching; after it exists pass flagKey for the authoritative server-resolved policy. Returns winningPolicy, winningReleaseMethod (immediate|progressive|guarded|null), any auto-attached metric/metric-group keys, and a warnings array. A non-empty warnings array (code missing_policy when nothing matched, or incomplete_policy when the matched policy lacks required stages/metrics) means you must NOT present the environment as a clean policy rollout — surface the warning and stop for clarification.", "input_schema": { "type": "object", "properties": { diff --git a/skills.json b/skills.json index 5a035ee..cf50171 100644 --- a/skills.json +++ b/skills.json @@ -191,7 +191,7 @@ "name": "flag-and-release-change", "description": "Drive a pull request's change end to end: decide it's flag-worthy, create the guarding flag, wire the new code path behind it on the PR branch, and record an automated release so the change ships safely when the PR merges. A portable orchestrator that composes should-flag-change, launchdarkly-flag-create, and flag-release. Keywords: flag a PR, wrap change in a flag, dark launch, kill switch, auto-release, automated rollout, end-to-end flag workflow.", "path": "skills/feature-flags/flag-and-release-change", - "version": "0.1.0", + "version": "0.2.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server and a git CLI with access to the PR's repository", "tags": [ @@ -209,9 +209,9 @@ }, { "name": "flag-release", - "description": "Record an automated rollout for an existing LaunchDarkly flag that guards a pull request's change, so the change releases safely when the PR merges. Honors a stated release intent (release now / hold / notBefore / segment / prerequisite) and defers per-environment to the project's release policies. Use as the release step once the guarding flag exists and its code is wired. Keywords: record release, automated rollout, release policy, guarded rollout, staged rollout, simple vs policy, release intent, hold release, dark launch.", + "description": "Record an automated rollout for an existing LaunchDarkly flag that guards a pull request's change, so the change releases safely when the PR merges. Honors a stated release intent (release now / hold / notBefore / segment / prerequisite) and grounds each environment's rollout in the project's configured release policies via match-release-policies \u2014 never inventing methods, stages, or metrics, and stopping when a policy is missing or incomplete. Use as the release step once the guarding flag exists and its code is wired. Keywords: record release, automated rollout, release policy, match release policies, guarded rollout, staged rollout, simple vs policy, release intent, hold release, dark launch.", "path": "skills/feature-flags/flag-release", - "version": "0.1.0", + "version": "0.2.0", "license": "Apache-2.0", "compatibility": "Requires the remotely hosted LaunchDarkly MCP server. Operates on a flag that already exists; does not create flags or edit code.", "tags": [ diff --git a/skills/feature-flags/flag-and-release-change/SKILL.md b/skills/feature-flags/flag-and-release-change/SKILL.md index 07092ca..34da4ee 100644 --- a/skills/feature-flags/flag-and-release-change/SKILL.md +++ b/skills/feature-flags/flag-and-release-change/SKILL.md @@ -5,7 +5,7 @@ license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server and a git CLI with access to the PR's repository metadata: author: launchdarkly - version: "0.1.0" + version: "0.2.0" --- # Flag & Release a PR Change @@ -55,7 +55,7 @@ The three-dot diff (`base...head`) shows exactly what the PR introduces. Read th 1. **Confirm it should be flagged.** If [`should-flag-change`](../should-flag-change/SKILL.md) already ran, act on its verdict. Otherwise apply the same judgment: favor a flag for user-facing or risky changes; skip config-only, dependency-bump, infra, test-only, or docs changes. If a flag clearly isn't warranted, say so and stop. 2. **Understand the change and conventions.** Read the three-dot diff and changed files — what does it do, what's the blast radius? Then follow **flag-create's Step 1** to learn how this codebase already uses flags (SDK, wrapper, key constants, naming). Don't reinvent that exploration here. 3. **Design the flag.** Usually a single boolean kill-switch around the new path (flag-create's [flag-types](../launchdarkly-flag-create/references/flag-types.md) covers the choice). Don't propose more flags than the change needs. If Step 1 (or `should-flag-change`) surfaced a **dependency on a parent flag/feature that isn't live yet**, note it — the release step can couple them with a prerequisite. -4. **Plan the release.** Follow [`flag-release`](../flag-release/SKILL.md)'s plan phase: pick target environments, preview each with `match-release-policies`, and capture the human's **release intent** (release on merge / hold / `notBefore` / segment / prerequisite). Don't re-derive the rollout model here — that's flag-release's job. +4. **Plan the release.** Follow [`flag-release`](../flag-release/SKILL.md)'s plan phase: pick target environments and preview each with `match-release-policies`. The flag doesn't exist yet in this phase, so preview by the **proposed `flagTags`** (not `flagKey`). If a preview comes back with a `missing_policy` / `incomplete_policy` warning, surface it and ask how to proceed — don't invent a rollout. Capture the human's **release intent** (release on merge / hold / `notBefore` / segment / prerequisite). Don't re-derive the rollout model here — that's flag-release's job. 5. **Present the combined plan and stop.** Summarize: the flag (`key`, `name`, boolean, tags) and why it gates *this* change; where in the code the guard goes; the per-environment release plan + captured intent (and anything to be *held*). Then wait. Revise on feedback; proceed only on clear approval. Ask a focused question if you're genuinely missing something (project key, environments, a missing policy) rather than guessing. ## Implement Phase diff --git a/skills/feature-flags/flag-and-release-change/marketplace.json b/skills/feature-flags/flag-and-release-change/marketplace.json index 60b9733..1cf462c 100644 --- a/skills/feature-flags/flag-and-release-change/marketplace.json +++ b/skills/feature-flags/flag-and-release-change/marketplace.json @@ -1,7 +1,7 @@ { "name": "flag-and-release-change", "description": "End-to-end PR orchestrator: decide, create and wire the guarding flag, and record its automated release (composes should-flag-change, launchdarkly-flag-create, and flag-release)", - "version": "0.1.0", + "version": "0.2.0", "author": "LaunchDarkly", "repository": "https://github.com/launchdarkly/ai-tooling", "skills": ["./"], diff --git a/skills/feature-flags/flag-release/README.md b/skills/feature-flags/flag-release/README.md index ea68e3c..596e47c 100644 --- a/skills/feature-flags/flag-release/README.md +++ b/skills/feature-flags/flag-release/README.md @@ -6,7 +6,8 @@ An Agent Skill that records an automated rollout for an **existing** LaunchDarkl This skill teaches agents how to: - Confirm the guarding flag exists and is OFF -- Preview each environment's release policy with `match-release-policies` (immediate / progressive / guarded) +- Preview each environment's release policy with `match-release-policies` (immediate / progressive / guarded) — by `flagKey` once the flag exists, or by proposed `flagTags` before it does — grounding the plan in the configured policy rather than inventing methods, stages, or metrics +- Surface a `missing_policy` / `incomplete_policy` warning and **stop for clarification** instead of presenting a clean policy rollout when none is backed - Capture the human's **release intent** (release on merge / hold / `notBefore` / segment / prerequisite) and honor it — recording only the environments the intent clears, holding the rest - Record the rollout with `create-automated-rollout-config` diff --git a/skills/feature-flags/flag-release/SKILL.md b/skills/feature-flags/flag-release/SKILL.md index d2be60b..1b1265d 100644 --- a/skills/feature-flags/flag-release/SKILL.md +++ b/skills/feature-flags/flag-release/SKILL.md @@ -1,11 +1,11 @@ --- name: flag-release -description: "Record an automated rollout for an existing LaunchDarkly flag that guards a pull request's change, so the change releases safely when the PR merges. Honors a stated release intent (release now / hold / notBefore / segment / prerequisite) and defers per-environment to the project's release policies. Use as the release step once the guarding flag exists and its code is wired. Keywords: record release, automated rollout, release policy, guarded rollout, staged rollout, simple vs policy, release intent, hold release, dark launch." +description: "Record an automated rollout for an existing LaunchDarkly flag that guards a pull request's change, so the change releases safely when the PR merges. Honors a stated release intent (release now / hold / notBefore / segment / prerequisite) and grounds each environment's rollout in the project's configured release policies via match-release-policies — never inventing methods, stages, or metrics, and stopping when a policy is missing or incomplete. Use as the release step once the guarding flag exists and its code is wired. Keywords: record release, automated rollout, release policy, match release policies, guarded rollout, staged rollout, simple vs policy, release intent, hold release, dark launch." license: Apache-2.0 compatibility: Requires the remotely hosted LaunchDarkly MCP server. Operates on a flag that already exists; does not create flags or edit code. metadata: author: launchdarkly - version: "0.1.0" + version: "0.2.0" --- # Record a Flag's Automated Release @@ -34,7 +34,7 @@ By the time this skill runs, the flag exists (OFF) and the guarding code is wire **MCP tools this skill uses:** - `create-automated-rollout-config` — record the rollout for the flag against the PR *(the deliverable)* -- `match-release-policies` — resolve which release policy governs each environment (call before proposing the plan) +- `match-release-policies` — resolve which release policy governs each environment (call before proposing the plan; by `flagKey` once the flag exists, or by proposed `flagTags` before it does). Read its `warnings` — a `missing_policy` / `incomplete_policy` warning means **stop, don't fabricate a rollout**. - `list-release-policies` — see the project's release policies and the metrics they auto-attach - `get-flag` — confirm the flag exists and is OFF before recording @@ -46,9 +46,15 @@ Full release model — `simple` vs `policy`, precedence, previewing, prerequisit 1. **Confirm the flag.** `get-flag` to verify the guarding flag exists and is OFF. If it doesn't exist yet, stop — creation is [`launchdarkly-flag-create`](../launchdarkly-flag-create/SKILL.md)'s job, and recording a rollout for a missing flag fails confusingly. 2. **Pick the target environments.** Use the environments named by the user or harness. Don't hardcode a set — a given change can't always release to every environment. If none are named, enumerate the project's real keys and confirm the set rather than assuming. -3. **Preview each environment's policy.** Call `match-release-policies` (by `flagKey` + `environmentKey`) to resolve, deterministically, what a `policy` release will do per environment — `winningReleaseMethod` (immediate / progressive / guarded / none). Don't reason about policy scope by hand. For a **guarded** winner, check the auto-attached metrics can actually compare this change (see the metric-adequacy note in [references/auto-release.md](references/auto-release.md)). +3. **Preview each environment's policy — call `match-release-policies` for every environment you plan to release.** It resolves, deterministically, what a `policy` release will do per environment — `winningReleaseMethod` (immediate / progressive / guarded / none). Don't reason about policy scope, methods, stages, percentages, or metrics by hand — take them only from this tool. + - **Existing flag** (the usual case for this skill): pass `flagKey` + `environmentKey` for the authoritative, server-resolved policy. + - **Flag not created yet** (you're previewing inside the [`flag-and-release-change`](../flag-and-release-change/SKILL.md) plan phase, before the flag exists): pass the proposed `flagTags` + `environmentKey` instead — there's no flag key to resolve yet. Re-confirm with `flagKey` once the flag has been created. + - **Read the `warnings`.** A non-empty `warnings` list means `missing_policy` (nothing matched this environment) or `incomplete_policy` (e.g. a guarded/progressive policy with no stages, or a guarded policy with no metrics). When a warning is present, **do not present that environment as a clean `policy` rollout** — surface the warning and **stop for clarification** (step 5). Never fabricate a rollout or silently fall back to a default. + - For a **guarded** winner with no warnings, still confirm the auto-attached metrics can actually compare this change (see the metric-adequacy note in [references/auto-release.md](references/auto-release.md)). 4. **Capture the human's release intent.** Ask (briefly, only if not already stated): release **on merge**, **hold** (recorded but not released yet), or wait until a **`notBefore`** date? A **cohort/segment** to target first? A **prerequisite** parent flag this must not precede? Intent sits above the policy in precedence and is **honored or explicitly held — never silently dropped**. -5. **Present the per-environment plan and stop.** For each environment, state either the `releaseType` it will be recorded with (`simple` / `policy`, and what that does on merge) **or** that it will be **held** — omitted from the recorded config so the flag stays OFF there — with the reason. Wait for confirmation; revise on feedback. +5. **Present the per-environment plan and stop.** For each environment, state either the `releaseType` it will be recorded with (`simple` / `policy`, and what that does on merge, quoting the previewed `winningReleaseMethod` and any guarded metrics) **or** that it will be **held** — omitted from the recorded config so the flag stays OFF there — with the reason. If any environment came back with a `warnings` entry, present the warning and ask how to proceed rather than proposing a rollout for it. This plan is a **preview**: because you record `policy`, LaunchDarkly re-resolves the then-current policy at merge. Wait for confirmation; revise on feedback. + +**Re-run after any refinement.** If feedback changes the flag key, the flag's tags, or the environment set, call `match-release-policies` again for the affected environments before re-presenting — a stale preview can propose a rollout the current policy no longer produces. ## Implement Phase @@ -90,16 +96,19 @@ Only after confirmation: | Flag doesn't exist yet | Stop — creation is `launchdarkly-flag-create`. Recording a rollout for a missing flag fails confusingly. | | A rollout config already exists for this flag + PR | Don't record a second one — a duplicate confuses the scheduler. Point the user at the existing config to change the plan. | | Registering before the PR exists | `simple` envs work without a PR, but `policy` envs need `repoFullName`/`prNumber` to trigger on merge. Prefer recording *after* the PR is open; if you record early, say `policy` won't fire until the PR is wired. | -| No release policy matches an env | `policy` falls back to defaults (often immediate). Tell the user; offer `simple`, or point at release-policy setup. | +| No release policy matches an env (`missing_policy` warning) | Surface the warning and **stop for clarification** — do NOT record the env as `policy` and do NOT assume a default. Offer `simple`, or point at release-policy setup, and proceed only if the user explicitly accepts the manual follow-up. | +| A policy matches but is incomplete (`incomplete_policy` warning — e.g. guarded/progressive with no stages, or guarded with no metrics) | Surface the warning and stop; don't claim a clean rollout. Fix the policy, or use `simple` / an explicit override, before recording. | | User wants to hold, or set a `notBefore` date | Skip the releasing plan for those environments; report them as held with the reason. Never silently release against stated intent. | | Change depends on a parent flag not yet live | Couple them with a prerequisite (set it if the MCP surface supports it); otherwise report the coupling as a required manual step. Don't let this flag release before its parent. | -| A `policy` env resolves to guarded but has no relevant metric | Say so — a guarded rollout with no meaningful metric guards nothing. Recommend `simple`, or point at metric setup. | +| A `policy` env resolves to guarded but has no relevant metric | The tool flags this as an `incomplete_policy` warning; say so — a guarded rollout with no meaningful metric guards nothing. Recommend `simple`, or point at metric setup. | ## What NOT to Do - **Don't create the flag or edit code** — that's `launchdarkly-flag-create`. This skill only records the release. - **Don't turn the flag on yourself, or toggle it after recording the config.** The rollout owns that; double-toggling causes audit noise and confuses the scheduler. - **Don't skip `match-release-policies`.** Proposing `policy` without knowing what it resolves to is guessing. +- **Don't invent rollout details.** Methods, stages, durations, percentages, and metrics come from `match-release-policies` — never fill them in yourself. +- **Don't present a `policy` rollout when `match-release-policies` returned a warning.** A `missing_policy` / `incomplete_policy` warning means stop and clarify — surface it; never bury it or fabricate a rollout around it. - **Don't silently release against a stated hold/`notBefore`.** Honor intent or hold — never drop it. - **Don't handle or print credentials.** Access is injected by the environment. diff --git a/skills/feature-flags/flag-release/marketplace.json b/skills/feature-flags/flag-release/marketplace.json index a9085ed..ebc4ccb 100644 --- a/skills/feature-flags/flag-release/marketplace.json +++ b/skills/feature-flags/flag-release/marketplace.json @@ -1,7 +1,7 @@ { "name": "flag-release", - "description": "Record an automated rollout for an existing LaunchDarkly flag guarding a PR, honoring release intent and per-environment release policies", - "version": "0.1.0", + "description": "Record an automated rollout for an existing LaunchDarkly flag guarding a PR, honoring release intent and grounding each environment's rollout in the configured release policies (via match-release-policies) rather than inventing rollout details", + "version": "0.2.0", "author": "LaunchDarkly", "repository": "https://github.com/launchdarkly/ai-tooling", "skills": ["./"], diff --git a/skills/feature-flags/flag-release/references/auto-release.md b/skills/feature-flags/flag-release/references/auto-release.md index 75e347e..797f2b8 100644 --- a/skills/feature-flags/flag-release/references/auto-release.md +++ b/skills/feature-flags/flag-release/references/auto-release.md @@ -29,9 +29,15 @@ Always call `match-release-policies` before recommending a `policy` environment, - **Before the flag exists** — pass `projectKey`, `environmentKey`, and the proposed `flagTags`. This does client-side matching against the project's policies and previews the winner. - **After the flag exists** — pass `projectKey`, `environmentKey`, and `flagKey`. This hits the server-side release-settings endpoint and returns the authoritative resolved policy. -It returns the `winningPolicy`, the `winningReleaseMethod` (immediate / progressive / guarded), and any `autoAttachedMetricKeys` / `autoAttachedMetricGroupKeys`. Use `list-release-policies` to see every policy in the project and what each attaches. +It returns the `winningPolicy`, the `winningReleaseMethod` (immediate / progressive / guarded), any `autoAttachedMetricKeys` / `autoAttachedMetricGroupKeys`, and a `warnings` list. Use `list-release-policies` to see every policy in the project and what each attaches. -**If nothing matches**, `policy` falls back to project defaults (often an immediate release). Tell the user — they may want to pick `simple` instead, or set up a release policy first. +**Read `warnings` — they are a stop signal, not a fallback.** A non-empty `warnings` list means: +- `missing_policy` — nothing matched this environment. Do NOT assume a project default; surface the warning and stop for clarification. The user may want `simple` instead, or to set up a release policy first — proceed only if they accept the manual follow-up. +- `incomplete_policy` — a policy matched but can't govern a rollout as-is (e.g. a guarded/progressive policy with no stages, or a guarded policy with no metrics). Surface it and stop; don't claim a clean rollout. Fix the policy, or use `simple` / an explicit override. + +Never present an environment with warnings as a clean `policy` rollout, and never fabricate the missing detail. + +**Re-run on refinement.** The result is a preview of the *current* policy; `policy` re-resolves at merge. If a refinement changes the flag, its tags, or the environment set, call `match-release-policies` again for the affected environments — don't carry a stale preview forward. **A guarded release is only as good as its metrics.** If a `policy` env resolves to a **guarded** method, check that `autoAttachedMetricKeys` is non-empty and actually relevant to this change — a guarded rollout with no meaningful metric guards nothing. Watch for the **net-new path** case (from `should-flag-change`'s output, if present): when the flag-off control renders nothing, feature-specific before/after comparisons are "one-armed" and can't detect a regression, so a guarded release must lean on existing global/service metrics. If the attached metrics can't compare treatment vs. control for this change, say so and recommend `simple` for that env (or point the user at metric setup) rather than presenting a guarded rollout that can't actually guard.