Ground flag-release plans in matched release policies (EMSR-1930)

evals/flag-and-release-change/promptfooconfig.yaml

@@ -266,3 +266,137 @@ tests:
           Score 1.0 if both. Deduct 0.5 per criterion missed.
         metric: rationale_quality
         weight: 2
+
+  # ==================================================================
+  # POLICY PREVIEW BY flagTags — in the plan phase the flag doesn't exist
+  # yet, so the release preview must call match-release-policies by the
+  # proposed flagTags (NOT a flagKey), for every planned environment. It
+  # must still create/record nothing. (EMSR-1930)
+  # ==================================================================
+  - description: "Plan phase previews each environment's policy by flagTags before the flag exists"
+    providers: ["orchestrator"]
+    vars:
+      user_request: >
+        PLAN how to flag and release this change — do NOT create the flag or record
+        anything yet. Project `default`; target environments `staging` and
+        `production`. Include the per-environment release plan grounded in the
+        configured release policies.
+      codebase_context: >
+        Node.js Express API using the LaunchDarkly Node server SDK
+        (client.variation). Routes live in src/routes/. The PR is checked out
+        locally; the diff is below.
+      git_diff: |
+        diff --git a/src/routes/export.js b/src/routes/export.js
+        new file mode 100644
+        index 0000000..7de4a11
+        --- /dev/null
+        +++ b/src/routes/export.js
+        @@ -0,0 +1,10 @@
+        +const express = require('express');
+        +const router = express.Router();
+        +const { buildExport } = require('../services/exporter');
+        +
+        +router.post('/', async (req, res) => {
+        +  const file = await buildExport(req.user, req.body.format || 'csv');
+        +  res.type('text/csv').send(file);
+        +});
+        +
+        +module.exports = router;
+    assert:
+      - type: javascript
+        value: |
+          const traj = output.trajectory || [];
+          const calls = traj.filter(t => t.tool === 'match-release-policies');
+          if (calls.length === 0) return { pass: false, score: 0, reason: 'never called match-release-policies' };
+          // Pre-creation preview must use flagTags, not a (nonexistent) flagKey.
+          const byTags = calls.filter(c => Array.isArray(c.arguments && c.arguments.flagTags) && !c.arguments.flagKey);
+          const pass = byTags.length === calls.length;
+          return { pass, score: pass ? 1 : 0, reason: `match-release-policies calls=${calls.length}, by-flagTags(no flagKey)=${byTags.length}` };
+        metric: previewed_by_flagtags
+        weight: 3
+
+      - type: javascript
+        value: |
+          const traj = output.trajectory || [];
+          const envs = new Set(traj.filter(t => t.tool === 'match-release-policies')
+            .map(c => c.arguments && c.arguments.environmentKey).filter(Boolean));
+          const pass = envs.has('staging') && envs.has('production');
+          return { pass, score: pass ? 1 : 0, reason: `previewed envs=[${[...envs].join(', ')}] (want staging + production)` };
+        metric: previewed_each_env
+        weight: 3
+
+      - type: javascript
+        value: |
+          const tools = output.tools_called || [];
+          const forbidden = ['create-flag','create-feature-flag','create-automated-rollout-config','update-flag-settings','update-feature-flag','toggle-flag','delete-flag'];
+          const called = forbidden.filter(f => tools.includes(f));
+          const pass = called.length === 0;
+          return { pass, score: pass ? 1 : 0, reason: pass ? 'no mutations during planning' : 'mutated during plan: ' + called.join(', ') };
+        metric: no_mutation_in_plan
+        weight: 2
+
+  # ==================================================================
+  # RE-RUN ON REFINEMENT — when the requested environment set changes,
+  # policy matching must run for the newly added environment too, not just
+  # the originally named ones. Still side-effect-free. (EMSR-1930)
+  # ==================================================================
+  - description: "A changed environment set re-runs policy matching for the added environment"
+    providers: ["orchestrator"]
+    vars:
+      user_request: >
+        PLAN how to flag and release this change — create/record nothing yet. Project
+        `default`, environments `staging` and `production`. On reflection, also add
+        `eu-production` to the release plan and preview what each environment's policy
+        would do.
+      codebase_context: >
+        Node.js Express API using the LaunchDarkly Node server SDK
+        (client.variation). Routes live in src/routes/. The PR is checked out
+        locally; the diff is below.
+      git_diff: |
+        diff --git a/src/routes/export.js b/src/routes/export.js
+        new file mode 100644
+        index 0000000..7de4a11
+        --- /dev/null
+        +++ b/src/routes/export.js
+        @@ -0,0 +1,10 @@
+        +const express = require('express');
+        +const router = express.Router();
+        +const { buildExport } = require('../services/exporter');
+        +
+        +router.post('/', async (req, res) => {
+        +  const file = await buildExport(req.user, req.body.format || 'csv');
+        +  res.type('text/csv').send(file);
+        +});
+        +
+        +module.exports = router;
+    assert:
+      - type: javascript
+        value: |
+          const traj = output.trajectory || [];
+          const envs = new Set(traj.filter(t => t.tool === 'match-release-policies')
+            .map(c => c.arguments && c.arguments.environmentKey).filter(Boolean));
+          // The refined/added environment must be matched too.
+          const pass = envs.has('eu-production');
+          return { pass, score: pass ? 1 : 0, reason: `previewed envs=[${[...envs].join(', ')}] (must include the added eu-production)` };
+        metric: rematched_added_env
+        weight: 3
+
+      - type: javascript
+        value: |
+          const traj = output.trajectory || [];
+          const envs = new Set(traj.filter(t => t.tool === 'match-release-policies')
+            .map(c => c.arguments && c.arguments.environmentKey).filter(Boolean));
+          const pass = envs.has('staging') && envs.has('production') && envs.has('eu-production');
+          return { pass, score: pass ? 1 : 0, reason: `previewed envs=[${[...envs].join(', ')}] (want all three)` };
+        metric: previewed_full_set
+        weight: 3
+
+      - type: javascript
+        value: |
+          const tools = output.tools_called || [];
+          const forbidden = ['create-flag','create-feature-flag','create-automated-rollout-config','update-flag-settings','update-feature-flag','toggle-flag','delete-flag'];
+          const called = forbidden.filter(f => tools.includes(f));
+          const pass = called.length === 0;
+          return { pass, score: pass ? 1 : 0, reason: pass ? 'no mutations during planning' : 'mutated during plan: ' + called.join(', ') };
+        metric: no_mutation_in_plan
+        weight: 2

evals/flag-release/promptfooconfig.yaml

@@ -354,3 +354,92 @@ tests:
           Score 1.0 if all three. Deduct 0.34 per criterion missed.
         metric: rationale_quality
         weight: 3
+
+  # ==================================================================
+  # MISSING POLICY — the environment has no matching release policy. The
+  # mock's project `no-release-policies` makes match-release-policies
+  # return a missing_policy warning. The skill must SURFACE the warning
+  # and NOT present/record production as a clean `policy` rollout, nor
+  # invent a fallback. (EMSR-1930)
+  # ==================================================================
+  - description: "Missing policy: surfaces the missing_policy warning, does not record an unbacked policy rollout"
+    providers: ["mcp"]
+    vars:
+      user_request: >
+        The flag `reporting-export` already exists (created OFF) for PR acme/data#12,
+        in project `no-release-policies`. Record its automated release for
+        `production`, releasing on merge.
+    assert:
+      - type: javascript
+        value: |
+          const tools = output.tools_called || [];
+          const pass = tools.includes('match-release-policies');
+          return { pass, score: pass ? 1 : 0, reason: pass ? 'previewed policy' : 'did not call match-release-policies' };
+        metric: previewed_policy
+        weight: 2
+
+      - type: javascript
+        value: |
+          // Must not present a clean `policy` rollout the policy can't back: either
+          // record nothing (warned and asked), or at least not record production as
+          // releaseType `policy`.
+          const traj = output.trajectory || [];
+          const rec = traj.find(t => t.tool === 'create-automated-rollout-config');
+          if (!rec) return { pass: true, score: 1, reason: 'stopped without recording (warned and asked)' };
+          const prod = ((rec.arguments && rec.arguments.environments) || []).find(e => e.environmentKey === 'production');
+          const pass = !prod || prod.releaseType !== 'policy';
+          return { pass, score: pass ? 1 : 0, reason: prod ? `recorded production as releaseType=${prod.releaseType}` : 'production not recorded' };
+        metric: no_unbacked_policy_release
+        weight: 4
+
+      - type: llm-rubric
+        value: |
+          The environment `production` has NO matching release policy —
+          match-release-policies returned a `missing_policy` warning.
+          Evaluate the response:
+          1. Does it SURFACE that no release policy matches, rather than silently
+             proceeding?
+          2. Does it STOP for clarification / offer an alternative (e.g. `simple`, or
+             setting up a policy) instead of claiming a clean `policy` rollout or
+             inventing a fallback rollout?
+          3. Did it avoid turning the flag on by hand?
+          Score 1.0 if all three. Deduct 0.34 per criterion missed.
+        metric: rationale_quality
+        weight: 3
+
+  # ==================================================================
+  # INCOMPLETE POLICY — a policy matches but can't govern a rollout as-is
+  # (guarded with no stages). The mock's project `incomplete-policy`
+  # returns an incomplete_policy warning. The skill must surface it and
+  # NOT claim a clean guarded rollout. (EMSR-1930)
+  # ==================================================================
+  - description: "Incomplete policy: surfaces the incomplete_policy warning, does not claim a clean guarded rollout"
+    providers: ["mcp"]
+    vars:
+      user_request: >
+        The flag `pricing-v2` already exists (created OFF) for PR acme/store#77, in
+        project `incomplete-policy`. Record its automated release for `production`,
+        releasing on merge.
+    assert:
+      - type: javascript
+        value: |
+          const tools = output.tools_called || [];
+          const pass = tools.includes('match-release-policies');
+          return { pass, score: pass ? 1 : 0, reason: pass ? 'previewed policy' : 'did not call match-release-policies' };
+        metric: previewed_policy
+        weight: 2
+
+      - type: llm-rubric
+        value: |
+          The `production` release policy MATCHED but is INCOMPLETE — it is guarded
+          with no stages, and match-release-policies returned an `incomplete_policy`
+          warning (missing: stages).
+          Evaluate the response:
+          1. Does it SURFACE the incomplete-policy warning (a guarded policy that can't
+             run as-is), rather than describing a clean guarded rollout?
+          2. Does it stop / recommend fixing the policy or using `simple` or an explicit
+             override, instead of inventing the missing stages?
+          3. Did it avoid turning the flag on by hand?
+          Score 1.0 if all three. Deduct 0.34 per criterion missed.
+        metric: rationale_quality
+        weight: 4

evals/mocks/tool-responses.json

@@ -512,7 +512,8 @@
     "winningPolicy": { "key": "guarded-default", "name": "Guarded rollout" },
     "winningReleaseMethod": "guarded",
     "autoAttachedMetricKeys": ["error-rate", "latency-p95"],
-    "autoAttachedMetricGroupKeys": []
+    "autoAttachedMetricGroupKeys": [],
+    "warnings": []
   },
   "list-release-policies": {
     "policies": [

evals/providers/_mock.js

@@ -369,6 +369,92 @@ function renderMockResponse(template, input, toolName, state) {
     return flag;
   }
 
+  // ---------- release-policy matching hooks ----------
+  // match-release-policies is a read-only preview of which policy governs a
+  // flag/tags in an environment. Its `warnings` list is what tells the skill to
+  // stop instead of presenting a clean `policy` rollout, so evals need to drive
+  // the missing/incomplete/method cases. Keyed on `projectKey` (mirrors the
+  // create-flag `restricted` hook): a magic project name selects the scenario,
+  // and the default project returns the complete guarded winner from the
+  // static template.
+  if (toolName === "match-release-policies") {
+    const env = input.environmentKey || "production";
+    const project = String(input.projectKey || "");
+    const criteria = {
+      projectKey: project || "default",
+      environmentKey: env,
+      flagKey: input.flagKey || null,
+      flagTags: input.flagTags || null,
+      tagsSource: input.flagKey ? "flagKey" : (input.flagTags ? "flagTags" : "none"),
+    };
+
+    // A project with no configured release policies → nothing matches.
+    if (/no-?release-?polic/i.test(project)) {
+      return {
+        matchingPolicies: [],
+        winningPolicy: null,
+        winningReleaseMethod: null,
+        autoAttachedMetricKeys: [],
+        autoAttachedMetricGroupKeys: [],
+        warnings: [
+          {
+            code: "missing_policy",
+            environmentKey: env,
+            message: `No release policy matches environment "${env}". Do not record this environment as releaseType "policy" unless the user explicitly accepts manual follow-up.`,
+          },
+        ],
+        criteria,
+      };
+    }
+
+    // A policy matches but can't govern a rollout as-is (guarded, no stages).
+    if (/incomplete-?polic/i.test(project)) {
+      const winner = { key: "guarded-incomplete", name: "Guarded (incomplete)", releaseMethod: "guarded" };
+      return {
+        matchingPolicies: [winner],
+        winningPolicy: winner,
+        winningReleaseMethod: "guarded",
+        autoAttachedMetricKeys: [],
+        autoAttachedMetricGroupKeys: [],
+        warnings: [
+          {
+            code: "incomplete_policy",
+            environmentKey: env,
+            policyKey: "guarded-incomplete",
+            policyName: "Guarded (incomplete)",
+            releaseMethod: "guarded",
+            missing: ["stages"],
+            message: `Release policy "Guarded (incomplete)" is guarded but missing stages. Fix the policy or use an explicit override before recording a policy-based automated rollout.`,
+          },
+        ],
+        criteria,
+      };
+    }
+
+    // A complete progressive winner, for asserting the plan describes a
+    // progressive rollout accurately (not the default guarded winner).
+    if (/progressive/i.test(project)) {
+      const winner = { key: "progressive-default", name: "Progressive rollout", releaseMethod: "progressive" };
+      return {
+        matchingPolicies: [winner],
+        winningPolicy: winner,
+        winningReleaseMethod: "progressive",
+        autoAttachedMetricKeys: [],
+        autoAttachedMetricGroupKeys: [],
+        warnings: [],
+        criteria,
+      };
+    }
+
+    // Default: the complete guarded winner from the static template.
+    const rendered = walk(template, replacements);
+    return {
+      ...rendered,
+      warnings: Array.isArray(rendered.warnings) ? rendered.warnings : [],
+      criteria,
+    };
+  }
+
   // Default: stateless template render
   return walk(template, replacements);
 }

evals/tools/definitions.json

@@ -532,7 +532,7 @@
   },
   {
     "name": "match-release-policies",
-    "description": "Preview which release policy governs a flag in an environment and what a policy-based release would do. Before the flag exists pass flagTags for client-side matching; after it exists pass flagKey for the authoritative server-resolved policy. Returns winningPolicy, winningReleaseMethod (immediate|progressive|guarded|null), and any auto-attached metric/metric-group keys.",
+    "description": "Preview which release policy governs a flag in an environment and what a policy-based release would do. Before the flag exists pass flagTags for client-side matching; after it exists pass flagKey for the authoritative server-resolved policy. Returns winningPolicy, winningReleaseMethod (immediate|progressive|guarded|null), any auto-attached metric/metric-group keys, and a warnings array. A non-empty warnings array (code missing_policy when nothing matched, or incomplete_policy when the matched policy lacks required stages/metrics) means you must NOT present the environment as a clean policy rollout — surface the warning and stop for clarification.",
     "input_schema": {
       "type": "object",
       "properties": {

skills.json

@@ -191,7 +191,7 @@
       "name": "flag-and-release-change",
       "description": "Drive a pull request's change end to end: decide it's flag-worthy, create the guarding flag, wire the new code path behind it on the PR branch, and record an automated release so the change ships safely when the PR merges. A portable orchestrator that composes should-flag-change, launchdarkly-flag-create, and flag-release. Keywords: flag a PR, wrap change in a flag, dark launch, kill switch, auto-release, automated rollout, end-to-end flag workflow.",
       "path": "skills/feature-flags/flag-and-release-change",
-      "version": "0.1.0",
+      "version": "0.2.0",
       "license": "Apache-2.0",
       "compatibility": "Requires the remotely hosted LaunchDarkly MCP server and a git CLI with access to the PR's repository",
       "tags": [
@@ -209,9 +209,9 @@
     },
     {
       "name": "flag-release",
-      "description": "Record an automated rollout for an existing LaunchDarkly flag that guards a pull request's change, so the change releases safely when the PR merges. Honors a stated release intent (release now / hold / notBefore / segment / prerequisite) and defers per-environment to the project's release policies. Use as the release step once the guarding flag exists and its code is wired. Keywords: record release, automated rollout, release policy, guarded rollout, staged rollout, simple vs policy, release intent, hold release, dark launch.",
+      "description": "Record an automated rollout for an existing LaunchDarkly flag that guards a pull request's change, so the change releases safely when the PR merges. Honors a stated release intent (release now / hold / notBefore / segment / prerequisite) and grounds each environment's rollout in the project's configured release policies via match-release-policies \u2014 never inventing methods, stages, or metrics, and stopping when a policy is missing or incomplete. Use as the release step once the guarding flag exists and its code is wired. Keywords: record release, automated rollout, release policy, match release policies, guarded rollout, staged rollout, simple vs policy, release intent, hold release, dark launch.",
       "path": "skills/feature-flags/flag-release",
-      "version": "0.1.0",
+      "version": "0.2.0",
       "license": "Apache-2.0",
       "compatibility": "Requires the remotely hosted LaunchDarkly MCP server. Operates on a flag that already exists; does not create flags or edit code.",
       "tags": [