Actions: Add experimental queries for AI output validation CWE 1426

actions/ql/lib/change-notes/2026-04-08-ai-output-validation-library.md

@@ -0,0 +1,4 @@
+---
+category: feature
+---
+* Added `ImproperValidationOfAiOutputQuery.qll` library and `ai_inference_actions.model.yml` models-as-data file for detecting improper validation of AI-generated output (CWE-1426) in GitHub Actions workflows.

actions/ql/lib/codeql/actions/security/ImproperValidationOfAiOutputQuery.qll

@@ -0,0 +1,92 @@
+/**
+ * Provides classes and predicates for detecting improper validation of
+ * generative AI output in GitHub Actions workflows (CWE-1426).
+ *
+ * This library identifies cases where AI-generated output flows unsanitized
+ * into code execution sinks (LOTP gadgets) or subsequent AI prompts.
+ */
+
+private import actions
+private import codeql.actions.TaintTracking
+private import codeql.actions.dataflow.ExternalFlow
+import codeql.actions.dataflow.FlowSources
+import codeql.actions.DataFlow
+import codeql.actions.security.ControlChecks
+
+/**
+ * A source representing AI-generated output from AI inference actions.
+ * This models CWE-1426 where AI output is used without proper validation,
+ * potentially allowing an attacker to chain prompt injection into code execution.
+ */
+class AiInferenceOutputSource extends DataFlow::Node {
+  UsesStep aiStep;
+
+  AiInferenceOutputSource() {
+    exists(StepsExpression stepRef, string action |
+      this.asExpr() = stepRef and
+      stepRef.getStepId() = aiStep.getId() and
+      actionsSinkModel(action, _, _, "ai-inference", _) and
+      aiStep.getCallee() = action
+    )
+  }
+
+  /** Gets the AI inference step that produces this output. */
+  UsesStep getAiStep() { result = aiStep }
+}
+
+/**
+ * A sink for improper validation of AI output (CWE-1426).
+ * AI output flowing unsanitized to code execution (LOTP gadgets),
+ * subsequent AI prompts, or environment manipulation.
+ */
+class ImproperAiOutputSink extends DataFlow::Node {
+  ImproperAiOutputSink() {
+    // Code injection sinks (run steps) — LOTP gadgets
+    exists(Run e | e.getAnScriptExpr() = this.asExpr())
+    or
+    // MaD-defined code injection sinks
+    madSink(this, "code-injection")
+    or
+    // AI inference sinks (AI output flowing to another AI prompt = chained injection)
+    madSink(this, "ai-inference")
+  }
+}
+
+/**
+ * Gets the relevant event for sinks in a privileged context.
+ */
+Event getRelevantEventForAiOutputSink(DataFlow::Node sink) {
+  inPrivilegedContext(sink.asExpr(), result) and
+  not exists(ControlCheck check | check.protects(sink.asExpr(), result, "code-injection"))
+}
+
+/**
+ * Holds when a critical-severity AI output validation issue exists.
+ */
+predicate criticalAiOutputInjection(
+  ImproperAiOutputFlow::PathNode source, ImproperAiOutputFlow::PathNode sink, Event event
+) {
+  ImproperAiOutputFlow::flowPath(source, sink) and
+  event = getRelevantEventForAiOutputSink(sink.getNode())
+}
+
+/**
+ * A taint-tracking configuration for AI-generated output
+ * that flows unsanitized to code execution or subsequent AI prompts.
+ */
+private module ImproperAiOutputConfig implements DataFlow::ConfigSig {
+  predicate isSource(DataFlow::Node source) { source instanceof AiInferenceOutputSource }
+
+  predicate isSink(DataFlow::Node sink) { sink instanceof ImproperAiOutputSink }
+
+  predicate observeDiffInformedIncrementalMode() { any() }
+
+  Location getASelectedSinkLocation(DataFlow::Node sink) {
+    result = sink.getLocation()
+    or
+    result = getRelevantEventForAiOutputSink(sink).getLocation()
+  }
+}
+
+/** Tracks flow of AI-generated output to code execution sinks. */
+module ImproperAiOutputFlow = TaintTracking::Global<ImproperAiOutputConfig>;

actions/ql/lib/ext/manual/ai_inference_actions.model.yml

@@ -0,0 +1,32 @@
+extensions:
+  - addsTo:
+      pack: codeql/actions-all
+      extensible: actionsSinkModel
+    # AI inference actions whose output should be treated as untrusted.
+    # Used by CWE-1426 (ImproperValidationOfAiOutput) to identify AI action steps
+    # whose outputs may flow unsanitized to code execution sinks (LOTP gadgets).
+    # source: https://boostsecurityio.github.io/lotp/
+    # source: https://github.com/marketplace?type=actions&category=ai-assisted
+    data:
+      # === GitHub official ===
+      - ["actions/ai-inference", "*", "input.prompt", "ai-inference", "manual"]
+      - ["github/ai-moderator", "*", "input.prompt", "ai-inference", "manual"]
+      # === Anthropic ===
+      - ["anthropics/claude-code-action", "*", "input.prompt", "ai-inference", "manual"]
+      # === Google ===
+      - ["google/gemini-code-assist-action", "*", "input.prompt", "ai-inference", "manual"]
+      - ["google-gemini/code-assist-action", "*", "input.prompt", "ai-inference", "manual"]
+      # === OpenAI ===
+      - ["openai/chat-completion-action", "*", "input.prompt", "ai-inference", "manual"]
+      # === Community AI review/inference actions ===
+      - ["coderabbitai/ai-pr-reviewer", "*", "input.prompt", "ai-inference", "manual"]
+      - ["CodiumAI/pr-agent", "*", "input.prompt", "ai-inference", "manual"]
+      - ["platisd/openai-pr-description", "*", "input.prompt", "ai-inference", "manual"]
+      - ["austenstone/openai-completion-action", "*", "input.prompt", "ai-inference", "manual"]
+      - ["github/copilot-text-inference", "*", "input.prompt", "ai-inference", "manual"]
+      - ["huggingface/inference-action", "*", "input.prompt", "ai-inference", "manual"]
+      - ["replicate/action", "*", "input.prompt", "ai-inference", "manual"]
+      # === Google (GitHub Actions org) ===
+      - ["google-github-actions/run-gemini-cli", "*", "input.prompt", "ai-inference", "manual"]
+      # === Warp ===
+      - ["warpdotdev/oz-agent-action", "*", "input.prompt", "ai-inference", "manual"]

actions/ql/src/change-notes/2026-04-08-ai-output-validation-query.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Added new experimental query `actions/improper-ai-output-handling/critical` to detect improper validation of AI-generated output (CWE-1426) in GitHub Actions workflows where AI action output flows unsanitized to code execution sinks.

actions/ql/src/experimental/Security/CWE-1426/ImproperValidationOfAiOutputCritical.md

@@ -0,0 +1,67 @@
+## Overview
+
+Using AI-generated output without validation in GitHub Actions workflows enables **chained injection attacks** where an attacker's prompt injection in one step produces malicious AI output that executes as code in a subsequent step. When AI output flows unsanitized into shell commands, build scripts, package installations, or subsequent AI prompts, an attacker who controls the AI's input effectively controls the code that runs in your CI/CD pipeline.
+
+AI output should always be treated as untrusted data. This is especially dangerous because the malicious payload is generated dynamically by the AI and may bypass traditional static analysis or code review.
+
+## Recommendation
+
+Treat all AI-generated output as untrusted. Before using AI output in any executable context:
+
+- **Validate the format** — check that the output matches an expected schema or pattern before use.
+- **Never interpolate AI output directly into `run:` steps** — use environment variables and validate before execution.
+- **Limit AI action permissions** — restrict `GITHUB_TOKEN` scope and avoid passing secrets to workflows that consume AI output.
+- **Use structured output formats** (e.g. JSON with a defined schema) to constrain AI responses and make validation easier.
+- **Avoid chaining AI calls** without validating intermediate output. Each AI step's output is a potential injection vector for the next.
+
+## Example
+
+### Incorrect Usage
+
+The following example executes AI output directly as a shell command. An attacker who controls the AI's input (via the issue body) can cause the AI to output arbitrary shell commands:
+
+```yaml
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  ai-task:
+    runs-on: ubuntu-latest
+    steps:
+      - name: AI inference
+        id: ai
+        uses: actions/ai-inference@v1
+        with:
+          prompt: |
+            Suggest a fix for: ${{ github.event.issue.body }}
+
+      - name: Apply fix
+        run: |
+          ${{ steps.ai.outputs.response }}
+```
+
+### Correct Usage
+
+The following example validates the AI output format before taking any action:
+
+```yaml
+      - name: Validate and apply
+        run: |
+          RESPONSE="${AI_RESPONSE}"
+          # Only accept responses that match a safe pattern
+          if echo "$RESPONSE" | grep -qE '^(fix|patch|update):'; then
+            echo "Valid response format, proceeding"
+          else
+            echo "::warning::Unexpected AI output format, skipping execution"
+            exit 0
+          fi
+        env:
+          AI_RESPONSE: ${{ steps.ai.outputs.response }}
+```
+
+## References
+
+- Common Weakness Enumeration: [CWE-1426](https://cwe.mitre.org/data/definitions/1426.html).
+- [OWASP LLM02: Insecure Output Handling](https://genai.owasp.org/llmrisk/llm02-insecure-output-handling/).
+- GitHub Docs: [Security hardening for GitHub Actions](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions).

actions/ql/src/experimental/Security/CWE-1426/ImproperValidationOfAiOutputCritical.ql

@@ -0,0 +1,24 @@
+/**
+ * @name Improper validation of AI-generated output
+ * @description AI-generated output flowing unsanitized to code execution or
+ *              subsequent AI prompts may allow chained prompt injection attacks.
+ * @kind path-problem
+ * @problem.severity error
+ * @security-severity 9.0
+ * @precision high
+ * @id actions/improper-validation-of-ai-output/critical
+ * @tags actions
+ *       security
+ *       experimental
+ *       external/cwe/cwe-1426
+ */
+
+import actions
+import codeql.actions.security.ImproperValidationOfAiOutputQuery
+import ImproperAiOutputFlow::PathGraph
+
+from ImproperAiOutputFlow::PathNode source, ImproperAiOutputFlow::PathNode sink, Event event
+where criticalAiOutputInjection(source, sink, event)
+select sink.getNode(), source, sink,
+  "AI-generated output flows unsanitized to $@, which may allow chained injection ($@).", sink,
+  sink.getNode().asExpr().(Expression).getRawExpression(), event, event.getName()

actions/ql/test/query-tests/Security/CWE-1426/.github/workflows/safe1.yml

@@ -0,0 +1,19 @@
+name: Safe AI Output Usage
+on:
+  push:
+    branches: [main]
+
+jobs:
+  analyze:
+    runs-on: ubuntu-latest
+    steps:
+      - name: AI inference
+        id: ai
+        uses: actions/ai-inference@v1
+        with:
+          prompt: |
+            Analyze this repository.
+
+      - name: Display result only
+        run: |
+          echo "AI said something"

actions/ql/test/query-tests/Security/CWE-1426/.github/workflows/safe2.yml

@@ -0,0 +1,28 @@
+name: Safe AI Output to Comment Only
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  respond:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+      - name: AI analysis
+        id: ai
+        uses: actions/ai-inference@v1
+        with:
+          prompt: |
+            Summarize this issue.
+
+      - name: Post comment with AI response
+        uses: actions/github-script@v7
+        with:
+          script: |
+            github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: 'AI Summary posted'
+            })

actions/ql/test/query-tests/Security/CWE-1426/.github/workflows/vulnerable1.yml

@@ -0,0 +1,22 @@
+name: AI Output to Shell
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  ai-task:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      models: read
+    steps:
+      - name: AI inference
+        id: ai
+        uses: actions/ai-inference@v1
+        with:
+          prompt: |
+            Suggest a fix for: ${{ github.event.issue.body }}
+
+      - name: Apply fix unsanitized
+        run: |
+          ${{ steps.ai.outputs.response }}

actions/ql/test/query-tests/Security/CWE-1426/.github/workflows/vulnerable2.yml

@@ -0,0 +1,26 @@
+name: AI Output Chain
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  ai-task:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      models: read
+    steps:
+      - name: First AI inference
+        id: ai1
+        uses: actions/ai-inference@v1
+        with:
+          prompt: |
+            Summarize: ${{ github.event.issue.body }}
+
+      - name: Second AI inference using first AI output
+        id: ai2
+        uses: actions/ai-inference@v1
+        with:
+          prompt: |
+            Improve this summary:
+            ${{ steps.ai1.outputs.response }}

actions/ql/test/query-tests/Security/CWE-1426/.github/workflows/vulnerable3.yml

@@ -0,0 +1,23 @@
+name: Claude Output to Shell
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  fix:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      issues: write
+    steps:
+      - name: Claude analysis
+        id: claude
+        uses: anthropics/claude-code-action@v1
+        with:
+          prompt: |
+            Suggest a shell command to fix this issue:
+            ${{ github.event.issue.title }}
+
+      - name: Execute AI suggestion
+        run: |
+          ${{ steps.claude.outputs.response }}

actions/ql/test/query-tests/Security/CWE-1426/.github/workflows/vulnerable4.yml

@@ -0,0 +1,22 @@
+name: Gemini Output to Shell
+on:
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  apply-fix:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Gemini review
+        id: gemini
+        uses: google-github-actions/run-gemini-cli@v1
+        with:
+          prompt: |
+            Suggest a patch for this PR.
+
+      - name: Apply Gemini suggestion
+        run: |
+          echo "${{ steps.gemini.outputs.response }}" | patch -p1

actions/ql/test/query-tests/Security/CWE-1426/ImproperValidationOfAiOutputCritical.expected

@@ -0,0 +1,12 @@
+edges
+nodes
+| .github/workflows/vulnerable1.yml:22:12:22:43 | steps.ai.outputs.response | semmle.label | steps.ai.outputs.response |
+| .github/workflows/vulnerable2.yml:26:13:26:45 | steps.ai1.outputs.response | semmle.label | steps.ai1.outputs.response |
+| .github/workflows/vulnerable3.yml:23:12:23:47 | steps.claude.outputs.response | semmle.label | steps.claude.outputs.response |
+| .github/workflows/vulnerable4.yml:22:18:22:53 | steps.gemini.outputs.response | semmle.label | steps.gemini.outputs.response |
+subpaths
+#select
+| .github/workflows/vulnerable1.yml:22:12:22:43 | steps.ai.outputs.response | .github/workflows/vulnerable1.yml:22:12:22:43 | steps.ai.outputs.response | .github/workflows/vulnerable1.yml:22:12:22:43 | steps.ai.outputs.response | AI-generated output flows unsanitized to $@, which may allow chained injection ($@). | .github/workflows/vulnerable1.yml:22:12:22:43 | steps.ai.outputs.response | ${{ steps.ai.outputs.response }} | .github/workflows/vulnerable1.yml:3:3:3:8 | issues | issues |
+| .github/workflows/vulnerable2.yml:26:13:26:45 | steps.ai1.outputs.response | .github/workflows/vulnerable2.yml:26:13:26:45 | steps.ai1.outputs.response | .github/workflows/vulnerable2.yml:26:13:26:45 | steps.ai1.outputs.response | AI-generated output flows unsanitized to $@, which may allow chained injection ($@). | .github/workflows/vulnerable2.yml:26:13:26:45 | steps.ai1.outputs.response | ${{ steps.ai1.outputs.response }} | .github/workflows/vulnerable2.yml:3:3:3:8 | issues | issues |
+| .github/workflows/vulnerable3.yml:23:12:23:47 | steps.claude.outputs.response | .github/workflows/vulnerable3.yml:23:12:23:47 | steps.claude.outputs.response | .github/workflows/vulnerable3.yml:23:12:23:47 | steps.claude.outputs.response | AI-generated output flows unsanitized to $@, which may allow chained injection ($@). | .github/workflows/vulnerable3.yml:23:12:23:47 | steps.claude.outputs.response | ${{ steps.claude.outputs.response }} | .github/workflows/vulnerable3.yml:3:3:3:8 | issues | issues |
+| .github/workflows/vulnerable4.yml:22:18:22:53 | steps.gemini.outputs.response | .github/workflows/vulnerable4.yml:22:18:22:53 | steps.gemini.outputs.response | .github/workflows/vulnerable4.yml:22:18:22:53 | steps.gemini.outputs.response | AI-generated output flows unsanitized to $@, which may allow chained injection ($@). | .github/workflows/vulnerable4.yml:22:18:22:53 | steps.gemini.outputs.response | ${{ steps.gemini.outputs.response }} | .github/workflows/vulnerable4.yml:3:3:3:21 | pull_request_review | pull_request_review |

actions/ql/test/query-tests/Security/CWE-1426/ImproperValidationOfAiOutputCritical.qlref

@@ -0,0 +1 @@
+experimental/Security/CWE-1426/ImproperValidationOfAiOutputCritical.ql