From 9533f69d3f0612caa8c5c783b12695b1019353bb Mon Sep 17 00:00:00 2001 From: Chris Burns <29541485+ChrisJBurns@users.noreply.github.com> Date: Wed, 15 Apr 2026 21:17:18 +0100 Subject: [PATCH 1/4] Add dry-run doc validation skill New /test-docs-dryrun skill for fast CRD schema validation of all YAML blocks in K8s and vMCP documentation. Extracts YAML from docs, runs kubectl apply --dry-run=server, and reports pass/fail per doc. No resources created. Completes in under 5 minutes for all docs. Includes: - SKILL.md with scope selection (single page, section, all) - Python extraction script for YAML blocks from .mdx files - Lightweight prereqs checker (only needs CRDs, not operator) Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude/skills/test-docs-dryrun/SKILL.md | 146 ++++++++++++++++++ .../test-docs-dryrun/scripts/check-prereqs.sh | 70 +++++++++ .../test-docs-dryrun/scripts/extract-yaml.py | 71 +++++++++ 3 files changed, 287 insertions(+) create mode 100644 .claude/skills/test-docs-dryrun/SKILL.md create mode 100644 .claude/skills/test-docs-dryrun/scripts/check-prereqs.sh create mode 100644 .claude/skills/test-docs-dryrun/scripts/extract-yaml.py diff --git a/.claude/skills/test-docs-dryrun/SKILL.md b/.claude/skills/test-docs-dryrun/SKILL.md new file mode 100644 index 00000000..d7262fde --- /dev/null +++ b/.claude/skills/test-docs-dryrun/SKILL.md @@ -0,0 +1,146 @@ +--- +name: test-docs-dryrun +description: > + Fast CRD schema validation for ToolHive documentation. Extracts all YAML blocks from K8s and vMCP docs, runs kubectl apply --dry-run=server to catch field name errors, type mismatches, and schema drift. No cluster resources are created. Use for: "dry-run the docs", "validate the YAML", "check for schema issues", "run a quick doc check", or after any CRD/API change to catch doc rot. Requires a Kubernetes cluster with ToolHive CRDs installed. +--- + +# Test Docs - Dry Run + +Fast CRD schema validation for all ToolHive documentation YAML blocks. Extracts every YAML block containing `toolhive.stacklok.dev` resources from the docs, runs `kubectl apply --dry-run=server` on each, and reports pass/fail. No resources are created. Typically completes in under 5 minutes for all docs. + +## When to use + +- After a ToolHive release (CRD fields may have changed) +- After docs changes that touch YAML examples +- As a quick regression check before a full execution test +- In CI to validate docs against the current CRD schema + +## Workflow + +1. **Determine scope** - which docs to validate +2. **Check prerequisites** - cluster with CRDs must exist +3. **Extract YAML** - pull all ToolHive CRD YAML blocks from docs +4. **Validate** - dry-run each block against the K8s API server +5. **Report** - pass/fail per block, grouped by doc + +## Step 1: Determine scope + +The user can request validation at three levels: + +- **Single page** - validate YAML from one doc file +- **Section** - validate all YAML from a section +- **All** - validate all YAML across all sections + +### Section definitions + +**Kubernetes Operator** (`docs/toolhive/guides-k8s/`): all `.mdx` files + +**Virtual MCP Server** (`docs/toolhive/guides-vmcp/`): all `.mdx` files + +When the user says "all", "everything", or "dry-run the docs", validate both sections. + +### Asking the user + +When invoked without specifying scope, ask: + +- **Kubernetes Operator** - all pages in `guides-k8s/` +- **Virtual MCP Server** - all pages in `guides-vmcp/` +- **All** - both sections + +## Step 2: Check prerequisites + +A Kubernetes cluster with ToolHive CRDs installed is required. The cluster does NOT need the operator running - only the CRDs for schema validation. + +Run the prerequisites checker: + +```bash +bash /scripts/check-prereqs.sh +``` + +If no cluster exists, offer to create one: + +1. `kind create cluster --name toolhive` +2. `helm upgrade --install toolhive-operator-crds oci://ghcr.io/stacklok/toolhive/toolhive-operator-crds -n toolhive-system --create-namespace` + +The operator itself is NOT needed for dry-run validation. + +## Step 3: Extract YAML + +Use the extraction script to pull all YAML blocks from the target docs: + +```bash +python3 /scripts/extract-yaml.py +``` + +Run this for every `.mdx` file in scope. The script: + +- Finds all ` ```yaml ` / ` ```yml ` fenced code blocks +- Splits multi-document blocks on `---` +- Keeps only documents containing `toolhive.stacklok.dev` +- Writes each as a separate `.yaml` file named `_.yaml` + +## Step 4: Validate + +For each extracted YAML file, run: + +```bash +kubectl apply --dry-run=server -f +``` + +Classify results: + +- **PASS**: dry-run succeeds (exit 0) +- **PASS (namespace missing)**: fails only because namespace doesn't exist (`namespaces "..." not found`) - schema is valid +- **EXPECTED FAIL**: placeholder names like `` or missing required fields on intentionally partial snippets (`spec.incomingAuth: Required value`) +- **REAL FAIL**: schema validation error (unknown field, wrong type, etc.) + +## Step 5: Report + +Always output a per-doc breakdown table to the user showing every doc that had YAML blocks extracted. This table is the primary output of the skill and must always be displayed, not just written to a file. + +```text +## Dry-Run Results + +| Doc | Section | Blocks | Pass | Fail | Expected | Skip | +|-----|---------|--------|------|------|----------|------| +| auth-k8s | K8s | 10 | 10 | 0 | 0 | 0 | +| connect-clients | K8s | 3 | 1 | 0 | 0 | 2 | +| customize-tools | K8s | 8 | 8 | 0 | 0 | 0 | +| ... | ... | ... | ... | ... | ... | ... | +| authentication | vMCP | 7 | 7 | 0 | 0 | 0 | +| quickstart | vMCP | 4 | 4 | 0 | 0 | 0 | +| ... | ... | ... | ... | ... | ... | ... | +| **TOTAL** | | **N** | **N** | **N** | **N** | **N** | +``` + +Rules for the table: + +- Include EVERY doc that had at least 1 YAML block extracted +- Omit docs with 0 blocks (intro, index, deploy-operator, etc.) +- Group K8s docs first, then vMCP docs +- Sort alphabetically within each section +- Bold the TOTAL row +- Bold any non-zero Fail count to draw attention + +After the table, list each REAL FAIL with: + +- The file and block index +- The error message +- The likely fix (field rename, type change, etc.) + +### Write results to file + +- Single page: `TEST_DRYRUN_.md` +- Section: `TEST_DRYRUN_.md` +- All: `TEST_DRYRUN_ALL.md` + +## Example session + +User: "dry-run the docs" + +1. Check cluster exists, CRDs installed +2. Extract 108 YAML blocks from 26 docs +3. Dry-run validate all blocks (~2 minutes) +4. Report: 97 PASS, 4 EXPECTED FAIL, 7 REAL FAIL +5. List each real failure with the fix needed +6. Write `TEST_DRYRUN_ALL.md` diff --git a/.claude/skills/test-docs-dryrun/scripts/check-prereqs.sh b/.claude/skills/test-docs-dryrun/scripts/check-prereqs.sh new file mode 100644 index 00000000..1ae9fe04 --- /dev/null +++ b/.claude/skills/test-docs-dryrun/scripts/check-prereqs.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# Check prerequisites for dry-run documentation validation. +# Only requires kubectl and a cluster with CRDs installed (no operator needed). + +set -euo pipefail + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +NC='\033[0m' + +ERRORS=0 +WARNINGS=0 + +echo "=== Dry-Run Validation Prerequisites ===" +echo "" + +# Check kubectl +if command -v kubectl &>/dev/null; then + echo -e "${GREEN}[OK]${NC} kubectl available" +else + echo -e "${RED}[MISSING]${NC} kubectl (required)" + ERRORS=$((ERRORS + 1)) +fi + +# Check cluster connectivity +echo "" +echo "--- Kubernetes cluster ---" +if kubectl cluster-info &>/dev/null; then + CONTEXT=$(kubectl config current-context 2>/dev/null || echo "unknown") + echo -e "${GREEN}[OK]${NC} Cluster reachable (context: $CONTEXT)" +else + echo -e "${RED}[MISSING]${NC} No Kubernetes cluster reachable" + ERRORS=$((ERRORS + 1)) +fi + +# Check ToolHive CRDs installed +echo "" +echo "--- ToolHive CRDs ---" +CRD_COUNT=$(kubectl get crd 2>/dev/null | grep -c toolhive.stacklok.dev || true) +if [ "$CRD_COUNT" -gt 0 ]; then + echo -e "${GREEN}[OK]${NC} $CRD_COUNT ToolHive CRDs installed" +else + echo -e "${RED}[MISSING]${NC} No ToolHive CRDs found" + echo " Install with: helm upgrade --install toolhive-operator-crds oci://ghcr.io/stacklok/toolhive/toolhive-operator-crds -n toolhive-system --create-namespace" + ERRORS=$((ERRORS + 1)) +fi + +# Check python3 for extraction script +echo "" +echo "--- Tools ---" +if command -v python3 &>/dev/null; then + echo -e "${GREEN}[OK]${NC} python3 available" +else + echo -e "${RED}[MISSING]${NC} python3 (required for YAML extraction)" + ERRORS=$((ERRORS + 1)) +fi + +echo "" +echo "=== Summary ===" +if [ "$ERRORS" -gt 0 ]; then + echo -e "${RED}$ERRORS error(s)${NC}" + exit 1 +else + echo -e "${GREEN}All prerequisites met${NC}" + exit 0 +fi diff --git a/.claude/skills/test-docs-dryrun/scripts/extract-yaml.py b/.claude/skills/test-docs-dryrun/scripts/extract-yaml.py new file mode 100644 index 00000000..239c0a9c --- /dev/null +++ b/.claude/skills/test-docs-dryrun/scripts/extract-yaml.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +# SPDX-License-Identifier: Apache-2.0 + +"""Extract ToolHive CRD YAML blocks from Docusaurus .mdx files. + +Usage: + python3 extract-yaml.py + +Finds all ```yaml/```yml fenced code blocks, splits multi-document +blocks on ---, and writes each document containing +toolhive.stacklok.dev as a separate .yaml file. +""" + +import os +import re +import sys + + +def extract_yaml_blocks(filepath): + """Extract YAML blocks containing toolhive apiVersion from an mdx file.""" + with open(filepath) as f: + content = f.read() + + # Match ```yaml or ```yml code blocks (with optional metadata after yaml/yml) + pattern = r"```ya?ml[^\n]*\n(.*?)```" + blocks = re.findall(pattern, content, re.DOTALL) + + results = [] + for block in blocks: + # Split multi-document blocks + docs = re.split(r"^---\s*$", block, flags=re.MULTILINE) + for doc in docs: + doc = doc.strip() + if not doc: + continue + # Strip highlight comments that Docusaurus uses + doc = re.sub( + r"^\s*#\s*highlight-(start|end|next-line)\s*$", + "", + doc, + flags=re.MULTILINE, + ) + doc = doc.strip() + if "toolhive.stacklok.dev" in doc: + results.append(doc) + + return results + + +def main(): + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + filepath = sys.argv[1] + output_dir = sys.argv[2] + basename = os.path.basename(filepath).replace(".mdx", "") + + blocks = extract_yaml_blocks(filepath) + + for i, block in enumerate(blocks): + outfile = os.path.join(output_dir, f"{basename}_{i}.yaml") + with open(outfile, "w") as f: + f.write(block + "\n") + + print(f" {basename}: {len(blocks)} blocks") + + +if __name__ == "__main__": + main() From 4febafb3edfe5cb738658ed08fc8166803ccb9e9 Mon Sep 17 00:00:00 2001 From: Chris Burns <29541485+ChrisJBurns@users.noreply.github.com> Date: Wed, 15 Apr 2026 21:28:18 +0100 Subject: [PATCH 2/4] Improve dry-run skill with modular procedures - Split skill into SKILL.md (workflow) + 4 procedure files: cluster-setup.md, extract.md, validate.md, report.md - Fix duplicate filename bug: extraction script now takes --prefix flag to namespace output files by section (k8s-, vmcp-) - Fix cluster lifecycle: check for existing cluster first, default to keeping it after the run - Fix result tracking: use CSV file instead of bash associative arrays for zsh compatibility - Require per-doc breakdown table as primary output every run Co-Authored-By: Claude Opus 4.6 (1M context) --- .claude/skills/test-docs-dryrun/SKILL.md | 148 ++++-------------- .../procedures/cluster-setup.md | 42 +++++ .../test-docs-dryrun/procedures/extract.md | 61 ++++++++ .../test-docs-dryrun/procedures/report.md | 75 +++++++++ .../test-docs-dryrun/procedures/validate.md | 55 +++++++ .../test-docs-dryrun/scripts/extract-yaml.py | 36 ++++- 6 files changed, 291 insertions(+), 126 deletions(-) create mode 100644 .claude/skills/test-docs-dryrun/procedures/cluster-setup.md create mode 100644 .claude/skills/test-docs-dryrun/procedures/extract.md create mode 100644 .claude/skills/test-docs-dryrun/procedures/report.md create mode 100644 .claude/skills/test-docs-dryrun/procedures/validate.md diff --git a/.claude/skills/test-docs-dryrun/SKILL.md b/.claude/skills/test-docs-dryrun/SKILL.md index d7262fde..7c600158 100644 --- a/.claude/skills/test-docs-dryrun/SKILL.md +++ b/.claude/skills/test-docs-dryrun/SKILL.md @@ -6,141 +6,51 @@ description: > # Test Docs - Dry Run -Fast CRD schema validation for all ToolHive documentation YAML blocks. Extracts every YAML block containing `toolhive.stacklok.dev` resources from the docs, runs `kubectl apply --dry-run=server` on each, and reports pass/fail. No resources are created. Typically completes in under 5 minutes for all docs. - -## When to use - -- After a ToolHive release (CRD fields may have changed) -- After docs changes that touch YAML examples -- As a quick regression check before a full execution test -- In CI to validate docs against the current CRD schema +Fast CRD schema validation for all ToolHive documentation YAML blocks. Extracts every YAML block containing `toolhive.stacklok.dev` resources, runs `kubectl apply --dry-run=server` on each, and reports pass/fail in a per-doc table. No resources are created. Under 5 minutes for all docs. ## Workflow -1. **Determine scope** - which docs to validate -2. **Check prerequisites** - cluster with CRDs must exist -3. **Extract YAML** - pull all ToolHive CRD YAML blocks from docs -4. **Validate** - dry-run each block against the K8s API server -5. **Report** - pass/fail per block, grouped by doc - -## Step 1: Determine scope - -The user can request validation at three levels: - -- **Single page** - validate YAML from one doc file -- **Section** - validate all YAML from a section -- **All** - validate all YAML across all sections - -### Section definitions - -**Kubernetes Operator** (`docs/toolhive/guides-k8s/`): all `.mdx` files +Follow these steps in order. Each step references a procedure file in the `procedures/` directory - read that file and follow its instructions exactly. Do NOT improvise inline bash when a procedure exists. -**Virtual MCP Server** (`docs/toolhive/guides-vmcp/`): all `.mdx` files +### 1. Determine scope -When the user says "all", "everything", or "dry-run the docs", validate both sections. +Ask what to validate: -### Asking the user - -When invoked without specifying scope, ask: - -- **Kubernetes Operator** - all pages in `guides-k8s/` -- **Virtual MCP Server** - all pages in `guides-vmcp/` +- **Kubernetes Operator** - all `.mdx` in `docs/toolhive/guides-k8s/` +- **Virtual MCP Server** - all `.mdx` in `docs/toolhive/guides-vmcp/` - **All** - both sections -## Step 2: Check prerequisites - -A Kubernetes cluster with ToolHive CRDs installed is required. The cluster does NOT need the operator running - only the CRDs for schema validation. - -Run the prerequisites checker: - -```bash -bash /scripts/check-prereqs.sh -``` - -If no cluster exists, offer to create one: - -1. `kind create cluster --name toolhive` -2. `helm upgrade --install toolhive-operator-crds oci://ghcr.io/stacklok/toolhive/toolhive-operator-crds -n toolhive-system --create-namespace` - -The operator itself is NOT needed for dry-run validation. - -## Step 3: Extract YAML - -Use the extraction script to pull all YAML blocks from the target docs: - -```bash -python3 /scripts/extract-yaml.py -``` - -Run this for every `.mdx` file in scope. The script: - -- Finds all ` ```yaml ` / ` ```yml ` fenced code blocks -- Splits multi-document blocks on `---` -- Keeps only documents containing `toolhive.stacklok.dev` -- Writes each as a separate `.yaml` file named `_.yaml` - -## Step 4: Validate - -For each extracted YAML file, run: - -```bash -kubectl apply --dry-run=server -f -``` - -Classify results: - -- **PASS**: dry-run succeeds (exit 0) -- **PASS (namespace missing)**: fails only because namespace doesn't exist (`namespaces "..." not found`) - schema is valid -- **EXPECTED FAIL**: placeholder names like `` or missing required fields on intentionally partial snippets (`spec.incomingAuth: Required value`) -- **REAL FAIL**: schema validation error (unknown field, wrong type, etc.) - -## Step 5: Report - -Always output a per-doc breakdown table to the user showing every doc that had YAML blocks extracted. This table is the primary output of the skill and must always be displayed, not just written to a file. +### 2. Cluster setup -```text -## Dry-Run Results +Follow `procedures/cluster-setup.md`. Key points: -| Doc | Section | Blocks | Pass | Fail | Expected | Skip | -|-----|---------|--------|------|------|----------|------| -| auth-k8s | K8s | 10 | 10 | 0 | 0 | 0 | -| connect-clients | K8s | 3 | 1 | 0 | 0 | 2 | -| customize-tools | K8s | 8 | 8 | 0 | 0 | 0 | -| ... | ... | ... | ... | ... | ... | ... | -| authentication | vMCP | 7 | 7 | 0 | 0 | 0 | -| quickstart | vMCP | 4 | 4 | 0 | 0 | 0 | -| ... | ... | ... | ... | ... | ... | ... | -| **TOTAL** | | **N** | **N** | **N** | **N** | **N** | -``` +- Check for existing `toolhive` kind cluster FIRST +- Only create if missing +- Only CRDs needed, not the operator +- Default to KEEPING the cluster after the run -Rules for the table: +### 3. Extract YAML -- Include EVERY doc that had at least 1 YAML block extracted -- Omit docs with 0 blocks (intro, index, deploy-operator, etc.) -- Group K8s docs first, then vMCP docs -- Sort alphabetically within each section -- Bold the TOTAL row -- Bold any non-zero Fail count to draw attention +Follow `procedures/extract.md`. Key points: -After the table, list each REAL FAIL with: +- Always use `scripts/extract-yaml.py` with the `--prefix` flag +- Use `--prefix k8s` for K8s docs, `--prefix vmcp` for vMCP docs +- This prevents filename collisions between sections -- The file and block index -- The error message -- The likely fix (field rename, type change, etc.) +### 4. Validate -### Write results to file +Follow `procedures/validate.md`. Key points: -- Single page: `TEST_DRYRUN_.md` -- Section: `TEST_DRYRUN_.md` -- All: `TEST_DRYRUN_ALL.md` +- Write results to a CSV file (not bash variables) +- Classify as pass/fail/expected/skip per the procedure +- One CSV line per YAML block -## Example session +### 5. Report -User: "dry-run the docs" +Follow `procedures/report.md`. Key points: -1. Check cluster exists, CRDs installed -2. Extract 108 YAML blocks from 26 docs -3. Dry-run validate all blocks (~2 minutes) -4. Report: 97 PASS, 4 EXPECTED FAIL, 7 REAL FAIL -5. List each real failure with the fix needed -6. Write `TEST_DRYRUN_ALL.md` +- Always output the per-doc breakdown table to the user +- The table is the PRIMARY output of this skill +- Bold non-zero fail counts +- List real failures with error messages and fix suggestions +- Write results to `TEST_DRYRUN_*.md` diff --git a/.claude/skills/test-docs-dryrun/procedures/cluster-setup.md b/.claude/skills/test-docs-dryrun/procedures/cluster-setup.md new file mode 100644 index 00000000..100b09d7 --- /dev/null +++ b/.claude/skills/test-docs-dryrun/procedures/cluster-setup.md @@ -0,0 +1,42 @@ +# Cluster setup + +How to manage the kind cluster for dry-run validation. The goal is to avoid unnecessary cluster creation/deletion between runs. + +## Check for existing cluster + +Always check first. Do NOT create a cluster without checking. + +```bash +kind get clusters 2>/dev/null | grep -q "^toolhive$" +``` + +If the cluster exists, verify CRDs are installed: + +```bash +kubectl get crd --context kind-toolhive 2>/dev/null | grep -c toolhive.stacklok.dev +``` + +If CRDs are present (count > 0), skip straight to extraction. No setup needed. + +## Create cluster only if missing + +If no `toolhive` cluster exists: + +```bash +kind create cluster --name toolhive +helm upgrade --install toolhive-operator-crds \ + oci://ghcr.io/stacklok/toolhive/toolhive-operator-crds \ + -n toolhive-system --create-namespace +``` + +The operator is NOT needed for dry-run validation. Only install CRDs. + +## After the run: keep the cluster + +Default to keeping the cluster after the run. Do NOT delete it unless the user explicitly asks. This avoids the 2-minute cluster creation penalty on the next run. + +If the user asks to clean up, delete with: + +```bash +kind delete cluster --name toolhive +``` diff --git a/.claude/skills/test-docs-dryrun/procedures/extract.md b/.claude/skills/test-docs-dryrun/procedures/extract.md new file mode 100644 index 00000000..517cb211 --- /dev/null +++ b/.claude/skills/test-docs-dryrun/procedures/extract.md @@ -0,0 +1,61 @@ +# Extract YAML blocks + +How to extract ToolHive CRD YAML blocks from documentation files. Always use the extraction script - do NOT write inline bash to parse YAML. + +## Commands + +Use the `--prefix` flag to avoid filename collisions between sections. This is critical because both sections have files with the same name (e.g., `telemetry-and-metrics.mdx`). + +```bash +SKILL_PATH="" +YAML_DIR="$(mktemp -d)/yaml" +mkdir -p "$YAML_DIR" + +# K8s Operator docs +for f in docs/toolhive/guides-k8s/*.mdx; do + python3 "$SKILL_PATH/scripts/extract-yaml.py" "$f" "$YAML_DIR" --prefix k8s +done + +# vMCP docs +for f in docs/toolhive/guides-vmcp/*.mdx; do + python3 "$SKILL_PATH/scripts/extract-yaml.py" "$f" "$YAML_DIR" --prefix vmcp +done +``` + +Replace `` with the absolute path to this skill's directory. + +## Output + +The script writes one file per YAML block: + +```text +k8s-auth-k8s_0.yaml +k8s-auth-k8s_1.yaml +k8s-run-mcp-k8s_0.yaml +vmcp-authentication_0.yaml +vmcp-telemetry-and-metrics_0.yaml +``` + +The prefix ensures `k8s-telemetry-and-metrics_0.yaml` and `vmcp-telemetry-and-metrics_0.yaml` don't collide. + +## For single section + +If only validating one section, use just the relevant loop: + +```bash +# K8s only +for f in docs/toolhive/guides-k8s/*.mdx; do + python3 "$SKILL_PATH/scripts/extract-yaml.py" "$f" "$YAML_DIR" --prefix k8s +done + +# vMCP only +for f in docs/toolhive/guides-vmcp/*.mdx; do + python3 "$SKILL_PATH/scripts/extract-yaml.py" "$f" "$YAML_DIR" --prefix vmcp +done +``` + +## For single page + +```bash +python3 "$SKILL_PATH/scripts/extract-yaml.py" "$YAML_DIR" --prefix single +``` diff --git a/.claude/skills/test-docs-dryrun/procedures/report.md b/.claude/skills/test-docs-dryrun/procedures/report.md new file mode 100644 index 00000000..cc082e2f --- /dev/null +++ b/.claude/skills/test-docs-dryrun/procedures/report.md @@ -0,0 +1,75 @@ +# Report results + +How to produce the per-doc breakdown table from the results CSV. + +## Build the table from CSV + +The validate step writes a CSV at `$RESULTS` with columns: `doc,filename,result,detail` + +Aggregate it into the per-doc table using this bash: + +```bash +RESULTS="" + +echo "| Doc | Section | Blocks | Pass | Fail | Expected | Skip |" +echo "|-----|---------|--------|------|------|----------|------|" + +TBLOCKS=0; TPASS=0; TFAIL=0; TEXPECTED=0; TSKIP=0 + +for doc in $(cut -d',' -f1 "$RESULTS" | sort -u); do + blocks=$(grep -c "^$doc," "$RESULTS") + pass=$(grep -c "^$doc,[^,]*,pass" "$RESULTS" || true) + fail=$(grep -c "^$doc,[^,]*,fail" "$RESULTS" || true) + expected=$(grep -c "^$doc,[^,]*,expected" "$RESULTS" || true) + skip=$(grep -c "^$doc,[^,]*,skip" "$RESULTS" || true) + + # Determine section from prefix + section="K8s" + echo "$doc" | grep -q "^vmcp-" && section="vMCP" + + # Bold non-zero fail counts + fail_display="$fail" + [ "$fail" -gt 0 ] && fail_display="**$fail**" + + echo "| ${doc} | ${section} | ${blocks} | ${pass} | ${fail_display} | ${expected} | ${skip} |" + + TBLOCKS=$((TBLOCKS + blocks)) + TPASS=$((TPASS + pass)) + TFAIL=$((TFAIL + fail)) + TEXPECTED=$((TEXPECTED + expected)) + TSKIP=$((TSKIP + skip)) +done + +echo "| **TOTAL** | | **$TBLOCKS** | **$TPASS** | **$TFAIL** | **$TEXPECTED** | **$TSKIP** |" +``` + +## Table rules + +- Include every doc that had at least 1 YAML block extracted +- Omit docs with 0 blocks +- Group K8s docs first (prefix `k8s-`), then vMCP docs (prefix `vmcp-`) +- Sort alphabetically within each section +- Bold the TOTAL row +- Bold any non-zero Fail count + +## After the table + +List each real failure with: + +```text +### Real failures + +**k8s-mcp-server-entry_1.yaml**: `strict decoding error: unknown field "spec.remoteURL"` + Fix: Change `remoteURL` to `remoteUrl` + +**vmcp-optimizer_3.yaml**: `strict decoding error: unknown field "spec.modelCache.storageSize"` + Fix: Change `storageSize` to `size` +``` + +## Write to file + +Always write the table and failure details to a markdown file: + +- Single page: `TEST_DRYRUN_.md` +- Section: `TEST_DRYRUN_.md` +- All: `TEST_DRYRUN_ALL.md` diff --git a/.claude/skills/test-docs-dryrun/procedures/validate.md b/.claude/skills/test-docs-dryrun/procedures/validate.md new file mode 100644 index 00000000..de20f671 --- /dev/null +++ b/.claude/skills/test-docs-dryrun/procedures/validate.md @@ -0,0 +1,55 @@ +# Validate YAML blocks + +How to dry-run validate extracted YAML blocks efficiently. + +## Command + +Run all files in a single batch using a loop with result tracking. Write results to a CSV file for reliable aggregation (avoids zsh associative array issues). + +```bash +YAML_DIR="" +RESULTS="$YAML_DIR/../results.csv" + +for f in "$YAML_DIR"/*.yaml; do + bn=$(basename "$f") + doc=$(echo "$bn" | sed 's/_[0-9]*\.yaml$//') + + # Skip incomplete resources (snippets without kind/apiVersion/name) + if ! grep -q 'kind:' "$f" || ! grep -q 'apiVersion:' "$f" || ! grep -q 'name:' "$f"; then + echo "$doc,$bn,skip,incomplete fragment" >> "$RESULTS" + continue + fi + + OUT=$(kubectl apply --dry-run=server -f "$f" 2>&1) + EC=$? + + if [ $EC -eq 0 ]; then + echo "$doc,$bn,pass," >> "$RESULTS" + elif echo "$OUT" | grep -q 'namespaces.*not found'; then + echo "$doc,$bn,pass,namespace missing but schema valid" >> "$RESULTS" + elif echo "$OUT" | grep -qE '||'; then + echo "$doc,$bn,expected,placeholder name" >> "$RESULTS" + elif echo "$OUT" | grep -qE 'spec\.incomingAuth: Required value|spec\.config\.compositeTools\[0\]\.steps: Required value'; then + echo "$doc,$bn,expected,partial snippet" >> "$RESULTS" + else + # Real failure - include error on a single line + ERR=$(echo "$OUT" | tr '\n' ' ' | head -c 200) + echo "$doc,$bn,fail,$ERR" >> "$RESULTS" + fi +done +``` + +## Result classification + +| Result | Meaning | Action needed? | +| ---------- | ---------------------------------------- | ----------------- | +| `pass` | Schema valid (or only namespace missing) | No | +| `expected` | Placeholder name or partial snippet | No | +| `fail` | Real schema error | Yes - fix the doc | +| `skip` | Incomplete fragment (no kind/name) | No | + +## Important notes + +- Use a CSV file for results, not bash associative arrays (zsh compatibility) +- Write one line per YAML block with: doc name, filename, result, detail +- The CSV is consumed by the report procedure to build the table diff --git a/.claude/skills/test-docs-dryrun/scripts/extract-yaml.py b/.claude/skills/test-docs-dryrun/scripts/extract-yaml.py index 239c0a9c..81c646f8 100644 --- a/.claude/skills/test-docs-dryrun/scripts/extract-yaml.py +++ b/.claude/skills/test-docs-dryrun/scripts/extract-yaml.py @@ -5,11 +5,22 @@ """Extract ToolHive CRD YAML blocks from Docusaurus .mdx files. Usage: - python3 extract-yaml.py + python3 extract-yaml.py [--prefix
] Finds all ```yaml/```yml fenced code blocks, splits multi-document blocks on ---, and writes each document containing toolhive.stacklok.dev as a separate .yaml file. + +The --prefix option adds a section prefix to output filenames to avoid +collisions when multiple sections have files with the same basename +(e.g., both guides-k8s/ and guides-vmcp/ have telemetry-and-metrics.mdx). + +Examples: + python3 extract-yaml.py docs/toolhive/guides-k8s/auth-k8s.mdx /tmp/yaml --prefix k8s + # Output: k8s-auth-k8s_0.yaml, k8s-auth-k8s_1.yaml, ... + + python3 extract-yaml.py docs/toolhive/guides-vmcp/optimizer.mdx /tmp/yaml --prefix vmcp + # Output: vmcp-optimizer_0.yaml, vmcp-optimizer_1.yaml, ... """ import os @@ -49,22 +60,33 @@ def extract_yaml_blocks(filepath): def main(): - if len(sys.argv) != 3: - print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + # Parse args: [--prefix
] + args = sys.argv[1:] + prefix = "" + if "--prefix" in args: + idx = args.index("--prefix") + prefix = args[idx + 1] + "-" + args = args[:idx] + args[idx + 2 :] + + if len(args) != 2: + print( + f"Usage: {sys.argv[0]} [--prefix
]", + file=sys.stderr, + ) sys.exit(1) - filepath = sys.argv[1] - output_dir = sys.argv[2] + filepath = args[0] + output_dir = args[1] basename = os.path.basename(filepath).replace(".mdx", "") blocks = extract_yaml_blocks(filepath) for i, block in enumerate(blocks): - outfile = os.path.join(output_dir, f"{basename}_{i}.yaml") + outfile = os.path.join(output_dir, f"{prefix}{basename}_{i}.yaml") with open(outfile, "w") as f: f.write(block + "\n") - print(f" {basename}: {len(blocks)} blocks") + print(f" {prefix}{basename}: {len(blocks)} blocks") if __name__ == "__main__": From daea02394054e71bc9b3ccb4ec1cc639ba6a9ba1 Mon Sep 17 00:00:00 2001 From: Chris Burns <29541485+ChrisJBurns@users.noreply.github.com> Date: Wed, 15 Apr 2026 22:02:25 +0100 Subject: [PATCH 3/4] Fix validation procedure to use script file Long-running for loops with nested if/elif and subshells get backgrounded or time out when run as inline Bash tool calls. The procedure now instructs writing the loop to a .sh file first, then executing with bash. Also uses || true after kubectl and checks output string instead of exit code for reliability. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../test-docs-dryrun/procedures/validate.md | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/.claude/skills/test-docs-dryrun/procedures/validate.md b/.claude/skills/test-docs-dryrun/procedures/validate.md index de20f671..6bafd7d6 100644 --- a/.claude/skills/test-docs-dryrun/procedures/validate.md +++ b/.claude/skills/test-docs-dryrun/procedures/validate.md @@ -2,13 +2,21 @@ How to dry-run validate extracted YAML blocks efficiently. -## Command +## Important: use a script file, not inline bash -Run all files in a single batch using a loop with result tracking. Write results to a CSV file for reliable aggregation (avoids zsh associative array issues). +Long-running `for` loops with nested `if/elif/else` and subshells do NOT work reliably as inline Bash tool calls - they get backgrounded or time out. Always write the validation loop to a temporary `.sh` file and execute it with `bash`. + +## Step 1: Write the validation script + +Write the following to a temp file (e.g., `$YAML_DIR/../validate.sh`). Replace `` with the actual YAML directory path. ```bash +#!/bin/bash +set -euo pipefail + YAML_DIR="" RESULTS="$YAML_DIR/../results.csv" +> "$RESULTS" for f in "$YAML_DIR"/*.yaml; do bn=$(basename "$f") @@ -20,25 +28,35 @@ for f in "$YAML_DIR"/*.yaml; do continue fi - OUT=$(kubectl apply --dry-run=server -f "$f" 2>&1) - EC=$? + OUT=$(kubectl apply --dry-run=server -f "$f" 2>&1) || true - if [ $EC -eq 0 ]; then + if echo "$OUT" | grep -q 'created (server dry run)'; then echo "$doc,$bn,pass," >> "$RESULTS" elif echo "$OUT" | grep -q 'namespaces.*not found'; then echo "$doc,$bn,pass,namespace missing but schema valid" >> "$RESULTS" elif echo "$OUT" | grep -qE '||'; then echo "$doc,$bn,expected,placeholder name" >> "$RESULTS" - elif echo "$OUT" | grep -qE 'spec\.incomingAuth: Required value|spec\.config\.compositeTools\[0\]\.steps: Required value'; then + elif echo "$OUT" | grep -qE 'incomingAuth: Required|compositeTools.*steps: Required'; then echo "$doc,$bn,expected,partial snippet" >> "$RESULTS" else - # Real failure - include error on a single line ERR=$(echo "$OUT" | tr '\n' ' ' | head -c 200) echo "$doc,$bn,fail,$ERR" >> "$RESULTS" fi done + +echo "Done: $(wc -l < "$RESULTS") lines" +``` + +## Step 2: Run the script + +Use the Write tool to create the script file, then execute it: + +```bash +bash $YAML_DIR/../validate.sh ``` +This runs in a single process and completes reliably regardless of how many YAML blocks are validated. + ## Result classification | Result | Meaning | Action needed? | @@ -50,6 +68,9 @@ done ## Important notes -- Use a CSV file for results, not bash associative arrays (zsh compatibility) +- ALWAYS write to a script file first, then execute with `bash` +- Use `|| true` after `kubectl apply` to prevent `set -e` from exiting on validation failures +- Check for `created (server dry run)` in output rather than exit code, because some versions of kubectl return non-zero even on dry-run success with warnings +- Use a CSV file for results, not bash variables or associative arrays - Write one line per YAML block with: doc name, filename, result, detail - The CSV is consumed by the report procedure to build the table From 6d71bc2a16b49b1e0e1c2c2906067acb93597367 Mon Sep 17 00:00:00 2001 From: Chris Burns <29541485+ChrisJBurns@users.noreply.github.com> Date: Wed, 15 Apr 2026 22:21:07 +0100 Subject: [PATCH 4/4] Address PR review feedback for dry-run skill Co-Authored-By: Claude Opus 4.6 (1M context) --- .../test-docs-dryrun/scripts/check-prereqs.sh | 20 ++++++++++++------- .../test-docs-dryrun/scripts/extract-yaml.py | 6 +++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.claude/skills/test-docs-dryrun/scripts/check-prereqs.sh b/.claude/skills/test-docs-dryrun/scripts/check-prereqs.sh index 1ae9fe04..fcd765a9 100644 --- a/.claude/skills/test-docs-dryrun/scripts/check-prereqs.sh +++ b/.claude/skills/test-docs-dryrun/scripts/check-prereqs.sh @@ -29,24 +29,30 @@ fi # Check cluster connectivity echo "" echo "--- Kubernetes cluster ---" +CLUSTER_REACHABLE=false if kubectl cluster-info &>/dev/null; then CONTEXT=$(kubectl config current-context 2>/dev/null || echo "unknown") echo -e "${GREEN}[OK]${NC} Cluster reachable (context: $CONTEXT)" + CLUSTER_REACHABLE=true else echo -e "${RED}[MISSING]${NC} No Kubernetes cluster reachable" ERRORS=$((ERRORS + 1)) fi -# Check ToolHive CRDs installed +# Check ToolHive CRDs installed (only if cluster is reachable) echo "" echo "--- ToolHive CRDs ---" -CRD_COUNT=$(kubectl get crd 2>/dev/null | grep -c toolhive.stacklok.dev || true) -if [ "$CRD_COUNT" -gt 0 ]; then - echo -e "${GREEN}[OK]${NC} $CRD_COUNT ToolHive CRDs installed" +if [ "$CLUSTER_REACHABLE" = true ]; then + CRD_COUNT=$(kubectl get crd 2>/dev/null | grep -c toolhive.stacklok.dev || true) + if [ "$CRD_COUNT" -gt 0 ]; then + echo -e "${GREEN}[OK]${NC} $CRD_COUNT ToolHive CRDs installed" + else + echo -e "${RED}[MISSING]${NC} No ToolHive CRDs found" + echo " Install with: helm upgrade --install toolhive-operator-crds oci://ghcr.io/stacklok/toolhive/toolhive-operator-crds -n toolhive-system --create-namespace" + ERRORS=$((ERRORS + 1)) + fi else - echo -e "${RED}[MISSING]${NC} No ToolHive CRDs found" - echo " Install with: helm upgrade --install toolhive-operator-crds oci://ghcr.io/stacklok/toolhive/toolhive-operator-crds -n toolhive-system --create-namespace" - ERRORS=$((ERRORS + 1)) + echo -e "${YELLOW}[SKIPPED]${NC} Cannot check CRDs (cluster unreachable)" fi # Check python3 for extraction script diff --git a/.claude/skills/test-docs-dryrun/scripts/extract-yaml.py b/.claude/skills/test-docs-dryrun/scripts/extract-yaml.py index 81c646f8..f368a772 100644 --- a/.claude/skills/test-docs-dryrun/scripts/extract-yaml.py +++ b/.claude/skills/test-docs-dryrun/scripts/extract-yaml.py @@ -30,7 +30,7 @@ def extract_yaml_blocks(filepath): """Extract YAML blocks containing toolhive apiVersion from an mdx file.""" - with open(filepath) as f: + with open(filepath, encoding="utf-8") as f: content = f.read() # Match ```yaml or ```yml code blocks (with optional metadata after yaml/yml) @@ -77,13 +77,13 @@ def main(): filepath = args[0] output_dir = args[1] - basename = os.path.basename(filepath).replace(".mdx", "") + basename, _ = os.path.splitext(os.path.basename(filepath)) blocks = extract_yaml_blocks(filepath) for i, block in enumerate(blocks): outfile = os.path.join(output_dir, f"{prefix}{basename}_{i}.yaml") - with open(outfile, "w") as f: + with open(outfile, "w", encoding="utf-8") as f: f.write(block + "\n") print(f" {prefix}{basename}: {len(blocks)} blocks")