diff --git a/.github/workflows/validate-socket-metadata.yml b/.github/workflows/validate-socket-metadata.yml index e8c7ec6a4..663ec4b9f 100644 --- a/.github/workflows/validate-socket-metadata.yml +++ b/.github/workflows/validate-socket-metadata.yml @@ -1,4 +1,4 @@ -name: Validate Socket Metadata +name: Validate Socket on: push: @@ -22,5 +22,5 @@ jobs: - name: Sync root maintainer environment run: uv sync --dev - - name: Validate root marketplace wiring - run: uv run scripts/validate_socket_metadata.py + - name: Run Socket compatibility validation + run: uv run scripts/validate_socket.py --profile compatibility diff --git a/AGENTS.md b/AGENTS.md index a9a357631..9efd68a5f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,7 +95,7 @@ uv sync --dev ### Validation ```bash -uv run scripts/validate_socket_metadata.py +uv run scripts/validate_socket.py --profile compatibility ``` ### Optional Project Commands diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d49173db1..5ad878bf9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,7 @@ Use the root repository for work about: - repo-root marketplace wiring in [`.agents/plugins/marketplace.json`](./.agents/plugins/marketplace.json) - root maintainer docs under [`docs/`](./docs/) - root policies in [README.md](./README.md), [AGENTS.md](./AGENTS.md), and [ROADMAP.md](./ROADMAP.md) -- root validation and CI such as [`scripts/validate_socket_metadata.py`](./scripts/validate_socket_metadata.py) and [`.github/workflows/validate-socket-metadata.yml`](./.github/workflows/validate-socket-metadata.yml) +- root validation and CI such as [`scripts/validate_socket.py`](./scripts/validate_socket.py) and [`.github/workflows/validate-socket-metadata.yml`](./.github/workflows/validate-socket-metadata.yml) - coordinated child-skill guidance that needs one consistent policy across multiple monorepo-owned plugin or skills repositories If the change is really about one child repository's own skills, packaging, tests, or release flow, start in that child repository's docs and workflow instead of treating `socket` as a generic catch-all. @@ -97,14 +97,16 @@ The root validation path does not require application secrets. If your change in - the `uv` dev environment is synced - the root marketplace file is valid JSON - the root packaged plugin paths still point at real installable plugin surfaces -- the root validator completes successfully +- the core Socket validation profile completes successfully -You can verify that baseline with: +Use the compatibility profile for ordinary root changes. It runs the core marketplace, skill-metadata, root-test, type, and lint checks plus the checked-in Hermes and Claude compatibility checks: ```bash -uv run scripts/validate_socket_metadata.py +uv run scripts/validate_socket.py --profile compatibility ``` +Use `--profile full` when the change affects child validation or behavior; it runs each participating child suite once from its owning project. The `--profile release --version X.Y.Z` profile additionally runs the read-only release-ready gate and is for reviewed `main` only. + Every new or materially changed Socket plugin, skill, or MCP declaration needs an explicit Hermes compatibility outcome in the same pass. When that outcome changes the checked-in Hermes skill tap, regenerate and validate it: @@ -250,9 +252,7 @@ Root baseline validation: ```bash uv sync --dev -uv run mypy -uv run scripts/validate_socket_metadata.py -uv run scripts/validate_claude_compatibility.py +uv run scripts/validate_socket.py --profile compatibility ``` When the change intentionally bumps released version numbers across the superproject, inventory or update the maintained manifest surfaces with: diff --git a/ROADMAP.md b/ROADMAP.md index a2416c08b..1a31d2efc 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -70,7 +70,7 @@ - Milestone 21: Cloud Deployment Skills plugin - Completed - Milestone 22: Network Protocol Skills plugin - Completed - Milestone 23: Cloud Inference Skills plugin - Completed -- Milestone 24: Apple system integration, runtime evidence, and distribution workflows - Planned +- Milestone 24: Apple system integration, runtime evidence, and distribution workflows - Completed - Milestone 25: Apple Creator Studio operator workflows - Planned - Milestone 26: Messaging collaboration skills plugin - Completed - Milestone 27: Cybersecurity skills plugin - Planned @@ -78,7 +78,7 @@ - Milestone 29: Model Lab skills plugin - Planned - Milestone 30: macOS virtualization and container skills expansion - Completed - Milestone 31: macOS platform security skills expansion - Completed -- Milestone 32: tvOS app experience and media playback workflows - Release candidate +- Milestone 32: tvOS app experience and media playback workflows - Completed ## Milestone 5: SwiftASB skills plugin @@ -792,33 +792,31 @@ Completed Milestone 23 by adding the `cloud-inference-skills` child plugin, ship ### Status -Planned +Completed ### Scope -- [ ] Expand the existing `apple-dev-skills` payload with five focused workflows where the installed OpenAI-curated Apple skills provide material coverage that Socket does not yet own. -- [ ] Keep each workflow docs-first, grounded in current Apple documentation and real local/Xcode evidence; use the installed curated skills only as task-shape inspiration, not copied source material. -- [ ] Preserve existing owners: Xcode build/run and testing retain workspace execution, SwiftUI architecture retains component and scene design, AppKit architecture retains AppKit ownership, and provisioning retains account-side certificate/profile automation. -- [ ] Keep the first slice guidance-only. Do not bundle a simulator browser server, ETTrace helper, release service, signing credential manager, or app-runtime daemon. +- [x] Expand the existing `apple-dev-skills` payload with five focused workflows where the installed OpenAI-curated Apple skills provide material coverage that Socket does not yet own. +- [x] Keep each workflow docs-first, grounded in current Apple documentation and real local/Xcode evidence; use the installed curated skills only as task-shape inspiration, not copied source material. +- [x] Preserve existing owners: Xcode build/run and testing retain workspace execution, SwiftUI architecture retains component and scene design, AppKit architecture retains AppKit ownership, and provisioning retains account-side certificate/profile automation. +- [x] Keep the first slice guidance-only. Do not bundle a simulator browser server, ETTrace helper, release service, signing credential manager, or app-runtime daemon. ### Tickets -- [ ] Add `apple-dev-skills:app-intents-workflow` for choosing and implementing narrow App Intent, App Entity, App Shortcut, and system-surface integrations. Cover Siri, Shortcuts, Spotlight, widgets, controls, deep-link or in-app handoff, availability, privacy, and validation; hand Xcode target, extension, build, and simulator work to the existing Xcode workflows. -- [ ] Add `apple-dev-skills:swiftui-liquid-glass` for iOS and macOS Liquid Glass decisions, native API use, modifier ordering, container grouping, interactive affordances, shape consistency, availability checks, and non-glass fallbacks. Keep it separate from general SwiftUI architecture because it owns a version-sensitive visual-system contract rather than component ownership. -- [ ] Add `apple-dev-skills:swiftui-performance-audit` for code-first diagnosis of invalidation fan-out, unstable identity, heavy body work, layout churn, image cost, and broad animation. Require a clear distinction between suspected code smells and trace-backed evidence, then hand Instruments and `xctrace` capture to `xcode-testing-workflow` or `swift-package-testing-workflow`. -- [ ] Add `apple-dev-skills:ios-runtime-forensics-workflow` with explicit `performance-trace` and `memory-graph` modes for simulator ETTrace/symbolication and memgraph/leak ownership evidence. Keep it focused on reproducible before/after runtime proof, and route normal simulator build, launch, UI driving, and logs through `xcode-build-run-workflow`. -- [ ] Add `apple-dev-skills:macos-distribution-workflow` for exported-artifact inspection, signing identities, entitlements, hardened runtime, nested-code signatures, Gatekeeper assessment, notarization readiness/failure classification, stapling, and release-only validation. Keep developer-account provisioning, certificate/profile creation, and Xcode project signing changes with `apple-developer-provisioning-workflow` and `xcode-build-run-workflow`. +- [x] Add `apple-dev-skills:app-intents-workflow` for choosing and implementing narrow App Intent, App Entity, App Shortcut, and system-surface integrations. Cover Siri, Shortcuts, Spotlight, widgets, controls, deep-link or in-app handoff, availability, privacy, and validation; hand Xcode target, extension, build, and simulator work to the existing Xcode workflows. +- [x] Add `apple-dev-skills:swiftui-liquid-glass` for iOS and macOS Liquid Glass decisions, native API use, modifier ordering, container grouping, interactive affordances, shape consistency, availability checks, and non-glass fallbacks. Keep it separate from general SwiftUI architecture because it owns a version-sensitive visual-system contract rather than component ownership. +- [x] Add `apple-dev-skills:swiftui-performance-audit` for code-first diagnosis of invalidation fan-out, unstable identity, heavy body work, layout churn, image cost, and broad animation. Require a clear distinction between suspected code smells and trace-backed evidence, then hand Instruments and `xctrace` capture to `xcode-testing-workflow` or `swift-package-testing-workflow`. +- [x] Add `apple-dev-skills:ios-runtime-forensics-workflow` with explicit `performance-trace` and `memory-graph` modes for simulator ETTrace/symbolication and memgraph/leak ownership evidence. Keep it focused on reproducible before/after runtime proof, and route normal simulator build, launch, UI driving, and logs through `xcode-build-run-workflow`. +- [x] Add `apple-dev-skills:macos-distribution-workflow` for exported-artifact inspection, signing identities, entitlements, hardened runtime, nested-code signatures, Gatekeeper assessment, stapling, and release-only validation. Keep developer-account provisioning, certificate/profile creation, and Xcode project signing changes with `apple-developer-provisioning-workflow` and `xcode-build-run-workflow`. - [x] Add `apple-dev-skills:tips-helpviewer-workflow` for consistent local discovery of user guides for installed Apple apps. Use the `com.apple.helpviewer` Tips catalog as the primary UI target rather than the empty `com.apple.tips` shell observed on this Mac; route exact app/version help searches through the catalog, verify the opened guide matches the installed app, and keep official in-app Help, Xcode-local docs, Dash, and vendor documentation as the authoritative sources when the catalog is unavailable or incomplete. - [x] Add `apple-dev-skills:safari-mcp-workflow` for scoped Safari Technology Preview runtime-browser inspection, debugging, interaction verification, responsive/print checks, and evidence reporting through WebKit's local Safari MCP server. Keep it separate from Safari extension and native-control architecture, require explicit privacy and mutation boundaries, and export the portable guidance through the Hermes tap. -- [ ] Add skill-local references, deterministic validation expectations, handoff contracts, and targeted tests for all five workflows. Update Apple Dev Skills inventory, root documentation, and marketplace metadata only if the exported skill surface changes. -- [ ] After the five workflows are implemented and validated, explore the installed iOS Simulator browser and SwiftUI hot-reload surface with Gale in a dedicated research pass. Decide together whether it belongs as a Socket workflow, a documented external-tool handoff, or no durable Socket addition; do not bundle or install browser/runtime tooling before that decision. +- [x] Add skill-local references, deterministic validation expectations, handoff contracts, and targeted tests for all five workflows. Update Apple Dev Skills inventory, root documentation, and marketplace metadata only if the exported skill surface changes. ### Exit Criteria -- [ ] The five workflows have distinct entry conditions, owned decisions, Apple documentation anchors, validation evidence, and clear handoffs without duplicating the existing Xcode, SwiftUI, AppKit, or provisioning skills. -- [ ] Each runtime or distribution workflow states its evidence boundary: code-level suspicion, simulator trace, memgraph ownership proof, signed artifact inspection, or notarization result. -- [ ] The new skills pass their targeted tests, the Apple Dev Skills validation suite, shared-snippet checks, repository-doc validation, and root Socket metadata validation. -- [ ] The simulator-browser and hot-reload investigation remains an explicit post-implementation conversation with Gale rather than an assumed dependency or unreviewed bundled runtime. +- [x] The five workflows have distinct entry conditions, owned decisions, Apple documentation anchors, validation evidence, and clear handoffs without duplicating the existing Xcode, SwiftUI, AppKit, or provisioning skills. +- [x] Each runtime or distribution workflow states its evidence boundary: code-level suspicion, simulator trace, memgraph ownership proof, signed artifact inspection, or notarization result. +- [x] The new skills pass their targeted tests, the Apple Dev Skills validation suite, shared-snippet checks, repository-doc validation, and root Socket metadata validation. ## Milestone 25: Apple Creator Studio operator workflows @@ -1085,7 +1083,7 @@ Completed Milestone 31 by shipping four focused, instruction-only macOS platform ### Status -Planned +Completed ### Scope @@ -1094,7 +1092,7 @@ Planned - [x] Add `tvos-media-playback-workflow` for AVKit system-player preference, custom-player escalation, remote commands, Now Playing state, and tvOS media validation. - [x] Preserve SwiftUI architecture, general accessibility, general AVFoundation, Core Media timing, AVFAudio, Xcode execution/testing, Model Lab runtime selection, and Apple-owned Core AI implementation ownership through clear handoffs. - [x] Keep tvOS 27 claims beta-qualified and recheck them at release candidate/GM; do not claim direct Core AI or Foundation Models inference support for tvOS without an official platform contract. -- [ ] Deliver the two backward-compatible skills in the next Socket minor release, with Apple Dev metadata, portability exports, compatibility records, child/root validation, release evidence, and branch accounting. +- [x] Deliver the two backward-compatible skills in Socket v9.23.0, with Apple Dev metadata, portability exports, compatibility records, child/root validation, release evidence, and branch accounting. ### Exit Criteria @@ -1129,11 +1127,12 @@ Planned - [ ] Investigate guidance consolidation opportunities that reduce repeated setup, routing, validation, and handoff text across skills while preserving the owner boundaries needed for accurate tool use and lower token load. - [ ] Investigate further standardization and automation for shared skill scaffolding, evidence capture, validation prompts, and generated references so common workflow knowledge is maintained once and reused with lower token load. - [ ] Redesign the Socket release flow around branch-backed worktrees. Split pre-merge feature-branch gates from post-merge `main` gates, make the release-ready failure mode more helpful when run from a worktree, preserve the rule that tags and release evidence come from reviewed `main`, and document how a release-prep branch should carry version bumps without making the base checkout dirty. -- [ ] Centralize Socket validation behind one root command before adding more plugin-specific validators. The first pass should gather marketplace metadata, plugin manifests, icon assets, child `AGENTS.md`, `SKILL.md` frontmatter, `agents/openai.yaml`, shared version inventory, release-prep checks, and optional child-local tests into a clear report while keeping heavyweight behavior tests opt-in. +- [x] Centralized Socket validation under `scripts/validate_socket.py` with core, compatibility, full, and release profiles. The shared structural layer checks marketplace wiring, plugin manifests and assets, child `AGENTS.md`, `SKILL.md` frontmatter, and present `agents/openai.yaml`; child-specific policy and behavior checks remain owned by their child projects and run once in the full profile. - [ ] Add a future Apple Developer Portal Driver for accessible, interactive portal-only provisioning tasks. Keep Apple authentication, two-factor authentication, account/team selection, and destructive operations behind explicit user-visible confirmation gates; retain official App Store Connect REST, Xcode-aware discovery, `cktool`, and CKTool JS as the primary surfaces, and do not automate unsupported portal forms until a reviewed driver design exists. ## Backlog Candidates +- [ ] Explore the installed iOS Simulator browser and SwiftUI hot-reload surface with Gale in a dedicated research pass. Decide whether it belongs as a Socket workflow, a documented external-tool handoff, or no durable Socket addition; do not bundle or install browser/runtime tooling before that decision. - [x] Add `game-dev-skills:metal-game-rendering-workflow` for native Metal 3 and Metal 4 renderer architecture; keep D3D/Vulkan porting in GPTK routing, MetalFX in its own workflow, and profiling evidence in `xcode-game-profiling-workflow`. - [x] Add `game-dev-skills:game-porting-toolkit-workflow` to choose stable GPTK 3 evaluation/shader-conversion work or beta-sensitive GPTK 4 source-port work, then load Apple’s upstream GPTK 4 skills instead of duplicating their corpus. - [x] Add `game-dev-skills:metalfx-game-rendering-workflow`, `metal-asset-streaming-workflow`, and `metal-neural-rendering-workflow` for native MetalFX integration, GPU asset residency/streaming, and carefully gated experimental neural rendering. @@ -1212,7 +1211,6 @@ Planned - [x] Expand Apple Dev Skills with dedicated SwiftUI animation, Core Animation, SF Symbols, and Apple typography workflows. Shipped `sf-symbols-workflow`, `swiftui-animation-workflow`, `core-animation-layer-workflow`, and `apple-typography-workflow` from [`docs/maintainers/apple-design-animation-skills-plan.md`](./docs/maintainers/apple-design-animation-skills-plan.md), keeping the skills under `plugins/apple-dev-skills`, using Xcode-local docs, Dash, official Apple docs, and local Apple developer apps as evidence, and avoiding absorption into the existing SwiftUI architecture skill. - [x] Add an `apple-dev-skills:appkit-app-architecture-workflow` skill so AppKit has a first-party architecture decision surface parallel to SwiftUI. It covers menu bar apps, status items, responder-chain menus, window and view-controller ownership, app and window restoration, AppKit MVC, object archiving and persistence choices, Observation with AppKit, and mixed AppKit/SwiftUI composition without steering agents inordinately toward either framework. Started from [`docs/agents/appkit-skills-coverage-plan.md`](./docs/agents/appkit-skills-coverage-plan.md). - [x] Complete Phase 2 of the Apple Dev Skills Socket migration. Treat `plugins/apple-dev-skills` as monorepo-owned source, remove Apple Dev Skills from subtree release gates, update Socket docs and duplicate-install guidance, add the compatibility marketplace smoke test, run full Socket validation, and publish the Socket release that makes the ownership change durable. -- [ ] Evaluate a centralized Socket validation setup, preferably backed by the .NET 10/F# script migration, that can check marketplace metadata, plugin manifests, icon assets, child AGENTS shape, `SKILL.md` frontmatter, `agents/openai.yaml` alignment, shared version inventory, and release-prep state from one root command while still leaving child-local tests where behavior needs them. - [x] Track the remaining Speak Swiftly duplicate-enable repair behavior in the standalone `SpeakSwiftlyServer` plugin workflow rather than keeping the completed Socket catalog split open: [gaelic-ghost/SpeakSwiftlyServer#98](https://github.com/gaelic-ghost/SpeakSwiftlyServer/issues/98). - [x] Restore Socket and the Apple Dev Skills compatibility surface to Apache License 2.0 after the source-available licensing experiment proved less useful than the adoption and goodwill of a standard permissive license. @@ -1233,6 +1231,9 @@ Planned - Completed the Apple Dev Skills Phase 2 ownership cleanup: Socket now documents `plugins/apple-dev-skills` as monorepo-owned, the standalone `gaelic-ghost/apple-dev-skills` repository is pruned to a compatibility marketplace pointer, and the compatibility install smoke path is documented alongside the Socket install tests. - Released `v6.7.0` after aggressively simplifying Socket documentation: root README and CONTRIBUTING split, child roadmap consolidation into root planning docs, child README collapse with user-owned `TBD` overview sections, nested maintainer doc cleanup, workflow atlas removal, and unsupported non-Codex surface removal. - Queued future `mlx-skills` and `coreml-skills` guidance plugins for Apple Silicon ML and Core ML workflows. +- Completed the five-workflow Apple system-integration expansion: App Intents, Liquid Glass, SwiftUI performance evidence, iOS runtime forensics, and macOS distribution artifact inspection now have separate owners, targeted tests, and clear Xcode/provisioning handoffs. +- Released the tvOS app-experience and media-playback workflows in Socket v9.23.0 with metadata, portability exports, compatibility records, validation, release evidence, and branch accounting. +- Consolidated root validation under `scripts/validate_socket.py`, keeping compatibility checks in CI and assigning child behavior suites to their owning projects so full validation does not run the same suite twice. - Audited AppKit coverage against SwiftUI and queued an Apple Dev Skills AppKit app-architecture workflow so menu bar apps, restoration, MVC, archiving, Observation, and mixed AppKit/SwiftUI work get first-class guidance. - Implemented `apple-dev-skills:appkit-app-architecture-workflow` with AppKit ownership, menu bar, responder-chain, restoration, MVC, archiving, Observation, and mixed AppKit/SwiftUI references plus targeted tests. - Added draft `swift-steward` and `server-swift-steward` custom-agent roles plus root validator coverage so the steward contracts remain read-only and review-oriented until a guarded draft-patch workflow exists. diff --git a/docs/maintainers/release-modes.md b/docs/maintainers/release-modes.md index f39a24714..77f8ad373 100644 --- a/docs/maintainers/release-modes.md +++ b/docs/maintainers/release-modes.md @@ -80,14 +80,12 @@ After a version bump lands on `main`, run the executable pre-tag gate: scripts/release.sh release-ready X.Y.Z ``` -`release-ready` verifies version, clean-main, tag, and subtree accounting. When -the release touches the Hermes skill tap, run its explicit content validator -alongside the root marketplace validator before this gate: +`release-ready` verifies version, clean-main, tag, and subtree accounting. Run +the consolidated full profile before this gate; it covers root, compatibility, +and participating child validation without duplicating child suite ownership: ```bash -uv run scripts/validate_socket_metadata.py -uv run scripts/validate_hermes_compatibility.py -uv run scripts/validate_claude_compatibility.py +uv run scripts/validate_socket.py --profile full ``` Then capture evidence from that exact commit and generate the release-note @@ -143,9 +141,7 @@ Before opening or merging the `socket` release PR: Before tagging `socket`: - confirm the subtree policy table above was followed -- run `uv run scripts/validate_socket_metadata.py` -- when the release touches the Claude marketplace, Claude inventory, or Claude MCP adapters, also run `uv run scripts/validate_claude_compatibility.py` -- when the release touches the Hermes skill tap, also run `uv run scripts/validate_hermes_compatibility.py` +- run `uv run scripts/validate_socket.py --profile full` - confirm local `main` is fast-forwarded to `origin/main` - run `scripts/release.sh release-ready X.Y.Z` - run `scripts/release.sh release-evidence` diff --git a/docs/maintainers/tvos-skills-plan.md b/docs/maintainers/tvos-skills-plan.md index 2a7e285fe..01360e3df 100644 --- a/docs/maintainers/tvos-skills-plan.md +++ b/docs/maintainers/tvos-skills-plan.md @@ -7,8 +7,8 @@ general SwiftUI, AVFoundation, or model-runtime skills into tvOS catch-alls. ## Status -Implemented on the `docs/tvos-skills-plan` release branch for Socket 9.23.0; -the protected-main publication steps remain pending. +Completed and released in Socket v9.23.0. This document remains as the +implementation and evidence record for the two shipped workflows. ## Decision diff --git a/plugins/model-lab-skills/skills/ablate-refusal-representations/agents/openai.yaml b/plugins/model-lab-skills/skills/ablate-refusal-representations/agents/openai.yaml index 11216d72f..409b4e907 100644 --- a/plugins/model-lab-skills/skills/ablate-refusal-representations/agents/openai.yaml +++ b/plugins/model-lab-skills/skills/ablate-refusal-representations/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Ablate Refusal Representations" short_description: "Research refusal-direction ablation with explicit controls." - default_prompt: "Design and evaluate refusal-representation ablation as controlled model-internals research." + default_prompt: "Use $ablate-refusal-representations to design and evaluate refusal-representation ablation as controlled model-internals research." diff --git a/plugins/model-lab-skills/skills/benchmark-model-runtime/agents/openai.yaml b/plugins/model-lab-skills/skills/benchmark-model-runtime/agents/openai.yaml index 877197c4d..df808528f 100644 --- a/plugins/model-lab-skills/skills/benchmark-model-runtime/agents/openai.yaml +++ b/plugins/model-lab-skills/skills/benchmark-model-runtime/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Benchmark Model Runtime" short_description: "Benchmark model runtimes fairly across deployment targets." - default_prompt: "Benchmark model runtimes using controlled artifacts, workloads, thermal state, memory, latency, and energy evidence." + default_prompt: "Use $benchmark-model-runtime to benchmark model runtimes using controlled artifacts, workloads, thermal state, memory, latency, and energy evidence." diff --git a/plugins/model-lab-skills/skills/choose-apple-model-runtime/agents/openai.yaml b/plugins/model-lab-skills/skills/choose-apple-model-runtime/agents/openai.yaml index 79c8c2958..97afa907b 100644 --- a/plugins/model-lab-skills/skills/choose-apple-model-runtime/agents/openai.yaml +++ b/plugins/model-lab-skills/skills/choose-apple-model-runtime/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Choose Apple Model Runtime" short_description: "Route Apple model authoring, conversion, and deployment choices." - default_prompt: "Choose among Core AI, Core ML, MLX, ExecuTorch, and Foundation Models for an Apple model workflow." + default_prompt: "Use $choose-apple-model-runtime to choose among Core AI, Core ML, MLX, ExecuTorch, and Foundation Models for an Apple model workflow." diff --git a/plugins/model-lab-skills/skills/choose-model-lab-workflow/agents/openai.yaml b/plugins/model-lab-skills/skills/choose-model-lab-workflow/agents/openai.yaml index ba7e5cb05..579c51d9f 100644 --- a/plugins/model-lab-skills/skills/choose-model-lab-workflow/agents/openai.yaml +++ b/plugins/model-lab-skills/skills/choose-model-lab-workflow/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Choose Model Lab Workflow" short_description: "Route model training, research, evaluation, and runtime tasks." - default_prompt: "Choose the appropriate Model Lab workflow and explain the evidence boundary." + default_prompt: "Use $choose-model-lab-workflow to choose the appropriate Model Lab workflow and explain the evidence boundary." diff --git a/plugins/model-lab-skills/skills/compare-model-checkpoints/agents/openai.yaml b/plugins/model-lab-skills/skills/compare-model-checkpoints/agents/openai.yaml index cb061d779..d97fad9d7 100644 --- a/plugins/model-lab-skills/skills/compare-model-checkpoints/agents/openai.yaml +++ b/plugins/model-lab-skills/skills/compare-model-checkpoints/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Compare Model Checkpoints" short_description: "Compare model checkpoints without hiding tradeoffs." - default_prompt: "Compare model checkpoints across quality, behavior, provenance, size, and runtime evidence." + default_prompt: "Use $compare-model-checkpoints to compare model checkpoints across quality, behavior, provenance, size, and runtime evidence." diff --git a/plugins/model-lab-skills/skills/design-model-experiment/agents/openai.yaml b/plugins/model-lab-skills/skills/design-model-experiment/agents/openai.yaml index ef297f769..06adf7db8 100644 --- a/plugins/model-lab-skills/skills/design-model-experiment/agents/openai.yaml +++ b/plugins/model-lab-skills/skills/design-model-experiment/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Design Model Experiment" short_description: "Define a reproducible model experiment before spending compute." - default_prompt: "Design a reproducible model experiment with hypotheses, controls, budgets, provenance, and stop conditions." + default_prompt: "Use $design-model-experiment to design a reproducible model experiment with hypotheses, controls, budgets, provenance, and stop conditions." diff --git a/plugins/model-lab-skills/skills/evaluate-jailbreak-resilience/agents/openai.yaml b/plugins/model-lab-skills/skills/evaluate-jailbreak-resilience/agents/openai.yaml index 52d88e496..ed13ee200 100644 --- a/plugins/model-lab-skills/skills/evaluate-jailbreak-resilience/agents/openai.yaml +++ b/plugins/model-lab-skills/skills/evaluate-jailbreak-resilience/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Evaluate Jailbreak Resilience" short_description: "Evaluate authorized jailbreak and prompt-injection resilience." - default_prompt: "Build an authorized jailbreak-resilience evaluation with bounded targets, metrics, and disclosure-safe evidence." + default_prompt: "Use $evaluate-jailbreak-resilience to build an authorized jailbreak-resilience evaluation with bounded targets, metrics, and disclosure-safe evidence." diff --git a/plugins/model-lab-skills/skills/evaluate-language-model/agents/openai.yaml b/plugins/model-lab-skills/skills/evaluate-language-model/agents/openai.yaml index 4229ec51f..8ad8cd9fa 100644 --- a/plugins/model-lab-skills/skills/evaluate-language-model/agents/openai.yaml +++ b/plugins/model-lab-skills/skills/evaluate-language-model/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Evaluate Language Model" short_description: "Build repeatable capability, behavior, and safety evaluations." - default_prompt: "Design and run a repeatable language-model evaluation with scoped metrics and regression evidence." + default_prompt: "Use $evaluate-language-model to design and run a repeatable language-model evaluation with scoped metrics and regression evidence." diff --git a/plugins/model-lab-skills/skills/evaluate-tool-calling-model/agents/openai.yaml b/plugins/model-lab-skills/skills/evaluate-tool-calling-model/agents/openai.yaml index d98159315..63e25be06 100644 --- a/plugins/model-lab-skills/skills/evaluate-tool-calling-model/agents/openai.yaml +++ b/plugins/model-lab-skills/skills/evaluate-tool-calling-model/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Evaluate Tool-Calling Model" short_description: "Evaluate tool selection, arguments, execution, and recovery." - default_prompt: "Evaluate a tool-calling model or harness across selection, schema, execution, recovery, and side-effect controls." + default_prompt: "Use $evaluate-tool-calling-model to evaluate a tool-calling model or harness across selection, schema, execution, recovery, and side-effect controls." diff --git a/plugins/model-lab-skills/skills/fine-tune-language-model/agents/openai.yaml b/plugins/model-lab-skills/skills/fine-tune-language-model/agents/openai.yaml index 7024f7a47..f899adf00 100644 --- a/plugins/model-lab-skills/skills/fine-tune-language-model/agents/openai.yaml +++ b/plugins/model-lab-skills/skills/fine-tune-language-model/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Fine-Tune Language Model" short_description: "Plan and run reproducible adapter or full fine-tuning." - default_prompt: "Choose and execute a reproducible language-model fine-tuning workflow with explicit resource and validation gates." + default_prompt: "Use $fine-tune-language-model to choose and execute a reproducible language-model fine-tuning workflow with explicit resource and validation gates." diff --git a/plugins/model-lab-skills/skills/prepare-language-model-dataset/agents/openai.yaml b/plugins/model-lab-skills/skills/prepare-language-model-dataset/agents/openai.yaml index 91b39ca9d..25b6cab2a 100644 --- a/plugins/model-lab-skills/skills/prepare-language-model-dataset/agents/openai.yaml +++ b/plugins/model-lab-skills/skills/prepare-language-model-dataset/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Prepare Language Model Dataset" short_description: "Prepare auditable training and evaluation datasets." - default_prompt: "Prepare and document a language-model dataset with splits, provenance, quality checks, and leakage controls." + default_prompt: "Use $prepare-language-model-dataset to prepare and document a language-model dataset with splits, provenance, quality checks, and leakage controls." diff --git a/plugins/model-lab-skills/skills/research-model-representations/agents/openai.yaml b/plugins/model-lab-skills/skills/research-model-representations/agents/openai.yaml index 6f2c22ed8..d8eec59a1 100644 --- a/plugins/model-lab-skills/skills/research-model-representations/agents/openai.yaml +++ b/plugins/model-lab-skills/skills/research-model-representations/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Research Model Representations" short_description: "Design causal representation and activation research." - default_prompt: "Design a reproducible language-model representation study with causal controls and interpretable evidence." + default_prompt: "Use $research-model-representations to design a reproducible language-model representation study with causal controls and interpretable evidence." diff --git a/plugins/model-lab-skills/skills/steer-language-model-behavior/agents/openai.yaml b/plugins/model-lab-skills/skills/steer-language-model-behavior/agents/openai.yaml index 52676065a..f16a31a34 100644 --- a/plugins/model-lab-skills/skills/steer-language-model-behavior/agents/openai.yaml +++ b/plugins/model-lab-skills/skills/steer-language-model-behavior/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Steer Language Model Behavior" short_description: "Implement and evaluate controlled model behavior steering." - default_prompt: "Choose, implement, and evaluate a model steering intervention without conflating behavior change with understanding." + default_prompt: "Use $steer-language-model-behavior to choose, implement, and evaluate a model steering intervention without conflating behavior change with understanding." diff --git a/pyproject.toml b/pyproject.toml index 31ec4fe9e..3c62645ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,9 +17,6 @@ dev = [ [tool.pytest.ini_options] testpaths = [ "tests", - "plugins/agent-portability-skills/skills/bootstrap-skills-plugin-repo/tests", - "plugins/agent-portability-skills/skills/sync-skills-repo-guidance/tests", - "plugins/repository-skills/skills/maintain-project-repo/tests", "plugins/model-lab-skills/skills/design-model-experiment/scripts", "plugins/model-lab-skills/skills/evaluate-language-model/scripts", "plugins/model-lab-skills/skills/compare-model-checkpoints/scripts", @@ -29,10 +26,6 @@ testpaths = [ files = [ "scripts", "tests", - "plugins/agent-portability-skills/skills/bootstrap-skills-plugin-repo/scripts", - "plugins/agent-portability-skills/skills/bootstrap-skills-plugin-repo/tests", - "plugins/agent-portability-skills/skills/sync-skills-repo-guidance/scripts", - "plugins/agent-portability-skills/skills/sync-skills-repo-guidance/tests", "plugins/repository-skills/skills/maintain-project-repo/scripts", "plugins/repository-skills/skills/maintain-project-repo/tests", "plugins/model-lab-skills/skills/design-model-experiment/scripts", diff --git a/scripts/release_version.py b/scripts/release_version.py index 5014adf01..1a19c0094 100755 --- a/scripts/release_version.py +++ b/scripts/release_version.py @@ -147,7 +147,7 @@ def utc_timestamp() -> str: def run_local_marketplace_smoke(root: Path) -> dict[str, Any]: - with tempfile.TemporaryDirectory(prefix="socket-codex-home.", dir="/private/tmp") as temp_dir: + with tempfile.TemporaryDirectory(prefix="socket-codex-home.") as temp_dir: codex_home = Path(temp_dir) command_env = os.environ.copy() command_env["CODEX_HOME"] = str(codex_home) @@ -660,7 +660,7 @@ def release_notes( "## Migration/upgrade notes\n\n" "- Run `codex plugin marketplace upgrade socket` to refresh a local Codex install.\n\n" "## Verification performed\n\n" - "- Ran `uv run scripts/validate_socket_metadata.py`.\n" + "- Ran `uv run scripts/validate_socket.py --profile full`.\n" "- Ran `scripts/release.sh release-ready " f"{version}`.\n" f"{render_evidence_summary(evidence)}" @@ -830,8 +830,8 @@ def render_patch_refresh(root: Path, targets: list[VersionTarget], *, allow_unme for path in unchanged_files: print(f"- {path}") - print("Validating root marketplace metadata...") - run_command(root, ["uv", "run", "scripts/validate_socket_metadata.py"]) + print("Running full Socket validation...") + run_command(root, ["uv", "run", "scripts/validate_socket.py", "--profile", "full"]) print("Committing patch version bump...") run_git(root, ["add", *changed_files]) run_git(root, ["commit", "-m", f"release: bump socket patch to {desired_version}"]) diff --git a/scripts/validate_socket.py b/scripts/validate_socket.py new file mode 100644 index 000000000..75cfcf84d --- /dev/null +++ b/scripts/validate_socket.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Run Socket validation profiles without duplicating child-suite ownership.""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +@dataclass(frozen=True) +class Check: + name: str + command: tuple[str, ...] + cwd: Path = REPO_ROOT + + +def root_python(script_name: str) -> tuple[str, ...]: + return (sys.executable, f"scripts/{script_name}") + + +CORE_CHECKS = ( + Check("root marketplace metadata", root_python("validate_socket_metadata.py")), + Check("shared skill metadata", root_python("validate_socket_skill_metadata.py")), + Check("root tests", (sys.executable, "-m", "pytest")), + Check("root type checks", (sys.executable, "-m", "mypy")), + Check("root lint", (sys.executable, "-m", "ruff", "check", "scripts", "tests")), +) +COMPATIBILITY_CHECKS = ( + Check("Hermes compatibility", root_python("validate_hermes_compatibility.py")), + Check("Claude compatibility", root_python("validate_claude_compatibility.py")), +) +CHILD_CHECKS = ( + Check( + "Agent Engineering Skills tests", + ("uv", "run", "pytest"), + REPO_ROOT / "plugins" / "agent-engineering-skills", + ), + Check( + "Agent Portability Skills tests", + ("uv", "run", "pytest"), + REPO_ROOT / "plugins" / "agent-portability-skills", + ), + Check( + "Agent Portability Skills lint", + ("uv", "run", "ruff", "check", "."), + REPO_ROOT / "plugins" / "agent-portability-skills", + ), + Check( + "Agent Portability Skills type checks", + ("uv", "run", "mypy", "."), + REPO_ROOT / "plugins" / "agent-portability-skills", + ), + Check( + "Apple Dev Skills docs", + ("bash", ".github/scripts/validate_repo_docs.sh"), + REPO_ROOT / "plugins" / "apple-dev-skills", + ), + Check( + "Apple Dev Skills tests", + ("uv", "run", "pytest"), + REPO_ROOT / "plugins" / "apple-dev-skills", + ), + Check( + "Documentation Skills tests", + ("uv", "run", "pytest"), + REPO_ROOT / "plugins" / "documentation-skills", + ), + Check( + "Professional Skills tests", + ("uv", "run", "pytest"), + REPO_ROOT / "plugins" / "professional-skills", + ), + Check( + "Python Skills metadata", + ("uv", "run", "scripts/validate_repo_metadata.py"), + REPO_ROOT / "plugins" / "python-skills", + ), + Check( + "Python Skills tests", + ("uv", "run", "pytest"), + REPO_ROOT / "plugins" / "python-skills", + ), + Check( + "Python Skills lint", + ("uv", "run", "ruff", "check", "."), + REPO_ROOT / "plugins" / "python-skills", + ), + Check( + "Python Skills type checks", + ("uv", "run", "mypy", "."), + REPO_ROOT / "plugins" / "python-skills", + ), + Check( + "Repository Skills tests", + ("uv", "run", "pytest"), + REPO_ROOT / "plugins" / "repository-skills", + ), + Check( + "Socket Steward tests", + ("uv", "run", "pytest"), + REPO_ROOT / ".agents" / "socket-steward", + ), + Check( + "Socket Steward lint", + ("uv", "run", "ruff", "check", "."), + REPO_ROOT / ".agents" / "socket-steward", + ), + Check( + "Socket Steward type checks", + ("uv", "run", "mypy", "."), + REPO_ROOT / ".agents" / "socket-steward", + ), + Check( + "Things app tests", + ("uv", "run", "pytest"), + REPO_ROOT / "plugins" / "things-app", + ), + Check( + "Cardhop MCP tests", + ("uv", "run", "pytest"), + REPO_ROOT / "plugins" / "cardhop-app" / "mcp", + ), + Check( + "Cardhop MCP lint", + ("uv", "run", "ruff", "check", "."), + REPO_ROOT / "plugins" / "cardhop-app" / "mcp", + ), + Check( + "Cardhop MCP type checks", + ("uv", "run", "mypy", "."), + REPO_ROOT / "plugins" / "cardhop-app" / "mcp", + ), + Check( + "Things MCP tests", + ("uv", "run", "pytest"), + REPO_ROOT / "plugins" / "things-app" / "mcp", + ), + Check( + "Things MCP lint", + ("uv", "run", "ruff", "check", "."), + REPO_ROOT / "plugins" / "things-app" / "mcp", + ), + Check( + "Things MCP type checks", + ("uv", "run", "mypy", "."), + REPO_ROOT / "plugins" / "things-app" / "mcp", + ), + Check( + "Cybersecurity Skills metadata", + ("uv", "run", "scripts/validate_repo_metadata.py"), + REPO_ROOT / "plugins" / "cybersecurity-skills", + ), + Check( + "Cybersecurity Skills tests", + ("uv", "run", "pytest", "tests"), + REPO_ROOT / "plugins" / "cybersecurity-skills", + ), + Check( + "Reverse Engineering Skills metadata", + ("uv", "run", "scripts/validate_repo_metadata.py"), + REPO_ROOT / "plugins" / "reverse-engineering-skills", + ), + Check( + "Reverse Engineering Skills tests", + ("uv", "run", "pytest", "tests"), + REPO_ROOT / "plugins" / "reverse-engineering-skills", + ), +) + + +def checks_for_profile(profile: str, version: str | None) -> tuple[Check, ...]: + checks: tuple[Check, ...] = CORE_CHECKS + if profile in {"compatibility", "full", "release"}: + checks += COMPATIBILITY_CHECKS + if profile in {"full", "release"}: + checks += CHILD_CHECKS + if profile == "release": + if version is None: + raise ValueError("--version is required with --profile release.") + checks += (Check("release readiness", ("scripts/release.sh", "release-ready", version)),) + return checks + + +def run_check(check: Check, *, dry_run: bool) -> None: + rendered_command = " ".join(check.command) + relative_cwd = check.cwd.relative_to(REPO_ROOT) + print(f"\n==> {check.name}\n cwd: {relative_cwd or '.'}\n {rendered_command}") + if not dry_run: + environment = None + if check.cwd != REPO_ROOT: + environment = os.environ.copy() + environment.pop("VIRTUAL_ENV", None) + subprocess.run(check.command, cwd=check.cwd, check=True, env=environment) + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--profile", + choices=("core", "compatibility", "full", "release"), + default="core", + help="Validation breadth; defaults to the fast PR-safe core profile.", + ) + parser.add_argument("--version", help="Expected semantic version for the release profile.") + parser.add_argument("--dry-run", action="store_true", help="Print checks without running them.") + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + try: + checks = checks_for_profile(args.profile, args.version) + except ValueError as error: + raise SystemExit(f"validate-socket: {error}") from error + for check in checks: + run_check(check, dry_run=args.dry_run) + print(f"\nSocket validation profile `{args.profile}` passed ({len(checks)} checks).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_socket_skill_metadata.py b/scripts/validate_socket_skill_metadata.py new file mode 100644 index 000000000..0cefa1b56 --- /dev/null +++ b/scripts/validate_socket_skill_metadata.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Validate shared skill and child-plugin metadata contracts across Socket.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path +from typing import Any, NoReturn + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parent.parent +SKILL_NAME_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +MAX_DESCRIPTION_LENGTH = 1024 +OPTIONAL_INTERFACE_FIELDS = ("display_name", "short_description") + + +def fail(message: str) -> NoReturn: + print(f"validate-socket-skill-metadata: {message}", file=sys.stderr) + raise SystemExit(1) + + +def load_yaml(path: Path) -> dict[str, Any]: + try: + value = yaml.safe_load(path.read_text(encoding="utf-8")) + except yaml.YAMLError as error: + fail(f"{path.relative_to(REPO_ROOT)} is not valid YAML: {error}") + if not isinstance(value, dict): + fail(f"{path.relative_to(REPO_ROOT)} must decode to a YAML mapping.") + return value + + +def read_frontmatter(path: Path) -> dict[str, Any]: + contents = path.read_text(encoding="utf-8") + if not contents.startswith("---\n"): + fail(f"{path.relative_to(REPO_ROOT)} must begin with YAML frontmatter.") + raw_frontmatter, separator, _ = contents[4:].partition("\n---\n") + if not separator: + fail(f"{path.relative_to(REPO_ROOT)} has unterminated YAML frontmatter.") + try: + value = yaml.safe_load(raw_frontmatter) + except yaml.YAMLError as error: + fail(f"{path.relative_to(REPO_ROOT)} has invalid YAML frontmatter: {error}") + if not isinstance(value, dict): + fail(f"{path.relative_to(REPO_ROOT)} frontmatter must be a YAML mapping.") + return value + + +def validate_skill(path: Path) -> None: + metadata = read_frontmatter(path) + expected_name = path.parent.name + name = metadata.get("name") + if name != expected_name: + fail( + f"{path.relative_to(REPO_ROOT)} must use its directory name " + f"{expected_name!r}, but found {name!r}." + ) + if not isinstance(name, str) or not SKILL_NAME_RE.fullmatch(name): + fail(f"{path.relative_to(REPO_ROOT)} must use a lowercase kebab-case skill name.") + description = metadata.get("description") + if not isinstance(description, str) or not description.strip(): + fail(f"{path.relative_to(REPO_ROOT)} must define a non-empty description.") + if len(description) > MAX_DESCRIPTION_LENGTH: + fail( + f"{path.relative_to(REPO_ROOT)} description exceeds " + f"{MAX_DESCRIPTION_LENGTH} characters." + ) + + openai_metadata = path.parent / "agents" / "openai.yaml" + if openai_metadata.exists(): + validate_openai_interface(openai_metadata, expected_name) + + +def validate_openai_interface(path: Path, skill_name: str) -> None: + metadata = load_yaml(path) + interface = metadata.get("interface") + if not isinstance(interface, dict) or not interface: + fail(f"{path.relative_to(REPO_ROOT)} must define a non-empty interface mapping.") + + default_prompt = interface.get("default_prompt") + if not isinstance(default_prompt, str) or not default_prompt.strip(): + fail(f"{path.relative_to(REPO_ROOT)} must define a non-empty interface.default_prompt.") + if f"${skill_name}" not in default_prompt: + fail( + f"{path.relative_to(REPO_ROOT)} interface.default_prompt must include " + f"the ${skill_name} invocation token." + ) + + for field_name in OPTIONAL_INTERFACE_FIELDS: + value = interface.get(field_name) + if value is not None and (not isinstance(value, str) or not value.strip()): + fail(f"{path.relative_to(REPO_ROOT)} interface.{field_name} must be a non-empty string.") + + +def validate_child_guidance(plugin_root: Path) -> None: + if not (plugin_root / "AGENTS.md").is_file(): + fail(f"{plugin_root.relative_to(REPO_ROOT)} is missing its child AGENTS.md guidance.") + + +def main() -> int: + plugin_roots = sorted( + path.parent.parent + for path in REPO_ROOT.glob("plugins/*/.codex-plugin/plugin.json") + if path.is_file() + ) + for plugin_root in plugin_roots: + validate_child_guidance(plugin_root) + + skill_paths = sorted(REPO_ROOT.glob("plugins/*/skills/*/SKILL.md")) + if not skill_paths: + fail("No authored plugin SKILL.md files were found.") + for skill_path in skill_paths: + validate_skill(skill_path) + + print( + "Socket shared skill metadata validation passed " + f"({len(plugin_roots)} plugins, {len(skill_paths)} skills)." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/ablate-refusal-representations/agents/openai.yaml b/skills/ablate-refusal-representations/agents/openai.yaml index 11216d72f..409b4e907 100644 --- a/skills/ablate-refusal-representations/agents/openai.yaml +++ b/skills/ablate-refusal-representations/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Ablate Refusal Representations" short_description: "Research refusal-direction ablation with explicit controls." - default_prompt: "Design and evaluate refusal-representation ablation as controlled model-internals research." + default_prompt: "Use $ablate-refusal-representations to design and evaluate refusal-representation ablation as controlled model-internals research." diff --git a/skills/benchmark-model-runtime/agents/openai.yaml b/skills/benchmark-model-runtime/agents/openai.yaml index 877197c4d..df808528f 100644 --- a/skills/benchmark-model-runtime/agents/openai.yaml +++ b/skills/benchmark-model-runtime/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Benchmark Model Runtime" short_description: "Benchmark model runtimes fairly across deployment targets." - default_prompt: "Benchmark model runtimes using controlled artifacts, workloads, thermal state, memory, latency, and energy evidence." + default_prompt: "Use $benchmark-model-runtime to benchmark model runtimes using controlled artifacts, workloads, thermal state, memory, latency, and energy evidence." diff --git a/skills/choose-apple-model-runtime/agents/openai.yaml b/skills/choose-apple-model-runtime/agents/openai.yaml index 79c8c2958..97afa907b 100644 --- a/skills/choose-apple-model-runtime/agents/openai.yaml +++ b/skills/choose-apple-model-runtime/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Choose Apple Model Runtime" short_description: "Route Apple model authoring, conversion, and deployment choices." - default_prompt: "Choose among Core AI, Core ML, MLX, ExecuTorch, and Foundation Models for an Apple model workflow." + default_prompt: "Use $choose-apple-model-runtime to choose among Core AI, Core ML, MLX, ExecuTorch, and Foundation Models for an Apple model workflow." diff --git a/skills/choose-model-lab-workflow/agents/openai.yaml b/skills/choose-model-lab-workflow/agents/openai.yaml index ba7e5cb05..579c51d9f 100644 --- a/skills/choose-model-lab-workflow/agents/openai.yaml +++ b/skills/choose-model-lab-workflow/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Choose Model Lab Workflow" short_description: "Route model training, research, evaluation, and runtime tasks." - default_prompt: "Choose the appropriate Model Lab workflow and explain the evidence boundary." + default_prompt: "Use $choose-model-lab-workflow to choose the appropriate Model Lab workflow and explain the evidence boundary." diff --git a/skills/compare-model-checkpoints/agents/openai.yaml b/skills/compare-model-checkpoints/agents/openai.yaml index cb061d779..d97fad9d7 100644 --- a/skills/compare-model-checkpoints/agents/openai.yaml +++ b/skills/compare-model-checkpoints/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Compare Model Checkpoints" short_description: "Compare model checkpoints without hiding tradeoffs." - default_prompt: "Compare model checkpoints across quality, behavior, provenance, size, and runtime evidence." + default_prompt: "Use $compare-model-checkpoints to compare model checkpoints across quality, behavior, provenance, size, and runtime evidence." diff --git a/skills/design-model-experiment/agents/openai.yaml b/skills/design-model-experiment/agents/openai.yaml index ef297f769..06adf7db8 100644 --- a/skills/design-model-experiment/agents/openai.yaml +++ b/skills/design-model-experiment/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Design Model Experiment" short_description: "Define a reproducible model experiment before spending compute." - default_prompt: "Design a reproducible model experiment with hypotheses, controls, budgets, provenance, and stop conditions." + default_prompt: "Use $design-model-experiment to design a reproducible model experiment with hypotheses, controls, budgets, provenance, and stop conditions." diff --git a/skills/evaluate-jailbreak-resilience/agents/openai.yaml b/skills/evaluate-jailbreak-resilience/agents/openai.yaml index 52d88e496..ed13ee200 100644 --- a/skills/evaluate-jailbreak-resilience/agents/openai.yaml +++ b/skills/evaluate-jailbreak-resilience/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Evaluate Jailbreak Resilience" short_description: "Evaluate authorized jailbreak and prompt-injection resilience." - default_prompt: "Build an authorized jailbreak-resilience evaluation with bounded targets, metrics, and disclosure-safe evidence." + default_prompt: "Use $evaluate-jailbreak-resilience to build an authorized jailbreak-resilience evaluation with bounded targets, metrics, and disclosure-safe evidence." diff --git a/skills/evaluate-language-model/agents/openai.yaml b/skills/evaluate-language-model/agents/openai.yaml index 4229ec51f..8ad8cd9fa 100644 --- a/skills/evaluate-language-model/agents/openai.yaml +++ b/skills/evaluate-language-model/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Evaluate Language Model" short_description: "Build repeatable capability, behavior, and safety evaluations." - default_prompt: "Design and run a repeatable language-model evaluation with scoped metrics and regression evidence." + default_prompt: "Use $evaluate-language-model to design and run a repeatable language-model evaluation with scoped metrics and regression evidence." diff --git a/skills/evaluate-tool-calling-model/agents/openai.yaml b/skills/evaluate-tool-calling-model/agents/openai.yaml index d98159315..63e25be06 100644 --- a/skills/evaluate-tool-calling-model/agents/openai.yaml +++ b/skills/evaluate-tool-calling-model/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Evaluate Tool-Calling Model" short_description: "Evaluate tool selection, arguments, execution, and recovery." - default_prompt: "Evaluate a tool-calling model or harness across selection, schema, execution, recovery, and side-effect controls." + default_prompt: "Use $evaluate-tool-calling-model to evaluate a tool-calling model or harness across selection, schema, execution, recovery, and side-effect controls." diff --git a/skills/fine-tune-language-model/agents/openai.yaml b/skills/fine-tune-language-model/agents/openai.yaml index 7024f7a47..f899adf00 100644 --- a/skills/fine-tune-language-model/agents/openai.yaml +++ b/skills/fine-tune-language-model/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Fine-Tune Language Model" short_description: "Plan and run reproducible adapter or full fine-tuning." - default_prompt: "Choose and execute a reproducible language-model fine-tuning workflow with explicit resource and validation gates." + default_prompt: "Use $fine-tune-language-model to choose and execute a reproducible language-model fine-tuning workflow with explicit resource and validation gates." diff --git a/skills/prepare-language-model-dataset/agents/openai.yaml b/skills/prepare-language-model-dataset/agents/openai.yaml index 91b39ca9d..25b6cab2a 100644 --- a/skills/prepare-language-model-dataset/agents/openai.yaml +++ b/skills/prepare-language-model-dataset/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Prepare Language Model Dataset" short_description: "Prepare auditable training and evaluation datasets." - default_prompt: "Prepare and document a language-model dataset with splits, provenance, quality checks, and leakage controls." + default_prompt: "Use $prepare-language-model-dataset to prepare and document a language-model dataset with splits, provenance, quality checks, and leakage controls." diff --git a/skills/research-model-representations/agents/openai.yaml b/skills/research-model-representations/agents/openai.yaml index 6f2c22ed8..d8eec59a1 100644 --- a/skills/research-model-representations/agents/openai.yaml +++ b/skills/research-model-representations/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Research Model Representations" short_description: "Design causal representation and activation research." - default_prompt: "Design a reproducible language-model representation study with causal controls and interpretable evidence." + default_prompt: "Use $research-model-representations to design a reproducible language-model representation study with causal controls and interpretable evidence." diff --git a/skills/steer-language-model-behavior/agents/openai.yaml b/skills/steer-language-model-behavior/agents/openai.yaml index 52676065a..f16a31a34 100644 --- a/skills/steer-language-model-behavior/agents/openai.yaml +++ b/skills/steer-language-model-behavior/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Steer Language Model Behavior" short_description: "Implement and evaluate controlled model behavior steering." - default_prompt: "Choose, implement, and evaluate a model steering intervention without conflating behavior change with understanding." + default_prompt: "Use $steer-language-model-behavior to choose, implement, and evaluate a model steering intervention without conflating behavior change with understanding." diff --git a/tests/test_release_version.py b/tests/test_release_version.py index dc7d93500..dd597c639 100644 --- a/tests/test_release_version.py +++ b/tests/test_release_version.py @@ -376,7 +376,7 @@ def fake_capture_release_evidence(repo_root: Path, output_path: Path) -> object: assert exit_code == 0 assert "Patch-refresh release completed for v1.2.4." in output assert commands[-1] == ("cmd", ("codex", "plugin", "marketplace", "upgrade", "socket")) - assert ("cmd", ("uv", "run", "scripts/validate_socket_metadata.py")) in commands + assert ("cmd", ("uv", "run", "scripts/validate_socket.py", "--profile", "full")) in commands assert ("git", ("push", "origin", "main")) in commands assert ("git", ("push", "origin", "v1.2.4")) in commands assert commands.index(("evidence", (str(root / ".socket-release-evidence.json"),))) < commands.index( diff --git a/tests/test_validate_hermes_compatibility.py b/tests/test_validate_hermes_compatibility.py index d705f20d5..8b38cf243 100644 --- a/tests/test_validate_hermes_compatibility.py +++ b/tests/test_validate_hermes_compatibility.py @@ -97,6 +97,7 @@ def configure_paths(repo_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(export_hermes_skills, "PYTHON_SOURCE_ROOT", repo_root / "plugins" / "python-skills" / "skills") monkeypatch.setattr(export_hermes_skills, "JVM_SOURCE_ROOT", repo_root / "plugins" / "server-side-jvm" / "skills") monkeypatch.setattr(export_hermes_skills, "CLOUD_DEPLOYMENT_SOURCE_ROOT", repo_root / "plugins" / "agent-portability-skills" / "skills") + monkeypatch.setattr(export_hermes_skills, "REPOSITORY_SOURCE_ROOT", repo_root / "plugins" / "agent-portability-skills" / "skills") monkeypatch.setattr(export_hermes_skills, "EXPORT_ROOT", repo_root / "skills") monkeypatch.setattr(validate_hermes_compatibility, "REPO_ROOT", repo_root) monkeypatch.setattr(validate_hermes_compatibility, "EXPORT_ROOT", repo_root / "skills") diff --git a/tests/test_validate_socket.py b/tests/test_validate_socket.py new file mode 100644 index 000000000..fbf82cfba --- /dev/null +++ b/tests/test_validate_socket.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parent.parent + + +def load_module(name: str, filename: str): + spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / filename) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +validate_socket = load_module("validate_socket", "validate_socket.py") +validate_skill_metadata = load_module( + "validate_socket_skill_metadata", "validate_socket_skill_metadata.py" +) + + +def write(path: Path, contents: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + + +def test_core_profile_uses_root_owned_checks_only() -> None: + checks = validate_socket.checks_for_profile("core", None) + + assert [check.name for check in checks] == [ + "root marketplace metadata", + "shared skill metadata", + "root tests", + "root type checks", + "root lint", + ] + + +def test_full_profile_adds_compatibility_and_child_checks_once() -> None: + checks = validate_socket.checks_for_profile("full", None) + names = [check.name for check in checks] + + assert names.count("Agent Portability Skills tests") == 1 + assert names.count("Repository Skills tests") == 1 + assert names.count("Cybersecurity Skills tests") == 1 + assert names.count("Reverse Engineering Skills tests") == 1 + assert "Hermes compatibility" in names + assert "Claude compatibility" in names + assert "release readiness" not in names + + +def test_release_profile_requires_a_version() -> None: + with pytest.raises(ValueError, match="--version"): + validate_socket.checks_for_profile("release", None) + + +def test_release_profile_adds_the_release_ready_gate() -> None: + checks = validate_socket.checks_for_profile("release", "9.32.0") + + assert checks[-1].name == "release readiness" + assert checks[-1].command == ("scripts/release.sh", "release-ready", "9.32.0") + + +def test_dry_run_does_not_execute_a_subprocess(monkeypatch: pytest.MonkeyPatch) -> None: + called = False + + def unexpected_run(*args: object, **kwargs: object) -> None: + nonlocal called + called = True + + monkeypatch.setattr(validate_socket.subprocess, "run", unexpected_run) + validate_socket.run_check(validate_socket.CORE_CHECKS[0], dry_run=True) + + assert not called + + +def test_shared_skill_validator_accepts_valid_plugin(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + plugin_root = tmp_path / "plugins" / "example-skills" + write(plugin_root / ".codex-plugin" / "plugin.json", "{}\n") + write(plugin_root / "AGENTS.md", "# Guidance\n") + write( + plugin_root / "skills" / "example-skill" / "SKILL.md", + "---\nname: example-skill\ndescription: A valid skill.\n---\n", + ) + write( + plugin_root / "skills" / "example-skill" / "agents" / "openai.yaml", + "interface:\n default_prompt: Use $example-skill.\n", + ) + monkeypatch.setattr(validate_skill_metadata, "REPO_ROOT", tmp_path) + + assert validate_skill_metadata.main() == 0 + + +def test_shared_skill_validator_rejects_directory_name_drift( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + plugin_root = tmp_path / "plugins" / "example-skills" + write(plugin_root / ".codex-plugin" / "plugin.json", "{}\n") + write(plugin_root / "AGENTS.md", "# Guidance\n") + write( + plugin_root / "skills" / "example-skill" / "SKILL.md", + "---\nname: wrong-name\ndescription: A valid skill.\n---\n", + ) + monkeypatch.setattr(validate_skill_metadata, "REPO_ROOT", tmp_path) + + with pytest.raises(SystemExit): + validate_skill_metadata.main() + + +@pytest.mark.parametrize( + ("interface", "match"), + [ + ("interface: {}\n", "non-empty interface"), + ("interface:\n default_prompt: Use this skill.\n", "invocation token"), + ("interface:\n default_prompt: Use $example-skill.\n display_name: ''\n", "display_name"), + ], +) +def test_shared_skill_validator_rejects_invalid_openai_interface( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + interface: str, + match: str, +) -> None: + plugin_root = tmp_path / "plugins" / "example-skills" + write(plugin_root / ".codex-plugin" / "plugin.json", "{}\n") + write(plugin_root / "AGENTS.md", "# Guidance\n") + write( + plugin_root / "skills" / "example-skill" / "SKILL.md", + "---\nname: example-skill\ndescription: A valid skill.\n---\n", + ) + write(plugin_root / "skills" / "example-skill" / "agents" / "openai.yaml", interface) + monkeypatch.setattr(validate_skill_metadata, "REPO_ROOT", tmp_path) + + with pytest.raises(SystemExit): + validate_skill_metadata.main() + assert match in capsys.readouterr().err