diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bac0a477..02e40a10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -577,3 +577,75 @@ jobs: - name: Run ${{ matrix.ecosystem }} Docker e2e test run: cargo test -p socket-patch-cli --features docker-e2e --test docker_e2e_${{ matrix.ecosystem }} + + # ---------------------------------------------------------------------- + # Experimental `setup`-flow matrix (NON-BLOCKING). + # + # For each ecosystem/package manager, drives the full intended flow — + # prepare deps + a committed patch set, run `socket-patch setup`, run + # the native install, check whether the patch was applied — plus the + # negative controls (no setup, empty/wrong/alt patch sets). See + # tests/setup_matrix/ and scripts/setup-matrix.sh. + # + # This is EXPERIMENTAL and intentionally not required to pass yet: + # `setup` only configures npm-family install hooks today, so most + # non-npm `baseline_with_setup` cases are EXPECTED to fail (a baseline + # of what `setup` must eventually support). `continue-on-error: true` + # means this job never blocks a PR — it must ALSO be left OUT of the + # repo's required status checks (configured in the branch-protection + # UI, not in this file). The orchestrator exits non-zero only on a + # *regression* vs the recorded baseline; the full per-case result set + # is uploaded as a JSON artifact for inspection. + # ---------------------------------------------------------------------- + setup-matrix: + runs-on: ubuntu-latest + continue-on-error: true + permissions: + contents: read + strategy: + fail-fast: false + matrix: + ecosystem: [npm, pypi, cargo, gem, golang, maven, composer, nuget, deno] + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Docker Buildx + # `driver: docker` — the per-ecosystem image's `FROM + # socket-patch-test-base:latest` only resolves when buildx talks + # directly to the host docker daemon (see e2e-docker above). + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + with: + driver: docker + + - name: Install Rust + run: rustup show + + - name: Build base image + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + file: tests/docker/Dockerfile.base + tags: socket-patch-test-base:latest + load: true + + - name: Build ${{ matrix.ecosystem }} image + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + file: tests/docker/Dockerfile.${{ matrix.ecosystem }} + tags: socket-patch-test-${{ matrix.ecosystem }}:latest + load: true + + - name: Run ${{ matrix.ecosystem }} setup-matrix + run: scripts/setup-matrix.sh run --ecosystem ${{ matrix.ecosystem }} --out "report-${{ matrix.ecosystem }}.json" + + - name: Upload ${{ matrix.ecosystem }} setup-matrix report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: setup-matrix-${{ matrix.ecosystem }} + path: report-${{ matrix.ecosystem }}.json + if-no-files-found: warn diff --git a/crates/socket-patch-cli/Cargo.toml b/crates/socket-patch-cli/Cargo.toml index 3ce2753d..fba6ef4e 100644 --- a/crates/socket-patch-cli/Cargo.toml +++ b/crates/socket-patch-cli/Cargo.toml @@ -39,6 +39,16 @@ deno = ["socket-patch-core/deno"] # `tests/docker_e2e_*.rs`. Tests in this suite require either a running # Docker daemon OR `SOCKET_PATCH_TEST_HOST=1` (host-toolchain mode). docker-e2e = [] +# Enables the experimental `setup` end-to-end test matrix under +# `tests/setup_matrix_*.rs`, which drives the `socket-patch setup` → +# native-install → patch-applied flow across every ecosystem/package +# manager via `tests/setup_matrix/run-case.sh`. Same runtime requirement +# as docker-e2e (Docker daemon OR `SOCKET_PATCH_TEST_HOST=1`). These +# tests are ASPIRATIONAL: they assert the ideal (install applies the +# patch) and are EXPECTED to fail for ecosystems whose install hooks +# `setup` does not yet configure. Kept off `--all-features`-required CI; +# the dedicated `setup-matrix` CI job runs them non-blocking. +setup-e2e = [] [dev-dependencies] sha2 = { workspace = true } diff --git a/crates/socket-patch-cli/tests/setup_matrix_cargo.rs b/crates/socket-patch-cli/tests/setup_matrix_cargo.rs new file mode 100644 index 00000000..ddea0b8a --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_cargo.rs @@ -0,0 +1,14 @@ +//! setup-matrix: cargo ecosystem. `setup` is a no-op for Rust projects +//! (no package.json) and cargo has no post-install hook, so the +//! with-setup cases are an EXPECTED BASELINE GAP. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_cargo` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn cargo() { + smc::run_pm("cargo", "cargo"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_common/mod.rs b/crates/socket-patch-cli/tests/setup_matrix_common/mod.rs new file mode 100644 index 00000000..d2fe02c1 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_common/mod.rs @@ -0,0 +1,318 @@ +//! Shared harness for the experimental `socket-patch setup` end-to-end +//! test matrix (`tests/setup_matrix_*.rs`, gated by the `setup-e2e` +//! feature). +//! +//! Each `setup_matrix_.rs` wrapper pulls this in with +//! `#[path = "setup_matrix_common/mod.rs"] mod smc;` and calls +//! [`run_pm`] for each package manager it covers. The wrappers are +//! thin; ALL the flow logic lives in the single bash driver +//! `tests/setup_matrix/run-case.sh`, which this module invokes either +//! inside a Docker container (default) or on the host +//! (`SOCKET_PATCH_TEST_HOST=1`). The declarative case list comes from +//! `tests/setup_matrix/matrix.json` — the same spec the +//! `scripts/setup-matrix.sh` orchestrator consumes. +//! +//! ASPIRATIONAL assertion: each case asserts the *ideal* — that after +//! `setup` + a native install, the patch is (or isn't) applied as the +//! scenario expects. For ecosystems whose install hooks `setup` does +//! not yet configure, the `baseline_with_setup` / `alt_content_patchset` +//! cases are EXPECTED to fail; the failure message tags them +//! `BASELINE GAP` so the red is understood as a TODO, not a surprise. +//! +//! `#![allow(dead_code)]` — wrappers use different subsets of this API. + +#![allow(dead_code)] + +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// Path to the built binary under test (host mode passes this to the +/// driver via `SOCKET_PATCH_BIN`). +fn binary() -> PathBuf { + env!("CARGO_BIN_EXE_socket-patch").into() +} + +/// Workspace root = two levels up from this crate's manifest dir. +fn workspace_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(|p| p.parent()) + .expect("workspace root") + .to_path_buf() +} + +fn driver_path() -> PathBuf { + workspace_root().join("tests/setup_matrix/run-case.sh") +} + +fn matrix_path() -> PathBuf { + workspace_root().join("tests/setup_matrix/matrix.json") +} + +/// Host mode runs the driver against host-installed toolchains instead +/// of a container. Mirrors the `docker_e2e_*` convention. +fn host_mode() -> bool { + std::env::var("SOCKET_PATCH_TEST_HOST").map(|v| v == "1").unwrap_or(false) +} + +fn docker_on_path() -> bool { + Command::new("docker") + .arg("--version") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +fn image_present(image: &str) -> bool { + Command::new("docker") + .args(["image", "inspect", &format!("socket-patch-test-{image}:latest")]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +/// One concrete case = a (target, scenario) pair from matrix.json. +#[derive(Clone)] +struct Case { + id: String, + ecosystem: String, + pm: String, + image: String, + scenario: String, + patchset: String, + run_setup: bool, + expect_applied: bool, + baseline_supported: bool, + package: String, + version: String, + purl: String, + manifest_key: String, + apply_ecosystems: String, + marker: String, + alt_marker: String, + layout: String, +} + +impl Case { + /// Baseline (currently-known) outcome under today's code: + /// `setup` only wires npm-family hooks, so applied is expected only + /// when the target advertises `baseline_supported` AND the scenario + /// aspires to apply. + fn baseline_applied(&self) -> bool { + self.expect_applied && self.baseline_supported + } + + fn sm_env(&self) -> Vec<(String, String)> { + vec![ + ("SM_ID".into(), self.id.clone()), + ("SM_ECOSYSTEM".into(), self.ecosystem.clone()), + ("SM_PM".into(), self.pm.clone()), + ("SM_SCENARIO".into(), self.scenario.clone()), + ("SM_PATCHSET".into(), self.patchset.clone()), + ("SM_RUN_SETUP".into(), if self.run_setup { "1" } else { "0" }.into()), + ("SM_EXPECT_APPLIED".into(), if self.expect_applied { "1" } else { "0" }.into()), + ("SM_PACKAGE".into(), self.package.clone()), + ("SM_VERSION".into(), self.version.clone()), + ("SM_PURL".into(), self.purl.clone()), + ("SM_MANIFEST_KEY".into(), self.manifest_key.clone()), + ("SM_APPLY_ECOSYSTEMS".into(), self.apply_ecosystems.clone()), + ("SM_MARKER".into(), self.marker.clone()), + ("SM_ALT_MARKER".into(), self.alt_marker.clone()), + ("SM_LAYOUT".into(), self.layout.clone()), + ] + } +} + +/// Load every case for a given (ecosystem, pm) by crossing the matching +/// target in `targets_key` with every scenario in `scenarios_key`, +/// tagging each with `layout`. `targets_key`/`scenarios_key` select the +/// spec section: ("targets","scenarios") for single projects, +/// ("workspace_targets","workspace_scenarios") for nested workspaces, +/// ("monorepo_targets","monorepo_scenarios") for the polyglot monorepo. +fn load_section( + targets_key: &str, + scenarios_key: &str, + layout: &str, + ecosystem: &str, + pm: &str, +) -> Vec { + let text = std::fs::read_to_string(matrix_path()) + .unwrap_or_else(|e| panic!("read matrix.json: {e}")); + let spec: serde_json::Value = + serde_json::from_str(&text).expect("parse matrix.json"); + let marker = spec["marker"].as_str().unwrap_or("").to_string(); + let alt_marker = spec["alt_marker"].as_str().unwrap_or("").to_string(); + + let target = spec[targets_key] + .as_array() + .unwrap_or_else(|| panic!("{targets_key} array missing")) + .iter() + .find(|t| t["ecosystem"] == ecosystem && t["pm"] == pm) + .unwrap_or_else(|| panic!("no {targets_key} entry for {ecosystem}/{pm}")); + + let mut cases = Vec::new(); + for s in spec[scenarios_key].as_array().expect("scenarios array") { + let scenario = s["id"].as_str().unwrap().to_string(); + cases.push(Case { + id: format!("{ecosystem}/{pm}/{scenario}"), + ecosystem: ecosystem.to_string(), + pm: pm.to_string(), + image: target["image"].as_str().unwrap().to_string(), + scenario, + patchset: s["patchset"].as_str().unwrap().to_string(), + run_setup: s["run_setup"].as_bool().unwrap(), + expect_applied: s["expect_applied"].as_bool().unwrap(), + baseline_supported: target["baseline_supported"].as_bool().unwrap(), + package: target["package"].as_str().unwrap().to_string(), + version: target["version"].as_str().unwrap().to_string(), + purl: target["purl"].as_str().unwrap().to_string(), + manifest_key: target["manifest_key"].as_str().unwrap().to_string(), + apply_ecosystems: target["apply_ecosystems"].as_str().unwrap().to_string(), + marker: marker.clone(), + alt_marker: alt_marker.clone(), + layout: layout.to_string(), + }); + } + cases +} + +struct RunResult { + actual_applied: bool, + raw: String, + parsed: Option, +} + +/// Execute one case via the bash driver (container or host) and parse +/// its JSON result line. +fn run_case(case: &Case) -> RunResult { + let driver = driver_path(); + let env = case.sm_env(); + + let output = if host_mode() { + let mut cmd = Command::new("bash"); + cmd.arg(&driver); + for (k, v) in &env { + cmd.env(k, v); + } + cmd.env("SOCKET_PATCH_BIN", binary()); + cmd.output().expect("spawn bash driver") + } else { + let script = std::fs::read_to_string(&driver) + .unwrap_or_else(|e| panic!("read driver: {e}")); + let mut cmd = Command::new("docker"); + cmd.args(["run", "--rm"]); + for (k, v) in &env { + cmd.args(["-e", &format!("{k}={v}")]); + } + cmd.arg(format!("socket-patch-test-{}:latest", case.image)); + cmd.args(["bash", "-c", &script]); + cmd.output().expect("spawn docker run") + }; + + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + + // The driver prints its result JSON as the last matching stdout line. + let line = stdout + .lines() + .rev() + .find(|l| l.trim_start().starts_with('{') && l.contains("actual_applied")); + + let parsed = line.and_then(|l| serde_json::from_str::(l).ok()); + let actual_applied = parsed + .as_ref() + .and_then(|v| v["actual_applied"].as_bool()) + .unwrap_or(false); + + RunResult { + actual_applied, + raw: format!("stdout:\n{stdout}\nstderr:\n{stderr}"), + parsed, + } +} + +/// Run the single-project scenarios for one (ecosystem, pm). +pub fn run_pm(ecosystem: &str, pm: &str) { + run_cases( + &format!("{ecosystem}/{pm}"), + load_section("targets", "scenarios", "single", ecosystem, pm), + ); +} + +/// Run the nested-workspace scenarios for one (ecosystem, pm). +pub fn run_workspace_pm(ecosystem: &str, pm: &str) { + run_cases( + &format!("{ecosystem}/{pm} [workspace]"), + load_section("workspace_targets", "workspace_scenarios", "workspace", ecosystem, pm), + ); +} + +/// Run the polyglot all-ecosystem monorepo scenarios. +pub fn run_monorepo() { + run_cases( + "monorepo", + load_section("monorepo_targets", "monorepo_scenarios", "monorepo", "monorepo", "mono"), + ); +} + +/// Execute a set of cases and assert each meets the ASPIRATIONAL +/// expectation. Soft-skips when Docker / the ecosystem image is +/// unavailable (container mode) — matching the `docker_e2e_*` convention +/// where Rust integration tests have no native "skipped". +fn run_cases(label: &str, cases: Vec) { + if !host_mode() && !docker_on_path() { + eprintln!("skip {label}: docker not on PATH (set SOCKET_PATCH_TEST_HOST=1 to run on host)"); + return; + } + if !host_mode() { + if let Some(c) = cases.first() { + if !image_present(&c.image) { + eprintln!( + "skip {label}: image socket-patch-test-{}:latest not present \ + (build it: scripts/setup-matrix.sh build --ecosystem {})", + c.image, c.image + ); + return; + } + } + } + + let mut failures = Vec::new(); + for case in &cases { + let res = run_case(case); + if res.actual_applied != case.expect_applied { + let tag = if case.baseline_applied() { + // We recorded this as working; failing now is a real regression. + "REGRESSION (baseline says this should apply)" + } else if case.expect_applied { + "BASELINE GAP (setup does not yet wire this package manager)" + } else { + "LEAK (patch applied without the hook configuring it)" + }; + failures.push(format!( + " - {}: expected applied={}, got {} [{}]\n{}", + case.id, case.expect_applied, res.actual_applied, tag, indent(&res.raw) + )); + } + } + + assert!( + failures.is_empty(), + "{}: {} of {} setup-matrix case(s) did not meet the aspirational \ + expectation. BASELINE GAP entries are the experimental TODO list \ + (this suite is non-blocking in CI); REGRESSION / LEAK entries are \ + real problems:\n{}", + label, + failures.len(), + cases.len(), + failures.join("\n") + ); +} + +fn indent(s: &str) -> String { + s.lines().map(|l| format!(" {l}")).collect::>().join("\n") +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_composer.rs b/crates/socket-patch-cli/tests/setup_matrix_composer.rs new file mode 100644 index 00000000..8ec68934 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_composer.rs @@ -0,0 +1,15 @@ +//! setup-matrix: composer ecosystem (PHP). Composer DOES expose a +//! `post-install-cmd` event hook, but `setup` does not wire it today, +//! so the with-setup cases are an EXPECTED BASELINE GAP — and a clear +//! candidate for the first non-npm ecosystem `setup` could support. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_composer` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn composer() { + smc::run_pm("composer", "composer"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_deno.rs b/crates/socket-patch-cli/tests/setup_matrix_deno.rs new file mode 100644 index 00000000..4cec9383 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_deno.rs @@ -0,0 +1,16 @@ +//! setup-matrix: deno ecosystem (deno install against a package.json, +//! npm-via-deno layout). `setup` DOES rewrite the package.json (deno +//! projects have one), but whether `deno install` runs the root +//! postinstall hook is uncertain — so the baseline records this as a +//! GAP. If it applies, the orchestrator flags it `progress`. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_deno` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn deno() { + smc::run_pm("deno", "deno"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_gem.rs b/crates/socket-patch-cli/tests/setup_matrix_gem.rs new file mode 100644 index 00000000..c5507b54 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_gem.rs @@ -0,0 +1,14 @@ +//! setup-matrix: gem ecosystem (bundler). No native post-install hook +//! and `setup` is a no-op, so the with-setup cases are an EXPECTED +//! BASELINE GAP. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_gem` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn bundler() { + smc::run_pm("gem", "bundler"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_golang.rs b/crates/socket-patch-cli/tests/setup_matrix_golang.rs new file mode 100644 index 00000000..c444e1d8 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_golang.rs @@ -0,0 +1,14 @@ +//! setup-matrix: golang ecosystem (go modules). No native post-install +//! hook and `setup` is a no-op, so the with-setup cases are an EXPECTED +//! BASELINE GAP. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_golang` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn go() { + smc::run_pm("golang", "go"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_maven.rs b/crates/socket-patch-cli/tests/setup_matrix_maven.rs new file mode 100644 index 00000000..ab08a169 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_maven.rs @@ -0,0 +1,15 @@ +//! setup-matrix: maven ecosystem (mvn). No native post-install hook, +//! `setup` is a no-op, and apply is additionally gated behind +//! `SOCKET_EXPERIMENTAL_MAVEN` (the driver sets it). The with-setup +//! cases are an EXPECTED BASELINE GAP. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_maven` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn mvn() { + smc::run_pm("maven", "mvn"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_monorepo.rs b/crates/socket-patch-cli/tests/setup_matrix_monorepo.rs new file mode 100644 index 00000000..f62cb508 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_monorepo.rs @@ -0,0 +1,20 @@ +//! setup-matrix: polyglot all-ecosystem monorepo. +//! +//! A single repo containing an npm workspace alongside +//! python/rust/go/php/ruby/nuget/deno manifests. Confirms `socket-patch +//! setup` works in this mixed environment — it must configure the npm +//! hooks and NOT choke on the foreign manifests; a root `npm install` +//! then applies the patch to the npm slice. Runs in the npm image (the +//! only one with the npm toolchain); the foreign manifests are present +//! to test setup's robustness, not installed. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_monorepo` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn monorepo() { + smc::run_monorepo(); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_npm.rs b/crates/socket-patch-cli/tests/setup_matrix_npm.rs new file mode 100644 index 00000000..eac266e6 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_npm.rs @@ -0,0 +1,55 @@ +//! setup-matrix: npm ecosystem (npm / yarn / pnpm / bun). +//! +//! These are the ecosystems `socket-patch setup` actually supports +//! today (it writes a package.json postinstall hook), so the +//! `baseline_with_setup` / `alt_content_patchset` cases are expected to +//! PASS here. See `setup_matrix_common/mod.rs` for the harness and +//! `tests/setup_matrix/matrix.json` for the case list. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_npm` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn npm() { + smc::run_pm("npm", "npm"); +} + +#[test] +fn yarn() { + smc::run_pm("npm", "yarn"); +} + +#[test] +fn pnpm() { + smc::run_pm("npm", "pnpm"); +} + +#[test] +fn bun() { + smc::run_pm("npm", "bun"); +} + +// ── Nested-workspace layouts ────────────────────────────────────────── +// A root + several members (incl. a deeply-nested one and a member with +// no dependency on the patched package). Exercises `setup`'s workspace +// handling (npm/yarn write the hook to every member; pnpm only to the +// root) plus the cross-workspace apply on the root install. These should +// PASS — they're real regression guards, not gap documentation. + +#[test] +fn npm_workspace() { + smc::run_workspace_pm("npm", "npm"); +} + +#[test] +fn pnpm_workspace() { + smc::run_workspace_pm("npm", "pnpm"); +} + +#[test] +fn yarn_workspace() { + smc::run_workspace_pm("npm", "yarn"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_nuget.rs b/crates/socket-patch-cli/tests/setup_matrix_nuget.rs new file mode 100644 index 00000000..06bf050a --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_nuget.rs @@ -0,0 +1,15 @@ +//! setup-matrix: nuget ecosystem (dotnet). No native post-install hook, +//! `setup` is a no-op, and apply is additionally gated behind +//! `SOCKET_EXPERIMENTAL_NUGET` (the driver sets it). The with-setup +//! cases are an EXPECTED BASELINE GAP. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_nuget` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn dotnet() { + smc::run_pm("nuget", "dotnet"); +} diff --git a/crates/socket-patch-cli/tests/setup_matrix_pypi.rs b/crates/socket-patch-cli/tests/setup_matrix_pypi.rs new file mode 100644 index 00000000..b1907433 --- /dev/null +++ b/crates/socket-patch-cli/tests/setup_matrix_pypi.rs @@ -0,0 +1,52 @@ +//! setup-matrix: pypi ecosystem (pip / uv / poetry / pdm / hatch). +//! +//! Python installers have no native post-install hook and `socket-patch +//! setup` is a no-op for them, so the `baseline_with_setup` / +//! `alt_content_patchset` cases are EXPECTED to fail here (BASELINE +//! GAP). The negative-control / empty / wrong-target cases should pass. +//! +//! Run: `cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_pypi` +#![cfg(feature = "setup-e2e")] + +#[path = "setup_matrix_common/mod.rs"] +mod smc; + +#[test] +fn pip() { + smc::run_pm("pypi", "pip"); +} + +#[test] +fn uv() { + smc::run_pm("pypi", "uv"); +} + +#[test] +fn poetry() { + smc::run_pm("pypi", "poetry"); +} + +#[test] +fn pdm() { + smc::run_pm("pypi", "pdm"); +} + +#[test] +fn hatch() { + smc::run_pm("pypi", "hatch"); +} + +// ── Nested-workspace layouts (EXPECTED BASELINE GAP) ────────────────── +// uv workspace (root + members, one shared .venv) and a pip +// nested-requirements monorepo. Python has no post-install hook, so +// these don't apply today — but the install itself must succeed. + +#[test] +fn pip_workspace() { + smc::run_workspace_pm("pypi", "pip"); +} + +#[test] +fn uv_workspace() { + smc::run_workspace_pm("pypi", "uv"); +} diff --git a/scripts/setup-matrix.sh b/scripts/setup-matrix.sh new file mode 100755 index 00000000..dbec628d --- /dev/null +++ b/scripts/setup-matrix.sh @@ -0,0 +1,284 @@ +#!/usr/bin/env bash +# ===================================================================== +# setup-matrix.sh — orchestrate and query the `socket-patch setup` +# end-to-end test matrix. +# +# The matrix asks, for every supported ecosystem/package-manager: +# "does `socket-patch setup` configure things so that a normal install +# applies the project's patches?" Each case runs the flow driver +# (tests/setup_matrix/run-case.sh) which prepares a project + committed +# patch set, optionally runs `socket-patch setup`, runs the native +# install, and checks whether the patch landed on disk. +# +# Results are classified against the recorded baseline in matrix.json: +# pass meets the ideal AND matches the recorded baseline +# known_gap fails the ideal but exactly as recorded (expected today) +# progress better than the recorded baseline (update baseline!) +# regression diverged from the baseline the wrong way (this is the +# only thing that makes `run` exit non-zero) +# error the driver could not produce a result +# +# Subcommands: +# build [--ecosystem E]... build base + per-ecosystem images +# run [--ecosystem E] [--pm P] [--scenario S] [--host] [--out FILE] [--verbose] +# list [--json] enumerate every matrix case +# query [--status S] [--ecosystem E] [--pm P] [--scenario S] filter latest results +# results print the latest aggregate +# +# CLI/agent-friendly: `list`/`query`/`results` emit JSON; `run` writes a +# machine-readable report to tests/setup_matrix/results/latest.json. +# ===================================================================== +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SM_DIR="$REPO_ROOT/tests/setup_matrix" +MATRIX="$SM_DIR/matrix.json" +DRIVER="$SM_DIR/run-case.sh" +RESULTS_DIR="$SM_DIR/results" +LATEST="$RESULTS_DIR/latest.json" + +ALL_ECOSYSTEMS=(npm pypi cargo gem golang maven composer nuget deno) + +die() { echo "error: $*" >&2; exit 1; } +need() { command -v "$1" >/dev/null 2>&1 || die "'$1' is required but not on PATH"; } + +usage() { sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; } + +need jq +[ -f "$MATRIX" ] || die "matrix spec not found: $MATRIX" + +# Emit one TSV row per case, honoring filters. Covers all three layouts: +# single (targets x scenarios), workspace (workspace_targets x +# workspace_scenarios) and monorepo (monorepo_targets x monorepo_scenarios). +# Columns: id eco pm image hook_family baseline_supported package version +# purl manifest_key apply_ecosystems scenario patchset run_setup +# expect_applied layout +cases_tsv() { # $1=eco-filter ("" = all) $2=pm-filter $3=scenario-filter + jq -r --arg eco "${1:-}" --arg pm "${2:-}" --arg scn "${3:-}" ' + def rows($targets; $scenarios; $layout): + $targets[] as $t | $scenarios[] as $s + | select($eco == "" or $t.ecosystem == $eco) + | select($pm == "" or $t.pm == $pm) + | select($scn == "" or $s.id == $scn) + | [ ($t.ecosystem + "/" + $t.pm + "/" + $s.id), + $t.ecosystem, $t.pm, $t.image, ($t.hook_family // ""), + ($t.baseline_supported|tostring), + $t.package, $t.version, $t.purl, $t.manifest_key, $t.apply_ecosystems, + $s.id, $s.patchset, ($s.run_setup|tostring), ($s.expect_applied|tostring), + $layout ] + | @tsv; + rows(.targets; .scenarios; "single"), + rows((.workspace_targets // []); (.workspace_scenarios // []); "workspace"), + rows((.monorepo_targets // []); (.monorepo_scenarios // []); "monorepo") + ' "$MATRIX" +} + +marker() { jq -r '.marker' "$MATRIX"; } +alt_marker() { jq -r '.alt_marker' "$MATRIX"; } + +# --------------------------------------------------------------------- build +cmd_build() { + local ecos=(); + while [ $# -gt 0 ]; do case "$1" in + --ecosystem) ecos+=("$2"); shift 2;; + *) die "build: unknown arg '$1'";; + esac; done + [ ${#ecos[@]} -eq 0 ] && ecos=("${ALL_ECOSYSTEMS[@]}") + need docker + echo ">> building base image" >&2 + docker build -f "$REPO_ROOT/tests/docker/Dockerfile.base" -t socket-patch-test-base:latest "$REPO_ROOT" \ + || die "base image build failed" + local e + for e in "${ecos[@]}"; do + echo ">> building $e image" >&2 + docker build -f "$REPO_ROOT/tests/docker/Dockerfile.$e" -t "socket-patch-test-$e:latest" "$REPO_ROOT" \ + || die "$e image build failed" + done + echo ">> done" >&2 +} + +# --------------------------------------------------------------------- list +cmd_list() { + local as_json=0 + while [ $# -gt 0 ]; do case "$1" in --json) as_json=1; shift;; *) die "list: unknown arg '$1'";; esac; done + if [ "$as_json" = 1 ]; then + jq '[ .targets[] as $t | .scenarios[] as $s | + { id: ($t.ecosystem+"/"+$t.pm+"/"+$s.id), ecosystem:$t.ecosystem, pm:$t.pm, + scenario:$s.id, image:$t.image, hook_family:$t.hook_family, + baseline_supported:$t.baseline_supported, expect_applied:$s.expect_applied } ]' "$MATRIX" + else + printf '%-46s %-9s %-8s %-11s %-22s %s\n' ID ECO PM LAYOUT SCENARIO EXPECT + cases_tsv "" "" "" | while IFS=$'\t' read -r id eco pm image hook bsup pkg ver purl key aeco scn pset rsetup expect layout; do + printf '%-46s %-9s %-8s %-11s %-22s %s\n' "$id" "$eco" "$pm" "$layout" "$scn" "$expect" + done + fi +} + +# --------------------------------------------------------------------- run +resolve_host_bin() { + if [ -n "${SOCKET_PATCH_BIN:-}" ]; then echo "$SOCKET_PATCH_BIN"; return; fi + for c in "$REPO_ROOT/target/release/socket-patch" "$REPO_ROOT/target/debug/socket-patch"; do + [ -x "$c" ] && { echo "$c"; return; } + done + command -v socket-patch 2>/dev/null || echo "" +} + +cmd_run() { + local eco="" pm="" scn="" host=0 out="$LATEST" verbose=0 + while [ $# -gt 0 ]; do case "$1" in + --ecosystem) eco="$2"; shift 2;; + --pm) pm="$2"; shift 2;; + --scenario) scn="$2"; shift 2;; + --host) host=1; shift;; + --out) out="$2"; shift 2;; + --verbose) verbose=1; shift;; + *) die "run: unknown arg '$1'";; + esac; done + + local MARK ALT; MARK="$(marker)"; ALT="$(alt_marker)" + mkdir -p "$RESULTS_DIR" + local jsonl; jsonl="$(mktemp)" + + if [ "$host" = 0 ]; then need docker; fi + local host_bin="" + if [ "$host" = 1 ]; then + host_bin="$(resolve_host_bin)" + [ -n "$host_bin" ] || die "host mode: no socket-patch binary found (build it or set SOCKET_PATCH_BIN)" + echo ">> host mode, binary: $host_bin" >&2 + fi + + local total=0 + while IFS=$'\t' read -r id eco_ pm_ image hook bsup pkg ver purl key aeco scn_ pset rsetup expect layout; do + [ -z "$id" ] && continue + total=$((total+1)) + echo ">> [$total] $id (layout=$layout)" >&2 + + # Common SM_* env for the driver. + local -a base_env=( + "SM_ID=$id" "SM_ECOSYSTEM=$eco_" "SM_PM=$pm_" "SM_SCENARIO=$scn_" + "SM_LAYOUT=$layout" + "SM_PATCHSET=$pset" "SM_RUN_SETUP=$([ "$rsetup" = true ] && echo 1 || echo 0)" + "SM_EXPECT_APPLIED=$([ "$expect" = true ] && echo 1 || echo 0)" + "SM_PACKAGE=$pkg" "SM_VERSION=$ver" "SM_PURL=$purl" + "SM_MANIFEST_KEY=$key" "SM_APPLY_ECOSYSTEMS=$aeco" + "SM_MARKER=$MARK" "SM_ALT_MARKER=$ALT" + ) + + local raw="" rc=0 + if [ "$host" = 1 ]; then + if [ "$verbose" = 1 ]; then + raw="$(env "${base_env[@]}" "SOCKET_PATCH_BIN=$host_bin" bash "$DRIVER")"; rc=$? + else + raw="$(env "${base_env[@]}" "SOCKET_PATCH_BIN=$host_bin" bash "$DRIVER" 2>/dev/null)"; rc=$? + fi + else + local -a docker_env=() + local kv; for kv in "${base_env[@]}"; do docker_env+=(-e "$kv"); done + if [ "$verbose" = 1 ]; then + raw="$(docker run --rm "${docker_env[@]}" "socket-patch-test-$image:latest" bash -c "$(cat "$DRIVER")")"; rc=$? + else + raw="$(docker run --rm "${docker_env[@]}" "socket-patch-test-$image:latest" bash -c "$(cat "$DRIVER")" 2>/dev/null)"; rc=$? + fi + fi + + # The driver prints the result JSON as the last line of stdout. + local result; result="$(printf '%s\n' "$raw" | grep -E '^\{.*"actual_applied"' | tail -n1)" + + # baseline_applied = expect_applied AND baseline_supported. + local bl=false + if [ "$expect" = true ] && [ "$bsup" = true ]; then bl=true; fi + + if [ -n "$result" ] && printf '%s' "$result" | jq -e . >/dev/null 2>&1; then + printf '%s\n' "$result" | jq -c --argjson bl "$bl" --arg img "$image" --arg hk "$hook" --arg lay "$layout" ' + . as $r | + ($r.actual_applied == $r.expect_applied) as $ideal | + ($r.actual_applied == $bl) as $base | + (if $ideal and $base then "pass" + elif $ideal and ($base|not) then "progress" + elif ($ideal|not) and $base then "known_gap" + else "regression" end) as $cls | + $r + {baseline_applied:$bl, classification:$cls, layout:$lay, image:$img, hook_family:$hk, driver_rc:'"$rc"'} + ' >> "$jsonl" + else + # No parseable result — surface as an error case. + jq -nc --arg id "$id" --arg eco "$eco_" --arg pm "$pm_" --arg scn "$scn_" \ + --arg pset "$pset" --arg img "$image" --arg hk "$hook" --arg lay "$layout" --argjson bl "$bl" ' + { id:$id, ecosystem:$eco, pm:$pm, scenario:$scn, patchset:$pset, + expect_applied:null, actual_applied:null, baseline_applied:$bl, + classification:"error", layout:$lay, image:$img, hook_family:$hk, driver_rc:'"$rc"', + notes:"driver produced no parseable result" }' >> "$jsonl" + fi + done < <(cases_tsv "$eco" "$pm" "$scn") + + # Aggregate + summarize. + jq -s --arg generated "$(date -u +%FT%TZ)" ' + { generated:$generated, + summary: ( reduce .[] as $c ( + {total:0,pass:0,known_gap:0,progress:0,regression:0,error:0}; + .total += 1 | .[$c.classification] += 1 ) ), + cases: . }' "$jsonl" > "$out" + rm -f "$jsonl" + [ "$out" != "$LATEST" ] && cp "$out" "$LATEST" + + print_summary "$out" + local regressions; regressions="$(jq -r '.summary.regression' "$out")" + if [ "$regressions" -gt 0 ]; then + echo "!! $regressions regression(s) — a case that should work no longer does" >&2 + return 1 + fi + return 0 +} + +print_summary() { # $1 = results file + local f="$1" + echo "" >&2 + printf '%-44s %-8s %-6s %-6s %s\n' CASE PM APPLIED EXPECT STATUS >&2 + jq -r '.cases[] | [ .id, .pm, (.actual_applied|tostring), (.expect_applied|tostring), .classification ] | @tsv' "$f" \ + | while IFS=$'\t' read -r id pm act exp cls; do + printf '%-44s %-8s %-6s %-6s %s\n' "$id" "$pm" "$act" "$exp" "$cls" >&2 + done + echo "" >&2 + jq -r '.summary | "total=\(.total) pass=\(.pass) known_gap=\(.known_gap) progress=\(.progress) regression=\(.regression) error=\(.error)"' "$f" >&2 + local prog; prog="$(jq -r '.summary.progress' "$f")" + [ "$prog" -gt 0 ] && echo ">> $prog case(s) now BETTER than baseline — consider updating baseline_supported in matrix.json" >&2 + echo ">> full report: $f" >&2 +} + +# --------------------------------------------------------------------- query / results +cmd_query() { + local status="" eco="" pm="" scn="" lay="" + while [ $# -gt 0 ]; do case "$1" in + --status) status="$2"; shift 2;; + --ecosystem) eco="$2"; shift 2;; + --pm) pm="$2"; shift 2;; + --scenario) scn="$2"; shift 2;; + --layout) lay="$2"; shift 2;; + *) die "query: unknown arg '$1'";; + esac; done + [ -f "$LATEST" ] || die "no results yet — run '$0 run' first" + jq --arg st "$status" --arg eco "$eco" --arg pm "$pm" --arg scn "$scn" --arg lay "$lay" ' + [ .cases[] + | select($st == "" or .classification == $st) + | select($eco == "" or .ecosystem == $eco) + | select($pm == "" or .pm == $pm) + | select($scn == "" or .scenario == $scn) + | select($lay == "" or .layout == $lay) ]' "$LATEST" +} + +cmd_results() { + [ -f "$LATEST" ] || die "no results yet — run '$0 run' first" + cat "$LATEST" +} + +# --------------------------------------------------------------------- dispatch +[ $# -ge 1 ] || { usage; exit 1; } +sub="$1"; shift || true +case "$sub" in + build) cmd_build "$@";; + run) cmd_run "$@";; + list) cmd_list "$@";; + query) cmd_query "$@";; + results) cmd_results "$@";; + -h|--help|help) usage;; + *) die "unknown subcommand '$sub' (try: build run list query results)";; +esac diff --git a/tests/docker/Dockerfile.npm b/tests/docker/Dockerfile.npm index 31b3d418..9ed0a413 100644 --- a/tests/docker/Dockerfile.npm +++ b/tests/docker/Dockerfile.npm @@ -21,6 +21,18 @@ RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ RUN curl -fsSL https://bun.sh/install | bash \ && ln -s /root/.bun/bin/bun /usr/local/bin/bun +# Enable pnpm and yarn via corepack (bundled with Node 20). The +# setup-matrix suite (tests/setup_matrix/) drives all four npm-family +# package managers — npm, yarn, pnpm, bun — through the `socket-patch +# setup` install-hook flow. `corepack prepare … --activate` pins and +# activates a known version so the binaries resolve without a network +# round-trip at test time. Additive: the existing docker_e2e_npm tests +# are unaffected. +RUN corepack enable \ + && corepack prepare pnpm@9.15.0 --activate \ + && corepack prepare yarn@1.22.22 --activate + # Verify versions are sane at image-build time so a broken NodeSource setup # fails the image build rather than every downstream test. -RUN node --version && npm --version && bun --version && socket-patch --version +RUN node --version && npm --version && bun --version \ + && pnpm --version && yarn --version && socket-patch --version diff --git a/tests/docker/Dockerfile.pypi b/tests/docker/Dockerfile.pypi index 8e7ea3ed..8cab8df2 100644 --- a/tests/docker/Dockerfile.pypi +++ b/tests/docker/Dockerfile.pypi @@ -1,14 +1,16 @@ -# pypi ecosystem test image: base + Python 3.11 + pip + venv + uv. +# pypi ecosystem test image: base + Python 3.11 + pip + venv + uv + +# poetry + pdm + hatch. # # Debian 12 ships Python 3.11. We use a venv inside each test to keep # pip from needing `--break-system-packages` and to match real-world # user flow. # -# uv is installed from PyPI (single self-contained wheel) so the same -# image can drive both the pip-based and uv-based e2e tests. The -# `--break-system-packages` flag is what Debian-packaged pip3 requires -# to install into the system site-packages; it's safe inside the -# disposable test container. +# uv/poetry/pdm/hatch are installed from PyPI so the one image can drive +# every Python package manager the setup-matrix suite exercises +# (tests/setup_matrix/). The `--break-system-packages` flag is what +# Debian-packaged pip3 requires to install into the system site-packages; +# it's safe inside the disposable test container. Additive: the existing +# docker_e2e_pypi tests (pip + uv) are unaffected. FROM socket-patch-test-base:latest RUN apt-get update \ @@ -17,7 +19,10 @@ RUN apt-get update \ python3-pip \ python3-venv \ && rm -rf /var/lib/apt/lists/* \ - && pip3 install --break-system-packages --no-cache-dir uv \ + && pip3 install --break-system-packages --no-cache-dir uv poetry pdm hatch \ && python3 --version \ && pip3 --version \ - && uv --version + && uv --version \ + && poetry --version \ + && pdm --version \ + && hatch --version diff --git a/tests/docker/README.md b/tests/docker/README.md index 8089d0f6..c4b128bc 100644 --- a/tests/docker/README.md +++ b/tests/docker/README.md @@ -108,3 +108,18 @@ Fixtures are synthetic. Real Socket patches are not required to exist for the tested PURLs — what's validated is that the crawler discovers real installed packages and the CLI dispatches correctly through the ecosystem. + +## Related: the `setup`-flow matrix + +A separate, **experimental** suite lives under `tests/setup_matrix/` and +reuses these same per-ecosystem images. Where `docker_e2e_*` drives +`scan → apply` explicitly, the setup-matrix instead runs `socket-patch +setup` and then a *native install* to check whether the configured +install hook applies the patch on its own — the thing `setup` is meant +to enable. It also adds the npm-family package managers (pnpm/yarn via +corepack) and the Python ones (uv/poetry/pdm/hatch), which is why +`Dockerfile.npm` and `Dockerfile.pypi` install those tools. See +`tests/setup_matrix/README.md` for details and the +`scripts/setup-matrix.sh` runner. That suite's CI job (`setup-matrix`) +is **non-blocking** (`continue-on-error: true`) and is expected to fail +for ecosystems whose hooks `setup` does not yet configure. diff --git a/tests/setup_matrix/README.md b/tests/setup_matrix/README.md new file mode 100644 index 00000000..1a66ce05 --- /dev/null +++ b/tests/setup_matrix/README.md @@ -0,0 +1,168 @@ +# `setup`-flow test matrix (experimental) + +This suite verifies the **intended** end-to-end behavior of +`socket-patch setup`: that after `setup` configures a project, a normal +package-manager install applies the project's patches *on its own*, with +no explicit `scan`/`apply` step. + +It is **experimental and non-blocking**. `setup` only configures +npm-family install hooks today, so most non-npm cases are *expected to +fail*. The suite encodes the **aspirational** end state and records a +per-case **baseline** of what works now — the failing cases are a TODO +list for `setup`, not a broken test. + +## The flow (per case) + +Every case runs the same four steps via the bash driver `run-case.sh`: + +0. **prepare** a throwaway project: declare the dependency and commit a + patch set (`.socket/manifest.json` + `.socket/blobs/`). +1. **`socket-patch setup`** — configure install hooks (skipped in the + `no_setup_control` scenario). +2. **native install** — `npm install` / `pip install` / `cargo fetch` / + … for the package manager under test. +3. **check** — is the patch's marker now on disk in the installed file? + +The apply step is fully offline (`SOCKET_OFFLINE=1 SOCKET_FORCE=1`, +inherited by the hook), so the only network use is the real package +install. No Socket API is contacted. + +## Dimensions + +`ecosystem × package-manager × scenario` — see `matrix.json` (the single +source of truth, consumed by both the runner script and the Rust +wrappers). + +- **Package managers:** npm, yarn, pnpm, bun · pip, uv, poetry, pdm, + hatch · cargo · bundler · go · mvn · composer · dotnet · deno. +- **Scenarios (single-project):** + - `baseline_with_setup` — setup + install ⇒ patch applied *(ideal)*. + - `no_setup_control` — **ablation (setup not run)**: install only ⇒ NOT applied *(the hook is the cause)*. + - `patch_missing` — **ablation (patch missing)**: setup runs and the hook fires, but no `.socket/` patch set is committed ⇒ runs UNPATCHED *(the committed patch is the cause)*. + - `empty_patchset` — manifest present but with zero patches ⇒ NOT applied. + - `wrong_target_patchset` — manifest targets a different package ⇒ NOT applied. + - `alt_content_patchset` — a second patch set ⇒ its marker applied *(content tracks the manifest)*. + + The two **ablations** are the controls that confirm `setup` is correct: + each is identical to `baseline_with_setup` except for the single removed + factor (the setup step, or the committed patch), and each must run + unpatched. The workspace and monorepo layouts carry the same pair + (`*_no_setup`, `*_patch_missing`). + +## Layouts + +The driver's `SM_LAYOUT` selects the project shape (each layout has its +own `*_targets` / `*_scenarios` sections in `matrix.json`): + +- **`single`** *(default)* — one project, one dependency. The 16-PM grid above. +- **`workspace`** — a **nested workspace/monorepo**: a root + several + members (incl. a deeply-nested one and a member that does *not* use the + patched package). Models real-world monorepo deployments and exercises + `setup`'s workspace handling — npm/yarn write the hook to **every** + member, pnpm only to the **root** — plus the cross-workspace apply on a + single root install. Covered PMs: **npm, pnpm, yarn** (apply; the + dependency hoists / lands in the pnpm store and is patched once) and + **pip** (nested `requirements.txt` files) + **uv** (uv workspace, one + shared `.venv`) as Python gaps. Scenarios: `workspace_with_setup`, + `workspace_no_setup`, `workspace_patch_missing`. +- **`monorepo`** — a **polyglot all-ecosystem repo**: an npm workspace + alongside python/rust/go/php/ruby/nuget/deno manifests. Confirms `setup` + works in a mixed environment — it must configure the npm hooks and + **not choke** on the foreign manifests; a root `npm install` then + patches the npm slice. Runs in the npm image (the only one with the npm + toolchain), so the foreign manifests are present to test setup's + robustness, not installed. Scenarios: `monorepo_with_setup`, + `monorepo_no_setup`, `monorepo_patch_missing`. + +> Real-world wiring note surfaced by the workspace layout: the install +> hook's `apply` must run with the package manager's per-script cwd (root +> for the project, the member dir for each member) — so member +> postinstalls find no manifest and no-op while the root applies. Forcing +> a single cwd makes every member target the root manifest and fail +> mid-install with "no packages found on disk". The driver therefore does +> **not** pin `SOCKET_CWD`. + +## Result classification + +Each case's `actual` is compared against both the aspirational `expect` +and the recorded `baseline`: + +| classification | meaning | +|---|---| +| `pass` | meets the ideal and matches the baseline | +| `known_gap` | fails the ideal, exactly as recorded — expected today, non-blocking | +| `progress` | better than the recorded baseline — update `baseline_supported` in `matrix.json`! | +| `regression` | diverged from the baseline the wrong way — the only thing that fails the runner | +| `error` | the driver produced no parseable result | + +The Rust wrappers (`tests/setup_matrix_.rs`) assert the **ideal** +(`actual == expect`), so they are red for `known_gap` cases — that is the +intended "TODO list" view. The `scripts/setup-matrix.sh` runner uses the +**baseline** view and only exits non-zero on a `regression`. + +## Running it + +Requires a Docker daemon (default) or host-installed toolchains +(`SOCKET_PATCH_TEST_HOST=1`). + +```sh +# Build the shared base + a per-ecosystem image. +scripts/setup-matrix.sh build --ecosystem npm + +# Run all npm-family cases and write a JSON report. +scripts/setup-matrix.sh run --ecosystem npm + +# Filter to a single package manager / scenario. +scripts/setup-matrix.sh run --ecosystem pypi --pm uv +scripts/setup-matrix.sh run --scenario no_setup_control + +# Run the nested-workspace and polyglot-monorepo cases. +scripts/setup-matrix.sh run --scenario workspace_with_setup +scripts/setup-matrix.sh run --scenario monorepo_with_setup + +# Query the last results (agent-friendly JSON). +scripts/setup-matrix.sh query --status known_gap +scripts/setup-matrix.sh query --status regression +scripts/setup-matrix.sh query --layout workspace +scripts/setup-matrix.sh list --json + +# Host mode (no Docker; needs the toolchains + a built binary on PATH). +SOCKET_PATCH_TEST_HOST=1 scripts/setup-matrix.sh run --ecosystem npm --host +``` + +Or via `cargo test` (the aspirational view; gated by the `setup-e2e` +feature; soft-skips when the image isn't built): + +```sh +cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_npm +SOCKET_PATCH_TEST_HOST=1 cargo test -p socket-patch-cli --features setup-e2e --test setup_matrix_npm +``` + +## Files + +- `matrix.json` — declarative case list: `targets`×`scenarios` (single), + `workspace_targets`×`workspace_scenarios`, `monorepo_targets`×`monorepo_scenarios`, + markers. +- `run-case.sh` — self-contained flow driver (one case → JSON result), + layout-aware (`SM_LAYOUT=single|workspace|monorepo`); generates the + runner shims inline so it can be piped into a container. +- `shims/{npx,pnpm}` — reference copies of the PATH shims that route + `npx`/`pnpm dlx @socketsecurity/socket-patch` to the locally-built + binary (so the hook runs the binary under test, not a registry fetch). +- `results/latest.json` — most recent aggregate report (git-ignored). +- `../docker/Dockerfile.{npm,pypi,…}` — the per-ecosystem images + (npm/pypi extended with the extra package managers). +- `../../crates/socket-patch-cli/tests/setup_matrix_.rs` — thin Rust + wrappers around the same driver (incl. `setup_matrix_monorepo.rs`; the + npm/pypi wrappers add `*_workspace` tests). + +## Adding a package manager / ecosystem + +1. Add a `targets[]` entry to `matrix.json` (image, package, purl, + manifest key, whether `setup` supports it today via + `baseline_supported`). +2. Teach `run-case.sh` how to scaffold + install + resolve the target + file for the new `pm` (the `scaffold_project` / `run_install` / + `resolve_target` case statements). +3. If a new toolchain is needed, add it to the relevant + `tests/docker/Dockerfile.`. +4. Add a `#[test]` for the `pm` in the matching `setup_matrix_.rs`. diff --git a/tests/setup_matrix/matrix.json b/tests/setup_matrix/matrix.json new file mode 100644 index 00000000..ed6ccbdc --- /dev/null +++ b/tests/setup_matrix/matrix.json @@ -0,0 +1,281 @@ +{ + "_comment": [ + "Declarative source of truth for the `socket-patch setup` end-to-end test matrix.", + "Consumed by BOTH scripts/setup-matrix.sh (jq) and the Rust wrappers", + "crates/socket-patch-cli/tests/setup_matrix_.rs (serde_json).", + "", + "A 'case' is the cross-product (target x scenario). expect_applied comes from", + "the scenario (the ASPIRATIONAL ideal); baseline_supported on the target says", + "whether `setup` ACTUALLY wires a working install hook today. The classifier in", + "the orchestrator compares actual vs both: meeting the ideal => pass; failing the", + "ideal but matching the recorded baseline => known_gap (non-blocking); diverging", + "from the baseline in the wrong direction => regression (blocking the optional job).", + "", + "Packages, PURLs, manifest keys and install layouts are reused verbatim from the", + "existing tests/docker_e2e_.rs so the fixtures are known-valid.", + "NOTE: pypi uses NO `package/` prefix in the manifest key (the python crawler", + "reports the site-packages root); every other ecosystem uses `package/`." + ], + + "marker": "SOCKET-PATCH-SETUP-MATRIX-MARKER", + "alt_marker": "SOCKET-PATCH-SETUP-MATRIX-ALT-MARKER", + + "scenarios": [ + { + "id": "baseline_with_setup", + "run_setup": true, + "patchset": "primary", + "expect_applied": true, + "description": "Prepare deps + committed patch set, run `socket-patch setup`, run the native install. The install hook should apply the patch (the ideal)." + }, + { + "id": "no_setup_control", + "run_setup": false, + "patchset": "primary", + "expect_applied": false, + "description": "Negative control: identical fixture but setup is NOT run. With no hook configured, the install must NOT apply the patch." + }, + { + "id": "empty_patchset", + "run_setup": true, + "patchset": "empty", + "expect_applied": false, + "description": "Different patch set: an empty manifest. Even with setup, nothing should be applied." + }, + { + "id": "wrong_target_patchset", + "run_setup": true, + "patchset": "wrong", + "expect_applied": false, + "description": "Different patch set: a manifest that patches a different, non-installed package. The installed package must be left untouched." + }, + { + "id": "alt_content_patchset", + "run_setup": true, + "patchset": "alt", + "expect_applied": true, + "description": "Different patch set: a second fixture whose blob carries the ALT marker. Proves the applied bytes track the active manifest (alt marker present, primary marker absent)." + }, + { + "id": "patch_missing", + "run_setup": true, + "patchset": "none", + "expect_applied": false, + "description": "Ablation: setup runs and the install hook fires, but NO patch set is committed (no .socket/). The install must run UNPATCHED — proving the committed patch is what changes the code, not setup/install alone." + } + ], + + "targets": [ + { + "ecosystem": "npm", "pm": "npm", "image": "npm", "hook_family": "npm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + { + "ecosystem": "npm", "pm": "yarn", "image": "npm", "hook_family": "npm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + { + "ecosystem": "npm", "pm": "pnpm", "image": "npm", "hook_family": "pnpm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + { + "ecosystem": "npm", "pm": "bun", "image": "npm", "hook_family": "npm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + + { + "ecosystem": "pypi", "pm": "pip", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + }, + { + "ecosystem": "pypi", "pm": "uv", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + }, + { + "ecosystem": "pypi", "pm": "poetry", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + }, + { + "ecosystem": "pypi", "pm": "pdm", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + }, + { + "ecosystem": "pypi", "pm": "hatch", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + }, + + { + "ecosystem": "cargo", "pm": "cargo", "image": "cargo", "hook_family": "none", + "baseline_supported": false, + "package": "cfg-if", "version": "1.0.0", "purl": "pkg:cargo/cfg-if@1.0.0", + "manifest_key": "package/src/lib.rs", "apply_ecosystems": "cargo" + }, + + { + "ecosystem": "gem", "pm": "bundler", "image": "gem", "hook_family": "none", + "baseline_supported": false, + "package": "colorize", "version": "1.1.0", "purl": "pkg:gem/colorize@1.1.0", + "manifest_key": "package/lib/colorize.rb", "apply_ecosystems": "gem" + }, + + { + "ecosystem": "golang", "pm": "go", "image": "golang", "hook_family": "none", + "baseline_supported": false, + "package": "github.com/gin-gonic/gin", "version": "v1.9.1", + "purl": "pkg:golang/github.com/gin-gonic/gin@v1.9.1", + "manifest_key": "package/gin.go", "apply_ecosystems": "golang" + }, + + { + "ecosystem": "maven", "pm": "mvn", "image": "maven", "hook_family": "none", + "baseline_supported": false, + "package": "org.apache.commons:commons-lang3", "version": "3.12.0", + "purl": "pkg:maven/org.apache.commons/commons-lang3@3.12.0", + "manifest_key": "package/commons-lang3-3.12.0.pom", "apply_ecosystems": "maven" + }, + + { + "ecosystem": "composer", "pm": "composer", "image": "composer", "hook_family": "composer-event", + "baseline_supported": false, + "package": "monolog/monolog", "version": "3.5.0", "purl": "pkg:composer/monolog/monolog@3.5.0", + "manifest_key": "package/src/Monolog/Logger.php", "apply_ecosystems": "composer" + }, + + { + "ecosystem": "nuget", "pm": "dotnet", "image": "nuget", "hook_family": "none", + "baseline_supported": false, + "package": "Newtonsoft.Json", "version": "13.0.3", "purl": "pkg:nuget/newtonsoft.json@13.0.3", + "manifest_key": "package/LICENSE.md", "apply_ecosystems": "nuget" + }, + + { + "ecosystem": "deno", "pm": "deno", "image": "deno", "hook_family": "npm-via-deno", + "baseline_supported": false, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + } + ], + + "_workspace_comment": [ + "Nested-workspace layouts (run-case.sh SM_LAYOUT=workspace): a root +", + "several workspace members (incl. a deeply-nested one and a member that", + "does NOT use the patched package). Models real monorepos and exercises", + "`setup`'s workspace handling — npm/yarn write the hook to every member,", + "pnpm only to the root — plus the cross-workspace apply on the root", + "install. npm/yarn/pnpm should apply (baseline_supported true); Python", + "workspaces (uv workspace, pip nested-requirements) are gaps." + ], + "workspace_scenarios": [ + { + "id": "workspace_with_setup", + "run_setup": true, + "patchset": "primary", + "expect_applied": true, + "description": "Nested workspace: setup at root, then a root-level install must apply the patch to the (hoisted/store-linked) dependency used across members." + }, + { + "id": "workspace_no_setup", + "run_setup": false, + "patchset": "primary", + "expect_applied": false, + "description": "Ablation (setup not run): workspace install without setup must NOT apply — the hook is the cause." + }, + { + "id": "workspace_patch_missing", + "run_setup": true, + "patchset": "none", + "expect_applied": false, + "description": "Ablation (patch missing): workspace setup + install with NO committed patch set must run unpatched across the workspace." + } + ], + "workspace_targets": [ + { + "ecosystem": "npm", "pm": "npm", "image": "npm", "hook_family": "npm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + { + "ecosystem": "npm", "pm": "pnpm", "image": "npm", "hook_family": "pnpm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + { + "ecosystem": "npm", "pm": "yarn", "image": "npm", "hook_family": "npm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + }, + { + "ecosystem": "pypi", "pm": "pip", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + }, + { + "ecosystem": "pypi", "pm": "uv", "image": "pypi", "hook_family": "none", + "baseline_supported": false, + "package": "six", "version": "1.16.0", "purl": "pkg:pypi/six@1.16.0", + "manifest_key": "six.py", "apply_ecosystems": "pypi" + } + ], + + "_monorepo_comment": [ + "Polyglot monorepo (SM_LAYOUT=monorepo): an npm workspace alongside", + "python/rust/go/php/ruby/nuget/deno manifests. Confirms `setup` works in", + "a mixed environment — it must configure the npm hooks and NOT choke on", + "the foreign manifests; a root `npm install` then patches the npm slice.", + "Runs in the npm image (the only one with the npm toolchain); the foreign", + "manifests are present to test setup's robustness, not installed." + ], + "monorepo_scenarios": [ + { + "id": "monorepo_with_setup", + "run_setup": true, + "patchset": "primary", + "expect_applied": true, + "description": "All ecosystems present: setup at root, then npm install applies the patch to the npm workspace dependency; setup must not error on the foreign manifests." + }, + { + "id": "monorepo_no_setup", + "run_setup": false, + "patchset": "primary", + "expect_applied": false, + "description": "Ablation (setup not run): polyglot monorepo install without setup must NOT apply." + }, + { + "id": "monorepo_patch_missing", + "run_setup": true, + "patchset": "none", + "expect_applied": false, + "description": "Ablation (patch missing): polyglot monorepo setup + install with NO committed patch set must run unpatched." + } + ], + "monorepo_targets": [ + { + "ecosystem": "monorepo", "pm": "mono", "image": "npm", "hook_family": "npm", + "baseline_supported": true, + "package": "minimist", "version": "1.2.2", "purl": "pkg:npm/minimist@1.2.2", + "manifest_key": "package/index.js", "apply_ecosystems": "npm" + } + ] +} diff --git a/tests/setup_matrix/results/.gitignore b/tests/setup_matrix/results/.gitignore new file mode 100644 index 00000000..d5ae1810 --- /dev/null +++ b/tests/setup_matrix/results/.gitignore @@ -0,0 +1,3 @@ +# Generated setup-matrix run reports; keep the directory, ignore contents. +* +!.gitignore diff --git a/tests/setup_matrix/run-case.sh b/tests/setup_matrix/run-case.sh new file mode 100755 index 00000000..328587be --- /dev/null +++ b/tests/setup_matrix/run-case.sh @@ -0,0 +1,543 @@ +#!/usr/bin/env bash +# ===================================================================== +# setup-matrix flow driver — runs ONE (ecosystem, pm, scenario) case of +# the `socket-patch setup` end-to-end matrix and emits a JSON result. +# +# This script is the single source of truth for the flow: +# 0. prepare a project with the dependency + a committed patch set +# 1. (optionally) run `socket-patch setup` to configure install hooks +# 2. run the native install command for the package manager +# 3. check whether the patch was applied (marker present on disk) +# +# It is invoked by BOTH scripts/setup-matrix.sh (orchestrator) and the +# Rust wrappers (crates/socket-patch-cli/tests/setup_matrix_.rs), +# either inside a Docker container (script piped to `bash -c`) or on the +# host. It is fully self-contained: it generates the npx/pnpm shims +# inline so no extra files need to be copied into the container. +# +# The driver only REPORTS (expected vs actual). Pass/fail/known-gap/ +# regression classification is done by the caller against the recorded +# baseline in matrix.json. +# +# Inputs (environment, all SM_*-prefixed): +# SM_ID stable case id (for the JSON result) +# SM_ECOSYSTEM npm|pypi|cargo|gem|golang|maven|composer|nuget|deno +# SM_PM npm|yarn|pnpm|bun|pip|uv|poetry|pdm|hatch|cargo| +# bundler|go|mvn|composer|dotnet|deno +# SM_SCENARIO scenario id (echoed back) +# SM_PATCHSET primary|alt|empty|wrong +# SM_RUN_SETUP 1|0 — run `socket-patch setup` before install +# SM_EXPECT_APPLIED 1|0 — the aspirational expectation +# SM_PACKAGE dependency name (e.g. minimist, six, cfg-if) +# SM_VERSION dependency version (e.g. 1.2.2) +# SM_PURL manifest key PURL (e.g. pkg:npm/minimist@1.2.2) +# SM_MANIFEST_KEY file key in the patch record (e.g. package/index.js, +# or `six.py` for pypi — NO package/ prefix) +# SM_APPLY_ECOSYSTEMS ecosystem token used to build the "wrong" PURL +# SM_MARKER primary marker string spliced into the patched blob +# SM_ALT_MARKER alternate marker (alt_content_patchset) +# SOCKET_PATCH_BIN path to the binary under test (default: socket-patch on PATH) +# SM_WORKDIR scratch dir (default: a fresh mktemp -d) +# ===================================================================== + +set -uo pipefail + +# Route all ordinary output to stderr; the final JSON goes to the saved +# stdout (fd 3) so the result line is the ONLY thing on real stdout. +exec 3>&1 1>&2 + +SM_ID="${SM_ID:-unknown}" +SM_ECOSYSTEM="${SM_ECOSYSTEM:-}" +SM_PM="${SM_PM:-}" +SM_SCENARIO="${SM_SCENARIO:-}" +SM_PATCHSET="${SM_PATCHSET:-primary}" +SM_RUN_SETUP="${SM_RUN_SETUP:-1}" +SM_EXPECT_APPLIED="${SM_EXPECT_APPLIED:-0}" +SM_PACKAGE="${SM_PACKAGE:-}" +SM_VERSION="${SM_VERSION:-}" +SM_PURL="${SM_PURL:-}" +SM_MANIFEST_KEY="${SM_MANIFEST_KEY:-package/index.js}" +SM_APPLY_ECOSYSTEMS="${SM_APPLY_ECOSYSTEMS:-npm}" +SM_MARKER="${SM_MARKER:-SOCKET-PATCH-SETUP-MATRIX-MARKER}" +SM_ALT_MARKER="${SM_ALT_MARKER:-SOCKET-PATCH-SETUP-MATRIX-ALT-MARKER}" +SM_LAYOUT="${SM_LAYOUT:-single}" + +ZEROHASH="0000000000000000000000000000000000000000000000000000000000000000" +UUID="aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" +WRONG_PURL="pkg:${SM_APPLY_ECOSYSTEMS}/sm-setup-matrix-absent@9.9.9" + +SP_BIN="${SOCKET_PATCH_BIN:-$(command -v socket-patch 2>/dev/null || echo socket-patch)}" +export SOCKET_PATCH_BIN="$SP_BIN" + +NOTES="" +note() { NOTES="${NOTES}${NOTES:+; }$*"; } +log() { printf '[setup-matrix:%s] %s\n' "$SM_ID" "$*"; } + +# --- JSON emit (hand-rolled; values are simple, sanitized) ------------ +json_str() { printf '%s' "$1" | tr -d '\r' | tr '\n' ' ' | sed 's/\\/\\\\/g; s/"/\\"/g'; } +emit_result() { + local actual="$1" primary_present="$2" setup_exit="$3" install_exit="$4" target="$5" status="$6" + printf '{"id":"%s","ecosystem":"%s","pm":"%s","scenario":"%s","patchset":"%s","run_setup":%s,"expect_applied":%s,"actual_applied":%s,"primary_marker_present":%s,"setup_exit":%s,"install_exit":%s,"target":"%s","status":"%s","notes":"%s"}\n' \ + "$(json_str "$SM_ID")" "$(json_str "$SM_ECOSYSTEM")" "$(json_str "$SM_PM")" \ + "$(json_str "$SM_SCENARIO")" "$(json_str "$SM_PATCHSET")" \ + "$([ "$SM_RUN_SETUP" = 1 ] && echo true || echo false)" \ + "$([ "$SM_EXPECT_APPLIED" = 1 ] && echo true || echo false)" \ + "$actual" "$primary_present" "$setup_exit" "$install_exit" \ + "$(json_str "$target")" "$(json_str "$status")" "$(json_str "$NOTES")" >&3 +} + +# --- git-sha256 (blob \0 + content) ------------------------------ +git_sha256() { # $1 = file + local len; len="$(wc -c < "$1")" + { printf 'blob %d\0' "$len"; cat "$1"; } | sha256sum | cut -d' ' -f1 +} + +# --- inline npx/pnpm shims (kept in sync with tests/setup_matrix/shims/) -- +write_shims() { # $1 = shim dir + local d="$1"; mkdir -p "$d" + cat > "$d/npx" <<'SHIM' +#!/usr/bin/env bash +set -uo pipefail +sp_bin="${SOCKET_PATCH_BIN:-socket-patch}" +shim_dir="${SETUP_MATRIX_SHIM_DIR:-}" +clean_path="$PATH" +[ -n "$shim_dir" ] && clean_path="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$shim_dir" | paste -sd: -)" +real_npx="$(PATH="$clean_path" command -v npx 2>/dev/null || true)" +i=0 +for arg in "$@"; do + case "$arg" in + @socketsecurity/socket-patch|@socketsecurity/socket-patch@*|@socketsecurity/socket-patch/*) + shift "$((i + 1))"; exec "$sp_bin" "$@" ;; + esac + i=$((i + 1)) +done +[ -n "$real_npx" ] && exec "$real_npx" "$@" +echo "setup-matrix npx shim: real npx not found: $*" >&2; exit 127 +SHIM + cat > "$d/pnpm" <<'SHIM' +#!/usr/bin/env bash +set -uo pipefail +sp_bin="${SOCKET_PATCH_BIN:-socket-patch}" +shim_dir="${SETUP_MATRIX_SHIM_DIR:-}" +clean_path="$PATH" +[ -n "$shim_dir" ] && clean_path="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$shim_dir" | paste -sd: -)" +real_pnpm="$(PATH="$clean_path" command -v pnpm 2>/dev/null || true)" +if [ "${1:-}" = "dlx" ] || [ "${1:-}" = "exec" ]; then + case "${2:-}" in + @socketsecurity/socket-patch|@socketsecurity/socket-patch@*) shift 2; exec "$sp_bin" "$@" ;; + esac +fi +[ -n "$real_pnpm" ] && exec "$real_pnpm" "$@" +echo "setup-matrix pnpm shim: real pnpm not found: $*" >&2; exit 127 +SHIM + chmod +x "$d/npx" "$d/pnpm" +} + +# --- committed patch fixture ------------------------------------------ +write_manifest() { # $1=purl $2=key $3=afterHash + cat > .socket/manifest.json < .socket/manifest.json + note "empty manifest" ;; + wrong) + # A patch for a package that is NOT installed: nothing should match. + local body="/* $SM_MARKER */"; printf '%s\n' "$body" > /tmp/sm_blob + local h; h="$(git_sha256 /tmp/sm_blob)"; cp /tmp/sm_blob ".socket/blobs/$h" + write_manifest "$WRONG_PURL" "$SM_MANIFEST_KEY" "$h" + note "manifest targets absent purl $WRONG_PURL" ;; + alt) + local body="/* $SM_ALT_MARKER */"; printf '%s\n' "$body" > /tmp/sm_blob + local h; h="$(git_sha256 /tmp/sm_blob)"; cp /tmp/sm_blob ".socket/blobs/$h" + write_manifest "$SM_PURL" "$SM_MANIFEST_KEY" "$h" ;; + *) # primary + local body="/* $SM_MARKER */"; printf '%s\n' "$body" > /tmp/sm_blob + local h; h="$(git_sha256 /tmp/sm_blob)"; cp /tmp/sm_blob ".socket/blobs/$h" + write_manifest "$SM_PURL" "$SM_MANIFEST_KEY" "$h" ;; + esac +} + +# --- per-PM project scaffold (must exist before setup runs) ----------- +scaffold_project() { + case "$SM_PM" in + npm|yarn|bun) + printf '{"name":"sm-proj","version":"0.0.0","private":true}\n' > package.json ;; + pnpm) + # pnpm only runs the ROOT postinstall on `pnpm install` (not on + # `pnpm add`), so the dependency is declared up front and installed + # via a bare `pnpm install`. The stub lockfile is the pnpm marker + # that makes `setup` detect pnpm and write the `pnpm dlx` hook. + cat > package.json < pnpm-lock.yaml ;; + deno) + cat > package.json < deno.json < pyproject.toml <"] +package-mode = false + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" +EOF + ;; + pdm) + cat > pyproject.toml < pyproject.toml < Cargo.toml < src/main.rs ;; + bundler) + cat > Gemfile < go.mod ;; + mvn|composer|dotnet) : ;; + esac +} + +# --- per-PM native install (the hook, if configured, fires here) ------ +run_install() { + case "$SM_PM" in + npm) npm install --silent --no-audit --no-fund "$SM_PACKAGE@$SM_VERSION" ;; + yarn) yarn add --silent "$SM_PACKAGE@$SM_VERSION" ;; + pnpm) pnpm install --no-frozen-lockfile ;; + bun) bun add "$SM_PACKAGE@$SM_VERSION" ;; + deno) deno install --allow-scripts ;; + pip) python3 -m venv venv && ./venv/bin/pip install --disable-pip-version-check --quiet --no-cache-dir "$SM_PACKAGE==$SM_VERSION" ;; + uv) uv venv venv && uv pip install --python venv/bin/python --quiet "$SM_PACKAGE==$SM_VERSION" ;; + poetry) poetry config virtualenvs.in-project true --local && poetry add --no-interaction "$SM_PACKAGE@$SM_VERSION" ;; + pdm) pdm config python.use_venv true >/dev/null 2>&1; pdm add "$SM_PACKAGE==$SM_VERSION" ;; + hatch) HATCH_DATA_DIR="$PWD/.hatch" hatch env create && HATCH_DATA_DIR="$PWD/.hatch" hatch run python -c "import ${SM_PACKAGE//-/_}" ;; + cargo) cargo fetch ;; + bundler) bundle config set --local path vendor/bundle && bundle install ;; + go) GOFLAGS=-mod=mod go mod download "$SM_PACKAGE@$SM_VERSION" ;; + mvn) mvn -q -B dependency:get -Dartifact="$SM_PACKAGE:$SM_VERSION" ;; + composer) composer require --quiet --no-interaction "$SM_PACKAGE:$SM_VERSION" ;; + dotnet) dotnet new classlib -o . --force >/dev/null 2>&1 && dotnet add package "$SM_PACKAGE" --version "$SM_VERSION" ;; + *) echo "unknown pm: $SM_PM"; return 2 ;; + esac +} + +# --- resolve the on-disk file the patch would land in ----------------- +resolve_target() { + local rel="${SM_MANIFEST_KEY#package/}" + local base; base="$(basename "$rel")" + case "$SM_ECOSYSTEM" in + npm|deno) printf '%s\n' "$PWD/node_modules/$SM_PACKAGE/$rel" ;; + pypi) find "$PWD" -name "$base" 2>/dev/null | head -1 ;; + cargo) find "${CARGO_HOME:-$HOME/.cargo}/registry/src" -path "*/${SM_PACKAGE}-${SM_VERSION}/${rel}" 2>/dev/null | head -1 ;; + gem) find "$PWD/vendor" -path "*/${SM_PACKAGE}-${SM_VERSION}/${rel}" 2>/dev/null | head -1 ;; + golang) local gmc; gmc="$(go env GOMODCACHE 2>/dev/null || echo "${GOPATH:-$HOME/go}/pkg/mod")"; find "$gmc" -path "*/$(basename "$SM_PACKAGE")@${SM_VERSION}/${rel}" 2>/dev/null | head -1 ;; + maven) find "$HOME/.m2/repository" -name "$base" 2>/dev/null | head -1 ;; + composer) printf '%s\n' "$PWD/vendor/${SM_PACKAGE}/${rel}" ;; + nuget) local lc; lc="$(printf '%s' "$SM_PACKAGE" | tr '[:upper:]' '[:lower:]')"; find "${NUGET_PACKAGES:-$HOME/.nuget/packages}" -path "*/${lc}/${SM_VERSION}/${rel}" 2>/dev/null | head -1 ;; + esac +} + +# --- workspace scaffold (root + nested members) ---------------------- +# Models a real monorepo where multiple (incl. deeply-nested) workspace +# members depend on the package being patched, plus a member that does +# NOT — so `setup`'s workspace handling (npm: every member; pnpm: root +# only) and the root install's cross-workspace apply are both exercised. +ws_member_js() { # $1=dir $2=name (declares the dep) + mkdir -p "$1" + cat > "$1/package.json" < package.json <<'EOF' +{ "name": "sm-root", "version": "0.0.0", "private": true, + "workspaces": ["packages/*", "packages/group/*"] } +EOF + ws_member_js packages/app "@sm/app" + ws_member_js packages/lib "@sm/lib" + ws_member_js packages/group/nested "@sm/nested" + mkdir -p packages/util # member with NO dependency on the patched pkg + printf '{ "name": "@sm/util", "version": "0.0.0", "private": true }\n' > packages/util/package.json ;; + pnpm) + printf '{ "name": "sm-root", "version": "0.0.0", "private": true }\n' > package.json + cat > pnpm-workspace.yaml <<'EOF' +packages: + - 'packages/*' + - 'packages/group/*' +EOF + ws_member_js packages/app "@sm/app" + ws_member_js packages/lib "@sm/lib" + ws_member_js packages/group/nested "@sm/nested" + mkdir -p packages/util + printf '{ "name": "@sm/util", "version": "0.0.0", "private": true }\n' > packages/util/package.json ;; + uv) + # uv workspace: virtual root + members; the shared dep is installed + # into one root .venv by `uv sync`. + cat > pyproject.toml < "packages/$m/pyproject.toml" < packages/app/requirements.txt + echo "$SM_PACKAGE==$SM_VERSION" > packages/lib/requirements.txt + printf -- '-r packages/app/requirements.txt\n-r packages/lib/requirements.txt\n' > requirements.txt ;; + esac +} + +run_install_workspace() { + case "$SM_PM" in + npm) npm install --silent --no-audit --no-fund ;; + yarn) yarn install --silent ;; + pnpm) pnpm install --no-frozen-lockfile ;; + uv) uv sync ;; + pip) python3 -m venv venv && ./venv/bin/pip install --disable-pip-version-check --quiet --no-cache-dir -r requirements.txt ;; + esac +} + +# --- all-ecosystem monorepo scaffold --------------------------------- +# A polyglot repo: an npm workspace (the slice `setup` supports AND the +# npm image can install) alongside python/rust/go/php/ruby/nuget/deno +# manifests. The point is to confirm `setup` works in this environment — +# it must configure the npm hooks and NOT choke on the foreign manifests. +scaffold_monorepo() { + cat > package.json <<'EOF' +{ "name": "sm-monorepo", "version": "0.0.0", "private": true, + "workspaces": ["packages/js-*"] } +EOF + ws_member_js packages/js-app "@mono/js-app" + ws_member_js packages/js-nested "@mono/js-nested" + mkdir -p packages/py-svc + cat > packages/py-svc/pyproject.toml <<'EOF' +[project] +name = "py-svc" +version = "0.0.0" +requires-python = ">=3.9" +dependencies = ["six==1.16.0"] +EOF + printf 'six==1.16.0\n' > packages/py-svc/requirements.txt + mkdir -p packages/rust-lib/src + printf '[package]\nname = "rust-lib"\nversion = "0.0.0"\nedition = "2021"\n\n[dependencies]\ncfg-if = "=1.0.0"\n' > packages/rust-lib/Cargo.toml + printf '// lib\n' > packages/rust-lib/src/lib.rs + mkdir -p packages/go-mod && printf 'module mono/go\n\ngo 1.21\n' > packages/go-mod/go.mod + mkdir -p packages/php-web && printf '{ "name": "mono/php", "require": { "monolog/monolog": "3.5.0" } }\n' > packages/php-web/composer.json + mkdir -p packages/ruby-gem && printf "source 'https://rubygems.org'\ngem 'colorize', '1.1.0'\n" > packages/ruby-gem/Gemfile + mkdir -p packages/deno-app && printf '{ "name": "mono/deno", "version": "0.0.0" }\n' > packages/deno-app/deno.json + mkdir -p packages/nuget-app && printf '\n' > packages/nuget-app/app.csproj +} + +run_install_monorepo() { + npm install --silent --no-audit --no-fund +} + +# --- resolve candidate on-disk file(s) for verification -------------- +# For single layout: one path. For workspace/monorepo: search the tree +# (hoisted root node_modules, pnpm store, member dirs, shared venv). +resolve_targets() { + local rel="${SM_MANIFEST_KEY#package/}" + local base; base="$(basename "$rel")" + if [ "$SM_LAYOUT" = single ]; then + resolve_target + return + fi + case "$SM_ECOSYSTEM" in + npm|deno|monorepo) find "$PWD" -path "*/node_modules/$SM_PACKAGE/$rel" 2>/dev/null ;; + pypi) find "$PWD" -name "$base" 2>/dev/null ;; + *) resolve_target ;; + esac +} + +# ============================ main ==================================== +log "binary: $SP_BIN ($("$SP_BIN" --version 2>/dev/null || echo '??')) layout=$SM_LAYOUT" + +WORKDIR="${SM_WORKDIR:-$(mktemp -d)}" +PROJ="$WORKDIR/proj" +mkdir -p "$PROJ" +cd "$PROJ" || { emit_result false null null null "" fail; exit 0; } +note "proj=$PROJ" + +# 0. dependencies + committed patch set +case "$SM_LAYOUT" in + workspace) scaffold_workspace ;; + monorepo) scaffold_monorepo ;; + *) scaffold_project ;; +esac +build_fixture + +# npm-family (incl. deno-via-npm and the monorepo's npm slice) need the +# runner shim so the hook's `npx`/`pnpm dlx @socketsecurity/socket-patch` +# resolves to $SP_BIN instead of the npm registry. +if [[ "$SM_PM" =~ ^(npm|yarn|pnpm|bun|deno)$ ]] || [ "$SM_LAYOUT" = monorepo ]; then + SHIM_DIR="$PROJ/.sp-shims" + write_shims "$SHIM_DIR" + export SETUP_MATRIX_SHIM_DIR="$SHIM_DIR" + export PATH="$SHIM_DIR:$PATH" + log "shims installed at $SHIM_DIR (PATH prepended)" +fi + +# Hermetic apply env inherited by the install hook's `socket-patch apply`. +# NOTE: SOCKET_OFFLINE/SOCKET_FORCE must be "true"/"false" — the apply +# `--force` flag (unlike its siblings) has no boolish value parser, so +# SOCKET_FORCE=1 is rejected with "invalid value '1' for '--force'". +# The SOCKET_EXPERIMENTAL_* gates are read directly from the env and use +# "1". +export SOCKET_OFFLINE=true SOCKET_FORCE=true SOCKET_API_TOKEN=fake SOCKET_ORG_SLUG=test-org +export SOCKET_TELEMETRY_DISABLED=1 SOCKET_EXPERIMENTAL_MAVEN=1 SOCKET_EXPERIMENTAL_NUGET=1 +# NOTE: deliberately do NOT export SOCKET_CWD. The install hook's apply +# must run with whatever cwd the package manager sets for the lifecycle +# script — the project root for a single project, and the *member* dir +# for each workspace member. In a workspace, member postinstalls thus +# find no manifest in their own dir and no-op (exit 0), while the root +# postinstall (manifest present) applies. Forcing SOCKET_CWD=root would +# make every member apply target the root manifest and fail with "no +# packages found on disk" mid-install, breaking `npm install`. + +# 1. setup (configures hooks; no-op where there is no package.json) +SETUP_EXIT="null" +if [ "$SM_RUN_SETUP" = 1 ]; then + log "running: socket-patch setup --yes" + "$SP_BIN" setup --yes --json; SETUP_EXIT=$? + log "setup exit=$SETUP_EXIT" + [ -f package.json ] && { log "package.json scripts after setup:"; grep -A6 '"scripts"' package.json || true; } +fi + +# 2. native install (this is where a configured hook fires) +log "running install for pm=$SM_PM (layout=$SM_LAYOUT)" +case "$SM_LAYOUT" in + workspace) run_install_workspace ;; + monorepo) run_install_monorepo ;; + *) run_install ;; +esac +INSTALL_EXIT=$? +log "install exit=$INSTALL_EXIT" + +# 3. verify — applied if ANY discovered copy of the patched file carries +# the expected marker (covers hoisting, the pnpm store, member dirs and +# the shared venv in workspace/monorepo layouts). +check_marker="$SM_MARKER" +[ "$SM_PATCHSET" = alt ] && check_marker="$SM_ALT_MARKER" +APPLIED=false +PRIMARY_PRESENT=null +TARGET="" +n_found=0 +while IFS= read -r cand; do + [ -n "$cand" ] && [ -f "$cand" ] || continue + n_found=$((n_found + 1)) + [ -z "$TARGET" ] && TARGET="$cand" + if grep -q "$check_marker" "$cand" 2>/dev/null; then APPLIED=true; TARGET="$cand"; fi + if grep -q "$SM_MARKER" "$cand" 2>/dev/null; then PRIMARY_PRESENT=true; fi +done < <(resolve_targets) +[ "$PRIMARY_PRESENT" = null ] && [ "$n_found" -gt 0 ] && PRIMARY_PRESENT=false +note "candidate files found: $n_found" +log "resolved target: ${TARGET:-} (candidates=$n_found)" +[ "$n_found" -eq 0 ] && note "target file not found" +log "marker '$check_marker' present: $APPLIED" + +# Driver-level status: did actual match the aspirational expectation? +want=$([ "$SM_EXPECT_APPLIED" = 1 ] && echo true || echo false) +STATUS=fail +[ "$APPLIED" = "$want" ] && STATUS=pass + +emit_result "$APPLIED" "$PRIMARY_PRESENT" "$SETUP_EXIT" "$INSTALL_EXIT" "${TARGET:-}" "$STATUS" +exit 0 diff --git a/tests/setup_matrix/shims/npx b/tests/setup_matrix/shims/npx new file mode 100755 index 00000000..fb158c29 --- /dev/null +++ b/tests/setup_matrix/shims/npx @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# setup-matrix test shim for `npx`. +# +# The hook that `socket-patch setup` writes into package.json is +# npx @socketsecurity/socket-patch apply --silent --ecosystems npm +# In a hermetic test we do NOT want `npx` to fetch the published wrapper +# package from the npm registry — we want it to run the locally-built +# binary under test. This shim, prepended to PATH, intercepts exactly that +# invocation and execs the local binary; every other `npx` call is +# delegated to the real `npx`. +# +# This is a TEST FIXTURE, not a change to socket-patch behavior. It is the +# standalone/reference copy; tests/setup_matrix/run-case.sh embeds an +# identical copy inline so the driver is self-contained when piped into a +# container. Keep the two in sync. +set -uo pipefail + +sp_bin="${SOCKET_PATCH_BIN:-socket-patch}" +shim_dir="${SETUP_MATRIX_SHIM_DIR:-}" + +# Resolve the real npx by searching PATH with our own shim dir removed. +clean_path="$PATH" +if [ -n "$shim_dir" ]; then + clean_path="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$shim_dir" | paste -sd: -)" +fi +real_npx="$(PATH="$clean_path" command -v npx 2>/dev/null || true)" + +# If any argument names our package, drop everything up to and including it +# and exec the local binary with the remaining apply args. +i=0 +for arg in "$@"; do + case "$arg" in + @socketsecurity/socket-patch|@socketsecurity/socket-patch@*|@socketsecurity/socket-patch/*) + shift "$((i + 1))" + exec "$sp_bin" "$@" + ;; + esac + i=$((i + 1)) +done + +if [ -n "$real_npx" ]; then + exec "$real_npx" "$@" +fi +echo "setup-matrix npx shim: real npx not found and args are not our package: $*" >&2 +exit 127 diff --git a/tests/setup_matrix/shims/pnpm b/tests/setup_matrix/shims/pnpm new file mode 100755 index 00000000..7f342f71 --- /dev/null +++ b/tests/setup_matrix/shims/pnpm @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# setup-matrix test shim for `pnpm`. +# +# For pnpm projects `socket-patch setup` writes the hook +# pnpm dlx @socketsecurity/socket-patch apply --silent --ecosystems npm +# `pnpm dlx` always downloads from the registry, so it cannot be satisfied +# from a local file: dependency. This shim, prepended to PATH, intercepts +# `pnpm dlx @socketsecurity/socket-patch …` (and `pnpm exec …`) and execs +# the locally-built binary; every other `pnpm` invocation — crucially the +# real `pnpm install` / `pnpm add` — is delegated unchanged to the real +# pnpm. +# +# TEST FIXTURE only. Reference copy; run-case.sh embeds an identical copy. +set -uo pipefail + +sp_bin="${SOCKET_PATCH_BIN:-socket-patch}" +shim_dir="${SETUP_MATRIX_SHIM_DIR:-}" + +clean_path="$PATH" +if [ -n "$shim_dir" ]; then + clean_path="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$shim_dir" | paste -sd: -)" +fi +real_pnpm="$(PATH="$clean_path" command -v pnpm 2>/dev/null || true)" + +if [ "${1:-}" = "dlx" ] || [ "${1:-}" = "exec" ]; then + case "${2:-}" in + @socketsecurity/socket-patch|@socketsecurity/socket-patch@*) + shift 2 + exec "$sp_bin" "$@" + ;; + esac +fi + +if [ -n "$real_pnpm" ]; then + exec "$real_pnpm" "$@" +fi +echo "setup-matrix pnpm shim: real pnpm not found: $*" >&2 +exit 127