From f3d5685ffd6859c8ece2e92a8f7bfe68a0926b20 Mon Sep 17 00:00:00 2001 From: Pengyu Zhang Date: Mon, 24 Aug 2026 10:26:56 -0700 Subject: [PATCH] feat(discovery): add candidate enumeration --- .../adr_discovery/enumerator/__init__.py | 52 ++++++ Discovery/adr_discovery/enumerator/markers.py | 155 ++++++++++++++++++ Discovery/adr_discovery/enumerator/roots.py | 96 +++++++++++ .../enumerator/sources/__init__.py | 0 .../enumerator/sources/appstate.py | 127 ++++++++++++++ .../enumerator/sources/binaries.py | 73 +++++++++ .../enumerator/sources/execjournal.py | 40 +++++ .../enumerator/sources/modelstores.py | 34 ++++ .../enumerator/sources/network.py | 71 ++++++++ .../enumerator/sources/registries.py | 99 +++++++++++ Discovery/adr_discovery/enumerator/sweep.py | 54 ++++++ 11 files changed, 801 insertions(+) create mode 100644 Discovery/adr_discovery/enumerator/__init__.py create mode 100644 Discovery/adr_discovery/enumerator/markers.py create mode 100644 Discovery/adr_discovery/enumerator/roots.py create mode 100644 Discovery/adr_discovery/enumerator/sources/__init__.py create mode 100644 Discovery/adr_discovery/enumerator/sources/appstate.py create mode 100644 Discovery/adr_discovery/enumerator/sources/binaries.py create mode 100644 Discovery/adr_discovery/enumerator/sources/execjournal.py create mode 100644 Discovery/adr_discovery/enumerator/sources/modelstores.py create mode 100644 Discovery/adr_discovery/enumerator/sources/network.py create mode 100644 Discovery/adr_discovery/enumerator/sources/registries.py create mode 100644 Discovery/adr_discovery/enumerator/sweep.py diff --git a/Discovery/adr_discovery/enumerator/__init__.py b/Discovery/adr_discovery/enumerator/__init__.py new file mode 100644 index 0000000..d9b66ed --- /dev/null +++ b/Discovery/adr_discovery/enumerator/__init__.py @@ -0,0 +1,52 @@ +"""M2 -- where should we look. + +Answers that question once, for the whole pipeline. Nothing downstream may +decide it for itself, and nothing here knows what any particular tool is: +`enumerate_candidates` is run with an empty catalog in its own test set, +and must produce the same candidates it produces with a full one. +""" + +from __future__ import annotations + +from ..contracts.records import Candidate +from .roots import homes +from .sources.appstate import from_app_state +from .sources.binaries import from_binaries +from .sources.execjournal import from_exec_journal +from .sources.modelstores import from_model_stores +from .sources.network import from_network +from .sources.registries import from_applications, from_kernel, from_packages +from .sweep import sweep + +__all__ = ["enumerate_candidates"] + + +def enumerate_candidates(gate, include_dependency_caches: bool = False) -> tuple[Candidate, ...]: + """Registries first, then the sweep. + + The order is the optimisation: most of the search is over before the + walk starts, and every registry hit arrives with provenance attached. + """ + found: list[Candidate] = [] + + # Half one -- ask what already has the answer. + found.extend(from_packages(gate)) + found.extend(from_applications(gate)) + kernel = from_kernel(gate) + found.extend(kernel) + found.extend(from_network(gate, kernel)) + found.extend(from_exec_journal(gate)) + found.extend(from_app_state(gate, homes(gate))) + found.extend(from_binaries(gate, homes(gate))) + found.extend(from_model_stores(gate, homes(gate))) + + registry_entries = gate.budget.entries_used + + # Half two -- sweep only what no registry indexes. + found.extend(sweep(gate, include_dependency_caches)) + + gate.ledger.probe( + "enumerator", "ran", + f"{len(found)} candidates; {registry_entries} entries before the sweep", + ) + return tuple(found) diff --git a/Discovery/adr_discovery/enumerator/markers.py b/Discovery/adr_discovery/enumerator/markers.py new file mode 100644 index 0000000..a75ca70 --- /dev/null +++ b/Discovery/adr_discovery/enumerator/markers.py @@ -0,0 +1,155 @@ +"""The marker set, as data. + +Traversal is keyed on markers rather than on remembered paths, which is +what lets a repository in /opt/checkouts be found by the same rule that +finds one in ~/Projects. + +Nothing here names a tool. A marker says *where an agent works*; deciding +what the agent is belongs to M4, and M2 must stay passable with an empty +catalog (U2-03). +""" + +from __future__ import annotations + +#: Directory names that mark a surface worth reading. +DIR_MARKERS: frozenset[str] = frozenset( + { + ".git", ".claude", ".cursor", ".windsurf", ".aider", ".continue", + ".codeium", ".gemini", ".goose", ".opencode", ".zed", + "agents", "skills", "commands", "prompts", "output-styles", "plugins", + ".github", ".devcontainer", ".vscode", + } +) + +#: File names that mark a surface worth reading. +FILE_MARKERS: frozenset[str] = frozenset( + { + ".mcp.json", ".claude.json", "mcp.json", "settings.json", "settings.local.json", + "config.toml", "config.yaml", "mcp_settings.json", "mcp_config.json", + "cline_mcp_settings.json", "managed-settings.json", "managed-mcp.json", + "claude_desktop_config.json", "opencode.json", + } +) + +#: Workflow files are read by suffix rather than by name -- nobody agrees +#: on what a workflow is called, only on where it lives. +WORKFLOW_DIR = "/.github/workflows/" +WORKFLOW_SUFFIXES = (".yml", ".yaml") + +#: Instruction filenames are programmable-surface records. Their contents are +#: never collected; only path, scope and host-facing name leave the endpoint. +INSTRUCTION_MARKERS: frozenset[str] = frozenset( + { + "CLAUDE.md", "AGENTS.md", "GEMINI.md", "AGENT.md", + ".cursorrules", ".windsurfrules", "copilot-instructions.md", + } +) + +LOCATOR_ONLY: frozenset[str] = frozenset({".cursorrules", ".windsurfrules"}) + +#: State directories a host application keeps per user. +STATE_ROOTS: tuple[str, ...] = ( + "~/.claude", "~/.codex", "~/.cursor", "~/.aider", "~/.continue", + "~/.gemini", "~/.config/goose", "~/.config/opencode", "~/.ollama", + "~/Library/Application Support/Claude", + "~/Library/Application Support/Code/User", + "~/.config/Code/User", + "~/.vscode/extensions", "~/.vscode-server/extensions", +) + +#: Config files loaded directly by known agent hosts. These are enumerated +#: independently of the breadth sweep so a dependency cache cannot hide them. +CONFIG_FILE_TEMPLATES: tuple[str, ...] = ( + "~/.claude.json", + "~/.config/claude-desktop/claude_desktop_config.json", + "~/.cursor/mcp.json", + "~/.codeium/windsurf/mcp_config.json", + "~/.config/Code/User/mcp.json", + "~/.config/Code/User/globalStorage/saoudrizwan.claude-dev/settings/cline_mcp_settings.json", + "~/.config/zed/settings.json", + "~/.config/JetBrains/options/mcp.json", + "~/.config/opencode/opencode.json", + "~/.codex/config.toml", + "~/.config/goose/config.yaml", + "~/.bashrc", + "~/.zshrc", + "/etc/claude-code/managed-settings.json", + "/etc/adr/managed-mcp.json", +) + +#: Browser profile parents. Every profile, not just the default -- a large +#: share of real shadow AI lives on a second profile. +BROWSER_PROFILE_ROOTS: tuple[str, ...] = ( + "~/Library/Application Support/Google/Chrome", + "~/Library/Application Support/BraveSoftware/Brave-Browser", + "~/Library/Application Support/Microsoft Edge", + "~/Library/Application Support/Arc/User Data", + "~/.config/google-chrome", + "~/.config/chromium", + "~/.config/microsoft-edge", +) + +FIREFOX_PROFILE_ROOTS: tuple[str, ...] = ( + "~/Library/Application Support/Firefox/Profiles", + "~/.mozilla/firefox", +) + +EDITOR_EXTENSION_ROOTS: tuple[str, ...] = ( + "~/.vscode/extensions", "~/.vscode-server/extensions", + "~/.cursor/extensions", "~/.windsurf/extensions", + "~/.trae/extensions", "~/.kilo/extensions", +) + +#: Hosts that answer for a model provider. Landscape data, not identity: +#: a connection here says *something on this machine talks to a model*, +#: which is a candidate. What it is remains M4's question. +MODEL_PROVIDER_SUFFIXES: tuple[str, ...] = ( + "api.anthropic.com", "api.openai.com", "openai.azure.com", + "generativelanguage.googleapis.com", "aiplatform.googleapis.com", + "bedrock-runtime.amazonaws.com", "api.mistral.ai", "api.cohere.ai", + "api.groq.com", "api.together.xyz", "api.deepseek.com", + "api.x.ai", "openrouter.ai", "huggingface.co", +) + +#: Bundles and portable executables carry their own runtime, so nothing +#: else on disk reveals them. +BUNDLE_SUFFIXES: tuple[str, ...] = (".AppImage", ".app", ".exe") + +#: Ports a local model runtime answers on. +LOCAL_MODEL_PORTS: frozenset[int] = frozenset({11434, 1234, 8080, 8000, 5000, 7860}) + + +def is_model_provider(host: str) -> bool: + h = host.lower().rstrip(".") + return any(h == s or h.endswith("." + s) for s in MODEL_PROVIDER_SUFFIXES) + + +def is_loose_executable(entry, name: str) -> bool: + """An executable nothing else on disk accounts for. + + Extensionless is the test that keeps this from matching every script in + every repository: real CLI tools ship as `claude`, not `claude.sh`. + """ + if entry.is_dir: + return name.endswith(BUNDLE_SUFFIXES) + if name.endswith(BUNDLE_SUFFIXES): + return True + return entry.is_exec and "." not in name + + +def marker_kind(name: str, path: str = "") -> str | None: + if WORKFLOW_DIR in path and path.endswith(WORKFLOW_SUFFIXES): + return "marker_file" + if name in DIR_MARKERS: + return "marker_dir" + if name in FILE_MARKERS: + return "marker_file" + if name in INSTRUCTION_MARKERS: + return "instruction_file" + if name in (".bashrc", ".zshrc"): + return "shell_profile" + if name == "manifest.json" and "/.mcpb/" in path: + return "marker_file" + if name in LOCATOR_ONLY: + return "locator" + return None diff --git a/Discovery/adr_discovery/enumerator/roots.py b/Discovery/adr_discovery/enumerator/roots.py new file mode 100644 index 0000000..684275c --- /dev/null +++ b/Discovery/adr_discovery/enumerator/roots.py @@ -0,0 +1,96 @@ +"""Priority roots -- one definition. + +There were five copies of this tuple, in five probe files, none of which +reported that it had a boundary. Roots now *order* the sweep so the common +case stays fast; they no longer decide what exists. +""" + +from __future__ import annotations + +from ..contracts.records import Priority + +#: (template, priority). `~` is expanded per discovered home, not per the +#: user running the scan -- the owner of an asset is a person, never whoever +#: happened to launch the collector. +ROOT_TEMPLATES: tuple[tuple[str, Priority], ...] = ( + ("~", Priority.HOME), + ("~/Projects", Priority.CODE_ROOT), + ("~/src", Priority.CODE_ROOT), + ("~/code", Priority.CODE_ROOT), + ("~/work", Priority.CODE_ROOT), + ("~/dev", Priority.CODE_ROOT), + ("~/git", Priority.CODE_ROOT), + ("~/repos", Priority.CODE_ROOT), + ("/opt", Priority.SYSTEM), + ("/srv", Priority.SYSTEM), + ("/usr/local", Priority.SYSTEM), + ("/workspace", Priority.SYSTEM), + ("/Users", Priority.BREADTH), + ("/home", Priority.BREADTH), +) + +#: Scope is policy, not a constant. Whether a dependency cache is in scope is +#: a real question with a defensible answer either way, so it lives here with +#: a stated default rather than in a tuple nobody can see. +DEPENDENCY_CACHES: tuple[str, ...] = ( + "node_modules", ".venv", "venv", "site-packages", "go/pkg/mod", + ".cargo/registry", "vendor", ".gradle", ".m2", +) + +SKIP_ALWAYS: tuple[str, ...] = ( + ".git/objects", ".Trash", "Library/Caches", ".cache", "__pycache__", + "/.npm/", "/.local/share/pipx/", "/.cargo/registry/", "/.gradle/", "/.m2/", +) + + +def homes(gate) -> tuple[str, ...]: + """Every home on the machine, not just the caller's. + + Where homes live is a platform question and is answered by M1's + provider, not by a tuple here -- which is the same rule that removed + the five copies of PROJECT_ROOTS. + """ + found: list[str] = [] + for base in gate.providers.home_roots(): + listing = gate.list_dir(base) + if not listing.ok: + continue + for entry in listing.value: + if entry.is_dir and not entry.path.rsplit("/", 1)[-1].startswith("."): + found.append(entry.path) + if not found: + home = gate.env.get("HOME") + if home: + found.append(home) + return tuple(found) + + +def ordered_roots(gate) -> tuple[tuple[str, Priority], ...]: + """Roots in sweep order: home first, then code roots, then breadth. + + Order is asserted by U2-06, because a budget exhausted late must still + have covered the likely places. + """ + out: list[tuple[str, Priority]] = [] + seen: set[str] = set() + for home in homes(gate): + for template, priority in ROOT_TEMPLATES: + if not template.startswith("~"): + continue + path = home + template[1:] + if path not in seen: + seen.add(path) + out.append((path, priority)) + for template, priority in ROOT_TEMPLATES: + if template.startswith("~") or template in seen: + continue + seen.add(template) + out.append((template, priority)) + out.sort(key=lambda pair: pair[1]) + return tuple(out) + + +def in_scope(path: str, include_dependency_caches: bool = False) -> bool: + if any(seg in path for seg in SKIP_ALWAYS): + return False + return include_dependency_caches or not any(seg in path for seg in DEPENDENCY_CACHES) diff --git a/Discovery/adr_discovery/enumerator/sources/__init__.py b/Discovery/adr_discovery/enumerator/sources/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Discovery/adr_discovery/enumerator/sources/appstate.py b/Discovery/adr_discovery/enumerator/sources/appstate.py new file mode 100644 index 0000000..da8d770 --- /dev/null +++ b/Discovery/adr_discovery/enumerator/sources/appstate.py @@ -0,0 +1,127 @@ +"""Application state -- per profile, rather than per default. + +Extensions and per-tool state live under directories a host application +maintains. The rule that matters here is *every* browser profile: a large +share of real shadow AI sits on a second profile, and a scan that reads +Default only reports a clean machine. +""" + +from __future__ import annotations + +import json + +from ...contracts.records import Candidate, Priority +from ..markers import ( + BROWSER_PROFILE_ROOTS, + CONFIG_FILE_TEMPLATES, + EDITOR_EXTENSION_ROOTS, + FIREFOX_PROFILE_ROOTS, + STATE_ROOTS, +) + + +def _expand(gate, template: str, homes: tuple[str, ...]) -> list[str]: + if not template.startswith("~"): + return [template] + return [home + template[1:] for home in homes] + + +def from_app_state(gate, homes: tuple[str, ...]) -> tuple[Candidate, ...]: + out: list[Candidate] = [] + + for template in CONFIG_FILE_TEMPLATES: + for path in _expand(gate, template, homes): + if gate.stat(path).ok: + kind = "shell_profile" if path.endswith(("/.bashrc", "/.zshrc")) else "marker_file" + out.append(Candidate(kind=kind, path=path, source="app_state:config", + priority=Priority.HOME, detail={"marker": path.rsplit("/", 1)[-1]})) + + for template in STATE_ROOTS: + for path in _expand(gate, template, homes): + listing = gate.list_dir(path) + if not listing.ok: + continue + out.append( + Candidate(kind="state_dir", path=path, source="app_state", + priority=Priority.HOME, detail={"entries": len(listing.value)}) + ) + + for template in BROWSER_PROFILE_ROOTS: + for browser_root in _expand(gate, template, homes): + profiles = gate.list_dir(browser_root) + if not profiles.ok: + continue + for profile in profiles.value: + if not profile.is_dir: + continue + name = profile.path.rsplit("/", 1)[-1] + if name != "Default" and not name.startswith("Profile "): + continue + ext_root = profile.path + "/Extensions" + extensions = gate.list_dir(ext_root) + if not extensions.ok: + continue + for ext in extensions.value: + if not ext.is_dir: + continue + out.append( + Candidate( + kind="extension", + path=ext.path, + source="app_state:browser", + priority=Priority.HOME, + detail={"extension_id": ext.path.rsplit("/", 1)[-1], + "profile": name, "browser": browser_root}, + ) + ) + + for template in EDITOR_EXTENSION_ROOTS: + for root in _expand(gate, template, homes): + extensions = gate.list_dir(root) + if not extensions.ok: + continue + for ext in extensions.value: + if not ext.is_dir: + continue + ident, version = _editor_identity(gate, ext.path) + out.append(Candidate( + kind="extension", path=ext.path, source="app_state:editor", + priority=Priority.HOME, + detail={"extension_id": ident, "version": version, "editor": root}, + )) + + for template in FIREFOX_PROFILE_ROOTS: + for root in _expand(gate, template, homes): + profiles = gate.list_dir(root) + if not profiles.ok: + continue + for profile in profiles.value: + if not profile.is_dir: + continue + extensions = gate.list_dir(profile.path + "/extensions") + if not extensions.ok: + continue + for ext in extensions.value: + if ext.is_dir or ext.path.endswith(".xpi"): + ident = ext.path.rsplit("/", 1)[-1].removesuffix(".xpi") + out.append(Candidate( + kind="extension", path=ext.path, source="app_state:firefox", + priority=Priority.HOME, + detail={"extension_id": ident, "profile": profile.path, + "browser": "firefox"}, + )) + return tuple(out) + + +def _editor_identity(gate, path: str) -> tuple[str, str | None]: + raw = gate.read_text(path + "/package.json", limit=1 << 20) + if raw.ok: + try: + manifest = json.loads(raw.value) + publisher, name = manifest.get("publisher"), manifest.get("name") + if publisher and name: + version = manifest.get("version") + return f"{publisher}.{name}", str(version) if version else None + except (ValueError, TypeError): + gate.ledger.probe("extension_manifest", "degraded", path) + return path.rsplit("/", 1)[-1], None diff --git a/Discovery/adr_discovery/enumerator/sources/binaries.py b/Discovery/adr_discovery/enumerator/sources/binaries.py new file mode 100644 index 0000000..fe73e36 --- /dev/null +++ b/Discovery/adr_discovery/enumerator/sources/binaries.py @@ -0,0 +1,73 @@ +"""Executables on disk. + +The registries answer for everything a package manager installed, and the +sweep answers for everything a marker locates. Neither answers for a bare +executable: a tarball unpacked into /opt, an AppImage in Downloads, a +binary copied out of a container. Those have no package record and sit +beside no marker, and without this source they are invisible. + +Bounded on purpose. Only bin-shaped directories are read, one level deep, +and the shared entry ceiling applies -- this must not become a second +filesystem sweep wearing a different name. +""" + +from __future__ import annotations + +from ...contracts.records import Candidate, Priority +from ..markers import BUNDLE_SUFFIXES + +#: Directories that hold executables by convention, plus whatever PATH says. +BIN_ROOTS: tuple[str, ...] = ( + "/usr/local/bin", "/usr/bin", "/bin", "/opt/homebrew/bin", "/opt/local/bin", + "/snap/bin", "~/.local/bin", "~/bin", "~/.cargo/bin", "~/go/bin", + "~/.npm-global/bin", "~/.bun/bin", "~/.deno/bin", +) + +MAX_PER_ROOT = 2_000 + + +def from_binaries(gate, homes: tuple[str, ...]) -> tuple[Candidate, ...]: + out: list[Candidate] = [] + seen: set[str] = set() + + for root in _roots(gate, homes): + listing = gate.list_dir(root) + if not listing.ok: + continue + kept = 0 + for entry in listing.value: + if kept >= MAX_PER_ROOT: + gate.ledger.truncate(root, kept, len(listing.value)) + break + if not (entry.is_exec or entry.path.endswith(BUNDLE_SUFFIXES)): + continue + if not gate.budget.take_entries(): + gate.ledger.boundary(root, "budget_exhausted", "binaries not fully read") + return tuple(out) + if entry.path in seen: + continue + seen.add(entry.path) + kept += 1 + out.append( + Candidate( + kind="binary", + path=entry.path, + source="binaries", + priority=Priority.HOME, + detail={"name": entry.path.rsplit("/", 1)[-1], "symlink": entry.is_symlink}, + ) + ) + return tuple(out) + + +def _roots(gate, homes: tuple[str, ...]) -> list[str]: + roots: list[str] = [] + for template in BIN_ROOTS: + if template.startswith("~"): + roots.extend(home + template[1:] for home in homes) + else: + roots.append(template) + for entry in (gate.env.get("PATH") or "").split(":"): + if entry and entry not in roots: + roots.append(entry) + return roots diff --git a/Discovery/adr_discovery/enumerator/sources/execjournal.py b/Discovery/adr_discovery/enumerator/sources/execjournal.py new file mode 100644 index 0000000..252510f --- /dev/null +++ b/Discovery/adr_discovery/enumerator/sources/execjournal.py @@ -0,0 +1,40 @@ +"""Exec events -- what ran between scans. + +A snapshot finds an agent that happens to be running when the scan fires. +An agent that runs forty seconds a night is absent from every daily scan +and present on the machine the whole time. + +This source is conditional on a privileged collector. Its absence is a +coverage fact and must never read as "nothing ran" -- which is what the +`unavailable` record written by the provider guarantees (U2-10). +""" + +from __future__ import annotations + +from ...contracts.records import Candidate, Priority +from ...redact.rules import scrub_argv + + +def from_exec_journal(gate) -> tuple[Candidate, ...]: + result = gate.exec_journal() + if not result.ok: + gate.ledger.probe("exec_journal", "degraded", result.reason) + return () + gate.ledger.probe("exec_journal", "ran", f"{len(result.value)} events") + out = [] + for ev in result.value: + argv = scrub_argv(ev.argv) + out.append(Candidate( + kind="exec_event", + path=ev.exe, + source="exec_journal", + priority=Priority.HOME, + detail={ + "argv": argv, "ppid": ev.ppid, + "parent_exe": ev.parent_exe, "started": ev.started, + "unattended": bool(set(argv) & { + "--dangerously-skip-permissions", "--yolo", "--auto-approve", "--no-confirm", + }), + }, + )) + return tuple(out) diff --git a/Discovery/adr_discovery/enumerator/sources/modelstores.py b/Discovery/adr_discovery/enumerator/sources/modelstores.py new file mode 100644 index 0000000..1ea2484 --- /dev/null +++ b/Discovery/adr_discovery/enumerator/sources/modelstores.py @@ -0,0 +1,34 @@ +"""Local model stores that package and application registries do not index.""" + +from __future__ import annotations + +from ...contracts.records import Candidate, Priority + +MODEL_ROOTS = ( + "~/.ollama/models", "~/.cache/huggingface/hub", "~/.cache/lm-studio/models", + "~/Library/Application Support/LM Studio/models", "~/.local/share/Jan/models", + "~/.cache/gpt4all", +) +MODEL_SUFFIXES = (".gguf", ".safetensors") +MAX_MODELS = 500 + + +def from_model_stores(gate, homes: tuple[str, ...]) -> tuple[Candidate, ...]: + out: list[Candidate] = [] + for template in MODEL_ROOTS: + for home in homes: + root = home + template[1:] + if not gate.list_dir(root).ok: + continue + for entry in gate.walk(root, max_depth=4): + if entry.is_dir: + continue + name = entry.path.rsplit("/", 1)[-1] + if not (name.endswith(MODEL_SUFFIXES) or name.startswith("sha256-")): + continue + out.append(Candidate("model_weight_candidate", entry.path, "model_store", + Priority.HOME, {"name": name, "size": entry.size})) + if len(out) >= MAX_MODELS: + gate.ledger.truncate(root, len(out), len(out) + 1) + return tuple(out) + return tuple(out) diff --git a/Discovery/adr_discovery/enumerator/sources/network.py b/Discovery/adr_discovery/enumerator/sources/network.py new file mode 100644 index 0000000..d2d7910 --- /dev/null +++ b/Discovery/adr_discovery/enumerator/sources/network.py @@ -0,0 +1,71 @@ +"""Network -- what the machine talks to. + +Listening sockets find a *server*. Almost every AI tool is a *client*, and +the connection it opens is the one piece of evidence it cannot suppress and +still function -- which makes this the only source that yields anything at +all for a tool the catalog has never heard of. + +The resolver cache matters more than the connection table, because it +covers a window rather than an instant and so survives a tool that ran an +hour before the scan. +""" + +from __future__ import annotations + +from ...contracts.records import Candidate, Priority +from ..markers import LOCAL_MODEL_PORTS, is_model_provider + + +def from_network(gate, kernel_candidates: tuple[Candidate, ...] = ()) -> tuple[Candidate, ...]: + out: list[Candidate] = [] + processes = { + c.detail.get("pid"): c + for c in kernel_candidates + if c.kind == "process" and c.detail.get("pid") is not None + } + + socks = gate.sockets() + if socks.ok: + for s in socks.value: + if s.state == "ESTABLISHED" and is_model_provider(s.remote_host): + process = processes.get(s.pid) + out.append( + Candidate( + kind="network_peer", + path=process.path if process is not None else s.remote_host, + source="network:established", + priority=Priority.HOME, + detail={ + "pid": s.pid, "port": s.remote_port, "provider": True, + "remote_host": s.remote_host, + "env_names": process.detail.get("env_names", ()) if process else (), + "unattended": process.detail.get("unattended", False) if process else False, + }, + ) + ) + elif s.state == "LISTEN" and s.local_port in LOCAL_MODEL_PORTS: + out.append( + Candidate( + kind="model_port", + path=f"tcp:{s.local_port}", + source="network:listening", + priority=Priority.HOME, + detail={"port": s.local_port, "pid": s.pid}, + ) + ) + + cache = gate.dns_cache() + if cache.ok: + gate.ledger.probe("dns_cache", "ran", f"{len(cache.value)} entries") + for entry in cache.value: + if is_model_provider(entry.hostname): + out.append( + Candidate( + kind="dns_peer", + path=entry.hostname, + source="network:resolver_cache", + priority=Priority.HOME, + detail={"provider": True}, + ) + ) + return tuple(out) diff --git a/Discovery/adr_discovery/enumerator/sources/registries.py b/Discovery/adr_discovery/enumerator/sources/registries.py new file mode 100644 index 0000000..0dcfd9c --- /dev/null +++ b/Discovery/adr_discovery/enumerator/sources/registries.py @@ -0,0 +1,99 @@ +"""Ask the system first. + +Package databases, application registries and the kernel have already +catalogued most of what is installed, with provenance attached. Querying +them is cheaper and more complete than searching for it, and every hit +arrives with the provenance M4 needs anyway. + +Each function returns candidates and leaves a coverage record when its +surface could not be read -- an unavailable registry is never an empty one. +""" + +from __future__ import annotations + +from ...contracts.records import Candidate, Priority +from ...redact.rules import scrub_argv + + +def from_packages(gate) -> tuple[Candidate, ...]: + result = gate.packages() + if not result.ok: + return () + gate.ledger.probe("packages", "ran", f"{len(result.value)} records") + return tuple( + Candidate( + kind="package", + path=pkg.path or pkg.name, + source=f"package:{pkg.manager}", + priority=Priority.HOME, + detail={"manager": pkg.manager, "name": pkg.name, "version": pkg.version}, + ) + for pkg in result.value + ) + + +def from_applications(gate) -> tuple[Candidate, ...]: + result = gate.applications() + if not result.ok: + return () + gate.ledger.probe("applications", "ran", f"{len(result.value)} records") + return tuple( + Candidate( + kind="application", + path=app.path or app.ident, + source="app_registry", + priority=Priority.HOME, + detail={"ident": app.ident, "name": app.name, "version": app.version, "vendor": app.vendor}, + ) + for app in result.value + ) + + +def from_kernel(gate) -> tuple[Candidate, ...]: + """What is running, from which binary, and what it is serving. + + The exe path is carried through verbatim. Resolving a process *name* + against PATH is the defect this source exists to avoid. + """ + out: list[Candidate] = [] + procs = gate.processes() + if procs.ok: + gate.ledger.probe("processes", "ran", f"{len(procs.value)} pids") + for p in procs.value: + argv = scrub_argv(p.argv) + out.append( + Candidate( + kind="process", + path=p.exe, + source="kernel", + priority=Priority.HOME, + detail={ + "pid": p.pid, "ppid": p.ppid, "argv": argv, "cwd": p.cwd, + "user": p.user, "env_names": p.env_names, + "unattended": _is_unattended(argv), + }, + ) + ) + socks = gate.sockets() + if socks.ok: + gate.ledger.probe("sockets", "ran", f"{len(socks.value)} sockets") + for s in socks.value: + if s.state != "LISTEN": + continue + out.append( + Candidate( + kind="listening_socket", + path=f"tcp:{s.local_port}", + source="kernel", + priority=Priority.HOME, + detail={"port": s.local_port, "pid": s.pid}, + ) + ) + return tuple(out) + + +def _is_unattended(argv: tuple[str, ...]) -> bool: + flags = set(argv) + return bool(flags & { + "--dangerously-skip-permissions", "--yolo", "--auto-approve", "--no-confirm", + }) diff --git a/Discovery/adr_discovery/enumerator/sweep.py b/Discovery/adr_discovery/enumerator/sweep.py new file mode 100644 index 0000000..06d8423 --- /dev/null +++ b/Discovery/adr_discovery/enumerator/sweep.py @@ -0,0 +1,54 @@ +"""Sweep only what no registry indexes. + +Repositories, agent directories and skill folders are found by traversal +keyed on markers, not on remembered paths. This is the only part of M2 that +can ruin the budget, so it carries the budget. +""" + +from __future__ import annotations + +from ..contracts.records import Candidate +from .markers import is_loose_executable, marker_kind +from .roots import in_scope, ordered_roots + + +def sweep(gate, include_dependency_caches: bool = False) -> tuple[Candidate, ...]: + """Breadth-ordered over priority roots, under one shared ceiling. + + Marker matching is a name test on entries already being listed, not a + second pass -- the walk is the cost, and this adds nothing to it. + """ + out: list[Candidate] = [] + seen: set[str] = set() + + for root, priority in ordered_roots(gate): + if gate.budget.entries_exhausted: + gate.ledger.boundary(root, "budget_exhausted", "root not swept") + continue + probe = gate.list_dir(root) + if not probe.ok: + continue + for entry in gate.walk(root): + if not in_scope(entry.path, include_dependency_caches): + continue + name = entry.path.rsplit("/", 1)[-1] + kind = marker_kind(name, entry.path) + if kind is None and is_loose_executable(entry, name): + # A tarball unpacked into /opt, a binary copied out of a + # container: no package record, no marker beside it. The + # walk is already listing this entry, so noticing costs + # nothing beyond the name test that follows it. + kind = "binary" + if kind is None or entry.path in seen: + continue + seen.add(entry.path) + out.append( + Candidate( + kind=kind, + path=entry.path, + source="sweep", + priority=priority, + detail={"marker": name, "is_dir": entry.is_dir, "name": name}, + ) + ) + return tuple(out)