diff --git a/.env b/.env index eb941a35e..f3455c589 100644 --- a/.env +++ b/.env @@ -12,7 +12,7 @@ PROJECT_NAME="airstack" # If you've run ./airstack.sh setup, then this will auto-generate from the git commit hash every time a change is made # to a Dockerfile or docker-compose.yaml file. Otherwise this can also be set explicitly to make a release version. # auto-generated from git commit hash -VERSION="0.20.0-alpha.9" +VERSION="0.20.0-alpha.10" # Image-tag discriminator ONLY (appears in the image tag suffix, e.g. ..._robot-x86-64_dev). # No Dockerfile consumes it: "prebuilt" does NOT bake the built ros_ws into the image today — # a real prebuilt (workspace-baked) stage is future work. Keep "dev" (mounted code, built live). diff --git a/.github/orchestrator/README.md b/.github/orchestrator/README.md index 4c98a535b..1626e74cf 100644 --- a/.github/orchestrator/README.md +++ b/.github/orchestrator/README.md @@ -119,7 +119,7 @@ sudo bash /tmp/airstack/.github/orchestrator/setup.sh | `cpu` / `gpu` / `memory` / `storage` | Resource request for the worker | size for full stack + sim | | `privileged` | Must be `true` (docker compose inside the pod) | — | | `priority` | `HIGH` \| `NORMAL` \| `LOW` | — | -| `repo` | `owner/name` of the repo to poll | from GitHub URL | +| `repos` | list of `owner/name` repos to poll (legacy `repo:` accepted) | from GitHub URLs | | `runner_version` | Runner version baked into `runner_image` | matches step 1 | | `max_concurrent` | Max simultaneous in-flight workflows | — | | `max_job_minutes` | Straggler cancel ceiling | exceed the longest job | @@ -131,7 +131,7 @@ sudo systemctl enable --now airstack-orchestrator.service journalctl -u airstack-orchestrator.service -f ``` -You should see `orchestrator started (OSMO backend): repo=... pool=... max_concurrent=N`, an `osmo login succeeded` line, and then periodic poll activity. +You should see `orchestrator started (OSMO backend): repos=[...] pool=... max_concurrent=N`, an `osmo login succeeded` line, and then periodic poll activity. ## End-to-end verification @@ -155,20 +155,29 @@ osmo workflow list --name gha-runner- --pool airstack-ci --status RUNNING PENDIN Module repos (`asm_*`) that call trunk's reusable [`module-system-tests.yml`](https://github.com/castacks/AirStack/blob/main/.github/workflows/module-system-tests.yml) with the default `runs-on: [self-hosted, airstack-ephemeral]` queue jobs **in their own -repo**, and the orchestrator polls exactly one `repo:` per instance. To add an -`asm_` repo to the poll list, run a second orchestrator instance against it: +repo**. One orchestrator instance polls them all — list every repo under +`repos:` in `/etc/airstack-orchestrator/config.yaml`: + +```yaml +repos: + - "castacks/AirStack" + - "castacks/asm_dfm2_disturbances" + - "castacks/asm_optitrack" + - "castacks/asm_macvo" +``` + +(The legacy singular `repo:` key is still accepted.) Two requirements when +adding a repo: -1. **Extend the PAT.** The fine-grained GitHub PAT must also cover the module +1. **Extend the PAT.** The fine-grained GitHub PAT must cover each listed repo with `Actions: read/write` + `Administration: read/write` (JIT runner - registration is per-repo). Reuse the existing PAT file if it covers the - repo, else stage a second one. -2. **Copy the config.** `/etc/airstack-orchestrator/config.yaml` → - `config-asm-.yaml` with `repo: "castacks/asm_"` and a - **distinct `workflow_name_prefix`** (e.g. `gha-runner-asm-`) so the - two instances' orphan sweeps don't cancel each other's OSMO workflows. -3. **Run a second service instance** pointing at the new config and its own - state file (copy `airstack-orchestrator.service`, adjust `ExecStart`'s - `--config` and `--state`, e.g. `--state /var/lib/airstack-orchestrator/state-asm-.json`). + registration is per-repo). +2. **Restart the service** (`systemctl restart airstack-orchestrator`) and + confirm the startup line lists every repo: + `orchestrator started (OSMO backend): repos=[...]`. + +The shared `max_concurrent` cap and the orphan sweep span all polled repos — +no per-repo instances, prefixes, or state files needed. First-party only: the reusable workflow refuses callers outside the castacks org, mirroring the fork-PR block. Org-level polling across registered repos diff --git a/.github/orchestrator/config.example.yaml b/.github/orchestrator/config.example.yaml index ba1c64384..4a17fe8de 100644 --- a/.github/orchestrator/config.example.yaml +++ b/.github/orchestrator/config.example.yaml @@ -64,8 +64,15 @@ runner_version: "2.336.0" # --- GitHub --- -# owner/name of the repo whose queued workflow_jobs to pick up. -repo: "castacks/AirStack" +# Repos whose queued workflow_jobs to pick up. Trunk plus every module repo +# whose CI calls the reusable module-system-tests workflow. The PAT must have +# admin (self-hosted runner) rights on each. The legacy singular `repo:` key +# is still accepted. +repos: + - "castacks/AirStack" + - "castacks/asm_dfm2_disturbances" + - "castacks/asm_optitrack" + - "castacks/asm_macvo" # Labels the orchestrator polls for. A queued workflow_job whose `labels` # array is a superset of this list gets a workflow submitted for it. These are diff --git a/.github/orchestrator/orchestrator.py b/.github/orchestrator/orchestrator.py index 4af62555b..887f27f0f 100644 --- a/.github/orchestrator/orchestrator.py +++ b/.github/orchestrator/orchestrator.py @@ -327,8 +327,16 @@ def __init__(self, config: dict, pat: str, state_path: str, template_path: str): self.privileged = bool(config.get("privileged", True)) self.host_network = bool(config.get("host_network", False)) - # GitHub. - self.repo = config["repo"] + # GitHub. `repos:` (list) is the primary key; the legacy singular + # `repo:` is still accepted. Every listed repo is polled for queued + # jobs and must be visible to the PAT with admin (JIT runner) rights + # — trunk plus the asm_* module repos calling the reusable workflow. + repos = config.get("repos") or config.get("repo") + if isinstance(repos, str): + repos = [repos] + if not repos: + raise KeyError("config needs `repos:` (list) or legacy `repo:`") + self.repos: list[str] = list(repos) self.runner_labels = config["runner_labels"] # Limits / timing. @@ -493,13 +501,17 @@ def spawn_once(self) -> None: active = len(state["jobs"]) if active >= self.max_concurrent: return - try: - queued = find_queued_jobs(self.repo, self.runner_labels, self.pat) - except Exception as e: # noqa: BLE001 - log.warning("find_queued_jobs failed: %s", e) - return + queued: list[tuple[str, dict]] = [] + for repo in self.repos: + try: + queued.extend( + (repo, j) + for j in find_queued_jobs(repo, self.runner_labels, self.pat) + ) + except Exception as e: # noqa: BLE001 + log.warning("find_queued_jobs(%s) failed: %s", repo, e) - for job in queued: + for repo, job in queued: if active >= self.max_concurrent: break job_id = job["job_id"] @@ -512,7 +524,7 @@ def spawn_once(self) -> None: tmp_path: str | None = None try: jit = mint_jit_config( - self.repo, workflow_name, self.runner_labels, self.pat + repo, workflow_name, self.runner_labels, self.pat ) workflow_yaml = self.render_workflow(workflow_name, jit) tmp_path = self._write_temp_workflow(workflow_name, workflow_yaml) @@ -528,6 +540,7 @@ def spawn_once(self) -> None: pass state["jobs"][job_id] = { + "repo": repo, "run_id": job["run_id"], "workflow_id": workflow_id, "workflow_name": live_name, @@ -550,8 +563,11 @@ def reap_once(self) -> None: for job_id in list(state["jobs"].keys()): entry = state["jobs"][job_id] wid = entry["workflow_id"] + # Entries written before multi-repo support carry no repo field; + # they can only have come from the first (primary) repo. + entry_repo = entry.get("repo", self.repos[0]) try: - job = get_job_status(self.repo, job_id, self.pat) + job = get_job_status(entry_repo, job_id, self.pat) except Exception as e: # noqa: BLE001 log.warning("get_job_status(%s) failed: %s", job_id, e) continue @@ -607,9 +623,9 @@ def reap_once(self) -> None: def run(self) -> None: log.info( - "orchestrator started (OSMO backend): repo=%s labels=%s pool=%s " + "orchestrator started (OSMO backend): repos=%s labels=%s pool=%s " "platform=%s max_concurrent=%d", - self.repo, self.runner_labels, self.pool, + self.repos, self.runner_labels, self.pool, self.platform or "(pool default)", self.max_concurrent, ) last_spawn = 0.0 diff --git a/docs/release_notes/index.md b/docs/release_notes/index.md index d577eadde..2516a929c 100644 --- a/docs/release_notes/index.md +++ b/docs/release_notes/index.md @@ -187,6 +187,9 @@ Feature docs deliberately cite none of these — the design sources live here: - Unit-test documentation matches the co-located layout: C++ gtests run via `colcon test` under the `build_packages` mark; Python via the root harness (`conftest.py` applies the `unit` mark by file location) +- The CI orchestrator polls a `repos:` list (one instance covers trunk and + every asm_* module repo; the singular `repo:` key still works) — module CI + jobs on `airstack-ephemeral` no longer need per-repo orchestrator instances - Ephemeral CI GPU runners spawn via NVIDIA OSMO as a drop-in replacement for the earlier OpenStack-Nova backend: the GitHub side (labels, JIT tokens, fork guard) is unchanged; only the spawn target moved. The OSMO