From a25b2b29485a38b7469f87a0d9d3633f7593e52b Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 15:55:35 +0200 Subject: [PATCH 01/19] docs: add ADR-001 warm pool with stop/hibernate for idle runners - ADR documenting the warm pool concept with three instance states (hot/warm/cold) - Implementation plan with phased approach - User-facing warm pool configuration guide Signed-off-by: Brend Smits --- docs/adr/001-warm-pool-hibernation.md | 309 ++++++++++++ docs/adr/001-warm-pool-implementation-plan.md | 452 ++++++++++++++++++ docs/warm-pool.md | 201 ++++++++ 3 files changed, 962 insertions(+) create mode 100644 docs/adr/001-warm-pool-hibernation.md create mode 100644 docs/adr/001-warm-pool-implementation-plan.md create mode 100644 docs/warm-pool.md diff --git a/docs/adr/001-warm-pool-hibernation.md b/docs/adr/001-warm-pool-hibernation.md new file mode 100644 index 0000000000..b718a229b6 --- /dev/null +++ b/docs/adr/001-warm-pool-hibernation.md @@ -0,0 +1,309 @@ +# ADR-001: Warm Pool with Stop/Hibernate for Idle Pool Runners + +## Status + +Proposed + +## Date + +2026-05-22 + +## Context + +The current pool system maintains a configurable number of idle GitHub Actions runners via the pool lambda. These runners are launched on a cron schedule and sit running (consuming compute costs) until either: + +1. A job picks them up, or +2. The scale-down lambda terminates them after the idle count is exceeded. + +This "always running" model has significant cost implications: + +- **Waste during idle periods**: Pool runners that never receive a job still consume full EC2 compute costs until scale-down terminates them. +- **Cold start latency**: When all idle runners are consumed and new ones must be launched, users experience full boot time (AMI boot + runner registration + tool installation). This can be 2-5 minutes depending on instance type and AMI. +- **Binary choice**: Currently the only options are "running" (full cost) or "terminated" (zero cost but full cold start). There is no middle ground. + +AWS provides EC2 Stop and Hibernate capabilities that preserve the EBS root volume (and optionally RAM state). A stopped instance: + +- Incurs **zero compute cost** (no per-second billing). +- Retains its **EBS volume** (small storage cost, typically $0.08-0.10/GB-month for gp3). +- Can be **restarted in 10-30 seconds** vs. 2-5 minutes for a fresh launch. +- Preserves the instance ID, network interfaces, and private IP. + +This makes stopped instances an ideal "warm" tier between fully running (hot) and terminated (cold). + +## Decision + +We will implement a **warm pool system** with three instance states, controlled by a feature flag: + +### Instance States + +| State | Cost | Startup Time | Description | +|-------|------|-------------|-------------| +| **Hot** (running) | Full compute + EBS | 0s (already running) | Currently idle, registered with GitHub, ready to accept jobs | +| **Warm** (stopped) | EBS only | 10-30s | Stopped instance with preserved volume, must be started + re-registered | +| **Cold** (terminated) | None | 2-5 min | No instance exists, must launch from scratch | + +### Behavior Changes + +#### Scale-Down Lambda (modified) + +When the feature flag `enable_warm_pool` is enabled: + +1. **Instead of terminating** idle runners that exceed `idleCount`, the scale-down lambda will **stop** them and record their state in a **DynamoDB table**. +2. Before stopping, the runner is **deregistered from GitHub** (existing behavior preserved). +3. A new config parameter `warm_pool_config` controls: + - `maxWarmInstances`: Maximum number of stopped instances to retain (prevents EBS cost explosion). Default: same as pool size. + - `maxWarmAgeHours`: Maximum age (in hours) of a warm instance before it is terminated instead of kept. Default: 168 (7 days). Prevents stale AMIs from accumulating. +4. If a runner exceeds `maxWarmInstances` or `maxWarmAgeHours`, it is **terminated** (existing behavior). + +#### Scale-Up Lambda (modified) + +When a job is queued and no hot runner is available: + +1. **Before creating a new instance**, query the **DynamoDB warm pool table** for available stopped instances matching the runner config (owner, environment). +2. If a warm instance exists: + - **Start** the instance (EC2 `StartInstances` API). + - Remove the DynamoDB record (atomic delete prevents race conditions). + - The instance's existing user-data/startup script re-registers with GitHub on boot. + - Tag: set `ghr:started-from-warm-pool=true` for metrics. +3. If no warm instance exists (or start fails), fall through to the existing `createRunner()` flow (cold start). + +#### Pool Lambda (modified, separate feature flag) + +The pool lambda gets its own strategy setting (`pool_strategy`) independent of the scale-down warm pool behavior. This enables three distinct operational modes: + +| `pool_strategy` | `warm_pool_config.enabled` | Behavior | +|-----------------|---------------------------|----------| +| `hot` (default) | `false` | Current behavior: pool creates running instances, scale-down terminates them | +| `hot` | `true` | Pool creates running instances, scale-down stops them into warm tier | +| `warm` | `true` | Pool maintains **stopped** instances only (no idle compute). Scale-up/webhook starts them on demand | +| `warm` | `false` | Invalid — rejected at plan time | + +When `pool_strategy = "warm"`: + +1. The pool lambda **creates instances and waits `warm_pool_ready_delay_seconds`** (default 30s) before stopping them. This grace period gives the instance time to boot, register with GitHub, and potentially pick up a queued job. If the runner picks up a job during this window, it is **not stopped** — it runs normally. +2. After the grace period, if the runner is still idle (not busy), it is deregistered from GitHub, stopped, and tagged as warm. The pool target represents the number of **warm** (stopped) instances to maintain. +3. No permanently running idle runners exist from the pool — zero long-term compute waste. +4. Scale-up and webhook flows start warm instances when jobs arrive (10-30s startup). +5. The `idle_config` / `idleCount` setting becomes irrelevant for pool runners since none are kept running long-term. + +When `pool_strategy = "hot"` (default, backward compatible): + +1. Pool creates running instances as today. +2. If `warm_pool_config.enabled = true`, scale-down stops excess idle runners into the warm tier instead of terminating. +3. Scale-up can still use warm instances as a fast fallback before cold-launching. + +### Feature Flag + +- Terraform variable: `warm_pool_config.enabled` (bool, default `false`) — controls whether scale-down stops instances into a warm tier. +- Terraform variable: `pool_strategy` (string, `"hot"` or `"warm"`, default `"hot"`) — controls whether the pool lambda maintains running or stopped instances. **Independent from `warm_pool_config`** to allow warm-only deployments with zero idle compute. +- Both are passed as environment variables to the relevant lambdas. +- When both are at defaults, behavior is identical to today (no breaking changes). + +### Configuration + +New Terraform variable structure (nested in existing runner config patterns): + +```hcl +variable "pool_strategy" { + description = "Strategy for the pool lambda. 'hot' keeps runners running (current behavior). 'warm' maintains stopped instances only — zero idle compute, 10-30s start on demand." + type = string + default = "hot" + validation { + condition = contains(["hot", "warm"], var.pool_strategy) + error_message = "pool_strategy must be 'hot' or 'warm'." + } +} + +variable "warm_pool_config" { + description = "Configuration for the warm pool tier. Controls how stopped instances are managed." + type = object({ + enabled = bool + max_warm_instances = number + max_warm_age_hours = number + warm_pool_ready_delay_seconds = number + }) + default = { + enabled = false + max_warm_instances = 3 + max_warm_age_hours = 168 + warm_pool_ready_delay_seconds = 30 + } +} +``` + +**Validation**: `pool_strategy = "warm"` requires `warm_pool_config.enabled = true`. Terraform will error at plan time if this invariant is violated. + +### State Store (DynamoDB) + +Warm pool state is managed in a **DynamoDB table** rather than EC2 instance tags or DescribeInstances API calls. This avoids rate limiting, provides single-digit millisecond lookups, and enables atomic operations to prevent race conditions between concurrent lambda invocations. + +**Table**: `{prefix}-warm-pool` (PAY_PER_REQUEST billing) + +| Attribute | Type | Purpose | +|-----------|------|---------| +| `instanceId` (PK) | String | EC2 instance ID | +| `runnerOwner` (GSI PK) | String | GitHub org/repo owner | +| `stoppedAt` (GSI SK) | String | ISO 8601 — enables newest-first selection | +| `environment` | String | Runner environment | +| `runnerType` | String | `Org` or `Repo` | +| `amiId` | String | AMI at time of stop (staleness detection) | +| `instanceType` | String | For informational/filtering purposes | +| `az` | String | Availability zone | +| `expiresAt` (TTL) | Number | Auto-cleanup of stale records | + +DynamoDB TTL auto-deletes records past `max_warm_age_hours`. The scale-down lambda also actively terminates instances during its eviction pass. + +### Tags + +EC2 instance tags are still used for **observability and cost allocation** (not for state lookups): + +| Tag | Value | Purpose | +|-----|-------|---------| +| `ghr:started-from-warm-pool` | `true` | Instance was started from warm pool by scale-up (metrics) | +| `ghr:warm-pool-grace-hit` | `true` | Instance picked up a job during the pool lambda's grace window | + +**Observability scenarios:** + +| Scenario | State change | How to track | +|----------|-------------|-------------| +| Scale-up restarts a stopped warm instance | DynamoDB record deleted + EC2 tag `ghr:started-from-warm-pool=true` | Count instances with tag / CloudWatch metric | +| Pool lambda grace window: instance picks up job before being stopped | EC2 tag `ghr:warm-pool-grace-hit=true` | Count instances with tag / CloudWatch metric | +| Instance stopped into warm tier | DynamoDB record created | Query DynamoDB for current warm pool size | +| Warm instance terminated (age/cap/AMI eviction) | DynamoDB record deleted + EC2 instance terminated | CloudWatch metric `WarmPoolEvictions` | + +The pool lambda sets `ghr:warm-pool-grace-hit=true` when it detects a runner became busy during the `warm_pool_ready_delay_seconds` window. This lets operators measure how often the grace window saves a stop/start cycle — a high rate of grace hits means jobs are arriving frequently and the `warm` strategy is working efficiently even without going through the stopped state. + +### IAM Permissions + +The Lambda execution roles need additional permissions: + +**EC2** (scoped to `ghr:Application=github-action-runner` tag): +- `ec2:StartInstances` — to start warm instances +- `ec2:StopInstances` — to stop instances instead of terminating + +**DynamoDB** (scoped to warm pool table ARN): +- `dynamodb:PutItem` — record new warm pool entries (scale-down, pool lambda) +- `dynamodb:DeleteItem` — remove entries on start or termination (scale-up, scale-down, pool lambda) +- `dynamodb:Query` — find available warm instances by owner (scale-up, pool lambda) +- `dynamodb:GetItem` — check specific instance state (all lambdas) + +## Consequences + +### Positive + +- **Cost reduction**: Idle pool runners that never get a job now cost only EBS storage (~$0.80/month for a 10GB gp3 volume) instead of full EC2 compute ($30-150+/month depending on instance type). +- **Faster startup**: Warm instances start in 10-30s vs. 2-5 minutes for cold launches. The EBS volume already contains the OS, runner binary, and any pre-installed tools. +- **Preserved AMI investment**: Pre-baked AMIs with heavy toolchains (Docker images, SDKs) don't need to be re-downloaded. +- **Backward compatible**: Feature flag ensures no behavior change for existing users. +- **Graceful degradation**: If warm instances fail to start, the system falls through to cold launch. + +### Negative + +- **EBS costs accumulate**: Each warm instance retains its EBS volume. With `max_warm_instances` and `max_warm_age_hours` controls, this is bounded, but operators must be aware. +- **Stale state risk**: A stopped instance may have outdated packages, expired credentials, or stale Docker caches. The startup script must handle re-registration and basic validation. +- **Instance type lock-in**: A stopped instance retains its instance type. If the launch template or instance type config changes, warm instances become invalid and must be terminated. +- **Complexity**: Three-state lifecycle is more complex than the current two-state (running/terminated) model. +- **Spot instances**: Stopped spot instances can be reclaimed by AWS at any time. More critically, **one-time spot requests (the module default) cannot be stopped at all** — the EC2 API returns `UnsupportedOperation`. The warm pool works only with on-demand or persistent spot instances. The scale-down lambda handles this gracefully by falling back to terminate, but operators must set `instance_target_capacity_type = "on-demand"` for the warm pool to actually accumulate stopped instances. See [EC2 Instance Lifecycle and Warm Pool Compatibility](#ec2-instance-lifecycle-and-warm-pool-compatibility) for details. + +### Risks & Mitigations + +| Risk | Mitigation | +|------|-----------| +| EBS cost explosion | `max_warm_instances` hard cap + `max_warm_age_hours` TTL | +| Stale AMI on warm instance | Compare instance's AMI ID against current launch template; terminate if mismatched | +| Spot reclamation | Warm pool is best-effort for spot — if AWS reclaims a stopped spot instance, scale-up falls through to cold launch. Future: `warm_pool_capacity_type_override` to force on-demand for pool instances | +| Runner binary outdated | Startup script always pulls latest runner version (existing behavior with `DISABLE_RUNNER_AUTOUPDATE=false`) | +| GitHub registration failure on restart | If re-registration fails, tag as orphan and let next scale-down cycle terminate | + +## Alternatives Considered + +### 1. EC2 Hibernate (instead of Stop) + +Hibernate preserves RAM state, enabling even faster resume (~5-10s). However: + +- Requires specific instance types and AMI configuration +- Requires pre-allocated EBS space for RAM dump +- More complex setup and not all instance families support it +- The marginal benefit over Stop (5-10s vs 10-30s) doesn't justify the complexity + +**Decision**: Start with Stop. Hibernate can be a future enhancement for specific instance types. + +### 2. AWS EC2 Auto Scaling Warm Pools + +AWS natively supports warm pools in Auto Scaling Groups. However: + +- This module uses EC2 Fleet API, not ASGs +- Would require fundamental architecture change +- Less control over tagging and lifecycle integration with GitHub +- Doesn't integrate with the existing pool/scale-down lambda logic + +**Decision**: Implement warm pool logic in the existing lambda functions for tighter integration. + +### 3. EBS Snapshot + Fast Launch + +Take EBS snapshots of pool instances and use them for fast launch: + +- Still requires new instance creation (no IP/instance preservation) +- Snapshot management adds complexity +- Fast Launch has availability zone constraints + +**Decision**: Stop/Start is simpler and achieves comparable startup times. + +## EC2 Instance Lifecycle and Warm Pool Compatibility + +The warm pool relies on the EC2 `StopInstances` / `StartInstances` APIs. Not all EC2 purchase options support stop/start: + +### Instance Types by Purchase Option + +| Purchase Option | Can Stop/Start? | Warm Pool Compatible? | Cost vs On-Demand | +|----------------|----------------|----------------------|-------------------| +| **On-demand** | Yes | Yes (fully reliable) | 100% (baseline) | +| **Persistent Spot** | Yes | Yes (best-effort) | 60-90% discount | +| **One-time Spot** | No | No — must terminate | 60-90% discount | + +### What is a Persistent Spot Request? + +AWS Spot Instances come in two request types: + +- **One-time request** (default in this module via EC2 Fleet): AWS fulfills the request once. The instance **cannot be stopped** — only terminated. If interrupted by AWS, it is terminated and gone. This is what `instance_target_capacity_type = "spot"` uses today. + +- **Persistent request**: AWS keeps the request active. The instance **can be stopped and restarted**. When you stop a persistent spot instance, the underlying capacity is released (no compute cost), but the request remains open. When you start it, AWS attempts to re-acquire capacity at the current spot price. If capacity is unavailable, the start may be delayed. + +### Cost Comparison for Warm Pool + +Assuming a `m5.large` in eu-west-1 (~$0.096/hr on-demand, ~$0.035/hr spot): + +| Strategy | Running Cost | Stopped Cost | Monthly idle cost (1 instance, 50% idle) | +|----------|-------------|-------------|------------------------------------------| +| Hot pool (on-demand) | $0.096/hr | N/A (always running) | ~$34.56 | +| Hot pool (spot) | $0.035/hr | N/A (always running) | ~$12.60 | +| Warm pool (on-demand) | $0.096/hr when active | ~$0.80/mo EBS only | ~$0.80 + active hours | +| Warm pool (persistent spot) | $0.035/hr when active | ~$0.80/mo EBS only | ~$0.80 + active hours | + +The warm pool's value proposition is strongest when runners spend significant time idle: the stopped EBS cost ($0.80/mo for 10GB gp3) vs running compute ($12-35/mo). + +### Persistent Spot Caveats + +1. **Restart not guaranteed**: When starting a stopped persistent spot instance, AWS may not have capacity at the current spot price. The instance stays in `pending` until capacity is available. The scale-up lambda should implement a timeout and fall back to cold launch. + +2. **Price changes**: If the spot price has risen above your max price since the instance was stopped, the start will fail. + +3. **Reclamation while stopped**: AWS can reclaim (terminate) a stopped spot instance if it needs the capacity, though this is rare for stopped instances. + +4. **Implemented in this module**: When `warm_pool_config.enabled = true` and `instance_target_capacity_type = "spot"`, the module uses `RunInstances` with `InstanceMarketOptions.SpotOptions.SpotInstanceType = "persistent"` and `InstanceInterruptionBehavior = "stop"`. It also overrides `InstanceInitiatedShutdownBehavior = "stop"` (since the launch template defaults to `"terminate"`, which conflicts with the spot stop behavior). This is gated behind the `ENABLE_PERSISTENT_SPOT` environment variable, which is derived automatically by Terraform. + +### Recommended Configuration + +For warm pool deployments, use one of: + +1. **On-demand** (`instance_target_capacity_type = "on-demand"`): Fully reliable, higher cost when active but zero cost when stopped. Best for critical workloads. + +2. **Spot for scale-up, on-demand for pool** (future `warm_pool_capacity_type_override`): Scale-up launches cheap spot instances for burst jobs; pool maintains on-demand instances that reliably stop/start. Best cost-to-reliability ratio. + +3. **On-demand with warm pool only** (no hot pool): Set `pool_strategy = "warm"` with on-demand instances. Pool creates instances, immediately stops them. Scale-up starts them on demand. Zero idle compute cost, 10-30s start time, fully reliable. + +> **Resolved**: When `instance_target_capacity_type = "spot"` and `warm_pool_config.enabled = true`, the module automatically switches from `CreateFleet` (one-time spot) to `RunInstances` with `SpotInstanceType = "persistent"` and `InstanceInterruptionBehavior = "stop"`. This is transparent to the user — no additional configuration is needed. The Terraform variable `ENABLE_PERSISTENT_SPOT` is derived automatically. + +## Implementation Plan + +See [docs/adr/001-warm-pool-implementation-plan.md](001-warm-pool-implementation-plan.md) for the phased implementation plan. diff --git a/docs/adr/001-warm-pool-implementation-plan.md b/docs/adr/001-warm-pool-implementation-plan.md new file mode 100644 index 0000000000..cea9be567c --- /dev/null +++ b/docs/adr/001-warm-pool-implementation-plan.md @@ -0,0 +1,452 @@ +# ADR-001: Implementation Plan — Warm Pool with Stop/Hibernate + +## Overview + +This document details the phased implementation plan for the warm pool feature described in [ADR-001](001-warm-pool-hibernation.md). + +## Phase 1: Foundation (Terraform + IAM + Feature Flag + DynamoDB) + +**Goal**: Wire the feature flag, permissions, and state store without changing any runtime behavior. + +### Tasks + +1. **Add `warm_pool_config` variable** to `modules/runners/variables.tf` + ```hcl + variable "warm_pool_config" { + description = "Configuration for the warm pool tier. Controls how stopped instances are managed." + type = object({ + enabled = bool + max_warm_instances = number + max_warm_age_hours = number + warm_pool_ready_delay_seconds = number + }) + default = { + enabled = false + max_warm_instances = 3 + max_warm_age_hours = 168 + warm_pool_ready_delay_seconds = 30 + } + } + ``` + +2. **Add `pool_strategy` variable** to `modules/runners/variables.tf` + ```hcl + variable "pool_strategy" { + description = "Strategy for the pool lambda. 'hot' keeps runners running. 'warm' maintains stopped instances only." + type = string + default = "hot" + validation { + condition = contains(["hot", "warm"], var.pool_strategy) + error_message = "pool_strategy must be 'hot' or 'warm'." + } + } + ``` + +3. **Add cross-variable validation** (lifecycle precondition or `check` block): + - `pool_strategy = "warm"` requires `warm_pool_config.enabled = true` + +4. **Create DynamoDB table** for warm pool state (`modules/runners/warm-pool.tf`): + ```hcl + resource "aws_dynamodb_table" "warm_pool" { + count = var.warm_pool_config.enabled ? 1 : 0 + name = "${var.prefix}-warm-pool" + billing_mode = "PAY_PER_REQUEST" + hash_key = "instanceId" + + attribute { + name = "instanceId" + type = "S" + } + + attribute { + name = "runnerOwner" + type = "S" + } + + attribute { + name = "stoppedAt" + type = "S" + } + + global_secondary_index { + name = "by-owner" + hash_key = "runnerOwner" + range_key = "stoppedAt" + projection_type = "ALL" + } + + ttl { + attribute_name = "expiresAt" + enabled = true + } + + tags = local.tags + } + ``` + + **Table schema:** + + | Attribute | Type | Purpose | + |-----------|------|---------| + | `instanceId` (PK) | String | EC2 instance ID | + | `runnerOwner` (GSI PK) | String | GitHub org/repo owner (for filtering) | + | `stoppedAt` (GSI SK) | String | ISO 8601 timestamp when stopped (for newest-first selection + age eviction) | + | `environment` | String | Runner environment tag | + | `runnerType` | String | `Org` or `Repo` | + | `amiId` | String | AMI ID at time of stopping (for staleness check) | + | `instanceType` | String | EC2 instance type | + | `az` | String | Availability zone (instance can only restart in same AZ) | + | `expiresAt` | Number | Unix epoch for DynamoDB TTL auto-deletion (set to `stoppedAt + max_warm_age_hours`) | + + DynamoDB TTL automatically cleans up stale records — no lambda logic needed for age-based eviction of the DB record itself. The scale-down lambda still terminates the actual EC2 instance. + +5. **Add IAM permissions** to the Lambda execution roles: + - `ec2:StopInstances` (scale-down lambda) + - `ec2:StartInstances` (scale-up and pool lambdas) + - `dynamodb:PutItem`, `dynamodb:DeleteItem`, `dynamodb:GetItem` (scale-down, scale-up, pool lambdas) + - `dynamodb:Query` on GSI `by-owner` (scale-up and pool lambdas) + - Condition on EC2 actions: `ec2:ResourceTag/ghr:Application = github-action-runner` + - Condition on DynamoDB: resource ARN scoped to the warm pool table + +6. **Pass environment variables** to all three lambdas: + - `ENABLE_WARM_POOL` → `var.warm_pool_config.enabled` + - `WARM_POOL_MAX_INSTANCES` → `var.warm_pool_config.max_warm_instances` + - `WARM_POOL_MAX_AGE_HOURS` → `var.warm_pool_config.max_warm_age_hours` + - `WARM_POOL_READY_DELAY_SECONDS` → `var.warm_pool_config.warm_pool_ready_delay_seconds` (pool lambda only) + - `POOL_STRATEGY` → `var.pool_strategy` (pool lambda only) + - `WARM_POOL_TABLE_NAME` → DynamoDB table name (all three lambdas) + +7. **Wire through multi-runner module** (`modules/multi-runner/runners.tf`): + - Add `warm_pool_config` and `pool_strategy` to the `multi_runner_config` object type + - Pass through to the runners module + +### Files Modified +- `modules/runners/variables.tf` +- `modules/runners/warm-pool.tf` (new — DynamoDB table + IAM) +- `modules/runners/scale-down.tf` (env vars) +- `modules/runners/scale-up.tf` (env vars) +- `modules/runners/pool/main.tf` (env vars) +- `modules/runners/policies/lambda-scale-down.json` (IAM — EC2 Stop + DynamoDB) +- `modules/runners/policies/lambda-scale-up.json` (IAM — EC2 Start + DynamoDB) +- `modules/multi-runner/variables.tf` +- `modules/multi-runner/runners.tf` + +--- + +## Phase 2: Scale-Down — Stop Instead of Terminate + +**Goal**: Idle pool runners are stopped instead of terminated when warm pool is enabled. + +### Tasks + +1. **Add `stopRunner()` function** to `lambdas/functions/control-plane/src/aws/runners.ts`: + ```typescript + export async function stopRunner(instanceId: string): Promise { + const ec2 = getTracedAWSV3Client(new EC2Client({ region: process.env.AWS_REGION })); + await ec2.send(new StopInstancesCommand({ InstanceIds: [instanceId] })); + } + ``` + +2. **Create `lambdas/functions/control-plane/src/aws/warm-pool.ts`** — DynamoDB client for warm pool state: + ```typescript + import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; + import { DynamoDBDocumentClient, PutCommand, DeleteCommand, QueryCommand } from '@aws-sdk/lib-dynamodb'; + + const client = DynamoDBDocumentClient.from( + getTracedAWSV3Client(new DynamoDBClient({ region: process.env.AWS_REGION })) + ); + const TABLE_NAME = process.env.WARM_POOL_TABLE_NAME; + + export interface WarmPoolEntry { + instanceId: string; + runnerOwner: string; + runnerType: string; + environment: string; + stoppedAt: string; + amiId: string; + instanceType: string; + az: string; + expiresAt: number; + } + + export async function addToWarmPool(entry: WarmPoolEntry): Promise { ... } + export async function removeFromWarmPool(instanceId: string): Promise { ... } + export async function getWarmRunners(runnerOwner: string): Promise { ... } + export async function getWarmPoolCount(runnerOwner: string): Promise { ... } + ``` + + **Key operations:** + - `addToWarmPool()` — PutItem after stopping an instance + - `removeFromWarmPool()` — DeleteItem after starting or terminating a warm instance + - `getWarmRunners()` — Query GSI `by-owner` with ScanIndexForward=false (newest first) + - `getWarmPoolCount()` — Query with Select=COUNT for checking against max limit + +3. **Modify `removeRunner()`** in `scale-down.ts`: + - If `ENABLE_WARM_POOL=true`: + - Query DynamoDB for current warm pool count (`getWarmPoolCount()`) + - If under `WARM_POOL_MAX_INSTANCES`: deregister from GitHub → stop instance → `addToWarmPool()` with instance metadata + - If at/over limit: terminate as today + - Note: spot instances are stopped too (best-effort; if AWS reclaims them, scale-up handles gracefully) + - If `ENABLE_WARM_POOL=false`: existing behavior (terminate) + +4. **Add warm pool eviction** to `scaleDown()`: + - After active runner evaluation, add warm pool cleanup phase: + - Query DynamoDB for all warm entries for the environment + - For each entry older than `WARM_POOL_MAX_AGE_HOURS`: terminate EC2 instance + `removeFromWarmPool()` + - If total warm count > `WARM_POOL_MAX_INSTANCES`: terminate + remove oldest + - Note: DynamoDB TTL will eventually auto-delete very stale records, but we actively terminate EC2 instances + +5. **Add AMI staleness check**: + - `amiId` is stored in DynamoDB when stopping + - During eviction, compare stored `amiId` against current launch template AMI (passed as env var `LAUNCH_TEMPLATE_AMI_ID`) + - If mismatched: terminate + remove from DynamoDB + +### Files Modified +- `lambdas/functions/control-plane/src/aws/runners.ts` (`stopRunner()`) +- `lambdas/functions/control-plane/src/aws/warm-pool.ts` (new — DynamoDB client) +- `lambdas/functions/control-plane/src/scale-runners/scale-down.ts` (core logic) + +### Tests +- Unit tests for `stopRunner()` +- Unit tests for DynamoDB warm pool operations (add, remove, query, count) +- Unit tests for warm pool eviction logic (age, count, AMI mismatch) +- Integration test: idle runner is stopped and DynamoDB record created +- Integration test: warm pool count cap is respected + +--- + +## Phase 3: Scale-Up — Start Warm Instances + +**Goal**: When a job is queued, start a warm instance before creating a new one. + +### Tasks + +1. **Add `startRunner()` function** to `runners.ts`: + ```typescript + export async function startRunner(instanceId: string): Promise { + const ec2 = getTracedAWSV3Client(new EC2Client({ region: process.env.AWS_REGION })); + await ec2.send(new StartInstancesCommand({ InstanceIds: [instanceId] })); + } + ``` + +2. **Add `findAndStartWarmRunner()` function** to `scale-up.ts`: + ```typescript + export async function findAndStartWarmRunner(params: { + environment: string; + runnerOwner: string; + runnerType: RunnerType; + }): Promise { + // Query DynamoDB GSI 'by-owner' for runnerOwner, newest first (ScanIndexForward=false) + // Pick the first (most recently stopped) entry + // Call startRunner(entry.instanceId) + // Remove entry from DynamoDB (removeFromWarmPool) + // Tag instance with ghr:started-from-warm-pool=true + // Return instanceId, or null if table is empty / start fails + } + ``` + + **Why DynamoDB instead of DescribeInstances:** + - Single-digit millisecond query latency vs. 100-500ms+ for DescribeInstances + - No API rate limiting concerns (DynamoDB scales with PAY_PER_REQUEST) + - Consistent reads — no eventual consistency window + - GSI enables efficient filtering by owner without scanning all instances + - Atomic deletes prevent race conditions (two scale-up invocations won't pick the same instance) + + **Handling stale DynamoDB entries (instance already terminated by AWS/spot reclaim):** + - If `StartInstances` returns `InvalidInstanceID.NotFound` or instance is in `terminated` state → delete DynamoDB record, try next entry + - This self-heals any drift between DynamoDB and actual EC2 state + +3. **Modify `scaleUpHandler()`** in `scale-up.ts`: + - Before calling `createRunners()`, attempt `findAndStartWarmRunner()` + - Decrement `numberOfRunners` needed for each warm instance started + - If all needed runners filled from warm pool, skip fleet creation entirely + - On `StartInstances` failure: log warning, fall through to cold launch + +4. **Handle startup script re-registration**: + - The existing `start-runner.sh` / user-data script runs on boot + - On a **restarted** instance, it must: + - Detect it's a warm-pool restart (check `ghr:started-from-warm-pool` tag via instance metadata or a marker file) + - Re-fetch a registration token from GitHub + - Re-register the runner + - This may require a minor change to the startup script or a systemd unit that triggers on boot + +### Files Modified +- `lambdas/functions/control-plane/src/aws/runners.ts` (`startRunner()`) +- `lambdas/functions/control-plane/src/aws/warm-pool.ts` (query logic) +- `lambdas/functions/control-plane/src/scale-runners/scale-up.ts` (integration) +- `images/start-runner.sh` (handle restart case) + +### Tests +- Unit test: warm runner is started before new creation (DynamoDB queried first) +- Unit test: if DynamoDB is empty, falls through to createRunners +- Unit test: stale DynamoDB entry (terminated instance) is deleted and next entry tried +- Unit test: warm runner count decrements needed runners +- Unit test: concurrent scale-up invocations don't pick same instance (conditional delete) +- E2E test: job triggers warm start, runner registers, job completes + +--- + +## Phase 4: Pool Lambda — Warm-Only Strategy + +**Goal**: Pool lambda can maintain stopped (warm) instances instead of running ones, enabling zero idle compute. + +### Tasks + +1. **Modify `adjust()` in pool.ts** to check `POOL_STRATEGY` env var: + - If `POOL_STRATEGY=hot` (default): + - Before creating new running instances, check for warm instances and start them first + - Create remaining deficit as new running instances (current behavior + warm preference) + - If `POOL_STRATEGY=warm`: + - Target pool size refers to **warm** (stopped) instances + - Count existing warm (stopped) instances + - If below target: create new instances, let them boot + register with GitHub + - **Wait `WARM_POOL_READY_DELAY_SECONDS`** (default 30s) before evaluating + - After the delay: check if runner is busy (picked up a job) — if busy, leave it alone + - If still idle after delay: deregister from GitHub + stop instance + tag as warm + - If above target: do nothing (scale-down eviction handles excess) + - No permanently running idle instances are maintained + +2. **Add grace period logic** (for `POOL_STRATEGY=warm`): + - After creating/starting instances, wait `WARM_POOL_READY_DELAY_SECONDS` + - Query GitHub API for each runner's busy state + - Only stop runners that are confirmed idle after the grace window + - Runners that picked up a job during the window continue running normally + - This prevents unnecessary stop/start cycles when jobs arrive during pool top-up + +3. **Pass `POOL_STRATEGY` and `WARM_POOL_READY_DELAY_SECONDS` env vars** from Terraform to pool lambda + +### Files Modified +- `lambdas/functions/control-plane/src/pool/pool.ts` + +### Tests +- Unit test: pool prefers warm instances (hot strategy) +- Unit test: pool creates new when no warm instances available +- Unit test: warm strategy — instance stopped after grace period when idle +- Unit test: warm strategy — instance NOT stopped if it picked up a job during grace window +- Unit test: warm strategy — `ghr:warm-pool-grace-hit` tag set on grace window job pickup + +--- + +## Phase 5: Observability & Metrics + +**Goal**: Operators can monitor warm pool behavior. + +### Tasks + +1. **CloudWatch metrics** (via existing PowerTools integration): + - `WarmPoolSize` — current number of stopped warm instances + - `WarmPoolStarts` — count of instances started from warm pool + - `WarmPoolGraceHits` — count of instances that picked up a job during grace window + - `WarmPoolEvictions` — count of warm instances terminated (age/cap/AMI) + - `WarmPoolStartLatency` — time from start API call to instance running + +2. **Logging**: + - All warm pool operations logged with structured fields + - `source: 'warm-pool'` field for easy filtering + +3. **Tags for cost allocation**: + - `ghr:warm-pool=true` enables cost explorer filtering + - Operators can see EBS costs attributed to warm pool + +### Files Modified +- All lambda source files (metric emissions) +- Terraform outputs for CloudWatch dashboard (optional) + +--- + +## Phase 6: Documentation & Multi-Runner Integration + +**Goal**: Feature is documented and fully integrated with multi-runner module. + +### Tasks + +1. Update `docs/configuration.md` with warm pool section +2. Add example in `examples/multi-runner/` showing warm pool config +3. Update `modules/runners/scale-down-state-diagram.md` with new states +4. Add CHANGELOG entry + +--- + +## Sequencing & Dependencies + +``` +Phase 1 (Foundation) + ↓ +Phase 2 (Scale-Down: Stop) ← can be deployed independently + ↓ +Phase 3 (Scale-Up: Start) ← requires Phase 2 (needs warm instances to exist) + ↓ +Phase 4 (Pool: Start) ← requires Phase 2 + ↓ +Phase 5 (Observability) ← can run in parallel with Phase 3/4 + ↓ +Phase 6 (Docs) ← after all phases +``` + +## Key Design Decisions + +### Why DynamoDB instead of EC2 DescribeInstances? + +The existing codebase uses `DescribeInstances` with tag filters to discover runner state. This works for the current scale but has fundamental problems: + +| Concern | DescribeInstances | DynamoDB | +|---------|------------------|----------| +| Latency | 100-500ms+ per call | Single-digit ms | +| Rate limits | 100 calls/sec shared across all EC2 API usage | Effectively unlimited (PAY_PER_REQUEST) | +| Consistency | Eventually consistent | Strongly consistent reads available | +| Race conditions | Two lambdas can pick same instance | Conditional deletes prevent double-claim | +| Filtering | Tag filters are slow at scale | GSI gives instant owner-based lookup | +| Cost | Free (but rate-limited) | ~$1.25 per million requests | + +For the warm pool specifically, the scale-up lambda is on the **hot path** — every millisecond of delay adds to job queue time. A DynamoDB query returning available warm instances in <5ms is critical for the feature's value proposition. + +**Future migration note**: The existing `listEC2Runners()` calls in scale-down and pool lambdas also suffer from these DescribeInstances limitations. DynamoDB can eventually replace those too, with the warm pool table serving as the pattern/proof-of-concept. + +### Why deregister from GitHub before stopping? + +A stopped runner cannot respond to GitHub health checks. If left registered, GitHub would mark it offline/stale. Deregistering cleanly and re-registering on start is the correct lifecycle. + +### Why newest-first for warm pool selection? + +The most recently stopped instance has the freshest state (packages, Docker cache, etc.). Starting it minimizes stale-state risk. + +### What about spot instances in warm pool? + +The warm pool uses whatever capacity type the runner config specifies — including spot. This means stopped spot instances may be reclaimed by AWS at any time. If a warm spot instance is reclaimed, the scale-up lambda simply falls through to cold-launching a new instance. This is acceptable because: + +- The warm pool is a best-effort optimization, not a guarantee +- Most of the time, stopped spot instances are not reclaimed immediately +- The fallback (cold launch) is the same behavior as today without warm pool + +**Future enhancement**: A `warm_pool_capacity_type_override` setting will allow forcing on-demand for pool-created instances even when the runner config uses spot. This gives users cheap spot for reactive scale-up while maintaining a reliable warm pool on on-demand. + +### What about the `idle_config` interaction? + +The `idle_config` still controls how many runners remain **running** (hot) when `pool_strategy = "hot"`. With `pool_strategy = "warm"`, the `idle_config` becomes irrelevant for pool runners since none are kept running. + +Summary of interactions: + +| `pool_strategy` | `idle_config.idleCount` | `warm_pool_config.max_warm_instances` | Result | +|-----------------|------------------------|--------------------------------------|--------| +| `hot` | 2 | 3 | 2 running idle + up to 3 stopped warm | +| `warm` | _(ignored)_ | 5 | 0 running idle + up to 5 stopped warm | + +The `pool_strategy = "warm"` mode is ideal for users who: +- Want zero idle compute cost +- Accept 10-30s startup latency for the first job +- Have expensive instance types where idle cost is significant + +### What about EBS encryption? + +Stopped instances retain their encrypted EBS volumes. No change needed — encryption is controlled by the launch template and works identically for stopped instances. + +## Estimated Complexity + +| Phase | Effort | Risk | +|-------|--------|------| +| 1 - Foundation | Low | Low (config only) | +| 2 - Scale-Down | Medium | Medium (core logic change) | +| 3 - Scale-Up | Medium | Medium (startup script changes) | +| 4 - Pool | Low | Low (reuses Phase 3 logic) | +| 5 - Observability | Low | Low | +| 6 - Docs | Low | Low | diff --git a/docs/warm-pool.md b/docs/warm-pool.md new file mode 100644 index 0000000000..f3d058fed2 --- /dev/null +++ b/docs/warm-pool.md @@ -0,0 +1,201 @@ +# Warm Pool + +The warm pool feature reduces runner startup latency from 2–5 minutes to 10–30 seconds by **stopping** idle EC2 instances instead of terminating them. Stopped instances retain their EBS volume (OS, runner binary, caches) and can be restarted on demand when a new job arrives. + +## Overview + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ Instance Lifecycle │ +├─────────────┬──────────────┬────────────────────────────────────────────┤ +│ State │ Cost │ Startup Time │ +├─────────────┼──────────────┼────────────────────────────────────────────┤ +│ Hot │ Full compute│ 0s (already running, registered) │ +│ Warm │ EBS only │ 10–30s (start + re-register) │ +│ Cold │ None │ 2–5 min (launch + boot + register) │ +└─────────────┴──────────────┴────────────────────────────────────────────┘ +``` + +## Quick Start + +Enable the warm pool with spot instances for maximum cost savings: + +```hcl +multi_runner_config = { + "linux-x64" = { + runner_config = { + # ... other config ... + instance_target_capacity_type = "spot" # or "on-demand" + + warm_pool_config = { + enabled = true + max_warm_instances = 3 + max_warm_age_hours = 168 # 7 days + warm_pool_ready_delay_seconds = 30 + } + + pool_strategy = "warm" # or "hot" + } + } +} +``` + +## How It Works + +### Job Arrives → Scale-Up + +1. Scale-up lambda receives a `workflow_job` event. +2. Queries DynamoDB for available stopped instances matching the runner owner/environment. +3. If a warm instance exists: **starts** it (10–30s), tags it `ghr:started-from-warm-pool=true`. +4. If no warm instance is available: creates a new instance via the standard cold-start path. + +### Job Completes → Scale-Down + +1. Scale-down lambda detects the runner is idle and deregistered from GitHub. +2. Instead of terminating, it **stops** the instance. +3. Records the instance in the DynamoDB warm pool table (with TTL for auto-expiry). +4. If `max_warm_instances` is exceeded, oldest warm instances are terminated. + +### Pool Lambda (Warm Strategy) + +When `pool_strategy = "warm"`: + +1. Pool lambda creates instances normally. +2. After `warm_pool_ready_delay_seconds` (default 30s), checks if the runner picked up a job. +3. If still idle: deregisters from GitHub, stops the instance, adds to DynamoDB. +4. If busy: leaves it running (tags `ghr:warm-pool-grace-hit=true` for metrics). +5. Maintains a target count of **stopped** instances — zero idle compute cost. + +## Spot Instance Support + +The module fully supports spot instances with the warm pool. This provides the best cost optimization: **60–90% EC2 discount** from spot pricing + **zero compute cost** while stopped. + +### How It Works with Spot + +When both `warm_pool_config.enabled = true` and `instance_target_capacity_type = "spot"` are set, the module automatically: + +1. Uses the `RunInstances` API with **persistent spot requests** instead of the default `CreateFleet` (which creates one-time spot requests that cannot be stopped). +2. Sets `InstanceInterruptionBehavior = "stop"` so AWS stops (rather than terminates) the instance on capacity reclaim. +3. Overrides `InstanceInitiatedShutdownBehavior = "stop"` (the launch template defaults to `"terminate"`, which would conflict). + +This is fully transparent — no additional user configuration is needed. + +### Why Not CreateFleet? + +The `CreateFleet` API (used for non-warm-pool instances) always creates **one-time** spot requests. One-time spot instances: + +- Cannot be stopped (`ec2:StopInstances` returns `UnsupportedOperation`) +- Can only be terminated +- Are incompatible with warm pool stop/start cycling + +The `CreateFleet` API does not expose a `SpotInstanceType` parameter at all. The only APIs that support persistent spot are `RunInstances` and the legacy `RequestSpotInstances`. + +Additionally, `CreateFleet` with `type = "maintain"` is not suitable because: +- The fleet manages capacity automatically — if you stop an instance, the fleet detects reduced capacity and launches a replacement (the opposite of warm pool intent). + +### Trade-offs vs CreateFleet + +| Feature | CreateFleet (default) | RunInstances (persistent spot) | +|---------|----------------------|-------------------------------| +| Multi-AZ diversification | Yes (spreads across subnets) | Single subnet per instance | +| Multi-instance-type | Yes (all types tried) | First configured type only | +| Allocation strategy | Configurable (lowest-price, etc.) | N/A (single type) | +| Can be stopped | No | Yes | +| Warm pool compatible | No | Yes | + +For warm pool use cases, this trade-off is acceptable: the goal is fast restart from a known state, not placement diversity. + +### Caveats + +- **Restart not guaranteed**: When starting a stopped persistent spot instance, AWS may not have capacity at the current spot price. If the start fails, scale-up falls back to cold-launching a new instance. +- **Price fluctuations**: If the spot price has risen above your max price since the instance was stopped, the start may fail. +- **Reclamation while stopped**: AWS can terminate a stopped spot instance if capacity is needed (rare, but possible). The warm pool handles this gracefully — if an instance is gone, it's removed from DynamoDB and scale-up creates a new one. + +## Configuration Reference + +### `warm_pool_config` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `enabled` | bool | `false` | Enable the warm pool feature | +| `max_warm_instances` | number | `3` | Maximum stopped instances per runner owner | +| `max_warm_age_hours` | number | `168` | Hours before a warm instance expires (DynamoDB TTL) | +| `warm_pool_ready_delay_seconds` | number | `30` | Grace period before pool lambda stops a new instance | + +### `pool_strategy` + +| Value | Description | +|-------|-------------| +| `"hot"` (default) | Pool creates running instances. If warm pool is enabled, scale-down stops them. | +| `"warm"` | Pool creates instances then immediately stops them. Zero idle compute. Requires `warm_pool_config.enabled = true`. | + +### `instance_target_capacity_type` + +| Value | Warm Pool Behavior | +|-------|-------------------| +| `"on-demand"` | Fully reliable stop/start. No capacity concerns. | +| `"spot"` | Automatic persistent spot. Lower cost but restart subject to capacity availability. | + +## Cost Comparison + +Assuming `m5.large` in eu-west-1 (~$0.096/hr on-demand, ~$0.035/hr spot), 10GB gp3 EBS ($0.80/month): + +| Configuration | Monthly cost per idle runner | +|---|---| +| Hot pool, on-demand | ~$69 (always running) | +| Hot pool, spot | ~$25 (always running) | +| Warm pool, on-demand | ~$0.80 (EBS only) + compute while active | +| Warm pool, spot | ~$0.80 (EBS only) + compute while active | + +The warm pool is most beneficial when runners spend significant time idle between jobs. + +## Observability + +### Tags + +| Tag | When Set | Purpose | +|-----|----------|---------| +| `ghr:started-from-warm-pool=true` | Instance started from warm pool | Track warm starts in cost explorer / metrics | +| `ghr:warm-pool-grace-hit=true` | Instance picked up a job during grace window | Measure grace window effectiveness | + +### CloudWatch Metrics + +When `metrics.enable = true` and `metrics.metric.enable_warm_pool = true`: + +| Metric | Description | +|--------|-------------| +| `WarmPoolInstanceStopped` | Instance stopped and added to warm pool | +| `WarmPoolInstanceStarted` | Warm instance successfully restarted | +| `WarmPoolStartFailed` | Failed to restart a warm instance | + +### DynamoDB + +The warm pool table (`{prefix}-warm-pool`) can be queried directly for current pool state: + +```bash +aws dynamodb scan --table-name "your-prefix-warm-pool" \ + --projection-expression "instanceId, runnerOwner, stoppedAt, instanceType" +``` + +## Troubleshooting + +### Warm instances not accumulating + +- Verify `warm_pool_config.enabled = true` in your runner config. +- Check the scale-down lambda logs for stop errors. +- If using spot: confirm the lambda logs show "Created persistent spot instance" (not "Create fleet"). + +### Warm instance fails to start + +- Check EC2 console → Spot Requests for the instance's spot request status. +- If `Status = capacity-not-available`: spot capacity is exhausted. Scale-up will fall back to cold start. +- If the instance shows as "terminated": AWS reclaimed it. The DynamoDB record will be cleaned up on next access. + +### Slow restart times (>30s) + +- The runner re-registration with GitHub adds latency. This is typically 5–10s. +- If the instance was stopped for a long time, the OS may run updates on boot. Consider disabling automatic updates in your AMI. + +## Architecture Decision Record + +For full design rationale, alternatives considered, and implementation details, see [ADR-001: Warm Pool with Stop/Hibernate](adr/001-warm-pool-hibernation.md). From c617a8660ecc250ab834f890c5ec40b18ab873a4 Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 15:59:36 +0200 Subject: [PATCH 02/19] feat(warm-pool): add Terraform foundation - DynamoDB table, variables, IAM - Add warm_pool_config and pool_strategy variables to root, runners, and multi-runner modules - Create DynamoDB table with GSI by-owner and TTL for warm pool state - Add IAM policies for scale-down, scale-up, and pool lambdas (DynamoDB + EC2 stop/start) - Wire WARM_POOL_CONFIG, WARM_POOL_TABLE_NAME, POOL_STRATEGY env vars to all lambdas - Add enable_warm_pool metric toggle - Cross-variable validation: pool_strategy=warm requires warm_pool_config.enabled=true Signed-off-by: Brend Smits --- main.tf | 2 + modules/multi-runner/runners.tf | 2 + modules/multi-runner/variables.tf | 9 +++ .../runners/policies/lambda-warm-pool.json | 48 +++++++++++ modules/runners/pool.tf | 9 +++ modules/runners/pool/main.tf | 9 +++ modules/runners/pool/variables.tf | 9 +++ modules/runners/scale-down.tf | 9 +++ modules/runners/scale-up.tf | 9 +++ modules/runners/variables.tf | 45 +++++++++++ modules/runners/warm-pool.tf | 81 +++++++++++++++++++ variables.tf | 23 ++++++ 12 files changed, 255 insertions(+) create mode 100644 modules/runners/policies/lambda-warm-pool.json create mode 100644 modules/runners/warm-pool.tf diff --git a/main.tf b/main.tf index bcfa4e4032..ce13401e4e 100644 --- a/main.tf +++ b/main.tf @@ -280,6 +280,8 @@ module "runners" { pool_lambda_timeout = var.pool_lambda_timeout pool_runner_owner = var.pool_runner_owner pool_lambda_reserved_concurrent_executions = var.pool_lambda_reserved_concurrent_executions + pool_strategy = var.pool_strategy + warm_pool_config = var.warm_pool_config ssm_housekeeper = var.runners_ssm_housekeeper ebs_optimized = var.runners_ebs_optimized diff --git a/modules/multi-runner/runners.tf b/modules/multi-runner/runners.tf index 892113dcc7..99eb5ca008 100644 --- a/modules/multi-runner/runners.tf +++ b/modules/multi-runner/runners.tf @@ -120,6 +120,8 @@ module "runners" { pool_lambda_timeout = var.pool_lambda_timeout pool_runner_owner = each.value.runner_config.pool_runner_owner pool_lambda_reserved_concurrent_executions = var.pool_lambda_reserved_concurrent_executions + pool_strategy = try(each.value.runner_config.pool_strategy, "hot") + warm_pool_config = try(each.value.runner_config.warm_pool_config, { enabled = false, max_warm_instances = 3, max_warm_age_hours = 168, warm_pool_ready_delay_seconds = 30 }) associate_public_ipv4_address = var.associate_public_ipv4_address ssm_housekeeper = var.runners_ssm_housekeeper diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf index 0ff62b0083..c52c4934aa 100644 --- a/modules/multi-runner/variables.tf +++ b/modules/multi-runner/variables.tf @@ -182,6 +182,13 @@ variable "multi_runner_config" { schedule_expression_timezone = optional(string) size = number })), []) + pool_strategy = optional(string, "hot") + warm_pool_config = optional(object({ + enabled = optional(bool, false) + max_warm_instances = optional(number, 3) + max_warm_age_hours = optional(number, 168) + warm_pool_ready_delay_seconds = optional(number, 30) + }), {}) job_retry = optional(object({ enable = optional(bool, false) delay_in_seconds = optional(number, 300) @@ -279,6 +286,8 @@ variable "multi_runner_config" { block_device_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`." job_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app." pool_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)." + pool_strategy: "Strategy for maintaining idle runners: `hot` (default, traditional) or `warm` (stop instead of terminate)." + warm_pool_config: "Configuration for the warm pool feature. When enabled, idle runners are stopped instead of terminated for faster restart." iam_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances." } matcherConfig: { diff --git a/modules/runners/policies/lambda-warm-pool.json b/modules/runners/policies/lambda-warm-pool.json new file mode 100644 index 0000000000..d14c504acd --- /dev/null +++ b/modules/runners/policies/lambda-warm-pool.json @@ -0,0 +1,48 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "dynamodb:PutItem", + "dynamodb:DeleteItem", + "dynamodb:GetItem", + "dynamodb:Query" + ], + "Resource": [ + "${dynamodb_table_arn}", + "${dynamodb_table_arn}/index/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "ec2:StopInstances", + "ec2:StartInstances", + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Resource": "*", + "Condition": { + "StringEquals": { + "ec2:ResourceTag/ghr:Application": "github-action-runner" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:StopInstances", + "ec2:StartInstances", + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Resource": "*", + "Condition": { + "StringEquals": { + "ec2:ResourceTag/gh:environment": "${environment}" + } + } + } + ] +} diff --git a/modules/runners/pool.tf b/modules/runners/pool.tf index d9899fd21c..e8410c5373 100644 --- a/modules/runners/pool.tf +++ b/modules/runners/pool.tf @@ -64,6 +64,15 @@ module "pool" { tags = local.tags lambda_tags = var.lambda_tags arn_ssm_parameters_path_config = local.arn_ssm_parameters_path_config + warm_pool_table_name = var.warm_pool_config.enabled ? aws_dynamodb_table.warm_pool[0].name : "" + warm_pool_config = { + enabled = var.warm_pool_config.enabled + max_warm_instances = var.warm_pool_config.max_warm_instances + max_warm_age_hours = var.warm_pool_config.max_warm_age_hours + warm_pool_ready_delay_seconds = var.warm_pool_config.warm_pool_ready_delay_seconds + } + pool_strategy = var.pool_strategy + enable_metric_warm_pool = var.metrics.enable && var.metrics.metric.enable_warm_pool } aws_partition = var.aws_partition diff --git a/modules/runners/pool/main.tf b/modules/runners/pool/main.tf index efdf537a96..d4d9e79153 100644 --- a/modules/runners/pool/main.tf +++ b/modules/runners/pool/main.tf @@ -60,6 +60,15 @@ resource "aws_lambda_function" "pool" { SSM_PARAMETER_STORE_TAGS = var.config.lambda.parameter_store_tags SCALE_ERRORS = jsonencode(var.config.runner.scale_errors) USE_DEDICATED_HOST = var.config.runner.use_dedicated_host + WARM_POOL_CONFIG = jsonencode({ + enabled = var.config.warm_pool_config.enabled + maxWarmInstances = var.config.warm_pool_config.max_warm_instances + maxWarmAgeHours = var.config.warm_pool_config.max_warm_age_hours + warmPoolReadyDelaySeconds = var.config.warm_pool_config.warm_pool_ready_delay_seconds + }) + WARM_POOL_TABLE_NAME = var.config.warm_pool_table_name + POOL_STRATEGY = var.config.pool_strategy + ENABLE_METRIC_WARM_POOL = var.config.enable_metric_warm_pool } } diff --git a/modules/runners/pool/variables.tf b/modules/runners/pool/variables.tf index 2666444ae9..bbf361fbcb 100644 --- a/modules/runners/pool/variables.tf +++ b/modules/runners/pool/variables.tf @@ -72,6 +72,15 @@ variable "config" { arn_ssm_parameters_path_config = string lambda_tags = map(string) user_agent = string + warm_pool_table_name = string + warm_pool_config = object({ + enabled = bool + max_warm_instances = number + max_warm_age_hours = number + warm_pool_ready_delay_seconds = number + }) + pool_strategy = string + enable_metric_warm_pool = bool }) } diff --git a/modules/runners/scale-down.tf b/modules/runners/scale-down.tf index 449d1970ed..d5c841a47d 100644 --- a/modules/runners/scale-down.tf +++ b/modules/runners/scale-down.tf @@ -27,6 +27,7 @@ resource "aws_lambda_function" "scale_down" { variables = { ENVIRONMENT = var.prefix ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.metrics.enable && var.metrics.metric.enable_github_app_rate_limit + ENABLE_METRIC_WARM_POOL = var.metrics.enable && var.metrics.metric.enable_warm_pool GHES_URL = var.ghes_url USER_AGENT = var.user_agent LOG_LEVEL = var.log_level @@ -42,6 +43,14 @@ resource "aws_lambda_function" "scale_down" { POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error + WARM_POOL_CONFIG = jsonencode({ + enabled = var.warm_pool_config.enabled + maxWarmInstances = var.warm_pool_config.max_warm_instances + maxWarmAgeHours = var.warm_pool_config.max_warm_age_hours + warmPoolReadyDelaySeconds = var.warm_pool_config.warm_pool_ready_delay_seconds + }) + WARM_POOL_TABLE_NAME = var.warm_pool_config.enabled ? aws_dynamodb_table.warm_pool[0].name : "" + POOL_STRATEGY = var.pool_strategy } } diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf index cbc9beaa64..638072615a 100644 --- a/modules/runners/scale-up.tf +++ b/modules/runners/scale-up.tf @@ -64,6 +64,15 @@ resource "aws_lambda_function" "scale_up" { SCALE_ERRORS = jsonencode(var.scale_errors) JOB_RETRY_CONFIG = jsonencode(local.job_retry_config) USE_DEDICATED_HOST = var.use_dedicated_host + ENABLE_METRIC_WARM_POOL = var.metrics.enable && var.metrics.metric.enable_warm_pool + WARM_POOL_CONFIG = jsonencode({ + enabled = var.warm_pool_config.enabled + maxWarmInstances = var.warm_pool_config.max_warm_instances + maxWarmAgeHours = var.warm_pool_config.max_warm_age_hours + warmPoolReadyDelaySeconds = var.warm_pool_config.warm_pool_ready_delay_seconds + }) + WARM_POOL_TABLE_NAME = var.warm_pool_config.enabled ? aws_dynamodb_table.warm_pool[0].name : "" + POOL_STRATEGY = var.pool_strategy } } diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index 01f2e2834c..14b94e61a3 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -796,6 +796,7 @@ variable "metrics" { enable_github_app_rate_limit = optional(bool, true) enable_job_retry = optional(bool, true) enable_spot_termination_warning = optional(bool, true) + enable_warm_pool = optional(bool, true) }), {}) }) default = {} @@ -870,3 +871,47 @@ variable "use_dedicated_host" { type = bool default = false } + +variable "warm_pool_config" { + description = <<-EOF + Configuration for the warm pool feature. When enabled, idle runners are stopped instead of terminated, + allowing 10-30s restart times instead of 2-5 minute cold starts. + + `enabled`: Enable or disable the warm pool feature. When enabled, a DynamoDB table is created to track stopped instances. + `max_warm_instances`: Maximum number of stopped instances to keep in the warm pool per runner owner. + `max_warm_age_hours`: Maximum age in hours before a warm instance is terminated by TTL. + `warm_pool_ready_delay_seconds`: Grace period in seconds before a newly launched instance is eligible for warm pool (allows time to pick up jobs). + EOF + type = object({ + enabled = optional(bool, false) + max_warm_instances = optional(number, 3) + max_warm_age_hours = optional(number, 168) + warm_pool_ready_delay_seconds = optional(number, 30) + }) + default = {} + + validation { + condition = var.warm_pool_config.max_warm_instances >= 1 + error_message = "max_warm_instances must be at least 1." + } + + validation { + condition = var.warm_pool_config.max_warm_age_hours >= 1 + error_message = "max_warm_age_hours must be at least 1 hour." + } +} + +variable "pool_strategy" { + description = <<-EOF + Strategy for maintaining idle runners. + - `hot`: (default) Traditional behavior. Pool lambda creates fresh instances on a schedule. Idle instances are terminated by scale-down. + - `warm`: Idle instances are stopped (hibernated) instead of terminated. Scale-up will restart warm instances before creating new ones. Requires `warm_pool_config.enabled = true`. + EOF + type = string + default = "hot" + + validation { + condition = contains(["hot", "warm"], var.pool_strategy) + error_message = "pool_strategy must be either \"hot\" or \"warm\"." + } +} diff --git a/modules/runners/warm-pool.tf b/modules/runners/warm-pool.tf new file mode 100644 index 0000000000..a02e9b356d --- /dev/null +++ b/modules/runners/warm-pool.tf @@ -0,0 +1,81 @@ +# Warm Pool DynamoDB table and IAM policies +# Only created when warm_pool_config.enabled = true + +# Cross-variable validation +check "warm_pool_strategy_requires_enabled" { + assert { + condition = var.pool_strategy != "warm" || var.warm_pool_config.enabled + error_message = "pool_strategy = \"warm\" requires warm_pool_config.enabled = true." + } +} + +resource "aws_dynamodb_table" "warm_pool" { + count = var.warm_pool_config.enabled ? 1 : 0 + + name = "${var.prefix}-warm-pool" + billing_mode = "PAY_PER_REQUEST" + hash_key = "instanceId" + + attribute { + name = "instanceId" + type = "S" + } + + attribute { + name = "runnerOwner" + type = "S" + } + + attribute { + name = "stoppedAt" + type = "S" + } + + global_secondary_index { + name = "by-owner" + hash_key = "runnerOwner" + range_key = "stoppedAt" + projection_type = "ALL" + } + + ttl { + attribute_name = "expiresAt" + enabled = true + } + + tags = local.tags +} + +# IAM policy for warm pool operations (DynamoDB + EC2 stop/start) +resource "aws_iam_role_policy" "scale_down_warm_pool" { + count = var.warm_pool_config.enabled ? 1 : 0 + + name = "warm-pool-policy" + role = aws_iam_role.scale_down.name + policy = templatefile("${path.module}/policies/lambda-warm-pool.json", { + dynamodb_table_arn = aws_dynamodb_table.warm_pool[0].arn + environment = var.prefix + }) +} + +resource "aws_iam_role_policy" "scale_up_warm_pool" { + count = var.warm_pool_config.enabled ? 1 : 0 + + name = "warm-pool-policy" + role = aws_iam_role.scale_up.name + policy = templatefile("${path.module}/policies/lambda-warm-pool.json", { + dynamodb_table_arn = aws_dynamodb_table.warm_pool[0].arn + environment = var.prefix + }) +} + +resource "aws_iam_role_policy" "pool_warm_pool" { + count = var.warm_pool_config.enabled && length(var.pool_config) > 0 ? 1 : 0 + + name = "warm-pool-policy" + role = module.pool[0].role_pool.name + policy = templatefile("${path.module}/policies/lambda-warm-pool.json", { + dynamodb_table_arn = aws_dynamodb_table.warm_pool[0].arn + environment = var.prefix + }) +} diff --git a/variables.tf b/variables.tf index b7081761a5..3bb9ecc55f 100644 --- a/variables.tf +++ b/variables.tf @@ -847,6 +847,28 @@ variable "pool_config" { default = [] } +variable "pool_strategy" { + description = "Strategy for maintaining idle runners: `hot` (default, traditional) or `warm` (stop instead of terminate). Requires `warm_pool_config.enabled = true` when set to `warm`." + type = string + default = "hot" + + validation { + condition = contains(["hot", "warm"], var.pool_strategy) + error_message = "pool_strategy must be either \"hot\" or \"warm\"." + } +} + +variable "warm_pool_config" { + description = "Configuration for the warm pool feature. When enabled, idle runners are stopped instead of terminated, allowing 10-30s restart times instead of 2-5 minute cold starts." + type = object({ + enabled = optional(bool, false) + max_warm_instances = optional(number, 3) + max_warm_age_hours = optional(number, 168) + warm_pool_ready_delay_seconds = optional(number, 30) + }) + default = {} +} + variable "aws_partition" { description = "(optiona) partition in the arn namespace to use if not 'aws'" type = string @@ -1047,6 +1069,7 @@ variable "metrics" { enable_github_app_rate_limit = optional(bool, true) enable_job_retry = optional(bool, true) enable_spot_termination_warning = optional(bool, true) + enable_warm_pool = optional(bool, true) }), {}) }) default = {} From 996abcca405972bbe2e235bbbb6e0b0ffa67f037 Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 16:01:04 +0200 Subject: [PATCH 03/19] feat(warm-pool): add DynamoDB client and EC2 stop/start functions - Add warm-pool.ts with DynamoDB operations (add, remove, get, list, count) - Add stopRunner() and startRunner() to runners.ts - Add emitWarmPoolMetric() for CloudWatch metric publishing - Export getWarmPoolConfig() and getPoolStrategy() helpers Signed-off-by: Brend Smits --- .../control-plane/src/aws/runners.ts | 16 ++ .../control-plane/src/aws/warm-pool.ts | 158 ++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 lambdas/functions/control-plane/src/aws/warm-pool.ts diff --git a/lambdas/functions/control-plane/src/aws/runners.ts b/lambdas/functions/control-plane/src/aws/runners.ts index 799bc0d4a2..869ffd562d 100644 --- a/lambdas/functions/control-plane/src/aws/runners.ts +++ b/lambdas/functions/control-plane/src/aws/runners.ts @@ -13,6 +13,8 @@ import { FleetLaunchTemplateOverridesRequest, FleetOnDemandAllocationStrategy, SpotAllocationStrategy, + StartInstancesCommand, + StopInstancesCommand, Tag, TerminateInstancesCommand, _InstanceType, @@ -115,6 +117,20 @@ export async function terminateRunner(instanceId: string): Promise { logger.debug(`Runner ${instanceId} has been terminated.`); } +export async function stopRunner(instanceId: string): Promise { + logger.info(`Runner '${instanceId}' will be stopped.`); + const ec2 = getTracedAWSV3Client(new EC2Client({ region: process.env.AWS_REGION })); + await ec2.send(new StopInstancesCommand({ InstanceIds: [instanceId] })); + logger.info(`Runner '${instanceId}' has been stopped.`); +} + +export async function startRunner(instanceId: string): Promise { + logger.info(`Runner '${instanceId}' will be started.`); + const ec2 = getTracedAWSV3Client(new EC2Client({ region: process.env.AWS_REGION })); + await ec2.send(new StartInstancesCommand({ InstanceIds: [instanceId] })); + logger.info(`Runner '${instanceId}' has been started.`); +} + export async function tag(instanceId: string, tags: Tag[]): Promise { logger.debug(`Tagging '${instanceId}'`, { tags }); const ec2 = getTracedAWSV3Client(new EC2Client({ region: process.env.AWS_REGION })); diff --git a/lambdas/functions/control-plane/src/aws/warm-pool.ts b/lambdas/functions/control-plane/src/aws/warm-pool.ts new file mode 100644 index 0000000000..775beb06e1 --- /dev/null +++ b/lambdas/functions/control-plane/src/aws/warm-pool.ts @@ -0,0 +1,158 @@ +import { + DynamoDBClient, + PutItemCommand, + DeleteItemCommand, + QueryCommand, + GetItemCommand, +} from '@aws-sdk/client-dynamodb'; +import { createChildLogger, createSingleMetric } from '@aws-github-runner/aws-powertools-util'; +import { getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; +import { MetricUnit } from '@aws-lambda-powertools/metrics'; +import yn from 'yn'; + +const logger = createChildLogger('warm-pool'); + +export interface WarmPoolConfig { + enabled: boolean; + maxWarmInstances: number; + maxWarmAgeHours: number; + warmPoolReadyDelaySeconds: number; +} + +export interface WarmPoolEntry { + instanceId: string; + runnerOwner: string; + environment: string; + runnerType: string; + instanceType?: string; + az?: string; + stoppedAt: string; + expiresAt: number; +} + +function getClient(): DynamoDBClient { + return getTracedAWSV3Client(new DynamoDBClient({ region: process.env.AWS_REGION })); +} + +function getTableName(): string { + return process.env.WARM_POOL_TABLE_NAME || ''; +} + +export function getWarmPoolConfig(): WarmPoolConfig { + const raw = process.env.WARM_POOL_CONFIG; + if (!raw) { + return { enabled: false, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }; + } + return JSON.parse(raw) as WarmPoolConfig; +} + +export function getPoolStrategy(): string { + return process.env.POOL_STRATEGY || 'hot'; +} + +export async function addToWarmPool(entry: Omit): Promise { + const config = getWarmPoolConfig(); + const tableName = getTableName(); + const now = new Date(); + const stoppedAt = now.toISOString(); + const expiresAt = Math.floor(now.getTime() / 1000) + config.maxWarmAgeHours * 3600; + + const client = getClient(); + await client.send( + new PutItemCommand({ + TableName: tableName, + Item: { + instanceId: { S: entry.instanceId }, + runnerOwner: { S: entry.runnerOwner }, + environment: { S: entry.environment }, + runnerType: { S: entry.runnerType }, + ...(entry.instanceType && { instanceType: { S: entry.instanceType } }), + ...(entry.az && { az: { S: entry.az } }), + stoppedAt: { S: stoppedAt }, + expiresAt: { N: String(expiresAt) }, + }, + }), + ); + logger.info(`Added instance '${entry.instanceId}' to warm pool for owner '${entry.runnerOwner}'`); +} + +export async function removeFromWarmPool(instanceId: string): Promise { + const client = getClient(); + await client.send( + new DeleteItemCommand({ + TableName: getTableName(), + Key: { instanceId: { S: instanceId } }, + }), + ); + logger.info(`Removed instance '${instanceId}' from warm pool`); +} + +export async function getWarmInstance(instanceId: string): Promise { + const client = getClient(); + const result = await client.send( + new GetItemCommand({ + TableName: getTableName(), + Key: { instanceId: { S: instanceId } }, + }), + ); + if (!result.Item) return null; + return itemToEntry(result.Item); +} + +export async function listWarmInstancesByOwner(runnerOwner: string): Promise { + const client = getClient(); + const result = await client.send( + new QueryCommand({ + TableName: getTableName(), + IndexName: 'by-owner', + KeyConditionExpression: 'runnerOwner = :owner', + ExpressionAttributeValues: { + ':owner': { S: runnerOwner }, + }, + ScanIndexForward: true, // oldest first + }), + ); + return (result.Items || []).map(itemToEntry); +} + +export async function countWarmInstancesByOwner(runnerOwner: string): Promise { + const client = getClient(); + const result = await client.send( + new QueryCommand({ + TableName: getTableName(), + IndexName: 'by-owner', + KeyConditionExpression: 'runnerOwner = :owner', + ExpressionAttributeValues: { + ':owner': { S: runnerOwner }, + }, + Select: 'COUNT', + }), + ); + return result.Count || 0; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function itemToEntry(item: any): WarmPoolEntry { + return { + instanceId: item.instanceId.S, + runnerOwner: item.runnerOwner.S, + environment: item.environment.S, + runnerType: item.runnerType.S, + instanceType: item.instanceType?.S, + az: item.az?.S, + stoppedAt: item.stoppedAt.S, + expiresAt: Number(item.expiresAt.N), + }; +} + +export function emitWarmPoolMetric( + metricName: 'WarmPoolInstanceStopped' | 'WarmPoolInstanceStarted' | 'WarmPoolStartFailed' | 'WarmPoolSize', + value: number, + dimensions: Record = {}, +): void { + const enabled = yn(process.env.ENABLE_METRIC_WARM_POOL, { default: false }); + if (!enabled) return; + + const environment = process.env.ENVIRONMENT || ''; + createSingleMetric(metricName, MetricUnit.Count, value, { Environment: environment, ...dimensions }); +} From cd63be58a44963caf89b2af56f19013d8eb3b605 Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 16:02:25 +0200 Subject: [PATCH 04/19] feat(warm-pool): implement scale-down stop logic and warm pool eviction - Modify removeRunner() to stop instances into warm pool when enabled - Add DynamoDB record on stop with ghr:warm-pool-member tag - Respect maxWarmInstances cap (terminate if full) - Add evictStaleWarmInstances() for periodic cleanup: - Evict instances exceeding maxWarmAgeHours - Evict instances exceeding maxWarmInstances count - Emit WarmPoolSize metric after eviction Signed-off-by: Brend Smits --- .../src/scale-runners/scale-down.ts | 97 ++++++++++++++++++- 1 file changed, 94 insertions(+), 3 deletions(-) diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index 5fecc14c99..9eb1a896fa 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -5,12 +5,21 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import moment from 'moment'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; -import { bootTimeExceeded, listEC2Runners, tag, untag, terminateRunner } from './../aws/runners'; +import { bootTimeExceeded, listEC2Runners, tag, untag, terminateRunner, stopRunner } from './../aws/runners'; import { RunnerInfo, RunnerList } from './../aws/runners.d'; import { GhRunners, githubCache } from './cache'; import { ScalingDownConfig, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config'; import { metricGitHubAppRateLimit } from '../github/rate-limit'; import { getGitHubEnterpriseApiUrl } from './scale-up'; +import { + addToWarmPool, + countWarmInstancesByOwner, + listWarmInstancesByOwner, + removeFromWarmPool, + getWarmPoolConfig, + getPoolStrategy, + emitWarmPoolMetric, +} from '../aws/warm-pool'; const logger = createChildLogger('scale-down'); @@ -188,8 +197,38 @@ async function removeRunner(ec2runner: RunnerInfo, ghRunnerIds: number[]): Promi const failedRunners = results.filter((r) => !r.success); if (allSucceeded) { - await terminateRunner(ec2runner.instanceId); - logger.info(`AWS runner instance '${ec2runner.instanceId}' is terminated and GitHub runner is de-registered.`); + const warmPoolConfig = getWarmPoolConfig(); + const poolStrategy = getPoolStrategy(); + + if (warmPoolConfig.enabled && poolStrategy === 'warm') { + const warmCount = await countWarmInstancesByOwner(ec2runner.owner); + if (warmCount < warmPoolConfig.maxWarmInstances) { + await stopRunner(ec2runner.instanceId); + await addToWarmPool({ + instanceId: ec2runner.instanceId, + runnerOwner: ec2runner.owner, + environment: process.env.ENVIRONMENT || '', + runnerType: ec2runner.type, + }); + await tag(ec2runner.instanceId, [{ Key: 'ghr:warm-pool-member', Value: 'true' }]); + emitWarmPoolMetric('WarmPoolInstanceStopped', 1, { Owner: ec2runner.owner }); + logger.info( + `Runner '${ec2runner.instanceId}' stopped and added to warm pool ` + + `(${warmCount + 1}/${warmPoolConfig.maxWarmInstances}).`, + ); + } else { + await terminateRunner(ec2runner.instanceId); + logger.info( + `Runner '${ec2runner.instanceId}' terminated (warm pool full: ` + + `${warmCount}/${warmPoolConfig.maxWarmInstances}).`, + ); + } + } else { + await terminateRunner(ec2runner.instanceId); + logger.info( + `AWS runner instance '${ec2runner.instanceId}' is terminated and GitHub runner is de-registered.`, + ); + } } else { // Only terminate EC2 if we successfully de-registered from GitHub // Otherwise, leave the instance running so the next scale-down cycle can retry @@ -352,6 +391,55 @@ function filterRunners(ec2runners: RunnerList[]): RunnerInfo[] { return ec2runners.filter((ec2Runner) => ec2Runner.type && !ec2Runner.orphan) as RunnerInfo[]; } +async function evictStaleWarmInstances(environment: string): Promise { + const warmPoolConfig = getWarmPoolConfig(); + if (!warmPoolConfig.enabled) return; + + const ownerTags = new Set(); + const ec2runners = await listEC2Runners({ environment }); + for (const runner of ec2runners) { + if (runner.owner) ownerTags.add(runner.owner); + } + + for (const owner of ownerTags) { + try { + const warmInstances = await listWarmInstancesByOwner(owner); + if (warmInstances.length === 0) continue; + + const now = Date.now() / 1000; + let evictedCount = 0; + + for (const entry of warmInstances) { + const ageHours = (now - new Date(entry.stoppedAt).getTime() / 1000) / 3600; + const exceedsAge = ageHours > warmPoolConfig.maxWarmAgeHours; + const exceedsCount = warmInstances.length - evictedCount > warmPoolConfig.maxWarmInstances; + + if (exceedsAge || exceedsCount) { + try { + await terminateRunner(entry.instanceId); + await removeFromWarmPool(entry.instanceId); + evictedCount++; + logger.info( + `Evicted warm instance '${entry.instanceId}' (age: ${ageHours.toFixed(1)}h, ` + + `reason: ${exceedsAge ? 'max_age_exceeded' : 'max_count_exceeded'}).`, + ); + } catch (e) { + logger.warn(`Failed to evict warm instance '${entry.instanceId}'`, { error: e }); + // Remove stale DynamoDB record anyway if EC2 termination fails (instance may already be gone) + await removeFromWarmPool(entry.instanceId).catch(() => {}); + } + } + } + + if (evictedCount > 0) { + emitWarmPoolMetric('WarmPoolSize', warmInstances.length - evictedCount, { Owner: owner }); + } + } catch (e) { + logger.warn(`Failed to process warm pool eviction for owner '${owner}'`, { error: e }); + } + } +} + export async function scaleDown(): Promise { githubCache.reset(); const environment = process.env.ENVIRONMENT; @@ -374,6 +462,9 @@ export async function scaleDown(): Promise { const runners = filterRunners(ec2Runners); await evaluateAndRemoveRunners(runners, scaleDownConfigs); + // Evict warm pool instances that exceed age or count limits + await evictStaleWarmInstances(environment); + const activeEc2RunnersCountAfter = (await listRunners(environment)).length; logger.info(`Found: '${activeEc2RunnersCountAfter}' active GitHub EC2 runners instances after clean-up.`); } From 2a7968306ef8e2af598255cc467ab955b68a4304 Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 16:04:46 +0200 Subject: [PATCH 05/19] feat(warm-pool): implement scale-up warm instance restart - Add findAndStartWarmRunners() to query DynamoDB and start stopped instances - Integrate warm pool lookup before cold-launching new runners - Handle org-level fallback when repo-level owner has no warm instances - Write runner registration config (SSM) for restarted warm instances - Tag started instances with ghr:started-from-warm-pool, remove ghr:warm-pool-member - Emit WarmPoolInstanceStarted/WarmPoolStartFailed metrics - Self-heal stale DynamoDB records on start failure Signed-off-by: Brend Smits --- .../src/scale-runners/scale-up.ts | 140 +++++++++++++++--- 1 file changed, 118 insertions(+), 22 deletions(-) diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 3340c9a865..c7e3740ae0 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -8,10 +8,17 @@ import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util'; import yn from 'yn'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; -import { createRunner, listEC2Runners, tag, terminateRunner } from './../aws/runners'; +import { createRunner, listEC2Runners, startRunner, tag, untag, terminateRunner } from './../aws/runners'; import { Ec2OverrideConfig, RunnerInputParameters } from './../aws/runners.d'; import { metricGitHubAppRateLimit } from '../github/rate-limit'; import { publishRetryMessage } from './job-retry'; +import { + getWarmPoolConfig, + getPoolStrategy, + listWarmInstancesByOwner, + removeFromWarmPool, + emitWarmPoolMetric, +} from '../aws/warm-pool'; import { _InstanceType, Tenancy, @@ -298,6 +305,64 @@ async function getRunnerGroupByName(ghClient: Octokit, githubRunnerConfig: Creat return runnerGroupId; } +export async function findAndStartWarmRunners( + runnerOwner: string, + count: number, + githubRunnerConfig?: CreateGitHubRunnerConfig, + ghClient?: Octokit, +): Promise { + const warmPoolConfig = getWarmPoolConfig(); + const poolStrategy = getPoolStrategy(); + + if (!warmPoolConfig.enabled || poolStrategy !== 'warm' || count <= 0) { + return []; + } + + let warmInstances = await listWarmInstancesByOwner(runnerOwner); + // If no warm instances found and owner contains a repo (org/repo), try org-level lookup + if (warmInstances.length === 0 && runnerOwner.includes('/')) { + const orgOwner = runnerOwner.split('/')[0]; + warmInstances = await listWarmInstancesByOwner(orgOwner); + if (warmInstances.length > 0) { + logger.info(`Found ${warmInstances.length} warm instances under org owner '${orgOwner}'`); + } + } + + const startedInstances: string[] = []; + + for (const entry of warmInstances) { + if (startedInstances.length >= count) break; + + try { + await startRunner(entry.instanceId); + await removeFromWarmPool(entry.instanceId); + startedInstances.push(entry.instanceId); + emitWarmPoolMetric('WarmPoolInstanceStarted', 1, { Owner: runnerOwner }); + logger.info(`Started warm instance '${entry.instanceId}' for owner '${runnerOwner}'`); + + // Observability tags (best-effort) + await Promise.all([ + tag(entry.instanceId, [{ Key: 'ghr:started-from-warm-pool', Value: 'true' }]), + untag(entry.instanceId, [{ Key: 'ghr:warm-pool-member' }]), + ]).catch((e) => { + logger.warn(`Failed to update tags on '${entry.instanceId}', continuing`, { error: e }); + }); + } catch (e) { + logger.warn(`Failed to start warm instance '${entry.instanceId}', skipping`, { error: e as Error }); + emitWarmPoolMetric('WarmPoolStartFailed', 1, { Owner: runnerOwner }); + // Remove stale DynamoDB record — instance may already be terminated + await removeFromWarmPool(entry.instanceId).catch(() => {}); + } + } + + // Write runner config (registration tokens) for started warm instances + if (startedInstances.length > 0 && githubRunnerConfig && ghClient) { + await createStartRunnerConfig(githubRunnerConfig, startedInstances, ghClient); + } + + return startedInstances; +} + export async function createRunners( githubRunnerConfig: CreateGitHubRunnerConfig, ec2RunnerConfig: CreateEC2RunnerConfig, @@ -612,7 +677,10 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise 0) { + logger.info(`Started ${warmInstances.length} warm runners, need ${remainingRunners} more from cold start`); + } + + let instances: string[] = [...warmInstances]; + + if (remainingRunners > 0) { + const coldInstances = await createRunners( + { + ephemeral: ephemeralEnabled, + enableJitConfig, + ghesBaseUrl, + runnerLabels: groupRunnerLabels, + runnerGroup, + runnerNamePrefix, + runnerOwner: runnerOwner, + runnerType, + disableAutoUpdate, + ssmTokenPath, + ssmConfigPath, + ssmParameterStoreTags, + }, + { + ec2instanceCriteria: { + instanceTypes, + instanceTypePriorities, + targetCapacityType: instanceTargetCapacityType, + maxSpotPrice: instanceMaxSpotPrice, + instanceAllocationStrategy: instanceAllocationStrategy, + }, + ec2OverrideConfig, + environment, + launchTemplateName, + subnets, + amiIdSsmParameterName, + tracingEnabled, + onDemandFailoverOnError, + scaleErrors, + useDedicatedHost, + }, + remainingRunners, + githubInstallationClient, + 'scale-up-lambda', + ); + instances = [...instances, ...coldInstances]; + } // Not all runners we wanted were created, let's reject enough items so that // number of entries will be retried. From f411c77d3be44e8a9f0749ad5bc50e1b03aa2d7b Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 16:05:51 +0200 Subject: [PATCH 06/19] feat(warm-pool): implement pool lambda warm strategy - Pool counts warm (stopped) instances toward pool target when pool_strategy=warm - Pool tries to start warm instances before cold-launching new ones - Remaining deficit is filled by createRunners (cold start) - Scale-down will stop excess idle runners into warm pool after they run Signed-off-by: Brend Smits --- .../functions/control-plane/src/pool/pool.ts | 47 +++++++++++++++++-- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index ccc6d93e26..daa86d2f66 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -5,8 +5,9 @@ import yn from 'yn'; import { bootTimeExceeded, listEC2Runners } from '../aws/runners'; import { RunnerList } from '../aws/runners.d'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; -import { createRunners, getGitHubEnterpriseApiUrl } from '../scale-runners/scale-up'; +import { createRunners, findAndStartWarmRunners, getGitHubEnterpriseApiUrl } from '../scale-runners/scale-up'; import { validateSsmParameterStoreTags } from '../scale-runners/scale-up'; +import { getPoolStrategy, getWarmPoolConfig, countWarmInstancesByOwner } from '../aws/warm-pool'; const logger = createChildLogger('pool'); @@ -76,7 +77,18 @@ export async function adjust(event: PoolEvent): Promise { }); const numberOfRunnersInPool = calculatePooSize(ec2runners, runnerStatusses); - let topUp = event.poolSize - numberOfRunnersInPool; + const poolStrategy = getPoolStrategy(); + const warmPoolConfig = getWarmPoolConfig(); + + // For warm strategy, count warm (stopped) instances toward pool target + let effectivePoolSize = numberOfRunnersInPool; + if (poolStrategy === 'warm' && warmPoolConfig.enabled) { + const warmCount = await countWarmInstancesByOwner(runnerOwner); + effectivePoolSize = numberOfRunnersInPool + warmCount; + logger.info(`Warm strategy: ${numberOfRunnersInPool} running idle + ${warmCount} warm stopped = ${effectivePoolSize} effective pool size`); + } + + let topUp = event.poolSize - effectivePoolSize; // The pool must never push the total number of runners (busy + idle) past the configured maximum. // ec2runners contains every running runner for this type, so its length is the current total and no @@ -95,7 +107,31 @@ export async function adjust(event: PoolEvent): Promise { if (topUp > 0) { logger.info(`The pool will be topped up with ${topUp} runners.`); - await createRunners( + + // Try warm instances first (applies to both hot and warm strategies) + const warmRunnerConfig = { + ephemeral, + enableJitConfig, + ghesBaseUrl, + runnerLabels, + runnerGroup, + runnerNamePrefix, + runnerOwner, + runnerType: 'Org' as const, + disableAutoUpdate, + ssmTokenPath, + ssmConfigPath, + ssmParameterStoreTags, + }; + const warmInstances = await findAndStartWarmRunners(runnerOwner, topUp, warmRunnerConfig, githubInstallationClient); + const remainingTopUp = topUp - warmInstances.length; + + if (warmInstances.length > 0) { + logger.info(`Started ${warmInstances.length} warm runners for pool, need ${remainingTopUp} more from cold start`); + } + + if (remainingTopUp > 0) { + await createRunners( { ephemeral, enableJitConfig, @@ -126,12 +162,13 @@ export async function adjust(event: PoolEvent): Promise { onDemandFailoverOnError, scaleErrors, }, - topUp, + remainingTopUp, githubInstallationClient, 'pool-lambda', ); + } } else { - logger.info(`Pool will not be topped up. Found ${numberOfRunnersInPool} managed idle runners.`); + logger.info(`Pool will not be topped up. Found ${effectivePoolSize} effective pool runners (${numberOfRunnersInPool} running + warm).`); } } From 423e4e6f4ce18b59623b0fc54b9035b73f5f6276 Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 16:06:47 +0200 Subject: [PATCH 07/19] test(warm-pool): add unit tests for findAndStartWarmRunners - Test disabled/hot strategy returns empty - Test starting and tagging warm instances - Test failure handling (skip failed, cleanup DynamoDB) - Test org-level fallback for repo-level owners - Test multiple instance start up to count - Test tag failure as non-fatal (best-effort) Signed-off-by: Brend Smits --- .../find-and-start-warm-runners.test.ts | 287 ++++++++++++++++++ 1 file changed, 287 insertions(+) create mode 100644 lambdas/functions/control-plane/src/scale-runners/find-and-start-warm-runners.test.ts diff --git a/lambdas/functions/control-plane/src/scale-runners/find-and-start-warm-runners.test.ts b/lambdas/functions/control-plane/src/scale-runners/find-and-start-warm-runners.test.ts new file mode 100644 index 0000000000..ce239c77b1 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/find-and-start-warm-runners.test.ts @@ -0,0 +1,287 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { startRunner, tag, untag } from './../aws/runners'; +import { + getWarmPoolConfig, + getPoolStrategy, + listWarmInstancesByOwner, + removeFromWarmPool, + emitWarmPoolMetric, +} from '../aws/warm-pool'; +import { findAndStartWarmRunners } from './scale-up'; + +vi.mock('./../aws/runners', async () => ({ + createRunner: vi.fn(), + listEC2Runners: vi.fn(), + startRunner: vi.fn(), + tag: vi.fn(), + untag: vi.fn(), + terminateRunner: vi.fn(), +})); + +vi.mock('../aws/warm-pool', async () => ({ + getWarmPoolConfig: vi.fn(), + getPoolStrategy: vi.fn(), + listWarmInstancesByOwner: vi.fn(), + removeFromWarmPool: vi.fn(), + emitWarmPoolMetric: vi.fn(), + addToWarmPool: vi.fn(), + countWarmInstancesByOwner: vi.fn(), +})); + +vi.mock('./../github/auth', async () => ({ + createGithubAppAuth: vi.fn(), + createGithubInstallationAuth: vi.fn(), + createOctokitClient: vi.fn(), +})); + +vi.mock('@aws-github-runner/aws-ssm-util', async () => ({ + getParameter: vi.fn(), + putParameter: vi.fn(), +})); + +vi.mock('./job-retry', () => ({ + publishRetryMessage: vi.fn(), + checkAndRetryJob: vi.fn(), +})); + +const mockStartRunner = vi.mocked(startRunner); +const mockTag = vi.mocked(tag); +const mockUntag = vi.mocked(untag); +const mockGetWarmPoolConfig = vi.mocked(getWarmPoolConfig); +const mockGetPoolStrategy = vi.mocked(getPoolStrategy); +const mockListWarmInstances = vi.mocked(listWarmInstancesByOwner); +const mockRemoveFromWarmPool = vi.mocked(removeFromWarmPool); +const mockEmitWarmPoolMetric = vi.mocked(emitWarmPoolMetric); + +describe('findAndStartWarmRunners', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env.ENVIRONMENT = 'test-env'; + + mockGetWarmPoolConfig.mockReturnValue({ + enabled: true, + maxWarmInstances: 3, + maxWarmAgeHours: 168, + warmPoolReadyDelaySeconds: 30, + }); + mockGetPoolStrategy.mockReturnValue('warm'); + mockStartRunner.mockResolvedValue(undefined); + mockRemoveFromWarmPool.mockResolvedValue(undefined); + mockTag.mockResolvedValue(undefined); + }); + + it('should return empty array when warm pool is disabled', async () => { + mockGetWarmPoolConfig.mockReturnValue({ + enabled: false, + maxWarmInstances: 3, + maxWarmAgeHours: 168, + warmPoolReadyDelaySeconds: 30, + }); + + const result = await findAndStartWarmRunners('my-org', 1); + + expect(result).toEqual([]); + expect(mockListWarmInstances).not.toHaveBeenCalled(); + }); + + it('should return empty array when pool strategy is not warm', async () => { + mockGetPoolStrategy.mockReturnValue('hot'); + + const result = await findAndStartWarmRunners('my-org', 1); + + expect(result).toEqual([]); + expect(mockListWarmInstances).not.toHaveBeenCalled(); + }); + + it('should return empty array when count is 0', async () => { + const result = await findAndStartWarmRunners('my-org', 0); + + expect(result).toEqual([]); + expect(mockListWarmInstances).not.toHaveBeenCalled(); + }); + + it('should return empty array when no warm instances available', async () => { + mockListWarmInstances.mockResolvedValue([]); + + const result = await findAndStartWarmRunners('my-org', 1); + + expect(result).toEqual([]); + expect(mockStartRunner).not.toHaveBeenCalled(); + }); + + it('should start a warm instance and remove from pool', async () => { + mockListWarmInstances.mockResolvedValue([ + { + instanceId: 'i-warm-1', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: '2026-01-01T00:00:00Z', + expiresAt: 9999999999, + }, + ]); + + const result = await findAndStartWarmRunners('my-org', 1); + + expect(result).toEqual(['i-warm-1']); + expect(mockStartRunner).toHaveBeenCalledWith('i-warm-1'); + expect(mockRemoveFromWarmPool).toHaveBeenCalledWith('i-warm-1'); + expect(mockEmitWarmPoolMetric).toHaveBeenCalledWith('WarmPoolInstanceStarted', 1, { Owner: 'my-org' }); + }); + + it('should tag instance as started-from-warm-pool after successful start', async () => { + mockListWarmInstances.mockResolvedValue([ + { + instanceId: 'i-warm-1', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: '2026-01-01T00:00:00Z', + expiresAt: 9999999999, + }, + ]); + + const result = await findAndStartWarmRunners('my-org', 1); + + expect(result).toEqual(['i-warm-1']); + expect(mockTag).toHaveBeenCalledWith('i-warm-1', [{ Key: 'ghr:started-from-warm-pool', Value: 'true' }]); + expect(mockUntag).toHaveBeenCalledWith('i-warm-1', [{ Key: 'ghr:warm-pool-member' }]); + }); + + it('should succeed even if tag fails (best-effort)', async () => { + mockTag.mockRejectedValue(new Error('UnauthorizedOperation')); + mockListWarmInstances.mockResolvedValue([ + { + instanceId: 'i-warm-1', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: '2026-01-01T00:00:00Z', + expiresAt: 9999999999, + }, + ]); + + const result = await findAndStartWarmRunners('my-org', 1); + + // Instance should still be in the result — untag failure is non-fatal + expect(result).toEqual(['i-warm-1']); + expect(mockStartRunner).toHaveBeenCalledWith('i-warm-1'); + expect(mockRemoveFromWarmPool).toHaveBeenCalledWith('i-warm-1'); + }); + + it('should start multiple instances up to the requested count', async () => { + mockListWarmInstances.mockResolvedValue([ + { + instanceId: 'i-warm-1', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: '2026-01-01T00:00:00Z', + expiresAt: 9999999999, + }, + { + instanceId: 'i-warm-2', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: '2026-01-01T01:00:00Z', + expiresAt: 9999999999, + }, + { + instanceId: 'i-warm-3', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: '2026-01-01T02:00:00Z', + expiresAt: 9999999999, + }, + ]); + + const result = await findAndStartWarmRunners('my-org', 2); + + expect(result).toEqual(['i-warm-1', 'i-warm-2']); + expect(mockStartRunner).toHaveBeenCalledTimes(2); + expect(mockStartRunner).toHaveBeenCalledWith('i-warm-1'); + expect(mockStartRunner).toHaveBeenCalledWith('i-warm-2'); + }); + + it('should skip failed instances and continue with next', async () => { + mockStartRunner + .mockRejectedValueOnce(new Error('Instance terminated')) + .mockResolvedValueOnce(undefined); + mockListWarmInstances.mockResolvedValue([ + { + instanceId: 'i-bad', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: '2026-01-01T00:00:00Z', + expiresAt: 9999999999, + }, + { + instanceId: 'i-good', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: '2026-01-01T01:00:00Z', + expiresAt: 9999999999, + }, + ]); + + const result = await findAndStartWarmRunners('my-org', 2); + + expect(result).toEqual(['i-good']); + expect(mockEmitWarmPoolMetric).toHaveBeenCalledWith('WarmPoolStartFailed', 1, { Owner: 'my-org' }); + }); + + it('should remove failed instance from DynamoDB', async () => { + mockStartRunner.mockRejectedValue(new Error('Instance terminated')); + mockListWarmInstances.mockResolvedValue([ + { + instanceId: 'i-gone', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: '2026-01-01T00:00:00Z', + expiresAt: 9999999999, + }, + ]); + + const result = await findAndStartWarmRunners('my-org', 1); + + expect(result).toEqual([]); + // removeFromWarmPool called in the catch block for cleanup + expect(mockRemoveFromWarmPool).toHaveBeenCalledWith('i-gone'); + }); + + it('should fallback to org-level lookup when repo-level owner has no instances', async () => { + mockListWarmInstances + .mockResolvedValueOnce([]) // repo-level lookup: empty + .mockResolvedValueOnce([ + { + instanceId: 'i-org-warm', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: '2026-01-01T00:00:00Z', + expiresAt: 9999999999, + }, + ]); + + const result = await findAndStartWarmRunners('my-org/my-repo', 1); + + expect(result).toEqual(['i-org-warm']); + expect(mockListWarmInstances).toHaveBeenCalledWith('my-org/my-repo'); + expect(mockListWarmInstances).toHaveBeenCalledWith('my-org'); + }); + + it('should not fallback to org-level when owner has no slash', async () => { + mockListWarmInstances.mockResolvedValue([]); + + const result = await findAndStartWarmRunners('my-org', 1); + + expect(result).toEqual([]); + expect(mockListWarmInstances).toHaveBeenCalledTimes(1); + expect(mockListWarmInstances).toHaveBeenCalledWith('my-org'); + }); +}); From aa61edbad5d7d3a979a8e69d94a090c3a875fbc0 Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 16:09:14 +0200 Subject: [PATCH 08/19] test(warm-pool): add scale-down eviction and pool warm strategy tests Scale-down eviction tests: - Skip eviction when warm pool disabled - Evict instances exceeding max age - Evict instances exceeding max count - Emit WarmPoolSize metric after eviction Pool warm strategy tests: - Count warm instances toward pool target - Try warm instances before cold launching - Skip cold launch when warm satisfies full top-up - Don't count warm instances with hot strategy Signed-off-by: Brend Smits --- .../src/pool/pool-warm-strategy.test.ts | 144 +++++++++++++ .../scale-down-warm-pool.test.ts | 203 ++++++++++++++++++ 2 files changed, 347 insertions(+) create mode 100644 lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts diff --git a/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts b/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts new file mode 100644 index 0000000000..ac0f7e5dc5 --- /dev/null +++ b/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts @@ -0,0 +1,144 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { listEC2Runners } from '../aws/runners'; +import * as ghAuth from '../github/auth'; +import { createRunners, findAndStartWarmRunners, getGitHubEnterpriseApiUrl } from '../scale-runners/scale-up'; +import { getPoolStrategy, getWarmPoolConfig, countWarmInstancesByOwner } from '../aws/warm-pool'; +import { adjust } from './pool'; + +const mockOctokit = { + paginate: vi.fn().mockResolvedValue([]), + actions: { + createRegistrationTokenForOrg: vi.fn(), + }, + apps: { + getOrgInstallation: vi.fn().mockResolvedValue({ data: { id: 1 } }), + }, +}; + +vi.mock('@octokit/rest', () => ({ + Octokit: vi.fn().mockImplementation(function () { + return mockOctokit; + }), +})); + +vi.mock('./../aws/runners', async () => ({ + listEC2Runners: vi.fn().mockResolvedValue([]), + bootTimeExceeded: vi.fn().mockReturnValue(false), +})); + +vi.mock('./../github/auth', async () => ({ + createGithubAppAuth: vi.fn().mockResolvedValue({ token: 'app-token', appId: 1, type: 'app' }), + createGithubInstallationAuth: vi.fn().mockResolvedValue({ token: 'install-token', type: 'token' }), + createOctokitClient: vi.fn().mockImplementation(() => mockOctokit), +})); + +vi.mock('../scale-runners/scale-up', async () => ({ + scaleUp: vi.fn(), + createRunners: vi.fn(), + findAndStartWarmRunners: vi.fn().mockResolvedValue([]), + getGitHubEnterpriseApiUrl: vi.fn().mockReturnValue({ ghesApiUrl: '', ghesBaseUrl: '' }), + validateSsmParameterStoreTags: vi.fn().mockReturnValue([]), +})); + +vi.mock('../aws/warm-pool', async () => ({ + getPoolStrategy: vi.fn().mockReturnValue('hot'), + getWarmPoolConfig: vi.fn().mockReturnValue({ enabled: false, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }), + countWarmInstancesByOwner: vi.fn().mockResolvedValue(0), +})); + +const mockListRunners = vi.mocked(listEC2Runners); +const mockCreateRunners = vi.mocked(createRunners); +const mockFindAndStartWarmRunners = vi.mocked(findAndStartWarmRunners); +const mockGetPoolStrategy = vi.mocked(getPoolStrategy); +const mockGetWarmPoolConfig = vi.mocked(getWarmPoolConfig); +const mockCountWarmInstances = vi.mocked(countWarmInstancesByOwner); + +describe('pool warm strategy', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env.ENVIRONMENT = 'test-env'; + process.env.RUNNER_OWNER = 'my-org'; + process.env.RUNNER_LABELS = 'linux,x64'; + process.env.RUNNER_GROUP_NAME = ''; + process.env.RUNNER_NAME_PREFIX = ''; + process.env.SSM_TOKEN_PATH = '/runners/token'; + process.env.SSM_CONFIG_PATH = '/runners/config'; + process.env.SUBNET_IDS = 'subnet-1,subnet-2'; + process.env.INSTANCE_TYPES = 'm5.large'; + process.env.INSTANCE_TARGET_CAPACITY_TYPE = 'on-demand'; + process.env.LAUNCH_TEMPLATE_NAME = 'test-lt'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + process.env.ENABLE_JIT_CONFIG = 'false'; + process.env.DISABLE_RUNNER_AUTOUPDATE = 'false'; + process.env.INSTANCE_MAX_SPOT_PRICE = ''; + process.env.INSTANCE_ALLOCATION_STRATEGY = 'lowest-price'; + process.env.RUNNERS_MAXIMUM_COUNT = '-1'; + process.env.SCALE_ERRORS = '["InsufficientInstanceCapacity"]'; + process.env.POWERTOOLS_TRACE_ENABLED = 'false'; + process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = '[]'; + process.env.AMI_ID_SSM_PARAMETER_NAME = ''; + process.env.SSM_PARAMETER_STORE_TAGS = ''; + + mockListRunners.mockResolvedValue([]); + mockOctokit.paginate.mockResolvedValue([]); + mockFindAndStartWarmRunners.mockResolvedValue([]); + }); + + it('should count warm instances toward pool target with warm strategy', async () => { + mockGetPoolStrategy.mockReturnValue('warm'); + mockGetWarmPoolConfig.mockReturnValue({ enabled: true, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }); + mockCountWarmInstances.mockResolvedValue(2); + // 0 running + 2 warm = 2 effective, pool size = 2 → no top-up needed + mockListRunners.mockResolvedValue([]); + + await adjust({ poolSize: 2 }); + + expect(mockCountWarmInstances).toHaveBeenCalledWith('my-org'); + expect(mockCreateRunners).not.toHaveBeenCalled(); + expect(mockFindAndStartWarmRunners).not.toHaveBeenCalled(); + }); + + it('should try warm instances first when topping up', async () => { + mockGetPoolStrategy.mockReturnValue('warm'); + mockGetWarmPoolConfig.mockReturnValue({ enabled: true, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }); + mockCountWarmInstances.mockResolvedValue(0); + // Pool wants 3, has 0 → needs 3, warm start returns 2 → 1 cold needed + mockFindAndStartWarmRunners.mockResolvedValue(['i-warm-1', 'i-warm-2']); + + await adjust({ poolSize: 3 }); + + expect(mockFindAndStartWarmRunners).toHaveBeenCalledWith('my-org', 3, expect.any(Object), expect.anything()); + expect(mockCreateRunners).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Object), + 1, // remainingTopUp = 3 - 2 + expect.anything(), + 'pool-lambda', + ); + }); + + it('should not cold-launch if warm instances satisfy the full top-up', async () => { + mockGetPoolStrategy.mockReturnValue('warm'); + mockGetWarmPoolConfig.mockReturnValue({ enabled: true, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }); + mockCountWarmInstances.mockResolvedValue(0); + mockFindAndStartWarmRunners.mockResolvedValue(['i-warm-1', 'i-warm-2']); + + await adjust({ poolSize: 2 }); + + expect(mockFindAndStartWarmRunners).toHaveBeenCalledWith('my-org', 2, expect.any(Object), expect.anything()); + expect(mockCreateRunners).not.toHaveBeenCalled(); + }); + + it('should not count warm instances when strategy is hot', async () => { + mockGetPoolStrategy.mockReturnValue('hot'); + mockGetWarmPoolConfig.mockReturnValue({ enabled: true, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }); + // With hot strategy, warm instances should NOT be counted even if warm pool is enabled + mockListRunners.mockResolvedValue([]); + + await adjust({ poolSize: 2 }); + + expect(mockCountWarmInstances).not.toHaveBeenCalled(); + // Should need to top-up 2 instances (no warm counting) + expect(mockFindAndStartWarmRunners).toHaveBeenCalledWith('my-org', 2, expect.any(Object), expect.anything()); + }); +}); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts new file mode 100644 index 0000000000..9b2cf0c228 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts @@ -0,0 +1,203 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { terminateRunner, stopRunner, tag, listEC2Runners } from './../aws/runners'; +import { + getWarmPoolConfig, + getPoolStrategy, + addToWarmPool, + countWarmInstancesByOwner, + listWarmInstancesByOwner, + removeFromWarmPool, + emitWarmPoolMetric, +} from '../aws/warm-pool'; + +vi.mock('./../aws/runners', async () => ({ + createRunner: vi.fn(), + listEC2Runners: vi.fn().mockResolvedValue([]), + startRunner: vi.fn(), + stopRunner: vi.fn(), + tag: vi.fn(), + untag: vi.fn(), + terminateRunner: vi.fn(), + bootTimeExceeded: vi.fn().mockReturnValue(false), +})); + +vi.mock('../aws/warm-pool', async () => ({ + getWarmPoolConfig: vi.fn(), + getPoolStrategy: vi.fn(), + addToWarmPool: vi.fn(), + countWarmInstancesByOwner: vi.fn(), + listWarmInstancesByOwner: vi.fn(), + removeFromWarmPool: vi.fn(), + emitWarmPoolMetric: vi.fn(), + getWarmInstance: vi.fn(), +})); + +vi.mock('./../github/auth', async () => ({ + createGithubAppAuth: vi.fn().mockResolvedValue({ token: 'test-token' }), + createGithubInstallationAuth: vi.fn().mockResolvedValue({ token: 'test-token' }), + createOctokitClient: vi.fn().mockResolvedValue({ + apps: { getOrgInstallation: vi.fn().mockResolvedValue({ data: { id: 1 } }) }, + actions: { + listSelfHostedRunnersForOrg: vi.fn().mockResolvedValue({ data: { runners: [] } }), + getSelfHostedRunnerForOrg: vi.fn().mockResolvedValue({ data: { busy: false }, headers: {} }), + deleteSelfHostedRunnerFromOrg: vi.fn().mockResolvedValue({ status: 204 }), + }, + paginate: vi.fn().mockResolvedValue([]), + }), +})); + +vi.mock('../github/rate-limit', () => ({ + metricGitHubAppRateLimit: vi.fn(), +})); + +vi.mock('@aws-github-runner/aws-ssm-util', async () => ({ + getParameter: vi.fn(), + putParameter: vi.fn(), +})); + +vi.mock('./job-retry', () => ({ + publishRetryMessage: vi.fn(), + checkAndRetryJob: vi.fn(), +})); + +// Need to import scaleDown after mocks are set up +const { scaleDown } = await import('./scale-down'); + +const mockGetWarmPoolConfig = vi.mocked(getWarmPoolConfig); +const mockGetPoolStrategy = vi.mocked(getPoolStrategy); +const mockListWarmInstances = vi.mocked(listWarmInstancesByOwner); +const mockRemoveFromWarmPool = vi.mocked(removeFromWarmPool); +const mockTerminateRunner = vi.mocked(terminateRunner); +const mockEmitWarmPoolMetric = vi.mocked(emitWarmPoolMetric); +const mockListEC2Runners = vi.mocked(listEC2Runners); + +describe('scale-down warm pool eviction', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env.ENVIRONMENT = 'test-env'; + process.env.SCALE_DOWN_CONFIG = JSON.stringify([{ idleCount: 0, cron: '* * * * *', timeZone: 'UTC' }]); + process.env.MINIMUM_RUNNING_TIME_IN_MINUTES = '5'; + process.env.RUNNER_BOOT_TIME_IN_MINUTES = '5'; + + mockGetWarmPoolConfig.mockReturnValue({ + enabled: true, + maxWarmInstances: 2, + maxWarmAgeHours: 168, + warmPoolReadyDelaySeconds: 30, + }); + mockGetPoolStrategy.mockReturnValue('warm'); + mockListEC2Runners.mockResolvedValue([]); + }); + + it('should skip eviction when warm pool is disabled', async () => { + mockGetWarmPoolConfig.mockReturnValue({ + enabled: false, + maxWarmInstances: 3, + maxWarmAgeHours: 168, + warmPoolReadyDelaySeconds: 30, + }); + + await scaleDown(); + + expect(mockListWarmInstances).not.toHaveBeenCalled(); + }); + + it('should evict warm instances exceeding max age', async () => { + mockListEC2Runners.mockResolvedValue([ + { instanceId: 'i-running', owner: 'my-org', type: 'Org' } as any, + ]); + const oldDate = new Date(Date.now() - 200 * 3600 * 1000).toISOString(); // 200 hours ago + mockListWarmInstances.mockResolvedValue([ + { + instanceId: 'i-old-warm', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: oldDate, + expiresAt: 9999999999, + }, + ]); + mockTerminateRunner.mockResolvedValue(undefined); + mockRemoveFromWarmPool.mockResolvedValue(undefined); + + await scaleDown(); + + expect(mockTerminateRunner).toHaveBeenCalledWith('i-old-warm'); + expect(mockRemoveFromWarmPool).toHaveBeenCalledWith('i-old-warm'); + }); + + it('should evict warm instances exceeding max count', async () => { + mockListEC2Runners.mockResolvedValue([ + { instanceId: 'i-running', owner: 'my-org', type: 'Org' } as any, + ]); + const recentDate = new Date(Date.now() - 1 * 3600 * 1000).toISOString(); // 1 hour ago + mockListWarmInstances.mockResolvedValue([ + { + instanceId: 'i-warm-1', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: recentDate, + expiresAt: 9999999999, + }, + { + instanceId: 'i-warm-2', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: recentDate, + expiresAt: 9999999999, + }, + { + instanceId: 'i-warm-3', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: recentDate, + expiresAt: 9999999999, + }, + ]); + mockTerminateRunner.mockResolvedValue(undefined); + mockRemoveFromWarmPool.mockResolvedValue(undefined); + + await scaleDown(); + + // maxWarmInstances is 2, so at least 1 warm instance should be evicted (i-warm-*) + const warmEvictionCalls = mockTerminateRunner.mock.calls.filter( + (call) => (call[0] as string).startsWith('i-warm-'), + ); + expect(warmEvictionCalls).toHaveLength(1); + }); + + it('should emit WarmPoolSize metric after eviction', async () => { + mockListEC2Runners.mockResolvedValue([ + { instanceId: 'i-running', owner: 'my-org', type: 'Org' } as any, + ]); + const oldDate = new Date(Date.now() - 200 * 3600 * 1000).toISOString(); + mockListWarmInstances.mockResolvedValue([ + { + instanceId: 'i-old', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: oldDate, + expiresAt: 9999999999, + }, + { + instanceId: 'i-recent', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: new Date().toISOString(), + expiresAt: 9999999999, + }, + ]); + mockTerminateRunner.mockResolvedValue(undefined); + mockRemoveFromWarmPool.mockResolvedValue(undefined); + + await scaleDown(); + + // One evicted (old), one remaining + expect(mockEmitWarmPoolMetric).toHaveBeenCalledWith('WarmPoolSize', 1, { Owner: 'my-org' }); + }); +}); From 80a8eba59da3932b51a2c1458262ccb2eb94bc52 Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 16:13:35 +0200 Subject: [PATCH 09/19] feat(warm-pool): add AMI staleness eviction and DynamoDB dependency - Add amiId field to WarmPoolEntry interface and DynamoDB storage - Store current AMI ID (from SSM parameter) when stopping instances - Evict warm instances with stale AMI during eviction cycle - Add @aws-sdk/client-dynamodb to control-plane dependencies - Pass AMI_ID_SSM_PARAMETER_NAME env var to scale-down lambda - Grant scale-down IAM permission to read AMI SSM parameter Signed-off-by: Brend Smits --- lambdas/functions/control-plane/package.json | 1 + .../control-plane/src/aws/warm-pool.ts | 3 + .../src/scale-runners/scale-down.ts | 27 +- lambdas/yarn.lock | 335 ++++++++++++++++++ modules/runners/scale-down.tf | 7 + 5 files changed, 370 insertions(+), 3 deletions(-) diff --git a/lambdas/functions/control-plane/package.json b/lambdas/functions/control-plane/package.json index 8f978c0d17..4324dcc2c8 100644 --- a/lambdas/functions/control-plane/package.json +++ b/lambdas/functions/control-plane/package.json @@ -33,6 +33,7 @@ "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", "@aws-lambda-powertools/parameters": "^2.31.0", + "@aws-sdk/client-dynamodb": "^3.1078.0", "@aws-sdk/client-ec2": "^3.1009.0", "@aws-sdk/client-sqs": "^3.1009.0", "@middy/core": "^6.4.5", diff --git a/lambdas/functions/control-plane/src/aws/warm-pool.ts b/lambdas/functions/control-plane/src/aws/warm-pool.ts index 775beb06e1..c9aff50d04 100644 --- a/lambdas/functions/control-plane/src/aws/warm-pool.ts +++ b/lambdas/functions/control-plane/src/aws/warm-pool.ts @@ -26,6 +26,7 @@ export interface WarmPoolEntry { runnerType: string; instanceType?: string; az?: string; + amiId?: string; stoppedAt: string; expiresAt: number; } @@ -68,6 +69,7 @@ export async function addToWarmPool(entry: Omit { if (runner.owner) ownerTags.add(runner.owner); } + // Resolve the current AMI ID for staleness comparison + let currentAmiId: string | undefined; + const amiSsmParam = process.env.AMI_ID_SSM_PARAMETER_NAME; + if (amiSsmParam) { + try { + currentAmiId = await getParameter(amiSsmParam); + } catch (e) { + logger.warn('Failed to resolve current AMI ID for warm pool staleness check', { error: e }); + } + } + for (const owner of ownerTags) { try { const warmInstances = await listWarmInstancesByOwner(owner); @@ -413,15 +433,16 @@ async function evictStaleWarmInstances(environment: string): Promise { const ageHours = (now - new Date(entry.stoppedAt).getTime() / 1000) / 3600; const exceedsAge = ageHours > warmPoolConfig.maxWarmAgeHours; const exceedsCount = warmInstances.length - evictedCount > warmPoolConfig.maxWarmInstances; + const staleAmi = currentAmiId && entry.amiId && entry.amiId !== currentAmiId; - if (exceedsAge || exceedsCount) { + if (exceedsAge || exceedsCount || staleAmi) { try { await terminateRunner(entry.instanceId); await removeFromWarmPool(entry.instanceId); evictedCount++; + const reason = staleAmi ? 'stale_ami' : exceedsAge ? 'max_age_exceeded' : 'max_count_exceeded'; logger.info( - `Evicted warm instance '${entry.instanceId}' (age: ${ageHours.toFixed(1)}h, ` + - `reason: ${exceedsAge ? 'max_age_exceeded' : 'max_count_exceeded'}).`, + `Evicted warm instance '${entry.instanceId}' (age: ${ageHours.toFixed(1)}h, reason: ${reason}).`, ); } catch (e) { logger.warn(`Failed to evict warm instance '${entry.instanceId}'`, { error: e }); diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index bfe69a07b9..111470d1c9 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -148,6 +148,7 @@ __metadata: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-lambda-powertools/parameters": "npm:^2.31.0" + "@aws-sdk/client-dynamodb": "npm:^3.1078.0" "@aws-sdk/client-ec2": "npm:^3.1009.0" "@aws-sdk/client-sqs": "npm:^3.1009.0" "@aws-sdk/types": "npm:^3.973.6" @@ -324,6 +325,24 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-dynamodb@npm:^3.1078.0": + version: 3.1078.0 + resolution: "@aws-sdk/client-dynamodb@npm:3.1078.0" + dependencies: + "@aws-sdk/core": "npm:^3.974.26" + "@aws-sdk/credential-provider-node": "npm:^3.972.61" + "@aws-sdk/dynamodb-codec": "npm:^3.973.26" + "@aws-sdk/middleware-endpoint-discovery": "npm:^3.972.22" + "@aws-sdk/types": "npm:^3.973.15" + "@smithy/core": "npm:^3.29.0" + "@smithy/fetch-http-handler": "npm:^5.6.2" + "@smithy/node-http-handler": "npm:^4.9.2" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/5609dd53e1e4c0ccbb1af69bbb3c2b16132ee8fe5e86ac0184e742b7574d1f301b548cf1a1fa77c3c5ce8739e8c7ea8c721974f745f901262569e09d609bda8e + languageName: node + linkType: hard + "@aws-sdk/client-ec2@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/client-ec2@npm:3.1014.0" @@ -602,6 +621,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/core@npm:^3.974.26": + version: 3.974.26 + resolution: "@aws-sdk/core@npm:3.974.26" + dependencies: + "@aws-sdk/types": "npm:^3.973.15" + "@aws-sdk/xml-builder": "npm:^3.972.33" + "@aws/lambda-invoke-store": "npm:^0.2.2" + "@smithy/core": "npm:^3.29.0" + "@smithy/signature-v4": "npm:^5.6.1" + "@smithy/types": "npm:^4.15.1" + bowser: "npm:^2.11.0" + tslib: "npm:^2.6.2" + checksum: 10c0/3e2f10c874b87cac48ccba38b4ec4bb7e2d1869f21a526044f4d32ed793eaab7dd4ed49f7ad02eb8a37a9042e2df24b08f8b6af645207522920416a10df62a2a + languageName: node + linkType: hard + "@aws-sdk/crc64-nvme@npm:^3.972.5": version: 3.972.5 resolution: "@aws-sdk/crc64-nvme@npm:3.972.5" @@ -625,6 +660,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-env@npm:^3.972.52": + version: 3.972.52 + resolution: "@aws-sdk/credential-provider-env@npm:3.972.52" + dependencies: + "@aws-sdk/core": "npm:^3.974.26" + "@aws-sdk/types": "npm:^3.973.15" + "@smithy/core": "npm:^3.29.0" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/1626bd328408a7c511386715f3a7071243787605a6f846a1299d8bc72db7cd70181bd4623d38035b3e84986c88e2ceef0f870e511725d4effea33f3050d8fa64 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-http@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-http@npm:3.972.23" @@ -643,6 +691,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-http@npm:^3.972.54": + version: 3.972.54 + resolution: "@aws-sdk/credential-provider-http@npm:3.972.54" + dependencies: + "@aws-sdk/core": "npm:^3.974.26" + "@aws-sdk/types": "npm:^3.973.15" + "@smithy/core": "npm:^3.29.0" + "@smithy/fetch-http-handler": "npm:^5.6.2" + "@smithy/node-http-handler": "npm:^4.9.2" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/600fbf04df77194b9fccffb61245b3f9f29218a61ebc5b0745d6cc01322697739cf298eafd91760a28780c11b6018347b566f8a045e602cd5a8f999e93dc37fa + languageName: node + linkType: hard + "@aws-sdk/credential-provider-ini@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-ini@npm:3.972.23" @@ -665,6 +728,27 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-ini@npm:^3.972.59": + version: 3.972.59 + resolution: "@aws-sdk/credential-provider-ini@npm:3.972.59" + dependencies: + "@aws-sdk/core": "npm:^3.974.26" + "@aws-sdk/credential-provider-env": "npm:^3.972.52" + "@aws-sdk/credential-provider-http": "npm:^3.972.54" + "@aws-sdk/credential-provider-login": "npm:^3.972.58" + "@aws-sdk/credential-provider-process": "npm:^3.972.52" + "@aws-sdk/credential-provider-sso": "npm:^3.972.58" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.58" + "@aws-sdk/nested-clients": "npm:^3.997.26" + "@aws-sdk/types": "npm:^3.973.15" + "@smithy/core": "npm:^3.29.0" + "@smithy/credential-provider-imds": "npm:^4.4.5" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/a3875f6b37942bfbac9dac1e57d7b9a5d642dfcbc8333014a66895377e47a21e2238709d97a9da0eb7bfdaf37307479a08ff30a0451a2ad7f984c8c89255e67e + languageName: node + linkType: hard + "@aws-sdk/credential-provider-login@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-login@npm:3.972.23" @@ -681,6 +765,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-login@npm:^3.972.58": + version: 3.972.58 + resolution: "@aws-sdk/credential-provider-login@npm:3.972.58" + dependencies: + "@aws-sdk/core": "npm:^3.974.26" + "@aws-sdk/nested-clients": "npm:^3.997.26" + "@aws-sdk/types": "npm:^3.973.15" + "@smithy/core": "npm:^3.29.0" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/9c3da8d17733920d765322d8fcea3d8b597989e35918662b6245482f65f33f900c7b549dece1da55171249b36664b3a7748b4d1d7512d243f22a0ac30ff566ba + languageName: node + linkType: hard + "@aws-sdk/credential-provider-node@npm:^3.972.24": version: 3.972.24 resolution: "@aws-sdk/credential-provider-node@npm:3.972.24" @@ -701,6 +799,25 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-node@npm:^3.972.61": + version: 3.972.61 + resolution: "@aws-sdk/credential-provider-node@npm:3.972.61" + dependencies: + "@aws-sdk/credential-provider-env": "npm:^3.972.52" + "@aws-sdk/credential-provider-http": "npm:^3.972.54" + "@aws-sdk/credential-provider-ini": "npm:^3.972.59" + "@aws-sdk/credential-provider-process": "npm:^3.972.52" + "@aws-sdk/credential-provider-sso": "npm:^3.972.58" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.58" + "@aws-sdk/types": "npm:^3.973.15" + "@smithy/core": "npm:^3.29.0" + "@smithy/credential-provider-imds": "npm:^4.4.5" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/eb86129c9b1f3fc96b504fbcf9d9405f3f7ec24cb10a59dd1a28b5fa1ecef908e8e3aa9df0f1939950ccc9669dbf8664d30f091cf4c7b79423dab9188a1d5367 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-process@npm:^3.972.21": version: 3.972.21 resolution: "@aws-sdk/credential-provider-process@npm:3.972.21" @@ -715,6 +832,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-process@npm:^3.972.52": + version: 3.972.52 + resolution: "@aws-sdk/credential-provider-process@npm:3.972.52" + dependencies: + "@aws-sdk/core": "npm:^3.974.26" + "@aws-sdk/types": "npm:^3.973.15" + "@smithy/core": "npm:^3.29.0" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/65b5772ec0e6ea1dd9be3819278aa10361abb3e79df4022316f60aff2ed2492db5b1ecc8ebc08727beede323b6b605b8a61e60b70a6c43b360370f09352b32b3 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-sso@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-sso@npm:3.972.23" @@ -731,6 +861,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-sso@npm:^3.972.58": + version: 3.972.58 + resolution: "@aws-sdk/credential-provider-sso@npm:3.972.58" + dependencies: + "@aws-sdk/core": "npm:^3.974.26" + "@aws-sdk/nested-clients": "npm:^3.997.26" + "@aws-sdk/token-providers": "npm:3.1078.0" + "@aws-sdk/types": "npm:^3.973.15" + "@smithy/core": "npm:^3.29.0" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/d12aa015435d99e08d7959c46ea3624141178e833d135b52ead89691526b3103e6b0e578bf62be85bb6a57c9f6018e77c420f64236379604f20c91042b71967c + languageName: node + linkType: hard + "@aws-sdk/credential-provider-web-identity@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.23" @@ -746,6 +891,42 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-web-identity@npm:^3.972.58": + version: 3.972.58 + resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.58" + dependencies: + "@aws-sdk/core": "npm:^3.974.26" + "@aws-sdk/nested-clients": "npm:^3.997.26" + "@aws-sdk/types": "npm:^3.973.15" + "@smithy/core": "npm:^3.29.0" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/3c4e185fac1ff6cc81494c4f0b8c9344c99935499c2f53c63e69276ff039ca4d336b0afae83223acb48230adc5f322d8b5807be13981f1a651a89965c0e100fb + languageName: node + linkType: hard + +"@aws-sdk/dynamodb-codec@npm:^3.973.26": + version: 3.973.26 + resolution: "@aws-sdk/dynamodb-codec@npm:3.973.26" + dependencies: + "@aws-sdk/core": "npm:^3.974.26" + "@smithy/core": "npm:^3.29.0" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/20517122d108dd5cb8eec9af787e6b4db2933d541f6489bded3bb8488f7f20b2bd3bdbf46aef92b84961e0e43c2c1275afe75cf27d7b5b53c427f64c1b21f6ed + languageName: node + linkType: hard + +"@aws-sdk/endpoint-cache@npm:^3.972.8": + version: 3.972.8 + resolution: "@aws-sdk/endpoint-cache@npm:3.972.8" + dependencies: + mnemonist: "npm:0.38.3" + tslib: "npm:^2.6.2" + checksum: 10c0/c8ea64ca773b3792a33d32432460741217ba1fa6442f921c99574e03b90c8034a1fd1c0f8485356445a67ec0ad5c9a145cc024faefe0f163c1f28357ebd4e297 + languageName: node + linkType: hard + "@aws-sdk/lib-storage@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/lib-storage@npm:3.1014.0" @@ -778,6 +959,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/middleware-endpoint-discovery@npm:^3.972.22": + version: 3.972.22 + resolution: "@aws-sdk/middleware-endpoint-discovery@npm:3.972.22" + dependencies: + "@aws-sdk/endpoint-cache": "npm:^3.972.8" + "@aws-sdk/types": "npm:^3.973.15" + "@smithy/core": "npm:^3.29.0" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/df7c9273fee4beac34aa1778e8a2397f7e25d8466eafb7e85060a2bf7cdee804a2b954914921d483e5f38682e965a7134c483772a2c9c0cf6b393498724a76ec + languageName: node + linkType: hard + "@aws-sdk/middleware-expect-continue@npm:^3.972.8": version: 3.972.8 resolution: "@aws-sdk/middleware-expect-continue@npm:3.972.8" @@ -984,6 +1178,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/nested-clients@npm:^3.997.26": + version: 3.997.26 + resolution: "@aws-sdk/nested-clients@npm:3.997.26" + dependencies: + "@aws-sdk/core": "npm:^3.974.26" + "@aws-sdk/signature-v4-multi-region": "npm:^3.996.38" + "@aws-sdk/types": "npm:^3.973.15" + "@smithy/core": "npm:^3.29.0" + "@smithy/fetch-http-handler": "npm:^5.6.2" + "@smithy/node-http-handler": "npm:^4.9.2" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/38c23c3f98926e4491aae973df851d503ca16bbd97b613d84aedbb391e6835e0ddba22c60a1b96047a1a3dda3ecd2175500d701076b5bd7446bb9a44a3a9989a + languageName: node + linkType: hard + "@aws-sdk/region-config-resolver@npm:^3.972.9": version: 3.972.9 resolution: "@aws-sdk/region-config-resolver@npm:3.972.9" @@ -1011,6 +1221,18 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/signature-v4-multi-region@npm:^3.996.38": + version: 3.996.38 + resolution: "@aws-sdk/signature-v4-multi-region@npm:3.996.38" + dependencies: + "@aws-sdk/types": "npm:^3.973.15" + "@smithy/signature-v4": "npm:^5.6.1" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/3cd2a7f751e02becc7230de6a549d85b785cb5ffecf5c3c4cc2c498a2dbd7e08969d149a5b669ba36af618c97a353c30b9b3441666682d8fb19b54589aea4396 + languageName: node + linkType: hard + "@aws-sdk/token-providers@npm:3.1014.0": version: 3.1014.0 resolution: "@aws-sdk/token-providers@npm:3.1014.0" @@ -1026,6 +1248,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/token-providers@npm:3.1078.0": + version: 3.1078.0 + resolution: "@aws-sdk/token-providers@npm:3.1078.0" + dependencies: + "@aws-sdk/core": "npm:^3.974.26" + "@aws-sdk/nested-clients": "npm:^3.997.26" + "@aws-sdk/types": "npm:^3.973.15" + "@smithy/core": "npm:^3.29.0" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/90ab6dd1460e0c0083305146d830d76faead11ab170a9d8d274c0157a323b45e0a45959b8f22cf6bf3ca0230eb3f731e0eab13fba755b20377495adee28fff38 + languageName: node + linkType: hard + "@aws-sdk/types@npm:^3.222.0, @aws-sdk/types@npm:^3.4.1, @aws-sdk/types@npm:^3.973.6": version: 3.973.6 resolution: "@aws-sdk/types@npm:3.973.6" @@ -1036,6 +1272,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/types@npm:^3.973.15": + version: 3.973.15 + resolution: "@aws-sdk/types@npm:3.973.15" + dependencies: + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/3e20ce06e83e6749c379cd822a2b00bca17ee0431d69ec96b5b086c65e33c91ea484d02d10758eef79caa67af2e5623c2e454be6083d2609f20b9b447f661e5e + languageName: node + linkType: hard + "@aws-sdk/util-arn-parser@npm:^3.972.3": version: 3.972.3 resolution: "@aws-sdk/util-arn-parser@npm:3.972.3" @@ -1121,6 +1367,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/xml-builder@npm:^3.972.33": + version: 3.972.33 + resolution: "@aws-sdk/xml-builder@npm:3.972.33" + dependencies: + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/58d9b21b16cb81180e90f2af9f28fba070086d08e84473fd1cb0bb5afd502ec1e847436ddf73fc08934512e867389f246d31bd0d61d280e10937e0f775710989 + languageName: node + linkType: hard + "@aws/lambda-invoke-store@npm:0.2.3, @aws/lambda-invoke-store@npm:^0.2.2": version: 0.2.3 resolution: "@aws/lambda-invoke-store@npm:0.2.3" @@ -4386,6 +4642,16 @@ __metadata: languageName: node linkType: hard +"@smithy/core@npm:^3.29.0": + version: 3.29.0 + resolution: "@smithy/core@npm:3.29.0" + dependencies: + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/4d1dfa9935bd7aecd53c70f8dcf340736cbd9e5893da65d4bf8ece038d62bdbc09ce9e8beef4bd35b808470d5408eaefd4cccba7dbffdb945526679846325081 + languageName: node + linkType: hard + "@smithy/credential-provider-imds@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/credential-provider-imds@npm:4.2.12" @@ -4399,6 +4665,17 @@ __metadata: languageName: node linkType: hard +"@smithy/credential-provider-imds@npm:^4.4.5": + version: 4.4.5 + resolution: "@smithy/credential-provider-imds@npm:4.4.5" + dependencies: + "@smithy/core": "npm:^3.29.0" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/c7847f74af51fd5bde047ffc30bfe32f04b8f54b2dc3631b28b009c593300ae7e9769c35acb11e1ee402941c4e3a31426aa5ba035deb21f850921739eb2d5030 + languageName: node + linkType: hard + "@smithy/eventstream-codec@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/eventstream-codec@npm:4.2.12" @@ -4467,6 +4744,17 @@ __metadata: languageName: node linkType: hard +"@smithy/fetch-http-handler@npm:^5.6.2": + version: 5.6.2 + resolution: "@smithy/fetch-http-handler@npm:5.6.2" + dependencies: + "@smithy/core": "npm:^3.29.0" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/adcc1bab05e2aad9d2e28adc359821dfdae1b76987cd0434b1003214b2e9453bbab666b989d9c070e184b1e2431ba1ba125af61fcdc92961f507b12170364d2d + languageName: node + linkType: hard + "@smithy/hash-blob-browser@npm:^4.2.13": version: 4.2.13 resolution: "@smithy/hash-blob-browser@npm:4.2.13" @@ -4632,6 +4920,17 @@ __metadata: languageName: node linkType: hard +"@smithy/node-http-handler@npm:^4.9.2": + version: 4.9.2 + resolution: "@smithy/node-http-handler@npm:4.9.2" + dependencies: + "@smithy/core": "npm:^3.29.0" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b619c3f79a762538a6ebf34a84abb60ac77ccbdf577449daf7358b2bb04ce791ca90a1ceeffbfde66a87165cceeabe1c92cec11b1a1eecaef2f8277480cb63d0 + languageName: node + linkType: hard + "@smithy/property-provider@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/property-provider@npm:4.2.12" @@ -4717,6 +5016,17 @@ __metadata: languageName: node linkType: hard +"@smithy/signature-v4@npm:^5.6.1": + version: 5.6.1 + resolution: "@smithy/signature-v4@npm:5.6.1" + dependencies: + "@smithy/core": "npm:^3.29.0" + "@smithy/types": "npm:^4.15.1" + tslib: "npm:^2.6.2" + checksum: 10c0/3442a4d98f339851daf7305f104b8a95cc0295e5bc3f6fe5462d29611764d6f485051f024803859821f36ce9c2ce75a405267ba8d0daabec04224372116ba6c8 + languageName: node + linkType: hard + "@smithy/smithy-client@npm:^4.12.7": version: 4.12.7 resolution: "@smithy/smithy-client@npm:4.12.7" @@ -4750,6 +5060,15 @@ __metadata: languageName: node linkType: hard +"@smithy/types@npm:^4.15.1": + version: 4.15.1 + resolution: "@smithy/types@npm:4.15.1" + dependencies: + tslib: "npm:^2.6.2" + checksum: 10c0/65cb70240fa13f37274796b065be48d86cccd182ec97e7b1a0da234e2c9bd5ee686b4f5f111c930fe2209a2a6f168b9a045d59fb80c5545f4bff885ba350afa2 + languageName: node + linkType: hard + "@smithy/url-parser@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/url-parser@npm:4.2.12" @@ -8823,6 +9142,15 @@ __metadata: languageName: node linkType: hard +"mnemonist@npm:0.38.3": + version: 0.38.3 + resolution: "mnemonist@npm:0.38.3" + dependencies: + obliterator: "npm:^1.6.1" + checksum: 10c0/064aa1ee1a89fce2754423b3617c598fd65bc34311eb3c01dc063976f6b819b073bd23532415cf8c92240157b4c8fbb7ec5d79d717f2bd4fcd95d8131cb23acb + languageName: node + linkType: hard + "moment-timezone@npm:^0.6.0": version: 0.6.0 resolution: "moment-timezone@npm:0.6.0" @@ -9225,6 +9553,13 @@ __metadata: languageName: node linkType: hard +"obliterator@npm:^1.6.1": + version: 1.6.1 + resolution: "obliterator@npm:1.6.1" + checksum: 10c0/5fad57319aae0ef6e34efa640541d41c2dd9790a7ab808f17dcb66c83a81333963fc2dfcfa6e1b62158e5cef6291cdcf15c503ad6c3de54b2227dd4c3d7e1b55 + languageName: node + linkType: hard + "obug@npm:^2.1.1": version: 2.1.1 resolution: "obug@npm:2.1.1" diff --git a/modules/runners/scale-down.tf b/modules/runners/scale-down.tf index d5c841a47d..92f87f8f88 100644 --- a/modules/runners/scale-down.tf +++ b/modules/runners/scale-down.tf @@ -51,6 +51,7 @@ resource "aws_lambda_function" "scale_down" { }) WARM_POOL_TABLE_NAME = var.warm_pool_config.enabled ? aws_dynamodb_table.warm_pool[0].name : "" POOL_STRATEGY = var.pool_strategy + AMI_ID_SSM_PARAMETER_NAME = local.ami_id_ssm_parameter_name } } @@ -130,6 +131,12 @@ resource "aws_iam_role_policy_attachment" "scale_down_vpc_execution_role" { policy_arn = "arn:${var.aws_partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole" } +resource "aws_iam_role_policy_attachment" "scale_down_ami_id_ssm_parameter_read" { + count = local.ami_id_ssm_parameter_name != null ? 1 : 0 + role = aws_iam_role.scale_down.name + policy_arn = aws_iam_policy.ami_id_ssm_parameter_read[0].arn +} + resource "aws_iam_role_policy" "scale_down_xray" { count = var.tracing_config.mode != null ? 1 : 0 name = "xray-policy" From 36085c0bf4c4ae00a5f51cfa33558d9670fd15d4 Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 16:14:54 +0200 Subject: [PATCH 10/19] test(pool): fix existing pool tests for warm pool imports Add findAndStartWarmRunners and warm-pool module mocks to existing pool.test.ts to prevent failures from new imports. Signed-off-by: Brend Smits --- lambdas/functions/control-plane/src/pool/pool.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index a60d787c11..1e6a04afaa 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -4,7 +4,7 @@ import * as nock from 'nock'; import { listEC2Runners } from '../aws/runners'; import * as ghAuth from '../github/auth'; -import { createRunners, getGitHubEnterpriseApiUrl } from '../scale-runners/scale-up'; +import { createRunners, findAndStartWarmRunners, getGitHubEnterpriseApiUrl } from '../scale-runners/scale-up'; import { adjust } from './pool'; import { describe, it, expect, beforeEach, vi, MockedClass } from 'vitest'; @@ -39,11 +39,18 @@ vi.mock('./../github/auth', async () => ({ vi.mock('../scale-runners/scale-up', async () => ({ scaleUp: vi.fn(), createRunners: vi.fn(), + findAndStartWarmRunners: vi.fn().mockResolvedValue([]), getGitHubEnterpriseApiUrl: vi.fn().mockReturnValue({ ghesApiUrl: '', ghesBaseUrl: '', }), - // Include any other functions that might be needed + validateSsmParameterStoreTags: vi.fn().mockReturnValue([]), +})); + +vi.mock('../aws/warm-pool', async () => ({ + getPoolStrategy: vi.fn().mockReturnValue('hot'), + getWarmPoolConfig: vi.fn().mockReturnValue({ enabled: false, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }), + countWarmInstancesByOwner: vi.fn().mockResolvedValue(0), })); const mocktokit = Octokit as MockedClass; From 7057b29b56a8a4f970944504e841802120ac809d Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 16:17:56 +0200 Subject: [PATCH 11/19] test(warm-pool): add Terraform tests for warm pool infrastructure - Verify DynamoDB table not created when disabled (default) - Verify DynamoDB table created with correct schema when enabled - Verify IAM policies attached to scale-down, scale-up, and pool roles - Verify pool_strategy defaults to 'hot' - Verify warm strategy + enabled config creates resources Signed-off-by: Brend Smits --- modules/runners/tests/warm-pool.tftest.hcl | 149 +++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 modules/runners/tests/warm-pool.tftest.hcl diff --git a/modules/runners/tests/warm-pool.tftest.hcl b/modules/runners/tests/warm-pool.tftest.hcl new file mode 100644 index 0000000000..c62fee46b3 --- /dev/null +++ b/modules/runners/tests/warm-pool.tftest.hcl @@ -0,0 +1,149 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" + } + } +} + +variables { + aws_region = "eu-west-1" + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + + instance_types = ["m5.large"] + + s3_runner_binaries = { + arn = "arn:aws:s3:::my-bucket" + id = "my-bucket" + key = "runners/linux/actions-runner.tar.gz" + } + + sqs_build_queue = { + arn = "arn:aws:sqs:eu-west-1:123456789012:build-queue" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/build-queue" + } + + enable_organization_runners = true + enable_ssm_on_runners = true + runner_labels = ["self-hosted", "linux", "x64"] + + lambda_s3_bucket = "my-lambda-bucket" + runners_lambda_s3_key = "runners.zip" + + github_app_parameters = { + key_base64 = { name = "/github-runner/key-base64", arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" } + id = { name = "/github-runner/app-id", arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" } + } + + ssm_paths = { + root = "/github-runner" + tokens = "tokens" + config = "config" + } + + pool_config = [{ + schedule_expression = "cron(0 8 * * ? *)" + size = 1 + }] +} + +run "warm_pool_disabled_by_default" { + command = plan + + assert { + condition = length(aws_dynamodb_table.warm_pool) == 0 + error_message = "DynamoDB warm pool table should not be created when warm_pool_config.enabled is false (default)" + } + + assert { + condition = length(aws_iam_role_policy.scale_down_warm_pool) == 0 + error_message = "Warm pool IAM policy should not be attached when disabled" + } +} + +run "warm_pool_enabled_creates_dynamodb" { + command = plan + + variables { + warm_pool_config = { + enabled = true + max_warm_instances = 5 + max_warm_age_hours = 168 + warm_pool_ready_delay_seconds = 30 + } + } + + assert { + condition = length(aws_dynamodb_table.warm_pool) == 1 + error_message = "DynamoDB warm pool table should be created when warm_pool_config.enabled = true" + } + + assert { + condition = aws_dynamodb_table.warm_pool[0].billing_mode == "PAY_PER_REQUEST" + error_message = "DynamoDB table should use PAY_PER_REQUEST billing" + } + + assert { + condition = aws_dynamodb_table.warm_pool[0].hash_key == "instanceId" + error_message = "DynamoDB table hash key should be instanceId" + } +} + +run "warm_pool_enabled_creates_iam_policies" { + command = plan + + variables { + warm_pool_config = { + enabled = true + max_warm_instances = 3 + max_warm_age_hours = 168 + warm_pool_ready_delay_seconds = 30 + } + } + + assert { + condition = length(aws_iam_role_policy.scale_down_warm_pool) == 1 + error_message = "Scale-down warm pool IAM policy should be created" + } + + assert { + condition = length(aws_iam_role_policy.scale_up_warm_pool) == 1 + error_message = "Scale-up warm pool IAM policy should be created" + } + + assert { + condition = length(aws_iam_role_policy.pool_warm_pool) == 1 + error_message = "Pool warm pool IAM policy should be created when pool_config is set" + } +} + +run "warm_pool_strategy_validation" { + command = plan + + variables { + pool_strategy = "warm" + warm_pool_config = { + enabled = true + max_warm_instances = 3 + max_warm_age_hours = 168 + warm_pool_ready_delay_seconds = 30 + } + } + + # The check block emits a warning but doesn't fail the plan, + # so we verify the resources are properly created for valid config + assert { + condition = length(aws_dynamodb_table.warm_pool) == 1 + error_message = "Warm strategy with enabled config should create DynamoDB table" + } +} + +run "pool_strategy_defaults_to_hot" { + command = plan + + assert { + condition = var.pool_strategy == "hot" + error_message = "pool_strategy should default to 'hot'" + } +} From fc97a6bebc95f07a99823b4d835082a8e90e60d0 Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 16:20:52 +0200 Subject: [PATCH 12/19] feat(warm-pool): add WarmPoolStartLatency and WarmPoolEvicted metrics - Measure and emit start latency (ms) for warm instance starts - Emit WarmPoolEvicted metric count during eviction cycles - Use MetricUnit.Milliseconds for latency metric Signed-off-by: Brend Smits --- lambdas/functions/control-plane/src/aws/warm-pool.ts | 11 +++++++++-- .../control-plane/src/scale-runners/scale-down.ts | 1 + .../control-plane/src/scale-runners/scale-up.ts | 5 ++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/lambdas/functions/control-plane/src/aws/warm-pool.ts b/lambdas/functions/control-plane/src/aws/warm-pool.ts index c9aff50d04..866d8d374e 100644 --- a/lambdas/functions/control-plane/src/aws/warm-pool.ts +++ b/lambdas/functions/control-plane/src/aws/warm-pool.ts @@ -149,7 +149,13 @@ function itemToEntry(item: any): WarmPoolEntry { } export function emitWarmPoolMetric( - metricName: 'WarmPoolInstanceStopped' | 'WarmPoolInstanceStarted' | 'WarmPoolStartFailed' | 'WarmPoolSize', + metricName: + | 'WarmPoolInstanceStopped' + | 'WarmPoolInstanceStarted' + | 'WarmPoolStartFailed' + | 'WarmPoolSize' + | 'WarmPoolStartLatency' + | 'WarmPoolEvicted', value: number, dimensions: Record = {}, ): void { @@ -157,5 +163,6 @@ export function emitWarmPoolMetric( if (!enabled) return; const environment = process.env.ENVIRONMENT || ''; - createSingleMetric(metricName, MetricUnit.Count, value, { Environment: environment, ...dimensions }); + const unit = metricName === 'WarmPoolStartLatency' ? MetricUnit.Milliseconds : MetricUnit.Count; + createSingleMetric(metricName, unit, value, { Environment: environment, ...dimensions }); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index e3c1fe9d45..a7bf142b05 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -453,6 +453,7 @@ async function evictStaleWarmInstances(environment: string): Promise { } if (evictedCount > 0) { + emitWarmPoolMetric('WarmPoolEvicted', evictedCount, { Owner: owner }); emitWarmPoolMetric('WarmPoolSize', warmInstances.length - evictedCount, { Owner: owner }); } } catch (e) { diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index c7e3740ae0..0195c1fa39 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -334,11 +334,14 @@ export async function findAndStartWarmRunners( if (startedInstances.length >= count) break; try { + const startTime = Date.now(); await startRunner(entry.instanceId); + const startLatencyMs = Date.now() - startTime; await removeFromWarmPool(entry.instanceId); startedInstances.push(entry.instanceId); emitWarmPoolMetric('WarmPoolInstanceStarted', 1, { Owner: runnerOwner }); - logger.info(`Started warm instance '${entry.instanceId}' for owner '${runnerOwner}'`); + emitWarmPoolMetric('WarmPoolStartLatency', startLatencyMs, { Owner: runnerOwner }); + logger.info(`Started warm instance '${entry.instanceId}' for owner '${runnerOwner}' (${startLatencyMs}ms)`); // Observability tags (best-effort) await Promise.all([ From cf2dc4aff1823f8946a6204e67a6c3edd957fbed Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 16:23:53 +0200 Subject: [PATCH 13/19] feat(warm-pool): implement pool lambda warm strategy grace period When pool_strategy=warm and new instances are cold-launched: - Wait warmPoolReadyDelaySeconds (default 30s) after creation - Re-check GitHub runner status for each new instance - If runner picked up a job: leave running, tag ghr:warm-pool-grace-hit - If runner is idle: stop instance and add to warm pool This enables the pool to maintain warm (stopped) instances at zero idle compute cost while giving jobs a chance to claim new runners. Signed-off-by: Brend Smits --- .../src/pool/pool-warm-strategy.test.ts | 6 +- .../control-plane/src/pool/pool.test.ts | 7 ++- .../functions/control-plane/src/pool/pool.ts | 59 ++++++++++++++++++- 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts b/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts index ac0f7e5dc5..12409c8a62 100644 --- a/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts @@ -24,6 +24,8 @@ vi.mock('@octokit/rest', () => ({ vi.mock('./../aws/runners', async () => ({ listEC2Runners: vi.fn().mockResolvedValue([]), bootTimeExceeded: vi.fn().mockReturnValue(false), + stopRunner: vi.fn().mockResolvedValue(undefined), + tag: vi.fn().mockResolvedValue(undefined), })); vi.mock('./../github/auth', async () => ({ @@ -34,7 +36,7 @@ vi.mock('./../github/auth', async () => ({ vi.mock('../scale-runners/scale-up', async () => ({ scaleUp: vi.fn(), - createRunners: vi.fn(), + createRunners: vi.fn().mockResolvedValue([]), findAndStartWarmRunners: vi.fn().mockResolvedValue([]), getGitHubEnterpriseApiUrl: vi.fn().mockReturnValue({ ghesApiUrl: '', ghesBaseUrl: '' }), validateSsmParameterStoreTags: vi.fn().mockReturnValue([]), @@ -44,6 +46,8 @@ vi.mock('../aws/warm-pool', async () => ({ getPoolStrategy: vi.fn().mockReturnValue('hot'), getWarmPoolConfig: vi.fn().mockReturnValue({ enabled: false, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }), countWarmInstancesByOwner: vi.fn().mockResolvedValue(0), + addToWarmPool: vi.fn().mockResolvedValue(undefined), + emitWarmPoolMetric: vi.fn(), })); const mockListRunners = vi.mocked(listEC2Runners); diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index 1e6a04afaa..af4a55cedd 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -27,8 +27,9 @@ vi.mock('@octokit/rest', () => ({ vi.mock('./../aws/runners', async () => ({ listEC2Runners: vi.fn(), - // Include any other functions from the module that might be used bootTimeExceeded: vi.fn(), + stopRunner: vi.fn().mockResolvedValue(undefined), + tag: vi.fn().mockResolvedValue(undefined), })); vi.mock('./../github/auth', async () => ({ createGithubAppAuth: vi.fn(), @@ -38,7 +39,7 @@ vi.mock('./../github/auth', async () => ({ vi.mock('../scale-runners/scale-up', async () => ({ scaleUp: vi.fn(), - createRunners: vi.fn(), + createRunners: vi.fn().mockResolvedValue([]), findAndStartWarmRunners: vi.fn().mockResolvedValue([]), getGitHubEnterpriseApiUrl: vi.fn().mockReturnValue({ ghesApiUrl: '', @@ -51,6 +52,8 @@ vi.mock('../aws/warm-pool', async () => ({ getPoolStrategy: vi.fn().mockReturnValue('hot'), getWarmPoolConfig: vi.fn().mockReturnValue({ enabled: false, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }), countWarmInstancesByOwner: vi.fn().mockResolvedValue(0), + addToWarmPool: vi.fn().mockResolvedValue(undefined), + emitWarmPoolMetric: vi.fn(), })); const mocktokit = Octokit as MockedClass; diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index daa86d2f66..101713c475 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -2,12 +2,12 @@ import { Octokit } from '@octokit/rest'; import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import yn from 'yn'; -import { bootTimeExceeded, listEC2Runners } from '../aws/runners'; +import { bootTimeExceeded, listEC2Runners, stopRunner, tag } from '../aws/runners'; import { RunnerList } from '../aws/runners.d'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; import { createRunners, findAndStartWarmRunners, getGitHubEnterpriseApiUrl } from '../scale-runners/scale-up'; import { validateSsmParameterStoreTags } from '../scale-runners/scale-up'; -import { getPoolStrategy, getWarmPoolConfig, countWarmInstancesByOwner } from '../aws/warm-pool'; +import { addToWarmPool, getPoolStrategy, getWarmPoolConfig, countWarmInstancesByOwner, emitWarmPoolMetric } from '../aws/warm-pool'; const logger = createChildLogger('pool'); @@ -131,7 +131,7 @@ export async function adjust(event: PoolEvent): Promise { } if (remainingTopUp > 0) { - await createRunners( + const newInstances = await createRunners( { ephemeral, enableJitConfig, @@ -166,12 +166,65 @@ export async function adjust(event: PoolEvent): Promise { githubInstallationClient, 'pool-lambda', ); + + // Warm strategy grace period: wait for runners to register, then stop idle ones + if (poolStrategy === 'warm' && warmPoolConfig.enabled && newInstances.length > 0) { + await warmPoolGracePeriod( + newInstances, + warmPoolConfig.warmPoolReadyDelaySeconds, + runnerOwner, + runnerNamePrefix, + environment, + githubInstallationClient, + ); + } } } else { logger.info(`Pool will not be topped up. Found ${effectivePoolSize} effective pool runners (${numberOfRunnersInPool} running + warm).`); } } +async function warmPoolGracePeriod( + instanceIds: string[], + delaySeconds: number, + runnerOwner: string, + runnerNamePrefix: string, + environment: string, + ghClient: Octokit, +): Promise { + logger.info(`Warm strategy: waiting ${delaySeconds}s grace period for ${instanceIds.length} new instances`); + await new Promise((resolve) => setTimeout(resolve, delaySeconds * 1000)); + + // Re-check runner statuses after grace period + const runnerStatuses = await getGitHubRegisteredRunnnerStatusses(ghClient, runnerOwner, runnerNamePrefix); + + for (const instanceId of instanceIds) { + const status = runnerStatuses.get(instanceId); + if (status?.busy) { + // Runner picked up a job during grace window — leave it running + logger.info(`Runner '${instanceId}' picked up a job during grace period, leaving running`); + await tag(instanceId, [{ Key: 'ghr:warm-pool-grace-hit', Value: 'true' }]).catch(() => {}); + emitWarmPoolMetric('WarmPoolInstanceStarted', 1, { Owner: runnerOwner }); + } else { + // Runner is idle after grace period — stop and add to warm pool + try { + await stopRunner(instanceId); + await addToWarmPool({ + instanceId, + runnerOwner, + environment, + runnerType: 'Org', + }); + await tag(instanceId, [{ Key: 'ghr:warm-pool-member', Value: 'true' }]).catch(() => {}); + emitWarmPoolMetric('WarmPoolInstanceStopped', 1, { Owner: runnerOwner }); + logger.info(`Warm strategy: stopped idle runner '${instanceId}' after grace period`); + } catch (e) { + logger.warn(`Failed to stop runner '${instanceId}' after grace period`, { error: e }); + } + } + } +} + async function getInstallationId(ghesApiUrl: string, org: string): Promise { const ghAuth = await createGithubAppAuth(undefined, ghesApiUrl); const githubClient = await createOctokitClient(ghAuth.token, ghesApiUrl); From de6799f240023425dbf34e3dd41651060fb620ed Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:25:21 +0000 Subject: [PATCH 14/19] docs: auto update terraform docs --- README.md | 7 +++++-- modules/multi-runner/README.md | 2 +- modules/runners/README.md | 12 ++++++++++-- modules/runners/pool/README.md | 2 +- modules/termination-watcher/notification/README.md | 4 ++++ modules/termination-watcher/termination/README.md | 1 + 6 files changed, 22 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1c7dec9270..3786acdbe5 100644 --- a/README.md +++ b/README.md @@ -142,11 +142,12 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh) | [github\_app](#input\_github\_app) | GitHub app parameters, see your github app.
You can optionally create the SSM parameters yourself and provide the ARN and name here, through the `*_ssm` attributes.
If you chose to provide the configuration values directly here,
please ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`).
Note: the provided SSM parameters arn and name have a precedence over the actual value (i.e `key_base64_ssm` has a precedence over `key_base64` etc). | 
object({
    key_base64 = optional(string)
    key_base64_ssm = optional(object({
      arn  = string
      name = string
    }))
    id = optional(string)
    id_ssm = optional(object({
      arn  = string
      name = string
    }))
    webhook_secret = optional(string)
    webhook_secret_ssm = optional(object({
      arn  = string
      name = string
    }))
  })
| n/a | yes | | [iam\_overrides](#input\_iam\_overrides) | This map provides the possibility to override some IAM defaults. Note that when using this variable, you are responsible for ensuring the role has necessary permissions to access required resources. `override_instance_profile`: When set to true, uses the instance profile name specified in `instance_profile_name` instead of creating a new instance profile. `override_runner_role`: When set to true, uses the role ARN specified in `runner_role_arn` instead of creating a new IAM role. | 
object({
    override_instance_profile = optional(bool, null)
    instance_profile_name     = optional(string, null)
    override_runner_role      = optional(bool, null)
    runner_role_arn           = optional(string, null)
  })
| 
{
  "instance_profile_name": null,
  "override_instance_profile": false,
  "override_runner_role": false,
  "runner_role_arn": null
}
| no | | [idle\_config](#input\_idle\_config) | List of time periods, defined as a cron expression, to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    cron             = string
    timeZone         = string
    idleCount        = number
    evictionStrategy = optional(string, "oldest_first")
  }))
| `[]` | no | -| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for spot instances. AWS recommends using `price-capacity-optimized` however the AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | +| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | | [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no | | [instance\_target\_capacity\_type](#input\_instance\_target\_capacity\_type) | Default lifecycle used for runner instances, can be either `spot` or `on-demand`. | `string` | `"spot"` | no | | [instance\_termination\_watcher](#input\_instance\_termination\_watcher) | Configuration for the instance termination watcher. This feature is Beta, changes will not trigger a major release as long in beta.

`enable`: Enable or disable the spot termination watcher.
'features': Enable or disable features of the termination watcher.
`enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated.
`memory_size`: Memory size limit in MB of the lambda.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`timeout`: Time out of the lambda in seconds.
`zip`: File location of the lambda zip file. | 
object({
    enable = optional(bool, false)
    features = optional(object({
      enable_spot_termination_handler              = optional(bool, true)
      enable_spot_termination_notification_watcher = optional(bool, true)
    }), {})
    enable_runner_deregistration = optional(bool, true)
    memory_size                  = optional(number, null)
    s3_key                       = optional(string, null)
    s3_object_version            = optional(string, null)
    timeout                      = optional(number, null)
    zip                          = optional(string, null)
  })
| `{}` | no | +| [instance\_type\_priorities](#input\_instance\_type\_priorities) | A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`. | `map(number)` | `null` | no | | [instance\_types](#input\_instance\_types) | List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win). | `list(string)` | 
[
  "m5.large",
  "c5.large"
]
| no | | [job\_queue\_retention\_in\_seconds](#input\_job\_queue\_retention\_in\_seconds) | The number of seconds the job is held in the queue before it is purged. | `number` | `86400` | no | | [job\_retry](#input\_job\_retry) | Experimental! Can be removed / changed without trigger a major release.Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app.

`enable`: Enable or disable the job retry feature.
`delay_in_seconds`: The delay in seconds before the job retry check lambda will check the job status.
`delay_backoff`: The backoff factor for the delay.
`lambda_memory_size`: Memory size limit in MB for the job retry check lambda.
`lambda_timeout`: Time out of the job retry check lambda in seconds.
`max_attempts`: The maximum number of attempts to retry the job. | 
object({
    enable             = optional(bool, false)
    delay_in_seconds   = optional(number, 300)
    delay_backoff      = optional(number, 2)
    lambda_memory_size = optional(number, 256)
    lambda_timeout     = optional(number, 30)
    max_attempts       = optional(number, 1)
  })
| `{}` | no | @@ -166,7 +167,7 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh) | [logging\_kms\_key\_id](#input\_logging\_kms\_key\_id) | Specifies the kms key id to encrypt the logs with. | `string` | `null` | no | | [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no | | [matcher\_config\_parameter\_store\_tier](#input\_matcher\_config\_parameter\_store\_tier) | The tier of the parameter store for the matcher configuration. Valid values are `Standard`, and `Advanced`. | `string` | `"Standard"` | no | -| [metrics](#input\_metrics) | Configuration for metrics created by the module, by default disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | 
object({
    enable    = optional(bool, false)
    namespace = optional(string, "GitHub Runners")
    metric = optional(object({
      enable_github_app_rate_limit    = optional(bool, true)
      enable_job_retry                = optional(bool, true)
      enable_spot_termination_warning = optional(bool, true)
    }), {})
  })
| `{}` | no | +| [metrics](#input\_metrics) | Configuration for metrics created by the module, by default disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | 
object({
    enable    = optional(bool, false)
    namespace = optional(string, "GitHub Runners")
    metric = optional(object({
      enable_github_app_rate_limit    = optional(bool, true)
      enable_job_retry                = optional(bool, true)
      enable_spot_termination_warning = optional(bool, true)
      enable_warm_pool                = optional(bool, true)
    }), {})
  })
| `{}` | no | | [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated, if not busy. | `number` | `null` | no | | [parameter\_store\_tags](#input\_parameter\_store\_tags) | Map of tags that will be added to all the SSM Parameter Store parameters created by the Lambda function. | `map(string)` | `{}` | no | | [pool\_config](#input\_pool\_config) | The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for weekdays to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC). | 
list(object({
    schedule_expression          = string
    schedule_expression_timezone = optional(string)
    size                         = number
  }))
| `[]` | no | @@ -174,6 +175,7 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh) | [pool\_lambda\_reserved\_concurrent\_executions](#input\_pool\_lambda\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no | | [pool\_lambda\_timeout](#input\_pool\_lambda\_timeout) | Time out for the pool lambda in seconds. | `number` | `60` | no | | [pool\_runner\_owner](#input\_pool\_runner\_owner) | The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported. | `string` | `null` | no | +| [pool\_strategy](#input\_pool\_strategy) | Strategy for maintaining idle runners: `hot` (default, traditional) or `warm` (stop instead of terminate). Requires `warm_pool_config.enabled = true` when set to `warm`. | `string` | `"hot"` | no | | [prefix](#input\_prefix) | The prefix used for naming resources | `string` | `"github-actions"` | no | | [queue\_encryption](#input\_queue\_encryption) | Configure how data on queues managed by the modules is encrypted at REST. Options are encrypted via SSE, non encrypted and via KMS. By default encrypted via SSE is enabled. See for more details the Terraform `aws_sqs_queue` resource https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue. | 
object({
    kms_data_key_reuse_period_seconds = number
    kms_master_key_id                 = string
    sqs_managed_sse_enabled           = bool
  })
| 
{
  "kms_data_key_reuse_period_seconds": null,
  "kms_master_key_id": null,
  "sqs_managed_sse_enabled": true
}
| no | | [queue\_selection\_strategy](#input\_queue\_selection\_strategy) | Strategy used to pick a queue when multiple runner configurations match a job equally well. `first` keeps the historical deterministic behaviour (the first matching queue by priority). `random` spreads jobs across the matching queues to avoid concentrating load on a single one. `all` scales up one runner per matching queue and lets the first to become available take the job (favouring speed over cost; this multiplies instance launches and runner registrations per job). | `string` | `"first"` | no | @@ -237,6 +239,7 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh) | [userdata\_pre\_install](#input\_userdata\_pre\_install) | Script to be ran before the GitHub Actions runner is installed on the EC2 instances | `string` | `""` | no | | [userdata\_template](#input\_userdata\_template) | Alternative user-data template file path, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored. | `string` | `null` | no | | [vpc\_id](#input\_vpc\_id) | The VPC for security groups of the action runners. | `string` | n/a | yes | +| [warm\_pool\_config](#input\_warm\_pool\_config) | Configuration for the warm pool feature. When enabled, idle runners are stopped instead of terminated, allowing 10-30s restart times instead of 2-5 minute cold starts. | 
object({
    enabled                       = optional(bool, false)
    max_warm_instances            = optional(number, 3)
    max_warm_age_hours            = optional(number, 168)
    warm_pool_ready_delay_seconds = optional(number, 30)
  })
| `{}` | no | | [webhook\_lambda\_apigateway\_access\_log\_settings](#input\_webhook\_lambda\_apigateway\_access\_log\_settings) | Access log settings for webhook API gateway. | 
object({
    destination_arn = string
    format          = string
  })
| `null` | no | | [webhook\_lambda\_memory\_size](#input\_webhook\_lambda\_memory\_size) | Memory size limit in MB for webhook lambda in. | `number` | `256` | no | | [webhook\_lambda\_s3\_key](#input\_webhook\_lambda\_s3\_key) | S3 key for webhook lambda function. Required if using S3 bucket to specify lambdas. | `string` | `null` | no | diff --git a/modules/multi-runner/README.md b/modules/multi-runner/README.md index 13085c2b3c..1131d184df 100644 --- a/modules/multi-runner/README.md +++ b/modules/multi-runner/README.md @@ -152,7 +152,7 @@ module "multi-runner" { | [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no | | [matcher\_config\_parameter\_store\_tier](#input\_matcher\_config\_parameter\_store\_tier) | The tier of the parameter store for the matcher configuration. Valid values are `Standard`, and `Advanced`. | `string` | `"Standard"` | no | | [metrics](#input\_metrics) | Configuration for metrics created by the module, by default metrics are disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | 
object({
    enable    = optional(bool, false)
    namespace = optional(string, "GitHub Runners")
    metric = optional(object({
      enable_github_app_rate_limit    = optional(bool, true)
      enable_job_retry                = optional(bool, true)
      enable_spot_termination_warning = optional(bool, true)
    }), {})
  })
| `{}` | no | -| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {
runner\_config: {
runner\_os: "The EC2 Operating System type to use for action runner instances (linux, osx, windows)."
runner\_architecture: "The platform architecture of the runner instance\_type."
runner\_metadata\_options: "(Optional) Metadata options for the ec2 runner instances."
ami: "(Optional) AMI configuration for the action runner instances. This object allows you to specify all AMI-related settings in one place."
create\_service\_linked\_role\_spot: (Optional) create the serviced linked role for spot instances that is required by the scale-up lambda.
credit\_specification: "(Optional) The credit specification of the runner instance\_type. Can be unset, `standard` or `unlimited`.
delay\_webhook\_event: "The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event."
disable\_runner\_autoupdate: "Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)"
ebs\_optimized: "The EC2 EBS optimized configuration."
enable\_ephemeral\_runners: "Enable ephemeral runners, runners will only be used once."
enable\_job\_queued\_check: Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners."
enable\_on\_demand\_failover\_for\_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later."
scale\_errors: "List of aws error codes that should trigger retry during scale up. This list will replace the default errors defined in the variable `defaultScaleErrors` in https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/lambdas/functions/control-plane/src/aws/runners.ts"
enable\_organization\_runners: "Register runners to organization, instead of repo level"
enable\_runner\_binaries\_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
enable\_ssm\_on\_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances."
enable\_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI."
instance\_allocation\_strategy: "The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`."
instance\_max\_spot\_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet."
instance\_target\_capacity\_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`."
instance\_types: "List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)."
job\_queue\_retention\_in\_seconds: "The number of seconds the job is held in the queue before it is purged"
minimum\_running\_time\_in\_minutes: "The time an ec2 action runner should be running at minimum before terminated if not busy."
pool\_runner\_owner: "The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported."
runner\_additional\_security\_group\_ids: "List of additional security groups IDs to apply to the runner. If added outside the multi\_runner\_config block, the additional security group(s) will be applied to all runner configs. If added inside the multi\_runner\_config, the additional security group(s) will be applied to the individual runner."
runner\_as\_root: "Run the action runner under the root user. Variable `runner_run_as` will be ignored."
runner\_boot\_time\_in\_minutes: "The minimum time for an EC2 runner to boot and register as a runner."
runner\_disable\_default\_labels: "Disable default labels for the runners (os, architecture and `self-hosted`). If enabled, the runner will only have the extra labels provided in `runner_extra_labels`. In case you on own start script is used, this configuration parameter needs to be parsed via SSM."
runner\_extra\_labels: "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `multi_runner_config.matcherConfig.exactMatch`. GitHub read-only labels should not be provided."
runner\_group\_name: "Name of the runner group."
runner\_name\_prefix: "Prefix for the GitHub runner name."
runner\_run\_as: "Run the GitHub actions agent as user."
runners\_maximum\_count: "The maximum number of runners that will be created. Setting the variable to `-1` disables the maximum check."
scale\_down\_schedule\_expression: "Scheduler expression to check every x for scale down."
scale\_up\_reserved\_concurrent\_executions: "Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations."
lambda\_event\_source\_mapping\_batch\_size: "(Optional) Maximum number of records per Lambda invocation for this runner flavor. Overrides the module-level `lambda_event_source_mapping_batch_size` when set."
lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds: "(Optional) Maximum seconds to gather records before invoking Lambda for this runner flavor. Overrides the module-level `lambda_event_source_mapping_maximum_batching_window_in_seconds` when set."
userdata\_template: "Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored."
enable\_jit\_config: "Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is available. In case you are upgrading from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI."
enable\_runner\_detailed\_monitoring: "Should detailed monitoring be enabled for the runner. Set this to true if you want to use detailed monitoring. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html for details."
enable\_cloudwatch\_agent: "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
cloudwatch\_config: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
userdata\_pre\_install: "Script to be ran before the GitHub Actions runner is installed on the EC2 instances"
userdata\_post\_install: "Script to be ran after the GitHub Actions runner is installed on the EC2 instances"
runner\_hook\_job\_started: "Script to be ran in the runner environment at the beginning of every job"
runner\_hook\_job\_completed: "Script to be ran in the runner environment at the end of every job"
runner\_ec2\_tags: "Map of tags that will be added to the launch template instance tag specifications."
runner\_iam\_role\_managed\_policy\_arns: "Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role"
vpc\_id: "The VPC for security groups of the action runners. If not set uses the value of `var.vpc_id`."
subnet\_ids: "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. If not set, uses the value of `var.subnet_ids`."
idle\_config: "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle."
license\_specifications: "Optional EC2 License Manager license configuration ARNs for the runner launch template. Required for macOS dedicated-host runners when the host resource group uses a Mac dedicated host license configuration."
use\_dedicated\_host: "Experimental! Can be removed / changed without trigger a major release. Whether to use EC2 dedicated hosts for the runners. Needed for macos runners Note that using dedicated hosts can increase cost significantly."
runner\_log\_files: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
block\_device\_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`."
job\_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app."
pool\_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)."
iam\_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances."
}
matcherConfig: {
labelMatchers: "The list of list of labels supported by the runner configuration. `[[self-hosted, linux, x64, example]]`"
exactMatch: "DEPRECATED: Use `bidirectionalLabelMatch` instead. If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ workflow label matches it will trigger the webhook. Note: this only checks that workflow labels are a subset of runner labels, not the reverse."
bidirectionalLabelMatch: "If set to true, the runner labels and workflow job labels must be an exact two-way match (same set, any order, no extras or missing labels). This is stricter than `exactMatch` which only checks that workflow labels are a subset of runner labels. When false, if __any__ workflow label matches it will trigger the webhook."
priority: "If set it defines the priority of the matcher, the matcher with the lowest priority will be evaluated first. Default is 999, allowed values 0-999."
enableDynamicLabels: "Experimental! When true the dispatcher allows `ghr-*` dynamic labels for jobs routed to this runner. Default false."
ec2DynamicLabelsPolicy: "Optional policy for `ghr-ec2-*` labels evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose EC2 dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`."
}
redrive\_build\_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries."
} | 
map(object({
    runner_config = object({
      runner_os           = string
      runner_architecture = string
      runner_metadata_options = optional(map(any), {
        instance_metadata_tags      = "enabled"
        http_endpoint               = "enabled"
        http_tokens                 = "required"
        http_put_response_hop_limit = 1
      })
      ami = optional(object({
        filter               = optional(map(list(string)), { state = ["available"] })
        owners               = optional(list(string), ["amazon"])
        id_ssm_parameter_arn = optional(string, null)
        kms_key_arn          = optional(string, null)
      }), null)
      create_service_linked_role_spot      = optional(bool, false)
      credit_specification                 = optional(string, null)
      delay_webhook_event                  = optional(number, 30)
      disable_runner_autoupdate            = optional(bool, false)
      ebs_optimized                        = optional(bool, false)
      enable_ephemeral_runners             = optional(bool, false)
      enable_job_queued_check              = optional(bool, null)
      enable_on_demand_failover_for_errors = optional(list(string), [])
      scale_errors = optional(list(string), [
        "UnfulfillableCapacity",
        "MaxSpotInstanceCountExceeded",
        "TargetCapacityLimitExceededException",
        "RequestLimitExceeded",
        "ResourceLimitExceeded",
        "MaxSpotInstanceCountExceeded",
        "MaxSpotFleetRequestCountExceeded",
        "InsufficientInstanceCapacity",
        "InsufficientCapacityOnHost",
      ])
      enable_organization_runners                                    = optional(bool, false)
      enable_runner_binaries_syncer                                  = optional(bool, true)
      enable_ssm_on_runners                                          = optional(bool, false)
      enable_userdata                                                = optional(bool, true)
      instance_allocation_strategy                                   = optional(string, "lowest-price")
      instance_max_spot_price                                        = optional(string, null)
      instance_target_capacity_type                                  = optional(string, "spot")
      instance_types                                                 = list(string)
      job_queue_retention_in_seconds                                 = optional(number, 86400)
      minimum_running_time_in_minutes                                = optional(number, null)
      pool_runner_owner                                              = optional(string, null)
      runner_as_root                                                 = optional(bool, false)
      runner_boot_time_in_minutes                                    = optional(number, 5)
      runner_disable_default_labels                                  = optional(bool, false)
      runner_extra_labels                                            = optional(list(string), [])
      runner_group_name                                              = optional(string, "Default")
      runner_name_prefix                                             = optional(string, "")
      runner_run_as                                                  = optional(string, "ec2-user")
      runners_maximum_count                                          = number
      runner_additional_security_group_ids                           = optional(list(string), [])
      scale_down_schedule_expression                                 = optional(string, "cron(*/5 * * * ? *)")
      scale_up_reserved_concurrent_executions                        = optional(number, 1)
      lambda_event_source_mapping_batch_size                         = optional(number, null)
      lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
      userdata_template                                              = optional(string, null)
      userdata_content                                               = optional(string, null)
      enable_jit_config                                              = optional(bool, null)
      enable_runner_detailed_monitoring                              = optional(bool, false)
      enable_cloudwatch_agent                                        = optional(bool, true)
      cloudwatch_config                                              = optional(string, null)
      userdata_pre_install                                           = optional(string, "")
      userdata_post_install                                          = optional(string, "")
      runner_hook_job_started                                        = optional(string, "")
      runner_hook_job_completed                                      = optional(string, "")
      runner_ec2_tags                                                = optional(map(string), {})
      runner_iam_role_managed_policy_arns                            = optional(list(string), [])
      vpc_id                                                         = optional(string, null)
      subnet_ids                                                     = optional(list(string), null)
      idle_config = optional(list(object({
        cron             = string
        timeZone         = string
        idleCount        = number
        evictionStrategy = optional(string, "oldest_first")
      })), [])
      cpu_options = optional(object({
        core_count            = optional(number)
        threads_per_core      = optional(number)
        amd_sev_snp           = optional(string)
        nested_virtualization = optional(string)
      }), null)
      placement = optional(object({
        affinity                = optional(string)
        availability_zone       = optional(string)
        group_id                = optional(string)
        group_name              = optional(string)
        host_id                 = optional(string)
        host_resource_group_arn = optional(string)
        spread_domain           = optional(string)
        tenancy                 = optional(string)
        partition_number        = optional(number)
      }), null)
      license_specifications = optional(list(object({
        license_configuration_arn = string
      })), [])
      use_dedicated_host = optional(bool, false)
      runner_log_files = optional(list(object({
        log_group_name   = string
        prefix_log_group = bool
        file_path        = string
        log_stream_name  = string
        log_class        = optional(string, "STANDARD")
      })), null)
      block_device_mappings = optional(list(object({
        delete_on_termination      = optional(bool, true)
        device_name                = optional(string, "/dev/xvda")
        encrypted                  = optional(bool, true)
        iops                       = optional(number)
        kms_key_id                 = optional(string)
        snapshot_id                = optional(string)
        throughput                 = optional(number)
        volume_initialization_rate = optional(number)
        volume_size                = number
        volume_type                = optional(string, "gp3")
        })), [{
        volume_size = 30
      }])
      pool_config = optional(list(object({
        schedule_expression          = string
        schedule_expression_timezone = optional(string)
        size                         = number
      })), [])
      job_retry = optional(object({
        enable             = optional(bool, false)
        delay_in_seconds   = optional(number, 300)
        delay_backoff      = optional(number, 2)
        lambda_memory_size = optional(number, 256)
        lambda_timeout     = optional(number, 30)
        max_attempts       = optional(number, 1)
      }), {})
      iam_overrides = optional(object({
        override_instance_profile = optional(bool, null)
        instance_profile_name     = optional(string, null)
        override_runner_role      = optional(bool, null)
        runner_role_arn           = optional(string, null)
        }), {
        override_instance_profile = false
        instance_profile_name     = null
        override_runner_role      = false
        runner_role_arn           = null
      })
    })
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = optional(bool, false)
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      ec2DynamicLabelsPolicy  = optional(any, null)
    })
    redrive_build_queue = optional(object({
      enabled         = bool
      maxReceiveCount = number
      }), {
      enabled         = false
      maxReceiveCount = null
    })
  }))
| n/a | yes | +| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {
runner\_config: {
runner\_os: "The EC2 Operating System type to use for action runner instances (linux, osx, windows)."
runner\_architecture: "The platform architecture of the runner instance\_type."
runner\_metadata\_options: "(Optional) Metadata options for the ec2 runner instances."
ami: "(Optional) AMI configuration for the action runner instances. This object allows you to specify all AMI-related settings in one place."
create\_service\_linked\_role\_spot: (Optional) create the serviced linked role for spot instances that is required by the scale-up lambda.
credit\_specification: "(Optional) The credit specification of the runner instance\_type. Can be unset, `standard` or `unlimited`.
delay\_webhook\_event: "The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event."
disable\_runner\_autoupdate: "Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)"
ebs\_optimized: "The EC2 EBS optimized configuration."
enable\_ephemeral\_runners: "Enable ephemeral runners, runners will only be used once."
enable\_job\_queued\_check: Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners."
enable\_on\_demand\_failover\_for\_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later."
scale\_errors: "List of aws error codes that should trigger retry during scale up. This list will replace the default errors defined in the variable `defaultScaleErrors` in https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/lambdas/functions/control-plane/src/aws/runners.ts"
enable\_organization\_runners: "Register runners to organization, instead of repo level"
enable\_runner\_binaries\_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
enable\_ssm\_on\_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances."
enable\_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI."
instance\_allocation\_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`."
instance\_type\_priorities: "A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`."
instance\_max\_spot\_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet."
instance\_target\_capacity\_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`."
instance\_types: "List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)."
job\_queue\_retention\_in\_seconds: "The number of seconds the job is held in the queue before it is purged"
minimum\_running\_time\_in\_minutes: "The time an ec2 action runner should be running at minimum before terminated if not busy."
pool\_runner\_owner: "The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported."
runner\_additional\_security\_group\_ids: "List of additional security groups IDs to apply to the runner. If added outside the multi\_runner\_config block, the additional security group(s) will be applied to all runner configs. If added inside the multi\_runner\_config, the additional security group(s) will be applied to the individual runner."
runner\_as\_root: "Run the action runner under the root user. Variable `runner_run_as` will be ignored."
runner\_boot\_time\_in\_minutes: "The minimum time for an EC2 runner to boot and register as a runner."
runner\_disable\_default\_labels: "Disable default labels for the runners (os, architecture and `self-hosted`). If enabled, the runner will only have the extra labels provided in `runner_extra_labels`. In case you on own start script is used, this configuration parameter needs to be parsed via SSM."
runner\_extra\_labels: "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `multi_runner_config.matcherConfig.exactMatch`. GitHub read-only labels should not be provided."
runner\_group\_name: "Name of the runner group."
runner\_name\_prefix: "Prefix for the GitHub runner name."
runner\_run\_as: "Run the GitHub actions agent as user."
runners\_maximum\_count: "The maximum number of runners that will be created. Setting the variable to `-1` disables the maximum check."
scale\_down\_schedule\_expression: "Scheduler expression to check every x for scale down."
scale\_up\_reserved\_concurrent\_executions: "Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations."
lambda\_event\_source\_mapping\_batch\_size: "(Optional) Maximum number of records per Lambda invocation for this runner flavor. Overrides the module-level `lambda_event_source_mapping_batch_size` when set."
lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds: "(Optional) Maximum seconds to gather records before invoking Lambda for this runner flavor. Overrides the module-level `lambda_event_source_mapping_maximum_batching_window_in_seconds` when set."
userdata\_template: "Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored."
enable\_jit\_config: "Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is available. In case you are upgrading from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI."
enable\_runner\_detailed\_monitoring: "Should detailed monitoring be enabled for the runner. Set this to true if you want to use detailed monitoring. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html for details."
enable\_cloudwatch\_agent: "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
cloudwatch\_config: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
userdata\_pre\_install: "Script to be ran before the GitHub Actions runner is installed on the EC2 instances"
userdata\_post\_install: "Script to be ran after the GitHub Actions runner is installed on the EC2 instances"
runner\_hook\_job\_started: "Script to be ran in the runner environment at the beginning of every job"
runner\_hook\_job\_completed: "Script to be ran in the runner environment at the end of every job"
runner\_ec2\_tags: "Map of tags that will be added to the launch template instance tag specifications."
runner\_iam\_role\_managed\_policy\_arns: "Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role"
vpc\_id: "The VPC for security groups of the action runners. If not set uses the value of `var.vpc_id`."
subnet\_ids: "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. If not set, uses the value of `var.subnet_ids`."
idle\_config: "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle."
license\_specifications: "Optional EC2 License Manager license configuration ARNs for the runner launch template. Required for macOS dedicated-host runners when the host resource group uses a Mac dedicated host license configuration."
use\_dedicated\_host: "Experimental! Can be removed / changed without trigger a major release. Whether to use EC2 dedicated hosts for the runners. Needed for macos runners Note that using dedicated hosts can increase cost significantly."
runner\_log\_files: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
block\_device\_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`."
job\_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app."
pool\_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)."
pool\_strategy: "Strategy for maintaining idle runners: `hot` (default, traditional) or `warm` (stop instead of terminate)."
warm\_pool\_config: "Configuration for the warm pool feature. When enabled, idle runners are stopped instead of terminated for faster restart."
iam\_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances."
}
matcherConfig: {
labelMatchers: "The list of list of labels supported by the runner configuration. `[[self-hosted, linux, x64, example]]`"
exactMatch: "DEPRECATED: Use `bidirectionalLabelMatch` instead. If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ workflow label matches it will trigger the webhook. Note: this only checks that workflow labels are a subset of runner labels, not the reverse."
bidirectionalLabelMatch: "If set to true, the runner labels and workflow job labels must be an exact two-way match (same set, any order, no extras or missing labels). This is stricter than `exactMatch` which only checks that workflow labels are a subset of runner labels. When false, if __any__ workflow label matches it will trigger the webhook."
priority: "If set it defines the priority of the matcher, the matcher with the lowest priority will be evaluated first. Default is 999, allowed values 0-999."
enableDynamicLabels: "Experimental! When true the dispatcher allows `ghr-*` dynamic labels for jobs routed to this runner. Default false."
ec2DynamicLabelsPolicy: "Optional policy for `ghr-ec2-*` labels evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose EC2 dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`."
}
redrive\_build\_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries."
} | 
map(object({
    runner_config = object({
      runner_os           = string
      runner_architecture = string
      runner_metadata_options = optional(map(any), {
        instance_metadata_tags      = "enabled"
        http_endpoint               = "enabled"
        http_tokens                 = "required"
        http_put_response_hop_limit = 1
      })
      ami = optional(object({
        filter               = optional(map(list(string)), { state = ["available"] })
        owners               = optional(list(string), ["amazon"])
        id_ssm_parameter_arn = optional(string, null)
        kms_key_arn          = optional(string, null)
      }), null)
      create_service_linked_role_spot      = optional(bool, false)
      credit_specification                 = optional(string, null)
      delay_webhook_event                  = optional(number, 30)
      disable_runner_autoupdate            = optional(bool, false)
      ebs_optimized                        = optional(bool, false)
      enable_ephemeral_runners             = optional(bool, false)
      enable_job_queued_check              = optional(bool, null)
      enable_on_demand_failover_for_errors = optional(list(string), [])
      scale_errors = optional(list(string), [
        "UnfulfillableCapacity",
        "MaxSpotInstanceCountExceeded",
        "TargetCapacityLimitExceededException",
        "RequestLimitExceeded",
        "ResourceLimitExceeded",
        "MaxSpotInstanceCountExceeded",
        "MaxSpotFleetRequestCountExceeded",
        "InsufficientInstanceCapacity",
        "InsufficientCapacityOnHost",
      ])
      enable_organization_runners                                    = optional(bool, false)
      enable_runner_binaries_syncer                                  = optional(bool, true)
      enable_ssm_on_runners                                          = optional(bool, false)
      enable_userdata                                                = optional(bool, true)
      instance_allocation_strategy                                   = optional(string, "lowest-price")
      instance_type_priorities                                       = optional(map(number), null)
      instance_max_spot_price                                        = optional(string, null)
      instance_target_capacity_type                                  = optional(string, "spot")
      instance_types                                                 = list(string)
      job_queue_retention_in_seconds                                 = optional(number, 86400)
      minimum_running_time_in_minutes                                = optional(number, null)
      pool_runner_owner                                              = optional(string, null)
      runner_as_root                                                 = optional(bool, false)
      runner_boot_time_in_minutes                                    = optional(number, 5)
      runner_disable_default_labels                                  = optional(bool, false)
      runner_extra_labels                                            = optional(list(string), [])
      runner_group_name                                              = optional(string, "Default")
      runner_name_prefix                                             = optional(string, "")
      runner_run_as                                                  = optional(string, "ec2-user")
      runners_maximum_count                                          = number
      runner_additional_security_group_ids                           = optional(list(string), [])
      scale_down_schedule_expression                                 = optional(string, "cron(*/5 * * * ? *)")
      scale_up_reserved_concurrent_executions                        = optional(number, 1)
      lambda_event_source_mapping_batch_size                         = optional(number, null)
      lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
      userdata_template                                              = optional(string, null)
      userdata_content                                               = optional(string, null)
      enable_jit_config                                              = optional(bool, null)
      enable_runner_detailed_monitoring                              = optional(bool, false)
      enable_cloudwatch_agent                                        = optional(bool, true)
      cloudwatch_config                                              = optional(string, null)
      userdata_pre_install                                           = optional(string, "")
      userdata_post_install                                          = optional(string, "")
      runner_hook_job_started                                        = optional(string, "")
      runner_hook_job_completed                                      = optional(string, "")
      runner_ec2_tags                                                = optional(map(string), {})
      runner_iam_role_managed_policy_arns                            = optional(list(string), [])
      vpc_id                                                         = optional(string, null)
      subnet_ids                                                     = optional(list(string), null)
      idle_config = optional(list(object({
        cron             = string
        timeZone         = string
        idleCount        = number
        evictionStrategy = optional(string, "oldest_first")
      })), [])
      cpu_options = optional(object({
        core_count            = optional(number)
        threads_per_core      = optional(number)
        amd_sev_snp           = optional(string)
        nested_virtualization = optional(string)
      }), null)
      placement = optional(object({
        affinity                = optional(string)
        availability_zone       = optional(string)
        group_id                = optional(string)
        group_name              = optional(string)
        host_id                 = optional(string)
        host_resource_group_arn = optional(string)
        spread_domain           = optional(string)
        tenancy                 = optional(string)
        partition_number        = optional(number)
      }), null)
      license_specifications = optional(list(object({
        license_configuration_arn = string
      })), [])
      use_dedicated_host = optional(bool, false)
      runner_log_files = optional(list(object({
        log_group_name   = string
        prefix_log_group = bool
        file_path        = string
        log_stream_name  = string
        log_class        = optional(string, "STANDARD")
      })), null)
      block_device_mappings = optional(list(object({
        delete_on_termination      = optional(bool, true)
        device_name                = optional(string, "/dev/xvda")
        encrypted                  = optional(bool, true)
        iops                       = optional(number)
        kms_key_id                 = optional(string)
        snapshot_id                = optional(string)
        throughput                 = optional(number)
        volume_initialization_rate = optional(number)
        volume_size                = number
        volume_type                = optional(string, "gp3")
        })), [{
        volume_size = 30
      }])
      pool_config = optional(list(object({
        schedule_expression          = string
        schedule_expression_timezone = optional(string)
        size                         = number
      })), [])
      pool_strategy = optional(string, "hot")
      warm_pool_config = optional(object({
        enabled                       = optional(bool, false)
        max_warm_instances            = optional(number, 3)
        max_warm_age_hours            = optional(number, 168)
        warm_pool_ready_delay_seconds = optional(number, 30)
      }), {})
      job_retry = optional(object({
        enable             = optional(bool, false)
        delay_in_seconds   = optional(number, 300)
        delay_backoff      = optional(number, 2)
        lambda_memory_size = optional(number, 256)
        lambda_timeout     = optional(number, 30)
        max_attempts       = optional(number, 1)
      }), {})
      iam_overrides = optional(object({
        override_instance_profile = optional(bool, null)
        instance_profile_name     = optional(string, null)
        override_runner_role      = optional(bool, null)
        runner_role_arn           = optional(string, null)
        }), {
        override_instance_profile = false
        instance_profile_name     = null
        override_runner_role      = false
        runner_role_arn           = null
      })
    })
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = optional(bool, false)
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      ec2DynamicLabelsPolicy  = optional(any, null)
    })
    redrive_build_queue = optional(object({
      enabled         = bool
      maxReceiveCount = number
      }), {
      enabled         = false
      maxReceiveCount = null
    })
  }))
| n/a | yes | | [parameter\_store\_tags](#input\_parameter\_store\_tags) | Map of tags that will be added to all the SSM Parameter Store parameters created by the Lambda function. | `map(string)` | `{}` | no | | [pool\_lambda\_reserved\_concurrent\_executions](#input\_pool\_lambda\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no | | [pool\_lambda\_timeout](#input\_pool\_lambda\_timeout) | Time out for the pool lambda in seconds. | `number` | `60` | no | diff --git a/modules/runners/README.md b/modules/runners/README.md index 112d2894ad..b13ec214dc 100644 --- a/modules/runners/README.md +++ b/modules/runners/README.md @@ -80,6 +80,7 @@ yarn run dist | [aws_cloudwatch_log_group.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | | [aws_cloudwatch_log_group.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | | [aws_cloudwatch_log_group.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_dynamodb_table.warm_pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table) | resource | | [aws_iam_instance_profile.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [aws_iam_policy.ami_id_ssm_parameter_read](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | @@ -92,12 +93,15 @@ yarn run dist | [aws_iam_role_policy.dist_bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.ec2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.job_retry_sqs_publish](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.pool_warm_pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.runner_session_manager_aws_managed](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.scale_down_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_down_warm_pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.scale_down_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.scale_up_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.scale_up_warm_pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.scale_up_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.service_linked_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | @@ -106,6 +110,7 @@ yarn run dist | [aws_iam_role_policy.ssm_parameters](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_iam_role_policy_attachment.ami_id_ssm_parameter_read](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.managed_policies](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.scale_down_ami_id_ssm_parameter_read](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.scale_down_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.scale_up_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.ssm_housekeeper_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | @@ -165,10 +170,11 @@ yarn run dist | [github\_app\_parameters](#input\_github\_app\_parameters) | Parameter Store for GitHub App Parameters. | 
object({
    key_base64 = map(string)
    id         = map(string)
  })
| n/a | yes | | [iam\_overrides](#input\_iam\_overrides) | This map provides the possibility to override some IAM defaults. The following attributes are supported: `instance_profile_name` overrides the instance profile name used in the launch template. `runner_role_arn` overrides the IAM role ARN used for the runner instances. | 
object({
    override_instance_profile = optional(bool, null)
    instance_profile_name     = optional(string, null)
    override_runner_role      = optional(bool, null)
    runner_role_arn           = optional(string, null)
  })
| 
{
  "instance_profile_name": null,
  "override_instance_profile": false,
  "override_runner_role": false,
  "runner_role_arn": null
}
| no | | [idle\_config](#input\_idle\_config) | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    cron             = string
    timeZone         = string
    idleCount        = number
    evictionStrategy = optional(string, "oldest_first")
  }))
| `[]` | no | -| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | +| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | | [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the prefix will be used. | `string` | `null` | no | | [instance\_target\_capacity\_type](#input\_instance\_target\_capacity\_type) | Default lifecycle used runner instances, can be either `spot` or `on-demand`. | `string` | `"spot"` | no | +| [instance\_type\_priorities](#input\_instance\_type\_priorities) | A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`. | `map(number)` | `null` | no | | [instance\_types](#input\_instance\_types) | List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win). | `list(string)` | `null` | no | | [job\_retry](#input\_job\_retry) | Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app.

`enable`: Enable or disable the job retry feature.
`delay_in_seconds`: The delay in seconds before the job retry check lambda will check the job status.
`delay_backoff`: The backoff factor for the delay.
`lambda_memory_size`: Memory size limit in MB for the job retry check lambda.
'lambda\_reserved\_concurrent\_executions': Amount of reserved concurrent executions for the job retry check lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations.
`lambda_timeout`: Time out of the job retry check lambda in seconds.
`max_attempts`: The maximum number of attempts to retry the job. | 
object({
    enable                                = optional(bool, false)
    delay_in_seconds                      = optional(number, 300)
    delay_backoff                         = optional(number, 2)
    lambda_memory_size                    = optional(number, 256)
    lambda_reserved_concurrent_executions = optional(number, 1)

    lambda_timeout = optional(number, 30)

    max_attempts = optional(number, 1)
  })
| `{}` | no | | [key\_name](#input\_key\_name) | Key pair name | `string` | `null` | no | @@ -192,7 +198,7 @@ yarn run dist | [logging\_kms\_key\_id](#input\_logging\_kms\_key\_id) | Specifies the kms key id to encrypt the logs with | `string` | `null` | no | | [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no | | [metadata\_options](#input\_metadata\_options) | Metadata options for the ec2 runner instances. By default, the module uses metadata tags for bootstrapping the runner, only disable `instance_metadata_tags` when using custom scripts for starting the runner. | `map(any)` | 
{
  "http_endpoint": "enabled",
  "http_put_response_hop_limit": 1,
  "http_tokens": "required",
  "instance_metadata_tags": "enabled"
}
| no | -| [metrics](#input\_metrics) | Configuration for metrics created by the module, by default metrics are disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | 
object({
    enable    = optional(bool, false)
    namespace = optional(string, "GitHub Runners")
    metric = optional(object({
      enable_github_app_rate_limit    = optional(bool, true)
      enable_job_retry                = optional(bool, true)
      enable_spot_termination_warning = optional(bool, true)
    }), {})
  })
| `{}` | no | +| [metrics](#input\_metrics) | Configuration for metrics created by the module, by default metrics are disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | 
object({
    enable    = optional(bool, false)
    namespace = optional(string, "GitHub Runners")
    metric = optional(object({
      enable_github_app_rate_limit    = optional(bool, true)
      enable_job_retry                = optional(bool, true)
      enable_spot_termination_warning = optional(bool, true)
      enable_warm_pool                = optional(bool, true)
    }), {})
  })
| `{}` | no | | [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated if non busy. If not set the default is calculated based on the OS. | `number` | `null` | no | | [overrides](#input\_overrides) | This map provides the possibility to override some defaults. The following attributes are supported: `name_sg` overrides the `Name` tag for all security groups created by this module. `name_runner_agent_instance` overrides the `Name` tag for the ec2 instance defined in the auto launch configuration. `name_docker_machine_runners` overrides the `Name` tag spot instances created by the runner agent. | `map(string)` | 
{
  "name_runner": "",
  "name_sg": ""
}
| no | | [parameter\_store\_tags](#input\_parameter\_store\_tags) | Map of tags that will be added to all the SSM Parameter Store parameters created by the Lambda function. | `map(string)` | `{}` | no | @@ -202,6 +208,7 @@ yarn run dist | [pool\_lambda\_reserved\_concurrent\_executions](#input\_pool\_lambda\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no | | [pool\_lambda\_timeout](#input\_pool\_lambda\_timeout) | Time out for the pool lambda in seconds. | `number` | `60` | no | | [pool\_runner\_owner](#input\_pool\_runner\_owner) | The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported. | `string` | `null` | no | +| [pool\_strategy](#input\_pool\_strategy) | Strategy for maintaining idle runners.
- `hot`: (default) Traditional behavior. Pool lambda creates fresh instances on a schedule. Idle instances are terminated by scale-down.
- `warm`: Idle instances are stopped (hibernated) instead of terminated. Scale-up will restart warm instances before creating new ones. Requires `warm_pool_config.enabled = true`. | `string` | `"hot"` | no | | [prefix](#input\_prefix) | The prefix used for naming resources | `string` | `"github-actions"` | no | | [role\_path](#input\_role\_path) | The path that will be added to the role; if not set, the prefix will be used. | `string` | `null` | no | | [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no | @@ -240,6 +247,7 @@ yarn run dist | [userdata\_pre\_install](#input\_userdata\_pre\_install) | User-data script snippet to insert before GitHub action runner install | `string` | `""` | no | | [userdata\_template](#input\_userdata\_template) | Alternative user-data template file path, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored. | `string` | `null` | no | | [vpc\_id](#input\_vpc\_id) | The VPC for the security groups. | `string` | n/a | yes | +| [warm\_pool\_config](#input\_warm\_pool\_config) | Configuration for the warm pool feature. When enabled, idle runners are stopped instead of terminated,
allowing 10-30s restart times instead of 2-5 minute cold starts.

`enabled`: Enable or disable the warm pool feature. When enabled, a DynamoDB table is created to track stopped instances.
`max_warm_instances`: Maximum number of stopped instances to keep in the warm pool per runner owner.
`max_warm_age_hours`: Maximum age in hours before a warm instance is terminated by TTL.
`warm_pool_ready_delay_seconds`: Grace period in seconds before a newly launched instance is eligible for warm pool (allows time to pick up jobs). | 
object({
    enabled                       = optional(bool, false)
    max_warm_instances            = optional(number, 3)
    max_warm_age_hours            = optional(number, 168)
    warm_pool_ready_delay_seconds = optional(number, 30)
  })
| `{}` | no | ## Outputs diff --git a/modules/runners/pool/README.md b/modules/runners/pool/README.md index 401c7f2cc8..71f57fcaa8 100644 --- a/modules/runners/pool/README.md +++ b/modules/runners/pool/README.md @@ -49,7 +49,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [aws\_partition](#input\_aws\_partition) | (optional) partition for the arn if not 'aws' | `string` | `"aws"` | no | -| [config](#input\_config) | Lookup details in parent module. | 
object({
    lambda = object({
      log_level                      = string
      logging_retention_in_days      = number
      logging_kms_key_id             = string
      log_class                      = string
      reserved_concurrent_executions = number
      s3_bucket                      = string
      s3_key                         = string
      s3_object_version              = string
      security_group_ids             = list(string)
      runtime                        = string
      architecture                   = string
      memory_size                    = number
      timeout                        = number
      zip                            = string
      subnet_ids                     = list(string)
      parameter_store_tags           = string
    })
    tags = map(string)
    ghes = object({
      url        = string
      ssl_verify = string
    })
    github_app_parameters = object({
      key_base64 = map(string)
      id         = map(string)
    })
    subnet_ids = list(string)
    runner = object({
      disable_runner_autoupdate            = bool
      ephemeral                            = bool
      enable_jit_config                    = bool
      enable_on_demand_failover_for_errors = list(string)
      scale_errors                         = list(string)
      boot_time_in_minutes                 = number
      labels                               = list(string)
      launch_template = object({
        name = string
      })
      group_name  = string
      name_prefix = string
      pool_owner  = string
      role = object({
        arn = string
      })
      use_dedicated_host = bool
    })
    runners_maximum_count         = number
    instance_types                = list(string)
    instance_target_capacity_type = string
    instance_allocation_strategy  = string
    instance_max_spot_price       = string
    prefix                        = string
    pool = list(object({
      schedule_expression          = string
      schedule_expression_timezone = string
      size                         = number
    }))
    role_permissions_boundary            = string
    kms_key_arn                          = string
    ami_kms_key_arn                      = string
    ami_id_ssm_parameter_arn             = string
    role_path                            = string
    ssm_token_path                       = string
    ssm_config_path                      = string
    ami_id_ssm_parameter_name            = string
    ami_id_ssm_parameter_read_policy_arn = string
    arn_ssm_parameters_path_config       = string
    lambda_tags                          = map(string)
    user_agent                           = string
  })
| n/a | yes | +| [config](#input\_config) | Lookup details in parent module. | 
object({
    lambda = object({
      log_level                      = string
      logging_retention_in_days      = number
      logging_kms_key_id             = string
      log_class                      = string
      reserved_concurrent_executions = number
      s3_bucket                      = string
      s3_key                         = string
      s3_object_version              = string
      security_group_ids             = list(string)
      runtime                        = string
      architecture                   = string
      memory_size                    = number
      timeout                        = number
      zip                            = string
      subnet_ids                     = list(string)
      parameter_store_tags           = string
    })
    tags = map(string)
    ghes = object({
      url        = string
      ssl_verify = string
    })
    github_app_parameters = object({
      key_base64 = map(string)
      id         = map(string)
    })
    subnet_ids = list(string)
    runner = object({
      disable_runner_autoupdate            = bool
      ephemeral                            = bool
      enable_jit_config                    = bool
      enable_on_demand_failover_for_errors = list(string)
      scale_errors                         = list(string)
      boot_time_in_minutes                 = number
      labels                               = list(string)
      launch_template = object({
        name = string
      })
      group_name  = string
      name_prefix = string
      pool_owner  = string
      role = object({
        arn = string
      })
      use_dedicated_host = bool
    })
    runners_maximum_count         = number
    instance_types                = list(string)
    instance_type_priorities      = optional(map(number))
    instance_target_capacity_type = string
    instance_allocation_strategy  = string
    instance_max_spot_price       = string
    prefix                        = string
    pool = list(object({
      schedule_expression          = string
      schedule_expression_timezone = string
      size                         = number
    }))
    role_permissions_boundary            = string
    kms_key_arn                          = string
    ami_kms_key_arn                      = string
    ami_id_ssm_parameter_arn             = string
    role_path                            = string
    ssm_token_path                       = string
    ssm_config_path                      = string
    ami_id_ssm_parameter_name            = string
    ami_id_ssm_parameter_read_policy_arn = string
    arn_ssm_parameters_path_config       = string
    lambda_tags                          = map(string)
    user_agent                           = string
    warm_pool_table_name                 = string
    warm_pool_config = object({
      enabled                       = bool
      max_warm_instances            = number
      max_warm_age_hours            = number
      warm_pool_ready_delay_seconds = number
    })
    pool_strategy           = string
    enable_metric_warm_pool = bool
  })
| n/a | yes | | [tracing\_config](#input\_tracing\_config) | Configuration for lambda tracing. | 
object({
    mode                  = optional(string, null)
    capture_http_requests = optional(bool, false)
    capture_error         = optional(bool, false)
  })
| `{}` | no | ## Outputs diff --git a/modules/termination-watcher/notification/README.md b/modules/termination-watcher/notification/README.md index 1a26de3bce..31df74586f 100644 --- a/modules/termination-watcher/notification/README.md +++ b/modules/termination-watcher/notification/README.md @@ -22,10 +22,14 @@ | Name | Type | |------|------| +| [aws_cloudwatch_event_rule.ec2_instance_state_change](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | | [aws_cloudwatch_event_rule.spot_instance_termination_warning](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | | [aws_cloudwatch_event_target.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_cloudwatch_event_target.state_change](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | | [aws_iam_role_policy.lambda_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.ssm_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_lambda_permission.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | +| [aws_lambda_permission.state_change](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | ## Inputs diff --git a/modules/termination-watcher/termination/README.md b/modules/termination-watcher/termination/README.md index 22912b9646..32b32aa54e 100644 --- a/modules/termination-watcher/termination/README.md +++ b/modules/termination-watcher/termination/README.md @@ -25,6 +25,7 @@ | [aws_cloudwatch_event_rule.spot_instance_termination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | | [aws_cloudwatch_event_target.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | | [aws_iam_role_policy.lambda_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.ssm_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_lambda_permission.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | ## Inputs From bb6f8c7394e648199680327aef20dc41da98b60c Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 16:37:27 +0200 Subject: [PATCH 15/19] docs: add warm pool to mkdocs navigation Signed-off-by: Brend Smits --- mkdocs.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mkdocs.yaml b/mkdocs.yaml index 9b98e84a36..663d3567ed 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -81,3 +81,6 @@ nav: - Lambda download: examples/lambda-download.md - Permissions boundary: examples/permissions-boundary.md - AMI examples: ami-examples/index.md + - Warm Pool: warm-pool.md + - Additional notes: additional_notes.md + - Mac runners: mac-runners.md From f535ed5123abab44d1b526c296b822d91261f7e8 Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 16:57:41 +0200 Subject: [PATCH 16/19] fix(warm-pool): address PR review feedback - Remove unused imports in test files (CodeQL warnings) - Move check block to variable validation (TF >= 1.3 compat) - Gate scale-up warm starts on warmPoolConfig.enabled only, not poolStrategy (scale-up should try warm starts regardless of pool lambda strategy) - Wrap scale-down warm pool stop in try/catch with fallback to terminate on failure - Include stopped instances in eviction owner lookup - Add AMI ID to pool-created warm instances for staleness eviction - Make removeFromWarmPool atomic with conditional delete to prevent concurrent scale-up race conditions - Add test for concurrent claim scenario Signed-off-by: Brend Smits --- .../control-plane/src/aws/warm-pool.ts | 27 +++++++--- .../src/pool/pool-warm-strategy.test.ts | 3 +- .../control-plane/src/pool/pool.test.ts | 2 +- .../functions/control-plane/src/pool/pool.ts | 11 ++++ .../find-and-start-warm-runners.test.ts | 39 ++++++++++++--- .../scale-down-warm-pool.test.ts | 4 +- .../src/scale-runners/scale-down.ts | 50 +++++++++++-------- .../src/scale-runners/scale-up.ts | 12 +++-- modules/runners/variables.tf | 5 ++ modules/runners/warm-pool.tf | 8 --- 10 files changed, 108 insertions(+), 53 deletions(-) diff --git a/lambdas/functions/control-plane/src/aws/warm-pool.ts b/lambdas/functions/control-plane/src/aws/warm-pool.ts index 866d8d374e..1418fae452 100644 --- a/lambdas/functions/control-plane/src/aws/warm-pool.ts +++ b/lambdas/functions/control-plane/src/aws/warm-pool.ts @@ -1,4 +1,5 @@ import { + ConditionalCheckFailedException, DynamoDBClient, PutItemCommand, DeleteItemCommand, @@ -78,15 +79,25 @@ export async function addToWarmPool(entry: Omit { +export async function removeFromWarmPool(instanceId: string): Promise { const client = getClient(); - await client.send( - new DeleteItemCommand({ - TableName: getTableName(), - Key: { instanceId: { S: instanceId } }, - }), - ); - logger.info(`Removed instance '${instanceId}' from warm pool`); + try { + await client.send( + new DeleteItemCommand({ + TableName: getTableName(), + Key: { instanceId: { S: instanceId } }, + ConditionExpression: 'attribute_exists(instanceId)', + }), + ); + logger.info(`Removed instance '${instanceId}' from warm pool`); + return true; + } catch (e) { + if (e instanceof ConditionalCheckFailedException) { + logger.info(`Instance '${instanceId}' already claimed from warm pool`); + return false; + } + throw e; + } } export async function getWarmInstance(instanceId: string): Promise { diff --git a/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts b/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts index 12409c8a62..f32340f542 100644 --- a/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts @@ -1,7 +1,6 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; import { listEC2Runners } from '../aws/runners'; -import * as ghAuth from '../github/auth'; -import { createRunners, findAndStartWarmRunners, getGitHubEnterpriseApiUrl } from '../scale-runners/scale-up'; +import { createRunners, findAndStartWarmRunners } from '../scale-runners/scale-up'; import { getPoolStrategy, getWarmPoolConfig, countWarmInstancesByOwner } from '../aws/warm-pool'; import { adjust } from './pool'; diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index af4a55cedd..196952a41c 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -4,7 +4,7 @@ import * as nock from 'nock'; import { listEC2Runners } from '../aws/runners'; import * as ghAuth from '../github/auth'; -import { createRunners, findAndStartWarmRunners, getGitHubEnterpriseApiUrl } from '../scale-runners/scale-up'; +import { createRunners, getGitHubEnterpriseApiUrl } from '../scale-runners/scale-up'; import { adjust } from './pool'; import { describe, it, expect, beforeEach, vi, MockedClass } from 'vitest'; diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 101713c475..300f8d4a93 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -8,6 +8,7 @@ import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient import { createRunners, findAndStartWarmRunners, getGitHubEnterpriseApiUrl } from '../scale-runners/scale-up'; import { validateSsmParameterStoreTags } from '../scale-runners/scale-up'; import { addToWarmPool, getPoolStrategy, getWarmPoolConfig, countWarmInstancesByOwner, emitWarmPoolMetric } from '../aws/warm-pool'; +import { getParameter } from '@aws-github-runner/aws-ssm-util'; const logger = createChildLogger('pool'); @@ -198,6 +199,15 @@ async function warmPoolGracePeriod( // Re-check runner statuses after grace period const runnerStatuses = await getGitHubRegisteredRunnnerStatusses(ghClient, runnerOwner, runnerNamePrefix); + // Resolve current AMI ID for staleness tracking (best-effort) + let amiId: string | undefined; + const amiSsmParam = process.env.AMI_ID_SSM_PARAMETER_NAME; + if (amiSsmParam) { + try { + amiId = await getParameter(amiSsmParam); + } catch { /* best-effort */ } + } + for (const instanceId of instanceIds) { const status = runnerStatuses.get(instanceId); if (status?.busy) { @@ -214,6 +224,7 @@ async function warmPoolGracePeriod( runnerOwner, environment, runnerType: 'Org', + amiId, }); await tag(instanceId, [{ Key: 'ghr:warm-pool-member', Value: 'true' }]).catch(() => {}); emitWarmPoolMetric('WarmPoolInstanceStopped', 1, { Owner: runnerOwner }); diff --git a/lambdas/functions/control-plane/src/scale-runners/find-and-start-warm-runners.test.ts b/lambdas/functions/control-plane/src/scale-runners/find-and-start-warm-runners.test.ts index ce239c77b1..1e1e84fad5 100644 --- a/lambdas/functions/control-plane/src/scale-runners/find-and-start-warm-runners.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/find-and-start-warm-runners.test.ts @@ -2,7 +2,6 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; import { startRunner, tag, untag } from './../aws/runners'; import { getWarmPoolConfig, - getPoolStrategy, listWarmInstancesByOwner, removeFromWarmPool, emitWarmPoolMetric, @@ -48,7 +47,6 @@ const mockStartRunner = vi.mocked(startRunner); const mockTag = vi.mocked(tag); const mockUntag = vi.mocked(untag); const mockGetWarmPoolConfig = vi.mocked(getWarmPoolConfig); -const mockGetPoolStrategy = vi.mocked(getPoolStrategy); const mockListWarmInstances = vi.mocked(listWarmInstancesByOwner); const mockRemoveFromWarmPool = vi.mocked(removeFromWarmPool); const mockEmitWarmPoolMetric = vi.mocked(emitWarmPoolMetric); @@ -64,9 +62,8 @@ describe('findAndStartWarmRunners', () => { maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30, }); - mockGetPoolStrategy.mockReturnValue('warm'); mockStartRunner.mockResolvedValue(undefined); - mockRemoveFromWarmPool.mockResolvedValue(undefined); + mockRemoveFromWarmPool.mockResolvedValue(true); mockTag.mockResolvedValue(undefined); }); @@ -84,8 +81,13 @@ describe('findAndStartWarmRunners', () => { expect(mockListWarmInstances).not.toHaveBeenCalled(); }); - it('should return empty array when pool strategy is not warm', async () => { - mockGetPoolStrategy.mockReturnValue('hot'); + it('should return empty array when pool strategy is hot and warm pool is disabled', async () => { + mockGetWarmPoolConfig.mockReturnValue({ + enabled: false, + maxWarmInstances: 3, + maxWarmAgeHours: 168, + warmPoolReadyDelaySeconds: 30, + }); const result = await findAndStartWarmRunners('my-org', 1); @@ -206,6 +208,9 @@ describe('findAndStartWarmRunners', () => { }); it('should skip failed instances and continue with next', async () => { + mockRemoveFromWarmPool + .mockResolvedValueOnce(true) // i-bad claimed + .mockResolvedValueOnce(true); // i-good claimed mockStartRunner .mockRejectedValueOnce(new Error('Instance terminated')) .mockResolvedValueOnce(undefined); @@ -235,6 +240,9 @@ describe('findAndStartWarmRunners', () => { }); it('should remove failed instance from DynamoDB', async () => { + mockRemoveFromWarmPool + .mockResolvedValueOnce(true) // claim succeeds + .mockResolvedValueOnce(true); // cleanup in catch mockStartRunner.mockRejectedValue(new Error('Instance terminated')); mockListWarmInstances.mockResolvedValue([ { @@ -254,6 +262,25 @@ describe('findAndStartWarmRunners', () => { expect(mockRemoveFromWarmPool).toHaveBeenCalledWith('i-gone'); }); + it('should skip instance already claimed by another invocation', async () => { + mockRemoveFromWarmPool.mockResolvedValue(false); + mockListWarmInstances.mockResolvedValue([ + { + instanceId: 'i-claimed', + runnerOwner: 'my-org', + environment: 'test-env', + runnerType: 'Org', + stoppedAt: '2026-01-01T00:00:00Z', + expiresAt: 9999999999, + }, + ]); + + const result = await findAndStartWarmRunners('my-org', 1); + + expect(result).toEqual([]); + expect(mockStartRunner).not.toHaveBeenCalled(); + }); + it('should fallback to org-level lookup when repo-level owner has no instances', async () => { mockListWarmInstances .mockResolvedValueOnce([]) // repo-level lookup: empty diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts index 9b2cf0c228..e545db32bb 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts @@ -1,10 +1,8 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { terminateRunner, stopRunner, tag, listEC2Runners } from './../aws/runners'; +import { terminateRunner, listEC2Runners } from './../aws/runners'; import { getWarmPoolConfig, getPoolStrategy, - addToWarmPool, - countWarmInstancesByOwner, listWarmInstancesByOwner, removeFromWarmPool, emitWarmPoolMetric, diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index a7bf142b05..5272a40452 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -204,27 +204,35 @@ async function removeRunner(ec2runner: RunnerInfo, ghRunnerIds: number[]): Promi if (warmPoolConfig.enabled && poolStrategy === 'warm') { const warmCount = await countWarmInstancesByOwner(ec2runner.owner); if (warmCount < warmPoolConfig.maxWarmInstances) { - await stopRunner(ec2runner.instanceId); - let amiId: string | undefined; - const amiSsmParam = process.env.AMI_ID_SSM_PARAMETER_NAME; - if (amiSsmParam) { - try { - amiId = await getParameter(amiSsmParam); - } catch { /* best-effort */ } + try { + await stopRunner(ec2runner.instanceId); + let amiId: string | undefined; + const amiSsmParam = process.env.AMI_ID_SSM_PARAMETER_NAME; + if (amiSsmParam) { + try { + amiId = await getParameter(amiSsmParam); + } catch { /* best-effort */ } + } + await addToWarmPool({ + instanceId: ec2runner.instanceId, + runnerOwner: ec2runner.owner, + environment: process.env.ENVIRONMENT || '', + runnerType: ec2runner.type, + amiId, + }); + await tag(ec2runner.instanceId, [{ Key: 'ghr:warm-pool-member', Value: 'true' }]); + emitWarmPoolMetric('WarmPoolInstanceStopped', 1, { Owner: ec2runner.owner }); + logger.info( + `Runner '${ec2runner.instanceId}' stopped and added to warm pool ` + + `(${warmCount + 1}/${warmPoolConfig.maxWarmInstances}).`, + ); + } catch (warmPoolError) { + logger.warn( + `Failed to stop runner '${ec2runner.instanceId}' into warm pool, terminating instead.`, + { error: warmPoolError }, + ); + await terminateRunner(ec2runner.instanceId); } - await addToWarmPool({ - instanceId: ec2runner.instanceId, - runnerOwner: ec2runner.owner, - environment: process.env.ENVIRONMENT || '', - runnerType: ec2runner.type, - amiId, - }); - await tag(ec2runner.instanceId, [{ Key: 'ghr:warm-pool-member', Value: 'true' }]); - emitWarmPoolMetric('WarmPoolInstanceStopped', 1, { Owner: ec2runner.owner }); - logger.info( - `Runner '${ec2runner.instanceId}' stopped and added to warm pool ` + - `(${warmCount + 1}/${warmPoolConfig.maxWarmInstances}).`, - ); } else { await terminateRunner(ec2runner.instanceId); logger.info( @@ -405,7 +413,7 @@ async function evictStaleWarmInstances(environment: string): Promise { if (!warmPoolConfig.enabled) return; const ownerTags = new Set(); - const ec2runners = await listEC2Runners({ environment }); + const ec2runners = await listEC2Runners({ environment, statuses: ['running', 'pending', 'stopped', 'stopping'] }); for (const runner of ec2runners) { if (runner.owner) ownerTags.add(runner.owner); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 0195c1fa39..3fda240f56 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -14,7 +14,6 @@ import { metricGitHubAppRateLimit } from '../github/rate-limit'; import { publishRetryMessage } from './job-retry'; import { getWarmPoolConfig, - getPoolStrategy, listWarmInstancesByOwner, removeFromWarmPool, emitWarmPoolMetric, @@ -312,9 +311,8 @@ export async function findAndStartWarmRunners( ghClient?: Octokit, ): Promise { const warmPoolConfig = getWarmPoolConfig(); - const poolStrategy = getPoolStrategy(); - if (!warmPoolConfig.enabled || poolStrategy !== 'warm' || count <= 0) { + if (!warmPoolConfig.enabled || count <= 0) { return []; } @@ -334,10 +332,16 @@ export async function findAndStartWarmRunners( if (startedInstances.length >= count) break; try { + // Atomically claim the warm instance — prevents concurrent scale-up from using the same one + const claimed = await removeFromWarmPool(entry.instanceId); + if (!claimed) { + logger.info(`Warm instance '${entry.instanceId}' already claimed by another invocation, skipping`); + continue; + } + const startTime = Date.now(); await startRunner(entry.instanceId); const startLatencyMs = Date.now() - startTime; - await removeFromWarmPool(entry.instanceId); startedInstances.push(entry.instanceId); emitWarmPoolMetric('WarmPoolInstanceStarted', 1, { Owner: runnerOwner }); emitWarmPoolMetric('WarmPoolStartLatency', startLatencyMs, { Owner: runnerOwner }); diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index 14b94e61a3..e777f7f131 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -914,4 +914,9 @@ variable "pool_strategy" { condition = contains(["hot", "warm"], var.pool_strategy) error_message = "pool_strategy must be either \"hot\" or \"warm\"." } + + validation { + condition = var.pool_strategy != "warm" || var.warm_pool_config.enabled + error_message = "pool_strategy = \"warm\" requires warm_pool_config.enabled = true." + } } diff --git a/modules/runners/warm-pool.tf b/modules/runners/warm-pool.tf index a02e9b356d..faadfdd8b3 100644 --- a/modules/runners/warm-pool.tf +++ b/modules/runners/warm-pool.tf @@ -1,14 +1,6 @@ # Warm Pool DynamoDB table and IAM policies # Only created when warm_pool_config.enabled = true -# Cross-variable validation -check "warm_pool_strategy_requires_enabled" { - assert { - condition = var.pool_strategy != "warm" || var.warm_pool_config.enabled - error_message = "pool_strategy = \"warm\" requires warm_pool_config.enabled = true." - } -} - resource "aws_dynamodb_table" "warm_pool" { count = var.warm_pool_config.enabled ? 1 : 0 From 7d918708f184ffe1bc88c5780008acc9b2e25265 Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 17:03:42 +0200 Subject: [PATCH 17/19] fix(warm-pool): remove cross-variable validation for TF 1.3 compat Cross-variable references in variable validation blocks require Terraform 1.9+. The repo supports >= 1.3.0, so this breaks CI on TF 1.5.6. The constraint (pool_strategy=warm requires enabled=true) is documented in the variable description and enforced at runtime by the lambda. Signed-off-by: Brend Smits --- modules/runners/variables.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index e777f7f131..14b94e61a3 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -914,9 +914,4 @@ variable "pool_strategy" { condition = contains(["hot", "warm"], var.pool_strategy) error_message = "pool_strategy must be either \"hot\" or \"warm\"." } - - validation { - condition = var.pool_strategy != "warm" || var.warm_pool_config.enabled - error_message = "pool_strategy = \"warm\" requires warm_pool_config.enabled = true." - } } From 6c3e235034afae1737d8b7fe7ccf2a1e0720072e Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 17:09:09 +0200 Subject: [PATCH 18/19] style: fix prettier and terraform formatting --- .../src/pool/pool-warm-strategy.test.ts | 32 +++++-- .../control-plane/src/pool/pool.test.ts | 4 +- .../functions/control-plane/src/pool/pool.ts | 86 +++++++++++-------- .../find-and-start-warm-runners.test.ts | 17 ++-- .../scale-down-warm-pool.test.ts | 16 ++-- .../src/scale-runners/scale-down.ts | 11 +-- .../src/scale-runners/scale-up.ts | 7 +- modules/runners/pool.tf | 4 +- modules/runners/pool/main.tf | 8 +- modules/runners/scale-down.tf | 8 +- modules/runners/scale-up.tf | 6 +- modules/runners/tests/pool.tftest.hcl | 2 +- modules/runners/variables.tf | 4 +- 13 files changed, 112 insertions(+), 93 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts b/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts index f32340f542..a5d0b49a60 100644 --- a/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool-warm-strategy.test.ts @@ -43,7 +43,9 @@ vi.mock('../scale-runners/scale-up', async () => ({ vi.mock('../aws/warm-pool', async () => ({ getPoolStrategy: vi.fn().mockReturnValue('hot'), - getWarmPoolConfig: vi.fn().mockReturnValue({ enabled: false, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }), + getWarmPoolConfig: vi + .fn() + .mockReturnValue({ enabled: false, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }), countWarmInstancesByOwner: vi.fn().mockResolvedValue(0), addToWarmPool: vi.fn().mockResolvedValue(undefined), emitWarmPoolMetric: vi.fn(), @@ -89,7 +91,12 @@ describe('pool warm strategy', () => { it('should count warm instances toward pool target with warm strategy', async () => { mockGetPoolStrategy.mockReturnValue('warm'); - mockGetWarmPoolConfig.mockReturnValue({ enabled: true, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }); + mockGetWarmPoolConfig.mockReturnValue({ + enabled: true, + maxWarmInstances: 3, + maxWarmAgeHours: 168, + warmPoolReadyDelaySeconds: 30, + }); mockCountWarmInstances.mockResolvedValue(2); // 0 running + 2 warm = 2 effective, pool size = 2 → no top-up needed mockListRunners.mockResolvedValue([]); @@ -103,7 +110,12 @@ describe('pool warm strategy', () => { it('should try warm instances first when topping up', async () => { mockGetPoolStrategy.mockReturnValue('warm'); - mockGetWarmPoolConfig.mockReturnValue({ enabled: true, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }); + mockGetWarmPoolConfig.mockReturnValue({ + enabled: true, + maxWarmInstances: 3, + maxWarmAgeHours: 168, + warmPoolReadyDelaySeconds: 30, + }); mockCountWarmInstances.mockResolvedValue(0); // Pool wants 3, has 0 → needs 3, warm start returns 2 → 1 cold needed mockFindAndStartWarmRunners.mockResolvedValue(['i-warm-1', 'i-warm-2']); @@ -122,7 +134,12 @@ describe('pool warm strategy', () => { it('should not cold-launch if warm instances satisfy the full top-up', async () => { mockGetPoolStrategy.mockReturnValue('warm'); - mockGetWarmPoolConfig.mockReturnValue({ enabled: true, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }); + mockGetWarmPoolConfig.mockReturnValue({ + enabled: true, + maxWarmInstances: 3, + maxWarmAgeHours: 168, + warmPoolReadyDelaySeconds: 30, + }); mockCountWarmInstances.mockResolvedValue(0); mockFindAndStartWarmRunners.mockResolvedValue(['i-warm-1', 'i-warm-2']); @@ -134,7 +151,12 @@ describe('pool warm strategy', () => { it('should not count warm instances when strategy is hot', async () => { mockGetPoolStrategy.mockReturnValue('hot'); - mockGetWarmPoolConfig.mockReturnValue({ enabled: true, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }); + mockGetWarmPoolConfig.mockReturnValue({ + enabled: true, + maxWarmInstances: 3, + maxWarmAgeHours: 168, + warmPoolReadyDelaySeconds: 30, + }); // With hot strategy, warm instances should NOT be counted even if warm pool is enabled mockListRunners.mockResolvedValue([]); diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index 196952a41c..a06aba762b 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -50,7 +50,9 @@ vi.mock('../scale-runners/scale-up', async () => ({ vi.mock('../aws/warm-pool', async () => ({ getPoolStrategy: vi.fn().mockReturnValue('hot'), - getWarmPoolConfig: vi.fn().mockReturnValue({ enabled: false, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }), + getWarmPoolConfig: vi + .fn() + .mockReturnValue({ enabled: false, maxWarmInstances: 3, maxWarmAgeHours: 168, warmPoolReadyDelaySeconds: 30 }), countWarmInstancesByOwner: vi.fn().mockResolvedValue(0), addToWarmPool: vi.fn().mockResolvedValue(undefined), emitWarmPoolMetric: vi.fn(), diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 300f8d4a93..3631b224aa 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -7,7 +7,13 @@ import { RunnerList } from '../aws/runners.d'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; import { createRunners, findAndStartWarmRunners, getGitHubEnterpriseApiUrl } from '../scale-runners/scale-up'; import { validateSsmParameterStoreTags } from '../scale-runners/scale-up'; -import { addToWarmPool, getPoolStrategy, getWarmPoolConfig, countWarmInstancesByOwner, emitWarmPoolMetric } from '../aws/warm-pool'; +import { + addToWarmPool, + getPoolStrategy, + getWarmPoolConfig, + countWarmInstancesByOwner, + emitWarmPoolMetric, +} from '../aws/warm-pool'; import { getParameter } from '@aws-github-runner/aws-ssm-util'; const logger = createChildLogger('pool'); @@ -86,7 +92,9 @@ export async function adjust(event: PoolEvent): Promise { if (poolStrategy === 'warm' && warmPoolConfig.enabled) { const warmCount = await countWarmInstancesByOwner(runnerOwner); effectivePoolSize = numberOfRunnersInPool + warmCount; - logger.info(`Warm strategy: ${numberOfRunnersInPool} running idle + ${warmCount} warm stopped = ${effectivePoolSize} effective pool size`); + logger.info( + `Warm strategy: ${numberOfRunnersInPool} running idle + ${warmCount} warm stopped = ${effectivePoolSize} effective pool size`, + ); } let topUp = event.poolSize - effectivePoolSize; @@ -133,40 +141,40 @@ export async function adjust(event: PoolEvent): Promise { if (remainingTopUp > 0) { const newInstances = await createRunners( - { - ephemeral, - enableJitConfig, - ghesBaseUrl, - runnerLabels, - runnerGroup, - runnerOwner, - runnerNamePrefix, - runnerType: 'Org', - disableAutoUpdate: disableAutoUpdate, - ssmTokenPath, - ssmConfigPath, - ssmParameterStoreTags, - }, - { - ec2instanceCriteria: { - instanceTypes, - instanceTypePriorities, - targetCapacityType: instanceTargetCapacityType, - maxSpotPrice: instanceMaxSpotPrice, - instanceAllocationStrategy: instanceAllocationStrategy, + { + ephemeral, + enableJitConfig, + ghesBaseUrl, + runnerLabels, + runnerGroup, + runnerOwner, + runnerNamePrefix, + runnerType: 'Org', + disableAutoUpdate: disableAutoUpdate, + ssmTokenPath, + ssmConfigPath, + ssmParameterStoreTags, }, - environment, - launchTemplateName, - subnets, - amiIdSsmParameterName, - tracingEnabled, - onDemandFailoverOnError, - scaleErrors, - }, - remainingTopUp, - githubInstallationClient, - 'pool-lambda', - ); + { + ec2instanceCriteria: { + instanceTypes, + instanceTypePriorities, + targetCapacityType: instanceTargetCapacityType, + maxSpotPrice: instanceMaxSpotPrice, + instanceAllocationStrategy: instanceAllocationStrategy, + }, + environment, + launchTemplateName, + subnets, + amiIdSsmParameterName, + tracingEnabled, + onDemandFailoverOnError, + scaleErrors, + }, + remainingTopUp, + githubInstallationClient, + 'pool-lambda', + ); // Warm strategy grace period: wait for runners to register, then stop idle ones if (poolStrategy === 'warm' && warmPoolConfig.enabled && newInstances.length > 0) { @@ -181,7 +189,9 @@ export async function adjust(event: PoolEvent): Promise { } } } else { - logger.info(`Pool will not be topped up. Found ${effectivePoolSize} effective pool runners (${numberOfRunnersInPool} running + warm).`); + logger.info( + `Pool will not be topped up. Found ${effectivePoolSize} effective pool runners (${numberOfRunnersInPool} running + warm).`, + ); } } @@ -205,7 +215,9 @@ async function warmPoolGracePeriod( if (amiSsmParam) { try { amiId = await getParameter(amiSsmParam); - } catch { /* best-effort */ } + } catch { + /* best-effort */ + } } for (const instanceId of instanceIds) { diff --git a/lambdas/functions/control-plane/src/scale-runners/find-and-start-warm-runners.test.ts b/lambdas/functions/control-plane/src/scale-runners/find-and-start-warm-runners.test.ts index 1e1e84fad5..55be93510b 100644 --- a/lambdas/functions/control-plane/src/scale-runners/find-and-start-warm-runners.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/find-and-start-warm-runners.test.ts @@ -1,11 +1,6 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; import { startRunner, tag, untag } from './../aws/runners'; -import { - getWarmPoolConfig, - listWarmInstancesByOwner, - removeFromWarmPool, - emitWarmPoolMetric, -} from '../aws/warm-pool'; +import { getWarmPoolConfig, listWarmInstancesByOwner, removeFromWarmPool, emitWarmPoolMetric } from '../aws/warm-pool'; import { findAndStartWarmRunners } from './scale-up'; vi.mock('./../aws/runners', async () => ({ @@ -209,11 +204,9 @@ describe('findAndStartWarmRunners', () => { it('should skip failed instances and continue with next', async () => { mockRemoveFromWarmPool - .mockResolvedValueOnce(true) // i-bad claimed + .mockResolvedValueOnce(true) // i-bad claimed .mockResolvedValueOnce(true); // i-good claimed - mockStartRunner - .mockRejectedValueOnce(new Error('Instance terminated')) - .mockResolvedValueOnce(undefined); + mockStartRunner.mockRejectedValueOnce(new Error('Instance terminated')).mockResolvedValueOnce(undefined); mockListWarmInstances.mockResolvedValue([ { instanceId: 'i-bad', @@ -241,8 +234,8 @@ describe('findAndStartWarmRunners', () => { it('should remove failed instance from DynamoDB', async () => { mockRemoveFromWarmPool - .mockResolvedValueOnce(true) // claim succeeds - .mockResolvedValueOnce(true); // cleanup in catch + .mockResolvedValueOnce(true) // claim succeeds + .mockResolvedValueOnce(true); // cleanup in catch mockStartRunner.mockRejectedValue(new Error('Instance terminated')); mockListWarmInstances.mockResolvedValue([ { diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts index e545db32bb..968e86c03c 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts @@ -101,9 +101,7 @@ describe('scale-down warm pool eviction', () => { }); it('should evict warm instances exceeding max age', async () => { - mockListEC2Runners.mockResolvedValue([ - { instanceId: 'i-running', owner: 'my-org', type: 'Org' } as any, - ]); + mockListEC2Runners.mockResolvedValue([{ instanceId: 'i-running', owner: 'my-org', type: 'Org' } as any]); const oldDate = new Date(Date.now() - 200 * 3600 * 1000).toISOString(); // 200 hours ago mockListWarmInstances.mockResolvedValue([ { @@ -125,9 +123,7 @@ describe('scale-down warm pool eviction', () => { }); it('should evict warm instances exceeding max count', async () => { - mockListEC2Runners.mockResolvedValue([ - { instanceId: 'i-running', owner: 'my-org', type: 'Org' } as any, - ]); + mockListEC2Runners.mockResolvedValue([{ instanceId: 'i-running', owner: 'my-org', type: 'Org' } as any]); const recentDate = new Date(Date.now() - 1 * 3600 * 1000).toISOString(); // 1 hour ago mockListWarmInstances.mockResolvedValue([ { @@ -161,16 +157,14 @@ describe('scale-down warm pool eviction', () => { await scaleDown(); // maxWarmInstances is 2, so at least 1 warm instance should be evicted (i-warm-*) - const warmEvictionCalls = mockTerminateRunner.mock.calls.filter( - (call) => (call[0] as string).startsWith('i-warm-'), + const warmEvictionCalls = mockTerminateRunner.mock.calls.filter((call) => + (call[0] as string).startsWith('i-warm-'), ); expect(warmEvictionCalls).toHaveLength(1); }); it('should emit WarmPoolSize metric after eviction', async () => { - mockListEC2Runners.mockResolvedValue([ - { instanceId: 'i-running', owner: 'my-org', type: 'Org' } as any, - ]); + mockListEC2Runners.mockResolvedValue([{ instanceId: 'i-running', owner: 'my-org', type: 'Org' } as any]); const oldDate = new Date(Date.now() - 200 * 3600 * 1000).toISOString(); mockListWarmInstances.mockResolvedValue([ { diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index 5272a40452..ce84d6e8b7 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -211,7 +211,9 @@ async function removeRunner(ec2runner: RunnerInfo, ghRunnerIds: number[]): Promi if (amiSsmParam) { try { amiId = await getParameter(amiSsmParam); - } catch { /* best-effort */ } + } catch { + /* best-effort */ + } } await addToWarmPool({ instanceId: ec2runner.instanceId, @@ -227,10 +229,9 @@ async function removeRunner(ec2runner: RunnerInfo, ghRunnerIds: number[]): Promi `(${warmCount + 1}/${warmPoolConfig.maxWarmInstances}).`, ); } catch (warmPoolError) { - logger.warn( - `Failed to stop runner '${ec2runner.instanceId}' into warm pool, terminating instead.`, - { error: warmPoolError }, - ); + logger.warn(`Failed to stop runner '${ec2runner.instanceId}' into warm pool, terminating instead.`, { + error: warmPoolError, + }); await terminateRunner(ec2runner.instanceId); } } else { diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 3fda240f56..4b891b51b7 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -12,12 +12,7 @@ import { createRunner, listEC2Runners, startRunner, tag, untag, terminateRunner import { Ec2OverrideConfig, RunnerInputParameters } from './../aws/runners.d'; import { metricGitHubAppRateLimit } from '../github/rate-limit'; import { publishRetryMessage } from './job-retry'; -import { - getWarmPoolConfig, - listWarmInstancesByOwner, - removeFromWarmPool, - emitWarmPoolMetric, -} from '../aws/warm-pool'; +import { getWarmPoolConfig, listWarmInstancesByOwner, removeFromWarmPool, emitWarmPoolMetric } from '../aws/warm-pool'; import { _InstanceType, Tenancy, diff --git a/modules/runners/pool.tf b/modules/runners/pool.tf index e8410c5373..7b0a9056ae 100644 --- a/modules/runners/pool.tf +++ b/modules/runners/pool.tf @@ -71,8 +71,8 @@ module "pool" { max_warm_age_hours = var.warm_pool_config.max_warm_age_hours warm_pool_ready_delay_seconds = var.warm_pool_config.warm_pool_ready_delay_seconds } - pool_strategy = var.pool_strategy - enable_metric_warm_pool = var.metrics.enable && var.metrics.metric.enable_warm_pool + pool_strategy = var.pool_strategy + enable_metric_warm_pool = var.metrics.enable && var.metrics.metric.enable_warm_pool } aws_partition = var.aws_partition diff --git a/modules/runners/pool/main.tf b/modules/runners/pool/main.tf index d4d9e79153..b9845bab3c 100644 --- a/modules/runners/pool/main.tf +++ b/modules/runners/pool/main.tf @@ -60,15 +60,15 @@ resource "aws_lambda_function" "pool" { SSM_PARAMETER_STORE_TAGS = var.config.lambda.parameter_store_tags SCALE_ERRORS = jsonencode(var.config.runner.scale_errors) USE_DEDICATED_HOST = var.config.runner.use_dedicated_host - WARM_POOL_CONFIG = jsonencode({ + WARM_POOL_CONFIG = jsonencode({ enabled = var.config.warm_pool_config.enabled maxWarmInstances = var.config.warm_pool_config.max_warm_instances maxWarmAgeHours = var.config.warm_pool_config.max_warm_age_hours warmPoolReadyDelaySeconds = var.config.warm_pool_config.warm_pool_ready_delay_seconds }) - WARM_POOL_TABLE_NAME = var.config.warm_pool_table_name - POOL_STRATEGY = var.config.pool_strategy - ENABLE_METRIC_WARM_POOL = var.config.enable_metric_warm_pool + WARM_POOL_TABLE_NAME = var.config.warm_pool_table_name + POOL_STRATEGY = var.config.pool_strategy + ENABLE_METRIC_WARM_POOL = var.config.enable_metric_warm_pool } } diff --git a/modules/runners/scale-down.tf b/modules/runners/scale-down.tf index 92f87f8f88..a7a567d56f 100644 --- a/modules/runners/scale-down.tf +++ b/modules/runners/scale-down.tf @@ -43,15 +43,15 @@ resource "aws_lambda_function" "scale_down" { POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error - WARM_POOL_CONFIG = jsonencode({ + WARM_POOL_CONFIG = jsonencode({ enabled = var.warm_pool_config.enabled maxWarmInstances = var.warm_pool_config.max_warm_instances maxWarmAgeHours = var.warm_pool_config.max_warm_age_hours warmPoolReadyDelaySeconds = var.warm_pool_config.warm_pool_ready_delay_seconds }) - WARM_POOL_TABLE_NAME = var.warm_pool_config.enabled ? aws_dynamodb_table.warm_pool[0].name : "" - POOL_STRATEGY = var.pool_strategy - AMI_ID_SSM_PARAMETER_NAME = local.ami_id_ssm_parameter_name + WARM_POOL_TABLE_NAME = var.warm_pool_config.enabled ? aws_dynamodb_table.warm_pool[0].name : "" + POOL_STRATEGY = var.pool_strategy + AMI_ID_SSM_PARAMETER_NAME = local.ami_id_ssm_parameter_name } } diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf index 638072615a..bd4874fb70 100644 --- a/modules/runners/scale-up.tf +++ b/modules/runners/scale-up.tf @@ -65,14 +65,14 @@ resource "aws_lambda_function" "scale_up" { JOB_RETRY_CONFIG = jsonencode(local.job_retry_config) USE_DEDICATED_HOST = var.use_dedicated_host ENABLE_METRIC_WARM_POOL = var.metrics.enable && var.metrics.metric.enable_warm_pool - WARM_POOL_CONFIG = jsonencode({ + WARM_POOL_CONFIG = jsonencode({ enabled = var.warm_pool_config.enabled maxWarmInstances = var.warm_pool_config.max_warm_instances maxWarmAgeHours = var.warm_pool_config.max_warm_age_hours warmPoolReadyDelaySeconds = var.warm_pool_config.warm_pool_ready_delay_seconds }) - WARM_POOL_TABLE_NAME = var.warm_pool_config.enabled ? aws_dynamodb_table.warm_pool[0].name : "" - POOL_STRATEGY = var.pool_strategy + WARM_POOL_TABLE_NAME = var.warm_pool_config.enabled ? aws_dynamodb_table.warm_pool[0].name : "" + POOL_STRATEGY = var.pool_strategy } } diff --git a/modules/runners/tests/pool.tftest.hcl b/modules/runners/tests/pool.tftest.hcl index 2c557f9392..c2282a7685 100644 --- a/modules/runners/tests/pool.tftest.hcl +++ b/modules/runners/tests/pool.tftest.hcl @@ -29,7 +29,7 @@ variables { runner_labels = ["self-hosted", "linux", "x64"] # Use S3 bucket to avoid filebase64sha256 needing local zip files - lambda_s3_bucket = "my-lambda-bucket" + lambda_s3_bucket = "my-lambda-bucket" runners_lambda_s3_key = "runners.zip" github_app_parameters = { diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index 14b94e61a3..18fc7a2c31 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -907,8 +907,8 @@ variable "pool_strategy" { - `hot`: (default) Traditional behavior. Pool lambda creates fresh instances on a schedule. Idle instances are terminated by scale-down. - `warm`: Idle instances are stopped (hibernated) instead of terminated. Scale-up will restart warm instances before creating new ones. Requires `warm_pool_config.enabled = true`. EOF - type = string - default = "hot" + type = string + default = "hot" validation { condition = contains(["hot", "warm"], var.pool_strategy) From 3b3be597d58d60d234d261bc9ad5fd1110bb324a Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Thu, 9 Jul 2026 17:15:52 +0200 Subject: [PATCH 19/19] fix: replace no-explicit-any with RunnerList type in warm pool tests --- .../src/scale-runners/scale-down-warm-pool.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts index 968e86c03c..89e56ae107 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-warm-pool.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; import { terminateRunner, listEC2Runners } from './../aws/runners'; +import { RunnerList } from '../aws/runners.d'; import { getWarmPoolConfig, getPoolStrategy, @@ -101,7 +102,7 @@ describe('scale-down warm pool eviction', () => { }); it('should evict warm instances exceeding max age', async () => { - mockListEC2Runners.mockResolvedValue([{ instanceId: 'i-running', owner: 'my-org', type: 'Org' } as any]); + mockListEC2Runners.mockResolvedValue([{ instanceId: 'i-running', owner: 'my-org', type: 'Org' } as RunnerList]); const oldDate = new Date(Date.now() - 200 * 3600 * 1000).toISOString(); // 200 hours ago mockListWarmInstances.mockResolvedValue([ { @@ -123,7 +124,7 @@ describe('scale-down warm pool eviction', () => { }); it('should evict warm instances exceeding max count', async () => { - mockListEC2Runners.mockResolvedValue([{ instanceId: 'i-running', owner: 'my-org', type: 'Org' } as any]); + mockListEC2Runners.mockResolvedValue([{ instanceId: 'i-running', owner: 'my-org', type: 'Org' } as RunnerList]); const recentDate = new Date(Date.now() - 1 * 3600 * 1000).toISOString(); // 1 hour ago mockListWarmInstances.mockResolvedValue([ { @@ -164,7 +165,7 @@ describe('scale-down warm pool eviction', () => { }); it('should emit WarmPoolSize metric after eviction', async () => { - mockListEC2Runners.mockResolvedValue([{ instanceId: 'i-running', owner: 'my-org', type: 'Org' } as any]); + mockListEC2Runners.mockResolvedValue([{ instanceId: 'i-running', owner: 'my-org', type: 'Org' } as RunnerList]); const oldDate = new Date(Date.now() - 200 * 3600 * 1000).toISOString(); mockListWarmInstances.mockResolvedValue([ {