diff --git a/README.md b/README.md
index 1c7dec9270..3786acdbe5 100644
--- a/README.md
+++ b/README.md
@@ -142,11 +142,12 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh)
| [github\_app](#input\_github\_app) | GitHub app parameters, see your github app.
You can optionally create the SSM parameters yourself and provide the ARN and name here, through the `*_ssm` attributes.
If you chose to provide the configuration values directly here,
please ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`).
Note: the provided SSM parameters arn and name have a precedence over the actual value (i.e `key_base64_ssm` has a precedence over `key_base64` etc). |
object({
key_base64 = optional(string)
key_base64_ssm = optional(object({
arn = string
name = string
}))
id = optional(string)
id_ssm = optional(object({
arn = string
name = string
}))
webhook_secret = optional(string)
webhook_secret_ssm = optional(object({
arn = string
name = string
}))
}) | n/a | yes |
| [iam\_overrides](#input\_iam\_overrides) | This map provides the possibility to override some IAM defaults. Note that when using this variable, you are responsible for ensuring the role has necessary permissions to access required resources. `override_instance_profile`: When set to true, uses the instance profile name specified in `instance_profile_name` instead of creating a new instance profile. `override_runner_role`: When set to true, uses the role ARN specified in `runner_role_arn` instead of creating a new IAM role. | object({
override_instance_profile = optional(bool, null)
instance_profile_name = optional(string, null)
override_runner_role = optional(bool, null)
runner_role_arn = optional(string, null)
}) | {
"instance_profile_name": null,
"override_instance_profile": false,
"override_runner_role": false,
"runner_role_arn": null
} | no |
| [idle\_config](#input\_idle\_config) | List of time periods, defined as a cron expression, to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | list(object({
cron = string
timeZone = string
idleCount = number
evictionStrategy = optional(string, "oldest_first")
})) | `[]` | no |
-| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for spot instances. AWS recommends using `price-capacity-optimized` however the AWS default is `lowest-price`. | `string` | `"lowest-price"` | no |
+| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`. | `string` | `"lowest-price"` | no |
| [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no |
| [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no |
| [instance\_target\_capacity\_type](#input\_instance\_target\_capacity\_type) | Default lifecycle used for runner instances, can be either `spot` or `on-demand`. | `string` | `"spot"` | no |
| [instance\_termination\_watcher](#input\_instance\_termination\_watcher) | Configuration for the instance termination watcher. This feature is Beta, changes will not trigger a major release as long in beta.object({
enable = optional(bool, false)
features = optional(object({
enable_spot_termination_handler = optional(bool, true)
enable_spot_termination_notification_watcher = optional(bool, true)
}), {})
enable_runner_deregistration = optional(bool, true)
memory_size = optional(number, null)
s3_key = optional(string, null)
s3_object_version = optional(string, null)
timeout = optional(number, null)
zip = optional(string, null)
}) | `{}` | no |
+| [instance\_type\_priorities](#input\_instance\_type\_priorities) | A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`. | `map(number)` | `null` | no |
| [instance\_types](#input\_instance\_types) | List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win). | `list(string)` | [| no | | [job\_queue\_retention\_in\_seconds](#input\_job\_queue\_retention\_in\_seconds) | The number of seconds the job is held in the queue before it is purged. | `number` | `86400` | no | | [job\_retry](#input\_job\_retry) | Experimental! Can be removed / changed without trigger a major release.Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app.
"m5.large",
"c5.large"
]
object({
enable = optional(bool, false)
delay_in_seconds = optional(number, 300)
delay_backoff = optional(number, 2)
lambda_memory_size = optional(number, 256)
lambda_timeout = optional(number, 30)
max_attempts = optional(number, 1)
}) | `{}` | no |
@@ -166,7 +167,7 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh)
| [logging\_kms\_key\_id](#input\_logging\_kms\_key\_id) | Specifies the kms key id to encrypt the logs with. | `string` | `null` | no |
| [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no |
| [matcher\_config\_parameter\_store\_tier](#input\_matcher\_config\_parameter\_store\_tier) | The tier of the parameter store for the matcher configuration. Valid values are `Standard`, and `Advanced`. | `string` | `"Standard"` | no |
-| [metrics](#input\_metrics) | Configuration for metrics created by the module, by default disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | object({
enable = optional(bool, false)
namespace = optional(string, "GitHub Runners")
metric = optional(object({
enable_github_app_rate_limit = optional(bool, true)
enable_job_retry = optional(bool, true)
enable_spot_termination_warning = optional(bool, true)
}), {})
}) | `{}` | no |
+| [metrics](#input\_metrics) | Configuration for metrics created by the module, by default disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | object({
enable = optional(bool, false)
namespace = optional(string, "GitHub Runners")
metric = optional(object({
enable_github_app_rate_limit = optional(bool, true)
enable_job_retry = optional(bool, true)
enable_spot_termination_warning = optional(bool, true)
enable_warm_pool = optional(bool, true)
}), {})
}) | `{}` | no |
| [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated, if not busy. | `number` | `null` | no |
| [parameter\_store\_tags](#input\_parameter\_store\_tags) | Map of tags that will be added to all the SSM Parameter Store parameters created by the Lambda function. | `map(string)` | `{}` | no |
| [pool\_config](#input\_pool\_config) | The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for weekdays to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC). | list(object({
schedule_expression = string
schedule_expression_timezone = optional(string)
size = number
})) | `[]` | no |
@@ -174,6 +175,7 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh)
| [pool\_lambda\_reserved\_concurrent\_executions](#input\_pool\_lambda\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no |
| [pool\_lambda\_timeout](#input\_pool\_lambda\_timeout) | Time out for the pool lambda in seconds. | `number` | `60` | no |
| [pool\_runner\_owner](#input\_pool\_runner\_owner) | The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported. | `string` | `null` | no |
+| [pool\_strategy](#input\_pool\_strategy) | Strategy for maintaining idle runners: `hot` (default, traditional) or `warm` (stop instead of terminate). Requires `warm_pool_config.enabled = true` when set to `warm`. | `string` | `"hot"` | no |
| [prefix](#input\_prefix) | The prefix used for naming resources | `string` | `"github-actions"` | no |
| [queue\_encryption](#input\_queue\_encryption) | Configure how data on queues managed by the modules is encrypted at REST. Options are encrypted via SSE, non encrypted and via KMS. By default encrypted via SSE is enabled. See for more details the Terraform `aws_sqs_queue` resource https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue. | object({
kms_data_key_reuse_period_seconds = number
kms_master_key_id = string
sqs_managed_sse_enabled = bool
}) | {
"kms_data_key_reuse_period_seconds": null,
"kms_master_key_id": null,
"sqs_managed_sse_enabled": true
} | no |
| [queue\_selection\_strategy](#input\_queue\_selection\_strategy) | Strategy used to pick a queue when multiple runner configurations match a job equally well. `first` keeps the historical deterministic behaviour (the first matching queue by priority). `random` spreads jobs across the matching queues to avoid concentrating load on a single one. `all` scales up one runner per matching queue and lets the first to become available take the job (favouring speed over cost; this multiplies instance launches and runner registrations per job). | `string` | `"first"` | no |
@@ -237,6 +239,7 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh)
| [userdata\_pre\_install](#input\_userdata\_pre\_install) | Script to be ran before the GitHub Actions runner is installed on the EC2 instances | `string` | `""` | no |
| [userdata\_template](#input\_userdata\_template) | Alternative user-data template file path, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored. | `string` | `null` | no |
| [vpc\_id](#input\_vpc\_id) | The VPC for security groups of the action runners. | `string` | n/a | yes |
+| [warm\_pool\_config](#input\_warm\_pool\_config) | Configuration for the warm pool feature. When enabled, idle runners are stopped instead of terminated, allowing 10-30s restart times instead of 2-5 minute cold starts. | object({
enabled = optional(bool, false)
max_warm_instances = optional(number, 3)
max_warm_age_hours = optional(number, 168)
warm_pool_ready_delay_seconds = optional(number, 30)
}) | `{}` | no |
| [webhook\_lambda\_apigateway\_access\_log\_settings](#input\_webhook\_lambda\_apigateway\_access\_log\_settings) | Access log settings for webhook API gateway. | object({
destination_arn = string
format = string
}) | `null` | no |
| [webhook\_lambda\_memory\_size](#input\_webhook\_lambda\_memory\_size) | Memory size limit in MB for webhook lambda in. | `number` | `256` | no |
| [webhook\_lambda\_s3\_key](#input\_webhook\_lambda\_s3\_key) | S3 key for webhook lambda function. Required if using S3 bucket to specify lambdas. | `string` | `null` | no |
diff --git a/docs/adr/001-warm-pool-hibernation.md b/docs/adr/001-warm-pool-hibernation.md
new file mode 100644
index 0000000000..b718a229b6
--- /dev/null
+++ b/docs/adr/001-warm-pool-hibernation.md
@@ -0,0 +1,309 @@
+# ADR-001: Warm Pool with Stop/Hibernate for Idle Pool Runners
+
+## Status
+
+Proposed
+
+## Date
+
+2026-05-22
+
+## Context
+
+The current pool system maintains a configurable number of idle GitHub Actions runners via the pool lambda. These runners are launched on a cron schedule and sit running (consuming compute costs) until either:
+
+1. A job picks them up, or
+2. The scale-down lambda terminates them after the idle count is exceeded.
+
+This "always running" model has significant cost implications:
+
+- **Waste during idle periods**: Pool runners that never receive a job still consume full EC2 compute costs until scale-down terminates them.
+- **Cold start latency**: When all idle runners are consumed and new ones must be launched, users experience full boot time (AMI boot + runner registration + tool installation). This can be 2-5 minutes depending on instance type and AMI.
+- **Binary choice**: Currently the only options are "running" (full cost) or "terminated" (zero cost but full cold start). There is no middle ground.
+
+AWS provides EC2 Stop and Hibernate capabilities that preserve the EBS root volume (and optionally RAM state). A stopped instance:
+
+- Incurs **zero compute cost** (no per-second billing).
+- Retains its **EBS volume** (small storage cost, typically $0.08-0.10/GB-month for gp3).
+- Can be **restarted in 10-30 seconds** vs. 2-5 minutes for a fresh launch.
+- Preserves the instance ID, network interfaces, and private IP.
+
+This makes stopped instances an ideal "warm" tier between fully running (hot) and terminated (cold).
+
+## Decision
+
+We will implement a **warm pool system** with three instance states, controlled by a feature flag:
+
+### Instance States
+
+| State | Cost | Startup Time | Description |
+|-------|------|-------------|-------------|
+| **Hot** (running) | Full compute + EBS | 0s (already running) | Currently idle, registered with GitHub, ready to accept jobs |
+| **Warm** (stopped) | EBS only | 10-30s | Stopped instance with preserved volume, must be started + re-registered |
+| **Cold** (terminated) | None | 2-5 min | No instance exists, must launch from scratch |
+
+### Behavior Changes
+
+#### Scale-Down Lambda (modified)
+
+When the feature flag `enable_warm_pool` is enabled:
+
+1. **Instead of terminating** idle runners that exceed `idleCount`, the scale-down lambda will **stop** them and record their state in a **DynamoDB table**.
+2. Before stopping, the runner is **deregistered from GitHub** (existing behavior preserved).
+3. A new config parameter `warm_pool_config` controls:
+ - `maxWarmInstances`: Maximum number of stopped instances to retain (prevents EBS cost explosion). Default: same as pool size.
+ - `maxWarmAgeHours`: Maximum age (in hours) of a warm instance before it is terminated instead of kept. Default: 168 (7 days). Prevents stale AMIs from accumulating.
+4. If a runner exceeds `maxWarmInstances` or `maxWarmAgeHours`, it is **terminated** (existing behavior).
+
+#### Scale-Up Lambda (modified)
+
+When a job is queued and no hot runner is available:
+
+1. **Before creating a new instance**, query the **DynamoDB warm pool table** for available stopped instances matching the runner config (owner, environment).
+2. If a warm instance exists:
+ - **Start** the instance (EC2 `StartInstances` API).
+ - Remove the DynamoDB record (atomic delete prevents race conditions).
+ - The instance's existing user-data/startup script re-registers with GitHub on boot.
+ - Tag: set `ghr:started-from-warm-pool=true` for metrics.
+3. If no warm instance exists (or start fails), fall through to the existing `createRunner()` flow (cold start).
+
+#### Pool Lambda (modified, separate feature flag)
+
+The pool lambda gets its own strategy setting (`pool_strategy`) independent of the scale-down warm pool behavior. This enables three distinct operational modes:
+
+| `pool_strategy` | `warm_pool_config.enabled` | Behavior |
+|-----------------|---------------------------|----------|
+| `hot` (default) | `false` | Current behavior: pool creates running instances, scale-down terminates them |
+| `hot` | `true` | Pool creates running instances, scale-down stops them into warm tier |
+| `warm` | `true` | Pool maintains **stopped** instances only (no idle compute). Scale-up/webhook starts them on demand |
+| `warm` | `false` | Invalid — rejected at plan time |
+
+When `pool_strategy = "warm"`:
+
+1. The pool lambda **creates instances and waits `warm_pool_ready_delay_seconds`** (default 30s) before stopping them. This grace period gives the instance time to boot, register with GitHub, and potentially pick up a queued job. If the runner picks up a job during this window, it is **not stopped** — it runs normally.
+2. After the grace period, if the runner is still idle (not busy), it is deregistered from GitHub, stopped, and tagged as warm. The pool target represents the number of **warm** (stopped) instances to maintain.
+3. No permanently running idle runners exist from the pool — zero long-term compute waste.
+4. Scale-up and webhook flows start warm instances when jobs arrive (10-30s startup).
+5. The `idle_config` / `idleCount` setting becomes irrelevant for pool runners since none are kept running long-term.
+
+When `pool_strategy = "hot"` (default, backward compatible):
+
+1. Pool creates running instances as today.
+2. If `warm_pool_config.enabled = true`, scale-down stops excess idle runners into the warm tier instead of terminating.
+3. Scale-up can still use warm instances as a fast fallback before cold-launching.
+
+### Feature Flag
+
+- Terraform variable: `warm_pool_config.enabled` (bool, default `false`) — controls whether scale-down stops instances into a warm tier.
+- Terraform variable: `pool_strategy` (string, `"hot"` or `"warm"`, default `"hot"`) — controls whether the pool lambda maintains running or stopped instances. **Independent from `warm_pool_config`** to allow warm-only deployments with zero idle compute.
+- Both are passed as environment variables to the relevant lambdas.
+- When both are at defaults, behavior is identical to today (no breaking changes).
+
+### Configuration
+
+New Terraform variable structure (nested in existing runner config patterns):
+
+```hcl
+variable "pool_strategy" {
+ description = "Strategy for the pool lambda. 'hot' keeps runners running (current behavior). 'warm' maintains stopped instances only — zero idle compute, 10-30s start on demand."
+ type = string
+ default = "hot"
+ validation {
+ condition = contains(["hot", "warm"], var.pool_strategy)
+ error_message = "pool_strategy must be 'hot' or 'warm'."
+ }
+}
+
+variable "warm_pool_config" {
+ description = "Configuration for the warm pool tier. Controls how stopped instances are managed."
+ type = object({
+ enabled = bool
+ max_warm_instances = number
+ max_warm_age_hours = number
+ warm_pool_ready_delay_seconds = number
+ })
+ default = {
+ enabled = false
+ max_warm_instances = 3
+ max_warm_age_hours = 168
+ warm_pool_ready_delay_seconds = 30
+ }
+}
+```
+
+**Validation**: `pool_strategy = "warm"` requires `warm_pool_config.enabled = true`. Terraform will error at plan time if this invariant is violated.
+
+### State Store (DynamoDB)
+
+Warm pool state is managed in a **DynamoDB table** rather than EC2 instance tags or DescribeInstances API calls. This avoids rate limiting, provides single-digit millisecond lookups, and enables atomic operations to prevent race conditions between concurrent lambda invocations.
+
+**Table**: `{prefix}-warm-pool` (PAY_PER_REQUEST billing)
+
+| Attribute | Type | Purpose |
+|-----------|------|---------|
+| `instanceId` (PK) | String | EC2 instance ID |
+| `runnerOwner` (GSI PK) | String | GitHub org/repo owner |
+| `stoppedAt` (GSI SK) | String | ISO 8601 — enables newest-first selection |
+| `environment` | String | Runner environment |
+| `runnerType` | String | `Org` or `Repo` |
+| `amiId` | String | AMI at time of stop (staleness detection) |
+| `instanceType` | String | For informational/filtering purposes |
+| `az` | String | Availability zone |
+| `expiresAt` (TTL) | Number | Auto-cleanup of stale records |
+
+DynamoDB TTL auto-deletes records past `max_warm_age_hours`. The scale-down lambda also actively terminates instances during its eviction pass.
+
+### Tags
+
+EC2 instance tags are still used for **observability and cost allocation** (not for state lookups):
+
+| Tag | Value | Purpose |
+|-----|-------|---------|
+| `ghr:started-from-warm-pool` | `true` | Instance was started from warm pool by scale-up (metrics) |
+| `ghr:warm-pool-grace-hit` | `true` | Instance picked up a job during the pool lambda's grace window |
+
+**Observability scenarios:**
+
+| Scenario | State change | How to track |
+|----------|-------------|-------------|
+| Scale-up restarts a stopped warm instance | DynamoDB record deleted + EC2 tag `ghr:started-from-warm-pool=true` | Count instances with tag / CloudWatch metric |
+| Pool lambda grace window: instance picks up job before being stopped | EC2 tag `ghr:warm-pool-grace-hit=true` | Count instances with tag / CloudWatch metric |
+| Instance stopped into warm tier | DynamoDB record created | Query DynamoDB for current warm pool size |
+| Warm instance terminated (age/cap/AMI eviction) | DynamoDB record deleted + EC2 instance terminated | CloudWatch metric `WarmPoolEvictions` |
+
+The pool lambda sets `ghr:warm-pool-grace-hit=true` when it detects a runner became busy during the `warm_pool_ready_delay_seconds` window. This lets operators measure how often the grace window saves a stop/start cycle — a high rate of grace hits means jobs are arriving frequently and the `warm` strategy is working efficiently even without going through the stopped state.
+
+### IAM Permissions
+
+The Lambda execution roles need additional permissions:
+
+**EC2** (scoped to `ghr:Application=github-action-runner` tag):
+- `ec2:StartInstances` — to start warm instances
+- `ec2:StopInstances` — to stop instances instead of terminating
+
+**DynamoDB** (scoped to warm pool table ARN):
+- `dynamodb:PutItem` — record new warm pool entries (scale-down, pool lambda)
+- `dynamodb:DeleteItem` — remove entries on start or termination (scale-up, scale-down, pool lambda)
+- `dynamodb:Query` — find available warm instances by owner (scale-up, pool lambda)
+- `dynamodb:GetItem` — check specific instance state (all lambdas)
+
+## Consequences
+
+### Positive
+
+- **Cost reduction**: Idle pool runners that never get a job now cost only EBS storage (~$0.80/month for a 10GB gp3 volume) instead of full EC2 compute ($30-150+/month depending on instance type).
+- **Faster startup**: Warm instances start in 10-30s vs. 2-5 minutes for cold launches. The EBS volume already contains the OS, runner binary, and any pre-installed tools.
+- **Preserved AMI investment**: Pre-baked AMIs with heavy toolchains (Docker images, SDKs) don't need to be re-downloaded.
+- **Backward compatible**: Feature flag ensures no behavior change for existing users.
+- **Graceful degradation**: If warm instances fail to start, the system falls through to cold launch.
+
+### Negative
+
+- **EBS costs accumulate**: Each warm instance retains its EBS volume. With `max_warm_instances` and `max_warm_age_hours` controls, this is bounded, but operators must be aware.
+- **Stale state risk**: A stopped instance may have outdated packages, expired credentials, or stale Docker caches. The startup script must handle re-registration and basic validation.
+- **Instance type lock-in**: A stopped instance retains its instance type. If the launch template or instance type config changes, warm instances become invalid and must be terminated.
+- **Complexity**: Three-state lifecycle is more complex than the current two-state (running/terminated) model.
+- **Spot instances**: Stopped spot instances can be reclaimed by AWS at any time. More critically, **one-time spot requests (the module default) cannot be stopped at all** — the EC2 API returns `UnsupportedOperation`. The warm pool works only with on-demand or persistent spot instances. The scale-down lambda handles this gracefully by falling back to terminate, but operators must set `instance_target_capacity_type = "on-demand"` for the warm pool to actually accumulate stopped instances. See [EC2 Instance Lifecycle and Warm Pool Compatibility](#ec2-instance-lifecycle-and-warm-pool-compatibility) for details.
+
+### Risks & Mitigations
+
+| Risk | Mitigation |
+|------|-----------|
+| EBS cost explosion | `max_warm_instances` hard cap + `max_warm_age_hours` TTL |
+| Stale AMI on warm instance | Compare instance's AMI ID against current launch template; terminate if mismatched |
+| Spot reclamation | Warm pool is best-effort for spot — if AWS reclaims a stopped spot instance, scale-up falls through to cold launch. Future: `warm_pool_capacity_type_override` to force on-demand for pool instances |
+| Runner binary outdated | Startup script always pulls latest runner version (existing behavior with `DISABLE_RUNNER_AUTOUPDATE=false`) |
+| GitHub registration failure on restart | If re-registration fails, tag as orphan and let next scale-down cycle terminate |
+
+## Alternatives Considered
+
+### 1. EC2 Hibernate (instead of Stop)
+
+Hibernate preserves RAM state, enabling even faster resume (~5-10s). However:
+
+- Requires specific instance types and AMI configuration
+- Requires pre-allocated EBS space for RAM dump
+- More complex setup and not all instance families support it
+- The marginal benefit over Stop (5-10s vs 10-30s) doesn't justify the complexity
+
+**Decision**: Start with Stop. Hibernate can be a future enhancement for specific instance types.
+
+### 2. AWS EC2 Auto Scaling Warm Pools
+
+AWS natively supports warm pools in Auto Scaling Groups. However:
+
+- This module uses EC2 Fleet API, not ASGs
+- Would require fundamental architecture change
+- Less control over tagging and lifecycle integration with GitHub
+- Doesn't integrate with the existing pool/scale-down lambda logic
+
+**Decision**: Implement warm pool logic in the existing lambda functions for tighter integration.
+
+### 3. EBS Snapshot + Fast Launch
+
+Take EBS snapshots of pool instances and use them for fast launch:
+
+- Still requires new instance creation (no IP/instance preservation)
+- Snapshot management adds complexity
+- Fast Launch has availability zone constraints
+
+**Decision**: Stop/Start is simpler and achieves comparable startup times.
+
+## EC2 Instance Lifecycle and Warm Pool Compatibility
+
+The warm pool relies on the EC2 `StopInstances` / `StartInstances` APIs. Not all EC2 purchase options support stop/start:
+
+### Instance Types by Purchase Option
+
+| Purchase Option | Can Stop/Start? | Warm Pool Compatible? | Cost vs On-Demand |
+|----------------|----------------|----------------------|-------------------|
+| **On-demand** | Yes | Yes (fully reliable) | 100% (baseline) |
+| **Persistent Spot** | Yes | Yes (best-effort) | 60-90% discount |
+| **One-time Spot** | No | No — must terminate | 60-90% discount |
+
+### What is a Persistent Spot Request?
+
+AWS Spot Instances come in two request types:
+
+- **One-time request** (default in this module via EC2 Fleet): AWS fulfills the request once. The instance **cannot be stopped** — only terminated. If interrupted by AWS, it is terminated and gone. This is what `instance_target_capacity_type = "spot"` uses today.
+
+- **Persistent request**: AWS keeps the request active. The instance **can be stopped and restarted**. When you stop a persistent spot instance, the underlying capacity is released (no compute cost), but the request remains open. When you start it, AWS attempts to re-acquire capacity at the current spot price. If capacity is unavailable, the start may be delayed.
+
+### Cost Comparison for Warm Pool
+
+Assuming a `m5.large` in eu-west-1 (~$0.096/hr on-demand, ~$0.035/hr spot):
+
+| Strategy | Running Cost | Stopped Cost | Monthly idle cost (1 instance, 50% idle) |
+|----------|-------------|-------------|------------------------------------------|
+| Hot pool (on-demand) | $0.096/hr | N/A (always running) | ~$34.56 |
+| Hot pool (spot) | $0.035/hr | N/A (always running) | ~$12.60 |
+| Warm pool (on-demand) | $0.096/hr when active | ~$0.80/mo EBS only | ~$0.80 + active hours |
+| Warm pool (persistent spot) | $0.035/hr when active | ~$0.80/mo EBS only | ~$0.80 + active hours |
+
+The warm pool's value proposition is strongest when runners spend significant time idle: the stopped EBS cost ($0.80/mo for 10GB gp3) vs running compute ($12-35/mo).
+
+### Persistent Spot Caveats
+
+1. **Restart not guaranteed**: When starting a stopped persistent spot instance, AWS may not have capacity at the current spot price. The instance stays in `pending` until capacity is available. The scale-up lambda should implement a timeout and fall back to cold launch.
+
+2. **Price changes**: If the spot price has risen above your max price since the instance was stopped, the start will fail.
+
+3. **Reclamation while stopped**: AWS can reclaim (terminate) a stopped spot instance if it needs the capacity, though this is rare for stopped instances.
+
+4. **Implemented in this module**: When `warm_pool_config.enabled = true` and `instance_target_capacity_type = "spot"`, the module uses `RunInstances` with `InstanceMarketOptions.SpotOptions.SpotInstanceType = "persistent"` and `InstanceInterruptionBehavior = "stop"`. It also overrides `InstanceInitiatedShutdownBehavior = "stop"` (since the launch template defaults to `"terminate"`, which conflicts with the spot stop behavior). This is gated behind the `ENABLE_PERSISTENT_SPOT` environment variable, which is derived automatically by Terraform.
+
+### Recommended Configuration
+
+For warm pool deployments, use one of:
+
+1. **On-demand** (`instance_target_capacity_type = "on-demand"`): Fully reliable, higher cost when active but zero cost when stopped. Best for critical workloads.
+
+2. **Spot for scale-up, on-demand for pool** (future `warm_pool_capacity_type_override`): Scale-up launches cheap spot instances for burst jobs; pool maintains on-demand instances that reliably stop/start. Best cost-to-reliability ratio.
+
+3. **On-demand with warm pool only** (no hot pool): Set `pool_strategy = "warm"` with on-demand instances. Pool creates instances, immediately stops them. Scale-up starts them on demand. Zero idle compute cost, 10-30s start time, fully reliable.
+
+> **Resolved**: When `instance_target_capacity_type = "spot"` and `warm_pool_config.enabled = true`, the module automatically switches from `CreateFleet` (one-time spot) to `RunInstances` with `SpotInstanceType = "persistent"` and `InstanceInterruptionBehavior = "stop"`. This is transparent to the user — no additional configuration is needed. The Terraform variable `ENABLE_PERSISTENT_SPOT` is derived automatically.
+
+## Implementation Plan
+
+See [docs/adr/001-warm-pool-implementation-plan.md](001-warm-pool-implementation-plan.md) for the phased implementation plan.
diff --git a/docs/adr/001-warm-pool-implementation-plan.md b/docs/adr/001-warm-pool-implementation-plan.md
new file mode 100644
index 0000000000..cea9be567c
--- /dev/null
+++ b/docs/adr/001-warm-pool-implementation-plan.md
@@ -0,0 +1,452 @@
+# ADR-001: Implementation Plan — Warm Pool with Stop/Hibernate
+
+## Overview
+
+This document details the phased implementation plan for the warm pool feature described in [ADR-001](001-warm-pool-hibernation.md).
+
+## Phase 1: Foundation (Terraform + IAM + Feature Flag + DynamoDB)
+
+**Goal**: Wire the feature flag, permissions, and state store without changing any runtime behavior.
+
+### Tasks
+
+1. **Add `warm_pool_config` variable** to `modules/runners/variables.tf`
+ ```hcl
+ variable "warm_pool_config" {
+ description = "Configuration for the warm pool tier. Controls how stopped instances are managed."
+ type = object({
+ enabled = bool
+ max_warm_instances = number
+ max_warm_age_hours = number
+ warm_pool_ready_delay_seconds = number
+ })
+ default = {
+ enabled = false
+ max_warm_instances = 3
+ max_warm_age_hours = 168
+ warm_pool_ready_delay_seconds = 30
+ }
+ }
+ ```
+
+2. **Add `pool_strategy` variable** to `modules/runners/variables.tf`
+ ```hcl
+ variable "pool_strategy" {
+ description = "Strategy for the pool lambda. 'hot' keeps runners running. 'warm' maintains stopped instances only."
+ type = string
+ default = "hot"
+ validation {
+ condition = contains(["hot", "warm"], var.pool_strategy)
+ error_message = "pool_strategy must be 'hot' or 'warm'."
+ }
+ }
+ ```
+
+3. **Add cross-variable validation** (lifecycle precondition or `check` block):
+ - `pool_strategy = "warm"` requires `warm_pool_config.enabled = true`
+
+4. **Create DynamoDB table** for warm pool state (`modules/runners/warm-pool.tf`):
+ ```hcl
+ resource "aws_dynamodb_table" "warm_pool" {
+ count = var.warm_pool_config.enabled ? 1 : 0
+ name = "${var.prefix}-warm-pool"
+ billing_mode = "PAY_PER_REQUEST"
+ hash_key = "instanceId"
+
+ attribute {
+ name = "instanceId"
+ type = "S"
+ }
+
+ attribute {
+ name = "runnerOwner"
+ type = "S"
+ }
+
+ attribute {
+ name = "stoppedAt"
+ type = "S"
+ }
+
+ global_secondary_index {
+ name = "by-owner"
+ hash_key = "runnerOwner"
+ range_key = "stoppedAt"
+ projection_type = "ALL"
+ }
+
+ ttl {
+ attribute_name = "expiresAt"
+ enabled = true
+ }
+
+ tags = local.tags
+ }
+ ```
+
+ **Table schema:**
+
+ | Attribute | Type | Purpose |
+ |-----------|------|---------|
+ | `instanceId` (PK) | String | EC2 instance ID |
+ | `runnerOwner` (GSI PK) | String | GitHub org/repo owner (for filtering) |
+ | `stoppedAt` (GSI SK) | String | ISO 8601 timestamp when stopped (for newest-first selection + age eviction) |
+ | `environment` | String | Runner environment tag |
+ | `runnerType` | String | `Org` or `Repo` |
+ | `amiId` | String | AMI ID at time of stopping (for staleness check) |
+ | `instanceType` | String | EC2 instance type |
+ | `az` | String | Availability zone (instance can only restart in same AZ) |
+ | `expiresAt` | Number | Unix epoch for DynamoDB TTL auto-deletion (set to `stoppedAt + max_warm_age_hours`) |
+
+ DynamoDB TTL automatically cleans up stale records — no lambda logic needed for age-based eviction of the DB record itself. The scale-down lambda still terminates the actual EC2 instance.
+
+5. **Add IAM permissions** to the Lambda execution roles:
+ - `ec2:StopInstances` (scale-down lambda)
+ - `ec2:StartInstances` (scale-up and pool lambdas)
+ - `dynamodb:PutItem`, `dynamodb:DeleteItem`, `dynamodb:GetItem` (scale-down, scale-up, pool lambdas)
+ - `dynamodb:Query` on GSI `by-owner` (scale-up and pool lambdas)
+ - Condition on EC2 actions: `ec2:ResourceTag/ghr:Application = github-action-runner`
+ - Condition on DynamoDB: resource ARN scoped to the warm pool table
+
+6. **Pass environment variables** to all three lambdas:
+ - `ENABLE_WARM_POOL` → `var.warm_pool_config.enabled`
+ - `WARM_POOL_MAX_INSTANCES` → `var.warm_pool_config.max_warm_instances`
+ - `WARM_POOL_MAX_AGE_HOURS` → `var.warm_pool_config.max_warm_age_hours`
+ - `WARM_POOL_READY_DELAY_SECONDS` → `var.warm_pool_config.warm_pool_ready_delay_seconds` (pool lambda only)
+ - `POOL_STRATEGY` → `var.pool_strategy` (pool lambda only)
+ - `WARM_POOL_TABLE_NAME` → DynamoDB table name (all three lambdas)
+
+7. **Wire through multi-runner module** (`modules/multi-runner/runners.tf`):
+ - Add `warm_pool_config` and `pool_strategy` to the `multi_runner_config` object type
+ - Pass through to the runners module
+
+### Files Modified
+- `modules/runners/variables.tf`
+- `modules/runners/warm-pool.tf` (new — DynamoDB table + IAM)
+- `modules/runners/scale-down.tf` (env vars)
+- `modules/runners/scale-up.tf` (env vars)
+- `modules/runners/pool/main.tf` (env vars)
+- `modules/runners/policies/lambda-scale-down.json` (IAM — EC2 Stop + DynamoDB)
+- `modules/runners/policies/lambda-scale-up.json` (IAM — EC2 Start + DynamoDB)
+- `modules/multi-runner/variables.tf`
+- `modules/multi-runner/runners.tf`
+
+---
+
+## Phase 2: Scale-Down — Stop Instead of Terminate
+
+**Goal**: Idle pool runners are stopped instead of terminated when warm pool is enabled.
+
+### Tasks
+
+1. **Add `stopRunner()` function** to `lambdas/functions/control-plane/src/aws/runners.ts`:
+ ```typescript
+ export async function stopRunner(instanceId: string): Promiseobject({
enable = optional(bool, false)
namespace = optional(string, "GitHub Runners")
metric = optional(object({
enable_github_app_rate_limit = optional(bool, true)
enable_job_retry = optional(bool, true)
enable_spot_termination_warning = optional(bool, true)
}), {})
}) | `{}` | no |
-| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {map(object({
runner_config = object({
runner_os = string
runner_architecture = string
runner_metadata_options = optional(map(any), {
instance_metadata_tags = "enabled"
http_endpoint = "enabled"
http_tokens = "required"
http_put_response_hop_limit = 1
})
ami = optional(object({
filter = optional(map(list(string)), { state = ["available"] })
owners = optional(list(string), ["amazon"])
id_ssm_parameter_arn = optional(string, null)
kms_key_arn = optional(string, null)
}), null)
create_service_linked_role_spot = optional(bool, false)
credit_specification = optional(string, null)
delay_webhook_event = optional(number, 30)
disable_runner_autoupdate = optional(bool, false)
ebs_optimized = optional(bool, false)
enable_ephemeral_runners = optional(bool, false)
enable_job_queued_check = optional(bool, null)
enable_on_demand_failover_for_errors = optional(list(string), [])
scale_errors = optional(list(string), [
"UnfulfillableCapacity",
"MaxSpotInstanceCountExceeded",
"TargetCapacityLimitExceededException",
"RequestLimitExceeded",
"ResourceLimitExceeded",
"MaxSpotInstanceCountExceeded",
"MaxSpotFleetRequestCountExceeded",
"InsufficientInstanceCapacity",
"InsufficientCapacityOnHost",
])
enable_organization_runners = optional(bool, false)
enable_runner_binaries_syncer = optional(bool, true)
enable_ssm_on_runners = optional(bool, false)
enable_userdata = optional(bool, true)
instance_allocation_strategy = optional(string, "lowest-price")
instance_max_spot_price = optional(string, null)
instance_target_capacity_type = optional(string, "spot")
instance_types = list(string)
job_queue_retention_in_seconds = optional(number, 86400)
minimum_running_time_in_minutes = optional(number, null)
pool_runner_owner = optional(string, null)
runner_as_root = optional(bool, false)
runner_boot_time_in_minutes = optional(number, 5)
runner_disable_default_labels = optional(bool, false)
runner_extra_labels = optional(list(string), [])
runner_group_name = optional(string, "Default")
runner_name_prefix = optional(string, "")
runner_run_as = optional(string, "ec2-user")
runners_maximum_count = number
runner_additional_security_group_ids = optional(list(string), [])
scale_down_schedule_expression = optional(string, "cron(*/5 * * * ? *)")
scale_up_reserved_concurrent_executions = optional(number, 1)
lambda_event_source_mapping_batch_size = optional(number, null)
lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
userdata_template = optional(string, null)
userdata_content = optional(string, null)
enable_jit_config = optional(bool, null)
enable_runner_detailed_monitoring = optional(bool, false)
enable_cloudwatch_agent = optional(bool, true)
cloudwatch_config = optional(string, null)
userdata_pre_install = optional(string, "")
userdata_post_install = optional(string, "")
runner_hook_job_started = optional(string, "")
runner_hook_job_completed = optional(string, "")
runner_ec2_tags = optional(map(string), {})
runner_iam_role_managed_policy_arns = optional(list(string), [])
vpc_id = optional(string, null)
subnet_ids = optional(list(string), null)
idle_config = optional(list(object({
cron = string
timeZone = string
idleCount = number
evictionStrategy = optional(string, "oldest_first")
})), [])
cpu_options = optional(object({
core_count = optional(number)
threads_per_core = optional(number)
amd_sev_snp = optional(string)
nested_virtualization = optional(string)
}), null)
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_id = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
spread_domain = optional(string)
tenancy = optional(string)
partition_number = optional(number)
}), null)
license_specifications = optional(list(object({
license_configuration_arn = string
})), [])
use_dedicated_host = optional(bool, false)
runner_log_files = optional(list(object({
log_group_name = string
prefix_log_group = bool
file_path = string
log_stream_name = string
log_class = optional(string, "STANDARD")
})), null)
block_device_mappings = optional(list(object({
delete_on_termination = optional(bool, true)
device_name = optional(string, "/dev/xvda")
encrypted = optional(bool, true)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = number
volume_type = optional(string, "gp3")
})), [{
volume_size = 30
}])
pool_config = optional(list(object({
schedule_expression = string
schedule_expression_timezone = optional(string)
size = number
})), [])
job_retry = optional(object({
enable = optional(bool, false)
delay_in_seconds = optional(number, 300)
delay_backoff = optional(number, 2)
lambda_memory_size = optional(number, 256)
lambda_timeout = optional(number, 30)
max_attempts = optional(number, 1)
}), {})
iam_overrides = optional(object({
override_instance_profile = optional(bool, null)
instance_profile_name = optional(string, null)
override_runner_role = optional(bool, null)
runner_role_arn = optional(string, null)
}), {
override_instance_profile = false
instance_profile_name = null
override_runner_role = false
runner_role_arn = null
})
})
matcherConfig = object({
labelMatchers = list(list(string))
exactMatch = optional(bool, false)
bidirectionalLabelMatch = optional(bool, false)
priority = optional(number, 999)
enableDynamicLabels = optional(bool, false)
ec2DynamicLabelsPolicy = optional(any, null)
})
redrive_build_queue = optional(object({
enabled = bool
maxReceiveCount = number
}), {
enabled = false
maxReceiveCount = null
})
})) | n/a | yes |
+| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {map(object({
runner_config = object({
runner_os = string
runner_architecture = string
runner_metadata_options = optional(map(any), {
instance_metadata_tags = "enabled"
http_endpoint = "enabled"
http_tokens = "required"
http_put_response_hop_limit = 1
})
ami = optional(object({
filter = optional(map(list(string)), { state = ["available"] })
owners = optional(list(string), ["amazon"])
id_ssm_parameter_arn = optional(string, null)
kms_key_arn = optional(string, null)
}), null)
create_service_linked_role_spot = optional(bool, false)
credit_specification = optional(string, null)
delay_webhook_event = optional(number, 30)
disable_runner_autoupdate = optional(bool, false)
ebs_optimized = optional(bool, false)
enable_ephemeral_runners = optional(bool, false)
enable_job_queued_check = optional(bool, null)
enable_on_demand_failover_for_errors = optional(list(string), [])
scale_errors = optional(list(string), [
"UnfulfillableCapacity",
"MaxSpotInstanceCountExceeded",
"TargetCapacityLimitExceededException",
"RequestLimitExceeded",
"ResourceLimitExceeded",
"MaxSpotInstanceCountExceeded",
"MaxSpotFleetRequestCountExceeded",
"InsufficientInstanceCapacity",
"InsufficientCapacityOnHost",
])
enable_organization_runners = optional(bool, false)
enable_runner_binaries_syncer = optional(bool, true)
enable_ssm_on_runners = optional(bool, false)
enable_userdata = optional(bool, true)
instance_allocation_strategy = optional(string, "lowest-price")
instance_type_priorities = optional(map(number), null)
instance_max_spot_price = optional(string, null)
instance_target_capacity_type = optional(string, "spot")
instance_types = list(string)
job_queue_retention_in_seconds = optional(number, 86400)
minimum_running_time_in_minutes = optional(number, null)
pool_runner_owner = optional(string, null)
runner_as_root = optional(bool, false)
runner_boot_time_in_minutes = optional(number, 5)
runner_disable_default_labels = optional(bool, false)
runner_extra_labels = optional(list(string), [])
runner_group_name = optional(string, "Default")
runner_name_prefix = optional(string, "")
runner_run_as = optional(string, "ec2-user")
runners_maximum_count = number
runner_additional_security_group_ids = optional(list(string), [])
scale_down_schedule_expression = optional(string, "cron(*/5 * * * ? *)")
scale_up_reserved_concurrent_executions = optional(number, 1)
lambda_event_source_mapping_batch_size = optional(number, null)
lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
userdata_template = optional(string, null)
userdata_content = optional(string, null)
enable_jit_config = optional(bool, null)
enable_runner_detailed_monitoring = optional(bool, false)
enable_cloudwatch_agent = optional(bool, true)
cloudwatch_config = optional(string, null)
userdata_pre_install = optional(string, "")
userdata_post_install = optional(string, "")
runner_hook_job_started = optional(string, "")
runner_hook_job_completed = optional(string, "")
runner_ec2_tags = optional(map(string), {})
runner_iam_role_managed_policy_arns = optional(list(string), [])
vpc_id = optional(string, null)
subnet_ids = optional(list(string), null)
idle_config = optional(list(object({
cron = string
timeZone = string
idleCount = number
evictionStrategy = optional(string, "oldest_first")
})), [])
cpu_options = optional(object({
core_count = optional(number)
threads_per_core = optional(number)
amd_sev_snp = optional(string)
nested_virtualization = optional(string)
}), null)
placement = optional(object({
affinity = optional(string)
availability_zone = optional(string)
group_id = optional(string)
group_name = optional(string)
host_id = optional(string)
host_resource_group_arn = optional(string)
spread_domain = optional(string)
tenancy = optional(string)
partition_number = optional(number)
}), null)
license_specifications = optional(list(object({
license_configuration_arn = string
})), [])
use_dedicated_host = optional(bool, false)
runner_log_files = optional(list(object({
log_group_name = string
prefix_log_group = bool
file_path = string
log_stream_name = string
log_class = optional(string, "STANDARD")
})), null)
block_device_mappings = optional(list(object({
delete_on_termination = optional(bool, true)
device_name = optional(string, "/dev/xvda")
encrypted = optional(bool, true)
iops = optional(number)
kms_key_id = optional(string)
snapshot_id = optional(string)
throughput = optional(number)
volume_initialization_rate = optional(number)
volume_size = number
volume_type = optional(string, "gp3")
})), [{
volume_size = 30
}])
pool_config = optional(list(object({
schedule_expression = string
schedule_expression_timezone = optional(string)
size = number
})), [])
pool_strategy = optional(string, "hot")
warm_pool_config = optional(object({
enabled = optional(bool, false)
max_warm_instances = optional(number, 3)
max_warm_age_hours = optional(number, 168)
warm_pool_ready_delay_seconds = optional(number, 30)
}), {})
job_retry = optional(object({
enable = optional(bool, false)
delay_in_seconds = optional(number, 300)
delay_backoff = optional(number, 2)
lambda_memory_size = optional(number, 256)
lambda_timeout = optional(number, 30)
max_attempts = optional(number, 1)
}), {})
iam_overrides = optional(object({
override_instance_profile = optional(bool, null)
instance_profile_name = optional(string, null)
override_runner_role = optional(bool, null)
runner_role_arn = optional(string, null)
}), {
override_instance_profile = false
instance_profile_name = null
override_runner_role = false
runner_role_arn = null
})
})
matcherConfig = object({
labelMatchers = list(list(string))
exactMatch = optional(bool, false)
bidirectionalLabelMatch = optional(bool, false)
priority = optional(number, 999)
enableDynamicLabels = optional(bool, false)
ec2DynamicLabelsPolicy = optional(any, null)
})
redrive_build_queue = optional(object({
enabled = bool
maxReceiveCount = number
}), {
enabled = false
maxReceiveCount = null
})
})) | n/a | yes |
| [parameter\_store\_tags](#input\_parameter\_store\_tags) | Map of tags that will be added to all the SSM Parameter Store parameters created by the Lambda function. | `map(string)` | `{}` | no |
| [pool\_lambda\_reserved\_concurrent\_executions](#input\_pool\_lambda\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no |
| [pool\_lambda\_timeout](#input\_pool\_lambda\_timeout) | Time out for the pool lambda in seconds. | `number` | `60` | no |
diff --git a/modules/multi-runner/runners.tf b/modules/multi-runner/runners.tf
index 892113dcc7..99eb5ca008 100644
--- a/modules/multi-runner/runners.tf
+++ b/modules/multi-runner/runners.tf
@@ -120,6 +120,8 @@ module "runners" {
pool_lambda_timeout = var.pool_lambda_timeout
pool_runner_owner = each.value.runner_config.pool_runner_owner
pool_lambda_reserved_concurrent_executions = var.pool_lambda_reserved_concurrent_executions
+ pool_strategy = try(each.value.runner_config.pool_strategy, "hot")
+ warm_pool_config = try(each.value.runner_config.warm_pool_config, { enabled = false, max_warm_instances = 3, max_warm_age_hours = 168, warm_pool_ready_delay_seconds = 30 })
associate_public_ipv4_address = var.associate_public_ipv4_address
ssm_housekeeper = var.runners_ssm_housekeeper
diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf
index 0ff62b0083..c52c4934aa 100644
--- a/modules/multi-runner/variables.tf
+++ b/modules/multi-runner/variables.tf
@@ -182,6 +182,13 @@ variable "multi_runner_config" {
schedule_expression_timezone = optional(string)
size = number
})), [])
+ pool_strategy = optional(string, "hot")
+ warm_pool_config = optional(object({
+ enabled = optional(bool, false)
+ max_warm_instances = optional(number, 3)
+ max_warm_age_hours = optional(number, 168)
+ warm_pool_ready_delay_seconds = optional(number, 30)
+ }), {})
job_retry = optional(object({
enable = optional(bool, false)
delay_in_seconds = optional(number, 300)
@@ -279,6 +286,8 @@ variable "multi_runner_config" {
block_device_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`."
job_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app."
pool_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)."
+ pool_strategy: "Strategy for maintaining idle runners: `hot` (default, traditional) or `warm` (stop instead of terminate)."
+ warm_pool_config: "Configuration for the warm pool feature. When enabled, idle runners are stopped instead of terminated for faster restart."
iam_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances."
}
matcherConfig: {
diff --git a/modules/runners/README.md b/modules/runners/README.md
index 112d2894ad..b13ec214dc 100644
--- a/modules/runners/README.md
+++ b/modules/runners/README.md
@@ -80,6 +80,7 @@ yarn run dist
| [aws_cloudwatch_log_group.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
| [aws_cloudwatch_log_group.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
| [aws_cloudwatch_log_group.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
+| [aws_dynamodb_table.warm_pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table) | resource |
| [aws_iam_instance_profile.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource |
| [aws_iam_policy.ami_id_ssm_parameter_read](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
| [aws_iam_role.runner](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
@@ -92,12 +93,15 @@ yarn run dist
| [aws_iam_role_policy.dist_bucket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy.ec2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy.job_retry_sqs_publish](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
+| [aws_iam_role_policy.pool_warm_pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy.runner_session_manager_aws_managed](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy.scale_down](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy.scale_down_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
+| [aws_iam_role_policy.scale_down_warm_pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy.scale_down_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy.scale_up](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy.scale_up_logging](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
+| [aws_iam_role_policy.scale_up_warm_pool](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy.scale_up_xray](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy.service_linked_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy.ssm_housekeeper](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
@@ -106,6 +110,7 @@ yarn run dist
| [aws_iam_role_policy.ssm_parameters](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_iam_role_policy_attachment.ami_id_ssm_parameter_read](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.managed_policies](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.scale_down_ami_id_ssm_parameter_read](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.scale_down_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.scale_up_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.ssm_housekeeper_vpc_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
@@ -165,10 +170,11 @@ yarn run dist
| [github\_app\_parameters](#input\_github\_app\_parameters) | Parameter Store for GitHub App Parameters. | object({
key_base64 = map(string)
id = map(string)
}) | n/a | yes |
| [iam\_overrides](#input\_iam\_overrides) | This map provides the possibility to override some IAM defaults. The following attributes are supported: `instance_profile_name` overrides the instance profile name used in the launch template. `runner_role_arn` overrides the IAM role ARN used for the runner instances. | object({
override_instance_profile = optional(bool, null)
instance_profile_name = optional(string, null)
override_runner_role = optional(bool, null)
runner_role_arn = optional(string, null)
}) | {
"instance_profile_name": null,
"override_instance_profile": false,
"override_runner_role": false,
"runner_role_arn": null
} | no |
| [idle\_config](#input\_idle\_config) | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | list(object({
cron = string
timeZone = string
idleCount = number
evictionStrategy = optional(string, "oldest_first")
})) | `[]` | no |
-| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`. | `string` | `"lowest-price"` | no |
+| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`. | `string` | `"lowest-price"` | no |
| [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no |
| [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the prefix will be used. | `string` | `null` | no |
| [instance\_target\_capacity\_type](#input\_instance\_target\_capacity\_type) | Default lifecycle used runner instances, can be either `spot` or `on-demand`. | `string` | `"spot"` | no |
+| [instance\_type\_priorities](#input\_instance\_type\_priorities) | A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`. | `map(number)` | `null` | no |
| [instance\_types](#input\_instance\_types) | List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win). | `list(string)` | `null` | no |
| [job\_retry](#input\_job\_retry) | Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app.object({
enable = optional(bool, false)
delay_in_seconds = optional(number, 300)
delay_backoff = optional(number, 2)
lambda_memory_size = optional(number, 256)
lambda_reserved_concurrent_executions = optional(number, 1)
lambda_timeout = optional(number, 30)
max_attempts = optional(number, 1)
}) | `{}` | no |
| [key\_name](#input\_key\_name) | Key pair name | `string` | `null` | no |
@@ -192,7 +198,7 @@ yarn run dist
| [logging\_kms\_key\_id](#input\_logging\_kms\_key\_id) | Specifies the kms key id to encrypt the logs with | `string` | `null` | no |
| [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no |
| [metadata\_options](#input\_metadata\_options) | Metadata options for the ec2 runner instances. By default, the module uses metadata tags for bootstrapping the runner, only disable `instance_metadata_tags` when using custom scripts for starting the runner. | `map(any)` | {
"http_endpoint": "enabled",
"http_put_response_hop_limit": 1,
"http_tokens": "required",
"instance_metadata_tags": "enabled"
} | no |
-| [metrics](#input\_metrics) | Configuration for metrics created by the module, by default metrics are disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | object({
enable = optional(bool, false)
namespace = optional(string, "GitHub Runners")
metric = optional(object({
enable_github_app_rate_limit = optional(bool, true)
enable_job_retry = optional(bool, true)
enable_spot_termination_warning = optional(bool, true)
}), {})
}) | `{}` | no |
+| [metrics](#input\_metrics) | Configuration for metrics created by the module, by default metrics are disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | object({
enable = optional(bool, false)
namespace = optional(string, "GitHub Runners")
metric = optional(object({
enable_github_app_rate_limit = optional(bool, true)
enable_job_retry = optional(bool, true)
enable_spot_termination_warning = optional(bool, true)
enable_warm_pool = optional(bool, true)
}), {})
}) | `{}` | no |
| [minimum\_running\_time\_in\_minutes](#input\_minimum\_running\_time\_in\_minutes) | The time an ec2 action runner should be running at minimum before terminated if non busy. If not set the default is calculated based on the OS. | `number` | `null` | no |
| [overrides](#input\_overrides) | This map provides the possibility to override some defaults. The following attributes are supported: `name_sg` overrides the `Name` tag for all security groups created by this module. `name_runner_agent_instance` overrides the `Name` tag for the ec2 instance defined in the auto launch configuration. `name_docker_machine_runners` overrides the `Name` tag spot instances created by the runner agent. | `map(string)` | {
"name_runner": "",
"name_sg": ""
} | no |
| [parameter\_store\_tags](#input\_parameter\_store\_tags) | Map of tags that will be added to all the SSM Parameter Store parameters created by the Lambda function. | `map(string)` | `{}` | no |
@@ -202,6 +208,7 @@ yarn run dist
| [pool\_lambda\_reserved\_concurrent\_executions](#input\_pool\_lambda\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no |
| [pool\_lambda\_timeout](#input\_pool\_lambda\_timeout) | Time out for the pool lambda in seconds. | `number` | `60` | no |
| [pool\_runner\_owner](#input\_pool\_runner\_owner) | The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported. | `string` | `null` | no |
+| [pool\_strategy](#input\_pool\_strategy) | Strategy for maintaining idle runners.object({
enabled = optional(bool, false)
max_warm_instances = optional(number, 3)
max_warm_age_hours = optional(number, 168)
warm_pool_ready_delay_seconds = optional(number, 30)
}) | `{}` | no |
## Outputs
diff --git a/modules/runners/policies/lambda-warm-pool.json b/modules/runners/policies/lambda-warm-pool.json
new file mode 100644
index 0000000000..d14c504acd
--- /dev/null
+++ b/modules/runners/policies/lambda-warm-pool.json
@@ -0,0 +1,48 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "dynamodb:PutItem",
+ "dynamodb:DeleteItem",
+ "dynamodb:GetItem",
+ "dynamodb:Query"
+ ],
+ "Resource": [
+ "${dynamodb_table_arn}",
+ "${dynamodb_table_arn}/index/*"
+ ]
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "ec2:StopInstances",
+ "ec2:StartInstances",
+ "ec2:CreateTags",
+ "ec2:DeleteTags"
+ ],
+ "Resource": "*",
+ "Condition": {
+ "StringEquals": {
+ "ec2:ResourceTag/ghr:Application": "github-action-runner"
+ }
+ }
+ },
+ {
+ "Effect": "Allow",
+ "Action": [
+ "ec2:StopInstances",
+ "ec2:StartInstances",
+ "ec2:CreateTags",
+ "ec2:DeleteTags"
+ ],
+ "Resource": "*",
+ "Condition": {
+ "StringEquals": {
+ "ec2:ResourceTag/gh:environment": "${environment}"
+ }
+ }
+ }
+ ]
+}
diff --git a/modules/runners/pool.tf b/modules/runners/pool.tf
index d9899fd21c..7b0a9056ae 100644
--- a/modules/runners/pool.tf
+++ b/modules/runners/pool.tf
@@ -64,6 +64,15 @@ module "pool" {
tags = local.tags
lambda_tags = var.lambda_tags
arn_ssm_parameters_path_config = local.arn_ssm_parameters_path_config
+ warm_pool_table_name = var.warm_pool_config.enabled ? aws_dynamodb_table.warm_pool[0].name : ""
+ warm_pool_config = {
+ enabled = var.warm_pool_config.enabled
+ max_warm_instances = var.warm_pool_config.max_warm_instances
+ max_warm_age_hours = var.warm_pool_config.max_warm_age_hours
+ warm_pool_ready_delay_seconds = var.warm_pool_config.warm_pool_ready_delay_seconds
+ }
+ pool_strategy = var.pool_strategy
+ enable_metric_warm_pool = var.metrics.enable && var.metrics.metric.enable_warm_pool
}
aws_partition = var.aws_partition
diff --git a/modules/runners/pool/README.md b/modules/runners/pool/README.md
index 401c7f2cc8..71f57fcaa8 100644
--- a/modules/runners/pool/README.md
+++ b/modules/runners/pool/README.md
@@ -49,7 +49,7 @@ No modules.
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| [aws\_partition](#input\_aws\_partition) | (optional) partition for the arn if not 'aws' | `string` | `"aws"` | no |
-| [config](#input\_config) | Lookup details in parent module. | object({
lambda = object({
log_level = string
logging_retention_in_days = number
logging_kms_key_id = string
log_class = string
reserved_concurrent_executions = number
s3_bucket = string
s3_key = string
s3_object_version = string
security_group_ids = list(string)
runtime = string
architecture = string
memory_size = number
timeout = number
zip = string
subnet_ids = list(string)
parameter_store_tags = string
})
tags = map(string)
ghes = object({
url = string
ssl_verify = string
})
github_app_parameters = object({
key_base64 = map(string)
id = map(string)
})
subnet_ids = list(string)
runner = object({
disable_runner_autoupdate = bool
ephemeral = bool
enable_jit_config = bool
enable_on_demand_failover_for_errors = list(string)
scale_errors = list(string)
boot_time_in_minutes = number
labels = list(string)
launch_template = object({
name = string
})
group_name = string
name_prefix = string
pool_owner = string
role = object({
arn = string
})
use_dedicated_host = bool
})
runners_maximum_count = number
instance_types = list(string)
instance_target_capacity_type = string
instance_allocation_strategy = string
instance_max_spot_price = string
prefix = string
pool = list(object({
schedule_expression = string
schedule_expression_timezone = string
size = number
}))
role_permissions_boundary = string
kms_key_arn = string
ami_kms_key_arn = string
ami_id_ssm_parameter_arn = string
role_path = string
ssm_token_path = string
ssm_config_path = string
ami_id_ssm_parameter_name = string
ami_id_ssm_parameter_read_policy_arn = string
arn_ssm_parameters_path_config = string
lambda_tags = map(string)
user_agent = string
}) | n/a | yes |
+| [config](#input\_config) | Lookup details in parent module. | object({
lambda = object({
log_level = string
logging_retention_in_days = number
logging_kms_key_id = string
log_class = string
reserved_concurrent_executions = number
s3_bucket = string
s3_key = string
s3_object_version = string
security_group_ids = list(string)
runtime = string
architecture = string
memory_size = number
timeout = number
zip = string
subnet_ids = list(string)
parameter_store_tags = string
})
tags = map(string)
ghes = object({
url = string
ssl_verify = string
})
github_app_parameters = object({
key_base64 = map(string)
id = map(string)
})
subnet_ids = list(string)
runner = object({
disable_runner_autoupdate = bool
ephemeral = bool
enable_jit_config = bool
enable_on_demand_failover_for_errors = list(string)
scale_errors = list(string)
boot_time_in_minutes = number
labels = list(string)
launch_template = object({
name = string
})
group_name = string
name_prefix = string
pool_owner = string
role = object({
arn = string
})
use_dedicated_host = bool
})
runners_maximum_count = number
instance_types = list(string)
instance_type_priorities = optional(map(number))
instance_target_capacity_type = string
instance_allocation_strategy = string
instance_max_spot_price = string
prefix = string
pool = list(object({
schedule_expression = string
schedule_expression_timezone = string
size = number
}))
role_permissions_boundary = string
kms_key_arn = string
ami_kms_key_arn = string
ami_id_ssm_parameter_arn = string
role_path = string
ssm_token_path = string
ssm_config_path = string
ami_id_ssm_parameter_name = string
ami_id_ssm_parameter_read_policy_arn = string
arn_ssm_parameters_path_config = string
lambda_tags = map(string)
user_agent = string
warm_pool_table_name = string
warm_pool_config = object({
enabled = bool
max_warm_instances = number
max_warm_age_hours = number
warm_pool_ready_delay_seconds = number
})
pool_strategy = string
enable_metric_warm_pool = bool
}) | n/a | yes |
| [tracing\_config](#input\_tracing\_config) | Configuration for lambda tracing. | object({
mode = optional(string, null)
capture_http_requests = optional(bool, false)
capture_error = optional(bool, false)
}) | `{}` | no |
## Outputs
diff --git a/modules/runners/pool/main.tf b/modules/runners/pool/main.tf
index efdf537a96..b9845bab3c 100644
--- a/modules/runners/pool/main.tf
+++ b/modules/runners/pool/main.tf
@@ -60,6 +60,15 @@ resource "aws_lambda_function" "pool" {
SSM_PARAMETER_STORE_TAGS = var.config.lambda.parameter_store_tags
SCALE_ERRORS = jsonencode(var.config.runner.scale_errors)
USE_DEDICATED_HOST = var.config.runner.use_dedicated_host
+ WARM_POOL_CONFIG = jsonencode({
+ enabled = var.config.warm_pool_config.enabled
+ maxWarmInstances = var.config.warm_pool_config.max_warm_instances
+ maxWarmAgeHours = var.config.warm_pool_config.max_warm_age_hours
+ warmPoolReadyDelaySeconds = var.config.warm_pool_config.warm_pool_ready_delay_seconds
+ })
+ WARM_POOL_TABLE_NAME = var.config.warm_pool_table_name
+ POOL_STRATEGY = var.config.pool_strategy
+ ENABLE_METRIC_WARM_POOL = var.config.enable_metric_warm_pool
}
}
diff --git a/modules/runners/pool/variables.tf b/modules/runners/pool/variables.tf
index 2666444ae9..bbf361fbcb 100644
--- a/modules/runners/pool/variables.tf
+++ b/modules/runners/pool/variables.tf
@@ -72,6 +72,15 @@ variable "config" {
arn_ssm_parameters_path_config = string
lambda_tags = map(string)
user_agent = string
+ warm_pool_table_name = string
+ warm_pool_config = object({
+ enabled = bool
+ max_warm_instances = number
+ max_warm_age_hours = number
+ warm_pool_ready_delay_seconds = number
+ })
+ pool_strategy = string
+ enable_metric_warm_pool = bool
})
}
diff --git a/modules/runners/scale-down.tf b/modules/runners/scale-down.tf
index 449d1970ed..a7a567d56f 100644
--- a/modules/runners/scale-down.tf
+++ b/modules/runners/scale-down.tf
@@ -27,6 +27,7 @@ resource "aws_lambda_function" "scale_down" {
variables = {
ENVIRONMENT = var.prefix
ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.metrics.enable && var.metrics.metric.enable_github_app_rate_limit
+ ENABLE_METRIC_WARM_POOL = var.metrics.enable && var.metrics.metric.enable_warm_pool
GHES_URL = var.ghes_url
USER_AGENT = var.user_agent
LOG_LEVEL = var.log_level
@@ -42,6 +43,15 @@ resource "aws_lambda_function" "scale_down" {
POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false
POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests
POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error
+ WARM_POOL_CONFIG = jsonencode({
+ enabled = var.warm_pool_config.enabled
+ maxWarmInstances = var.warm_pool_config.max_warm_instances
+ maxWarmAgeHours = var.warm_pool_config.max_warm_age_hours
+ warmPoolReadyDelaySeconds = var.warm_pool_config.warm_pool_ready_delay_seconds
+ })
+ WARM_POOL_TABLE_NAME = var.warm_pool_config.enabled ? aws_dynamodb_table.warm_pool[0].name : ""
+ POOL_STRATEGY = var.pool_strategy
+ AMI_ID_SSM_PARAMETER_NAME = local.ami_id_ssm_parameter_name
}
}
@@ -121,6 +131,12 @@ resource "aws_iam_role_policy_attachment" "scale_down_vpc_execution_role" {
policy_arn = "arn:${var.aws_partition}:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
}
+resource "aws_iam_role_policy_attachment" "scale_down_ami_id_ssm_parameter_read" {
+ count = local.ami_id_ssm_parameter_name != null ? 1 : 0
+ role = aws_iam_role.scale_down.name
+ policy_arn = aws_iam_policy.ami_id_ssm_parameter_read[0].arn
+}
+
resource "aws_iam_role_policy" "scale_down_xray" {
count = var.tracing_config.mode != null ? 1 : 0
name = "xray-policy"
diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf
index cbc9beaa64..bd4874fb70 100644
--- a/modules/runners/scale-up.tf
+++ b/modules/runners/scale-up.tf
@@ -64,6 +64,15 @@ resource "aws_lambda_function" "scale_up" {
SCALE_ERRORS = jsonencode(var.scale_errors)
JOB_RETRY_CONFIG = jsonencode(local.job_retry_config)
USE_DEDICATED_HOST = var.use_dedicated_host
+ ENABLE_METRIC_WARM_POOL = var.metrics.enable && var.metrics.metric.enable_warm_pool
+ WARM_POOL_CONFIG = jsonencode({
+ enabled = var.warm_pool_config.enabled
+ maxWarmInstances = var.warm_pool_config.max_warm_instances
+ maxWarmAgeHours = var.warm_pool_config.max_warm_age_hours
+ warmPoolReadyDelaySeconds = var.warm_pool_config.warm_pool_ready_delay_seconds
+ })
+ WARM_POOL_TABLE_NAME = var.warm_pool_config.enabled ? aws_dynamodb_table.warm_pool[0].name : ""
+ POOL_STRATEGY = var.pool_strategy
}
}
diff --git a/modules/runners/tests/pool.tftest.hcl b/modules/runners/tests/pool.tftest.hcl
index 2c557f9392..c2282a7685 100644
--- a/modules/runners/tests/pool.tftest.hcl
+++ b/modules/runners/tests/pool.tftest.hcl
@@ -29,7 +29,7 @@ variables {
runner_labels = ["self-hosted", "linux", "x64"]
# Use S3 bucket to avoid filebase64sha256 needing local zip files
- lambda_s3_bucket = "my-lambda-bucket"
+ lambda_s3_bucket = "my-lambda-bucket"
runners_lambda_s3_key = "runners.zip"
github_app_parameters = {
diff --git a/modules/runners/tests/warm-pool.tftest.hcl b/modules/runners/tests/warm-pool.tftest.hcl
new file mode 100644
index 0000000000..c62fee46b3
--- /dev/null
+++ b/modules/runners/tests/warm-pool.tftest.hcl
@@ -0,0 +1,149 @@
+mock_provider "aws" {
+ mock_data "aws_iam_policy_document" {
+ defaults = {
+ json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}"
+ }
+ }
+}
+
+variables {
+ aws_region = "eu-west-1"
+ vpc_id = "vpc-12345678"
+ subnet_ids = ["subnet-12345678"]
+
+ instance_types = ["m5.large"]
+
+ s3_runner_binaries = {
+ arn = "arn:aws:s3:::my-bucket"
+ id = "my-bucket"
+ key = "runners/linux/actions-runner.tar.gz"
+ }
+
+ sqs_build_queue = {
+ arn = "arn:aws:sqs:eu-west-1:123456789012:build-queue"
+ url = "https://sqs.eu-west-1.amazonaws.com/123456789012/build-queue"
+ }
+
+ enable_organization_runners = true
+ enable_ssm_on_runners = true
+ runner_labels = ["self-hosted", "linux", "x64"]
+
+ lambda_s3_bucket = "my-lambda-bucket"
+ runners_lambda_s3_key = "runners.zip"
+
+ github_app_parameters = {
+ key_base64 = { name = "/github-runner/key-base64", arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" }
+ id = { name = "/github-runner/app-id", arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" }
+ }
+
+ ssm_paths = {
+ root = "/github-runner"
+ tokens = "tokens"
+ config = "config"
+ }
+
+ pool_config = [{
+ schedule_expression = "cron(0 8 * * ? *)"
+ size = 1
+ }]
+}
+
+run "warm_pool_disabled_by_default" {
+ command = plan
+
+ assert {
+ condition = length(aws_dynamodb_table.warm_pool) == 0
+ error_message = "DynamoDB warm pool table should not be created when warm_pool_config.enabled is false (default)"
+ }
+
+ assert {
+ condition = length(aws_iam_role_policy.scale_down_warm_pool) == 0
+ error_message = "Warm pool IAM policy should not be attached when disabled"
+ }
+}
+
+run "warm_pool_enabled_creates_dynamodb" {
+ command = plan
+
+ variables {
+ warm_pool_config = {
+ enabled = true
+ max_warm_instances = 5
+ max_warm_age_hours = 168
+ warm_pool_ready_delay_seconds = 30
+ }
+ }
+
+ assert {
+ condition = length(aws_dynamodb_table.warm_pool) == 1
+ error_message = "DynamoDB warm pool table should be created when warm_pool_config.enabled = true"
+ }
+
+ assert {
+ condition = aws_dynamodb_table.warm_pool[0].billing_mode == "PAY_PER_REQUEST"
+ error_message = "DynamoDB table should use PAY_PER_REQUEST billing"
+ }
+
+ assert {
+ condition = aws_dynamodb_table.warm_pool[0].hash_key == "instanceId"
+ error_message = "DynamoDB table hash key should be instanceId"
+ }
+}
+
+run "warm_pool_enabled_creates_iam_policies" {
+ command = plan
+
+ variables {
+ warm_pool_config = {
+ enabled = true
+ max_warm_instances = 3
+ max_warm_age_hours = 168
+ warm_pool_ready_delay_seconds = 30
+ }
+ }
+
+ assert {
+ condition = length(aws_iam_role_policy.scale_down_warm_pool) == 1
+ error_message = "Scale-down warm pool IAM policy should be created"
+ }
+
+ assert {
+ condition = length(aws_iam_role_policy.scale_up_warm_pool) == 1
+ error_message = "Scale-up warm pool IAM policy should be created"
+ }
+
+ assert {
+ condition = length(aws_iam_role_policy.pool_warm_pool) == 1
+ error_message = "Pool warm pool IAM policy should be created when pool_config is set"
+ }
+}
+
+run "warm_pool_strategy_validation" {
+ command = plan
+
+ variables {
+ pool_strategy = "warm"
+ warm_pool_config = {
+ enabled = true
+ max_warm_instances = 3
+ max_warm_age_hours = 168
+ warm_pool_ready_delay_seconds = 30
+ }
+ }
+
+ # The check block emits a warning but doesn't fail the plan,
+ # so we verify the resources are properly created for valid config
+ assert {
+ condition = length(aws_dynamodb_table.warm_pool) == 1
+ error_message = "Warm strategy with enabled config should create DynamoDB table"
+ }
+}
+
+run "pool_strategy_defaults_to_hot" {
+ command = plan
+
+ assert {
+ condition = var.pool_strategy == "hot"
+ error_message = "pool_strategy should default to 'hot'"
+ }
+}
diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf
index 01f2e2834c..18fc7a2c31 100644
--- a/modules/runners/variables.tf
+++ b/modules/runners/variables.tf
@@ -796,6 +796,7 @@ variable "metrics" {
enable_github_app_rate_limit = optional(bool, true)
enable_job_retry = optional(bool, true)
enable_spot_termination_warning = optional(bool, true)
+ enable_warm_pool = optional(bool, true)
}), {})
})
default = {}
@@ -870,3 +871,47 @@ variable "use_dedicated_host" {
type = bool
default = false
}
+
+variable "warm_pool_config" {
+ description = <<-EOF
+ Configuration for the warm pool feature. When enabled, idle runners are stopped instead of terminated,
+ allowing 10-30s restart times instead of 2-5 minute cold starts.
+
+ `enabled`: Enable or disable the warm pool feature. When enabled, a DynamoDB table is created to track stopped instances.
+ `max_warm_instances`: Maximum number of stopped instances to keep in the warm pool per runner owner.
+ `max_warm_age_hours`: Maximum age in hours before a warm instance is terminated by TTL.
+ `warm_pool_ready_delay_seconds`: Grace period in seconds before a newly launched instance is eligible for warm pool (allows time to pick up jobs).
+ EOF
+ type = object({
+ enabled = optional(bool, false)
+ max_warm_instances = optional(number, 3)
+ max_warm_age_hours = optional(number, 168)
+ warm_pool_ready_delay_seconds = optional(number, 30)
+ })
+ default = {}
+
+ validation {
+ condition = var.warm_pool_config.max_warm_instances >= 1
+ error_message = "max_warm_instances must be at least 1."
+ }
+
+ validation {
+ condition = var.warm_pool_config.max_warm_age_hours >= 1
+ error_message = "max_warm_age_hours must be at least 1 hour."
+ }
+}
+
+variable "pool_strategy" {
+ description = <<-EOF
+ Strategy for maintaining idle runners.
+ - `hot`: (default) Traditional behavior. Pool lambda creates fresh instances on a schedule. Idle instances are terminated by scale-down.
+ - `warm`: Idle instances are stopped (hibernated) instead of terminated. Scale-up will restart warm instances before creating new ones. Requires `warm_pool_config.enabled = true`.
+ EOF
+ type = string
+ default = "hot"
+
+ validation {
+ condition = contains(["hot", "warm"], var.pool_strategy)
+ error_message = "pool_strategy must be either \"hot\" or \"warm\"."
+ }
+}
diff --git a/modules/runners/warm-pool.tf b/modules/runners/warm-pool.tf
new file mode 100644
index 0000000000..faadfdd8b3
--- /dev/null
+++ b/modules/runners/warm-pool.tf
@@ -0,0 +1,73 @@
+# Warm Pool DynamoDB table and IAM policies
+# Only created when warm_pool_config.enabled = true
+
+resource "aws_dynamodb_table" "warm_pool" {
+ count = var.warm_pool_config.enabled ? 1 : 0
+
+ name = "${var.prefix}-warm-pool"
+ billing_mode = "PAY_PER_REQUEST"
+ hash_key = "instanceId"
+
+ attribute {
+ name = "instanceId"
+ type = "S"
+ }
+
+ attribute {
+ name = "runnerOwner"
+ type = "S"
+ }
+
+ attribute {
+ name = "stoppedAt"
+ type = "S"
+ }
+
+ global_secondary_index {
+ name = "by-owner"
+ hash_key = "runnerOwner"
+ range_key = "stoppedAt"
+ projection_type = "ALL"
+ }
+
+ ttl {
+ attribute_name = "expiresAt"
+ enabled = true
+ }
+
+ tags = local.tags
+}
+
+# IAM policy for warm pool operations (DynamoDB + EC2 stop/start)
+resource "aws_iam_role_policy" "scale_down_warm_pool" {
+ count = var.warm_pool_config.enabled ? 1 : 0
+
+ name = "warm-pool-policy"
+ role = aws_iam_role.scale_down.name
+ policy = templatefile("${path.module}/policies/lambda-warm-pool.json", {
+ dynamodb_table_arn = aws_dynamodb_table.warm_pool[0].arn
+ environment = var.prefix
+ })
+}
+
+resource "aws_iam_role_policy" "scale_up_warm_pool" {
+ count = var.warm_pool_config.enabled ? 1 : 0
+
+ name = "warm-pool-policy"
+ role = aws_iam_role.scale_up.name
+ policy = templatefile("${path.module}/policies/lambda-warm-pool.json", {
+ dynamodb_table_arn = aws_dynamodb_table.warm_pool[0].arn
+ environment = var.prefix
+ })
+}
+
+resource "aws_iam_role_policy" "pool_warm_pool" {
+ count = var.warm_pool_config.enabled && length(var.pool_config) > 0 ? 1 : 0
+
+ name = "warm-pool-policy"
+ role = module.pool[0].role_pool.name
+ policy = templatefile("${path.module}/policies/lambda-warm-pool.json", {
+ dynamodb_table_arn = aws_dynamodb_table.warm_pool[0].arn
+ environment = var.prefix
+ })
+}
diff --git a/modules/termination-watcher/notification/README.md b/modules/termination-watcher/notification/README.md
index 1a26de3bce..31df74586f 100644
--- a/modules/termination-watcher/notification/README.md
+++ b/modules/termination-watcher/notification/README.md
@@ -22,10 +22,14 @@
| Name | Type |
|------|------|
+| [aws_cloudwatch_event_rule.ec2_instance_state_change](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
| [aws_cloudwatch_event_rule.spot_instance_termination_warning](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
| [aws_cloudwatch_event_target.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
+| [aws_cloudwatch_event_target.state_change](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
| [aws_iam_role_policy.lambda_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
+| [aws_iam_role_policy.ssm_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_lambda_permission.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
+| [aws_lambda_permission.state_change](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
## Inputs
diff --git a/modules/termination-watcher/termination/README.md b/modules/termination-watcher/termination/README.md
index 22912b9646..32b32aa54e 100644
--- a/modules/termination-watcher/termination/README.md
+++ b/modules/termination-watcher/termination/README.md
@@ -25,6 +25,7 @@
| [aws_cloudwatch_event_rule.spot_instance_termination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
| [aws_cloudwatch_event_target.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
| [aws_iam_role_policy.lambda_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
+| [aws_iam_role_policy.ssm_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
| [aws_lambda_permission.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
## Inputs
diff --git a/variables.tf b/variables.tf
index b7081761a5..3bb9ecc55f 100644
--- a/variables.tf
+++ b/variables.tf
@@ -847,6 +847,28 @@ variable "pool_config" {
default = []
}
+variable "pool_strategy" {
+ description = "Strategy for maintaining idle runners: `hot` (default, traditional) or `warm` (stop instead of terminate). Requires `warm_pool_config.enabled = true` when set to `warm`."
+ type = string
+ default = "hot"
+
+ validation {
+ condition = contains(["hot", "warm"], var.pool_strategy)
+ error_message = "pool_strategy must be either \"hot\" or \"warm\"."
+ }
+}
+
+variable "warm_pool_config" {
+ description = "Configuration for the warm pool feature. When enabled, idle runners are stopped instead of terminated, allowing 10-30s restart times instead of 2-5 minute cold starts."
+ type = object({
+ enabled = optional(bool, false)
+ max_warm_instances = optional(number, 3)
+ max_warm_age_hours = optional(number, 168)
+ warm_pool_ready_delay_seconds = optional(number, 30)
+ })
+ default = {}
+}
+
variable "aws_partition" {
description = "(optiona) partition in the arn namespace to use if not 'aws'"
type = string
@@ -1047,6 +1069,7 @@ variable "metrics" {
enable_github_app_rate_limit = optional(bool, true)
enable_job_retry = optional(bool, true)
enable_spot_termination_warning = optional(bool, true)
+ enable_warm_pool = optional(bool, true)
}), {})
})
default = {}