refactor(runners): split runner provider layout

docs/configuration.md

@@ -336,7 +336,7 @@ Below is an example of the log messages created.
 
 [!WARNING]
 **Security implication:** Dynamic labels are extracted from the `runs-on` labels in incoming `workflow_job` webhook events. These labels originate from what
-users define in their workflow files. Any user with permission to create or modify workflows can inject arbitrary EC2 configuration values — including instance types, AMI IDs, subnet IDs, EBS volumes, placement settings, and more. Unless constrained with `ec2_dynamic_labels_policy`, these values are not validated against label-specific rules before being passed to the EC2 CreateFleet API. This means a malicious or careless workflow author could, for example:
+users define in their workflow files. Any user with permission to create or modify workflows can inject arbitrary EC2 configuration values — including instance types, AMI IDs, subnet IDs, EBS volumes, placement settings, and more. Unless constrained with `aws_dynamic_labels_policy` in the root module, or `matcherConfig.awsDynamicLabelsPolicy` when configuring matcher entries directly, these values are not validated against label-specific rules before being passed to the EC2 CreateFleet API. This means a malicious or careless workflow author could, for example:
 
 - Launch expensive instance types (e.g., `p5.48xlarge`) to inflate costs
 - Override the AMI (`ghr-ec2-image-id`) to boot a compromised image
@@ -367,7 +367,7 @@ module "runners" {
 
   ...
   enable_dynamic_labels = true
-  ec2_dynamic_labels_policy = {
+  aws_dynamic_labels_policy = {
     blocked_keys = ["image-id", "subnet-id"]
 
     restricted_keys = {
@@ -385,6 +385,8 @@ module "runners" {
 }
 ```
 
+The root module variable is named `aws_dynamic_labels_policy`. The webhook matcher config receives the same policy under `matcherConfig.awsDynamicLabelsPolicy`. If you configure `runner_matcher_config` or `multi_runner_config.matcherConfig` directly, use `awsDynamicLabelsPolicy` for this policy.
+
 The policy is evaluated by dynamic label key:
 
 1. Keys in `blocked_keys` are always rejected.

lambdas/functions/control-plane/package.json

@@ -32,6 +32,7 @@
   "dependencies": {
     "@aws-github-runner/aws-powertools-util": "*",
     "@aws-github-runner/aws-ssm-util": "*",
+    "@aws-github-runner/runner-provider": "*",
     "@aws-lambda-powertools/parameters": "^2.31.0",
     "@aws-sdk/client-ec2": "^3.1009.0",
     "@aws-sdk/client-sqs": "^3.1009.0",

lambdas/functions/control-plane/src/aws/runners.d.ts → lambdas/functions/control-plane/src/aws/ec2-runners.d.ts

@@ -7,7 +7,7 @@ import {
   Placement,
   FleetBlockDeviceMappingRequest,
 } from '@aws-sdk/client-ec2';
-import { LambdaRunnerSource } from '../scale-runners/scale-up';
+import type { LambdaRunnerSource } from '../scale-runners/types';
 
 export type RunnerType = 'Org' | 'Repo';
 

lambdas/functions/control-plane/src/aws/runners.ts → lambdas/functions/control-plane/src/aws/ec2-runners.ts

@@ -23,7 +23,7 @@ import { getParameter } from '@aws-github-runner/aws-ssm-util';
 import moment from 'moment';
 
 import ScaleError from './../scale-runners/ScaleError';
-import * as Runners from './runners.d';
+import * as Runners from './ec2-runners.d';
 
 const logger = createChildLogger('runners');
 

lambdas/functions/control-plane/src/aws/runners.test.ts

@@ -22,9 +22,9 @@ import 'aws-sdk-client-mock-jest/vitest';
 
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 import ScaleError from './../scale-runners/ScaleError';
-import { createRunner, listEC2Runners, tag, terminateRunner, untag } from './runners';
-import type { Ec2OverrideConfig, RunnerInfo, RunnerInputParameters, RunnerType } from './runners.d';
-import { LambdaRunnerSource } from '../scale-runners/scale-up';
+import { createRunner, listEC2Runners, tag, terminateRunner, untag } from './ec2-runners';
+import type { Ec2OverrideConfig, RunnerInfo, RunnerInputParameters, RunnerType } from './ec2-runners.d';
+import type { LambdaRunnerSource } from '../scale-runners/types';
 
 process.env.AWS_REGION = 'eu-east-1';
 const mockEC2Client = mockClient(EC2Client);

lambdas/functions/control-plane/src/github/octokit.test.ts

@@ -1,5 +1,5 @@
 import { Octokit } from '@octokit/rest';
-import { ActionRequestMessage } from '../scale-runners/scale-up';
+import type { ActionRequestMessage } from '../scale-runners/types';
 import { getOctokit } from './octokit';
 import { describe, it, expect, beforeEach, vi } from 'vitest';
 import { createGithubAppAuth, createGithubInstallationAuth } from '../github/auth';

lambdas/functions/control-plane/src/github/octokit.ts

@@ -1,5 +1,5 @@
 import { Octokit } from '@octokit/rest';
-import { ActionRequestMessage } from '../scale-runners/scale-up';
+import type { ActionRequestMessage } from '../scale-runners/types';
 import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from './auth';
 
 function getErrorStatus(error: unknown): number | undefined {

lambdas/functions/control-plane/src/lambda.test.ts

@@ -5,7 +5,8 @@ import { addMiddleware, adjustPool, scaleDownHandler, scaleUpHandler, ssmHouseke
 import { adjust } from './pool/pool';
 import ScaleError from './scale-runners/ScaleError';
 import { scaleDown } from './scale-runners/scale-down';
-import { ActionRequestMessage, scaleUp } from './scale-runners/scale-up';
+import { scaleUp } from './scale-runners/scale-up';
+import type { ActionRequestMessage } from './scale-runners/types';
 import { cleanSSMTokens } from './scale-runners/ssm-housekeeper';
 import { checkAndRetryJob } from './scale-runners/job-retry';
 import { describe, it, expect, vi, MockedFunction, beforeEach } from 'vitest';
@@ -248,14 +249,14 @@ describe('Test scale down lambda wrapper.', () => {
 describe('Adjust pool.', () => {
   it('Receive message to adjust pool.', async () => {
     vi.mocked(adjust).mockResolvedValue();
-    await expect(adjustPool({ poolSize: 2 }, context)).resolves.not.toThrow();
+    await expect(adjustPool({ poolSize: 2, type: 'ec2' }, context)).resolves.not.toThrow();
   });
 
   it('Handle error for adjusting pool.', async () => {
     const error = new Error('Handle error for adjusting pool.');
     vi.mocked(adjust).mockRejectedValue(error);
     const logSpy = vi.spyOn(logger, 'error');
-    await adjustPool({ poolSize: 0 }, context);
+    await adjustPool({ poolSize: 0, type: 'ec2' }, context);
     expect(logSpy).toHaveBeenCalledWith(`Handle error for adjusting pool. ${error.message}`, { error });
   });
 });

lambdas/functions/control-plane/src/lambda.ts

@@ -6,7 +6,8 @@ import { Context, type SQSBatchItemFailure, type SQSBatchResponse, SQSEvent } fr
 import { PoolEvent, adjust } from './pool/pool';
 import ScaleError from './scale-runners/ScaleError';
 import { scaleDown } from './scale-runners/scale-down';
-import { type ActionRequestMessage, type ActionRequestMessageSQS, scaleUp } from './scale-runners/scale-up';
+import { scaleUp } from './scale-runners/scale-up';
+import type { ActionRequestMessage, ActionRequestMessageSQS } from './scale-runners/types';
 import { SSMCleanupOptions, cleanSSMTokens } from './scale-runners/ssm-housekeeper';
 import { checkAndRetryJob } from './scale-runners/job-retry';
 

lambdas/functions/control-plane/src/local-pool.ts

@@ -1,7 +1,7 @@
 import { adjust } from './pool/pool';
 
 export function run(): void {
-  adjust({ poolSize: 1 })
+  adjust({ poolSize: 1, type: 'ec2' })
     .then()
     .catch((e) => {
       console.log(e);

lambdas/functions/control-plane/src/modules.d.ts

@@ -16,6 +16,7 @@ declare namespace NodeJS {
     PARAMETER_GITHUB_APP_ID_NAME: string;
     PARAMETER_GITHUB_APP_KEY_BASE64_NAME: string;
     RUNNER_OWNER: string;
+    RUNNER_PROVIDER_TYPE?: string;
     SCALE_DOWN_CONFIG: string;
     SSM_TOKEN_PATH: string;
     SSM_CLEANUP_CONFIG: string;

lambdas/functions/control-plane/src/pool/ec2-pool.ts

@@ -0,0 +1,118 @@
+import { createChildLogger } from '@aws-github-runner/aws-powertools-util';
+import yn from 'yn';
+
+import { bootTimeExceeded, listEC2Runners } from '../aws/ec2-runners';
+import type { RunnerList } from '../aws/ec2-runners.d';
+import { createRunners } from '../scale-runners/ec2';
+import type { CreateEC2RunnerConfig } from '../scale-runners/ec2';
+import type { CreatePoolRunnersInput, ListPoolRunnersInput, PoolRunnerProvider, RunnerStatus } from './pool-provider';
+
+const logger = createChildLogger('pool');
+
+interface Ec2PoolProviderConfig {
+  environment: string;
+  subnets: string[];
+  launchTemplateName: string;
+  ec2instanceCriteria: CreateEC2RunnerConfig['ec2instanceCriteria'];
+  amiIdSsmParameterName?: string;
+  tracingEnabled?: boolean;
+  onDemandFailoverOnError: string[];
+  scaleErrors: string[];
+}
+
+function loadEc2PoolProviderConfig(): Ec2PoolProviderConfig {
+  return {
+    environment: process.env.ENVIRONMENT,
+    subnets: process.env.SUBNET_IDS.split(','),
+    launchTemplateName: process.env.LAUNCH_TEMPLATE_NAME,
+    ec2instanceCriteria: {
+      instanceTypes: process.env.INSTANCE_TYPES.split(','),
+      instanceTypePriorities: process.env.INSTANCE_TYPE_PRIORITIES
+        ? (JSON.parse(process.env.INSTANCE_TYPE_PRIORITIES) as Record<string, number>)
+        : undefined,
+      targetCapacityType: process.env.INSTANCE_TARGET_CAPACITY_TYPE,
+      maxSpotPrice: process.env.INSTANCE_MAX_SPOT_PRICE,
+      instanceAllocationStrategy: process.env.INSTANCE_ALLOCATION_STRATEGY || 'lowest-price',
+    },
+    amiIdSsmParameterName: process.env.AMI_ID_SSM_PARAMETER_NAME,
+    tracingEnabled: yn(process.env.POWERTOOLS_TRACE_ENABLED, { default: false }),
+    onDemandFailoverOnError: process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS
+      ? (JSON.parse(process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS) as [string])
+      : [],
+    scaleErrors: JSON.parse(process.env.SCALE_ERRORS) as [string],
+  };
+}
+
+async function listEc2PoolRunners({
+  environment,
+  runnerOwner,
+  runnerType,
+}: ListPoolRunnersInput): Promise<RunnerList[]> {
+  return await listEC2Runners({
+    environment,
+    runnerOwner,
+    runnerType,
+    statuses: ['running'],
+  });
+}
+
+async function createEc2PoolRunners({
+  githubRunnerConfig,
+  numberOfRunners,
+  githubInstallationClient,
+}: CreatePoolRunnersInput): Promise<string[]> {
+  const config = loadEc2PoolProviderConfig();
+
+  return await createRunners(
+    githubRunnerConfig,
+    {
+      ec2instanceCriteria: config.ec2instanceCriteria,
+      environment: config.environment,
+      launchTemplateName: config.launchTemplateName,
+      subnets: config.subnets,
+      amiIdSsmParameterName: config.amiIdSsmParameterName,
+      tracingEnabled: config.tracingEnabled,
+      onDemandFailoverOnError: config.onDemandFailoverOnError,
+      scaleErrors: config.scaleErrors,
+    },
+    numberOfRunners,
+    githubInstallationClient,
+    'pool-lambda',
+  );
+}
+
+export function createEc2PoolProvider(): Omit<PoolRunnerProvider, 'type'> {
+  return {
+    listRunners: listEc2PoolRunners,
+    countAvailableRunners: calculateEc2PoolSize,
+    createRunners: createEc2PoolRunners,
+  };
+}
+
+export function calculateEc2PoolSize(
+  ec2runners: RunnerList[],
+  runnerStatus: Map<string, RunnerStatus>,
+  includeBusyRunners = false,
+): number {
+  // Runner should be considered idle if it is still booting, or is idle in GitHub
+  let numberOfRunnersInPool = 0;
+  for (const ec2Instance of ec2runners) {
+    if (
+      (runnerStatus.get(ec2Instance.instanceId)?.busy === false || includeBusyRunners) &&
+      runnerStatus.get(ec2Instance.instanceId)?.status === 'online'
+    ) {
+      numberOfRunnersInPool++;
+      logger.debug(`Runner ${ec2Instance.instanceId} is idle in GitHub and counted as part of the pool`);
+    } else if (runnerStatus.get(ec2Instance.instanceId) != null) {
+      logger.debug(`Runner ${ec2Instance.instanceId} is not idle in GitHub and NOT counted as part of the pool`);
+    } else if (!bootTimeExceeded(ec2Instance)) {
+      numberOfRunnersInPool++;
+      logger.info(`Runner ${ec2Instance.instanceId} is still booting and counted as part of the pool`);
+    } else {
+      logger.debug(
+        `Runner ${ec2Instance.instanceId} is not idle in GitHub nor booting and not counted as part of the pool`,
+      );
+    }
+  }
+  return numberOfRunnersInPool;
+}

lambdas/functions/control-plane/src/pool/pool-provider.ts

@@ -0,0 +1,31 @@
+import type { Octokit } from '@octokit/rest';
+import type { RunnerProvider } from '@aws-github-runner/runner-provider';
+
+import type { CreateGitHubRunnerConfig, GitHubRunnerType } from '../scale-runners/types';
+
+export interface RunnerStatus {
+  busy: boolean;
+  status: string;
+}
+
+export interface ListPoolRunnersInput {
+  environment: string;
+  runnerOwner: string;
+  runnerType: GitHubRunnerType;
+}
+
+export interface CreatePoolRunnersInput {
+  githubRunnerConfig: CreateGitHubRunnerConfig;
+  numberOfRunners: number;
+  githubInstallationClient: Octokit;
+}
+
+export interface PoolRunnerProvider<TRunner = unknown> extends RunnerProvider {
+  listRunners(input: ListPoolRunnersInput): Promise<TRunner[]>;
+  countAvailableRunners(
+    runners: TRunner[],
+    runnerStatus: Map<string, RunnerStatus>,
+    includeBusyRunners: boolean,
+  ): number;
+  createRunners(input: CreatePoolRunnersInput): Promise<string[]>;
+}