From 4964069282d45104bff201571bf30bed6263d613 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 7 Jul 2026 22:04:31 +0200 Subject: [PATCH 01/46] refactor(aws): split ec2 runner module --- .../src/aws/{runners.d.ts => ec2-runners.d.ts} | 2 +- .../src/aws/{runners.test.ts => ec2-runners.test.ts} | 6 +++--- .../control-plane/src/aws/{runners.ts => ec2-runners.ts} | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename lambdas/functions/control-plane/src/aws/{runners.d.ts => ec2-runners.d.ts} (96%) rename lambdas/functions/control-plane/src/aws/{runners.test.ts => ec2-runners.test.ts} (99%) rename lambdas/functions/control-plane/src/aws/{runners.ts => ec2-runners.ts} (99%) diff --git a/lambdas/functions/control-plane/src/aws/runners.d.ts b/lambdas/functions/control-plane/src/aws/ec2-runners.d.ts similarity index 96% rename from lambdas/functions/control-plane/src/aws/runners.d.ts rename to lambdas/functions/control-plane/src/aws/ec2-runners.d.ts index 770106a98d..4df7d82fb5 100644 --- a/lambdas/functions/control-plane/src/aws/runners.d.ts +++ b/lambdas/functions/control-plane/src/aws/ec2-runners.d.ts @@ -7,7 +7,7 @@ import { Placement, FleetBlockDeviceMappingRequest, } from '@aws-sdk/client-ec2'; -import { LambdaRunnerSource } from '../scale-runners/scale-up'; +import type { LambdaRunnerSource } from '../scale-runners/types'; export type RunnerType = 'Org' | 'Repo'; diff --git a/lambdas/functions/control-plane/src/aws/runners.test.ts b/lambdas/functions/control-plane/src/aws/ec2-runners.test.ts similarity index 99% rename from lambdas/functions/control-plane/src/aws/runners.test.ts rename to lambdas/functions/control-plane/src/aws/ec2-runners.test.ts index 03d5b386e2..e95931033c 100644 --- a/lambdas/functions/control-plane/src/aws/runners.test.ts +++ b/lambdas/functions/control-plane/src/aws/ec2-runners.test.ts @@ -22,9 +22,9 @@ import 'aws-sdk-client-mock-jest/vitest'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import ScaleError from './../scale-runners/ScaleError'; -import { createRunner, listEC2Runners, tag, terminateRunner, untag } from './runners'; -import type { Ec2OverrideConfig, RunnerInfo, RunnerInputParameters, RunnerType } from './runners.d'; -import { LambdaRunnerSource } from '../scale-runners/scale-up'; +import { createRunner, listEC2Runners, tag, terminateRunner, untag } from './ec2-runners'; +import type { Ec2OverrideConfig, RunnerInfo, RunnerInputParameters, RunnerType } from './ec2-runners.d'; +import type { LambdaRunnerSource } from '../scale-runners/types'; process.env.AWS_REGION = 'eu-east-1'; const mockEC2Client = mockClient(EC2Client); diff --git a/lambdas/functions/control-plane/src/aws/runners.ts b/lambdas/functions/control-plane/src/aws/ec2-runners.ts similarity index 99% rename from lambdas/functions/control-plane/src/aws/runners.ts rename to lambdas/functions/control-plane/src/aws/ec2-runners.ts index 799bc0d4a2..8e83c93b33 100644 --- a/lambdas/functions/control-plane/src/aws/runners.ts +++ b/lambdas/functions/control-plane/src/aws/ec2-runners.ts @@ -23,7 +23,7 @@ import { getParameter } from '@aws-github-runner/aws-ssm-util'; import moment from 'moment'; import ScaleError from './../scale-runners/ScaleError'; -import * as Runners from './runners.d'; +import * as Runners from './ec2-runners.d'; const logger = createChildLogger('runners'); From b2f96f3069bd57892e3168d472e17878fcb686b4 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 7 Jul 2026 22:05:15 +0200 Subject: [PATCH 02/46] refactor(scale-runners): split ec2 scale-up provider --- .../control-plane/src/github/octokit.test.ts | 2 +- .../control-plane/src/github/octokit.ts | 2 +- .../control-plane/src/lambda.test.ts | 7 +- lambdas/functions/control-plane/src/lambda.ts | 3 +- .../functions/control-plane/src/modules.d.ts | 1 + .../src/scale-runners/ScaleError.test.ts | 2 +- .../src/scale-runners/ScaleError.ts | 2 +- .../src/scale-runners/ec2-labels.test.ts | 708 ++++ .../src/scale-runners/ec2-labels.ts | 388 ++ .../src/scale-runners/ec2-scale-up.test.ts | 2769 +++++++++++++ .../src/scale-runners/ec2-scale-up.ts | 94 + .../control-plane/src/scale-runners/ec2.ts | 129 + .../src/scale-runners/github-runner.ts | 371 ++ .../src/scale-runners/job-retry.test.ts | 2 +- .../src/scale-runners/job-retry.ts | 3 +- .../src/scale-runners/scale-up-config.test.ts | 13 + .../src/scale-runners/scale-up-config.ts | 8 + .../scale-up-provider-registry.ts | 26 + .../src/scale-runners/scale-up-provider.ts | 40 + .../src/scale-runners/scale-up.test.ts | 3575 +---------------- .../src/scale-runners/scale-up.ts | 1036 +---- .../control-plane/src/scale-runners/types.ts | 47 + 22 files changed, 4806 insertions(+), 4422 deletions(-) create mode 100644 lambdas/functions/control-plane/src/scale-runners/ec2-labels.test.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/ec2-labels.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.test.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/ec2.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/github-runner.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/scale-up-config.test.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/types.ts diff --git a/lambdas/functions/control-plane/src/github/octokit.test.ts b/lambdas/functions/control-plane/src/github/octokit.test.ts index c104b69641..e6bc0fd2e3 100644 --- a/lambdas/functions/control-plane/src/github/octokit.test.ts +++ b/lambdas/functions/control-plane/src/github/octokit.test.ts @@ -1,5 +1,5 @@ import { Octokit } from '@octokit/rest'; -import { ActionRequestMessage } from '../scale-runners/scale-up'; +import type { ActionRequestMessage } from '../scale-runners/types'; import { getOctokit } from './octokit'; import { describe, it, expect, beforeEach, vi } from 'vitest'; import { createGithubAppAuth, createGithubInstallationAuth } from '../github/auth'; diff --git a/lambdas/functions/control-plane/src/github/octokit.ts b/lambdas/functions/control-plane/src/github/octokit.ts index 588ec62150..28181e6acc 100644 --- a/lambdas/functions/control-plane/src/github/octokit.ts +++ b/lambdas/functions/control-plane/src/github/octokit.ts @@ -1,5 +1,5 @@ import { Octokit } from '@octokit/rest'; -import { ActionRequestMessage } from '../scale-runners/scale-up'; +import type { ActionRequestMessage } from '../scale-runners/types'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from './auth'; function getErrorStatus(error: unknown): number | undefined { diff --git a/lambdas/functions/control-plane/src/lambda.test.ts b/lambdas/functions/control-plane/src/lambda.test.ts index 2c9a98e420..5c79d78342 100644 --- a/lambdas/functions/control-plane/src/lambda.test.ts +++ b/lambdas/functions/control-plane/src/lambda.test.ts @@ -5,7 +5,8 @@ import { addMiddleware, adjustPool, scaleDownHandler, scaleUpHandler, ssmHouseke import { adjust } from './pool/pool'; import ScaleError from './scale-runners/ScaleError'; import { scaleDown } from './scale-runners/scale-down'; -import { ActionRequestMessage, scaleUp } from './scale-runners/scale-up'; +import { scaleUp } from './scale-runners/scale-up'; +import type { ActionRequestMessage } from './scale-runners/types'; import { cleanSSMTokens } from './scale-runners/ssm-housekeeper'; import { checkAndRetryJob } from './scale-runners/job-retry'; import { describe, it, expect, vi, MockedFunction, beforeEach } from 'vitest'; @@ -248,14 +249,14 @@ describe('Test scale down lambda wrapper.', () => { describe('Adjust pool.', () => { it('Receive message to adjust pool.', async () => { vi.mocked(adjust).mockResolvedValue(); - await expect(adjustPool({ poolSize: 2 }, context)).resolves.not.toThrow(); + await expect(adjustPool({ poolSize: 2, type: 'ec2' }, context)).resolves.not.toThrow(); }); it('Handle error for adjusting pool.', async () => { const error = new Error('Handle error for adjusting pool.'); vi.mocked(adjust).mockRejectedValue(error); const logSpy = vi.spyOn(logger, 'error'); - await adjustPool({ poolSize: 0 }, context); + await adjustPool({ poolSize: 0, type: 'ec2' }, context); expect(logSpy).toHaveBeenCalledWith(`Handle error for adjusting pool. ${error.message}`, { error }); }); }); diff --git a/lambdas/functions/control-plane/src/lambda.ts b/lambdas/functions/control-plane/src/lambda.ts index e2a0451c95..9a67ccd0f8 100644 --- a/lambdas/functions/control-plane/src/lambda.ts +++ b/lambdas/functions/control-plane/src/lambda.ts @@ -6,7 +6,8 @@ import { Context, type SQSBatchItemFailure, type SQSBatchResponse, SQSEvent } fr import { PoolEvent, adjust } from './pool/pool'; import ScaleError from './scale-runners/ScaleError'; import { scaleDown } from './scale-runners/scale-down'; -import { type ActionRequestMessage, type ActionRequestMessageSQS, scaleUp } from './scale-runners/scale-up'; +import { scaleUp } from './scale-runners/scale-up'; +import type { ActionRequestMessage, ActionRequestMessageSQS } from './scale-runners/types'; import { SSMCleanupOptions, cleanSSMTokens } from './scale-runners/ssm-housekeeper'; import { checkAndRetryJob } from './scale-runners/job-retry'; diff --git a/lambdas/functions/control-plane/src/modules.d.ts b/lambdas/functions/control-plane/src/modules.d.ts index 0ec63317db..53247cf6c6 100644 --- a/lambdas/functions/control-plane/src/modules.d.ts +++ b/lambdas/functions/control-plane/src/modules.d.ts @@ -16,6 +16,7 @@ declare namespace NodeJS { PARAMETER_GITHUB_APP_ID_NAME: string; PARAMETER_GITHUB_APP_KEY_BASE64_NAME: string; RUNNER_OWNER: string; + RUNNER_PROVIDER_TYPE?: string; SCALE_DOWN_CONFIG: string; SSM_TOKEN_PATH: string; SSM_CLEANUP_CONFIG: string; diff --git a/lambdas/functions/control-plane/src/scale-runners/ScaleError.test.ts b/lambdas/functions/control-plane/src/scale-runners/ScaleError.test.ts index 8490a80447..ca19c1722a 100644 --- a/lambdas/functions/control-plane/src/scale-runners/ScaleError.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/ScaleError.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import type { ActionRequestMessageSQS } from './scale-up'; +import type { ActionRequestMessageSQS } from './types'; import ScaleError from './ScaleError'; describe('ScaleError', () => { diff --git a/lambdas/functions/control-plane/src/scale-runners/ScaleError.ts b/lambdas/functions/control-plane/src/scale-runners/ScaleError.ts index 9c1f474d17..226f635eb5 100644 --- a/lambdas/functions/control-plane/src/scale-runners/ScaleError.ts +++ b/lambdas/functions/control-plane/src/scale-runners/ScaleError.ts @@ -1,5 +1,5 @@ import type { SQSBatchItemFailure } from 'aws-lambda'; -import type { ActionRequestMessageSQS } from './scale-up'; +import type { ActionRequestMessageSQS } from './types'; class ScaleError extends Error { constructor(public readonly failedInstanceCount: number = 1) { diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-labels.test.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-labels.test.ts new file mode 100644 index 0000000000..c2203d2c67 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-labels.test.ts @@ -0,0 +1,708 @@ +import { describe, expect, it } from 'vitest'; + +import { parseEc2OverrideConfig } from './ec2-labels'; + +describe('parseEc2OverrideConfig', () => { + describe('Basic Fleet Overrides', () => { + it('should parse instance-type label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-instance-type:c5.xlarge']); + expect(result?.InstanceType).toBe('c5.xlarge'); + }); + + it('should parse subnet-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-subnet-id:subnet-123456']); + expect(result?.SubnetId).toBe('subnet-123456'); + }); + + it('should parse availability-zone label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-availability-zone:us-east-1a']); + expect(result?.AvailabilityZone).toBe('us-east-1a'); + }); + + it('should parse availability-zone-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-availability-zone-id:use1-az1']); + expect(result?.AvailabilityZoneId).toBe('use1-az1'); + }); + + it('should parse max-price label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-max-price:0.50']); + expect(result?.MaxPrice).toBe('0.50'); + }); + + it('should parse priority label as number', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-priority:1']); + expect(result?.Priority).toBe(1); + }); + + it('should parse weighted-capacity label as number', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-weighted-capacity:2']); + expect(result?.WeightedCapacity).toBe(2); + }); + + it('should parse image-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-image-id:ami-12345678']); + expect(result?.ImageId).toBe('ami-12345678'); + }); + + it('should parse multiple basic fleet overrides', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-instance-type:r5.2xlarge', + 'ghr-ec2-max-price:1.00', + 'ghr-ec2-priority:2', + ]); + expect(result?.InstanceType).toBe('r5.2xlarge'); + expect(result?.MaxPrice).toBe('1.00'); + expect(result?.Priority).toBe(2); + }); + }); + + describe('Placement', () => { + it('should parse placement-group-name label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-group-name:my-placement-group']); + expect(result?.Placement?.GroupName).toBe('my-placement-group'); + }); + + it('should parse placement-group-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-group-id:pg-1234567890abcdef0']); + expect(result?.Placement?.GroupId).toBe('pg-1234567890abcdef0'); + }); + + it('should parse placement-tenancy label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-tenancy:dedicated']); + expect(result?.Placement?.Tenancy).toBe('dedicated'); + }); + + it('should parse placement-host-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-host-id:h-1234567890abcdef']); + expect(result?.Placement?.HostId).toBe('h-1234567890abcdef'); + }); + + it('should parse placement-affinity label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-affinity:host']); + expect(result?.Placement?.Affinity).toBe('host'); + }); + + it('should parse placement-partition-number label as number', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-partition-number:3']); + expect(result?.Placement?.PartitionNumber).toBe(3); + }); + + it('should parse placement-availability-zone label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-availability-zone:us-west-2b']); + expect(result?.Placement?.AvailabilityZone).toBe('us-west-2b'); + }); + + it('should parse placement-availability-zone-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-availability-zone-id:use1-az1']); + expect(result?.Placement?.AvailabilityZoneId).toBe('use1-az1'); + }); + + it('should parse placement-spread-domain label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-spread-domain:my-spread-domain']); + expect(result?.Placement?.SpreadDomain).toBe('my-spread-domain'); + }); + + it('should parse placement-host-resource-group-arn label', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-placement-host-resource-group-arn:arn:aws:ec2:us-east-1:123456789012:host-resource-group/hrg-1234', + ]); + expect(result?.Placement?.HostResourceGroupArn).toBe( + 'arn:aws:ec2:us-east-1:123456789012:host-resource-group/hrg-1234', + ); + }); + + it('should parse multiple placement labels', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-placement-group-name:group-1', + 'ghr-ec2-placement-tenancy:dedicated', + 'ghr-ec2-placement-availability-zone:us-east-1b', + ]); + expect(result?.Placement?.GroupName).toBe('group-1'); + expect(result?.Placement?.Tenancy).toBe('dedicated'); + expect(result?.Placement?.AvailabilityZone).toBe('us-east-1b'); + }); + }); + + describe('Block Device Mappings', () => { + it('should parse block-device-name label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-block-device-name:/dev/sdg']); + expect(result?.BlockDeviceMappings?.[0]?.DeviceName).toBe('/dev/sdg'); + }); + + it('should use default block device name when provided', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-volume-size:100'], '/dev/sda1'); + expect(result?.BlockDeviceMappings?.[0]?.DeviceName).toBe('/dev/sda1'); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(100); + }); + + it('should parse ebs-volume-size label as number', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-volume-size:100']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(100); + }); + + it('should parse ebs-volume-type label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-volume-type:gp3']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeType).toBe('gp3'); + }); + + it('should parse ebs-iops label as number', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-iops:3000']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Iops).toBe(3000); + }); + + it('should parse ebs-throughput label as number', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-throughput:250']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Throughput).toBe(250); + }); + + it('should parse ebs-encrypted label as boolean true', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-encrypted:true']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); + }); + + it('should parse ebs-encrypted label as boolean false', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-encrypted:false']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(false); + }); + + it('should parse ebs-kms-key-id label', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-ebs-kms-key-id:arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012', + ]); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.KmsKeyId).toBe( + 'arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012', + ); + }); + + it('should parse ebs-delete-on-termination label as boolean true', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-delete-on-termination:true']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.DeleteOnTermination).toBe(true); + }); + + it('should parse ebs-delete-on-termination label as boolean false', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-delete-on-termination:false']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.DeleteOnTermination).toBe(false); + }); + + it('should parse ebs-snapshot-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-snapshot-id:snap-1234567890abcdef']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.SnapshotId).toBe('snap-1234567890abcdef'); + }); + + it('should parse block-device-virtual-name label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-block-device-virtual-name:ephemeral0']); + expect(result?.BlockDeviceMappings?.[0]?.VirtualName).toBe('ephemeral0'); + }); + + it('should parse block-device-no-device label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-block-device-no-device:true']); + expect(result?.BlockDeviceMappings?.[0]?.NoDevice).toBe('true'); + }); + + it('should parse multiple block device mapping labels', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-ebs-volume-size:200', + 'ghr-ec2-ebs-volume-type:gp3', + 'ghr-ec2-ebs-iops:5000', + 'ghr-ec2-ebs-encrypted:true', + ]); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(200); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeType).toBe('gp3'); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Iops).toBe(5000); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); + }); + + it('should initialize BlockDeviceMappings when not present', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-volume-size:50']); + expect(result?.BlockDeviceMappings).toBeDefined(); + }); + }); + + describe('Instance Requirements - vCPU and Memory', () => { + it('should parse vcpu-count-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-vcpu-count-min:4']); + expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(4); + }); + + it('should parse vcpu-count-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-vcpu-count-max:16']); + expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(16); + }); + + it('should parse both vcpu-count-min and vcpu-count-max labels', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-vcpu-count-min:2', 'ghr-ec2-vcpu-count-max:8']); + expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(2); + expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(8); + }); + + it('should parse memory-mib-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-memory-mib-min:8192']); + expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(8192); + }); + + it('should parse memory-mib-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-memory-mib-max:32768']); + expect(result?.InstanceRequirements?.MemoryMiB?.Max).toBe(32768); + }); + + it('should parse both memory-mib-min and memory-mib-max labels', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-memory-mib-min:16384', 'ghr-ec2-memory-mib-max:65536']); + expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(16384); + expect(result?.InstanceRequirements?.MemoryMiB?.Max).toBe(65536); + }); + + it('should parse memory-gib-per-vcpu-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-memory-gib-per-vcpu-min:2']); + expect(result?.InstanceRequirements?.MemoryGiBPerVCpu?.Min).toBe(2); + }); + + it('should parse memory-gib-per-vcpu-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-memory-gib-per-vcpu-max:8']); + expect(result?.InstanceRequirements?.MemoryGiBPerVCpu?.Max).toBe(8); + }); + + it('should parse combined vCPU and memory requirements', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-vcpu-count-min:8', + 'ghr-ec2-vcpu-count-max:32', + 'ghr-ec2-memory-mib-min:32768', + 'ghr-ec2-memory-mib-max:131072', + ]); + expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(8); + expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(32); + expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(32768); + expect(result?.InstanceRequirements?.MemoryMiB?.Max).toBe(131072); + }); + }); + + describe('Instance Requirements - CPU and Performance', () => { + it('should parse cpu-manufacturers as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-cpu-manufacturers:intel']); + expect(result?.InstanceRequirements?.CpuManufacturers).toEqual(['intel']); + }); + + it('should parse cpu-manufacturers as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-cpu-manufacturers:intel;amd']); + expect(result?.InstanceRequirements?.CpuManufacturers).toEqual(['intel', 'amd']); + }); + + it('should parse instance-generations as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-instance-generations:current']); + expect(result?.InstanceRequirements?.InstanceGenerations).toEqual(['current']); + }); + + it('should parse instance-generations as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-instance-generations:current;previous']); + expect(result?.InstanceRequirements?.InstanceGenerations).toEqual(['current', 'previous']); + }); + + it('should parse excluded-instance-types as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-excluded-instance-types:t2.micro']); + expect(result?.InstanceRequirements?.ExcludedInstanceTypes).toEqual(['t2.micro']); + }); + + it('should parse excluded-instance-types as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-excluded-instance-types:t2.micro;t2.small']); + expect(result?.InstanceRequirements?.ExcludedInstanceTypes).toEqual(['t2.micro', 't2.small']); + }); + + it('should parse allowed-instance-types as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-allowed-instance-types:c5.xlarge']); + expect(result?.InstanceRequirements?.AllowedInstanceTypes).toEqual(['c5.xlarge']); + }); + + it('should parse allowed-instance-types as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-allowed-instance-types:c5.xlarge;c5.2xlarge']); + expect(result?.InstanceRequirements?.AllowedInstanceTypes).toEqual(['c5.xlarge', 'c5.2xlarge']); + }); + + it('should parse burstable-performance label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-burstable-performance:included']); + expect(result?.InstanceRequirements?.BurstablePerformance).toBe('included'); + }); + + it('should parse bare-metal label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-bare-metal:excluded']); + expect(result?.InstanceRequirements?.BareMetal).toBe('excluded'); + }); + }); + + describe('Instance Requirements - Accelerators', () => { + it('should parse accelerator-count-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-count-min:1']); + expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); + }); + + it('should parse accelerator-count-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-count-max:4']); + expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(4); + }); + + it('should parse both accelerator-count-min and accelerator-count-max', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-count-min:1', 'ghr-ec2-accelerator-count-max:2']); + expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); + expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(2); + }); + + it('should parse accelerator-types as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-types:gpu']); + expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu']); + }); + + it('should parse accelerator-types as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-types:gpu;fpga']); + expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu', 'fpga']); + }); + + it('should parse accelerator-manufacturers as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-manufacturers:nvidia']); + expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia']); + }); + + it('should parse accelerator-manufacturers as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-manufacturers:nvidia;amd']); + expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia', 'amd']); + }); + + it('should parse accelerator-names as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-names:a100']); + expect(result?.InstanceRequirements?.AcceleratorNames).toEqual(['a100']); + }); + + it('should parse accelerator-names as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-names:a100;v100']); + expect(result?.InstanceRequirements?.AcceleratorNames).toEqual(['a100', 'v100']); + }); + + it('should parse accelerator-total-memory-mib-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-total-memory-mib-min:8192']); + expect(result?.InstanceRequirements?.AcceleratorTotalMemoryMiB?.Min).toBe(8192); + }); + + it('should parse accelerator-total-memory-mib-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-total-memory-mib-max:40960']); + expect(result?.InstanceRequirements?.AcceleratorTotalMemoryMiB?.Max).toBe(40960); + }); + + it('should parse combined accelerator requirements', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-accelerator-count-min:1', + 'ghr-ec2-accelerator-count-max:2', + 'ghr-ec2-accelerator-types:gpu', + 'ghr-ec2-accelerator-manufacturers:nvidia', + ]); + expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); + expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(2); + expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu']); + expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia']); + }); + }); + + describe('Instance Requirements - Network and Storage', () => { + it('should parse network-interface-count-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-network-interface-count-min:2']); + expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Min).toBe(2); + }); + + it('should parse network-interface-count-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-network-interface-count-max:4']); + expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Max).toBe(4); + }); + + it('should parse network-bandwidth-gbps-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-network-bandwidth-gbps-min:5']); + expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Min).toBe(5); + }); + + it('should parse network-bandwidth-gbps-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-network-bandwidth-gbps-max:25']); + expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Max).toBe(25); + }); + + it('should parse local-storage label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-local-storage:included']); + expect(result?.InstanceRequirements?.LocalStorage).toBe('included'); + }); + + it('should parse local-storage-types as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-local-storage-types:ssd']); + expect(result?.InstanceRequirements?.LocalStorageTypes).toEqual(['ssd']); + }); + + it('should parse local-storage-types as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-local-storage-types:hdd;ssd']); + expect(result?.InstanceRequirements?.LocalStorageTypes).toEqual(['hdd', 'ssd']); + }); + + it('should parse total-local-storage-gb-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-total-local-storage-gb-min:100']); + expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Min).toBe(100); + }); + + it('should parse total-local-storage-gb-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-total-local-storage-gb-max:1000']); + expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Max).toBe(1000); + }); + + it('should parse baseline-ebs-bandwidth-mbps-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-baseline-ebs-bandwidth-mbps-min:500']); + expect(result?.InstanceRequirements?.BaselineEbsBandwidthMbps?.Min).toBe(500); + }); + + it('should parse baseline-ebs-bandwidth-mbps-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-baseline-ebs-bandwidth-mbps-max:2000']); + expect(result?.InstanceRequirements?.BaselineEbsBandwidthMbps?.Max).toBe(2000); + }); + }); + + describe('Instance Requirements - Pricing and Other', () => { + it('should parse spot-max-price-percentage-over-lowest-price label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-spot-max-price-percentage-over-lowest-price:50']); + expect(result?.InstanceRequirements?.SpotMaxPricePercentageOverLowestPrice).toBe(50); + }); + + it('should parse on-demand-max-price-percentage-over-lowest-price label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-on-demand-max-price-percentage-over-lowest-price:75']); + expect(result?.InstanceRequirements?.OnDemandMaxPricePercentageOverLowestPrice).toBe(75); + }); + + it('should parse max-spot-price-as-percentage-of-optimal-on-demand-price label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-max-spot-price-as-percentage-of-optimal-on-demand-price:60']); + expect(result?.InstanceRequirements?.MaxSpotPriceAsPercentageOfOptimalOnDemandPrice).toBe(60); + }); + + it('should parse require-hibernate-support label as boolean true', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-require-hibernate-support:true']); + expect(result?.InstanceRequirements?.RequireHibernateSupport).toBe(true); + }); + + it('should parse require-hibernate-support label as boolean false', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-require-hibernate-support:false']); + expect(result?.InstanceRequirements?.RequireHibernateSupport).toBe(false); + }); + + it('should parse require-encryption-in-transit label as boolean true', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-require-encryption-in-transit:true']); + expect(result?.InstanceRequirements?.RequireEncryptionInTransit).toBe(true); + }); + + it('should parse require-encryption-in-transit label as boolean false', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-require-encryption-in-transit:false']); + expect(result?.InstanceRequirements?.RequireEncryptionInTransit).toBe(false); + }); + + it('should parse baseline-performance-factors-cpu-reference-families label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-baseline-performance-factors-cpu-reference-families:intel']); + expect(result?.InstanceRequirements?.BaselinePerformanceFactors?.Cpu?.References?.[0]?.InstanceFamily).toBe( + 'intel', + ); + }); + it('should parse baseline-performance-factors-cpu-reference-families list label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-baseline-performance-factors-cpu-reference-families:intel;amd']); + expect(result?.InstanceRequirements?.BaselinePerformanceFactors?.Cpu?.References?.[0]?.InstanceFamily).toBe( + 'intel', + ); + expect(result?.InstanceRequirements?.BaselinePerformanceFactors?.Cpu?.References?.[1]?.InstanceFamily).toBe( + 'amd', + ); + }); + }); + + describe('Edge Cases', () => { + it('should return undefined when empty array is provided', () => { + const result = parseEc2OverrideConfig([]); + expect(result).toBeUndefined(); + }); + + it('should return undefined when no ghr-ec2 labels are provided', () => { + const result = parseEc2OverrideConfig(['self-hosted', 'linux', 'x64']); + expect(result).toBeUndefined(); + }); + + it('should ignore non-ghr-ec2 labels and only parse ghr-ec2 labels', () => { + const result = parseEc2OverrideConfig([ + 'self-hosted', + 'ghr-ec2-instance-type:m5.large', + 'linux', + 'ghr-ec2-max-price:0.30', + ]); + expect(result?.InstanceType).toBe('m5.large'); + expect(result?.MaxPrice).toBe('0.30'); + }); + + it('should handle labels with colons in values (ARNs)', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-ebs-kms-key-id:arn:aws:kms:us-east-1:123456789012:key/abc-def-ghi', + ]); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.KmsKeyId).toBe( + 'arn:aws:kms:us-east-1:123456789012:key/abc-def-ghi', + ); + }); + + it('should handle labels with colons in placement ARNs', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-placement-host-resource-group-arn:arn:aws:ec2:us-west-2:123456789012:host-resource-group/hrg-abc123', + ]); + expect(result?.Placement?.HostResourceGroupArn).toBe( + 'arn:aws:ec2:us-west-2:123456789012:host-resource-group/hrg-abc123', + ); + }); + + it('should handle labels without values gracefully', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-instance-type:', 'ghr-ec2-max-price:0.50']); + expect(result?.InstanceType).toBeUndefined(); + expect(result?.MaxPrice).toBe('0.50'); + }); + + it('should handle malformed labels (no colon) gracefully', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-instance-type-m5-large', 'ghr-ec2-max-price:0.50']); + expect(result?.MaxPrice).toBe('0.50'); + expect(result?.InstanceType).toBeUndefined(); + }); + + it('should handle numeric strings correctly for number fields', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-priority:5', + 'ghr-ec2-weighted-capacity:10', + 'ghr-ec2-vcpu-count-min:4', + ]); + expect(result?.Priority).toBe(5); + expect(result?.WeightedCapacity).toBe(10); + expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(4); + }); + + it('should handle boolean strings correctly for boolean fields', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-ebs-encrypted:true', + 'ghr-ec2-ebs-delete-on-termination:false', + 'ghr-ec2-require-hibernate-support:true', + ]); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.DeleteOnTermination).toBe(false); + expect(result?.InstanceRequirements?.RequireHibernateSupport).toBe(true); + }); + + it('should handle floating point numbers in max-price', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-max-price:0.12345']); + expect(result?.MaxPrice).toBe('0.12345'); + }); + + it('should handle whitespace in semicolon-separated lists', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-cpu-manufacturers: intel ; amd ']); + expect(result?.InstanceRequirements?.CpuManufacturers).toEqual([' intel ', ' amd ']); + }); + + it('should return config with all parsed labels', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-instance-type:c5.xlarge', + 'ghr-ec2-vcpu-count-min:4', + 'ghr-ec2-memory-mib-min:8192', + 'ghr-ec2-placement-tenancy:dedicated', + 'ghr-ec2-ebs-volume-size:100', + ]); + expect(result?.InstanceType).toBe('c5.xlarge'); + expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(4); + expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(8192); + expect(result?.Placement?.Tenancy).toBe('dedicated'); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(100); + }); + }); + + describe('Complex Scenarios', () => { + it('should handle comprehensive EC2 configuration with all categories', () => { + const result = parseEc2OverrideConfig([ + // Basic Fleet + 'ghr-ec2-instance-type:r5.2xlarge', + 'ghr-ec2-max-price:0.75', + 'ghr-ec2-priority:1', + // Placement + 'ghr-ec2-placement-group-name:my-group', + 'ghr-ec2-placement-tenancy:dedicated', + // Block Device + 'ghr-ec2-ebs-volume-size:200', + 'ghr-ec2-ebs-volume-type:gp3', + 'ghr-ec2-ebs-encrypted:true', + // Instance Requirements + 'ghr-ec2-vcpu-count-min:8', + 'ghr-ec2-vcpu-count-max:32', + 'ghr-ec2-memory-mib-min:32768', + 'ghr-ec2-cpu-manufacturers:intel;amd', + 'ghr-ec2-instance-generations:current', + ]); + + expect(result?.InstanceType).toBe('r5.2xlarge'); + expect(result?.MaxPrice).toBe('0.75'); + expect(result?.Priority).toBe(1); + expect(result?.Placement?.GroupName).toBe('my-group'); + expect(result?.Placement?.Tenancy).toBe('dedicated'); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(200); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeType).toBe('gp3'); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); + expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(8); + expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(32); + expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(32768); + expect(result?.InstanceRequirements?.CpuManufacturers).toEqual(['intel', 'amd']); + expect(result?.InstanceRequirements?.InstanceGenerations).toEqual(['current']); + }); + + it('should handle GPU instance configuration', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-accelerator-count-min:1', + 'ghr-ec2-accelerator-count-max:4', + 'ghr-ec2-accelerator-types:gpu', + 'ghr-ec2-accelerator-manufacturers:nvidia', + 'ghr-ec2-accelerator-names:a100;v100', + 'ghr-ec2-accelerator-total-memory-mib-min:16384', + ]); + + expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); + expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(4); + expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu']); + expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia']); + expect(result?.InstanceRequirements?.AcceleratorNames).toEqual(['a100', 'v100']); + expect(result?.InstanceRequirements?.AcceleratorTotalMemoryMiB?.Min).toBe(16384); + }); + + it('should handle network-optimized instance configuration', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-network-interface-count-min:2', + 'ghr-ec2-network-interface-count-max:8', + 'ghr-ec2-network-bandwidth-gbps-min:10', + 'ghr-ec2-network-bandwidth-gbps-max:100', + 'ghr-ec2-baseline-ebs-bandwidth-mbps-min:1000', + ]); + + expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Min).toBe(2); + expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Max).toBe(8); + expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Min).toBe(10); + expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Max).toBe(100); + expect(result?.InstanceRequirements?.BaselineEbsBandwidthMbps?.Min).toBe(1000); + }); + + it('should handle storage-optimized instance configuration', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-local-storage:included', + 'ghr-ec2-local-storage-types:ssd', + 'ghr-ec2-total-local-storage-gb-min:500', + 'ghr-ec2-total-local-storage-gb-max:2000', + ]); + + expect(result?.InstanceRequirements?.LocalStorage).toBe('included'); + expect(result?.InstanceRequirements?.LocalStorageTypes).toEqual(['ssd']); + expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Min).toBe(500); + expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Max).toBe(2000); + }); + + it('should handle spot instance configuration with pricing', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-max-price:0.50', + 'ghr-ec2-spot-max-price-percentage-over-lowest-price:100', + 'ghr-ec2-on-demand-max-price-percentage-over-lowest-price:150', + ]); + + expect(result?.MaxPrice).toBe('0.50'); + expect(result?.InstanceRequirements?.SpotMaxPricePercentageOverLowestPrice).toBe(100); + expect(result?.InstanceRequirements?.OnDemandMaxPricePercentageOverLowestPrice).toBe(150); + }); + }); +}); diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-labels.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-labels.ts new file mode 100644 index 0000000000..496b7dab46 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-labels.ts @@ -0,0 +1,388 @@ +import { + _InstanceType, + AcceleratorCountRequest, + AcceleratorManufacturer, + AcceleratorName, + AcceleratorTotalMemoryMiBRequest, + AcceleratorType, + BareMetal, + BaselineEbsBandwidthMbpsRequest, + BaselinePerformanceFactorsRequest, + BurstablePerformance, + CpuManufacturer, + CpuPerformanceFactorRequest, + DescribeLaunchTemplateVersionsCommand, + EC2Client, + FleetBlockDeviceMappingRequest, + FleetEbsBlockDeviceRequest, + InstanceGeneration, + InstanceRequirementsRequest, + LocalStorage, + LocalStorageType, + MemoryGiBPerVCpuRequest, + MemoryMiBRequest, + NetworkBandwidthGbpsRequest, + NetworkInterfaceCountRequest, + PerformanceFactorReferenceRequest, + Placement, + Tenancy, + TotalLocalStorageGBRequest, + VCpuCountRangeRequest, + VolumeType, +} from '@aws-sdk/client-ec2'; +import { getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; + +import { Ec2OverrideConfig } from './../aws/ec2-runners.d'; + +const EC2_OVERRIDE_LIST_VALUE_SEPARATOR = ';'; + +/** + * Parses EC2 override configuration from GitHub labels. + * + * Supported label formats: + * + * Basic Fleet Overrides: + * - ghr-ec2-instance-type: - Set specific instance type (e.g., c5.xlarge) + * - ghr-ec2-max-price: - Set maximum spot price + * - ghr-ec2-subnet-id: - Set subnet ID + * - ghr-ec2-availability-zone: - Set availability zone + * - ghr-ec2-availability-zone-id: - Set availability zone ID + * - ghr-ec2-weighted-capacity: - Set weighted capacity + * - ghr-ec2-priority: - Set launch priority + * - ghr-ec2-image-id: - Override AMI ID + * + * Instance Requirements (vCPU & Memory): + * - ghr-ec2-vcpu-count-min: - Set minimum vCPU count + * - ghr-ec2-vcpu-count-max: - Set maximum vCPU count + * - ghr-ec2-memory-mib-min: - Set minimum memory in MiB + * - ghr-ec2-memory-mib-max: - Set maximum memory in MiB + * - ghr-ec2-memory-gib-per-vcpu-min: - Set min memory per vCPU ratio + * - ghr-ec2-memory-gib-per-vcpu-max: - Set max memory per vCPU ratio + * + * Instance Requirements (CPU & Performance): + * - ghr-ec2-cpu-manufacturers: - CPU manufacturers (semicolon-separated: intel;amd;amazon-web-services) + * - ghr-ec2-instance-generations: - Instance generations (semicolon-separated: current;previous) + * - ghr-ec2-excluded-instance-types: - Exclude instance types (semicolon-separated) + * - ghr-ec2-allowed-instance-types: - Allow only specific instance types (semicolon-separated) + * - ghr-ec2-burstable-performance: - Burstable performance (included,excluded,required) + * - ghr-ec2-bare-metal: - Bare metal (included,excluded,required) + * + * Instance Requirements (Accelerators/GPU): + * - ghr-ec2-accelerator-types: - Accelerator types (semicolon-separated: gpu;fpga;inference) + * - ghr-ec2-accelerator-count-min: - Set minimum accelerator count + * - ghr-ec2-accelerator-count-max: - Set maximum accelerator count + * - ghr-ec2-accelerator-manufacturers: - Accelerator manufacturers (semicolon-separated: nvidia;amd;amazon-web-services;xilinx) + * - ghr-ec2-accelerator-names: - Specific accelerator names (semicolon-separated) + * - ghr-ec2-accelerator-total-memory-mib-min: - Min accelerator total memory in MiB + * - ghr-ec2-accelerator-total-memory-mib-max: - Max accelerator total memory in MiB + * + * Instance Requirements (Network & Storage): + * - ghr-ec2-network-interface-count-min: - Min network interfaces + * - ghr-ec2-network-interface-count-max: - Max network interfaces + * - ghr-ec2-network-bandwidth-gbps-min: - Min network bandwidth in Gbps + * - ghr-ec2-network-bandwidth-gbps-max: - Max network bandwidth in Gbps + * - ghr-ec2-local-storage: - Local storage (included,excluded,required) + * - ghr-ec2-local-storage-types: - Local storage types (semicolon-separated: hdd;ssd) + * - ghr-ec2-total-local-storage-gb-min: - Min total local storage in GB + * - ghr-ec2-total-local-storage-gb-max: - Max total local storage in GB + * - ghr-ec2-baseline-ebs-bandwidth-mbps-min: - Min baseline EBS bandwidth in Mbps + * - ghr-ec2-baseline-ebs-bandwidth-mbps-max: - Max baseline EBS bandwidth in Mbps + * + * Placement: + * - ghr-ec2-placement-group-name: - Placement group name + * - ghr-ec2-placement-group-id: - Placement group ID + * - ghr-ec2-placement-tenancy: - Tenancy (default,dedicated,host) + * - ghr-ec2-placement-host-id: - Dedicated host ID + * - ghr-ec2-placement-affinity: - Affinity (default,host) + * - ghr-ec2-placement-partition-number: - Partition number + * - ghr-ec2-placement-availability-zone: - Placement availability zone + * - ghr-ec2-placement-availability-zone-id: - Placement availability zone ID + * - ghr-ec2-placement-spread-domain: - Spread domain + * - ghr-ec2-placement-host-resource-group-arn: - Host resource group ARN + * + * Block Device Mappings: + * - ghr-ec2-block-device-name: - Block device name + * - ghr-ec2-ebs-volume-size: - EBS volume size in GB + * - ghr-ec2-ebs-volume-type: - EBS volume type (gp2,gp3,io1,io2,st1,sc1) + * - ghr-ec2-ebs-iops: - EBS IOPS + * - ghr-ec2-ebs-throughput: - EBS throughput in MB/s (gp3 only) + * - ghr-ec2-ebs-encrypted: - EBS encryption (true,false) + * - ghr-ec2-ebs-kms-key-id: - KMS key ID for encryption + * - ghr-ec2-ebs-delete-on-termination: - Delete on termination (true,false) + * - ghr-ec2-ebs-snapshot-id: - Snapshot ID for EBS volume + * - ghr-ec2-block-device-virtual-name: - Virtual device name (ephemeral storage) + * - ghr-ec2-block-device-no-device: - Suppresses device mapping + * + * Pricing & Advanced: + * - ghr-ec2-spot-max-price-percentage-over-lowest-price: - Spot max price as % over lowest price + * - ghr-ec2-on-demand-max-price-percentage-over-lowest-price: - On-demand max price as % over lowest price + * - ghr-ec2-max-spot-price-as-percentage-of-optimal-on-demand-price: - Max spot price as % of optimal on-demand + * - ghr-ec2-require-hibernate-support: - Require hibernate support (true,false) + * - ghr-ec2-require-encryption-in-transit: - Require encryption in-transit (true,false) + * - ghr-ec2-baseline-performance-factors-cpu-reference-families: - CPU baseline performance reference families (semicolon-separated) + * + * Example: + * runs-on: [self-hosted, linux, ghr-ec2-vcpu-count-min:4, ghr-ec2-memory-mib-min:16384, ghr-ec2-accelerator-types:gpu] + * + * @param labels - Array of GitHub workflow job labels + * @param defaultBlockDeviceName - Device name to use when dynamic block device labels create a mapping + * @returns EC2 override configuration object or undefined if no valid config found + */ +export function parseEc2OverrideConfig( + labels: string[], + defaultBlockDeviceName?: string, +): Ec2OverrideConfig | undefined { + const ec2Labels = labels.filter((l) => l.startsWith('ghr-ec2-')); + const config: Ec2OverrideConfig = {}; + + for (const label of ec2Labels) { + const [key, ...valueParts] = label.replace('ghr-ec2-', '').split(':'); + const value = valueParts.join(':'); + + if (!value) continue; + + // Basic Fleet Overrides + if (key === 'instance-type') { + config.InstanceType = value as _InstanceType; + } else if (key === 'subnet-id') { + config.SubnetId = value; + } else if (key === 'availability-zone') { + config.AvailabilityZone = value; + } else if (key === 'availability-zone-id') { + config.AvailabilityZoneId = value; + } else if (key === 'max-price') { + config.MaxPrice = value; + } else if (key === 'priority') { + config.Priority = parseFloat(value); + } else if (key === 'weighted-capacity') { + config.WeightedCapacity = parseFloat(value); + } else if (key === 'image-id') { + config.ImageId = value; + } + + // Placement + else if (key.startsWith('placement-')) { + config.Placement = config.Placement || ({} as Placement); + const placementKey = key.replace('placement-', ''); + if (placementKey === 'availability-zone-id') { + config.Placement.AvailabilityZoneId = value; + } else if (placementKey === 'affinity') { + config.Placement.Affinity = value; + } else if (placementKey === 'group-name') { + config.Placement.GroupName = value; + } else if (placementKey === 'partition-number') { + config.Placement.PartitionNumber = parseInt(value, 10); + } else if (placementKey === 'host-id') { + config.Placement.HostId = value; + } else if (placementKey === 'tenancy') { + config.Placement.Tenancy = value as Tenancy; + } else if (placementKey === 'spread-domain') { + config.Placement.SpreadDomain = value; + } else if (placementKey === 'host-resource-group-arn') { + config.Placement.HostResourceGroupArn = value; + } else if (placementKey === 'group-id') { + config.Placement.GroupId = value; + } else if (placementKey === 'availability-zone') { + config.Placement.AvailabilityZone = value; + } + } + + // Block Device Mappings + else if (key === 'block-device-name') { + getOrCreateBlockDeviceMapping(config, defaultBlockDeviceName).DeviceName = value; + } else if (key === 'block-device-virtual-name') { + getOrCreateBlockDeviceMapping(config, defaultBlockDeviceName).VirtualName = value; + } else if (key.startsWith('ebs-')) { + const blockDeviceMapping = getOrCreateBlockDeviceMapping(config, defaultBlockDeviceName); + const ebsKey = key.replace('ebs-', ''); + const ebs = blockDeviceMapping.Ebs || (blockDeviceMapping.Ebs = {} as FleetEbsBlockDeviceRequest); + + if (ebsKey === 'encrypted') { + ebs.Encrypted = value.toLowerCase() === 'true'; + } else if (ebsKey === 'delete-on-termination') { + ebs.DeleteOnTermination = value.toLowerCase() === 'true'; + } else if (ebsKey === 'iops') { + ebs.Iops = parseInt(value, 10); + } else if (ebsKey === 'throughput') { + ebs.Throughput = parseInt(value, 10); + } else if (ebsKey === 'kms-key-id') { + ebs.KmsKeyId = value; + } else if (ebsKey === 'snapshot-id') { + ebs.SnapshotId = value; + } else if (ebsKey === 'volume-size') { + ebs.VolumeSize = parseInt(value, 10); + } else if (ebsKey === 'volume-type') { + ebs.VolumeType = value as VolumeType; + } + } else if (key === 'block-device-no-device') { + getOrCreateBlockDeviceMapping(config, defaultBlockDeviceName).NoDevice = value; + } + + // Instance Requirements + else if (key.startsWith('vcpu-count-')) { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.VCpuCount = config.InstanceRequirements.VCpuCount || ({} as VCpuCountRangeRequest); + const subKey = key.replace('vcpu-count-', ''); + config.InstanceRequirements.VCpuCount![subKey === 'min' ? 'Min' : 'Max'] = parseInt(value, 10); + } else if (key.startsWith('memory-mib-')) { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.MemoryMiB = config.InstanceRequirements.MemoryMiB || ({} as MemoryMiBRequest); + const subKey = key.replace('memory-mib-', ''); + config.InstanceRequirements.MemoryMiB![subKey === 'min' ? 'Min' : 'Max'] = parseInt(value, 10); + } else if (key === 'cpu-manufacturers') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.CpuManufacturers = splitEc2OverrideListValue(value) as CpuManufacturer[]; + } else if (key.startsWith('memory-gib-per-vcpu-')) { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.MemoryGiBPerVCpu = + config.InstanceRequirements.MemoryGiBPerVCpu || ({} as MemoryGiBPerVCpuRequest); + const subKey = key.replace('memory-gib-per-vcpu-', ''); + config.InstanceRequirements.MemoryGiBPerVCpu![subKey === 'min' ? 'Min' : 'Max'] = parseFloat(value); + } else if (key === 'excluded-instance-types') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.ExcludedInstanceTypes = splitEc2OverrideListValue(value); + } else if (key === 'instance-generations') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.InstanceGenerations = splitEc2OverrideListValue(value) as InstanceGeneration[]; + } else if (key === 'spot-max-price-percentage-over-lowest-price') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.SpotMaxPricePercentageOverLowestPrice = parseInt(value, 10); + } else if (key === 'on-demand-max-price-percentage-over-lowest-price') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.OnDemandMaxPricePercentageOverLowestPrice = parseInt(value, 10); + } else if (key === 'bare-metal') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.BareMetal = value as BareMetal; + } else if (key === 'burstable-performance') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.BurstablePerformance = value as BurstablePerformance; + } else if (key === 'require-hibernate-support') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.RequireHibernateSupport = value.toLowerCase() === 'true'; + } else if (key.startsWith('network-interface-count-')) { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.NetworkInterfaceCount = + config.InstanceRequirements.NetworkInterfaceCount || ({} as NetworkInterfaceCountRequest); + const subKey = key.replace('network-interface-count-', ''); + config.InstanceRequirements.NetworkInterfaceCount![subKey === 'min' ? 'Min' : 'Max'] = parseInt(value, 10); + } else if (key === 'local-storage') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.LocalStorage = value as LocalStorage; + } else if (key === 'local-storage-types') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.LocalStorageTypes = splitEc2OverrideListValue(value) as LocalStorageType[]; + } else if (key.startsWith('total-local-storage-gb-')) { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.TotalLocalStorageGB = + config.InstanceRequirements.TotalLocalStorageGB || ({} as TotalLocalStorageGBRequest); + const subKey = key.replace('total-local-storage-gb-', ''); + config.InstanceRequirements.TotalLocalStorageGB![subKey === 'min' ? 'Min' : 'Max'] = parseFloat(value); + } else if (key.startsWith('baseline-ebs-bandwidth-mbps-')) { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.BaselineEbsBandwidthMbps = + config.InstanceRequirements.BaselineEbsBandwidthMbps || ({} as BaselineEbsBandwidthMbpsRequest); + const subKey = key.replace('baseline-ebs-bandwidth-mbps-', ''); + config.InstanceRequirements.BaselineEbsBandwidthMbps![subKey === 'min' ? 'Min' : 'Max'] = parseInt(value, 10); + } else if (key === 'accelerator-types') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.AcceleratorTypes = splitEc2OverrideListValue(value) as AcceleratorType[]; + } else if (key.startsWith('accelerator-count-')) { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.AcceleratorCount = + config.InstanceRequirements.AcceleratorCount || ({} as AcceleratorCountRequest); + const subKey = key.replace('accelerator-count-', ''); + config.InstanceRequirements.AcceleratorCount![subKey === 'min' ? 'Min' : 'Max'] = parseInt(value, 10); + } else if (key === 'accelerator-manufacturers') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.AcceleratorManufacturers = splitEc2OverrideListValue( + value, + ) as AcceleratorManufacturer[]; + } else if (key === 'accelerator-names') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.AcceleratorNames = splitEc2OverrideListValue(value) as AcceleratorName[]; + } else if (key.startsWith('accelerator-total-memory-mib-')) { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.AcceleratorTotalMemoryMiB = + config.InstanceRequirements.AcceleratorTotalMemoryMiB || ({} as AcceleratorTotalMemoryMiBRequest); + const subKey = key.replace('accelerator-total-memory-mib-', ''); + config.InstanceRequirements.AcceleratorTotalMemoryMiB![subKey === 'min' ? 'Min' : 'Max'] = parseInt(value, 10); + } else if (key.startsWith('network-bandwidth-gbps-')) { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.NetworkBandwidthGbps = + config.InstanceRequirements.NetworkBandwidthGbps || ({} as NetworkBandwidthGbpsRequest); + const subKey = key.replace('network-bandwidth-gbps-', ''); + config.InstanceRequirements.NetworkBandwidthGbps![subKey === 'min' ? 'Min' : 'Max'] = parseFloat(value); + } else if (key === 'allowed-instance-types') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.AllowedInstanceTypes = splitEc2OverrideListValue(value); + } else if (key === 'max-spot-price-as-percentage-of-optimal-on-demand-price') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.MaxSpotPriceAsPercentageOfOptimalOnDemandPrice = parseInt(value, 10); + } else if (key === 'baseline-performance-factors-cpu-reference-families') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.BaselinePerformanceFactors = + config.InstanceRequirements.BaselinePerformanceFactors || ({} as BaselinePerformanceFactorsRequest); + config.InstanceRequirements.BaselinePerformanceFactors.Cpu = + config.InstanceRequirements.BaselinePerformanceFactors.Cpu || ({} as CpuPerformanceFactorRequest); + config.InstanceRequirements.BaselinePerformanceFactors.Cpu.References = splitEc2OverrideListValue(value).map( + (family) => ({ InstanceFamily: family }), + ) as PerformanceFactorReferenceRequest[]; + } else if (key === 'require-encryption-in-transit') { + config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); + config.InstanceRequirements.RequireEncryptionInTransit = value.toLowerCase() === 'true'; + } + } + + return Object.keys(config).length > 0 ? config : undefined; +} + +function splitEc2OverrideListValue(value: string): string[] { + return value.split(EC2_OVERRIDE_LIST_VALUE_SEPARATOR); +} + +function getOrCreateBlockDeviceMapping( + config: Ec2OverrideConfig, + defaultBlockDeviceName?: string, +): FleetBlockDeviceMappingRequest { + config.BlockDeviceMappings = + config.BlockDeviceMappings || + ([defaultBlockDeviceName ? { DeviceName: defaultBlockDeviceName } : {}] as FleetBlockDeviceMappingRequest[]); + return config.BlockDeviceMappings[0]; +} + +export function shouldLoadLaunchTemplateBlockDeviceName(labels: string[]): boolean { + const blockDeviceNameLabel = 'ghr-ec2-block-device-name:'; + let hasBlockDeviceOverride = false; + let hasBlockDeviceName = false; + + for (const label of labels) { + hasBlockDeviceOverride = + hasBlockDeviceOverride || label.startsWith('ghr-ec2-ebs-') || label.startsWith('ghr-ec2-block-device-'); + + hasBlockDeviceName = + hasBlockDeviceName || (label.startsWith(blockDeviceNameLabel) && label.slice(blockDeviceNameLabel.length) !== ''); + } + + return hasBlockDeviceOverride && !hasBlockDeviceName; +} + +export async function getDefaultBlockDeviceNameFromLaunchTemplate(launchTemplateName: string): Promise { + const ec2Client = getTracedAWSV3Client(new EC2Client({ region: process.env.AWS_REGION })); + const launchTemplateVersions = await ec2Client.send( + new DescribeLaunchTemplateVersionsCommand({ + LaunchTemplateName: launchTemplateName, + Versions: ['$Default'], + }), + ); + const blockDeviceMappings = + launchTemplateVersions.LaunchTemplateVersions?.[0]?.LaunchTemplateData?.BlockDeviceMappings; + const blockDeviceName = + blockDeviceMappings?.find((blockDeviceMapping) => blockDeviceMapping.DeviceName && blockDeviceMapping.Ebs) + ?.DeviceName ?? blockDeviceMappings?.find((blockDeviceMapping) => blockDeviceMapping.DeviceName)?.DeviceName; + + if (!blockDeviceName) { + throw new Error(`Failed to determine block device name from launch template '${launchTemplateName}'.`); + } + + return blockDeviceName; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.test.ts new file mode 100644 index 0000000000..50f0e6776f --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.test.ts @@ -0,0 +1,2769 @@ +import { DescribeLaunchTemplateVersionsCommand, EC2Client } from '@aws-sdk/client-ec2'; +import { PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +// Using vi.mocked instead of jest-mock +import nock from 'nock'; +import { performance } from 'perf_hooks'; + +import * as ghAuth from '../github/auth'; +import { createRunner, listEC2Runners, tag } from './../aws/ec2-runners'; +import { RunnerInputParameters } from './../aws/ec2-runners.d'; +import * as scaleUpModule from './scale-up'; +import { EC2_TAG_VALUE_MAX_LENGTH, RUNNER_LABELS_TAG_MAX_COUNT } from './ec2'; +import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { publishRetryMessage } from './job-retry'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import type { Octokit } from '@octokit/rest'; +import type { ActionRequestMessageSQS } from './types'; + +const mockOctokit = { + paginate: vi.fn(), + checks: { get: vi.fn() }, + actions: { + createRegistrationTokenForOrg: vi.fn(), + createRegistrationTokenForRepo: vi.fn(), + getJobForWorkflowRun: vi.fn(), + generateRunnerJitconfigForOrg: vi.fn(), + generateRunnerJitconfigForRepo: vi.fn(), + }, + apps: { + getOrgInstallation: vi.fn(), + getRepoInstallation: vi.fn(), + }, +}; + +const mockCreateRunner = vi.mocked(createRunner); +const mockListRunners = vi.mocked(listEC2Runners); +const mockTag = vi.mocked(tag); +const mockEC2Client = mockClient(EC2Client); +const mockSSMClient = mockClient(SSMClient); +const mockSSMgetParameter = vi.mocked(getParameter); +const mockPublishRetryMessage = vi.mocked(publishRetryMessage); + +vi.mock('@octokit/rest', () => ({ + Octokit: vi.fn().mockImplementation(function () { + return mockOctokit; + }), +})); + +vi.mock('./../aws/ec2-runners', async () => ({ + createRunner: vi.fn(), + listEC2Runners: vi.fn(), + tag: vi.fn(), +})); + +vi.mock('./../github/auth', async () => ({ + createGithubAppAuth: vi.fn(), + createGithubInstallationAuth: vi.fn(), + createOctokitClient: vi.fn(), +})); + +vi.mock('@aws-github-runner/aws-ssm-util', async () => { + const actual = (await vi.importActual( + '@aws-github-runner/aws-ssm-util', + )) as typeof import('@aws-github-runner/aws-ssm-util'); + + return { + ...actual, + getParameter: vi.fn(), + }; +}); + +vi.mock('./job-retry', () => ({ + publishRetryMessage: vi.fn(), + checkAndRetryJob: vi.fn(), +})); + +export type RunnerType = 'ephemeral' | 'non-ephemeral'; + +// for ephemeral and non-ephemeral runners +const RUNNER_TYPES: RunnerType[] = ['ephemeral', 'non-ephemeral']; + +const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); +const mockedInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); +const mockCreateClient = vi.mocked(ghAuth.createOctokitClient); + +const TEST_DATA_SINGLE: ActionRequestMessageSQS = { + id: 1, + eventType: 'workflow_job', + repositoryName: 'hello-world', + repositoryOwner: 'Codertocat', + installationId: 2, + repoOwnerType: 'Organization', + messageId: 'foobar', +}; + +const TEST_DATA: ActionRequestMessageSQS[] = [ + { + ...TEST_DATA_SINGLE, + messageId: 'foobar', + }, +]; + +const cleanEnv = process.env; + +const EXPECTED_RUNNER_PARAMS: RunnerInputParameters = { + environment: 'unit-test-environment', + runnerType: 'Org', + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + numberOfRunners: 1, + launchTemplateName: 'lt-1', + ec2instanceCriteria: { + instanceTypes: ['m5.large'], + targetCapacityType: 'spot', + instanceAllocationStrategy: 'lowest-price', + }, + subnets: ['subnet-123'], + tracingEnabled: false, + onDemandFailoverOnError: [], + scaleErrors: ['UnfulfillableCapacity', 'MaxSpotInstanceCountExceeded', 'TargetCapacityLimitExceededException'], + source: 'scale-up-lambda', + useDedicatedHost: false, +}; +let expectedRunnerParams: RunnerInputParameters; + +function setDefaults() { + process.env = { ...cleanEnv }; + process.env.PARAMETER_GITHUB_APP_ID_NAME = 'github-app-id'; + process.env.GITHUB_APP_KEY_BASE64 = 'TEST_CERTIFICATE_DATA'; + process.env.GITHUB_APP_ID = '1337'; + process.env.GITHUB_APP_CLIENT_ID = 'TEST_CLIENT_ID'; + process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET'; + process.env.RUNNERS_MAXIMUM_COUNT = '3'; + process.env.ENVIRONMENT = EXPECTED_RUNNER_PARAMS.environment; + process.env.LAUNCH_TEMPLATE_NAME = 'lt-1'; + process.env.SUBNET_IDS = 'subnet-123'; + process.env.INSTANCE_TYPES = 'm5.large'; + process.env.INSTANCE_TARGET_CAPACITY_TYPE = 'spot'; + process.env.ENABLE_ON_DEMAND_FAILOVER = undefined; + process.env.SCALE_ERRORS = + '["UnfulfillableCapacity","MaxSpotInstanceCountExceeded","TargetCapacityLimitExceededException"]'; +} + +beforeEach(() => { + nock.disableNetConnect(); + vi.resetModules(); + vi.clearAllMocks(); + setDefaults(); + + mockEC2Client.reset(); + mockEC2Client.on(DescribeLaunchTemplateVersionsCommand).resolves({ + LaunchTemplateVersions: [ + { + LaunchTemplateData: { + BlockDeviceMappings: [ + { + DeviceName: '/dev/sda1', + Ebs: {}, + }, + ], + }, + }, + ], + }); + + defaultSSMGetParameterMockImpl(); + defaultOctokitMockImpl(); + + mockCreateRunner.mockImplementation(async () => { + return ['i-12345']; + }); + mockListRunners.mockImplementation(async () => [ + { + instanceId: 'i-1234', + launchTime: new Date(), + type: 'Org', + owner: TEST_DATA_SINGLE.repositoryOwner, + }, + ]); + + mockedAppAuth.mockResolvedValue({ + type: 'app', + token: 'token', + appId: TEST_DATA_SINGLE.installationId, + expiresAt: 'some-date', + }); + mockedInstallationAuth.mockResolvedValue({ + type: 'token', + tokenType: 'installation', + token: 'token', + createdAt: 'some-date', + expiresAt: 'some-date', + permissions: {}, + repositorySelection: 'all', + installationId: 0, + }); + + mockCreateClient.mockResolvedValue(mockOctokit as unknown as Octokit); +}); + +describe('scaleUp with GHES', () => { + beforeEach(() => { + process.env.GHES_URL = 'https://github.enterprise.something'; + }); + + it('checks queued workflows', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalledWith({ + job_id: TEST_DATA_SINGLE.id, + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('does not list runners when no workflows are queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ + data: { total_count: 0 }, + })); + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).not.toBeCalled(); + }); + + describe('on org level', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.RUNNER_NAME_PREFIX = 'unit-test-'; + process.env.RUNNER_GROUP_NAME = 'Default'; + process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; + process.env.RUNNER_LABELS = 'label1,label2'; + + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + mockSSMClient.reset(); + }); + + it('gets the current org level runners', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).toBeCalledWith({ + environment: 'unit-test-environment', + runnerType: 'Org', + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }); + }); + + it('does not create a token when maximum runners has been reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('does not create runners when current runners exceed maximum (race condition)', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '5'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + // Simulate race condition where pool lambda created more runners than max + mockListRunners.mockImplementation(async () => + Array.from({ length: 10 }, (_, i) => ({ + instanceId: `i-${i}`, + launchTime: new Date(), + type: 'Org', + owner: TEST_DATA_SINGLE.repositoryOwner, + })), + ); + await scaleUpModule.scaleUp(TEST_DATA); + // Should not attempt to create runners (would be negative without fix) + expect(createRunner).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + }); + + it('does create a runner if maximum is set to -1', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '-1'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).not.toHaveBeenCalled(); + expect(createRunner).toHaveBeenCalled(); + }); + + it('creates a token when maximum runners has not been reached', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + }); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a runner with correct config', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('tags the EC2 runner with the GitHub runner metadata returned by JIT config', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockTag).toHaveBeenCalledWith('i-12345', [ + { Key: 'ghr:github_runner_id', Value: '9876543210' }, + { Key: 'ghr:runner_labels', Value: 'label1,label2' }, + ]); + }); + + it('chunks comma-joined GitHub runner labels by the EC2 tag value max length', async () => { + const runnerLabels = ['a'.repeat(EC2_TAG_VALUE_MAX_LENGTH), 'b'].join(','); + process.env.RUNNER_LABELS = runnerLabels; + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockTag).toHaveBeenCalledWith('i-12345', [ + { Key: 'ghr:github_runner_id', Value: '9876543210' }, + { Key: 'ghr:runner_labels', Value: runnerLabels.slice(0, EC2_TAG_VALUE_MAX_LENGTH) }, + { + Key: 'ghr:runner_labels:2', + Value: runnerLabels.slice(EC2_TAG_VALUE_MAX_LENGTH), + }, + ]); + }); + + it('limits GitHub runner label metadata to five EC2 tags', async () => { + const runnerLabels = Array.from({ length: RUNNER_LABELS_TAG_MAX_COUNT + 1 }, (_, index) => + String.fromCharCode('a'.charCodeAt(0) + index).repeat(EC2_TAG_VALUE_MAX_LENGTH), + ).join(','); + process.env.RUNNER_LABELS = runnerLabels; + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockTag).toHaveBeenCalledWith('i-12345', [ + { Key: 'ghr:github_runner_id', Value: '9876543210' }, + ...Array.from({ length: RUNNER_LABELS_TAG_MAX_COUNT }, (_, index) => ({ + Key: index === 0 ? 'ghr:runner_labels' : `ghr:runner_labels:${index + 1}`, + Value: runnerLabels.slice(index * EC2_TAG_VALUE_MAX_LENGTH, (index + 1) * EC2_TAG_VALUE_MAX_LENGTH), + })), + ]); + }); + + it('uses a reserved separator when packing GitHub runner labels into EC2 tags', async () => { + process.env.RUNNER_LABELS = ['label:with/slash', 'label+with+plus'].join(','); + const testDataWithSemicolonDynamicLabel = [ + { + ...TEST_DATA_SINGLE, + labels: ['ghr-ec2-cpu-manufacturers:intel;amd'], + messageId: 'test-semicolon-dynamic-label', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithSemicolonDynamicLabel); + + expect(mockTag).toHaveBeenCalledWith('i-12345', [ + { Key: 'ghr:github_runner_id', Value: '9876543210' }, + { + Key: 'ghr:runner_labels', + Value: 'label:with/slash,label+with+plus,ghr-ec2-cpu-manufacturers:intel;amd', + }, + ]); + }); + + it('creates a runner with labels in a specific group', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner with ami id override from ssm parameter', async () => { + process.env.AMI_ID_SSM_PARAMETER_NAME = 'my-ami-id-param'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith({ ...expectedRunnerParams, amiIdSsmParameterName: 'my-ami-id-param' }); + }); + + it('Throws an error if runner group does not exist for ephemeral runners', async () => { + process.env.RUNNER_GROUP_NAME = 'test-runner-group'; + mockSSMgetParameter.mockImplementation(async () => { + throw new Error('ParameterNotFound'); + }); + await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toBeInstanceOf(Error); + expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); + }); + + it('Discards event if it is a User repo and org level runners is enabled', async () => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + const USER_REPO_TEST_DATA = structuredClone(TEST_DATA); + USER_REPO_TEST_DATA[0].repoOwnerType = 'User'; + await scaleUpModule.scaleUp(USER_REPO_TEST_DATA); + expect(createRunner).not.toHaveBeenCalled(); + }); + + it('create SSM parameter for runner group id if it does not exist', async () => { + mockSSMgetParameter.mockImplementation(async () => { + throw new Error('ParameterNotFound'); + }); + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); + expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 2); + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`, + Value: '1', + Type: 'String', + }); + }); + + it('Does not create SSM parameter for runner group id if it exists', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.paginate).toHaveBeenCalledTimes(0); + expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 1); + }); + + it('create start runner config for ephemeral runners ', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toBeCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + name: 'unit-test-i-12345', + runner_group_id: 1, + labels: ['label1', 'label2'], + }); + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: 'TEST_JIT_CONFIG_ORG', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('create start runner config for non-ephemeral runners ', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalled(); + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: + '--url https://github.enterprise.something/Codertocat --token 1234abcd ' + + '--labels label1,label2 --runnergroup Default', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('quotes runner labels with semicolon separators in non-ephemeral runner config', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + + await scaleUpModule.scaleUp([ + { + ...TEST_DATA_SINGLE, + labels: ['ghr-ec2-cpu-manufacturers:intel;amd'], + messageId: 'test-semicolon-labels', + }, + ]); + + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: + '--url https://github.enterprise.something/Codertocat --token 1234abcd ' + + "--labels 'label1,label2,ghr-ec2-cpu-manufacturers:intel;amd' --runnergroup Default", + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('should create JIT config for all remaining instances even when GitHub API fails for one instance', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '5'; + mockCreateRunner.mockImplementation(async () => { + return ['i-instance-1', 'i-instance-2', 'i-instance-3']; + }); + mockListRunners.mockImplementation(async () => { + return []; + }); + + mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ name }) => { + if (name === 'unit-test-i-instance-2') { + // Simulate a 503 Service Unavailable error from GitHub + const error = new Error('Service Unavailable') as Error & { + status: number; + response: { status: number; data: { message: string } }; + }; + error.status = 503; + error.response = { + status: 503, + data: { message: 'Service temporarily unavailable' }, + }; + throw error; + } + return { + data: { + runner: { id: 9876543210 }, + encoded_jit_config: `TEST_JIT_CONFIG_${name}`, + }, + headers: {}, + }; + }); + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toHaveBeenCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + name: 'unit-test-i-instance-1', + runner_group_id: 1, + labels: ['label1', 'label2'], + }); + + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toHaveBeenCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + name: 'unit-test-i-instance-2', + runner_group_id: 1, + labels: ['label1', 'label2'], + }); + + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toHaveBeenCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + name: 'unit-test-i-instance-3', + runner_group_id: 1, + labels: ['label1', 'label2'], + }); + + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-1', + Value: 'TEST_JIT_CONFIG_unit-test-i-instance-1', + Type: 'SecureString', + Tags: [{ Key: 'InstanceId', Value: 'i-instance-1' }], + }); + + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-3', + Value: 'TEST_JIT_CONFIG_unit-test-i-instance-3', + Type: 'SecureString', + Tags: [{ Key: 'InstanceId', Value: 'i-instance-3' }], + }); + + expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-2', + }); + }); + + it('should handle retryable errors with error handling logic', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '5'; + mockCreateRunner.mockImplementation(async () => { + return ['i-instance-1', 'i-instance-2']; + }); + mockListRunners.mockImplementation(async () => { + return []; + }); + + mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ name }) => { + if (name === 'unit-test-i-instance-1') { + const error = new Error('Internal Server Error') as Error & { + status: number; + response: { status: number; data: { message: string } }; + }; + error.status = 500; + error.response = { + status: 500, + data: { message: 'Internal server error' }, + }; + throw error; + } + return { + data: { + runner: { id: 9876543210 }, + encoded_jit_config: `TEST_JIT_CONFIG_${name}`, + }, + headers: {}, + }; + }); + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-2', + Value: 'TEST_JIT_CONFIG_unit-test-i-instance-2', + Type: 'SecureString', + Tags: [{ Key: 'InstanceId', Value: 'i-instance-2' }], + }); + + expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-1', + }); + }); + + it('should handle non-retryable 4xx errors gracefully', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '5'; + mockCreateRunner.mockImplementation(async () => { + return ['i-instance-1', 'i-instance-2']; + }); + mockListRunners.mockImplementation(async () => { + return []; + }); + + mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ name }) => { + if (name === 'unit-test-i-instance-1') { + // 404 is not retryable - will fail immediately + const error = new Error('Not Found') as Error & { + status: number; + response: { status: number; data: { message: string } }; + }; + error.status = 404; + error.response = { + status: 404, + data: { message: 'Resource not found' }, + }; + throw error; + } + return { + data: { + runner: { id: 9876543210 }, + encoded_jit_config: `TEST_JIT_CONFIG_${name}`, + }, + headers: {}, + }; + }); + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-2', + Value: 'TEST_JIT_CONFIG_unit-test-i-instance-2', + Type: 'SecureString', + Tags: [{ Key: 'InstanceId', Value: 'i-instance-2' }], + }); + + expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-1', + }); + }); + + it.each(RUNNER_TYPES)( + 'calls create start runner config of 40' + ' instances (ssm rate limit condition) to test time delay ', + async (type: RunnerType) => { + process.env.ENABLE_EPHEMERAL_RUNNERS = type === 'ephemeral' ? 'true' : 'false'; + process.env.RUNNERS_MAXIMUM_COUNT = '40'; + mockCreateRunner.mockImplementation(async () => { + return instances; + }); + mockListRunners.mockImplementation(async () => { + return []; + }); + const startTime = performance.now(); + const instances = [ + 'i-1234', + 'i-5678', + 'i-5567', + 'i-5569', + 'i-5561', + 'i-5560', + 'i-5566', + 'i-5536', + 'i-5526', + 'i-5516', + 'i-122', + 'i-123', + 'i-124', + 'i-125', + 'i-126', + 'i-127', + 'i-128', + 'i-129', + 'i-130', + 'i-131', + 'i-132', + 'i-133', + 'i-134', + 'i-135', + 'i-136', + 'i-137', + 'i-138', + 'i-139', + 'i-140', + 'i-141', + 'i-142', + 'i-143', + 'i-144', + 'i-145', + 'i-146', + 'i-147', + 'i-148', + 'i-149', + 'i-150', + 'i-151', + ]; + await scaleUpModule.scaleUp(TEST_DATA); + const endTime = performance.now(); + expect(endTime - startTime).toBeGreaterThan(1000); + expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 40); + }, + 10000, + ); + }); + + describe('Dynamic EC2 Configuration', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + process.env.RUNNER_LABELS = 'base-label'; + process.env.INSTANCE_TYPES = 't3.medium,t3.large'; + process.env.RUNNER_NAME_PREFIX = 'unit-test'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + mockSSMClient.reset(); + }); + + it('appends EC2 labels to existing runner labels when EC2 labels are present', async () => { + const testDataWithEc2Labels = [ + { + ...TEST_DATA_SINGLE, + labels: ['ghr-ec2-instance-type:c5.2xlarge', 'ghr-ec2-custom:value'], + messageId: 'test-1', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithEc2Labels); + + // Verify createRunner was called with EC2 instance type in override config + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ + instanceTypes: ['t3.medium', 't3.large'], + }), + ec2OverrideConfig: expect.objectContaining({ + InstanceType: 'c5.2xlarge', + }), + }), + ); + expect(mockTag).toHaveBeenCalledWith('i-12345', [ + { + Key: 'ghr:github_runner_id', + Value: '9876543210', + }, + { + Key: 'ghr:runner_labels', + Value: 'base-label,ghr-ec2-instance-type:c5.2xlarge,ghr-ec2-custom:value', + }, + ]); + }); + + it('uses default instance types when no instance type EC2 label is provided', async () => { + const testDataWithEc2Labels = [ + { + ...TEST_DATA_SINGLE, + labels: ['ghr-ec2-custom:value'], + messageId: 'test-3', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithEc2Labels); + + // Should use the default INSTANCE_TYPES from environment + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ + instanceTypes: ['t3.medium', 't3.large'], + }), + }), + ); + }); + + it('loads the launch template block device name for dynamic EBS labels without DeviceName', async () => { + mockEC2Client.on(DescribeLaunchTemplateVersionsCommand).resolves({ + LaunchTemplateVersions: [ + { + LaunchTemplateData: { + BlockDeviceMappings: [ + { + DeviceName: '/dev/sdb', + VirtualName: 'ephemeral0', + }, + { + DeviceName: '/dev/sdf', + Ebs: {}, + }, + ], + }, + }, + ], + }); + + const testDataWithEbsLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['ghr-ec2-ebs-volume-size:100', 'ghr-ec2-ebs-volume-type:gp3'], + messageId: 'test-ebs-device-name', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithEbsLabels); + + expect(mockEC2Client).toHaveReceivedCommandWith(DescribeLaunchTemplateVersionsCommand, { + LaunchTemplateName: 'lt-1', + Versions: ['$Default'], + }); + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + BlockDeviceMappings: [ + { + DeviceName: '/dev/sdf', + Ebs: { + VolumeSize: 100, + VolumeType: 'gp3', + }, + }, + ], + }), + }), + ); + }); + + it('does not load launch template block device name when DeviceName is provided by labels', async () => { + const testDataWithEbsLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['ghr-ec2-block-device-name:/dev/sdg', 'ghr-ec2-ebs-volume-size:100', 'ghr-ec2-ebs-volume-type:gp3'], + messageId: 'test-explicit-ebs-device-name', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithEbsLabels); + + expect(mockEC2Client).not.toHaveReceivedCommand(DescribeLaunchTemplateVersionsCommand); + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + BlockDeviceMappings: [ + { + DeviceName: '/dev/sdg', + Ebs: { + VolumeSize: 100, + VolumeType: 'gp3', + }, + }, + ], + }), + }), + ); + }); + + it('handles messages with no labels gracefully', async () => { + const testDataWithNoLabels = [ + { + ...TEST_DATA_SINGLE, + labels: undefined, + messageId: 'test-5', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithNoLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ + instanceTypes: ['t3.medium', 't3.large'], + }), + }), + ); + }); + + it('handles empty labels array', async () => { + const testDataWithEmptyLabels = [ + { + ...TEST_DATA_SINGLE, + labels: [], + messageId: 'test-6', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithEmptyLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ + instanceTypes: ['t3.medium', 't3.large'], + }), + }), + ); + }); + + it('handles multiple EC2 labels correctly', async () => { + const testDataWithMultipleEc2Labels = [ + { + ...TEST_DATA_SINGLE, + labels: ['regular-label', 'ghr-ec2-instance-type:r5.2xlarge', 'ghr-ec2-ami:custom-ami', 'ghr-ec2-disk:200'], + messageId: 'test-8', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithMultipleEc2Labels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ + instanceTypes: ['t3.medium', 't3.large'], + }), + ec2OverrideConfig: expect.objectContaining({ + InstanceType: 'r5.2xlarge', + }), + }), + ); + }); + + it('includes ec2OverrideConfig with VCpuCount requirements when specified', async () => { + const testDataWithVCpuLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-vcpu-count-min:4', 'ghr-ec2-vcpu-count-max:16'], + messageId: 'test-9', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithVCpuLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + InstanceRequirements: expect.objectContaining({ + VCpuCount: { + Min: 4, + Max: 16, + }, + }), + }), + }), + ); + }); + + it('includes ec2OverrideConfig with MemoryMiB requirements when specified', async () => { + const testDataWithMemoryLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-memory-mib-min:8192', 'ghr-ec2-memory-mib-max:32768'], + messageId: 'test-10', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithMemoryLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + InstanceRequirements: expect.objectContaining({ + MemoryMiB: { + Min: 8192, + Max: 32768, + }, + }), + }), + }), + ); + }); + + it('includes ec2OverrideConfig with CPU manufacturers when specified', async () => { + const testDataWithCpuLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-cpu-manufacturers:intel;amd'], + messageId: 'test-11', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithCpuLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + InstanceRequirements: expect.objectContaining({ + CpuManufacturers: ['intel', 'amd'], + }), + }), + }), + ); + }); + + it('includes ec2OverrideConfig with instance generations when specified', async () => { + const testDataWithGenerationLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-instance-generations:current'], + messageId: 'test-12', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithGenerationLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + InstanceRequirements: expect.objectContaining({ + InstanceGenerations: ['current'], + }), + }), + }), + ); + }); + + it('includes ec2OverrideConfig with accelerator requirements when specified', async () => { + const testDataWithAcceleratorLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-accelerator-count-min:1', 'ghr-ec2-accelerator-types:gpu'], + messageId: 'test-13', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithAcceleratorLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + InstanceRequirements: expect.objectContaining({ + AcceleratorCount: { + Min: 1, + }, + AcceleratorTypes: ['gpu'], + }), + }), + }), + ); + }); + + it('includes ec2OverrideConfig with max price when specified', async () => { + const testDataWithMaxPrice = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-max-price:0.50'], + messageId: 'test-14', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithMaxPrice); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + MaxPrice: '0.50', + }), + }), + ); + }); + + it('includes ec2OverrideConfig with priority and weighted capacity when specified', async () => { + const testDataWithPriorityWeight = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-priority:1', 'ghr-ec2-weighted-capacity:2'], + messageId: 'test-15', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithPriorityWeight); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + Priority: 1, + WeightedCapacity: 2, + }), + }), + ); + }); + + it('includes ec2OverrideConfig with combined requirements', async () => { + const testDataWithCombinedLabels = [ + { + ...TEST_DATA_SINGLE, + labels: [ + 'self-hosted', + 'linux', + 'ghr-ec2-vcpu-count-min:8', + 'ghr-ec2-memory-mib-min:16384', + 'ghr-ec2-cpu-manufacturers:intel', + 'ghr-ec2-instance-generations:current', + 'ghr-ec2-max-price:1.00', + ], + messageId: 'test-16', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithCombinedLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + InstanceRequirements: expect.objectContaining({ + VCpuCount: { Min: 8 }, + MemoryMiB: { Min: 16384 }, + CpuManufacturers: ['intel'], + InstanceGenerations: ['current'], + }), + MaxPrice: '1.00', + }), + }), + ); + }); + + it('includes both instance type and ec2OverrideConfig when both specified', async () => { + const testDataWithBoth = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-instance-type:c5.xlarge', 'ghr-ec2-vcpu-count-min:4'], + messageId: 'test-18', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithBoth); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ + instanceTypes: ['t3.medium', 't3.large'], + }), + ec2OverrideConfig: expect.objectContaining({ + InstanceType: 'c5.xlarge', + InstanceRequirements: expect.objectContaining({ + VCpuCount: { Min: 4 }, + }), + }), + }), + ); + }); + + it('does not accumulate labels across groups when multiple messages have different dynamic labels', async () => { + const testDataMultipleGroups = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:m7a.large', 'ghr-job-id:run-1-inst-0'], + messageId: 'msg-1', + }, + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:m7i.xlarge', 'ghr-job-id:run-1-inst-1'], + messageId: 'msg-2', + }, + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:c7a.large', 'ghr-job-id:run-1-inst-2'], + messageId: 'msg-3', + }, + ]; + + await scaleUpModule.scaleUp(testDataMultipleGroups); + + expect(createRunner).toBeCalledTimes(3); + + const jitCalls = mockOctokit.actions.generateRunnerJitconfigForOrg.mock.calls; + expect(jitCalls).toHaveLength(3); + + for (const call of jitCalls) { + const labels = call[0].labels as string[]; + + if (labels.includes('ghr-ec2-instance-type:m7a.large')) { + expect(labels).toContain('ghr-job-id:run-1-inst-0'); + expect(labels).not.toContain('ghr-job-id:run-1-inst-1'); + expect(labels).not.toContain('ghr-job-id:run-1-inst-2'); + expect(labels).not.toContain('ghr-ec2-instance-type:m7i.xlarge'); + expect(labels).not.toContain('ghr-ec2-instance-type:c7a.large'); + } else if (labels.includes('ghr-ec2-instance-type:m7i.xlarge')) { + expect(labels).toContain('ghr-job-id:run-1-inst-1'); + expect(labels).not.toContain('ghr-job-id:run-1-inst-0'); + expect(labels).not.toContain('ghr-job-id:run-1-inst-2'); + expect(labels).not.toContain('ghr-ec2-instance-type:m7a.large'); + expect(labels).not.toContain('ghr-ec2-instance-type:c7a.large'); + } else if (labels.includes('ghr-ec2-instance-type:c7a.large')) { + expect(labels).toContain('ghr-job-id:run-1-inst-2'); + expect(labels).not.toContain('ghr-job-id:run-1-inst-0'); + expect(labels).not.toContain('ghr-job-id:run-1-inst-1'); + expect(labels).not.toContain('ghr-ec2-instance-type:m7a.large'); + expect(labels).not.toContain('ghr-ec2-instance-type:m7i.xlarge'); + } else { + throw new Error(`Unexpected labels combination: ${labels.join(',')}`); + } + } + }); + + it('preserves base RUNNER_LABELS for each group without mutation', async () => { + process.env.RUNNER_LABELS = 'ubuntu-2404,x64'; + + const testDataTwoGroups = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-instance-type:m7a.large', 'ghr-team:alpha'], + messageId: 'msg-a', + }, + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-instance-type:c7i.large', 'ghr-team:beta'], + messageId: 'msg-b', + }, + ]; + + await scaleUpModule.scaleUp(testDataTwoGroups); + + expect(createRunner).toBeCalledTimes(2); + + const jitCalls = mockOctokit.actions.generateRunnerJitconfigForOrg.mock.calls; + expect(jitCalls).toHaveLength(2); + + for (const call of jitCalls) { + const labels = call[0].labels as string[]; + + expect(labels).toContain('ubuntu-2404'); + expect(labels).toContain('x64'); + + if (labels.includes('ghr-team:alpha')) { + expect(labels).not.toContain('ghr-team:beta'); + expect(labels).not.toContain('ghr-ec2-instance-type:c7i.large'); + } else if (labels.includes('ghr-team:beta')) { + expect(labels).not.toContain('ghr-team:alpha'); + expect(labels).not.toContain('ghr-ec2-instance-type:m7a.large'); + } else { + throw new Error(`Unexpected labels combination: ${labels.join(',')}`); + } + } + }); + }); + + describe('on repo level', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + process.env.RUNNER_NAME_PREFIX = 'unit-test'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + expectedRunnerParams.runnerType = 'Repo'; + expectedRunnerParams.runnerOwner = `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`; + // `--url https://github.enterprise.something/${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, + // `--token 1234abcd`, + // ]; + }); + + it('gets the current repo level runners', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).toBeCalledWith({ + environment: 'unit-test-environment', + runnerType: 'Repo', + runnerOwner: `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, + }); + }); + + it('does not create a token when maximum runners has been reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a token when maximum runners has not been reached', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('uses the default runner max count', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = undefined; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('creates a runner with correct config and labels', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner and ensure the group argument is ignored', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('Check error is thrown', async () => { + const mockCreateRunners = vi.mocked(createRunner); + mockCreateRunners.mockRejectedValue(new Error('no retry')); + await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow('no retry'); + mockCreateRunners.mockReset(); + }); + }); + + describe('Batch processing', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.RUNNERS_MAXIMUM_COUNT = '10'; + }); + + const createTestMessages = ( + count: number, + overrides: Partial[] = [], + ): ActionRequestMessageSQS[] => { + return Array.from({ length: count }, (_, i) => ({ + ...TEST_DATA_SINGLE, + id: i + 1, + messageId: `message-${i}`, + ...overrides[i], + })); + }; + + it('Should handle multiple messages for the same organization', async () => { + const messages = createTestMessages(3); + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(1); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 3, + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }), + ); + }); + + it('Should handle multiple messages for different organizations', async () => { + const messages = createTestMessages(3, [ + { repositoryOwner: 'org1' }, + { repositoryOwner: 'org2' }, + { repositoryOwner: 'org1' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(2); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, + runnerOwner: 'org1', + }), + ); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, + runnerOwner: 'org2', + }), + ); + }); + + it('Should handle multiple messages for different repositories when org-level is disabled', async () => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + const messages = createTestMessages(3, [ + { repositoryOwner: 'owner1', repositoryName: 'repo1' }, + { repositoryOwner: 'owner1', repositoryName: 'repo2' }, + { repositoryOwner: 'owner1', repositoryName: 'repo1' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(2); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, + runnerOwner: 'owner1/repo1', + }), + ); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, + runnerOwner: 'owner1/repo2', + }), + ); + }); + + it('Should reject messages when maximum runners limit is reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; // Set to 1 so with 1 existing, no new ones can be created + mockListRunners.mockImplementation(async () => [ + { + instanceId: 'i-existing', + launchTime: new Date(), + type: 'Org', + owner: TEST_DATA_SINGLE.repositoryOwner, + }, + ]); + + const messages = createTestMessages(3); + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).not.toHaveBeenCalled(); // No runners should be created + expect(rejectedMessages).toHaveLength(3); // All 3 messages should be rejected + }); + + it('Should handle partial EC2 instance creation failures', async () => { + mockCreateRunner.mockImplementation(async () => ['i-12345']); // Only creates 1 instead of requested 3 + + const messages = createTestMessages(3); + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(rejectedMessages).toHaveLength(2); // 3 requested - 1 created = 2 failed + expect(rejectedMessages).toEqual(['message-0', 'message-1']); + }); + + it('Should filter out invalid event types for ephemeral runners', async () => { + const messages = createTestMessages(3, [ + { eventType: 'workflow_job' }, + { eventType: 'check_run' }, + { eventType: 'workflow_job' }, + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only workflow_job events processed + }), + ); + expect(rejectedMessages).toContain('message-1'); // check_run event rejected + }); + + it('Should skip invalid repo owner types but not reject them', async () => { + const messages = createTestMessages(3, [ + { repoOwnerType: 'Organization' }, + { repoOwnerType: 'User' }, // Invalid for org-level runners + { repoOwnerType: 'Organization' }, + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only Organization events processed + }), + ); + expect(rejectedMessages).not.toContain('message-1'); // User repo not rejected, just skipped + }); + + it('Should skip messages when jobs are not queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { + const isQueued = params.job_id === 1 || params.job_id === 3; // Only jobs 1 and 3 are queued + return { + data: { + status: isQueued ? 'queued' : 'completed', + }, + }; + }); + + const messages = createTestMessages(3); + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only queued jobs processed + }), + ); + }); + + it('Should create separate GitHub clients for different installations', async () => { + // Override the default mock to return different installation IDs + mockOctokit.apps.getOrgInstallation.mockReset(); + mockOctokit.apps.getOrgInstallation.mockImplementation((params) => ({ + data: { + id: params.org === 'org1' ? 100 : 200, + }, + })); + + const messages = createTestMessages(2, [ + { repositoryOwner: 'org1', installationId: 0 }, + { repositoryOwner: 'org2', installationId: 0 }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients + expect(mockedInstallationAuth).toHaveBeenCalledWith(100, 'https://github.enterprise.something/api/v3'); + expect(mockedInstallationAuth).toHaveBeenCalledWith(200, 'https://github.enterprise.something/api/v3'); + }); + + it('Should reuse GitHub clients for same installation', async () => { + const messages = createTestMessages(3, [ + { repositoryOwner: 'same-org' }, + { repositoryOwner: 'same-org' }, + { repositoryOwner: 'same-org' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(mockCreateClient).toHaveBeenCalledTimes(2); // 1 app client, 1 installation client + expect(mockedInstallationAuth).toHaveBeenCalledTimes(1); + }); + + it('Should return empty array when no valid messages to process', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + const messages = createTestMessages(2, [ + { eventType: 'check_run' }, // Invalid for ephemeral + { eventType: 'check_run' }, // Invalid for ephemeral + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).not.toHaveBeenCalled(); + expect(rejectedMessages).toEqual(['message-0', 'message-1']); + }); + + it('Should handle unlimited runners configuration', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '-1'; + const messages = createTestMessages(10); + + await scaleUpModule.scaleUp(messages); + + expect(listEC2Runners).not.toHaveBeenCalled(); // No need to check current runners + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 10, // All messages processed + }), + ); + }); + }); +}); + +describe('scaleUp with public GH', () => { + it('checks queued workflows', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalledWith({ + job_id: TEST_DATA_SINGLE.id, + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('not checking queued workflows', async () => { + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); + }); + + it('does not list runners when no workflows are queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ + data: { status: 'completed' }, + })); + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).not.toBeCalled(); + }); + + describe('on org level', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.RUNNER_NAME_PREFIX = 'unit-test'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + }); + + it('gets the current org level runners', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).toBeCalledWith({ + environment: 'unit-test-environment', + runnerType: 'Org', + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }); + }); + + it('does not create a token when maximum runners has been reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a token when maximum runners has not been reached', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + }); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a runner with correct config', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner with labels in s specific group', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + }); + + describe('on repo level', () => { + beforeEach(() => { + mockSSMClient.reset(); + + process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + process.env.RUNNER_NAME_PREFIX = 'unit-test'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + expectedRunnerParams.runnerType = 'Repo'; + expectedRunnerParams.runnerOwner = `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`; + }); + + it('gets the current repo level runners', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).toBeCalledWith({ + environment: 'unit-test-environment', + runnerType: 'Repo', + runnerOwner: `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, + }); + }); + + it('does not create a token when maximum runners has been reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a token when maximum runners has not been reached', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('creates a runner with correct config and labels', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner with correct config and labels and on demand failover enabled.', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = JSON.stringify(['InsufficientInstanceCapacity']); + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith({ + ...expectedRunnerParams, + onDemandFailoverOnError: ['InsufficientInstanceCapacity'], + }); + }); + + it('creates a runner with correct config and labels and custom scale errors enabled.', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.SCALE_ERRORS = JSON.stringify(['RequestLimitExceeded']); + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith({ + ...expectedRunnerParams, + scaleErrors: ['RequestLimitExceeded'], + }); + }); + + it('creates a runner and ensure the group argument is ignored', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('ephemeral runners only run with workflow_job event, others should fail.', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + + const USER_REPO_TEST_DATA = structuredClone(TEST_DATA); + USER_REPO_TEST_DATA[0].eventType = 'check_run'; + + await expect(scaleUpModule.scaleUp(USER_REPO_TEST_DATA)).resolves.toEqual(['foobar']); + }); + + it('creates a ephemeral runner with JIT config.', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: 'TEST_JIT_CONFIG_REPO', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('creates a ephemeral runner with registration token.', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.ENABLE_JIT_CONFIG = 'false'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --ephemeral', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('JIT config is ignored for non-ephemeral runners.', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + process.env.ENABLE_JIT_CONFIG = 'true'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + process.env.RUNNER_LABELS = 'jit'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --labels jit', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('creates a ephemeral runner after checking job is queued.', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'true'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalled(); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('disable auto update on the runner.', async () => { + process.env.DISABLE_RUNNER_AUTOUPDATE = 'true'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('Scaling error should return failed message IDs so retry can be triggered.', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + await expect(scaleUpModule.scaleUp(TEST_DATA)).resolves.toEqual(['foobar']); + }); + }); + + describe('Batch processing', () => { + const createTestMessages = ( + count: number, + overrides: Partial[] = [], + ): ActionRequestMessageSQS[] => { + return Array.from({ length: count }, (_, i) => ({ + ...TEST_DATA_SINGLE, + id: i + 1, + messageId: `message-${i}`, + ...overrides[i], + })); + }; + + beforeEach(() => { + setDefaults(); + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.RUNNERS_MAXIMUM_COUNT = '10'; + }); + + it('Should handle multiple messages for the same organization', async () => { + const messages = createTestMessages(3); + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(1); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 3, + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }), + ); + }); + + it('Should handle multiple messages for different organizations', async () => { + const messages = createTestMessages(3, [ + { repositoryOwner: 'org1' }, + { repositoryOwner: 'org2' }, + { repositoryOwner: 'org1' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(2); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, + runnerOwner: 'org1', + }), + ); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, + runnerOwner: 'org2', + }), + ); + }); + + it('Should handle multiple messages for different repositories when org-level is disabled', async () => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + const messages = createTestMessages(3, [ + { repositoryOwner: 'owner1', repositoryName: 'repo1' }, + { repositoryOwner: 'owner1', repositoryName: 'repo2' }, + { repositoryOwner: 'owner1', repositoryName: 'repo1' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(2); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, + runnerOwner: 'owner1/repo1', + }), + ); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, + runnerOwner: 'owner1/repo2', + }), + ); + }); + + it('Should reject messages when maximum runners limit is reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; // Set to 1 so with 1 existing, no new ones can be created + mockListRunners.mockImplementation(async () => [ + { + instanceId: 'i-existing', + launchTime: new Date(), + type: 'Org', + owner: TEST_DATA_SINGLE.repositoryOwner, + }, + ]); + + const messages = createTestMessages(3); + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).not.toHaveBeenCalled(); // No runners should be created + expect(rejectedMessages).toHaveLength(3); // All 3 messages should be rejected + }); + + it('Should handle partial EC2 instance creation failures', async () => { + mockCreateRunner.mockImplementation(async () => ['i-12345']); // Only creates 1 instead of requested 3 + + const messages = createTestMessages(3); + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(rejectedMessages).toHaveLength(2); // 3 requested - 1 created = 2 failed + expect(rejectedMessages).toEqual(['message-0', 'message-1']); + }); + + it('Should filter out invalid event types for ephemeral runners', async () => { + const messages = createTestMessages(3, [ + { eventType: 'workflow_job' }, + { eventType: 'check_run' }, + { eventType: 'workflow_job' }, + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only workflow_job events processed + }), + ); + expect(rejectedMessages).toContain('message-1'); // check_run event rejected + }); + + it('Should skip invalid repo owner types but not reject them', async () => { + const messages = createTestMessages(3, [ + { repoOwnerType: 'Organization' }, + { repoOwnerType: 'User' }, // Invalid for org-level runners + { repoOwnerType: 'Organization' }, + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only Organization events processed + }), + ); + expect(rejectedMessages).not.toContain('message-1'); // User repo not rejected, just skipped + }); + + it('Should skip messages when jobs are not queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { + const isQueued = params.job_id === 1 || params.job_id === 3; // Only jobs 1 and 3 are queued + return { + data: { + status: isQueued ? 'queued' : 'completed', + }, + }; + }); + + const messages = createTestMessages(3); + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only queued jobs processed + }), + ); + }); + + it('Should create separate GitHub clients for different installations', async () => { + // Override the default mock to return different installation IDs + mockOctokit.apps.getOrgInstallation.mockReset(); + mockOctokit.apps.getOrgInstallation.mockImplementation((params) => ({ + data: { + id: params.org === 'org1' ? 100 : 200, + }, + })); + + const messages = createTestMessages(2, [ + { repositoryOwner: 'org1', installationId: 0 }, + { repositoryOwner: 'org2', installationId: 0 }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients + expect(mockedInstallationAuth).toHaveBeenCalledWith(100, ''); + expect(mockedInstallationAuth).toHaveBeenCalledWith(200, ''); + }); + + it('Should reuse GitHub clients for same installation', async () => { + const messages = createTestMessages(3, [ + { repositoryOwner: 'same-org' }, + { repositoryOwner: 'same-org' }, + { repositoryOwner: 'same-org' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(mockCreateClient).toHaveBeenCalledTimes(2); // 1 app client, 1 installation client + expect(mockedInstallationAuth).toHaveBeenCalledTimes(1); + }); + + it('Should return empty array when no valid messages to process', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + const messages = createTestMessages(2, [ + { eventType: 'check_run' }, // Invalid for ephemeral + { eventType: 'check_run' }, // Invalid for ephemeral + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).not.toHaveBeenCalled(); + expect(rejectedMessages).toEqual(['message-0', 'message-1']); + }); + + it('Should handle unlimited runners configuration', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '-1'; + const messages = createTestMessages(10); + + await scaleUpModule.scaleUp(messages); + + expect(listEC2Runners).not.toHaveBeenCalled(); // No need to check current runners + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 10, // All messages processed + }), + ); + }); + }); +}); + +describe('scaleUp with Github Data Residency', () => { + beforeEach(() => { + process.env.GHES_URL = 'https://companyname.ghe.com'; + }); + + it('checks queued workflows', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalledWith({ + job_id: TEST_DATA_SINGLE.id, + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('does not list runners when no workflows are queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ + data: { total_count: 0 }, + })); + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).not.toBeCalled(); + }); + + describe('on org level', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.RUNNER_NAME_PREFIX = 'unit-test-'; + process.env.RUNNER_GROUP_NAME = 'Default'; + process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; + process.env.RUNNER_LABELS = 'label1,label2'; + + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + mockSSMClient.reset(); + }); + + it('gets the current org level runners', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).toBeCalledWith({ + environment: 'unit-test-environment', + runnerType: 'Org', + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }); + }); + + it('does not create a token when maximum runners has been reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('does create a runner if maximum is set to -1', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '-1'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).not.toHaveBeenCalled(); + expect(createRunner).toHaveBeenCalled(); + }); + + it('creates a token when maximum runners has not been reached', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + }); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a runner with correct config', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner with labels in a specific group', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner with ami id override from ssm parameter', async () => { + process.env.AMI_ID_SSM_PARAMETER_NAME = 'my-ami-id-param'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith({ ...expectedRunnerParams, amiIdSsmParameterName: 'my-ami-id-param' }); + }); + + it('Throws an error if runner group does not exist for ephemeral runners', async () => { + process.env.RUNNER_GROUP_NAME = 'test-runner-group'; + mockSSMgetParameter.mockImplementation(async () => { + throw new Error('ParameterNotFound'); + }); + await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toBeInstanceOf(Error); + expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); + }); + + it('Discards event if it is a User repo and org level runners is enabled', async () => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + const USER_REPO_TEST_DATA = structuredClone(TEST_DATA); + USER_REPO_TEST_DATA[0].repoOwnerType = 'User'; + await scaleUpModule.scaleUp(USER_REPO_TEST_DATA); + expect(createRunner).not.toHaveBeenCalled(); + }); + + it('create SSM parameter for runner group id if it does not exist', async () => { + mockSSMgetParameter.mockImplementation(async () => { + throw new Error('ParameterNotFound'); + }); + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); + expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 2); + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`, + Value: '1', + Type: 'String', + }); + }); + + it('Does not create SSM parameter for runner group id if it exists', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.paginate).toHaveBeenCalledTimes(0); + expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 1); + }); + + it('create start runner config for ephemeral runners ', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toBeCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + name: 'unit-test-i-12345', + runner_group_id: 1, + labels: ['label1', 'label2'], + }); + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: 'TEST_JIT_CONFIG_ORG', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('create start runner config for non-ephemeral runners ', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalled(); + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: + '--url https://companyname.ghe.com/Codertocat --token 1234abcd ' + + '--labels label1,label2 --runnergroup Default', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + it.each(RUNNER_TYPES)( + 'calls create start runner config of 40' + ' instances (ssm rate limit condition) to test time delay ', + async (type: RunnerType) => { + process.env.ENABLE_EPHEMERAL_RUNNERS = type === 'ephemeral' ? 'true' : 'false'; + process.env.RUNNERS_MAXIMUM_COUNT = '40'; + mockCreateRunner.mockImplementation(async () => { + return instances; + }); + mockListRunners.mockImplementation(async () => { + return []; + }); + const startTime = performance.now(); + const instances = [ + 'i-1234', + 'i-5678', + 'i-5567', + 'i-5569', + 'i-5561', + 'i-5560', + 'i-5566', + 'i-5536', + 'i-5526', + 'i-5516', + 'i-122', + 'i-123', + 'i-124', + 'i-125', + 'i-126', + 'i-127', + 'i-128', + 'i-129', + 'i-130', + 'i-131', + 'i-132', + 'i-133', + 'i-134', + 'i-135', + 'i-136', + 'i-137', + 'i-138', + 'i-139', + 'i-140', + 'i-141', + 'i-142', + 'i-143', + 'i-144', + 'i-145', + 'i-146', + 'i-147', + 'i-148', + 'i-149', + 'i-150', + 'i-151', + ]; + await scaleUpModule.scaleUp(TEST_DATA); + const endTime = performance.now(); + expect(endTime - startTime).toBeGreaterThan(1000); + expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 40); + }, + 10000, + ); + }); + describe('on repo level', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + process.env.RUNNER_NAME_PREFIX = 'unit-test'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + expectedRunnerParams.runnerType = 'Repo'; + expectedRunnerParams.runnerOwner = `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`; + // `--url https://companyname.ghe.com${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, + // `--token 1234abcd`, + // ]; + }); + + it('gets the current repo level runners', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).toBeCalledWith({ + environment: 'unit-test-environment', + runnerType: 'Repo', + runnerOwner: `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, + }); + }); + + it('does not create a token when maximum runners has been reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a token when maximum runners has not been reached', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('uses the default runner max count', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = undefined; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('creates a runner with correct config and labels', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner and ensure the group argument is ignored', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('Check error is thrown', async () => { + const mockCreateRunners = vi.mocked(createRunner); + mockCreateRunners.mockRejectedValue(new Error('no retry')); + await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow('no retry'); + mockCreateRunners.mockReset(); + }); + }); + + describe('Batch processing', () => { + const createTestMessages = ( + count: number, + overrides: Partial[] = [], + ): ActionRequestMessageSQS[] => { + return Array.from({ length: count }, (_, i) => ({ + ...TEST_DATA_SINGLE, + id: i + 1, + messageId: `message-${i}`, + ...overrides[i], + })); + }; + + beforeEach(() => { + setDefaults(); + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.RUNNERS_MAXIMUM_COUNT = '10'; + }); + + it('Should handle multiple messages for the same organization', async () => { + const messages = createTestMessages(3); + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(1); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 3, + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }), + ); + }); + + it('Should handle multiple messages for different organizations', async () => { + const messages = createTestMessages(3, [ + { repositoryOwner: 'org1' }, + { repositoryOwner: 'org2' }, + { repositoryOwner: 'org1' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(2); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, + runnerOwner: 'org1', + }), + ); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, + runnerOwner: 'org2', + }), + ); + }); + + it('Should handle multiple messages for different repositories when org-level is disabled', async () => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + const messages = createTestMessages(3, [ + { repositoryOwner: 'owner1', repositoryName: 'repo1' }, + { repositoryOwner: 'owner1', repositoryName: 'repo2' }, + { repositoryOwner: 'owner1', repositoryName: 'repo1' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(2); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, + runnerOwner: 'owner1/repo1', + }), + ); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, + runnerOwner: 'owner1/repo2', + }), + ); + }); + + it('Should reject messages when maximum runners limit is reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + mockListRunners.mockImplementation(async () => [ + { + instanceId: 'i-existing', + launchTime: new Date(), + type: 'Org', + owner: TEST_DATA_SINGLE.repositoryOwner, + }, + ]); + + const messages = createTestMessages(5); + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, // 2 max - 1 existing = 1 new + }), + ); + expect(rejectedMessages).toHaveLength(4); // 5 requested - 1 created = 4 rejected + }); + + it('Should handle partial EC2 instance creation failures', async () => { + mockCreateRunner.mockImplementation(async () => ['i-12345']); // Only creates 1 instead of requested 3 + + const messages = createTestMessages(3); + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(rejectedMessages).toHaveLength(2); // 3 requested - 1 created = 2 failed + expect(rejectedMessages).toEqual(['message-0', 'message-1']); + }); + + it('Should filter out invalid event types for ephemeral runners', async () => { + const messages = createTestMessages(3, [ + { eventType: 'workflow_job' }, + { eventType: 'check_run' }, + { eventType: 'workflow_job' }, + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only workflow_job events processed + }), + ); + expect(rejectedMessages).toContain('message-1'); // check_run event rejected + }); + + it('Should skip invalid repo owner types but not reject them', async () => { + const messages = createTestMessages(3, [ + { repoOwnerType: 'Organization' }, + { repoOwnerType: 'User' }, // Invalid for org-level runners + { repoOwnerType: 'Organization' }, + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only Organization events processed + }), + ); + expect(rejectedMessages).not.toContain('message-1'); // User repo not rejected, just skipped + }); + + it('Should skip messages when jobs are not queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { + const isQueued = params.job_id === 1 || params.job_id === 3; // Only jobs 1 and 3 are queued + return { + data: { + status: isQueued ? 'queued' : 'completed', + }, + }; + }); + + const messages = createTestMessages(3); + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only queued jobs processed + }), + ); + }); + + it('Should create separate GitHub clients for different installations', async () => { + mockOctokit.apps.getOrgInstallation.mockImplementation((params) => ({ + data: { + id: params.org === 'org1' ? 100 : 200, + }, + })); + + const messages = createTestMessages(2, [ + { repositoryOwner: 'org1', installationId: 0 }, + { repositoryOwner: 'org2', installationId: 0 }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients + expect(mockedInstallationAuth).toHaveBeenCalledWith(100, ''); + expect(mockedInstallationAuth).toHaveBeenCalledWith(200, ''); + }); + + it('Should reuse GitHub clients for same installation', async () => { + const messages = createTestMessages(3, [ + { repositoryOwner: 'same-org' }, + { repositoryOwner: 'same-org' }, + { repositoryOwner: 'same-org' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(mockCreateClient).toHaveBeenCalledTimes(2); // 1 app client, 1 installation client + expect(mockedInstallationAuth).toHaveBeenCalledTimes(1); + }); + + it('Should return empty array when no valid messages to process', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + const messages = createTestMessages(2, [ + { eventType: 'check_run' }, // Invalid for ephemeral + { eventType: 'check_run' }, // Invalid for ephemeral + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).not.toHaveBeenCalled(); + expect(rejectedMessages).toEqual(['message-0', 'message-1']); + }); + + it('Should handle unlimited runners configuration', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '-1'; + const messages = createTestMessages(10); + + await scaleUpModule.scaleUp(messages); + + expect(listEC2Runners).not.toHaveBeenCalled(); // No need to check current runners + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 10, // All messages processed + }), + ); + }); + }); +}); + +describe('Retry mechanism tests', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'true'; + process.env.RUNNERS_MAXIMUM_COUNT = '10'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + mockSSMClient.reset(); + }); + + const createTestMessages = ( + count: number, + overrides: Partial[] = [], + ): ActionRequestMessageSQS[] => { + return Array.from({ length: count }, (_, i) => ({ + ...TEST_DATA_SINGLE, + id: i + 1, + messageId: `message-${i + 1}`, + ...overrides[i], + })); + }; + + it('calls publishRetryMessage for each valid message when job is queued', async () => { + const messages = createTestMessages(3); + mockCreateRunner.mockResolvedValue(['i-12345', 'i-67890', 'i-abcdef']); // Create all requested runners + + await scaleUpModule.scaleUp(messages); + + expect(mockPublishRetryMessage).toHaveBeenCalledTimes(3); + expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + id: 1, + messageId: 'message-1', + }), + ); + expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + id: 2, + messageId: 'message-2', + }), + ); + expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( + 3, + expect.objectContaining({ + id: 3, + messageId: 'message-3', + }), + ); + }); + + it('does not call publishRetryMessage when job is not queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { + const isQueued = params.job_id === 1; // Only job 1 is queued + return { + data: { + status: isQueued ? 'queued' : 'completed', + }, + }; + }); + + const messages = createTestMessages(3); + + await scaleUpModule.scaleUp(messages); + + // Only message with id 1 should trigger retry + expect(mockPublishRetryMessage).toHaveBeenCalledTimes(1); + expect(mockPublishRetryMessage).toHaveBeenCalledWith( + expect.objectContaining({ + id: 1, + messageId: 'message-1', + }), + ); + }); + + it('does not call publishRetryMessage when maximum runners is reached and messages are marked invalid', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '0'; // No runners can be created + + const messages = createTestMessages(2); + + await scaleUpModule.scaleUp(messages); + + // Verify listEC2Runners is called to check current runner count + expect(listEC2Runners).toHaveBeenCalledWith({ + environment: 'unit-test-environment', + runnerType: 'Org', + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }); + + // publishRetryMessage should NOT be called because messages are marked as invalid + // Invalid messages go back to the SQS queue and will be retried there + expect(mockPublishRetryMessage).not.toHaveBeenCalled(); + expect(createRunner).not.toHaveBeenCalled(); + }); + + it('calls publishRetryMessage with correct message structure including retry counter', async () => { + const message = { + ...TEST_DATA_SINGLE, + messageId: 'test-message-id', + retryCounter: 2, + }; + + await scaleUpModule.scaleUp([message]); + + expect(mockPublishRetryMessage).toHaveBeenCalledWith( + expect.objectContaining({ + id: message.id, + messageId: 'test-message-id', + retryCounter: 2, + }), + ); + }); + + it('calls publishRetryMessage when ENABLE_JOB_QUEUED_CHECK is false', async () => { + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + mockCreateRunner.mockResolvedValue(['i-12345', 'i-67890']); // Create all requested runners + + const messages = createTestMessages(2); + + await scaleUpModule.scaleUp(messages); + + // Should always call publishRetryMessage when queue check is disabled + expect(mockPublishRetryMessage).toHaveBeenCalledTimes(2); + expect(mockOctokit.actions.getJobForWorkflowRun).not.toHaveBeenCalled(); + }); + + it('calls publishRetryMessage for each message in a multi-runner scenario', async () => { + mockCreateRunner.mockResolvedValue(['i-12345', 'i-67890', 'i-abcdef', 'i-11111', 'i-22222']); // Create all requested runners + const messages = createTestMessages(5); + + await scaleUpModule.scaleUp(messages); + + expect(mockPublishRetryMessage).toHaveBeenCalledTimes(5); + messages.forEach((msg, index) => { + expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( + index + 1, + expect.objectContaining({ + id: msg.id, + messageId: msg.messageId, + }), + ); + }); + }); + + it('calls publishRetryMessage after runner creation', async () => { + const messages = createTestMessages(1); + mockCreateRunner.mockResolvedValue(['i-12345']); // Create the requested runner + + const callOrder: string[] = []; + mockPublishRetryMessage.mockImplementation(() => { + callOrder.push('publishRetryMessage'); + return Promise.resolve(); + }); + mockCreateRunner.mockImplementation(async () => { + callOrder.push('createRunner'); + return ['i-12345']; + }); + + await scaleUpModule.scaleUp(messages); + + expect(callOrder).toEqual(['createRunner', 'publishRetryMessage']); + }); +}); + +describe('useDedicatedHost', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.RUNNER_NAME_PREFIX = 'unit-test-'; + process.env.RUNNER_GROUP_NAME = 'Default'; + process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; + process.env.RUNNER_LABELS = 'label1,label2'; + }); + + it('defaults to false when USE_DEDICATED_HOST env var is not set', async () => { + delete process.env.USE_DEDICATED_HOST; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toHaveBeenCalledWith(expect.objectContaining({ useDedicatedHost: false })); + }); + + it('is true when USE_DEDICATED_HOST is "true"', async () => { + process.env.USE_DEDICATED_HOST = 'true'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toHaveBeenCalledWith(expect.objectContaining({ useDedicatedHost: true })); + }); + + it('is false when USE_DEDICATED_HOST is "false"', async () => { + process.env.USE_DEDICATED_HOST = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toHaveBeenCalledWith(expect.objectContaining({ useDedicatedHost: false })); + }); +}); + +function defaultOctokitMockImpl() { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ + data: { + status: 'queued', + }, + })); + mockOctokit.paginate.mockImplementation(() => [ + { + id: 1, + name: 'Default', + }, + ]); + mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ labels }: { labels: string[] }) => ({ + data: { + runner: { id: 9876543210, labels: labels.map((name: string) => ({ name })) }, + encoded_jit_config: 'TEST_JIT_CONFIG_ORG', + }, + })); + mockOctokit.actions.generateRunnerJitconfigForRepo.mockImplementation(({ labels }: { labels: string[] }) => ({ + data: { + runner: { id: 9876543210, labels: labels.map((name: string) => ({ name })) }, + encoded_jit_config: 'TEST_JIT_CONFIG_REPO', + }, + })); + mockOctokit.checks.get.mockImplementation(() => ({ + data: { + status: 'queued', + }, + })); + + const mockTokenReturnValue = { + data: { + token: '1234abcd', + }, + }; + const mockInstallationIdReturnValueOrgs = { + data: { + id: TEST_DATA_SINGLE.installationId, + }, + }; + const mockInstallationIdReturnValueRepos = { + data: { + id: TEST_DATA_SINGLE.installationId, + }, + }; + + mockOctokit.actions.createRegistrationTokenForOrg.mockImplementation(() => mockTokenReturnValue); + mockOctokit.actions.createRegistrationTokenForRepo.mockImplementation(() => mockTokenReturnValue); + mockOctokit.apps.getOrgInstallation.mockImplementation(() => mockInstallationIdReturnValueOrgs); + mockOctokit.apps.getRepoInstallation.mockImplementation(() => mockInstallationIdReturnValueRepos); +} + +function defaultSSMGetParameterMockImpl() { + mockSSMgetParameter.mockImplementation(async (name: string) => { + if (name === `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`) { + return '1'; + } else if (name === `${process.env.PARAMETER_GITHUB_APP_ID_NAME}`) { + return `${process.env.GITHUB_APP_ID}`; + } else { + throw new Error(`ParameterNotFound: ${name}`); + } + }); +} diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts new file mode 100644 index 0000000000..e6a62e958e --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts @@ -0,0 +1,94 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import yn from 'yn'; + +import { listEC2Runners } from '../aws/ec2-runners'; +import type { Ec2OverrideConfig } from './../aws/ec2-runners.d'; +import { + getDefaultBlockDeviceNameFromLaunchTemplate, + parseEc2OverrideConfig, + shouldLoadLaunchTemplateBlockDeviceName, +} from './ec2-labels'; +import { createRunners } from './ec2'; +import type { CreateEC2RunnerConfig } from './ec2'; +import type { ScaleUpRunnerProvider, ScaleUpRunnerProviderStrategy } from './scale-up-provider'; + +const logger = createChildLogger('ec2-scale-up'); + +type Ec2ScaleUpProviderConfig = Omit; + +interface Ec2ScaleUpState { + ec2OverrideConfig?: Ec2OverrideConfig; +} + +export function createEc2ScaleUpProvider(config: Ec2ScaleUpProviderConfig): ScaleUpRunnerProvider { + return { + type: 'ec2', + prepareGroup: async (messageLabels) => { + const trimmedLabels = messageLabels.map((label) => label.trim()); + const dynamicEC2Labels = trimmedLabels.filter((label) => label.startsWith('ghr-ec2-')); + const nonEc2DynamicLabels = trimmedLabels.filter( + (label) => label.startsWith('ghr-') && !label.startsWith('ghr-ec2-'), + ); + const runnerLabels = [...nonEc2DynamicLabels, ...dynamicEC2Labels]; + let ec2OverrideConfig: Ec2OverrideConfig | undefined; + + if (dynamicEC2Labels.length > 0) { + const defaultBlockDeviceName = shouldLoadLaunchTemplateBlockDeviceName(dynamicEC2Labels) + ? await getDefaultBlockDeviceNameFromLaunchTemplate(config.launchTemplateName) + : undefined; + + ec2OverrideConfig = parseEc2OverrideConfig(dynamicEC2Labels, defaultBlockDeviceName); + if (ec2OverrideConfig) { + logger.debug('EC2 override config parsed from labels', { ec2OverrideConfig }); + } + } + + return { runnerLabels, state: { ec2OverrideConfig } }; + }, + getCurrentRunners: async (_state, { runnerType, runnerOwner }) => + (await listEC2Runners({ environment: config.environment, runnerType, runnerOwner })).length, + createRunners: async ({ githubRunnerConfig, numberOfRunners, githubInstallationClient, state }) => { + const ec2State = state as Ec2ScaleUpState; + + return await createRunners( + githubRunnerConfig, + { + ...config, + ec2OverrideConfig: ec2State.ec2OverrideConfig, + }, + numberOfRunners, + githubInstallationClient, + 'scale-up-lambda', + ); + }, + }; +} + +export function createEc2ScaleUpProviderFromEnv(environment: string, scaleErrors: string[]): ScaleUpRunnerProvider { + return createEc2ScaleUpProvider({ + ec2instanceCriteria: { + instanceTypes: process.env.INSTANCE_TYPES.split(','), + instanceTypePriorities: process.env.INSTANCE_TYPE_PRIORITIES + ? (JSON.parse(process.env.INSTANCE_TYPE_PRIORITIES) as Record) + : undefined, + targetCapacityType: process.env.INSTANCE_TARGET_CAPACITY_TYPE, + maxSpotPrice: process.env.INSTANCE_MAX_SPOT_PRICE, + instanceAllocationStrategy: process.env.INSTANCE_ALLOCATION_STRATEGY || 'lowest-price', + }, + environment, + launchTemplateName: process.env.LAUNCH_TEMPLATE_NAME, + subnets: process.env.SUBNET_IDS.split(','), + amiIdSsmParameterName: process.env.AMI_ID_SSM_PARAMETER_NAME, + tracingEnabled: yn(process.env.POWERTOOLS_TRACE_ENABLED, { default: false }), + onDemandFailoverOnError: process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS + ? (JSON.parse(process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS) as [string]) + : [], + scaleErrors, + useDedicatedHost: yn(process.env.USE_DEDICATED_HOST, { default: false }), + }); +} + +export const ec2ScaleUpRunnerProviderStrategy: ScaleUpRunnerProviderStrategy = { + type: 'ec2', + createFromEnv: ({ environment, scaleErrors }) => createEc2ScaleUpProviderFromEnv(environment, scaleErrors), +}; diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2.ts b/lambdas/functions/control-plane/src/scale-runners/ec2.ts new file mode 100644 index 0000000000..b970b77b2f --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/ec2.ts @@ -0,0 +1,129 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { Octokit } from '@octokit/rest'; +import type { Tag } from '@aws-sdk/client-ec2'; + +import { createRunner, tag, terminateRunner } from '../aws/ec2-runners'; +import type { RunnerInputParameters } from '../aws/ec2-runners.d'; +import { createStartRunnerConfig } from './github-runner'; +import type { GitHubRunnerMetadata, StartRunnerConfigOptions } from './github-runner'; +import type { CreateGitHubRunnerConfig, LambdaRunnerSource } from './types'; + +const logger = createChildLogger('ec2-scale-up'); +const RUNNER_LABELS_TAG_KEY = 'ghr:runner_labels'; +const RUNNER_LABELS_TAG_VALUE_SEPARATOR = ','; +export const EC2_TAG_VALUE_MAX_LENGTH = 256; +export const RUNNER_LABELS_TAG_MAX_COUNT = 5; + +export interface CreateEC2RunnerConfig { + environment: string; + subnets: string[]; + launchTemplateName: string; + ec2instanceCriteria: RunnerInputParameters['ec2instanceCriteria']; + ec2OverrideConfig?: RunnerInputParameters['ec2OverrideConfig']; + numberOfRunners?: number; + amiIdSsmParameterName?: string; + tracingEnabled?: boolean; + onDemandFailoverOnError?: string[]; + scaleErrors: string[]; + useDedicatedHost?: boolean; +} + +export async function createRunners( + githubRunnerConfig: CreateGitHubRunnerConfig, + ec2RunnerConfig: CreateEC2RunnerConfig, + numberOfRunners: number, + ghClient: Octokit, + source: LambdaRunnerSource = 'scale-up-lambda', +): Promise { + const instances = await createRunner({ + runnerType: githubRunnerConfig.runnerType, + runnerOwner: githubRunnerConfig.runnerOwner, + numberOfRunners, + source, + ...ec2RunnerConfig, + }); + if (instances.length !== 0) { + const failedInstances = await createStartRunnerConfig( + githubRunnerConfig, + instances, + ghClient, + createEc2StartRunnerConfigOptions(), + ); + + // Terminate instances that failed to get configured to avoid waste + if (failedInstances.length > 0) { + logger.warn('Terminating instances that failed to get configured', { + failedInstances, + failedCount: failedInstances.length, + }); + + for (const instanceId of failedInstances) { + try { + await terminateRunner(instanceId); + } catch (error) { + logger.error('Failed to terminate instance', { + instanceId, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + // Remove failed instances from the returned list + return instances.filter((id) => !failedInstances.includes(id)); + } + } + + return instances; +} + +function createEc2StartRunnerConfigOptions(): StartRunnerConfigOptions { + return { + getSsmParameterTags: (instanceId) => [{ Key: 'InstanceId', Value: instanceId }], + onJitConfigCreated: async (instanceId, metadata) => await tagEc2RunnerMetadata(instanceId, metadata), + }; +} + +async function tagEc2RunnerMetadata(instanceId: string, metadata: GitHubRunnerMetadata): Promise { + const tags = [ + { Key: 'ghr:github_runner_id', Value: metadata.githubRunnerId }, + ...generateRunnerLabelsTags(metadata.runnerLabels), + ]; + + try { + await tag(instanceId, tags); + } catch (e) { + logger.error(`Failed to mark EC2 runner '${instanceId}' with GitHub runner metadata.`, { error: e }); + } +} + +function generateRunnerLabelsTags(labels: string[]): Tag[] { + if (labels.length === 0) { + return []; + } + + const generatedTagValues = packRunnerLabelsTagValues(labels); + const tagValues = generatedTagValues.slice(0, RUNNER_LABELS_TAG_MAX_COUNT); + + if (generatedTagValues.length > RUNNER_LABELS_TAG_MAX_COUNT) { + logger.warn('GitHub runner label EC2 tags were truncated to avoid exceeding EC2 tag limits.', { + maxRunnerLabelsTagCount: RUNNER_LABELS_TAG_MAX_COUNT, + }); + } + + return tagValues.map((value, index) => ({ + Key: index === 0 ? RUNNER_LABELS_TAG_KEY : `${RUNNER_LABELS_TAG_KEY}:${index + 1}`, + Value: value, + })); +} + +function packRunnerLabelsTagValues(labels: string[]): string[] { + const runnerLabelsValue = labels.join(RUNNER_LABELS_TAG_VALUE_SEPARATOR); + const characters = Array.from(runnerLabelsValue); + const tagValues: string[] = []; + + for (let start = 0; start < characters.length; start += EC2_TAG_VALUE_MAX_LENGTH) { + tagValues.push(characters.slice(start, start + EC2_TAG_VALUE_MAX_LENGTH).join('')); + } + + return tagValues; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts new file mode 100644 index 0000000000..773da3a269 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts @@ -0,0 +1,371 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util'; +import { Octokit } from '@octokit/rest'; + +import { metricGitHubAppRateLimit } from '../github/rate-limit'; +import { ActionRequestMessage, CreateGitHubRunnerConfig, EphemeralRunnerConfig, RunnerGroup } from './types'; + +const logger = createChildLogger('github-runner'); + +export interface GitHubRunnerMetadata { + githubRunnerId: string; + runnerLabels: string[]; +} + +export interface StartRunnerConfigOptions { + getSsmParameterTags?: (runnerId: string) => { Key: string; Value: string }[]; + onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise; +} + +function generateRunnerServiceConfig(githubRunnerConfig: CreateGitHubRunnerConfig, token: string) { + const config = [ + `--url ${githubRunnerConfig.ghesBaseUrl ?? 'https://github.com'}/${githubRunnerConfig.runnerOwner}`, + `--token ${token}`, + ]; + + if (githubRunnerConfig.runnerLabels) { + config.push(`--labels ${quoteRunnerLabelsForShell(githubRunnerConfig.runnerLabels)}`.trim()); + } + + if (githubRunnerConfig.disableAutoUpdate) { + config.push('--disableupdate'); + } + + if (githubRunnerConfig.runnerType === 'Org' && githubRunnerConfig.runnerGroup !== undefined) { + config.push(`--runnergroup ${githubRunnerConfig.runnerGroup}`); + } + + if (githubRunnerConfig.ephemeral) { + config.push(`--ephemeral`); + } + + return config; +} + +function quoteRunnerLabelsForShell(labels: string): string { + return /[\s;&|<>()$`"'*?[\\\]{}!]/.test(labels) ? quoteShellArg(labels) : labels; +} + +function quoteShellArg(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'`; +} + +export function validateSsmParameterStoreTags(tagsJson: string): { Key: string; Value: string }[] { + try { + const tags = JSON.parse(tagsJson); + + if (!Array.isArray(tags)) { + throw new Error('Tags must be an array'); + } + + if (tags.length === 0) { + return []; + } + + tags.forEach((tag, index) => { + if (typeof tag !== 'object' || tag === null) { + throw new Error(`Tag at index ${index} must be an object`); + } + if (!tag.Key || typeof tag.Key !== 'string' || tag.Key.trim() === '') { + throw new Error(`Tag at index ${index} has missing or invalid 'Key' property`); + } + if (!Object.prototype.hasOwnProperty.call(tag, 'Value') || typeof tag.Value !== 'string') { + throw new Error(`Tag at index ${index} has missing or invalid 'Value' property`); + } + }); + + return tags; + } catch (err) { + logger.error('Invalid SSM_PARAMETER_STORE_TAGS format', { error: err }); + throw new Error(`Failed to parse SSM_PARAMETER_STORE_TAGS: ${(err as Error).message}`); + } +} + +async function getGithubRunnerRegistrationToken(githubRunnerConfig: CreateGitHubRunnerConfig, ghClient: Octokit) { + const registrationToken = + githubRunnerConfig.runnerType === 'Org' + ? await ghClient.actions.createRegistrationTokenForOrg({ org: githubRunnerConfig.runnerOwner }) + : await ghClient.actions.createRegistrationTokenForRepo({ + owner: githubRunnerConfig.runnerOwner.split('/')[0], + repo: githubRunnerConfig.runnerOwner.split('/')[1], + }); + + return registrationToken.data.token; +} + +function removeTokenFromLogging(config: string[]): string[] { + const result: string[] = []; + config.forEach((e) => { + if (e.startsWith('--token')) { + result.push('--token '); + } else { + result.push(e); + } + }); + return result; +} + +export async function resolveInstallationId( + githubAppClient: Octokit, + enableOrgLevel: boolean, + payload: ActionRequestMessage, +): Promise { + return enableOrgLevel + ? ( + await githubAppClient.apps.getOrgInstallation({ + org: payload.repositoryOwner, + }) + ).data.id + : ( + await githubAppClient.apps.getRepoInstallation({ + owner: payload.repositoryOwner, + repo: payload.repositoryName, + }) + ).data.id; +} + +export async function getInstallationId( + githubAppClient: Octokit, + enableOrgLevel: boolean, + payload: ActionRequestMessage, +): Promise { + if (payload.installationId !== 0) { + return payload.installationId; + } + + return resolveInstallationId(githubAppClient, enableOrgLevel, payload); +} + +export async function isJobQueued(githubInstallationClient: Octokit, payload: ActionRequestMessage): Promise { + let isQueued = false; + if (payload.eventType === 'workflow_job') { + const jobForWorkflowRun = await githubInstallationClient.actions.getJobForWorkflowRun({ + job_id: payload.id, + owner: payload.repositoryOwner, + repo: payload.repositoryName, + }); + metricGitHubAppRateLimit(jobForWorkflowRun.headers); + isQueued = jobForWorkflowRun.data.status === 'queued'; + logger.debug(`The job ${payload.id} is${isQueued ? ' ' : 'not'} queued`); + } else { + throw Error(`Event ${payload.eventType} is not supported`); + } + return isQueued; +} + +export async function getRunnerGroupId( + githubRunnerConfig: CreateGitHubRunnerConfig, + ghClient: Octokit, +): Promise { + // if the runnerType is Repo, then runnerGroupId is default to 1 + let runnerGroupId: number | undefined = 1; + if (githubRunnerConfig.runnerType === 'Org' && githubRunnerConfig.runnerGroup !== undefined) { + let runnerGroup: string | undefined; + // check if runner group id is already stored in SSM Parameter Store and + // use it if it exists to avoid API call to GitHub + try { + runnerGroup = await getParameter( + `${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`, + ); + } catch (err) { + logger.debug('Handling error:', err as Error); + logger.warn( + `SSM Parameter "${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}" + for Runner group ${githubRunnerConfig.runnerGroup} does not exist`, + ); + } + if (runnerGroup === undefined) { + // get runner group id from GitHub + runnerGroupId = await getRunnerGroupByName(ghClient, githubRunnerConfig); + // store runner group id in SSM + try { + await putParameter( + `${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`, + runnerGroupId.toString(), + false, + { + tags: githubRunnerConfig.ssmParameterStoreTags, + }, + ); + } catch (err) { + logger.debug('Error storing runner group id in SSM Parameter Store', err as Error); + throw err; + } + } else { + runnerGroupId = parseInt(runnerGroup); + } + } + return runnerGroupId; +} + +async function getRunnerGroupByName(ghClient: Octokit, githubRunnerConfig: CreateGitHubRunnerConfig): Promise { + const runnerGroups: RunnerGroup[] = await ghClient.paginate(`GET /orgs/{org}/actions/runner-groups`, { + org: githubRunnerConfig.runnerOwner, + per_page: 100, + }); + const runnerGroupId = runnerGroups.find((runnerGroup) => runnerGroup.name === githubRunnerConfig.runnerGroup)?.id; + + if (runnerGroupId === undefined) { + throw new Error(`Runner group ${githubRunnerConfig.runnerGroup} does not exist`); + } + + return runnerGroupId; +} + +/** + * Creates the start configuration for runner targets by either generating JIT configs + * or registration tokens. + * + * @returns Array of runner IDs that failed to get configured + */ +export async function createStartRunnerConfig( + githubRunnerConfig: CreateGitHubRunnerConfig, + runnerIds: string[], + ghClient: Octokit, + options: StartRunnerConfigOptions = {}, +): Promise { + if (githubRunnerConfig.enableJitConfig && githubRunnerConfig.ephemeral) { + return await createJitConfig(githubRunnerConfig, runnerIds, ghClient, options); + } else { + return await createRegistrationTokenConfig(githubRunnerConfig, runnerIds, ghClient, options); + } +} + +function addDelay(runnerIds: string[]) { + const delay = async (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + const ssmParameterStoreMaxThroughput = 40; + const isDelay = runnerIds.length >= ssmParameterStoreMaxThroughput; + return { isDelay, delay }; +} + +/** + * Creates registration token configuration for non-ephemeral runners. + * + * @returns Empty array (this configuration method does not have failure cases) + */ +async function createRegistrationTokenConfig( + githubRunnerConfig: CreateGitHubRunnerConfig, + runnerIds: string[], + ghClient: Octokit, + options: StartRunnerConfigOptions, +): Promise { + const { isDelay, delay } = addDelay(runnerIds); + const token = await getGithubRunnerRegistrationToken(githubRunnerConfig, ghClient); + const runnerServiceConfig = generateRunnerServiceConfig(githubRunnerConfig, token); + + logger.debug('Runner service config for non-ephemeral runners', { + runner_service_config: removeTokenFromLogging(runnerServiceConfig), + }); + + for (const runnerId of runnerIds) { + await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerServiceConfig.join(' '), true, { + tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags], + }); + if (isDelay) { + // Delay to prevent AWS ssm rate limits by being within the max throughput limit + await delay(25); + } + } + + return []; +} + +/** + * Creates JIT (Just-In-Time) configuration for ephemeral runners. + * Continues processing remaining runners even if some fail. + * + * @returns Array of runner IDs that failed to get JIT configuration + */ +async function createJitConfig( + githubRunnerConfig: CreateGitHubRunnerConfig, + runnerIds: string[], + ghClient: Octokit, + options: StartRunnerConfigOptions, +): Promise { + const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient); + const { isDelay, delay } = addDelay(runnerIds); + const runnerLabels = githubRunnerConfig.runnerLabels.split(','); + const failedRunnerIds: string[] = []; + + logger.debug(`Runner group id: ${runnerGroupId}`); + logger.debug(`Runner labels: ${runnerLabels}`); + for (const runnerId of runnerIds) { + try { + // generate jit config for runner registration + const ephemeralRunnerConfig: EphemeralRunnerConfig = { + runnerName: `${githubRunnerConfig.runnerNamePrefix}${runnerId}`, + runnerGroupId: runnerGroupId, + runnerLabels: runnerLabels, + }; + logger.debug(`Runner name: ${ephemeralRunnerConfig.runnerName}`); + const runnerConfig = + githubRunnerConfig.runnerType === 'Org' + ? await ghClient.actions.generateRunnerJitconfigForOrg({ + org: githubRunnerConfig.runnerOwner, + name: ephemeralRunnerConfig.runnerName, + runner_group_id: ephemeralRunnerConfig.runnerGroupId, + labels: ephemeralRunnerConfig.runnerLabels, + }) + : await ghClient.actions.generateRunnerJitconfigForRepo({ + owner: githubRunnerConfig.runnerOwner.split('/')[0], + repo: githubRunnerConfig.runnerOwner.split('/')[1], + name: ephemeralRunnerConfig.runnerName, + runner_group_id: ephemeralRunnerConfig.runnerGroupId, + labels: ephemeralRunnerConfig.runnerLabels, + }); + + metricGitHubAppRateLimit(runnerConfig.headers); + + await options.onJitConfigCreated?.(runnerId, { + githubRunnerId: runnerConfig.data.runner.id.toString(), + runnerLabels, + }); + + // store jit config in ssm parameter store + logger.debug('Runner JIT config for ephemeral runner generated.', { + runnerId, + }); + await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerConfig.data.encoded_jit_config, true, { + tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags], + }); + if (isDelay) { + // Delay to prevent AWS ssm rate limits by being within the max throughput limit + await delay(25); + } + } catch (error) { + failedRunnerIds.push(runnerId); + logger.warn('Failed to create JIT config for runner, continuing with remaining runners', { + runnerId, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + if (failedRunnerIds.length > 0) { + logger.error('Failed to create JIT config for some runners', { + failedRunnerIds, + totalRunners: runnerIds.length, + successfulRunners: runnerIds.length - failedRunnerIds.length, + }); + } + + return failedRunnerIds; +} + +export function getGitHubEnterpriseApiUrl() { + const ghesBaseUrl = process.env.GHES_URL; + let ghesApiUrl = ''; + if (ghesBaseUrl) { + const url = new URL(ghesBaseUrl); + const domain = url.hostname; + if (domain.endsWith('.ghe.com')) { + // Data residency: Prepend 'api.' + ghesApiUrl = `https://api.${domain}`; + } else { + // GitHub Enterprise Server: Append '/api/v3' + ghesApiUrl = `${ghesBaseUrl}/api/v3`; + } + } + logger.debug(`Github Enterprise URLs: api_url - ${ghesApiUrl}; base_url - ${ghesBaseUrl}`); + return { ghesApiUrl, ghesBaseUrl }; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts b/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts index f807d06d8a..c4ff1e5d76 100644 --- a/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts @@ -1,6 +1,6 @@ import { publishMessage } from '../aws/sqs'; import { publishRetryMessage, checkAndRetryJob } from './job-retry'; -import { ActionRequestMessage, ActionRequestMessageRetry } from './scale-up'; +import type { ActionRequestMessage, ActionRequestMessageRetry } from './types'; import { getOctokit } from '../github/octokit'; import { jobRetryCheck } from '../lambda'; import { Octokit } from '@octokit/rest'; diff --git a/lambdas/functions/control-plane/src/scale-runners/job-retry.ts b/lambdas/functions/control-plane/src/scale-runners/job-retry.ts index bd9ebbd3b9..8f7d6e2289 100644 --- a/lambdas/functions/control-plane/src/scale-runners/job-retry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/job-retry.ts @@ -1,6 +1,7 @@ import { addPersistentContextToChildLogger, createSingleMetric, logger } from '@aws-github-runner/aws-powertools-util'; import { publishMessage } from '../aws/sqs'; -import { ActionRequestMessage, ActionRequestMessageRetry, isJobQueued, getGitHubEnterpriseApiUrl } from './scale-up'; +import { getGitHubEnterpriseApiUrl, isJobQueued } from './github-runner'; +import type { ActionRequestMessage, ActionRequestMessageRetry } from './types'; import { getOctokit } from '../github/octokit'; import { MetricUnit } from '@aws-lambda-powertools/metrics'; import yn from 'yn'; diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-config.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-config.test.ts new file mode 100644 index 0000000000..83fdb914b1 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-config.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from 'vitest'; + +import { getScaleUpRunnerProviderType } from './scale-up-config'; + +describe('getScaleUpRunnerProviderType', () => { + it('defaults to ec2 when no type is defined', () => { + expect(getScaleUpRunnerProviderType(undefined, 'ec2')).toEqual('ec2'); + }); + + it('uses configured ec2 type', () => { + expect(getScaleUpRunnerProviderType('ec2', 'microvm')).toEqual('ec2'); + }); +}); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts new file mode 100644 index 0000000000..58fe22b646 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts @@ -0,0 +1,8 @@ +import type { ScaleUpRunnerProviderType } from './scale-up-provider'; + +export function getScaleUpRunnerProviderType( + type: string | undefined, + defaultType: ScaleUpRunnerProviderType, +): ScaleUpRunnerProviderType { + return type ?? defaultType; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts new file mode 100644 index 0000000000..1931e0f383 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts @@ -0,0 +1,26 @@ +import { ec2ScaleUpRunnerProviderStrategy } from './ec2-scale-up'; +import type { + ScaleUpRunnerProvider, + ScaleUpRunnerProviderStrategy, + ScaleUpRunnerProviderType, +} from './scale-up-provider'; + +const scaleUpRunnerProviderStrategies: ScaleUpRunnerProviderStrategy[] = [ec2ScaleUpRunnerProviderStrategy]; + +export function getDefaultScaleUpRunnerProviderType(): ScaleUpRunnerProviderType { + return scaleUpRunnerProviderStrategies[0].type; +} + +export function createScaleUpRunnerProviderFromEnv( + type: ScaleUpRunnerProviderType, + environment: string, + scaleErrors: string[], +): ScaleUpRunnerProvider { + const strategy = scaleUpRunnerProviderStrategies.find((strategy) => strategy.type === type); + + if (!strategy) { + throw new Error(`Unsupported scale-up runner provider type: ${type}`); + } + + return strategy.createFromEnv({ environment, scaleErrors }); +} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts new file mode 100644 index 0000000000..4442f0e6cc --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts @@ -0,0 +1,40 @@ +import type { Octokit } from '@octokit/rest'; + +import type { ActionRequestMessageSQS, CreateGitHubRunnerConfig, GitHubRunnerType } from './types'; + +export interface CurrentRunnersInput { + runnerType: GitHubRunnerType; + runnerOwner: string; +} + +export interface CreateScaleUpRunnersInput { + githubRunnerConfig: CreateGitHubRunnerConfig; + numberOfRunners: number; + githubInstallationClient: Octokit; + messages: ActionRequestMessageSQS[]; + state: unknown; +} + +export interface PreparedScaleUpRunnerGroup { + runnerLabels: string[]; + state: unknown; +} + +export type ScaleUpRunnerProviderType = string; + +export interface ScaleUpRunnerProvider { + type: ScaleUpRunnerProviderType; + prepareGroup(messageLabels: string[]): Promise; + getCurrentRunners(state: unknown, input: CurrentRunnersInput): Promise; + createRunners(input: CreateScaleUpRunnersInput): Promise; +} + +export interface CreateScaleUpRunnerProviderInput { + environment: string; + scaleErrors: string[]; +} + +export interface ScaleUpRunnerProviderStrategy { + type: ScaleUpRunnerProviderType; + createFromEnv(input: CreateScaleUpRunnerProviderInput): ScaleUpRunnerProvider; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index f0a2706c1d..f97f8e8f6c 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -1,29 +1,22 @@ -import { DescribeLaunchTemplateVersionsCommand, EC2Client } from '@aws-sdk/client-ec2'; -import { PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; -import { mockClient } from 'aws-sdk-client-mock'; -import 'aws-sdk-client-mock-jest/vitest'; -// Using vi.mocked instead of jest-mock -import nock from 'nock'; -import { performance } from 'perf_hooks'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; import * as ghAuth from '../github/auth'; -import { createRunner, listEC2Runners, tag } from './../aws/runners'; -import { RunnerInputParameters } from './../aws/runners.d'; -import * as scaleUpModule from './scale-up'; -import { getParameter } from '@aws-github-runner/aws-ssm-util'; import { publishRetryMessage } from './job-retry'; -import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { scaleUp } from './scale-up'; +import { createScaleUpRunnerProviderFromEnv, getDefaultScaleUpRunnerProviderType } from './scale-up-provider-registry'; import type { Octokit } from '@octokit/rest'; +import type { ActionRequestMessageSQS } from './types'; + +const testProvider = vi.hoisted(() => ({ + type: 'test-provider', + prepareGroup: vi.fn(), + getCurrentRunners: vi.fn(), + createRunners: vi.fn(), +})); const mockOctokit = { - paginate: vi.fn(), - checks: { get: vi.fn() }, actions: { - createRegistrationTokenForOrg: vi.fn(), - createRegistrationTokenForRepo: vi.fn(), getJobForWorkflowRun: vi.fn(), - generateRunnerJitconfigForOrg: vi.fn(), - generateRunnerJitconfigForRepo: vi.fn(), }, apps: { getOrgInstallation: vi.fn(), @@ -31,3489 +24,195 @@ const mockOctokit = { }, }; -const mockCreateRunner = vi.mocked(createRunner); -const mockListRunners = vi.mocked(listEC2Runners); -const mockTag = vi.mocked(tag); -const mockEC2Client = mockClient(EC2Client); -const mockSSMClient = mockClient(SSMClient); -const mockSSMgetParameter = vi.mocked(getParameter); -const mockPublishRetryMessage = vi.mocked(publishRetryMessage); - -vi.mock('@octokit/rest', () => ({ - Octokit: vi.fn().mockImplementation(function () { - return mockOctokit; - }), -})); - -vi.mock('./../aws/runners', async () => ({ - createRunner: vi.fn(), - listEC2Runners: vi.fn(), - tag: vi.fn(), -})); - -vi.mock('./../github/auth', async () => ({ +vi.mock('../github/auth', async () => ({ createGithubAppAuth: vi.fn(), createGithubInstallationAuth: vi.fn(), createOctokitClient: vi.fn(), })); -vi.mock('@aws-github-runner/aws-ssm-util', async () => { - const actual = (await vi.importActual( - '@aws-github-runner/aws-ssm-util', - )) as typeof import('@aws-github-runner/aws-ssm-util'); - - return { - ...actual, - getParameter: vi.fn(), - }; -}); - vi.mock('./job-retry', () => ({ publishRetryMessage: vi.fn(), - checkAndRetryJob: vi.fn(), })); -export type RunnerType = 'ephemeral' | 'non-ephemeral'; - -// for ephemeral and non-ephemeral runners -const RUNNER_TYPES: RunnerType[] = ['ephemeral', 'non-ephemeral']; - -const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); -const mockedInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); -const mockCreateClient = vi.mocked(ghAuth.createOctokitClient); +vi.mock('./scale-up-provider-registry', () => ({ + createScaleUpRunnerProviderFromEnv: vi.fn(() => testProvider), + getDefaultScaleUpRunnerProviderType: vi.fn(() => 'test-provider'), +})); -const TEST_DATA_SINGLE: scaleUpModule.ActionRequestMessageSQS = { +const cleanEnv = process.env; +const mockedCreateGithubAppAuth = vi.mocked(ghAuth.createGithubAppAuth); +const mockedCreateGithubInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); +const mockedCreateOctokitClient = vi.mocked(ghAuth.createOctokitClient); +const mockedCreateScaleUpRunnerProviderFromEnv = vi.mocked(createScaleUpRunnerProviderFromEnv); +const mockedGetDefaultScaleUpRunnerProviderType = vi.mocked(getDefaultScaleUpRunnerProviderType); +const mockedPublishRetryMessage = vi.mocked(publishRetryMessage); + +const TEST_MESSAGE: ActionRequestMessageSQS = { id: 1, eventType: 'workflow_job', repositoryName: 'hello-world', repositoryOwner: 'Codertocat', installationId: 2, repoOwnerType: 'Organization', - messageId: 'foobar', + messageId: 'message-1', + labels: ['self-hosted', 'ghr-provider-size:large'], }; -const TEST_DATA: scaleUpModule.ActionRequestMessageSQS[] = [ - { - ...TEST_DATA_SINGLE, - messageId: 'foobar', - }, -]; - -const cleanEnv = process.env; - -const EXPECTED_RUNNER_PARAMS: RunnerInputParameters = { - environment: 'unit-test-environment', - runnerType: 'Org', - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - numberOfRunners: 1, - launchTemplateName: 'lt-1', - ec2instanceCriteria: { - instanceTypes: ['m5.large'], - targetCapacityType: 'spot', - instanceAllocationStrategy: 'lowest-price', - }, - subnets: ['subnet-123'], - tracingEnabled: false, - onDemandFailoverOnError: [], - scaleErrors: ['UnfulfillableCapacity', 'MaxSpotInstanceCountExceeded', 'TargetCapacityLimitExceededException'], - source: 'scale-up-lambda', - useDedicatedHost: false, -}; -let expectedRunnerParams: RunnerInputParameters; - function setDefaults() { process.env = { ...cleanEnv }; - process.env.PARAMETER_GITHUB_APP_ID_NAME = 'github-app-id'; process.env.GITHUB_APP_KEY_BASE64 = 'TEST_CERTIFICATE_DATA'; process.env.GITHUB_APP_ID = '1337'; process.env.GITHUB_APP_CLIENT_ID = 'TEST_CLIENT_ID'; process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET'; process.env.RUNNERS_MAXIMUM_COUNT = '3'; - process.env.ENVIRONMENT = EXPECTED_RUNNER_PARAMS.environment; - process.env.LAUNCH_TEMPLATE_NAME = 'lt-1'; - process.env.SUBNET_IDS = 'subnet-123'; - process.env.INSTANCE_TYPES = 'm5.large'; - process.env.INSTANCE_TARGET_CAPACITY_TYPE = 'spot'; - process.env.ENABLE_ON_DEMAND_FAILOVER = undefined; + process.env.ENVIRONMENT = 'unit-test-environment'; + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + process.env.RUNNER_NAME_PREFIX = 'unit-test-'; + process.env.RUNNER_GROUP_NAME = 'Default'; + process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; + process.env.RUNNER_LABELS = 'base-label'; process.env.SCALE_ERRORS = '["UnfulfillableCapacity","MaxSpotInstanceCountExceeded","TargetCapacityLimitExceededException"]'; } beforeEach(() => { - nock.disableNetConnect(); - vi.resetModules(); vi.clearAllMocks(); setDefaults(); - mockEC2Client.reset(); - mockEC2Client.on(DescribeLaunchTemplateVersionsCommand).resolves({ - LaunchTemplateVersions: [ - { - LaunchTemplateData: { - BlockDeviceMappings: [ - { - DeviceName: '/dev/sda1', - Ebs: {}, - }, - ], - }, - }, - ], - }); - - defaultSSMGetParameterMockImpl(); - defaultOctokitMockImpl(); - - mockCreateRunner.mockImplementation(async () => { - return ['i-12345']; + testProvider.prepareGroup.mockResolvedValue({ + runnerLabels: ['ghr-provider-size:large'], + state: { prepared: true }, }); - mockListRunners.mockImplementation(async () => [ - { - instanceId: 'i-1234', - launchTime: new Date(), - type: 'Org', - owner: TEST_DATA_SINGLE.repositoryOwner, - }, - ]); + testProvider.getCurrentRunners.mockResolvedValue(0); + testProvider.createRunners.mockResolvedValue(['runner-1']); - mockedAppAuth.mockResolvedValue({ + mockedCreateGithubAppAuth.mockResolvedValue({ type: 'app', - token: 'token', - appId: TEST_DATA_SINGLE.installationId, + token: 'app-token', + appId: 1, expiresAt: 'some-date', }); - mockedInstallationAuth.mockResolvedValue({ + mockedCreateGithubInstallationAuth.mockResolvedValue({ type: 'token', tokenType: 'installation', - token: 'token', + token: 'installation-token', createdAt: 'some-date', expiresAt: 'some-date', permissions: {}, repositorySelection: 'all', - installationId: 0, - }); - - mockCreateClient.mockResolvedValue(mockOctokit as unknown as Octokit); -}); - -describe('scaleUp with GHES', () => { - beforeEach(() => { - process.env.GHES_URL = 'https://github.enterprise.something'; - }); - - it('checks queued workflows', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalledWith({ - job_id: TEST_DATA_SINGLE.id, - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); + installationId: 2, }); - - it('does not list runners when no workflows are queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ - data: { total_count: 0 }, - })); - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).not.toBeCalled(); + mockedCreateOctokitClient.mockResolvedValue(mockOctokit as unknown as Octokit); + mockOctokit.actions.getJobForWorkflowRun.mockResolvedValue({ + data: { status: 'queued' }, + headers: {}, }); +}); - describe('on org level', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.RUNNER_NAME_PREFIX = 'unit-test-'; - process.env.RUNNER_GROUP_NAME = 'Default'; - process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; - process.env.RUNNER_LABELS = 'label1,label2'; - - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - mockSSMClient.reset(); - }); - - it('gets the current org level runners', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).toBeCalledWith({ - environment: 'unit-test-environment', +describe('scaleUp runner provider orchestration', () => { + it('creates the configured provider and forwards provider state into runner creation', async () => { + process.env.RUNNER_PROVIDER_TYPE = 'test-provider'; + + const rejectedMessages = await scaleUp([TEST_MESSAGE]); + + expect(rejectedMessages).toEqual([]); + expect(mockedGetDefaultScaleUpRunnerProviderType).toHaveBeenCalled(); + expect(mockedCreateScaleUpRunnerProviderFromEnv).toHaveBeenCalledWith('test-provider', 'unit-test-environment', [ + 'UnfulfillableCapacity', + 'MaxSpotInstanceCountExceeded', + 'TargetCapacityLimitExceededException', + ]); + expect(testProvider.prepareGroup).toHaveBeenCalledWith(TEST_MESSAGE.labels); + expect(testProvider.getCurrentRunners).toHaveBeenCalledWith( + { prepared: true }, + { + runnerOwner: TEST_MESSAGE.repositoryOwner, runnerType: 'Org', - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }); - }); - - it('does not create a token when maximum runners has been reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('does not create runners when current runners exceed maximum (race condition)', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '5'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - // Simulate race condition where pool lambda created more runners than max - mockListRunners.mockImplementation(async () => - Array.from({ length: 10 }, (_, i) => ({ - instanceId: `i-${i}`, - launchTime: new Date(), - type: 'Org', - owner: TEST_DATA_SINGLE.repositoryOwner, - })), - ); - await scaleUpModule.scaleUp(TEST_DATA); - // Should not attempt to create runners (would be negative without fix) - expect(createRunner).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - }); - - it('does create a runner if maximum is set to -1', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '-1'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).not.toHaveBeenCalled(); - expect(createRunner).toHaveBeenCalled(); - }); - - it('creates a token when maximum runners has not been reached', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - }); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a runner with correct config', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('tags the EC2 runner with the GitHub runner metadata returned by JIT config', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockTag).toHaveBeenCalledWith('i-12345', [ - { Key: 'ghr:github_runner_id', Value: '9876543210' }, - { Key: 'ghr:runner_labels', Value: 'label1,label2' }, - ]); - }); - - it('chunks comma-joined GitHub runner labels by the EC2 tag value max length', async () => { - const runnerLabels = ['a'.repeat(scaleUpModule.EC2_TAG_VALUE_MAX_LENGTH), 'b'].join(','); - process.env.RUNNER_LABELS = runnerLabels; - - await scaleUpModule.scaleUp(TEST_DATA); - - expect(mockTag).toHaveBeenCalledWith('i-12345', [ - { Key: 'ghr:github_runner_id', Value: '9876543210' }, - { Key: 'ghr:runner_labels', Value: runnerLabels.slice(0, scaleUpModule.EC2_TAG_VALUE_MAX_LENGTH) }, - { - Key: 'ghr:runner_labels:2', - Value: runnerLabels.slice(scaleUpModule.EC2_TAG_VALUE_MAX_LENGTH), - }, - ]); - }); - - it('limits GitHub runner label metadata to five EC2 tags', async () => { - const runnerLabels = Array.from({ length: scaleUpModule.RUNNER_LABELS_TAG_MAX_COUNT + 1 }, (_, index) => - String.fromCharCode('a'.charCodeAt(0) + index).repeat(scaleUpModule.EC2_TAG_VALUE_MAX_LENGTH), - ).join(','); - process.env.RUNNER_LABELS = runnerLabels; - - await scaleUpModule.scaleUp(TEST_DATA); - - expect(mockTag).toHaveBeenCalledWith('i-12345', [ - { Key: 'ghr:github_runner_id', Value: '9876543210' }, - ...Array.from({ length: scaleUpModule.RUNNER_LABELS_TAG_MAX_COUNT }, (_, index) => ({ - Key: index === 0 ? 'ghr:runner_labels' : `ghr:runner_labels:${index + 1}`, - Value: runnerLabels.slice( - index * scaleUpModule.EC2_TAG_VALUE_MAX_LENGTH, - (index + 1) * scaleUpModule.EC2_TAG_VALUE_MAX_LENGTH, - ), - })), - ]); - }); - - it('uses a reserved separator when packing GitHub runner labels into EC2 tags', async () => { - process.env.RUNNER_LABELS = ['label:with/slash', 'label+with+plus'].join(','); - const testDataWithSemicolonDynamicLabel = [ - { - ...TEST_DATA_SINGLE, - labels: ['ghr-ec2-cpu-manufacturers:intel;amd'], - messageId: 'test-semicolon-dynamic-label', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithSemicolonDynamicLabel); - - expect(mockTag).toHaveBeenCalledWith('i-12345', [ - { Key: 'ghr:github_runner_id', Value: '9876543210' }, - { - Key: 'ghr:runner_labels', - Value: 'label:with/slash,label+with+plus,ghr-ec2-cpu-manufacturers:intel;amd', - }, - ]); - }); - - it('creates a runner with labels in a specific group', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner with ami id override from ssm parameter', async () => { - process.env.AMI_ID_SSM_PARAMETER_NAME = 'my-ami-id-param'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith({ ...expectedRunnerParams, amiIdSsmParameterName: 'my-ami-id-param' }); - }); - - it('Throws an error if runner group does not exist for ephemeral runners', async () => { - process.env.RUNNER_GROUP_NAME = 'test-runner-group'; - mockSSMgetParameter.mockImplementation(async () => { - throw new Error('ParameterNotFound'); - }); - await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toBeInstanceOf(Error); - expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); - }); - - it('Discards event if it is a User repo and org level runners is enabled', async () => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - const USER_REPO_TEST_DATA = structuredClone(TEST_DATA); - USER_REPO_TEST_DATA[0].repoOwnerType = 'User'; - await scaleUpModule.scaleUp(USER_REPO_TEST_DATA); - expect(createRunner).not.toHaveBeenCalled(); - }); - - it('create SSM parameter for runner group id if it does not exist', async () => { - mockSSMgetParameter.mockImplementation(async () => { - throw new Error('ParameterNotFound'); - }); - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 2); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`, - Value: '1', - Type: 'String', - }); - }); - - it('Does not create SSM parameter for runner group id if it exists', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.paginate).toHaveBeenCalledTimes(0); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 1); - }); - - it('create start runner config for ephemeral runners ', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '2'; - - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toBeCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - name: 'unit-test-i-12345', - runner_group_id: 1, - labels: ['label1', 'label2'], - }); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: 'TEST_JIT_CONFIG_ORG', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('create start runner config for non-ephemeral runners ', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - process.env.RUNNERS_MAXIMUM_COUNT = '2'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalled(); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: - '--url https://github.enterprise.something/Codertocat --token 1234abcd ' + - '--labels label1,label2 --runnergroup Default', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('quotes runner labels with semicolon separators in non-ephemeral runner config', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - process.env.RUNNERS_MAXIMUM_COUNT = '2'; - - await scaleUpModule.scaleUp([ - { - ...TEST_DATA_SINGLE, - labels: ['ghr-ec2-cpu-manufacturers:intel;amd'], - messageId: 'test-semicolon-labels', - }, - ]); - - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: - '--url https://github.enterprise.something/Codertocat --token 1234abcd ' + - "--labels 'label1,label2,ghr-ec2-cpu-manufacturers:intel;amd' --runnergroup Default", - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('should create JIT config for all remaining instances even when GitHub API fails for one instance', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '5'; - mockCreateRunner.mockImplementation(async () => { - return ['i-instance-1', 'i-instance-2', 'i-instance-3']; - }); - mockListRunners.mockImplementation(async () => { - return []; - }); - - mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ name }) => { - if (name === 'unit-test-i-instance-2') { - // Simulate a 503 Service Unavailable error from GitHub - const error = new Error('Service Unavailable') as Error & { - status: number; - response: { status: number; data: { message: string } }; - }; - error.status = 503; - error.response = { - status: 503, - data: { message: 'Service temporarily unavailable' }, - }; - throw error; - } - return { - data: { - runner: { id: 9876543210 }, - encoded_jit_config: `TEST_JIT_CONFIG_${name}`, - }, - headers: {}, - }; - }); - - await scaleUpModule.scaleUp(TEST_DATA); - - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toHaveBeenCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - name: 'unit-test-i-instance-1', - runner_group_id: 1, - labels: ['label1', 'label2'], - }); - - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toHaveBeenCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - name: 'unit-test-i-instance-2', - runner_group_id: 1, - labels: ['label1', 'label2'], - }); - - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toHaveBeenCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - name: 'unit-test-i-instance-3', - runner_group_id: 1, - labels: ['label1', 'label2'], - }); - - expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-1', - Value: 'TEST_JIT_CONFIG_unit-test-i-instance-1', - Type: 'SecureString', - Tags: [{ Key: 'InstanceId', Value: 'i-instance-1' }], - }); - - expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-3', - Value: 'TEST_JIT_CONFIG_unit-test-i-instance-3', - Type: 'SecureString', - Tags: [{ Key: 'InstanceId', Value: 'i-instance-3' }], - }); - - expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-2', - }); - }); - - it('should handle retryable errors with error handling logic', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '5'; - mockCreateRunner.mockImplementation(async () => { - return ['i-instance-1', 'i-instance-2']; - }); - mockListRunners.mockImplementation(async () => { - return []; - }); - - mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ name }) => { - if (name === 'unit-test-i-instance-1') { - const error = new Error('Internal Server Error') as Error & { - status: number; - response: { status: number; data: { message: string } }; - }; - error.status = 500; - error.response = { - status: 500, - data: { message: 'Internal server error' }, - }; - throw error; - } - return { - data: { - runner: { id: 9876543210 }, - encoded_jit_config: `TEST_JIT_CONFIG_${name}`, - }, - headers: {}, - }; - }); - - await scaleUpModule.scaleUp(TEST_DATA); - - expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-2', - Value: 'TEST_JIT_CONFIG_unit-test-i-instance-2', - Type: 'SecureString', - Tags: [{ Key: 'InstanceId', Value: 'i-instance-2' }], - }); - - expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-1', - }); - }); - - it('should handle non-retryable 4xx errors gracefully', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '5'; - mockCreateRunner.mockImplementation(async () => { - return ['i-instance-1', 'i-instance-2']; - }); - mockListRunners.mockImplementation(async () => { - return []; - }); - - mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ name }) => { - if (name === 'unit-test-i-instance-1') { - // 404 is not retryable - will fail immediately - const error = new Error('Not Found') as Error & { - status: number; - response: { status: number; data: { message: string } }; - }; - error.status = 404; - error.response = { - status: 404, - data: { message: 'Resource not found' }, - }; - throw error; - } - return { - data: { - runner: { id: 9876543210 }, - encoded_jit_config: `TEST_JIT_CONFIG_${name}`, - }, - headers: {}, - }; - }); - - await scaleUpModule.scaleUp(TEST_DATA); - - expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-2', - Value: 'TEST_JIT_CONFIG_unit-test-i-instance-2', - Type: 'SecureString', - Tags: [{ Key: 'InstanceId', Value: 'i-instance-2' }], - }); - - expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-1', - }); - }); - - it.each(RUNNER_TYPES)( - 'calls create start runner config of 40' + ' instances (ssm rate limit condition) to test time delay ', - async (type: RunnerType) => { - process.env.ENABLE_EPHEMERAL_RUNNERS = type === 'ephemeral' ? 'true' : 'false'; - process.env.RUNNERS_MAXIMUM_COUNT = '40'; - mockCreateRunner.mockImplementation(async () => { - return instances; - }); - mockListRunners.mockImplementation(async () => { - return []; - }); - const startTime = performance.now(); - const instances = [ - 'i-1234', - 'i-5678', - 'i-5567', - 'i-5569', - 'i-5561', - 'i-5560', - 'i-5566', - 'i-5536', - 'i-5526', - 'i-5516', - 'i-122', - 'i-123', - 'i-124', - 'i-125', - 'i-126', - 'i-127', - 'i-128', - 'i-129', - 'i-130', - 'i-131', - 'i-132', - 'i-133', - 'i-134', - 'i-135', - 'i-136', - 'i-137', - 'i-138', - 'i-139', - 'i-140', - 'i-141', - 'i-142', - 'i-143', - 'i-144', - 'i-145', - 'i-146', - 'i-147', - 'i-148', - 'i-149', - 'i-150', - 'i-151', - ]; - await scaleUpModule.scaleUp(TEST_DATA); - const endTime = performance.now(); - expect(endTime - startTime).toBeGreaterThan(1000); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 40); }, - 10000, ); - }); - - describe('Dynamic EC2 Configuration', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - process.env.RUNNER_LABELS = 'base-label'; - process.env.INSTANCE_TYPES = 't3.medium,t3.large'; - process.env.RUNNER_NAME_PREFIX = 'unit-test'; - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - mockSSMClient.reset(); - }); - - it('appends EC2 labels to existing runner labels when EC2 labels are present', async () => { - const testDataWithEc2Labels = [ - { - ...TEST_DATA_SINGLE, - labels: ['ghr-ec2-instance-type:c5.2xlarge', 'ghr-ec2-custom:value'], - messageId: 'test-1', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithEc2Labels); - - // Verify createRunner was called with EC2 instance type in override config - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2instanceCriteria: expect.objectContaining({ - instanceTypes: ['t3.medium', 't3.large'], - }), - ec2OverrideConfig: expect.objectContaining({ - InstanceType: 'c5.2xlarge', - }), - }), - ); - expect(mockTag).toHaveBeenCalledWith('i-12345', [ - { - Key: 'ghr:github_runner_id', - Value: '9876543210', - }, - { - Key: 'ghr:runner_labels', - Value: 'base-label,ghr-ec2-instance-type:c5.2xlarge,ghr-ec2-custom:value', - }, - ]); - }); - - it('uses default instance types when no instance type EC2 label is provided', async () => { - const testDataWithEc2Labels = [ - { - ...TEST_DATA_SINGLE, - labels: ['ghr-ec2-custom:value'], - messageId: 'test-3', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithEc2Labels); - - // Should use the default INSTANCE_TYPES from environment - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2instanceCriteria: expect.objectContaining({ - instanceTypes: ['t3.medium', 't3.large'], - }), - }), - ); - }); - - it('loads the launch template block device name for dynamic EBS labels without DeviceName', async () => { - mockEC2Client.on(DescribeLaunchTemplateVersionsCommand).resolves({ - LaunchTemplateVersions: [ - { - LaunchTemplateData: { - BlockDeviceMappings: [ - { - DeviceName: '/dev/sdb', - VirtualName: 'ephemeral0', - }, - { - DeviceName: '/dev/sdf', - Ebs: {}, - }, - ], - }, - }, - ], - }); - - const testDataWithEbsLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['ghr-ec2-ebs-volume-size:100', 'ghr-ec2-ebs-volume-type:gp3'], - messageId: 'test-ebs-device-name', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithEbsLabels); - - expect(mockEC2Client).toHaveReceivedCommandWith(DescribeLaunchTemplateVersionsCommand, { - LaunchTemplateName: 'lt-1', - Versions: ['$Default'], - }); - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - BlockDeviceMappings: [ - { - DeviceName: '/dev/sdf', - Ebs: { - VolumeSize: 100, - VolumeType: 'gp3', - }, - }, - ], - }), - }), - ); - }); - - it('does not load launch template block device name when DeviceName is provided by labels', async () => { - const testDataWithEbsLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['ghr-ec2-block-device-name:/dev/sdg', 'ghr-ec2-ebs-volume-size:100', 'ghr-ec2-ebs-volume-type:gp3'], - messageId: 'test-explicit-ebs-device-name', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithEbsLabels); - - expect(mockEC2Client).not.toHaveReceivedCommand(DescribeLaunchTemplateVersionsCommand); - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - BlockDeviceMappings: [ - { - DeviceName: '/dev/sdg', - Ebs: { - VolumeSize: 100, - VolumeType: 'gp3', - }, - }, - ], - }), - }), - ); - }); - - it('handles messages with no labels gracefully', async () => { - const testDataWithNoLabels = [ - { - ...TEST_DATA_SINGLE, - labels: undefined, - messageId: 'test-5', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithNoLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2instanceCriteria: expect.objectContaining({ - instanceTypes: ['t3.medium', 't3.large'], - }), - }), - ); - }); - - it('handles empty labels array', async () => { - const testDataWithEmptyLabels = [ - { - ...TEST_DATA_SINGLE, - labels: [], - messageId: 'test-6', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithEmptyLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2instanceCriteria: expect.objectContaining({ - instanceTypes: ['t3.medium', 't3.large'], - }), - }), - ); - }); - - it('handles multiple EC2 labels correctly', async () => { - const testDataWithMultipleEc2Labels = [ - { - ...TEST_DATA_SINGLE, - labels: ['regular-label', 'ghr-ec2-instance-type:r5.2xlarge', 'ghr-ec2-ami:custom-ami', 'ghr-ec2-disk:200'], - messageId: 'test-8', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithMultipleEc2Labels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2instanceCriteria: expect.objectContaining({ - instanceTypes: ['t3.medium', 't3.large'], - }), - ec2OverrideConfig: expect.objectContaining({ - InstanceType: 'r5.2xlarge', - }), - }), - ); - }); - - it('includes ec2OverrideConfig with VCpuCount requirements when specified', async () => { - const testDataWithVCpuLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-vcpu-count-min:4', 'ghr-ec2-vcpu-count-max:16'], - messageId: 'test-9', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithVCpuLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - InstanceRequirements: expect.objectContaining({ - VCpuCount: { - Min: 4, - Max: 16, - }, - }), - }), - }), - ); - }); - - it('includes ec2OverrideConfig with MemoryMiB requirements when specified', async () => { - const testDataWithMemoryLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-memory-mib-min:8192', 'ghr-ec2-memory-mib-max:32768'], - messageId: 'test-10', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithMemoryLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - InstanceRequirements: expect.objectContaining({ - MemoryMiB: { - Min: 8192, - Max: 32768, - }, - }), - }), - }), - ); - }); - - it('includes ec2OverrideConfig with CPU manufacturers when specified', async () => { - const testDataWithCpuLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-cpu-manufacturers:intel;amd'], - messageId: 'test-11', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithCpuLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - InstanceRequirements: expect.objectContaining({ - CpuManufacturers: ['intel', 'amd'], - }), - }), - }), - ); - }); - - it('includes ec2OverrideConfig with instance generations when specified', async () => { - const testDataWithGenerationLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-instance-generations:current'], - messageId: 'test-12', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithGenerationLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - InstanceRequirements: expect.objectContaining({ - InstanceGenerations: ['current'], - }), - }), - }), - ); - }); - - it('includes ec2OverrideConfig with accelerator requirements when specified', async () => { - const testDataWithAcceleratorLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-accelerator-count-min:1', 'ghr-ec2-accelerator-types:gpu'], - messageId: 'test-13', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithAcceleratorLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - InstanceRequirements: expect.objectContaining({ - AcceleratorCount: { - Min: 1, - }, - AcceleratorTypes: ['gpu'], - }), - }), - }), - ); - }); - - it('includes ec2OverrideConfig with max price when specified', async () => { - const testDataWithMaxPrice = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-max-price:0.50'], - messageId: 'test-14', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithMaxPrice); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - MaxPrice: '0.50', - }), - }), - ); - }); - - it('includes ec2OverrideConfig with priority and weighted capacity when specified', async () => { - const testDataWithPriorityWeight = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-priority:1', 'ghr-ec2-weighted-capacity:2'], - messageId: 'test-15', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithPriorityWeight); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - Priority: 1, - WeightedCapacity: 2, - }), - }), - ); - }); - - it('includes ec2OverrideConfig with combined requirements', async () => { - const testDataWithCombinedLabels = [ - { - ...TEST_DATA_SINGLE, - labels: [ - 'self-hosted', - 'linux', - 'ghr-ec2-vcpu-count-min:8', - 'ghr-ec2-memory-mib-min:16384', - 'ghr-ec2-cpu-manufacturers:intel', - 'ghr-ec2-instance-generations:current', - 'ghr-ec2-max-price:1.00', - ], - messageId: 'test-16', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithCombinedLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - InstanceRequirements: expect.objectContaining({ - VCpuCount: { Min: 8 }, - MemoryMiB: { Min: 16384 }, - CpuManufacturers: ['intel'], - InstanceGenerations: ['current'], - }), - MaxPrice: '1.00', - }), - }), - ); - }); - - it('includes both instance type and ec2OverrideConfig when both specified', async () => { - const testDataWithBoth = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-instance-type:c5.xlarge', 'ghr-ec2-vcpu-count-min:4'], - messageId: 'test-18', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithBoth); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2instanceCriteria: expect.objectContaining({ - instanceTypes: ['t3.medium', 't3.large'], - }), - ec2OverrideConfig: expect.objectContaining({ - InstanceType: 'c5.xlarge', - InstanceRequirements: expect.objectContaining({ - VCpuCount: { Min: 4 }, - }), - }), + expect(testProvider.createRunners).toHaveBeenCalledWith( + expect.objectContaining({ + githubInstallationClient: mockOctokit, + messages: [TEST_MESSAGE], + numberOfRunners: 1, + state: { prepared: true }, + githubRunnerConfig: expect.objectContaining({ + runnerLabels: 'base-label,ghr-provider-size:large', + runnerOwner: TEST_MESSAGE.repositoryOwner, + runnerType: 'Org', }), - ); - }); - - it('does not accumulate labels across groups when multiple messages have different dynamic labels', async () => { - const testDataMultipleGroups = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:m7a.large', 'ghr-job-id:run-1-inst-0'], - messageId: 'msg-1', - }, - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:m7i.xlarge', 'ghr-job-id:run-1-inst-1'], - messageId: 'msg-2', - }, - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:c7a.large', 'ghr-job-id:run-1-inst-2'], - messageId: 'msg-3', - }, - ]; - - await scaleUpModule.scaleUp(testDataMultipleGroups); - - expect(createRunner).toBeCalledTimes(3); - - const jitCalls = mockOctokit.actions.generateRunnerJitconfigForOrg.mock.calls; - expect(jitCalls).toHaveLength(3); - - for (const call of jitCalls) { - const labels = call[0].labels as string[]; - - if (labels.includes('ghr-ec2-instance-type:m7a.large')) { - expect(labels).toContain('ghr-job-id:run-1-inst-0'); - expect(labels).not.toContain('ghr-job-id:run-1-inst-1'); - expect(labels).not.toContain('ghr-job-id:run-1-inst-2'); - expect(labels).not.toContain('ghr-ec2-instance-type:m7i.xlarge'); - expect(labels).not.toContain('ghr-ec2-instance-type:c7a.large'); - } else if (labels.includes('ghr-ec2-instance-type:m7i.xlarge')) { - expect(labels).toContain('ghr-job-id:run-1-inst-1'); - expect(labels).not.toContain('ghr-job-id:run-1-inst-0'); - expect(labels).not.toContain('ghr-job-id:run-1-inst-2'); - expect(labels).not.toContain('ghr-ec2-instance-type:m7a.large'); - expect(labels).not.toContain('ghr-ec2-instance-type:c7a.large'); - } else if (labels.includes('ghr-ec2-instance-type:c7a.large')) { - expect(labels).toContain('ghr-job-id:run-1-inst-2'); - expect(labels).not.toContain('ghr-job-id:run-1-inst-0'); - expect(labels).not.toContain('ghr-job-id:run-1-inst-1'); - expect(labels).not.toContain('ghr-ec2-instance-type:m7a.large'); - expect(labels).not.toContain('ghr-ec2-instance-type:m7i.xlarge'); - } else { - throw new Error(`Unexpected labels combination: ${labels.join(',')}`); - } - } - }); - - it('preserves base RUNNER_LABELS for each group without mutation', async () => { - process.env.RUNNER_LABELS = 'ubuntu-2404,x64'; - - const testDataTwoGroups = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-instance-type:m7a.large', 'ghr-team:alpha'], - messageId: 'msg-a', - }, - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-instance-type:c7i.large', 'ghr-team:beta'], - messageId: 'msg-b', - }, - ]; - - await scaleUpModule.scaleUp(testDataTwoGroups); - - expect(createRunner).toBeCalledTimes(2); - - const jitCalls = mockOctokit.actions.generateRunnerJitconfigForOrg.mock.calls; - expect(jitCalls).toHaveLength(2); - - for (const call of jitCalls) { - const labels = call[0].labels as string[]; - - expect(labels).toContain('ubuntu-2404'); - expect(labels).toContain('x64'); - - if (labels.includes('ghr-team:alpha')) { - expect(labels).not.toContain('ghr-team:beta'); - expect(labels).not.toContain('ghr-ec2-instance-type:c7i.large'); - } else if (labels.includes('ghr-team:beta')) { - expect(labels).not.toContain('ghr-team:alpha'); - expect(labels).not.toContain('ghr-ec2-instance-type:m7a.large'); - } else { - throw new Error(`Unexpected labels combination: ${labels.join(',')}`); - } - } - }); + }), + ); + expect(mockedPublishRetryMessage).toHaveBeenCalledWith(expect.objectContaining({ messageId: 'message-1' })); }); - describe('on repo level', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; - process.env.RUNNER_NAME_PREFIX = 'unit-test'; - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - expectedRunnerParams.runnerType = 'Repo'; - expectedRunnerParams.runnerOwner = `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`; - // `--url https://github.enterprise.something/${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, - // `--token 1234abcd`, - // ]; - }); - - it('gets the current repo level runners', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).toBeCalledWith({ - environment: 'unit-test-environment', - runnerType: 'Repo', - runnerOwner: `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, - }); - }); - - it('does not create a token when maximum runners has been reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a token when maximum runners has not been reached', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('uses the default runner max count', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = undefined; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('creates a runner with correct config and labels', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner and ensure the group argument is ignored', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); + it('uses the default provider type when no provider type is configured', async () => { + await scaleUp([TEST_MESSAGE]); - it('Check error is thrown', async () => { - const mockCreateRunners = vi.mocked(createRunner); - mockCreateRunners.mockRejectedValue(new Error('no retry')); - await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow('no retry'); - mockCreateRunners.mockReset(); - }); + expect(mockedCreateScaleUpRunnerProviderFromEnv).toHaveBeenCalledWith( + 'test-provider', + 'unit-test-environment', + expect.any(Array), + ); }); - describe('Batch processing', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.RUNNERS_MAXIMUM_COUNT = '10'; - }); + it('resolves installation again when the event installation belongs to another app', async () => { + process.env.GHES_URL = 'https://github.enterprise.something'; + mockOctokit.apps.getOrgInstallation.mockResolvedValue({ data: { id: 123 } }); + mockedCreateGithubInstallationAuth.mockRejectedValueOnce({ status: 404 }).mockResolvedValueOnce({ + type: 'token', + tokenType: 'installation', + token: 'installation-token', + createdAt: 'some-date', + expiresAt: 'some-date', + permissions: {}, + repositorySelection: 'all', + installationId: 123, + }); + + await scaleUp([TEST_MESSAGE]); + + expect(mockOctokit.apps.getOrgInstallation).toHaveBeenCalledWith({ org: TEST_MESSAGE.repositoryOwner }); + expect(mockedCreateGithubInstallationAuth).toHaveBeenNthCalledWith( + 1, + TEST_MESSAGE.installationId, + 'https://github.enterprise.something/api/v3', + ); + expect(mockedCreateGithubInstallationAuth).toHaveBeenNthCalledWith( + 2, + 123, + 'https://github.enterprise.something/api/v3', + ); + expect(testProvider.createRunners).toHaveBeenCalledTimes(1); + }); - const createTestMessages = ( - count: number, - overrides: Partial[] = [], - ): scaleUpModule.ActionRequestMessageSQS[] => { - return Array.from({ length: count }, (_, i) => ({ - ...TEST_DATA_SINGLE, - id: i + 1, - messageId: `message-${i}`, - ...overrides[i], - })); + it('does not query current provider runners when the maximum runner count is unlimited', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '-1'; + testProvider.createRunners.mockResolvedValue(['runner-1', 'runner-2']); + const secondMessage: ActionRequestMessageSQS = { + ...TEST_MESSAGE, + id: 2, + messageId: 'message-2', }; - it('Should handle multiple messages for the same organization', async () => { - const messages = createTestMessages(3); - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(1); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 3, - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }), - ); - }); - - it('Should handle multiple messages for different organizations', async () => { - const messages = createTestMessages(3, [ - { repositoryOwner: 'org1' }, - { repositoryOwner: 'org2' }, - { repositoryOwner: 'org1' }, - ]); - - await scaleUpModule.scaleUp(messages); + await scaleUp([TEST_MESSAGE, secondMessage]); - expect(createRunner).toHaveBeenCalledTimes(2); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, - runnerOwner: 'org1', - }), - ); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, - runnerOwner: 'org2', - }), - ); - }); - - it('Should handle multiple messages for different repositories when org-level is disabled', async () => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; - const messages = createTestMessages(3, [ - { repositoryOwner: 'owner1', repositoryName: 'repo1' }, - { repositoryOwner: 'owner1', repositoryName: 'repo2' }, - { repositoryOwner: 'owner1', repositoryName: 'repo1' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(2); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, - runnerOwner: 'owner1/repo1', - }), - ); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, - runnerOwner: 'owner1/repo2', - }), - ); - }); - - it('Should reject messages when maximum runners limit is reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; // Set to 1 so with 1 existing, no new ones can be created - mockListRunners.mockImplementation(async () => [ - { - instanceId: 'i-existing', - launchTime: new Date(), - type: 'Org', - owner: TEST_DATA_SINGLE.repositoryOwner, - }, - ]); - - const messages = createTestMessages(3); - const rejectedMessages = await scaleUpModule.scaleUp(messages); + expect(testProvider.getCurrentRunners).not.toHaveBeenCalled(); + expect(testProvider.createRunners).toHaveBeenCalledWith(expect.objectContaining({ numberOfRunners: 2 })); + }); - expect(createRunner).not.toHaveBeenCalled(); // No runners should be created - expect(rejectedMessages).toHaveLength(3); // All 3 messages should be rejected + it('does not ask the provider to create runners when no jobs are queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockResolvedValue({ + data: { status: 'completed' }, + headers: {}, }); - it('Should handle partial EC2 instance creation failures', async () => { - mockCreateRunner.mockImplementation(async () => ['i-12345']); // Only creates 1 instead of requested 3 - - const messages = createTestMessages(3); - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(rejectedMessages).toHaveLength(2); // 3 requested - 1 created = 2 failed - expect(rejectedMessages).toEqual(['message-0', 'message-1']); - }); + await scaleUp([TEST_MESSAGE]); - it('Should filter out invalid event types for ephemeral runners', async () => { - const messages = createTestMessages(3, [ - { eventType: 'workflow_job' }, - { eventType: 'check_run' }, - { eventType: 'workflow_job' }, - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only workflow_job events processed - }), - ); - expect(rejectedMessages).toContain('message-1'); // check_run event rejected - }); - - it('Should skip invalid repo owner types but not reject them', async () => { - const messages = createTestMessages(3, [ - { repoOwnerType: 'Organization' }, - { repoOwnerType: 'User' }, // Invalid for org-level runners - { repoOwnerType: 'Organization' }, - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only Organization events processed - }), - ); - expect(rejectedMessages).not.toContain('message-1'); // User repo not rejected, just skipped - }); - - it('Should skip messages when jobs are not queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { - const isQueued = params.job_id === 1 || params.job_id === 3; // Only jobs 1 and 3 are queued - return { - data: { - status: isQueued ? 'queued' : 'completed', - }, - }; - }); - - const messages = createTestMessages(3); - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only queued jobs processed - }), - ); - }); - - it('Should create separate GitHub clients for different installations', async () => { - // Override the default mock to return different installation IDs - mockOctokit.apps.getOrgInstallation.mockReset(); - mockOctokit.apps.getOrgInstallation.mockImplementation((params) => ({ - data: { - id: params.org === 'org1' ? 100 : 200, - }, - })); - - const messages = createTestMessages(2, [ - { repositoryOwner: 'org1', installationId: 0 }, - { repositoryOwner: 'org2', installationId: 0 }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients - expect(mockedInstallationAuth).toHaveBeenCalledWith(100, 'https://github.enterprise.something/api/v3'); - expect(mockedInstallationAuth).toHaveBeenCalledWith(200, 'https://github.enterprise.something/api/v3'); - }); - - it('Should resolve installation again when event installation belongs to another app', async () => { - mockOctokit.apps.getOrgInstallation.mockReset(); - mockOctokit.apps.getOrgInstallation.mockImplementation(() => ({ - data: { - id: 123, - }, - })); - - mockedInstallationAuth.mockRejectedValueOnce({ status: 404 }).mockResolvedValueOnce({ - type: 'token', - tokenType: 'installation', - token: 'token', - createdAt: 'some-date', - expiresAt: 'some-date', - permissions: {}, - repositorySelection: 'all', - installationId: 123, - }); - - await scaleUpModule.scaleUp(TEST_DATA); - - expect(mockOctokit.apps.getOrgInstallation).toHaveBeenCalledWith({ org: TEST_DATA_SINGLE.repositoryOwner }); - expect(mockedInstallationAuth).toHaveBeenNthCalledWith( - 1, - TEST_DATA_SINGLE.installationId, - 'https://github.enterprise.something/api/v3', - ); - expect(mockedInstallationAuth).toHaveBeenNthCalledWith(2, 123, 'https://github.enterprise.something/api/v3'); - }); - - it('Should reuse GitHub clients for same installation', async () => { - const messages = createTestMessages(3, [ - { repositoryOwner: 'same-org' }, - { repositoryOwner: 'same-org' }, - { repositoryOwner: 'same-org' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(mockCreateClient).toHaveBeenCalledTimes(2); // 1 app client, 1 installation client - expect(mockedInstallationAuth).toHaveBeenCalledTimes(1); - }); - - it('Should return empty array when no valid messages to process', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - const messages = createTestMessages(2, [ - { eventType: 'check_run' }, // Invalid for ephemeral - { eventType: 'check_run' }, // Invalid for ephemeral - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).not.toHaveBeenCalled(); - expect(rejectedMessages).toEqual(['message-0', 'message-1']); - }); - - it('Should handle unlimited runners configuration', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '-1'; - const messages = createTestMessages(10); - - await scaleUpModule.scaleUp(messages); - - expect(listEC2Runners).not.toHaveBeenCalled(); // No need to check current runners - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 10, // All messages processed - }), - ); - }); - }); -}); - -describe('scaleUp with public GH', () => { - it('checks queued workflows', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalledWith({ - job_id: TEST_DATA_SINGLE.id, - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('not checking queued workflows', async () => { - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); - }); - - it('does not list runners when no workflows are queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ - data: { status: 'completed' }, - })); - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).not.toBeCalled(); - }); - - describe('on org level', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.RUNNER_NAME_PREFIX = 'unit-test'; - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - }); - - it('gets the current org level runners', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).toBeCalledWith({ - environment: 'unit-test-environment', - runnerType: 'Org', - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }); - }); - - it('does not create a token when maximum runners has been reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a token when maximum runners has not been reached', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - }); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a runner with correct config', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner with labels in s specific group', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - }); - - describe('on repo level', () => { - beforeEach(() => { - mockSSMClient.reset(); - - process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; - process.env.RUNNER_NAME_PREFIX = 'unit-test'; - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - expectedRunnerParams.runnerType = 'Repo'; - expectedRunnerParams.runnerOwner = `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`; - }); - - it('gets the current repo level runners', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).toBeCalledWith({ - environment: 'unit-test-environment', - runnerType: 'Repo', - runnerOwner: `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, - }); - }); - - it('does not create a token when maximum runners has been reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a token when maximum runners has not been reached', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('creates a runner with correct config and labels', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner with correct config and labels and on demand failover enabled.', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = JSON.stringify(['InsufficientInstanceCapacity']); - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith({ - ...expectedRunnerParams, - onDemandFailoverOnError: ['InsufficientInstanceCapacity'], - }); - }); - - it('creates a runner with correct config and labels and custom scale errors enabled.', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.SCALE_ERRORS = JSON.stringify(['RequestLimitExceeded']); - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith({ - ...expectedRunnerParams, - scaleErrors: ['RequestLimitExceeded'], - }); - }); - - it('creates a runner and ensure the group argument is ignored', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('ephemeral runners only run with workflow_job event, others should fail.', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - - const USER_REPO_TEST_DATA = structuredClone(TEST_DATA); - USER_REPO_TEST_DATA[0].eventType = 'check_run'; - - await expect(scaleUpModule.scaleUp(USER_REPO_TEST_DATA)).resolves.toEqual(['foobar']); - }); - - it('creates a ephemeral runner with JIT config.', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: 'TEST_JIT_CONFIG_REPO', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('creates a ephemeral runner with registration token.', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.ENABLE_JIT_CONFIG = 'false'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --ephemeral', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('JIT config is ignored for non-ephemeral runners.', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - process.env.ENABLE_JIT_CONFIG = 'true'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - process.env.RUNNER_LABELS = 'jit'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --labels jit', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('creates a ephemeral runner after checking job is queued.', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'true'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalled(); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('disable auto update on the runner.', async () => { - process.env.DISABLE_RUNNER_AUTOUPDATE = 'true'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('Scaling error should return failed message IDs so retry can be triggered.', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - await expect(scaleUpModule.scaleUp(TEST_DATA)).resolves.toEqual(['foobar']); - }); - }); - - describe('Batch processing', () => { - const createTestMessages = ( - count: number, - overrides: Partial[] = [], - ): scaleUpModule.ActionRequestMessageSQS[] => { - return Array.from({ length: count }, (_, i) => ({ - ...TEST_DATA_SINGLE, - id: i + 1, - messageId: `message-${i}`, - ...overrides[i], - })); - }; - - beforeEach(() => { - setDefaults(); - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.RUNNERS_MAXIMUM_COUNT = '10'; - }); - - it('Should handle multiple messages for the same organization', async () => { - const messages = createTestMessages(3); - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(1); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 3, - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }), - ); - }); - - it('Should handle multiple messages for different organizations', async () => { - const messages = createTestMessages(3, [ - { repositoryOwner: 'org1' }, - { repositoryOwner: 'org2' }, - { repositoryOwner: 'org1' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(2); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, - runnerOwner: 'org1', - }), - ); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, - runnerOwner: 'org2', - }), - ); - }); - - it('Should handle multiple messages for different repositories when org-level is disabled', async () => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; - const messages = createTestMessages(3, [ - { repositoryOwner: 'owner1', repositoryName: 'repo1' }, - { repositoryOwner: 'owner1', repositoryName: 'repo2' }, - { repositoryOwner: 'owner1', repositoryName: 'repo1' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(2); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, - runnerOwner: 'owner1/repo1', - }), - ); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, - runnerOwner: 'owner1/repo2', - }), - ); - }); - - it('Should reject messages when maximum runners limit is reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; // Set to 1 so with 1 existing, no new ones can be created - mockListRunners.mockImplementation(async () => [ - { - instanceId: 'i-existing', - launchTime: new Date(), - type: 'Org', - owner: TEST_DATA_SINGLE.repositoryOwner, - }, - ]); - - const messages = createTestMessages(3); - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).not.toHaveBeenCalled(); // No runners should be created - expect(rejectedMessages).toHaveLength(3); // All 3 messages should be rejected - }); - - it('Should handle partial EC2 instance creation failures', async () => { - mockCreateRunner.mockImplementation(async () => ['i-12345']); // Only creates 1 instead of requested 3 - - const messages = createTestMessages(3); - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(rejectedMessages).toHaveLength(2); // 3 requested - 1 created = 2 failed - expect(rejectedMessages).toEqual(['message-0', 'message-1']); - }); - - it('Should filter out invalid event types for ephemeral runners', async () => { - const messages = createTestMessages(3, [ - { eventType: 'workflow_job' }, - { eventType: 'check_run' }, - { eventType: 'workflow_job' }, - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only workflow_job events processed - }), - ); - expect(rejectedMessages).toContain('message-1'); // check_run event rejected - }); - - it('Should skip invalid repo owner types but not reject them', async () => { - const messages = createTestMessages(3, [ - { repoOwnerType: 'Organization' }, - { repoOwnerType: 'User' }, // Invalid for org-level runners - { repoOwnerType: 'Organization' }, - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only Organization events processed - }), - ); - expect(rejectedMessages).not.toContain('message-1'); // User repo not rejected, just skipped - }); - - it('Should skip messages when jobs are not queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { - const isQueued = params.job_id === 1 || params.job_id === 3; // Only jobs 1 and 3 are queued - return { - data: { - status: isQueued ? 'queued' : 'completed', - }, - }; - }); - - const messages = createTestMessages(3); - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only queued jobs processed - }), - ); - }); - - it('Should create separate GitHub clients for different installations', async () => { - // Override the default mock to return different installation IDs - mockOctokit.apps.getOrgInstallation.mockReset(); - mockOctokit.apps.getOrgInstallation.mockImplementation((params) => ({ - data: { - id: params.org === 'org1' ? 100 : 200, - }, - })); - - const messages = createTestMessages(2, [ - { repositoryOwner: 'org1', installationId: 0 }, - { repositoryOwner: 'org2', installationId: 0 }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients - expect(mockedInstallationAuth).toHaveBeenCalledWith(100, ''); - expect(mockedInstallationAuth).toHaveBeenCalledWith(200, ''); - }); - - it('Should reuse GitHub clients for same installation', async () => { - const messages = createTestMessages(3, [ - { repositoryOwner: 'same-org' }, - { repositoryOwner: 'same-org' }, - { repositoryOwner: 'same-org' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(mockCreateClient).toHaveBeenCalledTimes(2); // 1 app client, 1 installation client - expect(mockedInstallationAuth).toHaveBeenCalledTimes(1); - }); - - it('Should return empty array when no valid messages to process', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - const messages = createTestMessages(2, [ - { eventType: 'check_run' }, // Invalid for ephemeral - { eventType: 'check_run' }, // Invalid for ephemeral - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).not.toHaveBeenCalled(); - expect(rejectedMessages).toEqual(['message-0', 'message-1']); - }); - - it('Should handle unlimited runners configuration', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '-1'; - const messages = createTestMessages(10); - - await scaleUpModule.scaleUp(messages); - - expect(listEC2Runners).not.toHaveBeenCalled(); // No need to check current runners - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 10, // All messages processed - }), - ); - }); - }); -}); - -describe('scaleUp with Github Data Residency', () => { - beforeEach(() => { - process.env.GHES_URL = 'https://companyname.ghe.com'; - }); - - it('checks queued workflows', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalledWith({ - job_id: TEST_DATA_SINGLE.id, - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('does not list runners when no workflows are queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ - data: { total_count: 0 }, - })); - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).not.toBeCalled(); - }); - - describe('on org level', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.RUNNER_NAME_PREFIX = 'unit-test-'; - process.env.RUNNER_GROUP_NAME = 'Default'; - process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; - process.env.RUNNER_LABELS = 'label1,label2'; - - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - mockSSMClient.reset(); - }); - - it('gets the current org level runners', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).toBeCalledWith({ - environment: 'unit-test-environment', - runnerType: 'Org', - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }); - }); - - it('does not create a token when maximum runners has been reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('does create a runner if maximum is set to -1', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '-1'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).not.toHaveBeenCalled(); - expect(createRunner).toHaveBeenCalled(); - }); - - it('creates a token when maximum runners has not been reached', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - }); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a runner with correct config', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner with labels in a specific group', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner with ami id override from ssm parameter', async () => { - process.env.AMI_ID_SSM_PARAMETER_NAME = 'my-ami-id-param'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith({ ...expectedRunnerParams, amiIdSsmParameterName: 'my-ami-id-param' }); - }); - - it('Throws an error if runner group does not exist for ephemeral runners', async () => { - process.env.RUNNER_GROUP_NAME = 'test-runner-group'; - mockSSMgetParameter.mockImplementation(async () => { - throw new Error('ParameterNotFound'); - }); - await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toBeInstanceOf(Error); - expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); - }); - - it('Discards event if it is a User repo and org level runners is enabled', async () => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - const USER_REPO_TEST_DATA = structuredClone(TEST_DATA); - USER_REPO_TEST_DATA[0].repoOwnerType = 'User'; - await scaleUpModule.scaleUp(USER_REPO_TEST_DATA); - expect(createRunner).not.toHaveBeenCalled(); - }); - - it('create SSM parameter for runner group id if it does not exist', async () => { - mockSSMgetParameter.mockImplementation(async () => { - throw new Error('ParameterNotFound'); - }); - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 2); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`, - Value: '1', - Type: 'String', - }); - }); - - it('Does not create SSM parameter for runner group id if it exists', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.paginate).toHaveBeenCalledTimes(0); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 1); - }); - - it('create start runner config for ephemeral runners ', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '2'; - - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toBeCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - name: 'unit-test-i-12345', - runner_group_id: 1, - labels: ['label1', 'label2'], - }); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: 'TEST_JIT_CONFIG_ORG', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('create start runner config for non-ephemeral runners ', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - process.env.RUNNERS_MAXIMUM_COUNT = '2'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalled(); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: - '--url https://companyname.ghe.com/Codertocat --token 1234abcd ' + - '--labels label1,label2 --runnergroup Default', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - it.each(RUNNER_TYPES)( - 'calls create start runner config of 40' + ' instances (ssm rate limit condition) to test time delay ', - async (type: RunnerType) => { - process.env.ENABLE_EPHEMERAL_RUNNERS = type === 'ephemeral' ? 'true' : 'false'; - process.env.RUNNERS_MAXIMUM_COUNT = '40'; - mockCreateRunner.mockImplementation(async () => { - return instances; - }); - mockListRunners.mockImplementation(async () => { - return []; - }); - const startTime = performance.now(); - const instances = [ - 'i-1234', - 'i-5678', - 'i-5567', - 'i-5569', - 'i-5561', - 'i-5560', - 'i-5566', - 'i-5536', - 'i-5526', - 'i-5516', - 'i-122', - 'i-123', - 'i-124', - 'i-125', - 'i-126', - 'i-127', - 'i-128', - 'i-129', - 'i-130', - 'i-131', - 'i-132', - 'i-133', - 'i-134', - 'i-135', - 'i-136', - 'i-137', - 'i-138', - 'i-139', - 'i-140', - 'i-141', - 'i-142', - 'i-143', - 'i-144', - 'i-145', - 'i-146', - 'i-147', - 'i-148', - 'i-149', - 'i-150', - 'i-151', - ]; - await scaleUpModule.scaleUp(TEST_DATA); - const endTime = performance.now(); - expect(endTime - startTime).toBeGreaterThan(1000); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 40); - }, - 10000, - ); - }); - describe('on repo level', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; - process.env.RUNNER_NAME_PREFIX = 'unit-test'; - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - expectedRunnerParams.runnerType = 'Repo'; - expectedRunnerParams.runnerOwner = `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`; - // `--url https://companyname.ghe.com${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, - // `--token 1234abcd`, - // ]; - }); - - it('gets the current repo level runners', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).toBeCalledWith({ - environment: 'unit-test-environment', - runnerType: 'Repo', - runnerOwner: `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, - }); - }); - - it('does not create a token when maximum runners has been reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a token when maximum runners has not been reached', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('uses the default runner max count', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = undefined; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('creates a runner with correct config and labels', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner and ensure the group argument is ignored', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('Check error is thrown', async () => { - const mockCreateRunners = vi.mocked(createRunner); - mockCreateRunners.mockRejectedValue(new Error('no retry')); - await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow('no retry'); - mockCreateRunners.mockReset(); - }); - }); - - describe('Batch processing', () => { - const createTestMessages = ( - count: number, - overrides: Partial[] = [], - ): scaleUpModule.ActionRequestMessageSQS[] => { - return Array.from({ length: count }, (_, i) => ({ - ...TEST_DATA_SINGLE, - id: i + 1, - messageId: `message-${i}`, - ...overrides[i], - })); - }; - - beforeEach(() => { - setDefaults(); - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.RUNNERS_MAXIMUM_COUNT = '10'; - }); - - it('Should handle multiple messages for the same organization', async () => { - const messages = createTestMessages(3); - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(1); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 3, - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }), - ); - }); - - it('Should handle multiple messages for different organizations', async () => { - const messages = createTestMessages(3, [ - { repositoryOwner: 'org1' }, - { repositoryOwner: 'org2' }, - { repositoryOwner: 'org1' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(2); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, - runnerOwner: 'org1', - }), - ); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, - runnerOwner: 'org2', - }), - ); - }); - - it('Should handle multiple messages for different repositories when org-level is disabled', async () => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; - const messages = createTestMessages(3, [ - { repositoryOwner: 'owner1', repositoryName: 'repo1' }, - { repositoryOwner: 'owner1', repositoryName: 'repo2' }, - { repositoryOwner: 'owner1', repositoryName: 'repo1' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(2); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, - runnerOwner: 'owner1/repo1', - }), - ); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, - runnerOwner: 'owner1/repo2', - }), - ); - }); - - it('Should reject messages when maximum runners limit is reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '2'; - mockListRunners.mockImplementation(async () => [ - { - instanceId: 'i-existing', - launchTime: new Date(), - type: 'Org', - owner: TEST_DATA_SINGLE.repositoryOwner, - }, - ]); - - const messages = createTestMessages(5); - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, // 2 max - 1 existing = 1 new - }), - ); - expect(rejectedMessages).toHaveLength(4); // 5 requested - 1 created = 4 rejected - }); - - it('Should handle partial EC2 instance creation failures', async () => { - mockCreateRunner.mockImplementation(async () => ['i-12345']); // Only creates 1 instead of requested 3 - - const messages = createTestMessages(3); - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(rejectedMessages).toHaveLength(2); // 3 requested - 1 created = 2 failed - expect(rejectedMessages).toEqual(['message-0', 'message-1']); - }); - - it('Should filter out invalid event types for ephemeral runners', async () => { - const messages = createTestMessages(3, [ - { eventType: 'workflow_job' }, - { eventType: 'check_run' }, - { eventType: 'workflow_job' }, - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only workflow_job events processed - }), - ); - expect(rejectedMessages).toContain('message-1'); // check_run event rejected - }); - - it('Should skip invalid repo owner types but not reject them', async () => { - const messages = createTestMessages(3, [ - { repoOwnerType: 'Organization' }, - { repoOwnerType: 'User' }, // Invalid for org-level runners - { repoOwnerType: 'Organization' }, - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only Organization events processed - }), - ); - expect(rejectedMessages).not.toContain('message-1'); // User repo not rejected, just skipped - }); - - it('Should skip messages when jobs are not queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { - const isQueued = params.job_id === 1 || params.job_id === 3; // Only jobs 1 and 3 are queued - return { - data: { - status: isQueued ? 'queued' : 'completed', - }, - }; - }); - - const messages = createTestMessages(3); - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only queued jobs processed - }), - ); - }); - - it('Should create separate GitHub clients for different installations', async () => { - mockOctokit.apps.getOrgInstallation.mockImplementation((params) => ({ - data: { - id: params.org === 'org1' ? 100 : 200, - }, - })); - - const messages = createTestMessages(2, [ - { repositoryOwner: 'org1', installationId: 0 }, - { repositoryOwner: 'org2', installationId: 0 }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients - expect(mockedInstallationAuth).toHaveBeenCalledWith(100, ''); - expect(mockedInstallationAuth).toHaveBeenCalledWith(200, ''); - }); - - it('Should reuse GitHub clients for same installation', async () => { - const messages = createTestMessages(3, [ - { repositoryOwner: 'same-org' }, - { repositoryOwner: 'same-org' }, - { repositoryOwner: 'same-org' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(mockCreateClient).toHaveBeenCalledTimes(2); // 1 app client, 1 installation client - expect(mockedInstallationAuth).toHaveBeenCalledTimes(1); - }); - - it('Should return empty array when no valid messages to process', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - const messages = createTestMessages(2, [ - { eventType: 'check_run' }, // Invalid for ephemeral - { eventType: 'check_run' }, // Invalid for ephemeral - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).not.toHaveBeenCalled(); - expect(rejectedMessages).toEqual(['message-0', 'message-1']); - }); - - it('Should handle unlimited runners configuration', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '-1'; - const messages = createTestMessages(10); - - await scaleUpModule.scaleUp(messages); - - expect(listEC2Runners).not.toHaveBeenCalled(); // No need to check current runners - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 10, // All messages processed - }), - ); - }); - }); -}); - -describe('Retry mechanism tests', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'true'; - process.env.RUNNERS_MAXIMUM_COUNT = '10'; - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - mockSSMClient.reset(); - }); - - const createTestMessages = ( - count: number, - overrides: Partial[] = [], - ): scaleUpModule.ActionRequestMessageSQS[] => { - return Array.from({ length: count }, (_, i) => ({ - ...TEST_DATA_SINGLE, - id: i + 1, - messageId: `message-${i + 1}`, - ...overrides[i], - })); - }; - - it('calls publishRetryMessage for each valid message when job is queued', async () => { - const messages = createTestMessages(3); - mockCreateRunner.mockResolvedValue(['i-12345', 'i-67890', 'i-abcdef']); // Create all requested runners - - await scaleUpModule.scaleUp(messages); - - expect(mockPublishRetryMessage).toHaveBeenCalledTimes(3); - expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ - id: 1, - messageId: 'message-1', - }), - ); - expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ - id: 2, - messageId: 'message-2', - }), - ); - expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( - 3, - expect.objectContaining({ - id: 3, - messageId: 'message-3', - }), - ); - }); - - it('does not call publishRetryMessage when job is not queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { - const isQueued = params.job_id === 1; // Only job 1 is queued - return { - data: { - status: isQueued ? 'queued' : 'completed', - }, - }; - }); - - const messages = createTestMessages(3); - - await scaleUpModule.scaleUp(messages); - - // Only message with id 1 should trigger retry - expect(mockPublishRetryMessage).toHaveBeenCalledTimes(1); - expect(mockPublishRetryMessage).toHaveBeenCalledWith( - expect.objectContaining({ - id: 1, - messageId: 'message-1', - }), - ); - }); - - it('does not call publishRetryMessage when maximum runners is reached and messages are marked invalid', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '0'; // No runners can be created - - const messages = createTestMessages(2); - - await scaleUpModule.scaleUp(messages); - - // Verify listEC2Runners is called to check current runner count - expect(listEC2Runners).toHaveBeenCalledWith({ - environment: 'unit-test-environment', - runnerType: 'Org', - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }); - - // publishRetryMessage should NOT be called because messages are marked as invalid - // Invalid messages go back to the SQS queue and will be retried there - expect(mockPublishRetryMessage).not.toHaveBeenCalled(); - expect(createRunner).not.toHaveBeenCalled(); - }); - - it('calls publishRetryMessage with correct message structure including retry counter', async () => { - const message = { - ...TEST_DATA_SINGLE, - messageId: 'test-message-id', - retryCounter: 2, - }; - - await scaleUpModule.scaleUp([message]); - - expect(mockPublishRetryMessage).toHaveBeenCalledWith( - expect.objectContaining({ - id: message.id, - messageId: 'test-message-id', - retryCounter: 2, - }), - ); - }); - - it('calls publishRetryMessage when ENABLE_JOB_QUEUED_CHECK is false', async () => { - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - mockCreateRunner.mockResolvedValue(['i-12345', 'i-67890']); // Create all requested runners - - const messages = createTestMessages(2); - - await scaleUpModule.scaleUp(messages); - - // Should always call publishRetryMessage when queue check is disabled - expect(mockPublishRetryMessage).toHaveBeenCalledTimes(2); - expect(mockOctokit.actions.getJobForWorkflowRun).not.toHaveBeenCalled(); - }); - - it('calls publishRetryMessage for each message in a multi-runner scenario', async () => { - mockCreateRunner.mockResolvedValue(['i-12345', 'i-67890', 'i-abcdef', 'i-11111', 'i-22222']); // Create all requested runners - const messages = createTestMessages(5); - - await scaleUpModule.scaleUp(messages); - - expect(mockPublishRetryMessage).toHaveBeenCalledTimes(5); - messages.forEach((msg, index) => { - expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( - index + 1, - expect.objectContaining({ - id: msg.id, - messageId: msg.messageId, - }), - ); - }); - }); - - it('calls publishRetryMessage after runner creation', async () => { - const messages = createTestMessages(1); - mockCreateRunner.mockResolvedValue(['i-12345']); // Create the requested runner - - const callOrder: string[] = []; - mockPublishRetryMessage.mockImplementation(() => { - callOrder.push('publishRetryMessage'); - return Promise.resolve(); - }); - mockCreateRunner.mockImplementation(async () => { - callOrder.push('createRunner'); - return ['i-12345']; - }); - - await scaleUpModule.scaleUp(messages); - - expect(callOrder).toEqual(['createRunner', 'publishRetryMessage']); - }); -}); - -describe('parseEc2OverrideConfig', () => { - describe('Basic Fleet Overrides', () => { - it('should parse instance-type label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-instance-type:c5.xlarge']); - expect(result?.InstanceType).toBe('c5.xlarge'); - }); - - it('should parse subnet-id label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-subnet-id:subnet-123456']); - expect(result?.SubnetId).toBe('subnet-123456'); - }); - - it('should parse availability-zone label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-availability-zone:us-east-1a']); - expect(result?.AvailabilityZone).toBe('us-east-1a'); - }); - - it('should parse availability-zone-id label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-availability-zone-id:use1-az1']); - expect(result?.AvailabilityZoneId).toBe('use1-az1'); - }); - - it('should parse max-price label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-max-price:0.50']); - expect(result?.MaxPrice).toBe('0.50'); - }); - - it('should parse priority label as number', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-priority:1']); - expect(result?.Priority).toBe(1); - }); - - it('should parse weighted-capacity label as number', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-weighted-capacity:2']); - expect(result?.WeightedCapacity).toBe(2); - }); - - it('should parse image-id label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-image-id:ami-12345678']); - expect(result?.ImageId).toBe('ami-12345678'); - }); - - it('should parse multiple basic fleet overrides', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-instance-type:r5.2xlarge', - 'ghr-ec2-max-price:1.00', - 'ghr-ec2-priority:2', - ]); - expect(result?.InstanceType).toBe('r5.2xlarge'); - expect(result?.MaxPrice).toBe('1.00'); - expect(result?.Priority).toBe(2); - }); - }); - - describe('Placement', () => { - it('should parse placement-group-name label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-placement-group-name:my-placement-group']); - expect(result?.Placement?.GroupName).toBe('my-placement-group'); - }); - - it('should parse placement-group-id label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-placement-group-id:pg-1234567890abcdef0']); - expect(result?.Placement?.GroupId).toBe('pg-1234567890abcdef0'); - }); - - it('should parse placement-tenancy label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-placement-tenancy:dedicated']); - expect(result?.Placement?.Tenancy).toBe('dedicated'); - }); - - it('should parse placement-host-id label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-placement-host-id:h-1234567890abcdef']); - expect(result?.Placement?.HostId).toBe('h-1234567890abcdef'); - }); - - it('should parse placement-affinity label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-placement-affinity:host']); - expect(result?.Placement?.Affinity).toBe('host'); - }); - - it('should parse placement-partition-number label as number', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-placement-partition-number:3']); - expect(result?.Placement?.PartitionNumber).toBe(3); - }); - - it('should parse placement-availability-zone label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-placement-availability-zone:us-west-2b']); - expect(result?.Placement?.AvailabilityZone).toBe('us-west-2b'); - }); - - it('should parse placement-availability-zone-id label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-placement-availability-zone-id:use1-az1']); - expect(result?.Placement?.AvailabilityZoneId).toBe('use1-az1'); - }); - - it('should parse placement-spread-domain label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-placement-spread-domain:my-spread-domain']); - expect(result?.Placement?.SpreadDomain).toBe('my-spread-domain'); - }); - - it('should parse placement-host-resource-group-arn label', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-placement-host-resource-group-arn:arn:aws:ec2:us-east-1:123456789012:host-resource-group/hrg-1234', - ]); - expect(result?.Placement?.HostResourceGroupArn).toBe( - 'arn:aws:ec2:us-east-1:123456789012:host-resource-group/hrg-1234', - ); - }); - - it('should parse multiple placement labels', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-placement-group-name:group-1', - 'ghr-ec2-placement-tenancy:dedicated', - 'ghr-ec2-placement-availability-zone:us-east-1b', - ]); - expect(result?.Placement?.GroupName).toBe('group-1'); - expect(result?.Placement?.Tenancy).toBe('dedicated'); - expect(result?.Placement?.AvailabilityZone).toBe('us-east-1b'); - }); - }); - - describe('Block Device Mappings', () => { - it('should parse block-device-name label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-block-device-name:/dev/sdg']); - expect(result?.BlockDeviceMappings?.[0]?.DeviceName).toBe('/dev/sdg'); - }); - - it('should use default block device name when provided', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-ebs-volume-size:100'], '/dev/sda1'); - expect(result?.BlockDeviceMappings?.[0]?.DeviceName).toBe('/dev/sda1'); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(100); - }); - - it('should parse ebs-volume-size label as number', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-ebs-volume-size:100']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(100); - }); - - it('should parse ebs-volume-type label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-ebs-volume-type:gp3']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeType).toBe('gp3'); - }); - - it('should parse ebs-iops label as number', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-ebs-iops:3000']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Iops).toBe(3000); - }); - - it('should parse ebs-throughput label as number', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-ebs-throughput:250']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Throughput).toBe(250); - }); - - it('should parse ebs-encrypted label as boolean true', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-ebs-encrypted:true']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); - }); - - it('should parse ebs-encrypted label as boolean false', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-ebs-encrypted:false']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(false); - }); - - it('should parse ebs-kms-key-id label', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-ebs-kms-key-id:arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012', - ]); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.KmsKeyId).toBe( - 'arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012', - ); - }); - - it('should parse ebs-delete-on-termination label as boolean true', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-ebs-delete-on-termination:true']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.DeleteOnTermination).toBe(true); - }); - - it('should parse ebs-delete-on-termination label as boolean false', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-ebs-delete-on-termination:false']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.DeleteOnTermination).toBe(false); - }); - - it('should parse ebs-snapshot-id label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-ebs-snapshot-id:snap-1234567890abcdef']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.SnapshotId).toBe('snap-1234567890abcdef'); - }); - - it('should parse block-device-virtual-name label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-block-device-virtual-name:ephemeral0']); - expect(result?.BlockDeviceMappings?.[0]?.VirtualName).toBe('ephemeral0'); - }); - - it('should parse block-device-no-device label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-block-device-no-device:true']); - expect(result?.BlockDeviceMappings?.[0]?.NoDevice).toBe('true'); - }); - - it('should parse multiple block device mapping labels', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-ebs-volume-size:200', - 'ghr-ec2-ebs-volume-type:gp3', - 'ghr-ec2-ebs-iops:5000', - 'ghr-ec2-ebs-encrypted:true', - ]); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(200); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeType).toBe('gp3'); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Iops).toBe(5000); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); - }); - - it('should initialize BlockDeviceMappings when not present', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-ebs-volume-size:50']); - expect(result?.BlockDeviceMappings).toBeDefined(); - }); - }); - - describe('Instance Requirements - vCPU and Memory', () => { - it('should parse vcpu-count-min label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-vcpu-count-min:4']); - expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(4); - }); - - it('should parse vcpu-count-max label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-vcpu-count-max:16']); - expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(16); - }); - - it('should parse both vcpu-count-min and vcpu-count-max labels', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-vcpu-count-min:2', 'ghr-ec2-vcpu-count-max:8']); - expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(2); - expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(8); - }); - - it('should parse memory-mib-min label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-memory-mib-min:8192']); - expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(8192); - }); - - it('should parse memory-mib-max label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-memory-mib-max:32768']); - expect(result?.InstanceRequirements?.MemoryMiB?.Max).toBe(32768); - }); - - it('should parse both memory-mib-min and memory-mib-max labels', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-memory-mib-min:16384', - 'ghr-ec2-memory-mib-max:65536', - ]); - expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(16384); - expect(result?.InstanceRequirements?.MemoryMiB?.Max).toBe(65536); - }); - - it('should parse memory-gib-per-vcpu-min label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-memory-gib-per-vcpu-min:2']); - expect(result?.InstanceRequirements?.MemoryGiBPerVCpu?.Min).toBe(2); - }); - - it('should parse memory-gib-per-vcpu-max label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-memory-gib-per-vcpu-max:8']); - expect(result?.InstanceRequirements?.MemoryGiBPerVCpu?.Max).toBe(8); - }); - - it('should parse combined vCPU and memory requirements', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-vcpu-count-min:8', - 'ghr-ec2-vcpu-count-max:32', - 'ghr-ec2-memory-mib-min:32768', - 'ghr-ec2-memory-mib-max:131072', - ]); - expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(8); - expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(32); - expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(32768); - expect(result?.InstanceRequirements?.MemoryMiB?.Max).toBe(131072); - }); - }); - - describe('Instance Requirements - CPU and Performance', () => { - it('should parse cpu-manufacturers as single value', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-cpu-manufacturers:intel']); - expect(result?.InstanceRequirements?.CpuManufacturers).toEqual(['intel']); - }); - - it('should parse cpu-manufacturers as semicolon-separated list', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-cpu-manufacturers:intel;amd']); - expect(result?.InstanceRequirements?.CpuManufacturers).toEqual(['intel', 'amd']); - }); - - it('should parse instance-generations as single value', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-instance-generations:current']); - expect(result?.InstanceRequirements?.InstanceGenerations).toEqual(['current']); - }); - - it('should parse instance-generations as semicolon-separated list', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-instance-generations:current;previous']); - expect(result?.InstanceRequirements?.InstanceGenerations).toEqual(['current', 'previous']); - }); - - it('should parse excluded-instance-types as single value', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-excluded-instance-types:t2.micro']); - expect(result?.InstanceRequirements?.ExcludedInstanceTypes).toEqual(['t2.micro']); - }); - - it('should parse excluded-instance-types as semicolon-separated list', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-excluded-instance-types:t2.micro;t2.small']); - expect(result?.InstanceRequirements?.ExcludedInstanceTypes).toEqual(['t2.micro', 't2.small']); - }); - - it('should parse allowed-instance-types as single value', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-allowed-instance-types:c5.xlarge']); - expect(result?.InstanceRequirements?.AllowedInstanceTypes).toEqual(['c5.xlarge']); - }); - - it('should parse allowed-instance-types as semicolon-separated list', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-allowed-instance-types:c5.xlarge;c5.2xlarge']); - expect(result?.InstanceRequirements?.AllowedInstanceTypes).toEqual(['c5.xlarge', 'c5.2xlarge']); - }); - - it('should parse burstable-performance label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-burstable-performance:included']); - expect(result?.InstanceRequirements?.BurstablePerformance).toBe('included'); - }); - - it('should parse bare-metal label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-bare-metal:excluded']); - expect(result?.InstanceRequirements?.BareMetal).toBe('excluded'); - }); - }); - - describe('Instance Requirements - Accelerators', () => { - it('should parse accelerator-count-min label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-accelerator-count-min:1']); - expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); - }); - - it('should parse accelerator-count-max label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-accelerator-count-max:4']); - expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(4); - }); - - it('should parse both accelerator-count-min and accelerator-count-max', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-accelerator-count-min:1', - 'ghr-ec2-accelerator-count-max:2', - ]); - expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); - expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(2); - }); - - it('should parse accelerator-types as single value', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-accelerator-types:gpu']); - expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu']); - }); - - it('should parse accelerator-types as semicolon-separated list', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-accelerator-types:gpu;fpga']); - expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu', 'fpga']); - }); - - it('should parse accelerator-manufacturers as single value', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-accelerator-manufacturers:nvidia']); - expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia']); - }); - - it('should parse accelerator-manufacturers as semicolon-separated list', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-accelerator-manufacturers:nvidia;amd']); - expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia', 'amd']); - }); - - it('should parse accelerator-names as single value', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-accelerator-names:a100']); - expect(result?.InstanceRequirements?.AcceleratorNames).toEqual(['a100']); - }); - - it('should parse accelerator-names as semicolon-separated list', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-accelerator-names:a100;v100']); - expect(result?.InstanceRequirements?.AcceleratorNames).toEqual(['a100', 'v100']); - }); - - it('should parse accelerator-total-memory-mib-min label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-accelerator-total-memory-mib-min:8192']); - expect(result?.InstanceRequirements?.AcceleratorTotalMemoryMiB?.Min).toBe(8192); - }); - - it('should parse accelerator-total-memory-mib-max label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-accelerator-total-memory-mib-max:40960']); - expect(result?.InstanceRequirements?.AcceleratorTotalMemoryMiB?.Max).toBe(40960); - }); - - it('should parse combined accelerator requirements', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-accelerator-count-min:1', - 'ghr-ec2-accelerator-count-max:2', - 'ghr-ec2-accelerator-types:gpu', - 'ghr-ec2-accelerator-manufacturers:nvidia', - ]); - expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); - expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(2); - expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu']); - expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia']); - }); - }); - - describe('Instance Requirements - Network and Storage', () => { - it('should parse network-interface-count-min label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-network-interface-count-min:2']); - expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Min).toBe(2); - }); - - it('should parse network-interface-count-max label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-network-interface-count-max:4']); - expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Max).toBe(4); - }); - - it('should parse network-bandwidth-gbps-min label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-network-bandwidth-gbps-min:5']); - expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Min).toBe(5); - }); - - it('should parse network-bandwidth-gbps-max label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-network-bandwidth-gbps-max:25']); - expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Max).toBe(25); - }); - - it('should parse local-storage label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-local-storage:included']); - expect(result?.InstanceRequirements?.LocalStorage).toBe('included'); - }); - - it('should parse local-storage-types as single value', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-local-storage-types:ssd']); - expect(result?.InstanceRequirements?.LocalStorageTypes).toEqual(['ssd']); - }); - - it('should parse local-storage-types as semicolon-separated list', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-local-storage-types:hdd;ssd']); - expect(result?.InstanceRequirements?.LocalStorageTypes).toEqual(['hdd', 'ssd']); - }); - - it('should parse total-local-storage-gb-min label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-total-local-storage-gb-min:100']); - expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Min).toBe(100); - }); - - it('should parse total-local-storage-gb-max label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-total-local-storage-gb-max:1000']); - expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Max).toBe(1000); - }); - - it('should parse baseline-ebs-bandwidth-mbps-min label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-baseline-ebs-bandwidth-mbps-min:500']); - expect(result?.InstanceRequirements?.BaselineEbsBandwidthMbps?.Min).toBe(500); - }); - - it('should parse baseline-ebs-bandwidth-mbps-max label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-baseline-ebs-bandwidth-mbps-max:2000']); - expect(result?.InstanceRequirements?.BaselineEbsBandwidthMbps?.Max).toBe(2000); - }); - }); - - describe('Instance Requirements - Pricing and Other', () => { - it('should parse spot-max-price-percentage-over-lowest-price label', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-spot-max-price-percentage-over-lowest-price:50']); - expect(result?.InstanceRequirements?.SpotMaxPricePercentageOverLowestPrice).toBe(50); - }); - - it('should parse on-demand-max-price-percentage-over-lowest-price label', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-on-demand-max-price-percentage-over-lowest-price:75', - ]); - expect(result?.InstanceRequirements?.OnDemandMaxPricePercentageOverLowestPrice).toBe(75); - }); - - it('should parse max-spot-price-as-percentage-of-optimal-on-demand-price label', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-max-spot-price-as-percentage-of-optimal-on-demand-price:60', - ]); - expect(result?.InstanceRequirements?.MaxSpotPriceAsPercentageOfOptimalOnDemandPrice).toBe(60); - }); - - it('should parse require-hibernate-support label as boolean true', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-require-hibernate-support:true']); - expect(result?.InstanceRequirements?.RequireHibernateSupport).toBe(true); - }); - - it('should parse require-hibernate-support label as boolean false', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-require-hibernate-support:false']); - expect(result?.InstanceRequirements?.RequireHibernateSupport).toBe(false); - }); - - it('should parse require-encryption-in-transit label as boolean true', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-require-encryption-in-transit:true']); - expect(result?.InstanceRequirements?.RequireEncryptionInTransit).toBe(true); - }); - - it('should parse require-encryption-in-transit label as boolean false', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-require-encryption-in-transit:false']); - expect(result?.InstanceRequirements?.RequireEncryptionInTransit).toBe(false); - }); - - it('should parse baseline-performance-factors-cpu-reference-families label', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-baseline-performance-factors-cpu-reference-families:intel', - ]); - expect(result?.InstanceRequirements?.BaselinePerformanceFactors?.Cpu?.References?.[0]?.InstanceFamily).toBe( - 'intel', - ); - }); - it('should parse baseline-performance-factors-cpu-reference-families list label', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-baseline-performance-factors-cpu-reference-families:intel;amd', - ]); - expect(result?.InstanceRequirements?.BaselinePerformanceFactors?.Cpu?.References?.[0]?.InstanceFamily).toBe( - 'intel', - ); - expect(result?.InstanceRequirements?.BaselinePerformanceFactors?.Cpu?.References?.[1]?.InstanceFamily).toBe( - 'amd', - ); - }); - }); - - describe('Edge Cases', () => { - it('should return undefined when empty array is provided', () => { - const result = scaleUpModule.parseEc2OverrideConfig([]); - expect(result).toBeUndefined(); - }); - - it('should return undefined when no ghr-ec2 labels are provided', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['self-hosted', 'linux', 'x64']); - expect(result).toBeUndefined(); - }); - - it('should ignore non-ghr-ec2 labels and only parse ghr-ec2 labels', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'self-hosted', - 'ghr-ec2-instance-type:m5.large', - 'linux', - 'ghr-ec2-max-price:0.30', - ]); - expect(result?.InstanceType).toBe('m5.large'); - expect(result?.MaxPrice).toBe('0.30'); - }); - - it('should handle labels with colons in values (ARNs)', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-ebs-kms-key-id:arn:aws:kms:us-east-1:123456789012:key/abc-def-ghi', - ]); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.KmsKeyId).toBe( - 'arn:aws:kms:us-east-1:123456789012:key/abc-def-ghi', - ); - }); - - it('should handle labels with colons in placement ARNs', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-placement-host-resource-group-arn:arn:aws:ec2:us-west-2:123456789012:host-resource-group/hrg-abc123', - ]); - expect(result?.Placement?.HostResourceGroupArn).toBe( - 'arn:aws:ec2:us-west-2:123456789012:host-resource-group/hrg-abc123', - ); - }); - - it('should handle labels without values gracefully', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-instance-type:', 'ghr-ec2-max-price:0.50']); - expect(result?.InstanceType).toBeUndefined(); - expect(result?.MaxPrice).toBe('0.50'); - }); - - it('should handle malformed labels (no colon) gracefully', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-instance-type-m5-large', 'ghr-ec2-max-price:0.50']); - expect(result?.MaxPrice).toBe('0.50'); - expect(result?.InstanceType).toBeUndefined(); - }); - - it('should handle numeric strings correctly for number fields', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-priority:5', - 'ghr-ec2-weighted-capacity:10', - 'ghr-ec2-vcpu-count-min:4', - ]); - expect(result?.Priority).toBe(5); - expect(result?.WeightedCapacity).toBe(10); - expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(4); - }); - - it('should handle boolean strings correctly for boolean fields', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-ebs-encrypted:true', - 'ghr-ec2-ebs-delete-on-termination:false', - 'ghr-ec2-require-hibernate-support:true', - ]); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.DeleteOnTermination).toBe(false); - expect(result?.InstanceRequirements?.RequireHibernateSupport).toBe(true); - }); - - it('should handle floating point numbers in max-price', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-max-price:0.12345']); - expect(result?.MaxPrice).toBe('0.12345'); - }); - - it('should handle whitespace in semicolon-separated lists', () => { - const result = scaleUpModule.parseEc2OverrideConfig(['ghr-ec2-cpu-manufacturers: intel ; amd ']); - expect(result?.InstanceRequirements?.CpuManufacturers).toEqual([' intel ', ' amd ']); - }); - - it('should return config with all parsed labels', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-instance-type:c5.xlarge', - 'ghr-ec2-vcpu-count-min:4', - 'ghr-ec2-memory-mib-min:8192', - 'ghr-ec2-placement-tenancy:dedicated', - 'ghr-ec2-ebs-volume-size:100', - ]); - expect(result?.InstanceType).toBe('c5.xlarge'); - expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(4); - expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(8192); - expect(result?.Placement?.Tenancy).toBe('dedicated'); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(100); - }); - }); - - describe('Complex Scenarios', () => { - it('should handle comprehensive EC2 configuration with all categories', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - // Basic Fleet - 'ghr-ec2-instance-type:r5.2xlarge', - 'ghr-ec2-max-price:0.75', - 'ghr-ec2-priority:1', - // Placement - 'ghr-ec2-placement-group-name:my-group', - 'ghr-ec2-placement-tenancy:dedicated', - // Block Device - 'ghr-ec2-ebs-volume-size:200', - 'ghr-ec2-ebs-volume-type:gp3', - 'ghr-ec2-ebs-encrypted:true', - // Instance Requirements - 'ghr-ec2-vcpu-count-min:8', - 'ghr-ec2-vcpu-count-max:32', - 'ghr-ec2-memory-mib-min:32768', - 'ghr-ec2-cpu-manufacturers:intel;amd', - 'ghr-ec2-instance-generations:current', - ]); - - expect(result?.InstanceType).toBe('r5.2xlarge'); - expect(result?.MaxPrice).toBe('0.75'); - expect(result?.Priority).toBe(1); - expect(result?.Placement?.GroupName).toBe('my-group'); - expect(result?.Placement?.Tenancy).toBe('dedicated'); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(200); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeType).toBe('gp3'); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); - expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(8); - expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(32); - expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(32768); - expect(result?.InstanceRequirements?.CpuManufacturers).toEqual(['intel', 'amd']); - expect(result?.InstanceRequirements?.InstanceGenerations).toEqual(['current']); - }); - - it('should handle GPU instance configuration', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-accelerator-count-min:1', - 'ghr-ec2-accelerator-count-max:4', - 'ghr-ec2-accelerator-types:gpu', - 'ghr-ec2-accelerator-manufacturers:nvidia', - 'ghr-ec2-accelerator-names:a100;v100', - 'ghr-ec2-accelerator-total-memory-mib-min:16384', - ]); - - expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); - expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(4); - expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu']); - expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia']); - expect(result?.InstanceRequirements?.AcceleratorNames).toEqual(['a100', 'v100']); - expect(result?.InstanceRequirements?.AcceleratorTotalMemoryMiB?.Min).toBe(16384); - }); - - it('should handle network-optimized instance configuration', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-network-interface-count-min:2', - 'ghr-ec2-network-interface-count-max:8', - 'ghr-ec2-network-bandwidth-gbps-min:10', - 'ghr-ec2-network-bandwidth-gbps-max:100', - 'ghr-ec2-baseline-ebs-bandwidth-mbps-min:1000', - ]); - - expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Min).toBe(2); - expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Max).toBe(8); - expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Min).toBe(10); - expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Max).toBe(100); - expect(result?.InstanceRequirements?.BaselineEbsBandwidthMbps?.Min).toBe(1000); - }); - - it('should handle storage-optimized instance configuration', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-local-storage:included', - 'ghr-ec2-local-storage-types:ssd', - 'ghr-ec2-total-local-storage-gb-min:500', - 'ghr-ec2-total-local-storage-gb-max:2000', - ]); - - expect(result?.InstanceRequirements?.LocalStorage).toBe('included'); - expect(result?.InstanceRequirements?.LocalStorageTypes).toEqual(['ssd']); - expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Min).toBe(500); - expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Max).toBe(2000); - }); - - it('should handle spot instance configuration with pricing', () => { - const result = scaleUpModule.parseEc2OverrideConfig([ - 'ghr-ec2-max-price:0.50', - 'ghr-ec2-spot-max-price-percentage-over-lowest-price:100', - 'ghr-ec2-on-demand-max-price-percentage-over-lowest-price:150', - ]); - - expect(result?.MaxPrice).toBe('0.50'); - expect(result?.InstanceRequirements?.SpotMaxPricePercentageOverLowestPrice).toBe(100); - expect(result?.InstanceRequirements?.OnDemandMaxPricePercentageOverLowestPrice).toBe(150); - }); + expect(testProvider.getCurrentRunners).not.toHaveBeenCalled(); + expect(testProvider.createRunners).not.toHaveBeenCalled(); + expect(mockedPublishRetryMessage).not.toHaveBeenCalled(); }); }); - -describe('useDedicatedHost', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.RUNNER_NAME_PREFIX = 'unit-test-'; - process.env.RUNNER_GROUP_NAME = 'Default'; - process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; - process.env.RUNNER_LABELS = 'label1,label2'; - }); - - it('defaults to false when USE_DEDICATED_HOST env var is not set', async () => { - delete process.env.USE_DEDICATED_HOST; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toHaveBeenCalledWith(expect.objectContaining({ useDedicatedHost: false })); - }); - - it('is true when USE_DEDICATED_HOST is "true"', async () => { - process.env.USE_DEDICATED_HOST = 'true'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toHaveBeenCalledWith(expect.objectContaining({ useDedicatedHost: true })); - }); - - it('is false when USE_DEDICATED_HOST is "false"', async () => { - process.env.USE_DEDICATED_HOST = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toHaveBeenCalledWith(expect.objectContaining({ useDedicatedHost: false })); - }); -}); - -function defaultOctokitMockImpl() { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ - data: { - status: 'queued', - }, - })); - mockOctokit.paginate.mockImplementation(() => [ - { - id: 1, - name: 'Default', - }, - ]); - mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ labels }: { labels: string[] }) => ({ - data: { - runner: { id: 9876543210, labels: labels.map((name: string) => ({ name })) }, - encoded_jit_config: 'TEST_JIT_CONFIG_ORG', - }, - })); - mockOctokit.actions.generateRunnerJitconfigForRepo.mockImplementation(({ labels }: { labels: string[] }) => ({ - data: { - runner: { id: 9876543210, labels: labels.map((name: string) => ({ name })) }, - encoded_jit_config: 'TEST_JIT_CONFIG_REPO', - }, - })); - mockOctokit.checks.get.mockImplementation(() => ({ - data: { - status: 'queued', - }, - })); - - const mockTokenReturnValue = { - data: { - token: '1234abcd', - }, - }; - const mockInstallationIdReturnValueOrgs = { - data: { - id: TEST_DATA_SINGLE.installationId, - }, - }; - const mockInstallationIdReturnValueRepos = { - data: { - id: TEST_DATA_SINGLE.installationId, - }, - }; - - mockOctokit.actions.createRegistrationTokenForOrg.mockImplementation(() => mockTokenReturnValue); - mockOctokit.actions.createRegistrationTokenForRepo.mockImplementation(() => mockTokenReturnValue); - mockOctokit.apps.getOrgInstallation.mockImplementation(() => mockInstallationIdReturnValueOrgs); - mockOctokit.apps.getRepoInstallation.mockImplementation(() => mockInstallationIdReturnValueRepos); -} - -function defaultSSMGetParameterMockImpl() { - mockSSMgetParameter.mockImplementation(async (name: string) => { - if (name === `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`) { - return '1'; - } else if (name === `${process.env.PARAMETER_GITHUB_APP_ID_NAME}`) { - return `${process.env.GITHUB_APP_ID}`; - } else { - throw new Error(`ParameterNotFound: ${name}`); - } - }); -} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 35d73c9364..81831390ab 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -1,206 +1,26 @@ +import { addPersistentContextToChildLogger, createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { Octokit } from '@octokit/rest'; -import { - addPersistentContextToChildLogger, - createChildLogger, - getTracedAWSV3Client, -} from '@aws-github-runner/aws-powertools-util'; -import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util'; import yn from 'yn'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; -import { createRunner, listEC2Runners, tag, terminateRunner } from './../aws/runners'; -import { Ec2OverrideConfig, RunnerInputParameters } from './../aws/runners.d'; -import { metricGitHubAppRateLimit } from '../github/rate-limit'; -import { publishRetryMessage } from './job-retry'; import { - _InstanceType, - Tenancy, - VolumeType, - CpuManufacturer, - InstanceGeneration, - BurstablePerformance, - BareMetal, - AcceleratorType, - AcceleratorManufacturer, - AcceleratorName, - LocalStorage, - LocalStorageType, - Placement, - BaselinePerformanceFactorsRequest, - FleetEbsBlockDeviceRequest, - CpuPerformanceFactorRequest, - PerformanceFactorReferenceRequest, - FleetBlockDeviceMappingRequest, - InstanceRequirementsRequest, - VCpuCountRangeRequest, - MemoryMiBRequest, - MemoryGiBPerVCpuRequest, - AcceleratorCountRequest, - AcceleratorTotalMemoryMiBRequest, - NetworkInterfaceCountRequest, - NetworkBandwidthGbpsRequest, - TotalLocalStorageGBRequest, - BaselineEbsBandwidthMbpsRequest, - DescribeLaunchTemplateVersionsCommand, - EC2Client, - Tag, -} from '@aws-sdk/client-ec2'; + getGitHubEnterpriseApiUrl, + getInstallationId, + resolveInstallationId, + isJobQueued, + validateSsmParameterStoreTags, +} from './github-runner'; +import { publishRetryMessage } from './job-retry'; +import { getScaleUpRunnerProviderType } from './scale-up-config'; +import { createScaleUpRunnerProviderFromEnv, getDefaultScaleUpRunnerProviderType } from './scale-up-provider-registry'; +import type { + ActionRequestMessage, + ActionRequestMessageRetry, + ActionRequestMessageSQS, + CreateGitHubRunnerConfig, +} from './types'; const logger = createChildLogger('scale-up'); -const RUNNER_LABELS_TAG_KEY = 'ghr:runner_labels'; -const RUNNER_LABELS_TAG_VALUE_SEPARATOR = ','; -const EC2_OVERRIDE_LIST_VALUE_SEPARATOR = ';'; -export const EC2_TAG_VALUE_MAX_LENGTH = 256; -export const RUNNER_LABELS_TAG_MAX_COUNT = 5; - -export type LambdaRunnerSource = 'scale-up-lambda' | 'pool-lambda'; - -export interface RunnerGroup { - name: string; - id: number; -} - -interface EphemeralRunnerConfig { - runnerName: string; - runnerGroupId: number; - runnerLabels: string[]; -} - -export interface ActionRequestMessage { - id: number; - eventType: 'check_run' | 'workflow_job'; - repositoryName: string; - repositoryOwner: string; - installationId: number; - repoOwnerType: string; - retryCounter?: number; - labels?: string[]; -} - -export interface ActionRequestMessageSQS extends ActionRequestMessage { - messageId: string; -} - -export interface ActionRequestMessageRetry extends ActionRequestMessage { - retryCounter: number; -} - -interface CreateGitHubRunnerConfig { - ephemeral: boolean; - ghesBaseUrl: string; - enableJitConfig: boolean; - runnerLabels: string; - runnerGroup: string; - runnerNamePrefix: string; - runnerOwner: string; - runnerType: 'Org' | 'Repo'; - disableAutoUpdate: boolean; - ssmTokenPath: string; - ssmConfigPath: string; - ssmParameterStoreTags: { Key: string; Value: string }[]; -} - -interface CreateEC2RunnerConfig { - environment: string; - subnets: string[]; - launchTemplateName: string; - ec2instanceCriteria: RunnerInputParameters['ec2instanceCriteria']; - ec2OverrideConfig?: RunnerInputParameters['ec2OverrideConfig']; - numberOfRunners?: number; - amiIdSsmParameterName?: string; - tracingEnabled?: boolean; - onDemandFailoverOnError?: string[]; - scaleErrors: string[]; - useDedicatedHost?: boolean; -} - -function generateRunnerServiceConfig(githubRunnerConfig: CreateGitHubRunnerConfig, token: string) { - const config = [ - `--url ${githubRunnerConfig.ghesBaseUrl ?? 'https://github.com'}/${githubRunnerConfig.runnerOwner}`, - `--token ${token}`, - ]; - - if (githubRunnerConfig.runnerLabels) { - config.push(`--labels ${quoteRunnerLabelsForShell(githubRunnerConfig.runnerLabels)}`.trim()); - } - - if (githubRunnerConfig.disableAutoUpdate) { - config.push('--disableupdate'); - } - - if (githubRunnerConfig.runnerType === 'Org' && githubRunnerConfig.runnerGroup !== undefined) { - config.push(`--runnergroup ${githubRunnerConfig.runnerGroup}`); - } - - if (githubRunnerConfig.ephemeral) { - config.push(`--ephemeral`); - } - - return config; -} - -function quoteRunnerLabelsForShell(labels: string): string { - return /[\s;&|<>()$`"'*?[\\\]{}!]/.test(labels) ? quoteShellArg(labels) : labels; -} - -function quoteShellArg(value: string): string { - return `'${value.replace(/'/g, `'\\''`)}'`; -} - -export function validateSsmParameterStoreTags(tagsJson: string): { Key: string; Value: string }[] { - try { - const tags = JSON.parse(tagsJson); - - if (!Array.isArray(tags)) { - throw new Error('Tags must be an array'); - } - - if (tags.length === 0) { - return []; - } - - tags.forEach((tag, index) => { - if (typeof tag !== 'object' || tag === null) { - throw new Error(`Tag at index ${index} must be an object`); - } - if (!tag.Key || typeof tag.Key !== 'string' || tag.Key.trim() === '') { - throw new Error(`Tag at index ${index} has missing or invalid 'Key' property`); - } - if (!Object.prototype.hasOwnProperty.call(tag, 'Value') || typeof tag.Value !== 'string') { - throw new Error(`Tag at index ${index} has missing or invalid 'Value' property`); - } - }); - - return tags; - } catch (err) { - logger.error('Invalid SSM_PARAMETER_STORE_TAGS format', { error: err }); - throw new Error(`Failed to parse SSM_PARAMETER_STORE_TAGS: ${(err as Error).message}`); - } -} - -async function getGithubRunnerRegistrationToken(githubRunnerConfig: CreateGitHubRunnerConfig, ghClient: Octokit) { - const registrationToken = - githubRunnerConfig.runnerType === 'Org' - ? await ghClient.actions.createRegistrationTokenForOrg({ org: githubRunnerConfig.runnerOwner }) - : await ghClient.actions.createRegistrationTokenForRepo({ - owner: githubRunnerConfig.runnerOwner.split('/')[0], - repo: githubRunnerConfig.runnerOwner.split('/')[1], - }); - - return registrationToken.data.token; -} - -function removeTokenFromLogging(config: string[]): string[] { - const result: string[] = []; - config.forEach((e) => { - if (e.startsWith('--token')) { - result.push('--token '); - } else { - result.push(e); - } - }); - return result; -} function getErrorStatus(error: unknown): number | undefined { if (typeof error !== 'object' || error === null) { @@ -211,185 +31,6 @@ function getErrorStatus(error: unknown): number | undefined { return errorWithStatus.status ?? errorWithStatus.response?.status; } -async function resolveInstallationId( - githubAppClient: Octokit, - enableOrgLevel: boolean, - payload: ActionRequestMessage, -): Promise { - return enableOrgLevel - ? ( - await githubAppClient.apps.getOrgInstallation({ - org: payload.repositoryOwner, - }) - ).data.id - : ( - await githubAppClient.apps.getRepoInstallation({ - owner: payload.repositoryOwner, - repo: payload.repositoryName, - }) - ).data.id; -} - -export async function getInstallationId( - githubAppClient: Octokit, - enableOrgLevel: boolean, - payload: ActionRequestMessage, -): Promise { - if (payload.installationId !== 0) { - return payload.installationId; - } - - return resolveInstallationId(githubAppClient, enableOrgLevel, payload); -} - -export async function isJobQueued(githubInstallationClient: Octokit, payload: ActionRequestMessage): Promise { - let isQueued = false; - if (payload.eventType === 'workflow_job') { - const jobForWorkflowRun = await githubInstallationClient.actions.getJobForWorkflowRun({ - job_id: payload.id, - owner: payload.repositoryOwner, - repo: payload.repositoryName, - }); - metricGitHubAppRateLimit(jobForWorkflowRun.headers); - isQueued = jobForWorkflowRun.data.status === 'queued'; - logger.debug(`The job ${payload.id} is${isQueued ? ' ' : 'not'} queued`); - } else { - throw Error(`Event ${payload.eventType} is not supported`); - } - return isQueued; -} - -async function getRunnerGroupId(githubRunnerConfig: CreateGitHubRunnerConfig, ghClient: Octokit): Promise { - // if the runnerType is Repo, then runnerGroupId is default to 1 - let runnerGroupId: number | undefined = 1; - if (githubRunnerConfig.runnerType === 'Org' && githubRunnerConfig.runnerGroup !== undefined) { - let runnerGroup: string | undefined; - // check if runner group id is already stored in SSM Parameter Store and - // use it if it exists to avoid API call to GitHub - try { - runnerGroup = await getParameter( - `${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`, - ); - } catch (err) { - logger.debug('Handling error:', err as Error); - logger.warn( - `SSM Parameter "${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}" - for Runner group ${githubRunnerConfig.runnerGroup} does not exist`, - ); - } - if (runnerGroup === undefined) { - // get runner group id from GitHub - runnerGroupId = await getRunnerGroupByName(ghClient, githubRunnerConfig); - // store runner group id in SSM - try { - await putParameter( - `${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`, - runnerGroupId.toString(), - false, - { - tags: githubRunnerConfig.ssmParameterStoreTags, - }, - ); - } catch (err) { - logger.debug('Error storing runner group id in SSM Parameter Store', err as Error); - throw err; - } - } else { - runnerGroupId = parseInt(runnerGroup); - } - } - return runnerGroupId; -} - -async function getRunnerGroupByName(ghClient: Octokit, githubRunnerConfig: CreateGitHubRunnerConfig): Promise { - const runnerGroups: RunnerGroup[] = await ghClient.paginate(`GET /orgs/{org}/actions/runner-groups`, { - org: githubRunnerConfig.runnerOwner, - per_page: 100, - }); - const runnerGroupId = runnerGroups.find((runnerGroup) => runnerGroup.name === githubRunnerConfig.runnerGroup)?.id; - - if (runnerGroupId === undefined) { - throw new Error(`Runner group ${githubRunnerConfig.runnerGroup} does not exist`); - } - - return runnerGroupId; -} - -export async function createRunners( - githubRunnerConfig: CreateGitHubRunnerConfig, - ec2RunnerConfig: CreateEC2RunnerConfig, - numberOfRunners: number, - ghClient: Octokit, - source: LambdaRunnerSource = 'scale-up-lambda', -): Promise { - const instances = await createRunner({ - runnerType: githubRunnerConfig.runnerType, - runnerOwner: githubRunnerConfig.runnerOwner, - numberOfRunners, - source, - ...ec2RunnerConfig, - }); - if (instances.length !== 0) { - const failedInstances = await createStartRunnerConfig(githubRunnerConfig, instances, ghClient); - - // Terminate instances that failed to get configured to avoid waste - if (failedInstances.length > 0) { - logger.warn('Terminating instances that failed to get configured', { - failedInstances, - failedCount: failedInstances.length, - }); - - for (const instanceId of failedInstances) { - try { - await terminateRunner(instanceId); - } catch (error) { - logger.error('Failed to terminate instance', { - instanceId, - error: error instanceof Error ? error.message : String(error), - }); - } - } - - // Remove failed instances from the returned list - return instances.filter((id) => !failedInstances.includes(id)); - } - } - - return instances; -} - -function generateRunnerLabelsTags(labels: string[]): Tag[] { - if (labels.length === 0) { - return []; - } - - const generatedTagValues = packRunnerLabelsTagValues(labels); - const tagValues = generatedTagValues.slice(0, RUNNER_LABELS_TAG_MAX_COUNT); - - if (generatedTagValues.length > RUNNER_LABELS_TAG_MAX_COUNT) { - logger.warn('GitHub runner label EC2 tags were truncated to avoid exceeding EC2 tag limits.', { - maxRunnerLabelsTagCount: RUNNER_LABELS_TAG_MAX_COUNT, - }); - } - - return tagValues.map((value, index) => ({ - Key: index === 0 ? RUNNER_LABELS_TAG_KEY : `${RUNNER_LABELS_TAG_KEY}:${index + 1}`, - Value: value, - })); -} - -function packRunnerLabelsTagValues(labels: string[]): string[] { - const runnerLabelsValue = labels.join(RUNNER_LABELS_TAG_VALUE_SEPARATOR); - const characters = Array.from(runnerLabelsValue); - const tagValues: string[] = []; - - for (let start = 0; start < characters.length; start += EC2_TAG_VALUE_MAX_LENGTH) { - tagValues.push(characters.slice(start, start + EC2_TAG_VALUE_MAX_LENGTH).join('')); - } - - return tagValues; -} - async function createGithubInstallationClient( githubAppClient: Octokit, enableOrgLevel: boolean, @@ -434,32 +75,22 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise) - : undefined; const enableJobQueuedCheck = yn(process.env.ENABLE_JOB_QUEUED_CHECK, { default: true }); - const amiIdSsmParameterName = process.env.AMI_ID_SSM_PARAMETER_NAME; const runnerNamePrefix = process.env.RUNNER_NAME_PREFIX || ''; const ssmConfigPath = process.env.SSM_CONFIG_PATH || ''; - const tracingEnabled = yn(process.env.POWERTOOLS_TRACE_ENABLED, { default: false }); - const onDemandFailoverOnError = process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS - ? (JSON.parse(process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS) as [string]) - : []; const ssmParameterStoreTags: { Key: string; Value: string }[] = process.env.SSM_PARAMETER_STORE_TAGS && process.env.SSM_PARAMETER_STORE_TAGS.trim() !== '' ? validateSsmParameterStoreTags(process.env.SSM_PARAMETER_STORE_TAGS) : []; const scaleErrors = JSON.parse(process.env.SCALE_ERRORS) as [string]; - const useDedicatedHost = yn(process.env.USE_DEDICATED_HOST, { default: false }); + const runnerProviderType = getScaleUpRunnerProviderType( + process.env.RUNNER_PROVIDER_TYPE, + getDefaultScaleUpRunnerProviderType(), + ); + const runnerProvider = createScaleUpRunnerProviderFromEnv(runnerProviderType, environment, scaleErrors); const { ghesApiUrl, ghesBaseUrl } = getGitHubEnterpriseApiUrl(); @@ -558,35 +189,19 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise 0 ? (messages[0].labels ?? []) : []; - const dynamicEC2Labels = messageLabels.map((l) => l.trim()).filter((l) => l.startsWith('ghr-ec2-')); - const nonEc2DynamicLabels = messageLabels - .map((l) => l.trim()) - .filter((l) => l.startsWith('ghr-') && !l.startsWith('ghr-ec2-')); - const allDynamicLabels = [...nonEc2DynamicLabels, ...dynamicEC2Labels]; + const preparedRunnerGroup = await runnerProvider.prepareGroup(messageLabels); + const dynamicLabels = preparedRunnerGroup.runnerLabels; - if (allDynamicLabels.length > 0) { - logger.debug('Dynamic labels present on message', { labels: allDynamicLabels }); + if (dynamicLabels.length > 0) { + logger.debug('Dynamic labels present on message', { labels: dynamicLabels }); groupRunnerLabels = groupRunnerLabels - ? `${groupRunnerLabels},${allDynamicLabels.join(',')}` - : allDynamicLabels.join(','); + ? `${groupRunnerLabels},${dynamicLabels.join(',')}` + : dynamicLabels.join(','); logger.debug('Updated runner labels', { runnerLabels: groupRunnerLabels }); - - if (dynamicEC2Labels.length > 0) { - const defaultBlockDeviceName = shouldLoadLaunchTemplateBlockDeviceName(dynamicEC2Labels) - ? await getDefaultBlockDeviceNameFromLaunchTemplate(launchTemplateName) - : undefined; - - ec2OverrideConfig = parseEc2OverrideConfig(dynamicEC2Labels, defaultBlockDeviceName); - if (ec2OverrideConfig) { - logger.debug('EC2 override config parsed from labels', { ec2OverrideConfig }); - } - } } for (const message of messages) { @@ -616,9 +231,11 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise 0) { + if (skippedRunnerCount > 0) { logger.info('Not all runners will be created for this group, maximum number of runners reached.', { desiredNewRunners: scaleUp, }); if (ephemeralEnabled) { - // This removes `missingInstanceCount` items from the start of the array + // This removes `skippedRunnerCount` items from the start of the array // so that, if we retry more messages later, we pick fresh ones. - const removedMessages = messages.splice(0, missingInstanceCount); + const removedMessages = messages.splice(0, skippedRunnerCount); removedMessages.forEach(({ messageId }) => rejectedMessageIds.add(messageId)); } - // No runners will be created, so skip calling the EC2 API. + // No runners will be created, so skip calling the provider. if (newRunners <= 0) { // Publish retry messages for messages that are not rejected for (const message of queuedMessages) { @@ -665,56 +282,41 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise rejectedMessageIds.add(messageId)); } @@ -729,524 +331,10 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise { - if (githubRunnerConfig.enableJitConfig && githubRunnerConfig.ephemeral) { - return await createJitConfig(githubRunnerConfig, instances, ghClient); - } else { - return await createRegistrationTokenConfig(githubRunnerConfig, instances, ghClient); - } -} - function isValidRepoOwnerTypeIfOrgLevelEnabled(payload: ActionRequestMessage, enableOrgLevel: boolean): boolean { return !(enableOrgLevel && payload.repoOwnerType !== 'Organization'); } -function addDelay(instances: string[]) { - const delay = async (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - const ssmParameterStoreMaxThroughput = 40; - const isDelay = instances.length >= ssmParameterStoreMaxThroughput; - return { isDelay, delay }; -} - -/** - * Creates registration token configuration for non-ephemeral runners. - * - * @returns Empty array (this configuration method does not have failure cases) - */ -async function createRegistrationTokenConfig( - githubRunnerConfig: CreateGitHubRunnerConfig, - instances: string[], - ghClient: Octokit, -): Promise { - const { isDelay, delay } = addDelay(instances); - const token = await getGithubRunnerRegistrationToken(githubRunnerConfig, ghClient); - const runnerServiceConfig = generateRunnerServiceConfig(githubRunnerConfig, token); - - logger.debug('Runner service config for non-ephemeral runners', { - runner_service_config: removeTokenFromLogging(runnerServiceConfig), - }); - - for (const instance of instances) { - await putParameter(`${githubRunnerConfig.ssmTokenPath}/${instance}`, runnerServiceConfig.join(' '), true, { - tags: [{ Key: 'InstanceId', Value: instance }, ...githubRunnerConfig.ssmParameterStoreTags], - }); - if (isDelay) { - // Delay to prevent AWS ssm rate limits by being within the max throughput limit - await delay(25); - } - } - - return []; -} - -async function tagRunnerMetadata(instanceId: string, runnerId: string, runnerLabels: string[]): Promise { - const tags = [{ Key: 'ghr:github_runner_id', Value: runnerId }, ...generateRunnerLabelsTags(runnerLabels)]; - - try { - await tag(instanceId, tags); - } catch (e) { - logger.error(`Failed to mark runner '${instanceId}' with GitHub runner metadata.`, { error: e }); - } -} - -/** - * Creates JIT (Just-In-Time) configuration for ephemeral runners. - * Continues processing remaining instances even if some fail. - * - * @returns Array of instance IDs that failed to get JIT configuration - */ -async function createJitConfig( - githubRunnerConfig: CreateGitHubRunnerConfig, - instances: string[], - ghClient: Octokit, -): Promise { - const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient); - const { isDelay, delay } = addDelay(instances); - const runnerLabels = githubRunnerConfig.runnerLabels.split(','); - const failedInstances: string[] = []; - - logger.debug(`Runner group id: ${runnerGroupId}`); - logger.debug(`Runner labels: ${runnerLabels}`); - for (const instance of instances) { - try { - // generate jit config for runner registration - const ephemeralRunnerConfig: EphemeralRunnerConfig = { - runnerName: `${githubRunnerConfig.runnerNamePrefix}${instance}`, - runnerGroupId: runnerGroupId, - runnerLabels: runnerLabels, - }; - logger.debug(`Runner name: ${ephemeralRunnerConfig.runnerName}`); - const runnerConfig = - githubRunnerConfig.runnerType === 'Org' - ? await ghClient.actions.generateRunnerJitconfigForOrg({ - org: githubRunnerConfig.runnerOwner, - name: ephemeralRunnerConfig.runnerName, - runner_group_id: ephemeralRunnerConfig.runnerGroupId, - labels: ephemeralRunnerConfig.runnerLabels, - }) - : await ghClient.actions.generateRunnerJitconfigForRepo({ - owner: githubRunnerConfig.runnerOwner.split('/')[0], - repo: githubRunnerConfig.runnerOwner.split('/')[1], - name: ephemeralRunnerConfig.runnerName, - runner_group_id: ephemeralRunnerConfig.runnerGroupId, - labels: ephemeralRunnerConfig.runnerLabels, - }); - - metricGitHubAppRateLimit(runnerConfig.headers); - - // tag the EC2 instance with GitHub runner metadata - await tagRunnerMetadata(instance, runnerConfig.data.runner.id.toString(), runnerLabels); - - // store jit config in ssm parameter store - logger.debug('Runner JIT config for ephemeral runner generated.', { - instance: instance, - }); - await putParameter(`${githubRunnerConfig.ssmTokenPath}/${instance}`, runnerConfig.data.encoded_jit_config, true, { - tags: [{ Key: 'InstanceId', Value: instance }, ...githubRunnerConfig.ssmParameterStoreTags], - }); - if (isDelay) { - // Delay to prevent AWS ssm rate limits by being within the max throughput limit - await delay(25); - } - } catch (error) { - failedInstances.push(instance); - logger.warn('Failed to create JIT config for instance, continuing with remaining instances', { - instance: instance, - error: error instanceof Error ? error.message : String(error), - }); - } - } - - if (failedInstances.length > 0) { - logger.error('Failed to create JIT config for some instances', { - failedInstances: failedInstances, - totalInstances: instances.length, - successfulInstances: instances.length - failedInstances.length, - }); - } - - return failedInstances; -} - -/** - * Parses EC2 override configuration from GitHub labels. - * - * Supported label formats: - * - * Basic Fleet Overrides: - * - ghr-ec2-instance-type: - Set specific instance type (e.g., c5.xlarge) - * - ghr-ec2-max-price: - Set maximum spot price - * - ghr-ec2-subnet-id: - Set subnet ID - * - ghr-ec2-availability-zone: - Set availability zone - * - ghr-ec2-availability-zone-id: - Set availability zone ID - * - ghr-ec2-weighted-capacity: - Set weighted capacity - * - ghr-ec2-priority: - Set launch priority - * - ghr-ec2-image-id: - Override AMI ID - * - * Instance Requirements (vCPU & Memory): - * - ghr-ec2-vcpu-count-min: - Set minimum vCPU count - * - ghr-ec2-vcpu-count-max: - Set maximum vCPU count - * - ghr-ec2-memory-mib-min: - Set minimum memory in MiB - * - ghr-ec2-memory-mib-max: - Set maximum memory in MiB - * - ghr-ec2-memory-gib-per-vcpu-min: - Set min memory per vCPU ratio - * - ghr-ec2-memory-gib-per-vcpu-max: - Set max memory per vCPU ratio - * - * Instance Requirements (CPU & Performance): - * - ghr-ec2-cpu-manufacturers: - CPU manufacturers (semicolon-separated: intel;amd;amazon-web-services) - * - ghr-ec2-instance-generations: - Instance generations (semicolon-separated: current;previous) - * - ghr-ec2-excluded-instance-types: - Exclude instance types (semicolon-separated) - * - ghr-ec2-allowed-instance-types: - Allow only specific instance types (semicolon-separated) - * - ghr-ec2-burstable-performance: - Burstable performance (included,excluded,required) - * - ghr-ec2-bare-metal: - Bare metal (included,excluded,required) - * - * Instance Requirements (Accelerators/GPU): - * - ghr-ec2-accelerator-types: - Accelerator types (semicolon-separated: gpu;fpga;inference) - * - ghr-ec2-accelerator-count-min: - Set minimum accelerator count - * - ghr-ec2-accelerator-count-max: - Set maximum accelerator count - * - ghr-ec2-accelerator-manufacturers: - Accelerator manufacturers (semicolon-separated: nvidia;amd;amazon-web-services;xilinx) - * - ghr-ec2-accelerator-names: - Specific accelerator names (semicolon-separated) - * - ghr-ec2-accelerator-total-memory-mib-min: - Min accelerator total memory in MiB - * - ghr-ec2-accelerator-total-memory-mib-max: - Max accelerator total memory in MiB - * - * Instance Requirements (Network & Storage): - * - ghr-ec2-network-interface-count-min: - Min network interfaces - * - ghr-ec2-network-interface-count-max: - Max network interfaces - * - ghr-ec2-network-bandwidth-gbps-min: - Min network bandwidth in Gbps - * - ghr-ec2-network-bandwidth-gbps-max: - Max network bandwidth in Gbps - * - ghr-ec2-local-storage: - Local storage (included,excluded,required) - * - ghr-ec2-local-storage-types: - Local storage types (semicolon-separated: hdd;ssd) - * - ghr-ec2-total-local-storage-gb-min: - Min total local storage in GB - * - ghr-ec2-total-local-storage-gb-max: - Max total local storage in GB - * - ghr-ec2-baseline-ebs-bandwidth-mbps-min: - Min baseline EBS bandwidth in Mbps - * - ghr-ec2-baseline-ebs-bandwidth-mbps-max: - Max baseline EBS bandwidth in Mbps - * - * Placement: - * - ghr-ec2-placement-group-name: - Placement group name - * - ghr-ec2-placement-group-id: - Placement group ID - * - ghr-ec2-placement-tenancy: - Tenancy (default,dedicated,host) - * - ghr-ec2-placement-host-id: - Dedicated host ID - * - ghr-ec2-placement-affinity: - Affinity (default,host) - * - ghr-ec2-placement-partition-number: - Partition number - * - ghr-ec2-placement-availability-zone: - Placement availability zone - * - ghr-ec2-placement-availability-zone-id: - Placement availability zone ID - * - ghr-ec2-placement-spread-domain: - Spread domain - * - ghr-ec2-placement-host-resource-group-arn: - Host resource group ARN - * - * Block Device Mappings: - * - ghr-ec2-block-device-name: - Block device name - * - ghr-ec2-ebs-volume-size: - EBS volume size in GB - * - ghr-ec2-ebs-volume-type: - EBS volume type (gp2,gp3,io1,io2,st1,sc1) - * - ghr-ec2-ebs-iops: - EBS IOPS - * - ghr-ec2-ebs-throughput: - EBS throughput in MB/s (gp3 only) - * - ghr-ec2-ebs-encrypted: - EBS encryption (true,false) - * - ghr-ec2-ebs-kms-key-id: - KMS key ID for encryption - * - ghr-ec2-ebs-delete-on-termination: - Delete on termination (true,false) - * - ghr-ec2-ebs-snapshot-id: - Snapshot ID for EBS volume - * - ghr-ec2-block-device-virtual-name: - Virtual device name (ephemeral storage) - * - ghr-ec2-block-device-no-device: - Suppresses device mapping - * - * Pricing & Advanced: - * - ghr-ec2-spot-max-price-percentage-over-lowest-price: - Spot max price as % over lowest price - * - ghr-ec2-on-demand-max-price-percentage-over-lowest-price: - On-demand max price as % over lowest price - * - ghr-ec2-max-spot-price-as-percentage-of-optimal-on-demand-price: - Max spot price as % of optimal on-demand - * - ghr-ec2-require-hibernate-support: - Require hibernate support (true,false) - * - ghr-ec2-require-encryption-in-transit: - Require encryption in-transit (true,false) - * - ghr-ec2-baseline-performance-factors-cpu-reference-families: - CPU baseline performance reference families (semicolon-separated) - * - * Example: - * runs-on: [self-hosted, linux, ghr-ec2-vcpu-count-min:4, ghr-ec2-memory-mib-min:16384, ghr-ec2-accelerator-types:gpu] - * - * @param labels - Array of GitHub workflow job labels - * @param defaultBlockDeviceName - Device name to use when dynamic block device labels create a mapping - * @returns EC2 override configuration object or undefined if no valid config found - */ -export function parseEc2OverrideConfig( - labels: string[], - defaultBlockDeviceName?: string, -): Ec2OverrideConfig | undefined { - const ec2Labels = labels.filter((l) => l.startsWith('ghr-ec2-')); - const config: Ec2OverrideConfig = {}; - - for (const label of ec2Labels) { - const [key, ...valueParts] = label.replace('ghr-ec2-', '').split(':'); - const value = valueParts.join(':'); - - if (!value) continue; - - // Basic Fleet Overrides - if (key === 'instance-type') { - config.InstanceType = value as _InstanceType; - } else if (key === 'subnet-id') { - config.SubnetId = value; - } else if (key === 'availability-zone') { - config.AvailabilityZone = value; - } else if (key === 'availability-zone-id') { - config.AvailabilityZoneId = value; - } else if (key === 'max-price') { - config.MaxPrice = value; - } else if (key === 'priority') { - config.Priority = parseFloat(value); - } else if (key === 'weighted-capacity') { - config.WeightedCapacity = parseFloat(value); - } else if (key === 'image-id') { - config.ImageId = value; - } - - // Placement - else if (key.startsWith('placement-')) { - config.Placement = config.Placement || ({} as Placement); - const placementKey = key.replace('placement-', ''); - if (placementKey === 'availability-zone-id') { - config.Placement.AvailabilityZoneId = value; - } else if (placementKey === 'affinity') { - config.Placement.Affinity = value; - } else if (placementKey === 'group-name') { - config.Placement.GroupName = value; - } else if (placementKey === 'partition-number') { - config.Placement.PartitionNumber = parseInt(value, 10); - } else if (placementKey === 'host-id') { - config.Placement.HostId = value; - } else if (placementKey === 'tenancy') { - config.Placement.Tenancy = value as Tenancy; - } else if (placementKey === 'spread-domain') { - config.Placement.SpreadDomain = value; - } else if (placementKey === 'host-resource-group-arn') { - config.Placement.HostResourceGroupArn = value; - } else if (placementKey === 'group-id') { - config.Placement.GroupId = value; - } else if (placementKey === 'availability-zone') { - config.Placement.AvailabilityZone = value; - } - } - - // Block Device Mappings - else if (key === 'block-device-name') { - getOrCreateBlockDeviceMapping(config, defaultBlockDeviceName).DeviceName = value; - } else if (key === 'block-device-virtual-name') { - getOrCreateBlockDeviceMapping(config, defaultBlockDeviceName).VirtualName = value; - } else if (key.startsWith('ebs-')) { - const blockDeviceMapping = getOrCreateBlockDeviceMapping(config, defaultBlockDeviceName); - const ebsKey = key.replace('ebs-', ''); - const ebs = blockDeviceMapping.Ebs || (blockDeviceMapping.Ebs = {} as FleetEbsBlockDeviceRequest); - - if (ebsKey === 'encrypted') { - ebs.Encrypted = value.toLowerCase() === 'true'; - } else if (ebsKey === 'delete-on-termination') { - ebs.DeleteOnTermination = value.toLowerCase() === 'true'; - } else if (ebsKey === 'iops') { - ebs.Iops = parseInt(value, 10); - } else if (ebsKey === 'throughput') { - ebs.Throughput = parseInt(value, 10); - } else if (ebsKey === 'kms-key-id') { - ebs.KmsKeyId = value; - } else if (ebsKey === 'snapshot-id') { - ebs.SnapshotId = value; - } else if (ebsKey === 'volume-size') { - ebs.VolumeSize = parseInt(value, 10); - } else if (ebsKey === 'volume-type') { - ebs.VolumeType = value as VolumeType; - } - } else if (key === 'block-device-no-device') { - getOrCreateBlockDeviceMapping(config, defaultBlockDeviceName).NoDevice = value; - } - - // Instance Requirements - else if (key.startsWith('vcpu-count-')) { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.VCpuCount = config.InstanceRequirements.VCpuCount || ({} as VCpuCountRangeRequest); - const subKey = key.replace('vcpu-count-', ''); - config.InstanceRequirements.VCpuCount![subKey === 'min' ? 'Min' : 'Max'] = parseInt(value, 10); - } else if (key.startsWith('memory-mib-')) { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.MemoryMiB = config.InstanceRequirements.MemoryMiB || ({} as MemoryMiBRequest); - const subKey = key.replace('memory-mib-', ''); - config.InstanceRequirements.MemoryMiB![subKey === 'min' ? 'Min' : 'Max'] = parseInt(value, 10); - } else if (key === 'cpu-manufacturers') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.CpuManufacturers = splitEc2OverrideListValue(value) as CpuManufacturer[]; - } else if (key.startsWith('memory-gib-per-vcpu-')) { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.MemoryGiBPerVCpu = - config.InstanceRequirements.MemoryGiBPerVCpu || ({} as MemoryGiBPerVCpuRequest); - const subKey = key.replace('memory-gib-per-vcpu-', ''); - config.InstanceRequirements.MemoryGiBPerVCpu![subKey === 'min' ? 'Min' : 'Max'] = parseFloat(value); - } else if (key === 'excluded-instance-types') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.ExcludedInstanceTypes = splitEc2OverrideListValue(value); - } else if (key === 'instance-generations') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.InstanceGenerations = splitEc2OverrideListValue(value) as InstanceGeneration[]; - } else if (key === 'spot-max-price-percentage-over-lowest-price') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.SpotMaxPricePercentageOverLowestPrice = parseInt(value, 10); - } else if (key === 'on-demand-max-price-percentage-over-lowest-price') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.OnDemandMaxPricePercentageOverLowestPrice = parseInt(value, 10); - } else if (key === 'bare-metal') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.BareMetal = value as BareMetal; - } else if (key === 'burstable-performance') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.BurstablePerformance = value as BurstablePerformance; - } else if (key === 'require-hibernate-support') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.RequireHibernateSupport = value.toLowerCase() === 'true'; - } else if (key.startsWith('network-interface-count-')) { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.NetworkInterfaceCount = - config.InstanceRequirements.NetworkInterfaceCount || ({} as NetworkInterfaceCountRequest); - const subKey = key.replace('network-interface-count-', ''); - config.InstanceRequirements.NetworkInterfaceCount![subKey === 'min' ? 'Min' : 'Max'] = parseInt(value, 10); - } else if (key === 'local-storage') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.LocalStorage = value as LocalStorage; - } else if (key === 'local-storage-types') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.LocalStorageTypes = splitEc2OverrideListValue(value) as LocalStorageType[]; - } else if (key.startsWith('total-local-storage-gb-')) { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.TotalLocalStorageGB = - config.InstanceRequirements.TotalLocalStorageGB || ({} as TotalLocalStorageGBRequest); - const subKey = key.replace('total-local-storage-gb-', ''); - config.InstanceRequirements.TotalLocalStorageGB![subKey === 'min' ? 'Min' : 'Max'] = parseFloat(value); - } else if (key.startsWith('baseline-ebs-bandwidth-mbps-')) { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.BaselineEbsBandwidthMbps = - config.InstanceRequirements.BaselineEbsBandwidthMbps || ({} as BaselineEbsBandwidthMbpsRequest); - const subKey = key.replace('baseline-ebs-bandwidth-mbps-', ''); - config.InstanceRequirements.BaselineEbsBandwidthMbps![subKey === 'min' ? 'Min' : 'Max'] = parseInt(value, 10); - } else if (key === 'accelerator-types') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.AcceleratorTypes = splitEc2OverrideListValue(value) as AcceleratorType[]; - } else if (key.startsWith('accelerator-count-')) { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.AcceleratorCount = - config.InstanceRequirements.AcceleratorCount || ({} as AcceleratorCountRequest); - const subKey = key.replace('accelerator-count-', ''); - config.InstanceRequirements.AcceleratorCount![subKey === 'min' ? 'Min' : 'Max'] = parseInt(value, 10); - } else if (key === 'accelerator-manufacturers') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.AcceleratorManufacturers = splitEc2OverrideListValue( - value, - ) as AcceleratorManufacturer[]; - } else if (key === 'accelerator-names') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.AcceleratorNames = splitEc2OverrideListValue(value) as AcceleratorName[]; - } else if (key.startsWith('accelerator-total-memory-mib-')) { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.AcceleratorTotalMemoryMiB = - config.InstanceRequirements.AcceleratorTotalMemoryMiB || ({} as AcceleratorTotalMemoryMiBRequest); - const subKey = key.replace('accelerator-total-memory-mib-', ''); - config.InstanceRequirements.AcceleratorTotalMemoryMiB![subKey === 'min' ? 'Min' : 'Max'] = parseInt(value, 10); - } else if (key.startsWith('network-bandwidth-gbps-')) { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.NetworkBandwidthGbps = - config.InstanceRequirements.NetworkBandwidthGbps || ({} as NetworkBandwidthGbpsRequest); - const subKey = key.replace('network-bandwidth-gbps-', ''); - config.InstanceRequirements.NetworkBandwidthGbps![subKey === 'min' ? 'Min' : 'Max'] = parseFloat(value); - } else if (key === 'allowed-instance-types') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.AllowedInstanceTypes = splitEc2OverrideListValue(value); - } else if (key === 'max-spot-price-as-percentage-of-optimal-on-demand-price') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.MaxSpotPriceAsPercentageOfOptimalOnDemandPrice = parseInt(value, 10); - } else if (key === 'baseline-performance-factors-cpu-reference-families') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.BaselinePerformanceFactors = - config.InstanceRequirements.BaselinePerformanceFactors || ({} as BaselinePerformanceFactorsRequest); - config.InstanceRequirements.BaselinePerformanceFactors.Cpu = - config.InstanceRequirements.BaselinePerformanceFactors.Cpu || ({} as CpuPerformanceFactorRequest); - config.InstanceRequirements.BaselinePerformanceFactors.Cpu.References = splitEc2OverrideListValue(value).map( - (family) => ({ InstanceFamily: family }), - ) as PerformanceFactorReferenceRequest[]; - } else if (key === 'require-encryption-in-transit') { - config.InstanceRequirements = config.InstanceRequirements || ({} as InstanceRequirementsRequest); - config.InstanceRequirements.RequireEncryptionInTransit = value.toLowerCase() === 'true'; - } - } - - return Object.keys(config).length > 0 ? config : undefined; -} - -function splitEc2OverrideListValue(value: string): string[] { - return value.split(EC2_OVERRIDE_LIST_VALUE_SEPARATOR); -} - -function getOrCreateBlockDeviceMapping( - config: Ec2OverrideConfig, - defaultBlockDeviceName?: string, -): FleetBlockDeviceMappingRequest { - config.BlockDeviceMappings = - config.BlockDeviceMappings || - ([defaultBlockDeviceName ? { DeviceName: defaultBlockDeviceName } : {}] as FleetBlockDeviceMappingRequest[]); - return config.BlockDeviceMappings[0]; -} - -function shouldLoadLaunchTemplateBlockDeviceName(labels: string[]): boolean { - const blockDeviceNameLabel = 'ghr-ec2-block-device-name:'; - let hasBlockDeviceOverride = false; - let hasBlockDeviceName = false; - - for (const label of labels) { - hasBlockDeviceOverride = - hasBlockDeviceOverride || label.startsWith('ghr-ec2-ebs-') || label.startsWith('ghr-ec2-block-device-'); - - hasBlockDeviceName = - hasBlockDeviceName || (label.startsWith(blockDeviceNameLabel) && label.slice(blockDeviceNameLabel.length) !== ''); - } - - return hasBlockDeviceOverride && !hasBlockDeviceName; -} - -async function getDefaultBlockDeviceNameFromLaunchTemplate(launchTemplateName: string): Promise { - const ec2Client = getTracedAWSV3Client(new EC2Client({ region: process.env.AWS_REGION })); - const launchTemplateVersions = await ec2Client.send( - new DescribeLaunchTemplateVersionsCommand({ - LaunchTemplateName: launchTemplateName, - Versions: ['$Default'], - }), - ); - const blockDeviceMappings = - launchTemplateVersions.LaunchTemplateVersions?.[0]?.LaunchTemplateData?.BlockDeviceMappings; - const blockDeviceName = - blockDeviceMappings?.find((blockDeviceMapping) => blockDeviceMapping.DeviceName && blockDeviceMapping.Ebs) - ?.DeviceName ?? blockDeviceMappings?.find((blockDeviceMapping) => blockDeviceMapping.DeviceName)?.DeviceName; - - if (!blockDeviceName) { - throw new Error(`Failed to determine block device name from launch template '${launchTemplateName}'.`); - } - - return blockDeviceName; -} - function labelsHash(labels: string[]): string { const prefix = 'ghr-'; diff --git a/lambdas/functions/control-plane/src/scale-runners/types.ts b/lambdas/functions/control-plane/src/scale-runners/types.ts new file mode 100644 index 0000000000..f3cb699b48 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/types.ts @@ -0,0 +1,47 @@ +export type LambdaRunnerSource = 'scale-up-lambda' | 'pool-lambda'; +export type GitHubRunnerType = 'Org' | 'Repo'; + +export interface RunnerGroup { + name: string; + id: number; +} + +export interface ActionRequestMessage { + id: number; + eventType: 'check_run' | 'workflow_job'; + repositoryName: string; + repositoryOwner: string; + installationId: number; + repoOwnerType: string; + retryCounter?: number; + labels?: string[]; +} + +export interface ActionRequestMessageSQS extends ActionRequestMessage { + messageId: string; +} + +export interface ActionRequestMessageRetry extends ActionRequestMessage { + retryCounter: number; +} + +export interface CreateGitHubRunnerConfig { + ephemeral: boolean; + ghesBaseUrl?: string; + enableJitConfig: boolean; + runnerLabels: string; + runnerGroup: string; + runnerNamePrefix: string; + runnerOwner: string; + runnerType: GitHubRunnerType; + disableAutoUpdate: boolean; + ssmTokenPath: string; + ssmConfigPath: string; + ssmParameterStoreTags: { Key: string; Value: string }[]; +} + +export interface EphemeralRunnerConfig { + runnerName: string; + runnerGroupId: number; + runnerLabels: string[]; +} From 344fbeb7bcd1fafd954db60c3f2ff17eec37a2a5 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 7 Jul 2026 22:05:49 +0200 Subject: [PATCH 03/46] refactor(scale-runners): split ec2 scale-down provider --- .../src/scale-runners/ec2-scale-down.test.ts | 890 ++++++++++++++++ .../src/scale-runners/ec2-scale-down.ts | 37 + .../scale-runners/scale-down-config.test.ts | 24 +- .../src/scale-runners/scale-down-config.ts | 18 + .../scale-down-provider-registry.ts | 22 + .../src/scale-runners/scale-down-provider.ts | 32 + .../src/scale-runners/scale-down.test.ts | 998 +++--------------- .../src/scale-runners/scale-down.ts | 203 ++-- 8 files changed, 1277 insertions(+), 947 deletions(-) create mode 100644 lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.test.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts create mode 100644 lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.test.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.test.ts new file mode 100644 index 0000000000..beda1af4e7 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.test.ts @@ -0,0 +1,890 @@ +import { Octokit } from '@octokit/rest'; +import { RequestError } from '@octokit/request-error'; +import moment from 'moment'; +import nock from 'nock'; + +import { RunnerInfo, RunnerList } from '../aws/ec2-runners.d'; +import * as ghAuth from '../github/auth'; +import { listEC2Runners, terminateRunner, tag, untag } from './../aws/ec2-runners'; +import { githubCache } from './cache'; +import { newestFirstStrategy, oldestFirstStrategy, scaleDown } from './scale-down'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; + +const mockOctokit = { + apps: { + getOrgInstallation: vi.fn(), + getRepoInstallation: vi.fn(), + }, + actions: { + listSelfHostedRunnersForRepo: vi.fn(), + listSelfHostedRunnersForOrg: vi.fn(), + deleteSelfHostedRunnerFromOrg: vi.fn(), + deleteSelfHostedRunnerFromRepo: vi.fn(), + getSelfHostedRunnerForOrg: vi.fn(), + getSelfHostedRunnerForRepo: vi.fn(), + }, + paginate: vi.fn(), +}; +vi.mock('@octokit/rest', () => ({ + Octokit: vi.fn().mockImplementation(function () { + return mockOctokit; + }), +})); + +vi.mock('./../aws/ec2-runners', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + tag: vi.fn(), + untag: vi.fn(), + terminateRunner: vi.fn(), + listEC2Runners: vi.fn(), + }; +}); +vi.mock('./../github/auth', async () => ({ + createGithubAppAuth: vi.fn(), + createGithubInstallationAuth: vi.fn(), + createOctokitClient: vi.fn(), +})); + +vi.mock('./cache', async () => ({ + githubCache: { + getRunner: vi.fn(), + addRunner: vi.fn(), + clients: new Map(), + runners: new Map(), + reset: vi.fn().mockImplementation(() => { + githubCache.clients.clear(); + githubCache.runners.clear(); + }), + }, +})); + +const mocktokit = Octokit as vi.MockedClass; +const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); +const mockedInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); +const mockCreateClient = vi.mocked(ghAuth.createOctokitClient); +const mockListRunners = vi.mocked(listEC2Runners); +const mockTagRunners = vi.mocked(tag); +const mockUntagRunners = vi.mocked(untag); +const mockTerminateRunners = vi.mocked(terminateRunner); + +export interface TestData { + repositoryName: string; + repositoryOwner: string; +} + +const cleanEnv = process.env; + +const ENVIRONMENT = 'unit-test-environment'; +const MINIMUM_TIME_RUNNING_IN_MINUTES = 30; +const MINIMUM_BOOT_TIME = 5; +const TEST_DATA: TestData = { + repositoryName: 'hello-world', + repositoryOwner: 'Codertocat', +}; + +interface RunnerTestItem extends RunnerList { + registered: boolean; + orphan: boolean; + shouldBeTerminated: boolean; +} + +describe('Scale down runners', () => { + beforeEach(() => { + process.env = { ...cleanEnv }; + process.env.GITHUB_APP_KEY_BASE64 = 'TEST_CERTIFICATE_DATA'; + process.env.GITHUB_APP_ID = '1337'; + process.env.GITHUB_APP_CLIENT_ID = 'TEST_CLIENT_ID'; + process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET'; + process.env.RUNNERS_MAXIMUM_COUNT = '3'; + process.env.SCALE_DOWN_CONFIG = '[]'; + process.env.ENVIRONMENT = ENVIRONMENT; + process.env.MINIMUM_RUNNING_TIME_IN_MINUTES = MINIMUM_TIME_RUNNING_IN_MINUTES.toString(); + process.env.RUNNER_BOOT_TIME_IN_MINUTES = MINIMUM_BOOT_TIME.toString(); + + nock.disableNetConnect(); + vi.clearAllMocks(); + vi.resetModules(); + githubCache.clients.clear(); + githubCache.runners.clear(); + mockOctokit.apps.getOrgInstallation.mockImplementation(() => ({ + data: { + id: 'ORG', + }, + })); + mockOctokit.apps.getRepoInstallation.mockImplementation(() => ({ + data: { + id: 'REPO', + }, + })); + + mockOctokit.paginate.mockResolvedValue([]); + mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation((repo) => { + // check if repo.runner_id contains the word "busy". If yes, throw an error else return 204 + if (repo.runner_id.includes('busy')) { + throw Error(); + } else { + return { status: 204 }; + } + }); + + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation((repo) => { + // check if repo.runner_id contains the word "busy". If yes, throw an error else return 204 + if (repo.runner_id.includes('busy')) { + throw Error(); + } else { + return { status: 204 }; + } + }); + + mockOctokit.actions.getSelfHostedRunnerForRepo.mockImplementation((repo) => { + if (repo.runner_id.includes('busy')) { + return { + data: { busy: true }, + }; + } else { + return { + data: { busy: false }, + }; + } + }); + mockOctokit.actions.getSelfHostedRunnerForOrg.mockImplementation((repo) => { + if (repo.runner_id.includes('busy')) { + return { + data: { busy: true }, + }; + } else { + return { + data: { busy: false }, + }; + } + }); + + mockTerminateRunners.mockImplementation(async () => { + return; + }); + mockedAppAuth.mockResolvedValue({ + type: 'app', + token: 'token', + appId: 1, + expiresAt: 'some-date', + }); + mockedInstallationAuth.mockResolvedValue({ + type: 'token', + tokenType: 'installation', + token: 'token', + createdAt: 'some-date', + expiresAt: 'some-date', + permissions: {}, + repositorySelection: 'all', + installationId: 0, + }); + mockCreateClient.mockResolvedValue(new mocktokit()); + }); + + const endpoints = ['https://api.github.com', 'https://github.enterprise.something', 'https://companyname.ghe.com']; + + describe.each(endpoints)('for %s', (endpoint) => { + beforeEach(() => { + if (endpoint.includes('enterprise') || endpoint.endsWith('.ghe.com')) { + process.env.GHES_URL = endpoint; + } + }); + + type RunnerType = 'Repo' | 'Org'; + const runnerTypes: RunnerType[] = ['Org', 'Repo']; + describe.each(runnerTypes)('For %s runners.', (type) => { + it('Should not call terminate when no runners online.', async () => { + // setup + mockAwsRunners([]); + + // act + await scaleDown(); + + // assert + expect(listEC2Runners).toHaveBeenCalledWith({ + environment: ENVIRONMENT, + }); + expect(terminateRunner).not.toHaveBeenCalled(); + expect(mockOctokit.apps.getRepoInstallation).not.toHaveBeenCalled(); + expect(mockOctokit.apps.getRepoInstallation).not.toHaveBeenCalled(); + }); + + it(`Should terminate runner without idle config ${type} runners.`, async () => { + // setup + const runners = [ + createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES - 1, true, false, false), + createRunnerTestData('idle-2', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 4, true, false, true), + createRunnerTestData('busy-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 3, true, false, false), + createRunnerTestData('booting-1', type, MINIMUM_BOOT_TIME - 1, false, false, false), + ]; + + mockGitHubRunners(runners); + mockListRunners.mockResolvedValue(runners); + mockAwsRunners(runners); + + await scaleDown(); + + // assert + expect(listEC2Runners).toHaveBeenCalledWith({ + environment: ENVIRONMENT, + }); + + if (type === 'Repo') { + expect(mockOctokit.apps.getRepoInstallation).toHaveBeenCalled(); + } else { + expect(mockOctokit.apps.getOrgInstallation).toHaveBeenCalled(); + } + + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it(`Should respect idle runner with minimum running time not exceeded.`, async () => { + // setup + const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES - 1, true, false, false)]; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await scaleDown(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it(`Should respect booting runner.`, async () => { + // setup + const runners = [createRunnerTestData('booting-1', type, MINIMUM_BOOT_TIME - 1, false, false, false)]; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await scaleDown(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it(`Should respect busy runner.`, async () => { + // setup + const runners = [createRunnerTestData('busy-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false)]; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await scaleDown(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it(`Should not terminate runner with bypass-removal tag set.`, async () => { + // setup + const runners = [ + createRunnerTestData('idle-with-bypass', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 10, true, false, false), + ]; + // Set bypass-removal tag + runners[0].bypassRemoval = true; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await scaleDown(); + + // assert + expect(terminateRunner).not.toHaveBeenCalled(); + checkNonTerminated(runners); + }); + + it(`Should not terminate orphaned runner with bypass-removal tag set.`, async () => { + // setup - orphan runner with bypass-removal tag + const orphanRunner = createRunnerTestData('orphan-bypass', type, MINIMUM_BOOT_TIME + 1, false, false, false); + orphanRunner.bypassRemoval = true; + + const idleRunner = createRunnerTestData('idle-1', type, MINIMUM_BOOT_TIME + 1, true, false, false); + const runners = [orphanRunner, idleRunner]; + + mockGitHubRunners([idleRunner]); + mockAwsRunners(runners); + + // act - first cycle marks orphan + await scaleDown(); + + // mark as orphan for next cycle + orphanRunner.orphan = true; + + // act - second cycle should skip termination due to bypass-removal + await scaleDown(); + + // assert - orphan runner should NOT be terminated + expect(terminateRunner).not.toHaveBeenCalledWith(orphanRunner.instanceId); + }); + + it(`Should not terminate a runner that became busy just before deregister runner.`, async () => { + // setup + const runners = [ + createRunnerTestData( + 'job-just-start-at-deregister-1', + type, + MINIMUM_TIME_RUNNING_IN_MINUTES + 1, + true, + false, + false, + ), + ]; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { + return { status: 500 }; + }); + + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { + return { status: 500 }; + }); + + // act and ensure no exception is thrown + await expect(scaleDown()).resolves.not.toThrow(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it(`Should terminate orphan (Non JIT)`, async () => { + // setup + const orphanRunner = createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, false, false); + const idleRunner = createRunnerTestData('idle-1', type, MINIMUM_BOOT_TIME + 1, true, false, false); + const runners = [orphanRunner, idleRunner]; + + mockGitHubRunners([idleRunner]); + mockAwsRunners(runners); + + // act + await scaleDown(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + + expect(mockTagRunners).toHaveBeenCalledWith(orphanRunner.instanceId, [ + { + Key: 'ghr:orphan', + Value: 'true', + }, + ]); + + expect(mockTagRunners).not.toHaveBeenCalledWith(idleRunner.instanceId, expect.anything()); + + // next cycle, update test data set orphan to true and terminate should be true + orphanRunner.orphan = true; + orphanRunner.shouldBeTerminated = true; + + // act + await scaleDown(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it('Should test if orphaned runner, untag if online and busy, else terminate (JIT)', async () => { + // arrange + const orphanRunner = createRunnerTestData( + 'orphan-jit', + type, + MINIMUM_BOOT_TIME + 1, + false, + true, + false, + undefined, + 1234567890, + ); + const runners = [orphanRunner]; + + mockGitHubRunners([]); + mockAwsRunners(runners); + + if (type === 'Repo') { + mockOctokit.actions.getSelfHostedRunnerForRepo.mockResolvedValueOnce({ + data: { id: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' }, + }); + } else { + mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ + data: { id: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' }, + }); + } + + // act + await scaleDown(); + + // assert + expect(mockUntagRunners).toHaveBeenCalledWith(orphanRunner.instanceId, [{ Key: 'ghr:orphan', Value: 'true' }]); + expect(mockTerminateRunners).not.toHaveBeenCalledWith(orphanRunner.instanceId); + + // arrange + if (type === 'Repo') { + mockOctokit.actions.getSelfHostedRunnerForRepo.mockResolvedValueOnce({ + data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'offline' }, + }); + } else { + mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ + data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'offline' }, + }); + } + + // act + await scaleDown(); + + // assert + expect(mockTerminateRunners).toHaveBeenCalledWith(orphanRunner.instanceId); + }); + + it('Should handle 404 error when checking orphaned runner (JIT) - treat as orphaned', async () => { + // arrange + const orphanRunner = createRunnerTestData( + 'orphan-jit-404', + type, + MINIMUM_BOOT_TIME + 1, + false, + true, + true, // should be terminated when 404 + undefined, + 1234567890, + ); + const runners = [orphanRunner]; + + mockGitHubRunners([]); + mockAwsRunners(runners); + + // Mock 404 error response + const error404 = new RequestError('Runner not found', 404, { + request: { + method: 'GET', + url: 'https://api.github.com/test', + headers: {}, + }, + }); + + if (type === 'Repo') { + mockOctokit.actions.getSelfHostedRunnerForRepo.mockRejectedValueOnce(error404); + } else { + mockOctokit.actions.getSelfHostedRunnerForOrg.mockRejectedValueOnce(error404); + } + + // act + await scaleDown(); + + // assert - should terminate since 404 means runner doesn't exist on GitHub + expect(mockTerminateRunners).toHaveBeenCalledWith(orphanRunner.instanceId); + }); + + it('Should handle 404 error when checking runner busy state - treat as not busy', async () => { + // arrange + const runner = createRunnerTestData( + 'runner-404', + type, + MINIMUM_TIME_RUNNING_IN_MINUTES + 1, + true, + false, + true, // should be terminated since not busy due to 404 + ); + const runners = [runner]; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // Mock 404 error response for busy state check + const error404 = new RequestError('Runner not found', 404, { + request: { + method: 'GET', + url: 'https://api.github.com/test', + headers: {}, + }, + }); + + if (type === 'Repo') { + mockOctokit.actions.getSelfHostedRunnerForRepo.mockRejectedValueOnce(error404); + } else { + mockOctokit.actions.getSelfHostedRunnerForOrg.mockRejectedValueOnce(error404); + } + + // act + await scaleDown(); + + // assert - should terminate since 404 means runner is not busy + checkTerminated(runners); + }); + + it('Should re-throw non-404 errors when checking runner state', async () => { + // arrange + const orphanRunner = createRunnerTestData( + 'orphan-error', + type, + MINIMUM_BOOT_TIME + 1, + false, + true, + false, + undefined, + 1234567890, + ); + const runners = [orphanRunner]; + + mockGitHubRunners([]); + mockAwsRunners(runners); + + // Mock non-404 error response + const error500 = new RequestError('Internal server error', 500, { + request: { + method: 'GET', + url: 'https://api.github.com/test', + headers: {}, + }, + }); + + if (type === 'Repo') { + mockOctokit.actions.getSelfHostedRunnerForRepo.mockRejectedValueOnce(error500); + } else { + mockOctokit.actions.getSelfHostedRunnerForOrg.mockRejectedValueOnce(error500); + } + + // act & assert - should not throw because error handling is in terminateOrphan + await expect(scaleDown()).resolves.not.toThrow(); + + // Should not terminate since the error was not a 404 + expect(terminateRunner).not.toHaveBeenCalledWith(orphanRunner.instanceId); + }); + + it(`Should ignore errors when termination orphan fails.`, async () => { + // setup + const orphanRunner = createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, true, true); + const runners = [orphanRunner]; + + mockGitHubRunners([]); + mockAwsRunners(runners); + mockTerminateRunners.mockImplementation(() => { + throw new Error('Failed to terminate'); + }); + + // act + await scaleDown(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + describe('When orphan termination fails', () => { + it(`Should not throw in case of list runner exception.`, async () => { + // setup + const runners = [createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, true, true)]; + + mockGitHubRunners([]); + mockListRunners.mockRejectedValueOnce(new Error('Failed to list runners')); + mockAwsRunners(runners); + + // ac + await scaleDown(); + + // assert + checkNonTerminated(runners); + }); + + it(`Should not throw in case of terminate runner exception.`, async () => { + // setup + const runners = [createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, true, true)]; + + mockGitHubRunners([]); + mockAwsRunners(runners); + mockTerminateRunners.mockRejectedValue(new Error('Failed to terminate')); + + // act and ensure no exception is thrown + await scaleDown(); + + // assert + checkNonTerminated(runners); + }); + }); + + it(`Should not terminate instance in case de-register fails.`, async () => { + // setup + const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false)]; + + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { + return { status: 500 }; + }); + mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { + return { status: 500 }; + }); + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act and should resolve + await expect(scaleDown()).resolves.not.toThrow(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it(`Should not throw an exception in case of failure during removing a runner.`, async () => { + // setup + const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, true, false)]; + + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { + throw new Error('Failed to delete runner'); + }); + mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { + throw new Error('Failed to delete runner'); + }); + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await expect(scaleDown()).resolves.not.toThrow(); + }); + + it(`Should not terminate instance when de-registration throws an error.`, async () => { + // setup - runner should NOT be terminated because de-registration fails + const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false)]; + + const error502 = new RequestError('Server Error', 502, { + request: { + method: 'DELETE', + url: 'https://api.github.com/test', + headers: {}, + }, + }); + + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { + throw error502; + }); + mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { + throw error502; + }); + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await expect(scaleDown()).resolves.not.toThrow(); + + // assert - should NOT terminate since de-registration failed + expect(terminateRunner).not.toHaveBeenCalled(); + }); + + const evictionStrategies = ['oldest_first', 'newest_first']; + describe.each(evictionStrategies)('When idle config defined', (evictionStrategy) => { + const defaultConfig = { + idleCount: 1, + cron: '* * * * * *', + timeZone: 'Europe/Amsterdam', + evictionStrategy, + }; + + beforeEach(() => { + process.env.SCALE_DOWN_CONFIG = JSON.stringify([defaultConfig]); + }); + + it(`Should terminate based on the the idle config with ${evictionStrategy} eviction strategy`, async () => { + // setup + const runnerToTerminateTime = + evictionStrategy === 'oldest_first' + ? MINIMUM_TIME_RUNNING_IN_MINUTES + 5 + : MINIMUM_TIME_RUNNING_IN_MINUTES + 1; + const runners = [ + createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 4, true, false, false), + createRunnerTestData('idle-to-terminate', type, runnerToTerminateTime, true, false, true), + ]; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await scaleDown(); + + // assert + const runnersToTerminate = runners.filter((r) => r.shouldBeTerminated); + for (const toTerminate of runnersToTerminate) { + expect(terminateRunner).toHaveBeenCalledWith(toTerminate.instanceId); + } + + const runnersNotToTerminate = runners.filter((r) => !r.shouldBeTerminated); + for (const notTerminated of runnersNotToTerminate) { + expect(terminateRunner).not.toHaveBeenCalledWith(notTerminated.instanceId); + } + }); + }); + }); + }); + + describe('When runners are sorted', () => { + const runners: RunnerInfo[] = [ + { + instanceId: '1', + launchTime: moment(new Date()).subtract(1, 'minute').toDate(), + owner: 'owner', + type: 'type', + }, + { + instanceId: '3', + launchTime: moment(new Date()).subtract(3, 'minute').toDate(), + owner: 'owner', + type: 'type', + }, + { + instanceId: '2', + launchTime: moment(new Date()).subtract(2, 'minute').toDate(), + owner: 'owner', + type: 'type', + }, + { + instanceId: '0', + launchTime: moment(new Date()).subtract(0, 'minute').toDate(), + owner: 'owner', + type: 'type', + }, + ]; + + it('Should sort runners descending for eviction strategy oldest first te keep the youngest.', () => { + runners.sort(oldestFirstStrategy); + expect(runners[0].instanceId).toEqual('0'); + expect(runners[1].instanceId).toEqual('1'); + expect(runners[2].instanceId).toEqual('2'); + expect(runners[3].instanceId).toEqual('3'); + }); + + it('Should sort runners ascending for eviction strategy newest first te keep oldest.', () => { + runners.sort(newestFirstStrategy); + expect(runners[0].instanceId).toEqual('3'); + expect(runners[1].instanceId).toEqual('2'); + expect(runners[2].instanceId).toEqual('1'); + expect(runners[3].instanceId).toEqual('0'); + }); + + it('Should sort runners with equal launch time.', () => { + const runnersTest = [...runners]; + const same = moment(new Date()).subtract(4, 'minute').toDate(); + runnersTest.push({ + instanceId: '4', + launchTime: same, + owner: 'owner', + type: 'type', + }); + runnersTest.push({ + instanceId: '5', + launchTime: same, + owner: 'owner', + type: 'type', + }); + runnersTest.sort(oldestFirstStrategy); + expect(runnersTest[3].launchTime).not.toEqual(same); + expect(runnersTest[4].launchTime).toEqual(same); + expect(runnersTest[5].launchTime).toEqual(same); + + runnersTest.sort(newestFirstStrategy); + expect(runnersTest[3].launchTime).not.toEqual(same); + expect(runnersTest[1].launchTime).toEqual(same); + expect(runnersTest[0].launchTime).toEqual(same); + }); + + it('Should sort runners even when launch time is undefined.', () => { + const runnersTest = [ + { + instanceId: '0', + launchTime: undefined, + owner: 'owner', + type: 'type', + }, + { + instanceId: '1', + launchTime: moment(new Date()).subtract(3, 'minute').toDate(), + owner: 'owner', + type: 'type', + }, + { + instanceId: '0', + launchTime: undefined, + owner: 'owner', + type: 'type', + }, + ]; + runnersTest.sort(oldestFirstStrategy); + expect(runnersTest[0].launchTime).toBeUndefined(); + expect(runnersTest[1].launchTime).toBeDefined(); + expect(runnersTest[2].launchTime).not.toBeDefined(); + }); + }); +}); + +function mockAwsRunners(runners: RunnerTestItem[]) { + mockListRunners.mockImplementation(async (filter) => { + return runners.filter((r) => !filter?.orphan || filter?.orphan === r.orphan); + }); +} + +function checkNonTerminated(runners: RunnerTestItem[]) { + const notTerminated = runners.filter((r) => !r.shouldBeTerminated); + for (const toTerminate of notTerminated) { + expect(terminateRunner).not.toHaveBeenCalledWith(toTerminate.instanceId); + } +} + +function checkTerminated(runners: RunnerTestItem[]) { + const runnersToTerminate = runners.filter((r) => r.shouldBeTerminated); + expect(terminateRunner).toHaveBeenCalledTimes(runnersToTerminate.length); + for (const toTerminate of runnersToTerminate) { + expect(terminateRunner).toHaveBeenCalledWith(toTerminate.instanceId); + } +} + +function mockGitHubRunners(runners: RunnerTestItem[]) { + mockOctokit.paginate.mockResolvedValue( + runners + .filter((r) => r.registered) + .map((r) => { + return { + id: r.instanceId, + name: r.instanceId, + }; + }), + ); +} + +function createRunnerTestData( + name: string, + type: 'Org' | 'Repo', + minutesLaunchedAgo: number, + registered: boolean, + orphan: boolean, + shouldBeTerminated: boolean, + owner?: string, + runnerId?: number, +): RunnerTestItem { + return { + instanceId: `i-${name}-${type.toLowerCase()}`, + launchTime: moment(new Date()).subtract(minutesLaunchedAgo, 'minutes').toDate(), + type, + owner: owner + ? owner + : type === 'Repo' + ? `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}` + : `${TEST_DATA.repositoryOwner}`, + registered, + orphan, + shouldBeTerminated, + runnerId: runnerId !== undefined ? String(runnerId) : undefined, + bypassRemoval: false, + }; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts new file mode 100644 index 0000000000..2747d7b149 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts @@ -0,0 +1,37 @@ +import { bootTimeExceeded, listEC2Runners, tag, terminateRunner, untag } from './../aws/ec2-runners'; +import type { RunnerList } from './../aws/ec2-runners.d'; +import type { + RunnerList as ScaleDownRunnerList, + ScaleDownRunnerProvider, + ScaleDownRunnerProviderStrategy, +} from './scale-down-provider'; + +export function createEc2ScaleDownProvider(): ScaleDownRunnerProvider { + return { + name: 'EC2', + list: async (environment, orphan) => (await listEC2Runners({ environment, orphan })).map(toScaleDownRunner), + bootTimeExceeded, + markOrphan: async (id) => await tag(id, [{ Key: 'ghr:orphan', Value: 'true' }]), + unmarkOrphan: async (id) => await untag(id, [{ Key: 'ghr:orphan', Value: 'true' }]), + terminate: terminateRunner, + }; +} + +export const ec2ScaleDownRunnerProviderStrategy: ScaleDownRunnerProviderStrategy = { + type: 'ec2', + create: createEc2ScaleDownProvider, +}; + +function toScaleDownRunner(runner: RunnerList): ScaleDownRunnerList { + return { + id: runner.instanceId, + launchTime: runner.launchTime, + owner: runner.owner, + type: runner.type, + repo: runner.repo, + org: runner.org, + orphan: runner.orphan, + githubRunnerId: runner.runnerId, + bypassRemoval: runner.bypassRemoval, + }; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts index ff2325128a..aa71d3173a 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts @@ -1,6 +1,12 @@ import moment from 'moment-timezone'; -import { EvictionStrategy, ScalingDownConfigList, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config'; +import { + EvictionStrategy, + ScalingDownConfigList, + getEvictionStrategy, + getIdleRunnerCount, + getScaleDownRunnerProviderType, +} from './scale-down-config'; import { describe, it, expect } from 'vitest'; const DEFAULT_TIMEZONE = 'America/Los_Angeles'; @@ -54,4 +60,20 @@ describe('scaleDownConfig', () => { expect(getEvictionStrategy(scaleDownConfig)).toEqual(DEFAULT_EVICTION_STRATEGY); }); }); + + describe('Determine runner provider type.', () => { + it('Defaults to ec2 when no idle config is defined', async () => { + expect(getScaleDownRunnerProviderType([], 'ec2')).toEqual('ec2'); + }); + + it('Defaults to ec2 when config has no type', async () => { + const scaleDownConfig = getConfig(['* * * * * *']); + expect(getScaleDownRunnerProviderType(scaleDownConfig, 'ec2')).toEqual('ec2'); + }); + + it('Uses configured ec2 type', async () => { + const scaleDownConfig = getConfig(['* * * * * *']).map((config) => ({ ...config, type: 'ec2' as const })); + expect(getScaleDownRunnerProviderType(scaleDownConfig, 'microvm')).toEqual('ec2'); + }); + }); }); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.ts index 2d28c38cfc..c3c51a6432 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.ts @@ -2,9 +2,12 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import parser from 'cron-parser'; import moment from 'moment'; +import type { ScaleDownRunnerProviderType } from './scale-down-provider'; + export type ScalingDownConfigList = ScalingDownConfig[]; export type EvictionStrategy = 'newest_first' | 'oldest_first'; export interface ScalingDownConfig { + type?: ScaleDownRunnerProviderType; cron: string; idleCount: number; timeZone: string; @@ -41,3 +44,18 @@ export function getEvictionStrategy(scalingDownConfigs: ScalingDownConfigList): } return 'oldest_first'; } + +export function getScaleDownRunnerProviderType( + scalingDownConfigs: ScalingDownConfigList, + defaultType: ScaleDownRunnerProviderType, +): ScaleDownRunnerProviderType { + const configuredTypes = new Set( + scalingDownConfigs.map((scalingDownConfig) => scalingDownConfig.type ?? defaultType), + ); + + if (configuredTypes.size > 1) { + throw new Error(`Multiple scale-down runner provider types are not supported in a single scale-down config.`); + } + + return configuredTypes.values().next().value ?? defaultType; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts new file mode 100644 index 0000000000..ed6501ff70 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts @@ -0,0 +1,22 @@ +import { ec2ScaleDownRunnerProviderStrategy } from './ec2-scale-down'; +import type { + ScaleDownRunnerProvider, + ScaleDownRunnerProviderStrategy, + ScaleDownRunnerProviderType, +} from './scale-down-provider'; + +const scaleDownRunnerProviderStrategies: ScaleDownRunnerProviderStrategy[] = [ec2ScaleDownRunnerProviderStrategy]; + +export function getDefaultScaleDownRunnerProviderType(): ScaleDownRunnerProviderType { + return scaleDownRunnerProviderStrategies[0].type; +} + +export function createScaleDownRunnerProvider(type: ScaleDownRunnerProviderType): ScaleDownRunnerProvider { + const strategy = scaleDownRunnerProviderStrategies.find((strategy) => strategy.type === type); + + if (!strategy) { + throw new Error(`Unsupported scale-down runner provider type: ${type}`); + } + + return strategy.create(); +} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts new file mode 100644 index 0000000000..712b281635 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts @@ -0,0 +1,32 @@ +export interface RunnerList { + id: string; + launchTime?: Date; + owner?: string; + type?: string; + repo?: string; + org?: string; + orphan?: boolean; + githubRunnerId?: string; + bypassRemoval?: boolean; +} + +export interface RunnerInfo extends RunnerList { + owner: string; + type: string; +} + +export type ScaleDownRunnerProviderType = string; + +export interface ScaleDownRunnerProvider { + name: string; + list(environment: string, orphan?: boolean): Promise; + bootTimeExceeded(runner: RunnerInfo): boolean; + markOrphan(id: string): Promise; + unmarkOrphan(id: string): Promise; + terminate(id: string): Promise; +} + +export interface ScaleDownRunnerProviderStrategy { + type: ScaleDownRunnerProviderType; + create(): ScaleDownRunnerProvider; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts index 730cf29bf7..a82f971b1a 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts @@ -1,14 +1,21 @@ -import { Octokit } from '@octokit/rest'; -import { RequestError } from '@octokit/request-error'; import moment from 'moment'; -import nock from 'nock'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { RunnerInfo, RunnerList } from '../aws/runners.d'; import * as ghAuth from '../github/auth'; -import { listEC2Runners, terminateRunner, tag, untag } from './../aws/runners'; import { githubCache } from './cache'; import { newestFirstStrategy, oldestFirstStrategy, scaleDown } from './scale-down'; -import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { createScaleDownRunnerProvider, getDefaultScaleDownRunnerProviderType } from './scale-down-provider-registry'; +import type { Octokit } from '@octokit/rest'; +import type { RunnerInfo, RunnerList } from './scale-down-provider'; + +const testProvider = vi.hoisted(() => ({ + name: 'TestProvider', + list: vi.fn(), + bootTimeExceeded: vi.fn(), + markOrphan: vi.fn(), + unmarkOrphan: vi.fn(), + terminate: vi.fn(), +})); const mockOctokit = { apps: { @@ -16,875 +23,178 @@ const mockOctokit = { getRepoInstallation: vi.fn(), }, actions: { - listSelfHostedRunnersForRepo: vi.fn(), listSelfHostedRunnersForOrg: vi.fn(), - deleteSelfHostedRunnerFromOrg: vi.fn(), - deleteSelfHostedRunnerFromRepo: vi.fn(), + listSelfHostedRunnersForRepo: vi.fn(), getSelfHostedRunnerForOrg: vi.fn(), getSelfHostedRunnerForRepo: vi.fn(), }, paginate: vi.fn(), }; -vi.mock('@octokit/rest', () => ({ - Octokit: vi.fn().mockImplementation(function () { - return mockOctokit; - }), -})); -vi.mock('./../aws/runners', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - tag: vi.fn(), - untag: vi.fn(), - terminateRunner: vi.fn(), - listEC2Runners: vi.fn(), - }; -}); -vi.mock('./../github/auth', async () => ({ +vi.mock('../github/auth', async () => ({ createGithubAppAuth: vi.fn(), createGithubInstallationAuth: vi.fn(), createOctokitClient: vi.fn(), })); -vi.mock('./cache', async () => ({ - githubCache: { - getRunner: vi.fn(), - addRunner: vi.fn(), - clients: new Map(), - runners: new Map(), - reset: vi.fn().mockImplementation(() => { - githubCache.clients.clear(); - githubCache.runners.clear(); - }), - }, +vi.mock('./scale-down-provider-registry', () => ({ + createScaleDownRunnerProvider: vi.fn(() => testProvider), + getDefaultScaleDownRunnerProviderType: vi.fn(() => 'test-provider'), })); -const mocktokit = Octokit as vi.MockedClass; -const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); -const mockedInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); -const mockCreateClient = vi.mocked(ghAuth.createOctokitClient); -const mockListRunners = vi.mocked(listEC2Runners); -const mockTagRunners = vi.mocked(tag); -const mockUntagRunners = vi.mocked(untag); -const mockTerminateRunners = vi.mocked(terminateRunner); - -export interface TestData { - repositoryName: string; - repositoryOwner: string; -} - const cleanEnv = process.env; - -const ENVIRONMENT = 'unit-test-environment'; -const MINIMUM_TIME_RUNNING_IN_MINUTES = 30; -const MINIMUM_BOOT_TIME = 5; -const TEST_DATA: TestData = { - repositoryName: 'hello-world', - repositoryOwner: 'Codertocat', -}; - -interface RunnerTestItem extends RunnerList { - registered: boolean; - orphan: boolean; - shouldBeTerminated: boolean; +const mockedCreateGithubAppAuth = vi.mocked(ghAuth.createGithubAppAuth); +const mockedCreateGithubInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); +const mockedCreateOctokitClient = vi.mocked(ghAuth.createOctokitClient); +const mockedCreateScaleDownRunnerProvider = vi.mocked(createScaleDownRunnerProvider); +const mockedGetDefaultScaleDownRunnerProviderType = vi.mocked(getDefaultScaleDownRunnerProviderType); + +function setDefaults() { + process.env = { ...cleanEnv }; + process.env.GITHUB_APP_KEY_BASE64 = 'TEST_CERTIFICATE_DATA'; + process.env.GITHUB_APP_ID = '1337'; + process.env.GITHUB_APP_CLIENT_ID = 'TEST_CLIENT_ID'; + process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET'; + process.env.SCALE_DOWN_CONFIG = '[]'; + process.env.ENVIRONMENT = 'unit-test-environment'; + process.env.MINIMUM_RUNNING_TIME_IN_MINUTES = '30'; + process.env.RUNNER_BOOT_TIME_IN_MINUTES = '5'; } -describe('Scale down runners', () => { - beforeEach(() => { - process.env = { ...cleanEnv }; - process.env.GITHUB_APP_KEY_BASE64 = 'TEST_CERTIFICATE_DATA'; - process.env.GITHUB_APP_ID = '1337'; - process.env.GITHUB_APP_CLIENT_ID = 'TEST_CLIENT_ID'; - process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET'; - process.env.RUNNERS_MAXIMUM_COUNT = '3'; - process.env.SCALE_DOWN_CONFIG = '[]'; - process.env.ENVIRONMENT = ENVIRONMENT; - process.env.MINIMUM_RUNNING_TIME_IN_MINUTES = MINIMUM_TIME_RUNNING_IN_MINUTES.toString(); - process.env.RUNNER_BOOT_TIME_IN_MINUTES = MINIMUM_BOOT_TIME.toString(); - - nock.disableNetConnect(); - vi.clearAllMocks(); - vi.resetModules(); - githubCache.clients.clear(); - githubCache.runners.clear(); - mockOctokit.apps.getOrgInstallation.mockImplementation(() => ({ - data: { - id: 'ORG', - }, - })); - mockOctokit.apps.getRepoInstallation.mockImplementation(() => ({ - data: { - id: 'REPO', - }, - })); - - mockOctokit.paginate.mockResolvedValue([]); - mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation((repo) => { - // check if repo.runner_id contains the word "busy". If yes, throw an error else return 204 - if (repo.runner_id.includes('busy')) { - throw Error(); - } else { - return { status: 204 }; - } - }); - - mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation((repo) => { - // check if repo.runner_id contains the word "busy". If yes, throw an error else return 204 - if (repo.runner_id.includes('busy')) { - throw Error(); - } else { - return { status: 204 }; - } - }); - - mockOctokit.actions.getSelfHostedRunnerForRepo.mockImplementation((repo) => { - if (repo.runner_id.includes('busy')) { - return { - data: { busy: true }, - }; - } else { - return { - data: { busy: false }, - }; - } - }); - mockOctokit.actions.getSelfHostedRunnerForOrg.mockImplementation((repo) => { - if (repo.runner_id.includes('busy')) { - return { - data: { busy: true }, - }; - } else { - return { - data: { busy: false }, - }; - } - }); - - mockTerminateRunners.mockImplementation(async () => { - return; - }); - mockedAppAuth.mockResolvedValue({ - type: 'app', - token: 'token', - appId: 1, - expiresAt: 'some-date', - }); - mockedInstallationAuth.mockResolvedValue({ - type: 'token', - tokenType: 'installation', - token: 'token', - createdAt: 'some-date', - expiresAt: 'some-date', - permissions: {}, - repositorySelection: 'all', - installationId: 0, - }); - mockCreateClient.mockResolvedValue(new mocktokit()); +beforeEach(() => { + vi.clearAllMocks(); + setDefaults(); + githubCache.clients.clear(); + githubCache.runners.clear(); + + testProvider.list.mockResolvedValue([]); + testProvider.bootTimeExceeded.mockReturnValue(false); + testProvider.markOrphan.mockResolvedValue(undefined); + testProvider.unmarkOrphan.mockResolvedValue(undefined); + testProvider.terminate.mockResolvedValue(undefined); + + mockedCreateGithubAppAuth.mockResolvedValue({ + type: 'app', + token: 'app-token', + appId: 1, + expiresAt: 'some-date', }); + mockedCreateGithubInstallationAuth.mockResolvedValue({ + type: 'token', + tokenType: 'installation', + token: 'installation-token', + createdAt: 'some-date', + expiresAt: 'some-date', + permissions: {}, + repositorySelection: 'all', + installationId: 2, + }); + mockedCreateOctokitClient.mockResolvedValue(mockOctokit as unknown as Octokit); + mockOctokit.apps.getOrgInstallation.mockResolvedValue({ data: { id: 2 } }); + mockOctokit.apps.getRepoInstallation.mockResolvedValue({ data: { id: 2 } }); + mockOctokit.paginate.mockResolvedValue([]); +}); - const endpoints = ['https://api.github.com', 'https://github.enterprise.something', 'https://companyname.ghe.com']; - - describe.each(endpoints)('for %s', (endpoint) => { - beforeEach(() => { - if (endpoint.includes('enterprise') || endpoint.endsWith('.ghe.com')) { - process.env.GHES_URL = endpoint; - } - }); - - type RunnerType = 'Repo' | 'Org'; - const runnerTypes: RunnerType[] = ['Org', 'Repo']; - describe.each(runnerTypes)('For %s runners.', (type) => { - it('Should not call terminate when no runners online.', async () => { - // setup - mockAwsRunners([]); - - // act - await scaleDown(); - - // assert - expect(listEC2Runners).toHaveBeenCalledWith({ - environment: ENVIRONMENT, - }); - expect(terminateRunner).not.toHaveBeenCalled(); - expect(mockOctokit.apps.getRepoInstallation).not.toHaveBeenCalled(); - expect(mockOctokit.apps.getRepoInstallation).not.toHaveBeenCalled(); - }); - - it(`Should terminate runner without idle config ${type} runners.`, async () => { - // setup - const runners = [ - createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES - 1, true, false, false), - createRunnerTestData('idle-2', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 4, true, false, true), - createRunnerTestData('busy-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 3, true, false, false), - createRunnerTestData('booting-1', type, MINIMUM_BOOT_TIME - 1, false, false, false), - ]; - - mockGitHubRunners(runners); - mockListRunners.mockResolvedValue(runners); - mockAwsRunners(runners); - - await scaleDown(); - - // assert - expect(listEC2Runners).toHaveBeenCalledWith({ - environment: ENVIRONMENT, - }); - - if (type === 'Repo') { - expect(mockOctokit.apps.getRepoInstallation).toHaveBeenCalled(); - } else { - expect(mockOctokit.apps.getOrgInstallation).toHaveBeenCalled(); - } - - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it(`Should respect idle runner with minimum running time not exceeded.`, async () => { - // setup - const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES - 1, true, false, false)]; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await scaleDown(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it(`Should respect booting runner.`, async () => { - // setup - const runners = [createRunnerTestData('booting-1', type, MINIMUM_BOOT_TIME - 1, false, false, false)]; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await scaleDown(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it(`Should respect busy runner.`, async () => { - // setup - const runners = [createRunnerTestData('busy-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false)]; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await scaleDown(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it(`Should not terminate runner with bypass-removal tag set.`, async () => { - // setup - const runners = [ - createRunnerTestData('idle-with-bypass', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 10, true, false, false), - ]; - // Set bypass-removal tag - runners[0].bypassRemoval = true; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await scaleDown(); - - // assert - expect(terminateRunner).not.toHaveBeenCalled(); - checkNonTerminated(runners); - }); - - it(`Should not terminate orphaned runner with bypass-removal tag set.`, async () => { - // setup - orphan runner with bypass-removal tag - const orphanRunner = createRunnerTestData('orphan-bypass', type, MINIMUM_BOOT_TIME + 1, false, false, false); - orphanRunner.bypassRemoval = true; - - const idleRunner = createRunnerTestData('idle-1', type, MINIMUM_BOOT_TIME + 1, true, false, false); - const runners = [orphanRunner, idleRunner]; - - mockGitHubRunners([idleRunner]); - mockAwsRunners(runners); - - // act - first cycle marks orphan - await scaleDown(); - - // mark as orphan for next cycle - orphanRunner.orphan = true; - - // act - second cycle should skip termination due to bypass-removal - await scaleDown(); - - // assert - orphan runner should NOT be terminated - expect(terminateRunner).not.toHaveBeenCalledWith(orphanRunner.instanceId); - }); - - it(`Should not terminate a runner that became busy just before deregister runner.`, async () => { - // setup - const runners = [ - createRunnerTestData( - 'job-just-start-at-deregister-1', - type, - MINIMUM_TIME_RUNNING_IN_MINUTES + 1, - true, - false, - false, - ), - ]; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { - return { status: 500 }; - }); - - mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { - return { status: 500 }; - }); - - // act and ensure no exception is thrown - await expect(scaleDown()).resolves.not.toThrow(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it(`Should terminate orphan (Non JIT)`, async () => { - // setup - const orphanRunner = createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, false, false); - const idleRunner = createRunnerTestData('idle-1', type, MINIMUM_BOOT_TIME + 1, true, false, false); - const runners = [orphanRunner, idleRunner]; - - mockGitHubRunners([idleRunner]); - mockAwsRunners(runners); - - // act - await scaleDown(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - - expect(mockTagRunners).toHaveBeenCalledWith(orphanRunner.instanceId, [ - { - Key: 'ghr:orphan', - Value: 'true', - }, - ]); - - expect(mockTagRunners).not.toHaveBeenCalledWith(idleRunner.instanceId, expect.anything()); - - // next cycle, update test data set orphan to true and terminate should be true - orphanRunner.orphan = true; - orphanRunner.shouldBeTerminated = true; - - // act - await scaleDown(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it('Should test if orphaned runner, untag if online and busy, else terminate (JIT)', async () => { - // arrange - const orphanRunner = createRunnerTestData( - 'orphan-jit', - type, - MINIMUM_BOOT_TIME + 1, - false, - true, - false, - undefined, - 1234567890, - ); - const runners = [orphanRunner]; - - mockGitHubRunners([]); - mockAwsRunners(runners); - - if (type === 'Repo') { - mockOctokit.actions.getSelfHostedRunnerForRepo.mockResolvedValueOnce({ - data: { id: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' }, - }); - } else { - mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ - data: { id: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' }, - }); - } - - // act - await scaleDown(); - - // assert - expect(mockUntagRunners).toHaveBeenCalledWith(orphanRunner.instanceId, [{ Key: 'ghr:orphan', Value: 'true' }]); - expect(mockTerminateRunners).not.toHaveBeenCalledWith(orphanRunner.instanceId); - - // arrange - if (type === 'Repo') { - mockOctokit.actions.getSelfHostedRunnerForRepo.mockResolvedValueOnce({ - data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'offline' }, - }); - } else { - mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ - data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'offline' }, - }); - } - - // act - await scaleDown(); - - // assert - expect(mockTerminateRunners).toHaveBeenCalledWith(orphanRunner.instanceId); - }); - - it('Should handle 404 error when checking orphaned runner (JIT) - treat as orphaned', async () => { - // arrange - const orphanRunner = createRunnerTestData( - 'orphan-jit-404', - type, - MINIMUM_BOOT_TIME + 1, - false, - true, - true, // should be terminated when 404 - undefined, - 1234567890, - ); - const runners = [orphanRunner]; - - mockGitHubRunners([]); - mockAwsRunners(runners); - - // Mock 404 error response - const error404 = new RequestError('Runner not found', 404, { - request: { - method: 'GET', - url: 'https://api.github.com/test', - headers: {}, - }, - }); - - if (type === 'Repo') { - mockOctokit.actions.getSelfHostedRunnerForRepo.mockRejectedValueOnce(error404); - } else { - mockOctokit.actions.getSelfHostedRunnerForOrg.mockRejectedValueOnce(error404); - } - - // act - await scaleDown(); - - // assert - should terminate since 404 means runner doesn't exist on GitHub - expect(mockTerminateRunners).toHaveBeenCalledWith(orphanRunner.instanceId); - }); - - it('Should handle 404 error when checking runner busy state - treat as not busy', async () => { - // arrange - const runner = createRunnerTestData( - 'runner-404', - type, - MINIMUM_TIME_RUNNING_IN_MINUTES + 1, - true, - false, - true, // should be terminated since not busy due to 404 - ); - const runners = [runner]; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // Mock 404 error response for busy state check - const error404 = new RequestError('Runner not found', 404, { - request: { - method: 'GET', - url: 'https://api.github.com/test', - headers: {}, - }, - }); - - if (type === 'Repo') { - mockOctokit.actions.getSelfHostedRunnerForRepo.mockRejectedValueOnce(error404); - } else { - mockOctokit.actions.getSelfHostedRunnerForOrg.mockRejectedValueOnce(error404); - } - - // act - await scaleDown(); - - // assert - should terminate since 404 means runner is not busy - checkTerminated(runners); - }); - - it('Should re-throw non-404 errors when checking runner state', async () => { - // arrange - const orphanRunner = createRunnerTestData( - 'orphan-error', - type, - MINIMUM_BOOT_TIME + 1, - false, - true, - false, - undefined, - 1234567890, - ); - const runners = [orphanRunner]; - - mockGitHubRunners([]); - mockAwsRunners(runners); - - // Mock non-404 error response - const error500 = new RequestError('Internal server error', 500, { - request: { - method: 'GET', - url: 'https://api.github.com/test', - headers: {}, - }, - }); - - if (type === 'Repo') { - mockOctokit.actions.getSelfHostedRunnerForRepo.mockRejectedValueOnce(error500); - } else { - mockOctokit.actions.getSelfHostedRunnerForOrg.mockRejectedValueOnce(error500); - } - - // act & assert - should not throw because error handling is in terminateOrphan - await expect(scaleDown()).resolves.not.toThrow(); - - // Should not terminate since the error was not a 404 - expect(terminateRunner).not.toHaveBeenCalledWith(orphanRunner.instanceId); - }); - - it(`Should ignore errors when termination orphan fails.`, async () => { - // setup - const orphanRunner = createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, true, true); - const runners = [orphanRunner]; - - mockGitHubRunners([]); - mockAwsRunners(runners); - mockTerminateRunners.mockImplementation(() => { - throw new Error('Failed to terminate'); - }); - - // act - await scaleDown(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - describe('When orphan termination fails', () => { - it(`Should not throw in case of list runner exception.`, async () => { - // setup - const runners = [createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, true, true)]; - - mockGitHubRunners([]); - mockListRunners.mockRejectedValueOnce(new Error('Failed to list runners')); - mockAwsRunners(runners); - - // ac - await scaleDown(); - - // assert - checkNonTerminated(runners); - }); - - it(`Should not throw in case of terminate runner exception.`, async () => { - // setup - const runners = [createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, true, true)]; - - mockGitHubRunners([]); - mockAwsRunners(runners); - mockTerminateRunners.mockRejectedValue(new Error('Failed to terminate')); - - // act and ensure no exception is thrown - await scaleDown(); - - // assert - checkNonTerminated(runners); - }); - }); - - it(`Should not terminate instance in case de-register fails.`, async () => { - // setup - const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false)]; - - mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { - return { status: 500 }; - }); - mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { - return { status: 500 }; - }); - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act and should resolve - await expect(scaleDown()).resolves.not.toThrow(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it(`Should not throw an exception in case of failure during removing a runner.`, async () => { - // setup - const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, true, false)]; - - mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { - throw new Error('Failed to delete runner'); - }); - mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { - throw new Error('Failed to delete runner'); - }); - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await expect(scaleDown()).resolves.not.toThrow(); - }); - - it(`Should not terminate instance when de-registration throws an error.`, async () => { - // setup - runner should NOT be terminated because de-registration fails - const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false)]; - - const error502 = new RequestError('Server Error', 502, { - request: { - method: 'DELETE', - url: 'https://api.github.com/test', - headers: {}, - }, - }); - - mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { - throw error502; - }); - mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { - throw error502; - }); - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await expect(scaleDown()).resolves.not.toThrow(); - - // assert - should NOT terminate since de-registration failed - expect(terminateRunner).not.toHaveBeenCalled(); - }); - - const evictionStrategies = ['oldest_first', 'newest_first']; - describe.each(evictionStrategies)('When idle config defined', (evictionStrategy) => { - const defaultConfig = { - idleCount: 1, - cron: '* * * * * *', - timeZone: 'Europe/Amsterdam', - evictionStrategy, - }; - - beforeEach(() => { - process.env.SCALE_DOWN_CONFIG = JSON.stringify([defaultConfig]); - }); - - it(`Should terminate based on the the idle config with ${evictionStrategy} eviction strategy`, async () => { - // setup - const runnerToTerminateTime = - evictionStrategy === 'oldest_first' - ? MINIMUM_TIME_RUNNING_IN_MINUTES + 5 - : MINIMUM_TIME_RUNNING_IN_MINUTES + 1; - const runners = [ - createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 4, true, false, false), - createRunnerTestData('idle-to-terminate', type, runnerToTerminateTime, true, false, true), - ]; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await scaleDown(); - - // assert - const runnersToTerminate = runners.filter((r) => r.shouldBeTerminated); - for (const toTerminate of runnersToTerminate) { - expect(terminateRunner).toHaveBeenCalledWith(toTerminate.instanceId); - } +describe('scaleDown runner provider orchestration', () => { + it('creates the default provider and lists orphan and active runners', async () => { + await scaleDown(); - const runnersNotToTerminate = runners.filter((r) => !r.shouldBeTerminated); - for (const notTerminated of runnersNotToTerminate) { - expect(terminateRunner).not.toHaveBeenCalledWith(notTerminated.instanceId); - } - }); - }); - }); + expect(mockedGetDefaultScaleDownRunnerProviderType).toHaveBeenCalled(); + expect(mockedCreateScaleDownRunnerProvider).toHaveBeenCalledWith('test-provider'); + expect(testProvider.list).toHaveBeenNthCalledWith(1, 'unit-test-environment', true); + expect(testProvider.list).toHaveBeenNthCalledWith(2, 'unit-test-environment'); + expect(testProvider.terminate).not.toHaveBeenCalled(); }); - describe('When runners are sorted', () => { - const runners: RunnerInfo[] = [ + it('creates the provider type configured on idle config', async () => { + process.env.SCALE_DOWN_CONFIG = JSON.stringify([ { - instanceId: '1', - launchTime: moment(new Date()).subtract(1, 'minute').toDate(), - owner: 'owner', - type: 'type', + idleCount: 0, + cron: '* * * * *', + timeZone: 'UTC', + type: 'custom-provider', }, - { - instanceId: '3', - launchTime: moment(new Date()).subtract(3, 'minute').toDate(), - owner: 'owner', - type: 'type', - }, - { - instanceId: '2', - launchTime: moment(new Date()).subtract(2, 'minute').toDate(), - owner: 'owner', - type: 'type', - }, - { - instanceId: '0', - launchTime: moment(new Date()).subtract(0, 'minute').toDate(), - owner: 'owner', - type: 'type', - }, - ]; - - it('Should sort runners descending for eviction strategy oldest first te keep the youngest.', () => { - runners.sort(oldestFirstStrategy); - expect(runners[0].instanceId).toEqual('0'); - expect(runners[1].instanceId).toEqual('1'); - expect(runners[2].instanceId).toEqual('2'); - expect(runners[3].instanceId).toEqual('3'); - }); + ]); - it('Should sort runners ascending for eviction strategy newest first te keep oldest.', () => { - runners.sort(newestFirstStrategy); - expect(runners[0].instanceId).toEqual('3'); - expect(runners[1].instanceId).toEqual('2'); - expect(runners[2].instanceId).toEqual('1'); - expect(runners[3].instanceId).toEqual('0'); - }); + await scaleDown(); - it('Should sort runners with equal launch time.', () => { - const runnersTest = [...runners]; - const same = moment(new Date()).subtract(4, 'minute').toDate(); - runnersTest.push({ - instanceId: '4', - launchTime: same, - owner: 'owner', - type: 'type', - }); - runnersTest.push({ - instanceId: '5', - launchTime: same, - owner: 'owner', - type: 'type', - }); - runnersTest.sort(oldestFirstStrategy); - expect(runnersTest[3].launchTime).not.toEqual(same); - expect(runnersTest[4].launchTime).toEqual(same); - expect(runnersTest[5].launchTime).toEqual(same); + expect(mockedCreateScaleDownRunnerProvider).toHaveBeenCalledWith('custom-provider'); + }); - runnersTest.sort(newestFirstStrategy); - expect(runnersTest[3].launchTime).not.toEqual(same); - expect(runnersTest[1].launchTime).toEqual(same); - expect(runnersTest[0].launchTime).toEqual(same); - }); + it('marks a provider runner as orphan when it is not registered and boot time is exceeded', async () => { + const runner: RunnerInfo = { + id: 'runner-1', + launchTime: moment(new Date()).subtract(10, 'minutes').toDate(), + owner: 'Codertocat', + type: 'Org', + }; + testProvider.list.mockImplementation(async (_environment: string, orphan?: boolean) => (orphan ? [] : [runner])); + testProvider.bootTimeExceeded.mockReturnValue(true); + + await scaleDown(); + + expect(testProvider.bootTimeExceeded).toHaveBeenCalledWith(runner); + expect(testProvider.markOrphan).toHaveBeenCalledWith('runner-1'); + expect(testProvider.terminate).not.toHaveBeenCalledWith('runner-1'); + }); - it('Should sort runners even when launch time is undefined.', () => { - const runnersTest = [ - { - instanceId: '0', - launchTime: undefined, - owner: 'owner', - type: 'type', - }, - { - instanceId: '1', - launchTime: moment(new Date()).subtract(3, 'minute').toDate(), - owner: 'owner', - type: 'type', - }, - { - instanceId: '0', - launchTime: undefined, - owner: 'owner', - type: 'type', - }, - ]; - runnersTest.sort(oldestFirstStrategy); - expect(runnersTest[0].launchTime).toBeUndefined(); - expect(runnersTest[1].launchTime).toBeDefined(); - expect(runnersTest[2].launchTime).not.toBeDefined(); - }); + it('terminates orphan runners through the selected provider', async () => { + const orphanRunner: RunnerList = { + id: 'orphan-1', + launchTime: moment(new Date()).subtract(10, 'minutes').toDate(), + owner: 'Codertocat', + type: 'Org', + orphan: true, + }; + testProvider.list.mockImplementation(async (_environment: string, orphan?: boolean) => + orphan ? [orphanRunner] : [], + ); + + await scaleDown(); + + expect(testProvider.terminate).toHaveBeenCalledWith('orphan-1'); }); }); -function mockAwsRunners(runners: RunnerTestItem[]) { - mockListRunners.mockImplementation(async (filter) => { - return runners.filter((r) => !filter?.orphan || filter?.orphan === r.orphan); +describe('scaleDown runner sort strategies', () => { + const createRunners = (): RunnerInfo[] => [ + { + id: '1', + launchTime: moment(new Date()).subtract(1, 'minute').toDate(), + owner: 'owner', + type: 'type', + }, + { + id: '3', + launchTime: moment(new Date()).subtract(3, 'minute').toDate(), + owner: 'owner', + type: 'type', + }, + { + id: '2', + launchTime: moment(new Date()).subtract(2, 'minute').toDate(), + owner: 'owner', + type: 'type', + }, + { + id: '0', + launchTime: moment(new Date()).subtract(0, 'minute').toDate(), + owner: 'owner', + type: 'type', + }, + ]; + + it('sorts runners descending for oldest first to keep the youngest', () => { + const runners = createRunners(); + + runners.sort(oldestFirstStrategy); + expect(runners.map((runner) => runner.id)).toEqual(['0', '1', '2', '3']); }); -} -function checkNonTerminated(runners: RunnerTestItem[]) { - const notTerminated = runners.filter((r) => !r.shouldBeTerminated); - for (const toTerminate of notTerminated) { - expect(terminateRunner).not.toHaveBeenCalledWith(toTerminate.instanceId); - } -} + it('sorts runners ascending for newest first to keep the oldest', () => { + const runners = createRunners(); -function checkTerminated(runners: RunnerTestItem[]) { - const runnersToTerminate = runners.filter((r) => r.shouldBeTerminated); - expect(terminateRunner).toHaveBeenCalledTimes(runnersToTerminate.length); - for (const toTerminate of runnersToTerminate) { - expect(terminateRunner).toHaveBeenCalledWith(toTerminate.instanceId); - } -} - -function mockGitHubRunners(runners: RunnerTestItem[]) { - mockOctokit.paginate.mockResolvedValue( - runners - .filter((r) => r.registered) - .map((r) => { - return { - id: r.instanceId, - name: r.instanceId, - }; - }), - ); -} - -function createRunnerTestData( - name: string, - type: 'Org' | 'Repo', - minutesLaunchedAgo: number, - registered: boolean, - orphan: boolean, - shouldBeTerminated: boolean, - owner?: string, - runnerId?: number, -): RunnerTestItem { - return { - instanceId: `i-${name}-${type.toLowerCase()}`, - launchTime: moment(new Date()).subtract(minutesLaunchedAgo, 'minutes').toDate(), - type, - owner: owner - ? owner - : type === 'Repo' - ? `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}` - : `${TEST_DATA.repositoryOwner}`, - registered, - orphan, - shouldBeTerminated, - runnerId: runnerId !== undefined ? String(runnerId) : undefined, - bypassRemoval: false, - }; -} + runners.sort(newestFirstStrategy); + expect(runners.map((runner) => runner.id)).toEqual(['3', '2', '1', '0']); + }); +}); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index 5fecc14c99..a759289058 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -5,12 +5,17 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import moment from 'moment'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; -import { bootTimeExceeded, listEC2Runners, tag, untag, terminateRunner } from './../aws/runners'; -import { RunnerInfo, RunnerList } from './../aws/runners.d'; import { GhRunners, githubCache } from './cache'; -import { ScalingDownConfig, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config'; +import { + ScalingDownConfig, + getEvictionStrategy, + getIdleRunnerCount, + getScaleDownRunnerProviderType, +} from './scale-down-config'; +import { createScaleDownRunnerProvider, getDefaultScaleDownRunnerProviderType } from './scale-down-provider-registry'; import { metricGitHubAppRateLimit } from '../github/rate-limit'; -import { getGitHubEnterpriseApiUrl } from './scale-up'; +import { getGitHubEnterpriseApiUrl } from './github-runner'; +import type { RunnerInfo, RunnerList, ScaleDownRunnerProvider } from './scale-down-provider'; const logger = createChildLogger('scale-down'); @@ -54,42 +59,40 @@ async function getOrCreateOctokit(runner: RunnerInfo): Promise { async function getGitHubSelfHostedRunnerState( client: Octokit, - ec2runner: RunnerInfo, + runner: RunnerInfo, runnerId: number, ): Promise { try { const state = - ec2runner.type === 'Org' + runner.type === 'Org' ? await client.actions.getSelfHostedRunnerForOrg({ runner_id: runnerId, - org: ec2runner.owner, + org: runner.owner, }) : await client.actions.getSelfHostedRunnerForRepo({ runner_id: runnerId, - owner: ec2runner.owner.split('/')[0], - repo: ec2runner.owner.split('/')[1], + owner: runner.owner.split('/')[0], + repo: runner.owner.split('/')[1], }); metricGitHubAppRateLimit(state.headers); return state.data; } catch (error) { if (error instanceof RequestError && error.status === 404) { - logger.info(`Runner '${ec2runner.instanceId}' with GitHub Runner ID '${runnerId}' not found on GitHub (404)`); + logger.info(`Runner '${runner.id}' with GitHub Runner ID '${runnerId}' not found on GitHub (404)`); return null; } throw error; } } -async function getGitHubRunnerBusyState(client: Octokit, ec2runner: RunnerInfo, runnerId: number): Promise { - const state = await getGitHubSelfHostedRunnerState(client, ec2runner, runnerId); +async function getGitHubRunnerBusyState(client: Octokit, runner: RunnerInfo, runnerId: number): Promise { + const state = await getGitHubSelfHostedRunnerState(client, runner, runnerId); if (state === null) { - logger.info( - `Runner '${ec2runner.instanceId}' - GitHub Runner ID '${runnerId}' - Not found on GitHub, treating as not busy`, - ); + logger.info(`Runner '${runner.id}' - GitHub Runner ID '${runnerId}' - Not found on GitHub, treating as not busy`); return false; } - logger.info(`Runner '${ec2runner.instanceId}' - GitHub Runner ID '${runnerId}' - Busy: ${state.busy}`); + logger.info(`Runner '${runner.id}' - GitHub Runner ID '${runnerId}' - Busy: ${state.busy}`); return state.busy; } @@ -132,18 +135,18 @@ function runnerMinimumTimeExceeded(runner: RunnerInfo): boolean { async function deleteGitHubRunner( githubInstallationClient: Octokit, - ec2runner: RunnerInfo, + runner: RunnerInfo, ghRunnerId: number, ): Promise<{ ghRunnerId: number; status: number; success: boolean }> { try { let response; - if (ec2runner.type === 'Org') { + if (runner.type === 'Org') { response = await githubInstallationClient.actions.deleteSelfHostedRunnerFromOrg({ runner_id: ghRunnerId, - org: ec2runner.owner, + org: runner.owner, }); } else { - const [owner, repo] = ec2runner.owner.split('/'); + const [owner, repo] = runner.owner.split('/'); response = await githubInstallationClient.actions.deleteSelfHostedRunnerFromRepo({ runner_id: ghRunnerId, owner, @@ -153,7 +156,7 @@ async function deleteGitHubRunner( return { ghRunnerId, status: response.status, success: response.status === 204 }; } catch (error) { logger.error( - `Failed to de-register GitHub runner ${ghRunnerId} for instance '${ec2runner.instanceId}'. ` + + `Failed to de-register GitHub runner ${ghRunnerId} for runner '${runner.id}'. ` + `Error: ${error instanceof Error ? error.message : String(error)}`, { error }, ); @@ -161,13 +164,16 @@ async function deleteGitHubRunner( } } -async function removeRunner(ec2runner: RunnerInfo, ghRunnerIds: number[]): Promise { - const githubInstallationClient = await getOrCreateOctokit(ec2runner); +async function removeRunner( + runner: RunnerInfo, + ghRunnerIds: number[], + runnerProvider: ScaleDownRunnerProvider, +): Promise { + const githubInstallationClient = await getOrCreateOctokit(runner); try { - const runnerList = ec2runner as unknown as RunnerList; - if (runnerList.bypassRemoval) { + if (runner.bypassRemoval) { logger.info( - `Runner '${ec2runner.instanceId}' has bypass-removal tag set, skipping removal. Remove the tag to allow scale-down.`, + `Runner '${runner.id}' has bypass-removal tag set, skipping removal. Remove the tag to allow scale-down.`, ); return; } @@ -175,153 +181,146 @@ async function removeRunner(ec2runner: RunnerInfo, ghRunnerIds: number[]): Promi const states = await Promise.all( ghRunnerIds.map(async (ghRunnerId) => { // Get busy state instead of using the output of listGitHubRunners(...) to minimize to race condition. - return await getGitHubRunnerBusyState(githubInstallationClient, ec2runner, ghRunnerId); + return await getGitHubRunnerBusyState(githubInstallationClient, runner, ghRunnerId); }), ); if (states.every((busy) => busy === false)) { const results = await Promise.all( - ghRunnerIds.map((ghRunnerId) => deleteGitHubRunner(githubInstallationClient, ec2runner, ghRunnerId)), + ghRunnerIds.map((ghRunnerId) => deleteGitHubRunner(githubInstallationClient, runner, ghRunnerId)), ); const allSucceeded = results.every((r) => r.success); const failedRunners = results.filter((r) => !r.success); if (allSucceeded) { - await terminateRunner(ec2runner.instanceId); - logger.info(`AWS runner instance '${ec2runner.instanceId}' is terminated and GitHub runner is de-registered.`); + await runnerProvider.terminate(runner.id); + logger.info(`${runnerProvider.name} runner '${runner.id}' is terminated and GitHub runner is de-registered.`); } else { - // Only terminate EC2 if we successfully de-registered from GitHub - // Otherwise, leave the instance running so the next scale-down cycle can retry + // Only terminate the provider runner if it was successfully de-registered from GitHub. logger.error( - `Failed to de-register ${failedRunners.length} GitHub runner(s) for instance '${ec2runner.instanceId}'. ` + - `Instance will NOT be terminated to allow retry on next scale-down cycle. ` + + `Failed to de-register ${failedRunners.length} GitHub runner(s) for runner '${runner.id}'. ` + + `Runner will NOT be terminated to allow retry on next scale-down cycle. ` + `Failed runner IDs: ${failedRunners.map((r) => r.ghRunnerId).join(', ')}`, ); } } else { - logger.info(`Runner '${ec2runner.instanceId}' cannot be de-registered, because it is still busy.`); + logger.info(`Runner '${runner.id}' cannot be de-registered, because it is still busy.`); } } catch (e) { logger.error( - `Runner '${ec2runner.instanceId}' cannot be de-registered. Error: ${e instanceof Error ? e.message : String(e)}`, + `Runner '${runner.id}' cannot be de-registered. Error: ${e instanceof Error ? e.message : String(e)}`, { error: e }, ); } } async function evaluateAndRemoveRunners( - ec2Runners: RunnerInfo[], + runners: RunnerInfo[], scaleDownConfigs: ScalingDownConfig[], + runnerProvider: ScaleDownRunnerProvider, ): Promise { let idleCounter = getIdleRunnerCount(scaleDownConfigs); const evictionStrategy = getEvictionStrategy(scaleDownConfigs); - const ownerTags = new Set(ec2Runners.map((runner) => runner.owner)); + const ownerTags = new Set(runners.map((runner) => runner.owner)); for (const ownerTag of ownerTags) { - const ec2RunnersFiltered = ec2Runners + const ownerRunners = runners .filter((runner) => runner.owner === ownerTag) .sort(evictionStrategy === 'oldest_first' ? oldestFirstStrategy : newestFirstStrategy); - logger.debug(`Found: '${ec2RunnersFiltered.length}' active GitHub runners with owner tag: '${ownerTag}'`); - logger.debug(`Active GitHub runners with owner tag: '${ownerTag}': ${JSON.stringify(ec2RunnersFiltered)}`); - for (const ec2Runner of ec2RunnersFiltered) { - if ((ec2Runner as unknown as RunnerList).bypassRemoval) { - logger.debug(`Runner '${ec2Runner.instanceId}' has bypass-removal tag set, skipping evaluation.`); + logger.debug(`Found: '${ownerRunners.length}' active GitHub runners with owner tag: '${ownerTag}'`); + logger.debug(`Active GitHub runners with owner tag: '${ownerTag}': ${JSON.stringify(ownerRunners)}`); + for (const runner of ownerRunners) { + if (runner.bypassRemoval) { + logger.debug(`Runner '${runner.id}' has bypass-removal tag set, skipping evaluation.`); continue; } - const ghRunners = await listGitHubRunners(ec2Runner); - const ghRunnersFiltered = ghRunners.filter((runner: { name: string }) => - runner.name.endsWith(ec2Runner.instanceId), - ); - logger.debug( - `Found: '${ghRunnersFiltered.length}' GitHub runners for AWS runner instance: '${ec2Runner.instanceId}'`, - ); - logger.debug( - `GitHub runners for AWS runner instance: '${ec2Runner.instanceId}': ${JSON.stringify(ghRunnersFiltered)}`, - ); + const ghRunners = await listGitHubRunners(runner); + const ghRunnersFiltered = ghRunners.filter((ghRunner: { name: string }) => ghRunner.name.endsWith(runner.id)); + logger.debug(`Found: '${ghRunnersFiltered.length}' GitHub runners for runner: '${runner.id}'`); + logger.debug(`GitHub runners for runner: '${runner.id}': ${JSON.stringify(ghRunnersFiltered)}`); if (ghRunnersFiltered.length) { - if (runnerMinimumTimeExceeded(ec2Runner)) { + if (runnerMinimumTimeExceeded(runner)) { if (idleCounter > 0) { idleCounter--; - logger.info(`Runner '${ec2Runner.instanceId}' will be kept idle.`); + logger.info(`Runner '${runner.id}' will be kept idle.`); } else { logger.info(`Terminating all non busy runners.`); await removeRunner( - ec2Runner, + runner, ghRunnersFiltered.map((runner: { id: number }) => runner.id), + runnerProvider, ); } } - } else if (bootTimeExceeded(ec2Runner)) { - await markOrphan(ec2Runner.instanceId); + } else if (runnerProvider.bootTimeExceeded(runner)) { + await markOrphan(runner.id, runnerProvider); } else { - logger.debug(`Runner ${ec2Runner.instanceId} has not yet booted.`); + logger.debug(`Runner ${runner.id} has not yet booted.`); } } } } -async function markOrphan(instanceId: string): Promise { +async function markOrphan(id: string, runnerProvider: ScaleDownRunnerProvider): Promise { try { - await tag(instanceId, [{ Key: 'ghr:orphan', Value: 'true' }]); - logger.info(`Runner '${instanceId}' tagged as orphan.`); + await runnerProvider.markOrphan(id); + logger.info(`Runner '${id}' tagged as orphan.`); } catch (e) { - logger.error(`Failed to tag runner '${instanceId}' as orphan.`, { error: e }); + logger.error(`Failed to tag runner '${id}' as orphan.`, { error: e }); } } -async function unMarkOrphan(instanceId: string): Promise { +async function unMarkOrphan(id: string, runnerProvider: ScaleDownRunnerProvider): Promise { try { - await untag(instanceId, [{ Key: 'ghr:orphan', Value: 'true' }]); - logger.info(`Runner '${instanceId}' untagged as orphan.`); + await runnerProvider.unmarkOrphan(id); + logger.info(`Runner '${id}' untagged as orphan.`); } catch (e) { - logger.error(`Failed to un-tag runner '${instanceId}' as orphan.`, { error: e }); + logger.error(`Failed to un-tag runner '${id}' as orphan.`, { error: e }); } } async function lastChanceCheckOrphanRunner(runner: RunnerList): Promise { - const client = await getOrCreateOctokit(runner as RunnerInfo); - const runnerId = parseInt(runner.runnerId || '0'); - const ec2Instance = runner as RunnerInfo; - const state = await getGitHubSelfHostedRunnerState(client, ec2Instance, runnerId); + const registeredRunner = runner as RunnerInfo; + const client = await getOrCreateOctokit(registeredRunner); + const runnerId = parseInt(runner.githubRunnerId || '0'); + const state = await getGitHubSelfHostedRunnerState(client, registeredRunner, runnerId); let isOrphan = false; if (state === null) { - logger.debug(`Runner '${runner.instanceId}' not found on GitHub, treating as orphaned.`); + logger.debug(`Runner '${runner.id}' not found on GitHub, treating as orphaned.`); isOrphan = true; } else { - logger.debug( - `Runner '${runner.instanceId}' is '${state.status}' and is currently '${state.busy ? 'busy' : 'idle'}'.`, - ); + logger.debug(`Runner '${runner.id}' is '${state.status}' and is currently '${state.busy ? 'busy' : 'idle'}'.`); const isOfflineAndBusy = state.status === 'offline' && state.busy; if (isOfflineAndBusy) { isOrphan = true; } } - logger.info(`Runner '${runner.instanceId}' is judged to ${isOrphan ? 'be' : 'not be'} orphaned.`); + logger.info(`Runner '${runner.id}' is judged to ${isOrphan ? 'be' : 'not be'} orphaned.`); return isOrphan; } -async function terminateOrphan(environment: string): Promise { +async function terminateOrphan(environment: string, runnerProvider: ScaleDownRunnerProvider): Promise { try { - const orphanRunners = await listEC2Runners({ environment, orphan: true }); + const orphanRunners = await runnerProvider.list(environment, true); for (const runner of orphanRunners) { if (runner.bypassRemoval) { - logger.info(`Orphan runner '${runner.instanceId}' has bypass-removal tag set, skipping termination.`); + logger.info(`Orphan runner '${runner.id}' has bypass-removal tag set, skipping termination.`); continue; } - if (runner.runnerId) { + if (runner.githubRunnerId) { const isOrphan = await lastChanceCheckOrphanRunner(runner); if (isOrphan) { - await terminateRunner(runner.instanceId); + await runnerProvider.terminate(runner.id); } else { - await unMarkOrphan(runner.instanceId); + await unMarkOrphan(runner.id, runnerProvider); } } else { - logger.info(`Terminating orphan runner '${runner.instanceId}'`); - await terminateRunner(runner.instanceId).catch((e) => { - logger.error(`Failed to terminate orphan runner '${runner.instanceId}'`, { error: e }); + logger.info(`Terminating orphan runner '${runner.id}'`); + await runnerProvider.terminate(runner.id).catch((e) => { + logger.error(`Failed to terminate orphan runner '${runner.id}'`, { error: e }); }); } } @@ -342,38 +341,38 @@ export function newestFirstStrategy(a: RunnerInfo, b: RunnerInfo): number { return oldestFirstStrategy(a, b) * -1; } -async function listRunners(environment: string) { - return await listEC2Runners({ - environment, - }); +async function listRunners(environment: string, runnerProvider: ScaleDownRunnerProvider) { + return await runnerProvider.list(environment); } -function filterRunners(ec2runners: RunnerList[]): RunnerInfo[] { - return ec2runners.filter((ec2Runner) => ec2Runner.type && !ec2Runner.orphan) as RunnerInfo[]; +function filterRunners(runners: RunnerList[]): RunnerInfo[] { + return runners.filter((runner) => runner.owner && runner.type && !runner.orphan) as RunnerInfo[]; } export async function scaleDown(): Promise { githubCache.reset(); const environment = process.env.ENVIRONMENT; const scaleDownConfigs = JSON.parse(process.env.SCALE_DOWN_CONFIG) as [ScalingDownConfig]; + const runnerProviderType = getScaleDownRunnerProviderType(scaleDownConfigs, getDefaultScaleDownRunnerProviderType()); + const runnerProvider = createScaleDownRunnerProvider(runnerProviderType); // first runners marked to be orphan. - await terminateOrphan(environment); + await terminateOrphan(environment, runnerProvider); // next scale down idle runners with respect to config and mark potential orphans - const ec2Runners = await listRunners(environment); - const activeEc2RunnersCount = ec2Runners.length; - logger.info(`Found: '${activeEc2RunnersCount}' active GitHub EC2 runner instances before clean-up.`); - logger.debug(`Active GitHub EC2 runner instances: ${JSON.stringify(ec2Runners)}`); + const providerRunners = await listRunners(environment, runnerProvider); + const activeProviderRunnersCount = providerRunners.length; + logger.info(`Found: '${activeProviderRunnersCount}' active ${runnerProvider.name} runners before clean-up.`); + logger.debug(`Active ${runnerProvider.name} runners: ${JSON.stringify(providerRunners)}`); - if (activeEc2RunnersCount === 0) { + if (activeProviderRunnersCount === 0) { logger.debug(`No active runners found for environment: '${environment}'`); return; } - const runners = filterRunners(ec2Runners); - await evaluateAndRemoveRunners(runners, scaleDownConfigs); + const runners = filterRunners(providerRunners); + await evaluateAndRemoveRunners(runners, scaleDownConfigs, runnerProvider); - const activeEc2RunnersCountAfter = (await listRunners(environment)).length; - logger.info(`Found: '${activeEc2RunnersCountAfter}' active GitHub EC2 runners instances after clean-up.`); + const activeProviderRunnersCountAfter = (await listRunners(environment, runnerProvider)).length; + logger.info(`Found: '${activeProviderRunnersCountAfter}' active ${runnerProvider.name} runners after clean-up.`); } From 8c8d9e4591183218b720619825b95c508ab99bcc Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 7 Jul 2026 22:08:52 +0200 Subject: [PATCH 04/46] refactor(pool): split ec2 pool provider --- .../functions/control-plane/src/local-pool.ts | 2 +- .../control-plane/src/pool/ec2-pool.test.ts | 54 +++++++++ .../control-plane/src/pool/ec2-pool.ts | 110 ++++++++++++++++++ .../src/pool/pool-provider-registry.ts | 14 +++ .../control-plane/src/pool/pool-provider.ts | 38 ++++++ .../control-plane/src/pool/pool.test.ts | 54 +++++---- .../functions/control-plane/src/pool/pool.ts | 102 ++++------------ 7 files changed, 272 insertions(+), 102 deletions(-) create mode 100644 lambdas/functions/control-plane/src/pool/ec2-pool.test.ts create mode 100644 lambdas/functions/control-plane/src/pool/ec2-pool.ts create mode 100644 lambdas/functions/control-plane/src/pool/pool-provider-registry.ts create mode 100644 lambdas/functions/control-plane/src/pool/pool-provider.ts diff --git a/lambdas/functions/control-plane/src/local-pool.ts b/lambdas/functions/control-plane/src/local-pool.ts index ab8c74a1a0..d743ac759b 100644 --- a/lambdas/functions/control-plane/src/local-pool.ts +++ b/lambdas/functions/control-plane/src/local-pool.ts @@ -1,7 +1,7 @@ import { adjust } from './pool/pool'; export function run(): void { - adjust({ poolSize: 1 }) + adjust({ poolSize: 1, type: 'ec2' }) .then() .catch((e) => { console.log(e); diff --git a/lambdas/functions/control-plane/src/pool/ec2-pool.test.ts b/lambdas/functions/control-plane/src/pool/ec2-pool.test.ts new file mode 100644 index 0000000000..64797408ea --- /dev/null +++ b/lambdas/functions/control-plane/src/pool/ec2-pool.test.ts @@ -0,0 +1,54 @@ +import { bootTimeExceeded } from '../aws/ec2-runners'; +import type { RunnerList } from '../aws/ec2-runners.d'; +import { calculateEc2PoolSize } from './ec2-pool'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../aws/ec2-runners', async () => ({ + bootTimeExceeded: vi.fn(), + listEC2Runners: vi.fn(), +})); + +vi.mock('../scale-runners/ec2', async () => ({ + createRunners: vi.fn(), +})); + +const mockBootTimeExceeded = vi.mocked(bootTimeExceeded); + +describe('calculateEc2PoolSize', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('counts registered online idle runners', () => { + const runners: RunnerList[] = [{ instanceId: 'i-idle' }]; + const runnerStatus = new Map([['i-idle', { busy: false, status: 'online' }]]); + + expect(calculateEc2PoolSize(runners, runnerStatus)).toBe(1); + expect(mockBootTimeExceeded).not.toHaveBeenCalled(); + }); + + it('does not count registered busy or offline runners', () => { + const runners: RunnerList[] = [{ instanceId: 'i-busy' }, { instanceId: 'i-offline' }]; + const runnerStatus = new Map([ + ['i-busy', { busy: true, status: 'online' }], + ['i-offline', { busy: false, status: 'offline' }], + ]); + + expect(calculateEc2PoolSize(runners, runnerStatus)).toBe(0); + expect(mockBootTimeExceeded).not.toHaveBeenCalled(); + }); + + it('counts unregistered runners that are still booting', () => { + const runners: RunnerList[] = [{ instanceId: 'i-booting' }]; + mockBootTimeExceeded.mockReturnValue(false); + + expect(calculateEc2PoolSize(runners, new Map())).toBe(1); + }); + + it('does not count unregistered runners whose boot time expired', () => { + const runners: RunnerList[] = [{ instanceId: 'i-expired' }]; + mockBootTimeExceeded.mockReturnValue(true); + + expect(calculateEc2PoolSize(runners, new Map())).toBe(0); + }); +}); diff --git a/lambdas/functions/control-plane/src/pool/ec2-pool.ts b/lambdas/functions/control-plane/src/pool/ec2-pool.ts new file mode 100644 index 0000000000..d1c222d935 --- /dev/null +++ b/lambdas/functions/control-plane/src/pool/ec2-pool.ts @@ -0,0 +1,110 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import yn from 'yn'; + +import { bootTimeExceeded, listEC2Runners } from '../aws/ec2-runners'; +import type { RunnerList } from '../aws/ec2-runners.d'; +import { createRunners } from '../scale-runners/ec2'; +import type { CreateEC2RunnerConfig } from '../scale-runners/ec2'; +import type { PoolRunnerProvider, PoolRunnerProviderStrategy, RunnerStatus } from './pool-provider'; + +const logger = createChildLogger('pool'); + +interface Ec2PoolProviderConfig { + environment: string; + subnets: string[]; + launchTemplateName: string; + ec2instanceCriteria: CreateEC2RunnerConfig['ec2instanceCriteria']; + amiIdSsmParameterName?: string; + tracingEnabled?: boolean; + onDemandFailoverOnError: string[]; + scaleErrors: string[]; +} + +export function createEc2PoolProviderFromEnv(): PoolRunnerProvider { + const scaleErrors = JSON.parse(process.env.SCALE_ERRORS) as [string]; + + return createEc2PoolProvider({ + environment: process.env.ENVIRONMENT, + subnets: process.env.SUBNET_IDS.split(','), + launchTemplateName: process.env.LAUNCH_TEMPLATE_NAME, + ec2instanceCriteria: { + instanceTypes: process.env.INSTANCE_TYPES.split(','), + instanceTypePriorities: process.env.INSTANCE_TYPE_PRIORITIES + ? (JSON.parse(process.env.INSTANCE_TYPE_PRIORITIES) as Record) + : undefined, + targetCapacityType: process.env.INSTANCE_TARGET_CAPACITY_TYPE, + maxSpotPrice: process.env.INSTANCE_MAX_SPOT_PRICE, + instanceAllocationStrategy: process.env.INSTANCE_ALLOCATION_STRATEGY || 'lowest-price', + }, + amiIdSsmParameterName: process.env.AMI_ID_SSM_PARAMETER_NAME, + tracingEnabled: yn(process.env.POWERTOOLS_TRACE_ENABLED, { default: false }), + onDemandFailoverOnError: process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS + ? (JSON.parse(process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS) as [string]) + : [], + scaleErrors, + }); +} + +export const ec2PoolRunnerProviderStrategy: PoolRunnerProviderStrategy = { + type: 'ec2', + createFromEnv: createEc2PoolProviderFromEnv, +}; + +function createEc2PoolProvider(config: Ec2PoolProviderConfig): PoolRunnerProvider { + return { + type: 'ec2', + listRunners: async ({ environment, runnerOwner, runnerType }) => + await listEC2Runners({ + environment, + runnerOwner, + runnerType, + statuses: ['running'], + }), + countAvailableRunners: calculateEc2PoolSize, + createRunners: async ({ githubRunnerConfig, numberOfRunners, githubInstallationClient }) => + await createRunners( + githubRunnerConfig, + { + ec2instanceCriteria: config.ec2instanceCriteria, + environment: config.environment, + launchTemplateName: config.launchTemplateName, + subnets: config.subnets, + amiIdSsmParameterName: config.amiIdSsmParameterName, + tracingEnabled: config.tracingEnabled, + onDemandFailoverOnError: config.onDemandFailoverOnError, + scaleErrors: config.scaleErrors, + }, + numberOfRunners, + githubInstallationClient, + 'pool-lambda', + ), + }; +} + +export function calculateEc2PoolSize( + ec2runners: RunnerList[], + runnerStatus: Map, + includeBusyRunners = false, +): number { + // Runner should be considered idle if it is still booting, or is idle in GitHub + let numberOfRunnersInPool = 0; + for (const ec2Instance of ec2runners) { + if ( + (runnerStatus.get(ec2Instance.instanceId)?.busy === false || includeBusyRunners) && + runnerStatus.get(ec2Instance.instanceId)?.status === 'online' + ) { + numberOfRunnersInPool++; + logger.debug(`Runner ${ec2Instance.instanceId} is idle in GitHub and counted as part of the pool`); + } else if (runnerStatus.get(ec2Instance.instanceId) != null) { + logger.debug(`Runner ${ec2Instance.instanceId} is not idle in GitHub and NOT counted as part of the pool`); + } else if (!bootTimeExceeded(ec2Instance)) { + numberOfRunnersInPool++; + logger.info(`Runner ${ec2Instance.instanceId} is still booting and counted as part of the pool`); + } else { + logger.debug( + `Runner ${ec2Instance.instanceId} is not idle in GitHub nor booting and not counted as part of the pool`, + ); + } + } + return numberOfRunnersInPool; +} diff --git a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts new file mode 100644 index 0000000000..d745c670ac --- /dev/null +++ b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts @@ -0,0 +1,14 @@ +import { ec2PoolRunnerProviderStrategy } from './ec2-pool'; +import type { PoolRunnerProvider, PoolRunnerProviderStrategy, PoolRunnerProviderType } from './pool-provider'; + +const poolRunnerProviderStrategies: PoolRunnerProviderStrategy[] = [ec2PoolRunnerProviderStrategy]; + +export function createPoolRunnerProviderFromEnv(type: PoolRunnerProviderType): PoolRunnerProvider { + const strategy = poolRunnerProviderStrategies.find((strategy) => strategy.type === type); + + if (!strategy) { + throw new Error(`Unsupported pool runner provider type '${type}'`); + } + + return strategy.createFromEnv(); +} diff --git a/lambdas/functions/control-plane/src/pool/pool-provider.ts b/lambdas/functions/control-plane/src/pool/pool-provider.ts new file mode 100644 index 0000000000..83ec25a93c --- /dev/null +++ b/lambdas/functions/control-plane/src/pool/pool-provider.ts @@ -0,0 +1,38 @@ +import type { Octokit } from '@octokit/rest'; + +import type { CreateGitHubRunnerConfig, GitHubRunnerType } from '../scale-runners/types'; + +export type PoolRunnerProviderType = string; + +export interface RunnerStatus { + busy: boolean; + status: string; +} + +export interface ListPoolRunnersInput { + environment: string; + runnerOwner: string; + runnerType: GitHubRunnerType; +} + +export interface CreatePoolRunnersInput { + githubRunnerConfig: CreateGitHubRunnerConfig; + numberOfRunners: number; + githubInstallationClient: Octokit; +} + +export interface PoolRunnerProvider { + type: PoolRunnerProviderType; + listRunners(input: ListPoolRunnersInput): Promise; + countAvailableRunners( + runners: TRunner[], + runnerStatus: Map, + includeBusyRunners: boolean, + ): number; + createRunners(input: CreatePoolRunnersInput): Promise; +} + +export interface PoolRunnerProviderStrategy { + type: PoolRunnerProviderType; + createFromEnv(): PoolRunnerProvider; +} diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index c326fe0f47..63cecbc346 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -2,9 +2,10 @@ import { Octokit } from '@octokit/rest'; import moment from 'moment-timezone'; import * as nock from 'nock'; -import { listEC2Runners } from '../aws/runners'; +import { listEC2Runners } from '../aws/ec2-runners'; import * as ghAuth from '../github/auth'; -import { createRunners, getGitHubEnterpriseApiUrl } from '../scale-runners/scale-up'; +import { createRunners } from '../scale-runners/ec2'; +import { getGitHubEnterpriseApiUrl } from '../scale-runners/github-runner'; import { adjust } from './pool'; import { describe, it, expect, beforeEach, vi, MockedClass } from 'vitest'; @@ -25,7 +26,7 @@ vi.mock('@octokit/rest', () => ({ }), })); -vi.mock('./../aws/runners', async () => ({ +vi.mock('./../aws/ec2-runners', async () => ({ listEC2Runners: vi.fn(), // Include any other functions from the module that might be used bootTimeExceeded: vi.fn(), @@ -36,14 +37,16 @@ vi.mock('./../github/auth', async () => ({ createOctokitClient: vi.fn(), })); -vi.mock('../scale-runners/scale-up', async () => ({ - scaleUp: vi.fn(), +vi.mock('../scale-runners/ec2', async () => ({ createRunners: vi.fn(), +})); + +vi.mock('../scale-runners/github-runner', async () => ({ getGitHubEnterpriseApiUrl: vi.fn().mockReturnValue({ ghesApiUrl: '', ghesBaseUrl: '', }), - // Include any other functions that might be needed + validateSsmParameterStoreTags: vi.fn().mockReturnValue([]), })); const mocktokit = Octokit as MockedClass; @@ -190,7 +193,7 @@ describe('Test simple pool.', () => { }); }); it('Top up pool with pool size 2 registered.', async () => { - await adjust({ poolSize: 3 }); + await adjust({ poolSize: 3, type: 'ec2' }); expect(createRunners).toHaveBeenCalledTimes(1); expect(createRunners).toHaveBeenCalledWith( expect.anything(), @@ -201,8 +204,19 @@ describe('Test simple pool.', () => { ); }); + it('uses the EC2 pool provider when the event type is ec2.', async () => { + await adjust({ poolSize: 3, type: 'ec2' }); + expect(createRunners).toHaveBeenCalledTimes(1); + expect(mockListRunners).toHaveBeenCalledWith({ + environment: 'unit-test-environment', + runnerOwner: ORG, + runnerType: 'Org', + statuses: ['running'], + }); + }); + it('Should not top up if pool size is reached.', async () => { - await adjust({ poolSize: 1 }); + await adjust({ poolSize: 1, type: 'ec2' }); expect(createRunners).not.toHaveBeenCalled(); }); @@ -228,7 +242,7 @@ describe('Test simple pool.', () => { ]); // 2 idle + 1 booting = 3, top up with 2 to match a pool of 5 - await adjust({ poolSize: 5 }); + await adjust({ poolSize: 5, type: 'ec2' }); expect(createRunners).toHaveBeenCalled(); // Access the numberOfRunners without assuming a specific position // Just test that the function was called @@ -258,7 +272,7 @@ describe('Test simple pool.', () => { }, ]); - await adjust({ poolSize: 2 }); + await adjust({ poolSize: 2, type: 'ec2' }); expect(createRunners).not.toHaveBeenCalled(); }); }); @@ -272,7 +286,7 @@ describe('Test simple pool.', () => { }); it('Top up if the pool size is set to 5', async () => { - await adjust({ poolSize: 5 }); + await adjust({ poolSize: 5, type: 'ec2' }); // 2 idle, top up with 3 to match a pool of 5 expect(createRunners).toHaveBeenCalledWith( expect.anything(), @@ -293,7 +307,7 @@ describe('Test simple pool.', () => { }); it('Top up if the pool size is set to 5', async () => { - await adjust({ poolSize: 5 }); + await adjust({ poolSize: 5, type: 'ec2' }); // 2 idle, top up with 3 to match a pool of 5 expect(createRunners).toHaveBeenCalledWith( expect.anything(), @@ -349,7 +363,7 @@ describe('Test simple pool.', () => { }, ]); - await adjust({ poolSize: 5 }); + await adjust({ poolSize: 5, type: 'ec2' }); // 2 idle, 2 prefixed idle top up with 1 to match a pool of 5 expect(createRunners).toHaveBeenCalledWith( expect.anything(), @@ -373,13 +387,13 @@ describe('Test simple pool.', () => { // 4 running runners (2 idle, 1 busy, 1 offline) already meet the maximum, so a large pool size // must not create more. This is the over-provisioning case from issue #5186. process.env.RUNNERS_MAXIMUM_COUNT = '4'; - await adjust({ poolSize: 10 }); + await adjust({ poolSize: 10, type: 'ec2' }); expect(createRunners).not.toHaveBeenCalled(); }); it('Should not top up when the total number of running runners exceeds the maximum.', async () => { process.env.RUNNERS_MAXIMUM_COUNT = '3'; - await adjust({ poolSize: 10 }); + await adjust({ poolSize: 10, type: 'ec2' }); expect(createRunners).not.toHaveBeenCalled(); }); @@ -387,7 +401,7 @@ describe('Test simple pool.', () => { // 4 running runners with a maximum of 6 leaves headroom for 2, even though the pool of 10 and the // 2 idle runners would otherwise request a top-up of 8. process.env.RUNNERS_MAXIMUM_COUNT = '6'; - await adjust({ poolSize: 10 }); + await adjust({ poolSize: 10, type: 'ec2' }); expect(createRunners).toHaveBeenCalledWith( expect.anything(), expect.anything(), @@ -401,7 +415,7 @@ describe('Test simple pool.', () => { // Headroom (6 - 4 = 2) is larger than the pool demand (5 - 2 idle = 3 would exceed it, so use a // pool that stays within headroom): pool of 3 with 2 idle requests 1, which is under the cap. process.env.RUNNERS_MAXIMUM_COUNT = '6'; - await adjust({ poolSize: 3 }); + await adjust({ poolSize: 3, type: 'ec2' }); expect(createRunners).toHaveBeenCalledWith( expect.anything(), expect.anything(), @@ -414,7 +428,7 @@ describe('Test simple pool.', () => { it('Should ignore the maximum when set to -1 (unlimited).', async () => { process.env.RUNNERS_MAXIMUM_COUNT = '-1'; // 2 idle of 4 running, pool of 10 tops up with 8 regardless of how many are already running. - await adjust({ poolSize: 10 }); + await adjust({ poolSize: 10, type: 'ec2' }); expect(createRunners).toHaveBeenCalledWith( expect.anything(), expect.anything(), @@ -432,12 +446,12 @@ describe('Test simple pool.', () => { it('Should not top up when pool size matches runners including busy online runners.', async () => { // Without INCLUDE_BUSY_RUNNERS: 2 in pool (i-1-idle, i-4-idle-older). With it: 3 (adds i-2-busy). - await adjust({ poolSize: 3 }); + await adjust({ poolSize: 3, type: 'ec2' }); expect(createRunners).not.toHaveBeenCalled(); }); it('Should top up by two runners when pool size is 5 and busy runners count toward the pool.', async () => { - await adjust({ poolSize: 5 }); + await adjust({ poolSize: 5, type: 'ec2' }); // 3 in pool (idle, busy, older idle); need 2 more expect(createRunners).toHaveBeenCalledWith( expect.anything(), diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 7d0cf483d4..626fd8567a 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -2,58 +2,40 @@ import { Octokit } from '@octokit/rest'; import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import yn from 'yn'; -import { bootTimeExceeded, listEC2Runners } from '../aws/runners'; -import { RunnerList } from '../aws/runners.d'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; -import { createRunners, getGitHubEnterpriseApiUrl } from '../scale-runners/scale-up'; -import { validateSsmParameterStoreTags } from '../scale-runners/scale-up'; +import { getGitHubEnterpriseApiUrl, validateSsmParameterStoreTags } from '../scale-runners/github-runner'; +import { createPoolRunnerProviderFromEnv } from './pool-provider-registry'; +import type { PoolRunnerProviderType, RunnerStatus } from './pool-provider'; const logger = createChildLogger('pool'); export interface PoolEvent { poolSize: number; -} - -interface RunnerStatus { - busy: boolean; - status: string; + type: PoolRunnerProviderType; } export async function adjust(event: PoolEvent): Promise { - logger.info(`Checking current pool size against pool of size: ${event.poolSize}`); + const runnerProviderType = event.type; + logger.info(`Checking current ${runnerProviderType} pool size against pool of size: ${event.poolSize}`); const runnerLabels = process.env.RUNNER_LABELS || ''; const runnerGroup = process.env.RUNNER_GROUP_NAME || ''; const runnerNamePrefix = process.env.RUNNER_NAME_PREFIX || ''; const environment = process.env.ENVIRONMENT; const ssmTokenPath = process.env.SSM_TOKEN_PATH; const ssmConfigPath = process.env.SSM_CONFIG_PATH || ''; - const subnets = process.env.SUBNET_IDS.split(','); - const instanceTypes = process.env.INSTANCE_TYPES.split(','); - const instanceTargetCapacityType = process.env.INSTANCE_TARGET_CAPACITY_TYPE; const ephemeral = yn(process.env.ENABLE_EPHEMERAL_RUNNERS, { default: false }); const enableJitConfig = yn(process.env.ENABLE_JIT_CONFIG, { default: ephemeral }); const disableAutoUpdate = yn(process.env.DISABLE_RUNNER_AUTOUPDATE, { default: false }); - const launchTemplateName = process.env.LAUNCH_TEMPLATE_NAME; - const instanceMaxSpotPrice = process.env.INSTANCE_MAX_SPOT_PRICE; - const instanceAllocationStrategy = process.env.INSTANCE_ALLOCATION_STRATEGY || 'lowest-price'; // same as AWS default - const instanceTypePriorities = process.env.INSTANCE_TYPE_PRIORITIES - ? (JSON.parse(process.env.INSTANCE_TYPE_PRIORITIES) as Record) - : undefined; const runnerOwner = process.env.RUNNER_OWNER; - const amiIdSsmParameterName = process.env.AMI_ID_SSM_PARAMETER_NAME; - const tracingEnabled = yn(process.env.POWERTOOLS_TRACE_ENABLED, { default: false }); - const onDemandFailoverOnError = process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS - ? (JSON.parse(process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS) as [string]) - : []; const ssmParameterStoreTags: { Key: string; Value: string }[] = process.env.SSM_PARAMETER_STORE_TAGS && process.env.SSM_PARAMETER_STORE_TAGS.trim() !== '' ? validateSsmParameterStoreTags(process.env.SSM_PARAMETER_STORE_TAGS) : []; - const scaleErrors = JSON.parse(process.env.SCALE_ERRORS) as [string]; // -1 disables the maximum check, matching the scale-up lambda's semantics. Defaults to unlimited // when unset so the pool keeps its previous behavior on stacks that do not provide the variable. const maximumRunners = parseInt(process.env.RUNNERS_MAXIMUM_COUNT || '-1'); const includeBusyRunners = yn(process.env.INCLUDE_BUSY_RUNNERS, { default: false }); + const runnerProvider = createPoolRunnerProviderFromEnv(runnerProviderType); const { ghesApiUrl, ghesBaseUrl } = getGitHubEnterpriseApiUrl(); @@ -68,27 +50,30 @@ export async function adjust(event: PoolEvent): Promise { runnerNamePrefix, ); - // Look up the managed ec2 runners in AWS, but running does not mean idle - const ec2runners = await listEC2Runners({ + // Look up the managed provider runners, but running does not mean idle. + const poolRunners = await runnerProvider.listRunners({ environment, runnerOwner, runnerType: 'Org', - statuses: ['running'], }); - const numberOfRunnersInPool = calculatePooSize(ec2runners, runnerStatusses, includeBusyRunners); + const numberOfRunnersInPool = runnerProvider.countAvailableRunners( + poolRunners, + runnerStatusses, + includeBusyRunners, + ); let topUp = event.poolSize - numberOfRunnersInPool; // The pool must never push the total number of runners (busy + idle) past the configured maximum. - // ec2runners contains every running runner for this type, so its length is the current total and no + // poolRunners contains every running runner for this type, so its length is the current total and no // extra API call is needed. Without this clamp the pool keeps topping up against idle-only counts and // can overshoot runners_maximum_count, while the scale-up lambda correctly refuses to launch. if (maximumRunners !== -1 && topUp > 0) { - const headroom = maximumRunners - ec2runners.length; + const headroom = maximumRunners - poolRunners.length; if (topUp > headroom) { logger.info( `Capping pool top-up from ${topUp} to ${Math.max(headroom, 0)} to respect the maximum of ` + - `${maximumRunners} runners (currently ${ec2runners.length} running).`, + `${maximumRunners} runners (currently ${poolRunners.length} running).`, ); topUp = headroom; } @@ -96,8 +81,8 @@ export async function adjust(event: PoolEvent): Promise { if (topUp > 0) { logger.info(`The pool will be topped up with ${topUp} runners.`); - await createRunners( - { + await runnerProvider.createRunners({ + githubRunnerConfig: { ephemeral, enableJitConfig, ghesBaseUrl, @@ -111,26 +96,9 @@ export async function adjust(event: PoolEvent): Promise { ssmConfigPath, ssmParameterStoreTags, }, - { - ec2instanceCriteria: { - instanceTypes, - instanceTypePriorities, - targetCapacityType: instanceTargetCapacityType, - maxSpotPrice: instanceMaxSpotPrice, - instanceAllocationStrategy: instanceAllocationStrategy, - }, - environment, - launchTemplateName, - subnets, - amiIdSsmParameterName, - tracingEnabled, - onDemandFailoverOnError, - scaleErrors, - }, - topUp, + numberOfRunners: topUp, githubInstallationClient, - 'pool-lambda', - ); + }); } else { logger.info(`Pool will not be topped up. Found ${numberOfRunnersInPool} managed idle runners.`); } @@ -147,34 +115,6 @@ async function getInstallationId(ghesApiUrl: string, org: string): Promise, - includeBusyRunners: boolean, -): number { - // Runner should be considered idle if it is still booting, or is idle in GitHub - let numberOfRunnersInPool = 0; - for (const ec2Instance of ec2runners) { - if ( - (runnerStatus.get(ec2Instance.instanceId)?.busy === false || includeBusyRunners) && - runnerStatus.get(ec2Instance.instanceId)?.status === 'online' - ) { - numberOfRunnersInPool++; - logger.debug(`Runner ${ec2Instance.instanceId} is idle in GitHub and counted as part of the pool`); - } else if (runnerStatus.get(ec2Instance.instanceId) != null) { - logger.debug(`Runner ${ec2Instance.instanceId} is not idle in GitHub and NOT counted as part of the pool`); - } else if (!bootTimeExceeded(ec2Instance)) { - numberOfRunnersInPool++; - logger.info(`Runner ${ec2Instance.instanceId} is still booting and counted as part of the pool`); - } else { - logger.debug( - `Runner ${ec2Instance.instanceId} is not idle in GitHub nor booting and not counted as part of the pool`, - ); - } - } - return numberOfRunnersInPool; -} - async function getGitHubRegisteredRunnnerStatusses( ghClient: Octokit, runnerOwner: string, From b7503003266d8593f9bfcfa191150504927ac6d9 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 7 Jul 2026 22:09:17 +0200 Subject: [PATCH 05/46] refactor(webhook): decouple dynamic label dispatch --- .../src/runners/aws-dynamic-labels-policy.ts | 15 +++ .../runners/aws-dynamic-labels-provider.ts | 17 +++ .../src/runners/aws-dynamic-labels.test.ts | 28 ++++ .../webhook/src/runners/aws-dynamic-labels.ts | 30 +++++ .../webhook/src/runners/dispatch.test.ts | 126 +----------------- .../functions/webhook/src/runners/dispatch.ts | 92 ++----------- ...t.ts => ec2-dynamic-labels-policy.test.ts} | 2 +- ...policy.ts => ec2-dynamic-labels-policy.ts} | 13 +- .../webhook/src/runners/ec2-dynamic-labels.ts | 44 ++++++ .../webhook/src/runners/labels.test.ts | 121 +++++++++++++++++ .../functions/webhook/src/runners/labels.ts | 75 +++++++++++ lambdas/functions/webhook/src/sqs/index.ts | 7 +- 12 files changed, 356 insertions(+), 214 deletions(-) create mode 100644 lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts create mode 100644 lambdas/functions/webhook/src/runners/aws-dynamic-labels-provider.ts create mode 100644 lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts create mode 100644 lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts rename lambdas/functions/webhook/src/runners/{dynamic-labels-policy.test.ts => ec2-dynamic-labels-policy.test.ts} (99%) rename lambdas/functions/webhook/src/runners/{dynamic-labels-policy.ts => ec2-dynamic-labels-policy.ts} (90%) create mode 100644 lambdas/functions/webhook/src/runners/ec2-dynamic-labels.ts create mode 100644 lambdas/functions/webhook/src/runners/labels.test.ts create mode 100644 lambdas/functions/webhook/src/runners/labels.ts diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts new file mode 100644 index 0000000000..61625bfb00 --- /dev/null +++ b/lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts @@ -0,0 +1,15 @@ +export interface AwsDynamicLabelsValueRule { + allowed?: string[]; + denied?: string[]; + max?: number | string; +} + +/** + * AWS dynamic labels policy schema. `blocked_keys` rejects keys outright; + * `restricted_keys` applies optional per-key value rules. Provider-specific + * evaluators decide which dynamic label prefixes the policy applies to. + */ +export interface AwsDynamicLabelsPolicy { + blocked_keys?: string[]; + restricted_keys?: Record; +} diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels-provider.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels-provider.ts new file mode 100644 index 0000000000..ae9fe26f3d --- /dev/null +++ b/lambdas/functions/webhook/src/runners/aws-dynamic-labels-provider.ts @@ -0,0 +1,17 @@ +import type { RunnerMatcherConfig, RunnerProvider } from '../sqs'; + +export interface AwsDynamicLabelDispatchTarget { + queue: RunnerMatcherConfig; + labels: string[]; +} + +export interface SelectAwsDynamicLabelQueueInput { + queue: RunnerMatcherConfig; + nonGhrLabels: string[]; + sanitizedGhrLabels: string[]; +} + +export interface AwsDynamicLabelProviderStrategy { + type: RunnerProvider; + selectQueue(input: SelectAwsDynamicLabelQueueInput): AwsDynamicLabelDispatchTarget | undefined; +} diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts new file mode 100644 index 0000000000..51b7e13020 --- /dev/null +++ b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest'; + +import type { RunnerMatcherConfig, RunnerProvider } from '../sqs'; +import { selectAwsDynamicLabelQueue } from './aws-dynamic-labels'; + +describe('selectAwsDynamicLabelQueue', () => { + it('defaults queues without a provider to EC2 dynamic label handling', () => { + const queue = runnerQueue('default-ec2'); + + expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ + queue, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + }); + }); +}); + +function runnerQueue(id: string, runnerProvider?: RunnerProvider): RunnerMatcherConfig { + return { + id, + arn: `arn:${id}`, + runnerProvider, + matcherConfig: { + labelMatchers: [['self-hosted', 'linux']], + exactMatch: true, + enableDynamicLabels: true, + }, + }; +} diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts new file mode 100644 index 0000000000..cd75154df8 --- /dev/null +++ b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts @@ -0,0 +1,30 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + +import { RunnerMatcherConfig, RunnerProvider } from '../sqs'; +import { AwsDynamicLabelDispatchTarget, AwsDynamicLabelProviderStrategy } from './aws-dynamic-labels-provider'; +import { ec2DynamicLabelProviderStrategy } from './ec2-dynamic-labels'; + +const logger = createChildLogger('handler'); + +const awsDynamicLabelProviderStrategies: AwsDynamicLabelProviderStrategy[] = [ec2DynamicLabelProviderStrategy]; + +export function selectAwsDynamicLabelQueue( + matches: RunnerMatcherConfig[], + nonGhrLabels: string[], + sanitizedGhrLabels: string[], +): AwsDynamicLabelDispatchTarget | undefined { + for (const queue of matches) { + const provider: RunnerProvider | string = queue.runnerProvider ?? 'ec2'; + const strategy = awsDynamicLabelProviderStrategies.find((strategy) => strategy.type === provider); + + if (!strategy) { + logger.warn(`Queue ${queue.id} has unsupported runner provider '${provider}'`); + continue; + } + + const target = strategy.selectQueue({ queue, nonGhrLabels, sanitizedGhrLabels }); + if (target) return target; + } + + return undefined; +} diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index b776dcc865..ae571da9d8 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -7,7 +7,7 @@ import workFlowJobEvent from '../../test/resources/github_workflowjob_event.json import runnerConfig from '../../test/resources/multi_runner_configurations.json'; import { RunnerConfig, sendActionRequest } from '../sqs'; -import { canRunJob, dispatch } from './dispatch'; +import { dispatch } from './dispatch'; import { ConfigDispatcher } from '../ConfigLoader'; import { logger } from '@aws-github-runner/aws-powertools-util'; import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; @@ -243,124 +243,6 @@ describe('Dispatcher', () => { }); }); - describe('decides can run job based on label and config (canRunJob)', () => { - it('should accept job with an exact match and identical labels.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should accept job with an exact match and identical labels, ignoring cases.', () => { - const workflowLabels = ['self-Hosted', 'Linux', 'X64', 'ubuntu-Latest']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should accept job with an exact match and runner supports requested capabilities.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should NOT accept job with an exact match and runner not matching requested capabilities.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(false); - }); - - it('should accept job with for a non exact match. Any label that matches will accept the job.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; - const runnerLabels = [['gpu']]; - expect(canRunJob(workflowLabels, runnerLabels, false)).toBe(true); - }); - - it('should NOT accept job with for an exact match. Not all requested capabilities are supported.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; - const runnerLabels = [['gpu']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(false); - }); - - it('should match when runner has more labels than workflow requests with exactMatch=true (unidirectional).', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404', 'on-demand']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should match when labels are exactly identical with exactMatch=true.', () => { - const workflowLabels = ['self-hosted', 'linux', 'on-demand']; - const runnerLabels = [['self-hosted', 'linux', 'on-demand']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should match with exactMatch=true when labels are in different order.', () => { - const workflowLabels = ['linux', 'self-hosted', 'x64']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should match with exactMatch=true when labels are completely shuffled.', () => { - const workflowLabels = ['x64', 'ubuntu-latest', 'self-hosted', 'linux']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should match with exactMatch=false when labels are in different order.', () => { - const workflowLabels = ['gpu', 'self-hosted']; - const runnerLabels = [['self-hosted', 'gpu']]; - expect(canRunJob(workflowLabels, runnerLabels, false)).toBe(true); - }); - - // bidirectionalLabelMatch tests - it('should NOT match when runner has more labels than workflow requests (bidirectionalLabelMatch=true).', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404', 'on-demand']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); - }); - - it('should NOT match when workflow has more labels than runner (bidirectionalLabelMatch=true).', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); - }); - - it('should match when labels are exactly identical with bidirectionalLabelMatch=true.', () => { - const workflowLabels = ['self-hosted', 'linux', 'on-demand']; - const runnerLabels = [['self-hosted', 'linux', 'on-demand']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); - }); - - it('should match with bidirectionalLabelMatch=true when labels are in different order.', () => { - const workflowLabels = ['linux', 'self-hosted', 'x64']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); - }); - - it('should match with bidirectionalLabelMatch=true when labels are completely shuffled.', () => { - const workflowLabels = ['x64', 'ubuntu-latest', 'self-hosted', 'linux']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); - }); - - it('should match with bidirectionalLabelMatch=true ignoring case.', () => { - const workflowLabels = ['Self-Hosted', 'Linux', 'X64']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); - }); - - it('should NOT match empty workflow labels with bidirectionalLabelMatch=true.', () => { - const workflowLabels: string[] = []; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); - }); - - it('bidirectionalLabelMatch takes precedence over exactMatch when both are true.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - // exactMatch alone would accept this (runner has extra labels), but bidirectional should reject - expect(canRunJob(workflowLabels, runnerLabels, true, true)).toBe(false); - }); - }); - describe('per-matcher dynamic labels handling', () => { const baseRunner = runnerConfig[0]; @@ -456,7 +338,7 @@ describe('Dispatcher', () => { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, enableDynamicLabels: true, - ec2DynamicLabelsPolicy: { + awsDynamicLabelsPolicy: { restricted_keys: { 'instance-type': { allowed: ['m5.*'] }, }, @@ -499,7 +381,7 @@ describe('Dispatcher', () => { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, enableDynamicLabels: true, - ec2DynamicLabelsPolicy: { + awsDynamicLabelsPolicy: { restricted_keys: { 'instance-type': { allowed: ['m5.*'] }, }, @@ -537,7 +419,7 @@ describe('Dispatcher', () => { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, enableDynamicLabels: true, - ec2DynamicLabelsPolicy: {}, + awsDynamicLabelsPolicy: {}, }, }, ]); diff --git a/lambdas/functions/webhook/src/runners/dispatch.ts b/lambdas/functions/webhook/src/runners/dispatch.ts index 16c65fb9a3..47c1f1bfc0 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.ts @@ -5,13 +5,11 @@ import { Response } from '../lambda'; import { RunnerMatcherConfig, sendActionRequest } from '../sqs'; import ValidationError from '../ValidationError'; import { ConfigDispatcher, ConfigWebhook, QueueSelectionStrategy } from '../ConfigLoader'; -import { violationsAgainstPolicy } from './dynamic-labels-policy'; +import { selectAwsDynamicLabelQueue } from './aws-dynamic-labels'; +import { canRunJob, splitWorkflowJobLabels } from './labels'; const logger = createChildLogger('handler'); -const GHR_LABEL_MAX_LENGTH = 128; -const GHR_LABEL_VALUE_PATTERN = /^[a-zA-Z0-9._/;\-:]+$/; - export async function dispatch( event: WorkflowJobEvent, eventType: string, @@ -55,11 +53,7 @@ async function handleWorkflowJob( return aStrict === bStrict ? 0 : aStrict ? -1 : 1; }); - const allLabels = body.workflow_job.labels; - const ghrLabels = allLabels.filter((l) => l.startsWith('ghr-')); - const sanitizedGhrLabels = sanitizeGhrLabels(ghrLabels); - const nonGhrLabels = allLabels.filter((l) => !l.startsWith('ghr-')); - const hasDynamicLabels = sanitizedGhrLabels.length > 0; + const { nonGhrLabels, sanitizedGhrLabels, hasDynamicLabels } = splitWorkflowJobLabels(body.workflow_job.labels); // 1. Collect all queues whose non-dynamic labels match the job. const matches: RunnerMatcherConfig[] = matcherConfig.filter((q) => @@ -87,29 +81,14 @@ async function handleWorkflowJob( targets = selectQueues(topMatches, queueSelectionStrategy); labelsToSend = nonGhrLabels; } else { - // Dynamic labels present: prefer the first match that has dynamic labels - // enabled AND accepts these labels under its policy. The queue selection - // strategy applies to standard jobs only; dynamic-label jobs always use the - // first compliant queue. - let compliant: RunnerMatcherConfig | undefined; - for (const q of matches) { - if (!q.matcherConfig.enableDynamicLabels) { - logger.warn(`Queue ${q.id} matches non-dynamic labels but does not allow dynamic labels; trying next match`); - continue; - } - const violations = violationsAgainstPolicy(sanitizedGhrLabels, q.matcherConfig.ec2DynamicLabelsPolicy); - if (violations.length === 0) { - compliant = q; - break; - } - for (const v of violations) { - logger.warn(`Queue ${q.id}: dynamic label '${v.label}' does not match policy (${v.reason}); trying next match`); - } - } - - if (compliant) { - targets = [compliant]; - labelsToSend = [...nonGhrLabels, ...sanitizedGhrLabels]; + // Dynamic labels present: prefer the first provider-compliant queue. The + // queue selection strategy applies to standard jobs only; dynamic-label jobs + // always use the first compliant queue. + const dynamicTarget = selectAwsDynamicLabelQueue(matches, nonGhrLabels, sanitizedGhrLabels); + + if (dynamicTarget) { + targets = [dynamicTarget.queue]; + labelsToSend = dynamicTarget.labels; } else { // No queue accepts the dynamic labels under its policy: refuse the job. logger.warn(`No queue accepts the dynamic labels for this job; not dispatching`, { @@ -152,7 +131,7 @@ async function handleWorkflowJob( * so a single pool's queue does not become a bottleneck. * - 'all' returns every candidate, scaling up one runner per matching pool and * letting the first to become available take the job (speed over cost). Note - * this multiplies instance launches and runner registrations per job. + * this multiplies AWS launches and runner registrations per job. */ function selectQueues(candidates: RunnerMatcherConfig[], strategy: QueueSelectionStrategy): RunnerMatcherConfig[] { switch (strategy) { @@ -174,50 +153,3 @@ function notAccepted(body: WorkflowJobEvent): Response { ); return { statusCode: 202, body: notAcceptedErrorMsg }; } - -function sanitizeGhrLabels(labels: string[]): string[] { - return labels.filter((label) => { - if (label.length > GHR_LABEL_MAX_LENGTH) { - logger.warn('Dynamic label exceeds max length, stripping', { label: label.substring(0, 40) }); - return false; - } - if (!GHR_LABEL_VALUE_PATTERN.test(label)) { - logger.warn('Dynamic label contains invalid characters, stripping', { label }); - return false; - } - return true; - }); -} - -/** - * Pure label match against a runner's `labelMatchers`. Caller is expected to - * pass only non-dynamic labels. - */ -export function canRunJob( - workflowJobLabels: string[], - runnerLabelsMatchers: string[][], - workflowLabelCheckAll: boolean, - bidirectionalLabelMatch = false, -): boolean { - const lowered = runnerLabelsMatchers.map((rl) => rl.map((l) => l.toLowerCase())); - - let match: boolean; - if (bidirectionalLabelMatch) { - const workflowLabelsLower = workflowJobLabels.map((wl) => wl.toLowerCase()); - match = lowered.some( - (rl) => workflowLabelsLower.every((wl) => rl.includes(wl)) && rl.every((r) => workflowLabelsLower.includes(r)), - ); - } else { - const matchLabels = workflowLabelCheckAll - ? lowered.some((rl) => workflowJobLabels.every((wl) => rl.includes(wl.toLowerCase()))) - : lowered.some((rl) => workflowJobLabels.some((wl) => rl.includes(wl.toLowerCase()))); - match = workflowJobLabels.length === 0 ? !matchLabels : matchLabels; - } - - logger.debug( - `Received workflow job event with labels: '${JSON.stringify(workflowJobLabels)}'. The event does ${ - match ? '' : 'NOT ' - }match the runner labels: '${Array.from(lowered).join(',')}'`, - ); - return match; -} diff --git a/lambdas/functions/webhook/src/runners/dynamic-labels-policy.test.ts b/lambdas/functions/webhook/src/runners/ec2-dynamic-labels-policy.test.ts similarity index 99% rename from lambdas/functions/webhook/src/runners/dynamic-labels-policy.test.ts rename to lambdas/functions/webhook/src/runners/ec2-dynamic-labels-policy.test.ts index d9d3c9325d..0497bee33a 100644 --- a/lambdas/functions/webhook/src/runners/dynamic-labels-policy.test.ts +++ b/lambdas/functions/webhook/src/runners/ec2-dynamic-labels-policy.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; -import { violationsAgainstPolicy, type Ec2DynamicLabelsPolicy } from './dynamic-labels-policy'; +import { violationsAgainstPolicy, type Ec2DynamicLabelsPolicy } from './ec2-dynamic-labels-policy'; describe('violationsAgainstPolicy', () => { it('returns [] when policy is null/undefined', () => { diff --git a/lambdas/functions/webhook/src/runners/dynamic-labels-policy.ts b/lambdas/functions/webhook/src/runners/ec2-dynamic-labels-policy.ts similarity index 90% rename from lambdas/functions/webhook/src/runners/dynamic-labels-policy.ts rename to lambdas/functions/webhook/src/runners/ec2-dynamic-labels-policy.ts index 54003690b7..15b6f4235d 100644 --- a/lambdas/functions/webhook/src/runners/dynamic-labels-policy.ts +++ b/lambdas/functions/webhook/src/runners/ec2-dynamic-labels-policy.ts @@ -1,8 +1,6 @@ -export interface Ec2DynamicLabelsValueRule { - allowed?: string[]; - denied?: string[]; - max?: number | string; -} +import type { AwsDynamicLabelsPolicy, AwsDynamicLabelsValueRule } from './aws-dynamic-labels-policy'; + +export type Ec2DynamicLabelsValueRule = AwsDynamicLabelsValueRule; /** * EC2 dynamic labels policy schema. `blocked_keys` rejects keys outright; @@ -10,10 +8,7 @@ export interface Ec2DynamicLabelsValueRule { * `` segment of a `ghr-ec2-:` label in the same hyphenated * form as the labels themselves (e.g. `instance-type`). */ -export interface Ec2DynamicLabelsPolicy { - blocked_keys?: string[]; - restricted_keys?: Record; -} +export type Ec2DynamicLabelsPolicy = AwsDynamicLabelsPolicy; function globToRegExp(glob: string): RegExp { const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&'); diff --git a/lambdas/functions/webhook/src/runners/ec2-dynamic-labels.ts b/lambdas/functions/webhook/src/runners/ec2-dynamic-labels.ts new file mode 100644 index 0000000000..fa27d4c559 --- /dev/null +++ b/lambdas/functions/webhook/src/runners/ec2-dynamic-labels.ts @@ -0,0 +1,44 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + +import { RunnerMatcherConfig } from '../sqs'; +import { AwsDynamicLabelDispatchTarget, AwsDynamicLabelProviderStrategy } from './aws-dynamic-labels-provider'; +import { violationsAgainstPolicy } from './ec2-dynamic-labels-policy'; + +const logger = createChildLogger('handler'); + +export type Ec2DynamicLabelDispatchTarget = AwsDynamicLabelDispatchTarget; + +export function selectEc2DynamicLabelQueue( + matches: RunnerMatcherConfig[], + nonGhrLabels: string[], + sanitizedGhrLabels: string[], +): Ec2DynamicLabelDispatchTarget | undefined { + for (const queue of matches) { + if (!queue.matcherConfig.enableDynamicLabels) { + logger.warn(`Queue ${queue.id} matches non-dynamic labels but does not allow dynamic labels; trying next match`); + continue; + } + + const violations = violationsAgainstPolicy(sanitizedGhrLabels, queue.matcherConfig.awsDynamicLabelsPolicy); + if (violations.length === 0) { + return { + queue, + labels: [...nonGhrLabels, ...sanitizedGhrLabels], + }; + } + + for (const violation of violations) { + logger.warn( + `Queue ${queue.id}: dynamic label '${violation.label}' does not match policy (${violation.reason}); trying next match`, + ); + } + } + + return undefined; +} + +export const ec2DynamicLabelProviderStrategy: AwsDynamicLabelProviderStrategy = { + type: 'ec2', + selectQueue: ({ queue, nonGhrLabels, sanitizedGhrLabels }) => + selectEc2DynamicLabelQueue([queue], nonGhrLabels, sanitizedGhrLabels), +}; diff --git a/lambdas/functions/webhook/src/runners/labels.test.ts b/lambdas/functions/webhook/src/runners/labels.test.ts new file mode 100644 index 0000000000..5263766063 --- /dev/null +++ b/lambdas/functions/webhook/src/runners/labels.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'vitest'; + +import { canRunJob } from './labels'; + +describe('decides can run job based on label and config (canRunJob)', () => { + it('should accept job with an exact match and identical labels.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should accept job with an exact match and identical labels, ignoring cases.', () => { + const workflowLabels = ['self-Hosted', 'Linux', 'X64', 'ubuntu-Latest']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should accept job with an exact match and runner supports requested capabilities.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should NOT accept job with an exact match and runner not matching requested capabilities.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(false); + }); + + it('should accept job with for a non exact match. Any label that matches will accept the job.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; + const runnerLabels = [['gpu']]; + expect(canRunJob(workflowLabels, runnerLabels, false)).toBe(true); + }); + + it('should NOT accept job with for an exact match. Not all requested capabilities are supported.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; + const runnerLabels = [['gpu']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(false); + }); + + it('should match when runner has more labels than workflow requests with exactMatch=true (unidirectional).', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404', 'on-demand']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should match when labels are exactly identical with exactMatch=true.', () => { + const workflowLabels = ['self-hosted', 'linux', 'on-demand']; + const runnerLabels = [['self-hosted', 'linux', 'on-demand']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should match with exactMatch=true when labels are in different order.', () => { + const workflowLabels = ['linux', 'self-hosted', 'x64']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should match with exactMatch=true when labels are completely shuffled.', () => { + const workflowLabels = ['x64', 'ubuntu-latest', 'self-hosted', 'linux']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should match with exactMatch=false when labels are in different order.', () => { + const workflowLabels = ['gpu', 'self-hosted']; + const runnerLabels = [['self-hosted', 'gpu']]; + expect(canRunJob(workflowLabels, runnerLabels, false)).toBe(true); + }); + + // bidirectionalLabelMatch tests + it('should NOT match when runner has more labels than workflow requests (bidirectionalLabelMatch=true).', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404', 'on-demand']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); + }); + + it('should NOT match when workflow has more labels than runner (bidirectionalLabelMatch=true).', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); + }); + + it('should match when labels are exactly identical with bidirectionalLabelMatch=true.', () => { + const workflowLabels = ['self-hosted', 'linux', 'on-demand']; + const runnerLabels = [['self-hosted', 'linux', 'on-demand']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); + }); + + it('should match with bidirectionalLabelMatch=true when labels are in different order.', () => { + const workflowLabels = ['linux', 'self-hosted', 'x64']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); + }); + + it('should match with bidirectionalLabelMatch=true when labels are completely shuffled.', () => { + const workflowLabels = ['x64', 'ubuntu-latest', 'self-hosted', 'linux']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); + }); + + it('should match with bidirectionalLabelMatch=true ignoring case.', () => { + const workflowLabels = ['Self-Hosted', 'Linux', 'X64']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); + }); + + it('should NOT match empty workflow labels with bidirectionalLabelMatch=true.', () => { + const workflowLabels: string[] = []; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); + }); + + it('bidirectionalLabelMatch takes precedence over exactMatch when both are true.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + // exactMatch alone would accept this (runner has extra labels), but bidirectional should reject + expect(canRunJob(workflowLabels, runnerLabels, true, true)).toBe(false); + }); +}); diff --git a/lambdas/functions/webhook/src/runners/labels.ts b/lambdas/functions/webhook/src/runners/labels.ts new file mode 100644 index 0000000000..d4094695aa --- /dev/null +++ b/lambdas/functions/webhook/src/runners/labels.ts @@ -0,0 +1,75 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + +const logger = createChildLogger('handler'); + +const GHR_LABEL_MAX_LENGTH = 128; +const GHR_LABEL_VALUE_PATTERN = /^[a-zA-Z0-9._/;\-:]+$/; + +export interface WorkflowJobLabels { + allLabels: string[]; + ghrLabels: string[]; + sanitizedGhrLabels: string[]; + nonGhrLabels: string[]; + hasDynamicLabels: boolean; +} + +export function splitWorkflowJobLabels(labels: string[]): WorkflowJobLabels { + const ghrLabels = labels.filter((label) => label.startsWith('ghr-')); + const sanitizedGhrLabels = sanitizeGhrLabels(ghrLabels); + const nonGhrLabels = labels.filter((label) => !label.startsWith('ghr-')); + + return { + allLabels: labels, + ghrLabels, + sanitizedGhrLabels, + nonGhrLabels, + hasDynamicLabels: sanitizedGhrLabels.length > 0, + }; +} + +export function sanitizeGhrLabels(labels: string[]): string[] { + return labels.filter((label) => { + if (label.length > GHR_LABEL_MAX_LENGTH) { + logger.warn('Dynamic label exceeds max length, stripping', { label: label.substring(0, 40) }); + return false; + } + if (!GHR_LABEL_VALUE_PATTERN.test(label)) { + logger.warn('Dynamic label contains invalid characters, stripping', { label }); + return false; + } + return true; + }); +} + +/** + * Pure label match against a runner's `labelMatchers`. Caller is expected to + * pass only non-dynamic labels. + */ +export function canRunJob( + workflowJobLabels: string[], + runnerLabelsMatchers: string[][], + workflowLabelCheckAll: boolean, + bidirectionalLabelMatch = false, +): boolean { + const lowered = runnerLabelsMatchers.map((rl) => rl.map((l) => l.toLowerCase())); + + let match: boolean; + if (bidirectionalLabelMatch) { + const workflowLabelsLower = workflowJobLabels.map((wl) => wl.toLowerCase()); + match = lowered.some( + (rl) => workflowLabelsLower.every((wl) => rl.includes(wl)) && rl.every((r) => workflowLabelsLower.includes(r)), + ); + } else { + const matchLabels = workflowLabelCheckAll + ? lowered.some((rl) => workflowJobLabels.every((wl) => rl.includes(wl.toLowerCase()))) + : lowered.some((rl) => workflowJobLabels.some((wl) => rl.includes(wl.toLowerCase()))); + match = workflowJobLabels.length === 0 ? !matchLabels : matchLabels; + } + + logger.debug( + `Received workflow job event with labels: '${JSON.stringify(workflowJobLabels)}'. The event does ${ + match ? '' : 'NOT ' + }match the runner labels: '${Array.from(lowered).join(',')}'`, + ); + return match; +} diff --git a/lambdas/functions/webhook/src/sqs/index.ts b/lambdas/functions/webhook/src/sqs/index.ts index 460d197e65..9d14fb7474 100644 --- a/lambdas/functions/webhook/src/sqs/index.ts +++ b/lambdas/functions/webhook/src/sqs/index.ts @@ -2,12 +2,14 @@ import { SQS, SendMessageCommandInput } from '@aws-sdk/client-sqs'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; import { createChildLogger, getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; -import { Ec2DynamicLabelsPolicy } from '../runners/dynamic-labels-policy'; +import type { AwsDynamicLabelsPolicy } from '../runners/aws-dynamic-labels-policy'; const logger = createChildLogger('sqs'); const sqsClientsByRegion = new Map(); +export type RunnerProvider = string; + export interface ActionRequestMessage { id: number; eventType: string; @@ -24,13 +26,14 @@ export interface MatcherConfig { exactMatch: boolean; bidirectionalLabelMatch?: boolean; enableDynamicLabels?: boolean; - ec2DynamicLabelsPolicy?: Ec2DynamicLabelsPolicy | null; + awsDynamicLabelsPolicy?: AwsDynamicLabelsPolicy | null; } export type RunnerConfig = RunnerMatcherConfig[]; export interface RunnerMatcherConfig { matcherConfig: MatcherConfig; + runnerProvider?: RunnerProvider; id: string; arn: string; } From 3a89176685e220826dffa0baae709e90b46b98c9 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 7 Jul 2026 22:10:08 +0200 Subject: [PATCH 06/46] chore(terraform): wire aws runner provider defaults --- README.md | 6 +++--- docs/configuration.md | 8 ++++++-- main.tf | 2 +- modules/multi-runner/README.md | 2 +- modules/multi-runner/variables.tf | 7 ++++--- modules/runners/README.md | 4 ++-- modules/runners/pool/main.tf | 1 + modules/runners/scale-up.tf | 1 + modules/runners/variables.tf | 3 ++- modules/webhook/README.md | 2 +- modules/webhook/variables.tf | 4 ++-- variables.tf | 18 ++++++++++-------- 12 files changed, 34 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 68f02946d6..9420ccd26f 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh) | [create\_service\_linked\_role\_spot](#input\_create\_service\_linked\_role\_spot) | (optional) create the service linked role for spot instances that is required by the scale-up lambda. | `bool` | `false` | no | | [delay\_webhook\_event](#input\_delay\_webhook\_event) | The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event. | `number` | `30` | no | | [disable\_runner\_autoupdate](#input\_disable\_runner\_autoupdate) | Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/) | `bool` | `false` | no | -| [ec2\_dynamic\_labels\_policy](#input\_ec2\_dynamic\_labels\_policy) | Experimental! Can be removed / changed without trigger a major release.
Optional policy for dynamic EC2 override labels evaluated by the webhook
dispatcher. Only effective when `enable_dynamic_labels = true`.

Jobs whose EC2 dynamic labels violate the policy are rejected with a 202 and a
warning is logged.

Evaluation:
1. Keys in `blocked_keys` are always rejected.
2. Keys in `restricted_keys` are allowed only when their value passes the rule.
3. Keys not listed in `blocked_keys` or `restricted_keys` are allowed.

Schema:
- `blocked_keys`: keys to reject outright.
- `restricted_keys`: map of key to value rule:
`{ allowed = [globs], denied = [globs], max = number|string }`.

Keys use the `ghr-ec2-*` dynamic label suffix, not the full label. For example, use
`instance-type` for `ghr-ec2-instance-type`. | `any` | `null` | no | +| [aws\_dynamic\_labels\_policy](#input\_aws\_dynamic\_labels\_policy) | Experimental! Can be removed / changed without trigger a major release.
Optional AWS dynamic label policy evaluated by the webhook dispatcher.
Only effective when `enable_dynamic_labels = true`.

Jobs whose provider-specific dynamic labels violate the policy are rejected
with a 202 and a warning is logged. Currently this policy applies to EC2
override labels using the `ghr-ec2-*` prefix.

Evaluation:
1. Keys in `blocked_keys` are always rejected.
2. Keys in `restricted_keys` are allowed only when their value passes the rule.
3. Keys not listed in `blocked_keys` or `restricted_keys` are allowed.

Schema:
- `blocked_keys`: keys to reject outright.
- `restricted_keys`: map of key to value rule:
`{ allowed = [globs], denied = [globs], max = number|string }`.

Keys use the provider dynamic label suffix, not the full label. For example,
use `instance-type` for `ghr-ec2-instance-type`. | `any` | `null` | no | | [enable\_ami\_housekeeper](#input\_enable\_ami\_housekeeper) | Option to disable the lambda to clean up old AMIs. | `bool` | `false` | no | | [enable\_cloudwatch\_agent](#input\_enable\_cloudwatch\_agent) | Enables the cloudwatch agent on the ec2 runner instances. The runner uses a default config that can be overridden via `cloudwatch_config`. | `bool` | `true` | no | | [enable\_dynamic\_labels](#input\_enable\_dynamic\_labels) | Experimental! Can be removed / changed without trigger a major release. Enable dynamic EC2 configs based on workflow job labels. When enabled, jobs can request specific configs via the 'ghr-ec2-:' label (e.g., 'ghr-ec2-instance-type:t3.large'). When enabled, labels starting with `ghr-` are ignored during webhook label matching. | `bool` | `false` | no | @@ -141,7 +141,7 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh) | [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. Example: https://github.internal.co - DO NOT SET IF USING PUBLIC GITHUB. However if you are using GitHub Enterprise Cloud with data-residency (ghe.com), set the endpoint here. Example - https://companyname.ghe.com | `string` | `null` | no | | [github\_app](#input\_github\_app) | GitHub app parameters, see your github app.
You can optionally create the SSM parameters yourself and provide the ARN and name here, through the `*_ssm` attributes.
If you chose to provide the configuration values directly here,
please ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`).
Note: the provided SSM parameters arn and name have a precedence over the actual value (i.e `key_base64_ssm` has a precedence over `key_base64` etc). | 
object({
    key_base64 = optional(string)
    key_base64_ssm = optional(object({
      arn  = string
      name = string
    }))
    id = optional(string)
    id_ssm = optional(object({
      arn  = string
      name = string
    }))
    webhook_secret = optional(string)
    webhook_secret_ssm = optional(object({
      arn  = string
      name = string
    }))
  })
| n/a | yes | | [iam\_overrides](#input\_iam\_overrides) | This map provides the possibility to override some IAM defaults. Note that when using this variable, you are responsible for ensuring the role has necessary permissions to access required resources. `override_instance_profile`: When set to true, uses the instance profile name specified in `instance_profile_name` instead of creating a new instance profile. `override_runner_role`: When set to true, uses the role ARN specified in `runner_role_arn` instead of creating a new IAM role. | 
object({
    override_instance_profile = optional(bool, null)
    instance_profile_name     = optional(string, null)
    override_runner_role      = optional(bool, null)
    runner_role_arn           = optional(string, null)
  })
| 
{
  "instance_profile_name": null,
  "override_instance_profile": false,
  "override_runner_role": false,
  "runner_role_arn": null
}
| no | -| [idle\_config](#input\_idle\_config) | List of time periods, defined as a cron expression, to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    cron             = string
    timeZone         = string
    idleCount        = number
    evictionStrategy = optional(string, "oldest_first")
  }))
| `[]` | no | +| [idle\_config](#input\_idle\_config) | List of time periods, defined as a cron expression, to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    type             = optional(string, "ec2")
    cron             = string
    timeZone         = string
    idleCount        = number
    evictionStrategy = optional(string, "oldest_first")
  }))
| `[]` | no | | [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for spot instances. AWS recommends using `price-capacity-optimized` however the AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | | [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no | @@ -222,7 +222,7 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh) | [runners\_scale\_up\_lambda\_timeout](#input\_runners\_scale\_up\_lambda\_timeout) | Time out for the scale up lambda in seconds. | `number` | `30` | no | | [runners\_ssm\_housekeeper](#input\_runners\_ssm\_housekeeper) | Configuration for the SSM housekeeper lambda. This lambda deletes token / JIT config from SSM.

`schedule_expression`: is used to configure the schedule for the lambda.
`enabled`: enable or disable the lambda trigger via the EventBridge.
`lambda_memory_size`: lambda memory size limit.
`lambda_timeout`: timeout for the lambda in seconds.
`config`: configuration for the lambda function. Token path will be read by default from the module. | 
object({
    schedule_expression = optional(string, "rate(1 day)")
    enabled             = optional(bool, true)
    lambda_memory_size  = optional(number, 512)
    lambda_timeout      = optional(number, 60)
    config = object({
      tokenPath      = optional(string)
      minimumDaysOld = optional(number, 1)
      dryRun         = optional(bool, false)
    })
  })
| 
{
  "config": {}
}
| no | | [scale\_down\_schedule\_expression](#input\_scale\_down\_schedule\_expression) | Scheduler expression to check every x for scale down. | `string` | `"cron(*/5 * * * ? *)"` | no | -| [scale\_errors](#input\_scale\_errors) | List of aws error codes that should trigger retry during scale up. This list will replace the default errors defined in the variable `defaultScaleErrors` in https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/lambdas/functions/control-plane/src/aws/runners.ts | `list(string)` | 
[
  "UnfulfillableCapacity",
  "MaxSpotInstanceCountExceeded",
  "TargetCapacityLimitExceededException",
  "RequestLimitExceeded",
  "ResourceLimitExceeded",
  "MaxSpotInstanceCountExceeded",
  "MaxSpotFleetRequestCountExceeded",
  "InsufficientInstanceCapacity",
  "InsufficientCapacityOnHost"
]
| no | +| [scale\_errors](#input\_scale\_errors) | List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors | `list(string)` | 
[
  "UnfulfillableCapacity",
  "MaxSpotInstanceCountExceeded",
  "TargetCapacityLimitExceededException",
  "RequestLimitExceeded",
  "ResourceLimitExceeded",
  "MaxSpotInstanceCountExceeded",
  "MaxSpotFleetRequestCountExceeded",
  "InsufficientInstanceCapacity",
  "InsufficientCapacityOnHost"
]
| no | | [scale\_up\_reserved\_concurrent\_executions](#input\_scale\_up\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no | | [ssm\_paths](#input\_ssm\_paths) | The root path used in SSM to store configuration and secrets. | 
object({
    root       = optional(string, "github-action-runners")
    app        = optional(string, "app")
    runners    = optional(string, "runners")
    webhook    = optional(string, "webhook")
    use_prefix = optional(bool, true)
  })
| `{}` | no | | [state\_event\_rule\_binaries\_syncer](#input\_state\_event\_rule\_binaries\_syncer) | Option to disable EventBridge Lambda trigger for the binary syncer, useful to stop automatic updates of binary distribution | `string` | `"ENABLED"` | no | diff --git a/docs/configuration.md b/docs/configuration.md index 2cdf2e95a9..62e9154bc5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -118,6 +118,8 @@ By default, the oldest instances are evicted. This helps keep your environment u ```hcl idle_config = [{ + # Defaults to 'ec2' + type = "ec2" cron = "* * 9-17 * * 1-5" timeZone = "Europe/Amsterdam" idleCount = 2 @@ -336,7 +338,7 @@ Below is an example of the log messages created. [!WARNING] **Security implication:** Dynamic labels are extracted from the `runs-on` labels in incoming `workflow_job` webhook events. These labels originate from what -users define in their workflow files. Any user with permission to create or modify workflows can inject arbitrary EC2 configuration values — including instance types, AMI IDs, subnet IDs, EBS volumes, placement settings, and more. Unless constrained with `ec2_dynamic_labels_policy`, these values are not validated against label-specific rules before being passed to the EC2 CreateFleet API. This means a malicious or careless workflow author could, for example: +users define in their workflow files. Any user with permission to create or modify workflows can inject arbitrary EC2 configuration values — including instance types, AMI IDs, subnet IDs, EBS volumes, placement settings, and more. Unless constrained with `aws_dynamic_labels_policy` in the root module, or `matcherConfig.awsDynamicLabelsPolicy` when configuring matcher entries directly, these values are not validated against label-specific rules before being passed to the EC2 CreateFleet API. This means a malicious or careless workflow author could, for example: - Launch expensive instance types (e.g., `p5.48xlarge`) to inflate costs - Override the AMI (`ghr-ec2-image-id`) to boot a compromised image @@ -367,7 +369,7 @@ module "runners" { ... enable_dynamic_labels = true - ec2_dynamic_labels_policy = { + aws_dynamic_labels_policy = { blocked_keys = ["image-id", "subnet-id"] restricted_keys = { @@ -385,6 +387,8 @@ module "runners" { } ``` +The root module variable is named `aws_dynamic_labels_policy`. The webhook matcher config receives the same policy under `matcherConfig.awsDynamicLabelsPolicy`. If you configure `runner_matcher_config` or `multi_runner_config.matcherConfig` directly, use `awsDynamicLabelsPolicy` for this policy. + The policy is evaluated by dynamic label key: 1. Keys in `blocked_keys` are always rejected. diff --git a/main.tf b/main.tf index 546270fe2d..ca83523285 100644 --- a/main.tf +++ b/main.tf @@ -116,7 +116,7 @@ module "webhook" { exactMatch : var.enable_runner_workflow_job_labels_check_all bidirectionalLabelMatch : var.enable_runner_bidirectional_label_match enableDynamicLabels : var.enable_dynamic_labels - ec2DynamicLabelsPolicy : var.ec2_dynamic_labels_policy + awsDynamicLabelsPolicy : var.aws_dynamic_labels_policy } } } diff --git a/modules/multi-runner/README.md b/modules/multi-runner/README.md index 13085c2b3c..e658ef7062 100644 --- a/modules/multi-runner/README.md +++ b/modules/multi-runner/README.md @@ -152,7 +152,7 @@ module "multi-runner" { | [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no | | [matcher\_config\_parameter\_store\_tier](#input\_matcher\_config\_parameter\_store\_tier) | The tier of the parameter store for the matcher configuration. Valid values are `Standard`, and `Advanced`. | `string` | `"Standard"` | no | | [metrics](#input\_metrics) | Configuration for metrics created by the module, by default metrics are disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | 
object({
    enable    = optional(bool, false)
    namespace = optional(string, "GitHub Runners")
    metric = optional(object({
      enable_github_app_rate_limit    = optional(bool, true)
      enable_job_retry                = optional(bool, true)
      enable_spot_termination_warning = optional(bool, true)
    }), {})
  })
| `{}` | no | -| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {
runner\_config: {
runner\_os: "The EC2 Operating System type to use for action runner instances (linux, osx, windows)."
runner\_architecture: "The platform architecture of the runner instance\_type."
runner\_metadata\_options: "(Optional) Metadata options for the ec2 runner instances."
ami: "(Optional) AMI configuration for the action runner instances. This object allows you to specify all AMI-related settings in one place."
create\_service\_linked\_role\_spot: (Optional) create the serviced linked role for spot instances that is required by the scale-up lambda.
credit\_specification: "(Optional) The credit specification of the runner instance\_type. Can be unset, `standard` or `unlimited`.
delay\_webhook\_event: "The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event."
disable\_runner\_autoupdate: "Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)"
ebs\_optimized: "The EC2 EBS optimized configuration."
enable\_ephemeral\_runners: "Enable ephemeral runners, runners will only be used once."
enable\_job\_queued\_check: Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners."
enable\_on\_demand\_failover\_for\_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later."
scale\_errors: "List of aws error codes that should trigger retry during scale up. This list will replace the default errors defined in the variable `defaultScaleErrors` in https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/lambdas/functions/control-plane/src/aws/runners.ts"
enable\_organization\_runners: "Register runners to organization, instead of repo level"
enable\_runner\_binaries\_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
enable\_ssm\_on\_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances."
enable\_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI."
instance\_allocation\_strategy: "The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`."
instance\_max\_spot\_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet."
instance\_target\_capacity\_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`."
instance\_types: "List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)."
job\_queue\_retention\_in\_seconds: "The number of seconds the job is held in the queue before it is purged"
minimum\_running\_time\_in\_minutes: "The time an ec2 action runner should be running at minimum before terminated if not busy."
pool\_runner\_owner: "The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported."
runner\_additional\_security\_group\_ids: "List of additional security groups IDs to apply to the runner. If added outside the multi\_runner\_config block, the additional security group(s) will be applied to all runner configs. If added inside the multi\_runner\_config, the additional security group(s) will be applied to the individual runner."
runner\_as\_root: "Run the action runner under the root user. Variable `runner_run_as` will be ignored."
runner\_boot\_time\_in\_minutes: "The minimum time for an EC2 runner to boot and register as a runner."
runner\_disable\_default\_labels: "Disable default labels for the runners (os, architecture and `self-hosted`). If enabled, the runner will only have the extra labels provided in `runner_extra_labels`. In case you on own start script is used, this configuration parameter needs to be parsed via SSM."
runner\_extra\_labels: "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `multi_runner_config.matcherConfig.exactMatch`. GitHub read-only labels should not be provided."
runner\_group\_name: "Name of the runner group."
runner\_name\_prefix: "Prefix for the GitHub runner name."
runner\_run\_as: "Run the GitHub actions agent as user."
runners\_maximum\_count: "The maximum number of runners that will be created. Setting the variable to `-1` disables the maximum check."
scale\_down\_schedule\_expression: "Scheduler expression to check every x for scale down."
scale\_up\_reserved\_concurrent\_executions: "Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations."
lambda\_event\_source\_mapping\_batch\_size: "(Optional) Maximum number of records per Lambda invocation for this runner flavor. Overrides the module-level `lambda_event_source_mapping_batch_size` when set."
lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds: "(Optional) Maximum seconds to gather records before invoking Lambda for this runner flavor. Overrides the module-level `lambda_event_source_mapping_maximum_batching_window_in_seconds` when set."
userdata\_template: "Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored."
enable\_jit\_config: "Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is available. In case you are upgrading from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI."
enable\_runner\_detailed\_monitoring: "Should detailed monitoring be enabled for the runner. Set this to true if you want to use detailed monitoring. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html for details."
enable\_cloudwatch\_agent: "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
cloudwatch\_config: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
userdata\_pre\_install: "Script to be ran before the GitHub Actions runner is installed on the EC2 instances"
userdata\_post\_install: "Script to be ran after the GitHub Actions runner is installed on the EC2 instances"
runner\_hook\_job\_started: "Script to be ran in the runner environment at the beginning of every job"
runner\_hook\_job\_completed: "Script to be ran in the runner environment at the end of every job"
runner\_ec2\_tags: "Map of tags that will be added to the launch template instance tag specifications."
runner\_iam\_role\_managed\_policy\_arns: "Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role"
vpc\_id: "The VPC for security groups of the action runners. If not set uses the value of `var.vpc_id`."
subnet\_ids: "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. If not set, uses the value of `var.subnet_ids`."
idle\_config: "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle."
license\_specifications: "Optional EC2 License Manager license configuration ARNs for the runner launch template. Required for macOS dedicated-host runners when the host resource group uses a Mac dedicated host license configuration."
use\_dedicated\_host: "Experimental! Can be removed / changed without trigger a major release. Whether to use EC2 dedicated hosts for the runners. Needed for macos runners Note that using dedicated hosts can increase cost significantly."
runner\_log\_files: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
block\_device\_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`."
job\_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app."
pool\_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)."
iam\_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances."
}
matcherConfig: {
labelMatchers: "The list of list of labels supported by the runner configuration. `[[self-hosted, linux, x64, example]]`"
exactMatch: "DEPRECATED: Use `bidirectionalLabelMatch` instead. If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ workflow label matches it will trigger the webhook. Note: this only checks that workflow labels are a subset of runner labels, not the reverse."
bidirectionalLabelMatch: "If set to true, the runner labels and workflow job labels must be an exact two-way match (same set, any order, no extras or missing labels). This is stricter than `exactMatch` which only checks that workflow labels are a subset of runner labels. When false, if __any__ workflow label matches it will trigger the webhook."
priority: "If set it defines the priority of the matcher, the matcher with the lowest priority will be evaluated first. Default is 999, allowed values 0-999."
enableDynamicLabels: "Experimental! When true the dispatcher allows `ghr-*` dynamic labels for jobs routed to this runner. Default false."
ec2DynamicLabelsPolicy: "Optional policy for `ghr-ec2-*` labels evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose EC2 dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`."
}
redrive\_build\_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries."
} | 
map(object({
    runner_config = object({
      runner_os           = string
      runner_architecture = string
      runner_metadata_options = optional(map(any), {
        instance_metadata_tags      = "enabled"
        http_endpoint               = "enabled"
        http_tokens                 = "required"
        http_put_response_hop_limit = 1
      })
      ami = optional(object({
        filter               = optional(map(list(string)), { state = ["available"] })
        owners               = optional(list(string), ["amazon"])
        id_ssm_parameter_arn = optional(string, null)
        kms_key_arn          = optional(string, null)
      }), null)
      create_service_linked_role_spot      = optional(bool, false)
      credit_specification                 = optional(string, null)
      delay_webhook_event                  = optional(number, 30)
      disable_runner_autoupdate            = optional(bool, false)
      ebs_optimized                        = optional(bool, false)
      enable_ephemeral_runners             = optional(bool, false)
      enable_job_queued_check              = optional(bool, null)
      enable_on_demand_failover_for_errors = optional(list(string), [])
      scale_errors = optional(list(string), [
        "UnfulfillableCapacity",
        "MaxSpotInstanceCountExceeded",
        "TargetCapacityLimitExceededException",
        "RequestLimitExceeded",
        "ResourceLimitExceeded",
        "MaxSpotInstanceCountExceeded",
        "MaxSpotFleetRequestCountExceeded",
        "InsufficientInstanceCapacity",
        "InsufficientCapacityOnHost",
      ])
      enable_organization_runners                                    = optional(bool, false)
      enable_runner_binaries_syncer                                  = optional(bool, true)
      enable_ssm_on_runners                                          = optional(bool, false)
      enable_userdata                                                = optional(bool, true)
      instance_allocation_strategy                                   = optional(string, "lowest-price")
      instance_max_spot_price                                        = optional(string, null)
      instance_target_capacity_type                                  = optional(string, "spot")
      instance_types                                                 = list(string)
      job_queue_retention_in_seconds                                 = optional(number, 86400)
      minimum_running_time_in_minutes                                = optional(number, null)
      pool_runner_owner                                              = optional(string, null)
      runner_as_root                                                 = optional(bool, false)
      runner_boot_time_in_minutes                                    = optional(number, 5)
      runner_disable_default_labels                                  = optional(bool, false)
      runner_extra_labels                                            = optional(list(string), [])
      runner_group_name                                              = optional(string, "Default")
      runner_name_prefix                                             = optional(string, "")
      runner_run_as                                                  = optional(string, "ec2-user")
      runners_maximum_count                                          = number
      runner_additional_security_group_ids                           = optional(list(string), [])
      scale_down_schedule_expression                                 = optional(string, "cron(*/5 * * * ? *)")
      scale_up_reserved_concurrent_executions                        = optional(number, 1)
      lambda_event_source_mapping_batch_size                         = optional(number, null)
      lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
      userdata_template                                              = optional(string, null)
      userdata_content                                               = optional(string, null)
      enable_jit_config                                              = optional(bool, null)
      enable_runner_detailed_monitoring                              = optional(bool, false)
      enable_cloudwatch_agent                                        = optional(bool, true)
      cloudwatch_config                                              = optional(string, null)
      userdata_pre_install                                           = optional(string, "")
      userdata_post_install                                          = optional(string, "")
      runner_hook_job_started                                        = optional(string, "")
      runner_hook_job_completed                                      = optional(string, "")
      runner_ec2_tags                                                = optional(map(string), {})
      runner_iam_role_managed_policy_arns                            = optional(list(string), [])
      vpc_id                                                         = optional(string, null)
      subnet_ids                                                     = optional(list(string), null)
      idle_config = optional(list(object({
        cron             = string
        timeZone         = string
        idleCount        = number
        evictionStrategy = optional(string, "oldest_first")
      })), [])
      cpu_options = optional(object({
        core_count            = optional(number)
        threads_per_core      = optional(number)
        amd_sev_snp           = optional(string)
        nested_virtualization = optional(string)
      }), null)
      placement = optional(object({
        affinity                = optional(string)
        availability_zone       = optional(string)
        group_id                = optional(string)
        group_name              = optional(string)
        host_id                 = optional(string)
        host_resource_group_arn = optional(string)
        spread_domain           = optional(string)
        tenancy                 = optional(string)
        partition_number        = optional(number)
      }), null)
      license_specifications = optional(list(object({
        license_configuration_arn = string
      })), [])
      use_dedicated_host = optional(bool, false)
      runner_log_files = optional(list(object({
        log_group_name   = string
        prefix_log_group = bool
        file_path        = string
        log_stream_name  = string
        log_class        = optional(string, "STANDARD")
      })), null)
      block_device_mappings = optional(list(object({
        delete_on_termination      = optional(bool, true)
        device_name                = optional(string, "/dev/xvda")
        encrypted                  = optional(bool, true)
        iops                       = optional(number)
        kms_key_id                 = optional(string)
        snapshot_id                = optional(string)
        throughput                 = optional(number)
        volume_initialization_rate = optional(number)
        volume_size                = number
        volume_type                = optional(string, "gp3")
        })), [{
        volume_size = 30
      }])
      pool_config = optional(list(object({
        schedule_expression          = string
        schedule_expression_timezone = optional(string)
        size                         = number
      })), [])
      job_retry = optional(object({
        enable             = optional(bool, false)
        delay_in_seconds   = optional(number, 300)
        delay_backoff      = optional(number, 2)
        lambda_memory_size = optional(number, 256)
        lambda_timeout     = optional(number, 30)
        max_attempts       = optional(number, 1)
      }), {})
      iam_overrides = optional(object({
        override_instance_profile = optional(bool, null)
        instance_profile_name     = optional(string, null)
        override_runner_role      = optional(bool, null)
        runner_role_arn           = optional(string, null)
        }), {
        override_instance_profile = false
        instance_profile_name     = null
        override_runner_role      = false
        runner_role_arn           = null
      })
    })
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = optional(bool, false)
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      ec2DynamicLabelsPolicy  = optional(any, null)
    })
    redrive_build_queue = optional(object({
      enabled         = bool
      maxReceiveCount = number
      }), {
      enabled         = false
      maxReceiveCount = null
    })
  }))
| n/a | yes | +| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {
runner\_config: {
runner\_os: "The EC2 Operating System type to use for action runner instances (linux, osx, windows)."
runner\_architecture: "The platform architecture of the runner instance\_type."
runner\_metadata\_options: "(Optional) Metadata options for the ec2 runner instances."
ami: "(Optional) AMI configuration for the action runner instances. This object allows you to specify all AMI-related settings in one place."
create\_service\_linked\_role\_spot: (Optional) create the serviced linked role for spot instances that is required by the scale-up lambda.
credit\_specification: "(Optional) The credit specification of the runner instance\_type. Can be unset, `standard` or `unlimited`.
delay\_webhook\_event: "The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event."
disable\_runner\_autoupdate: "Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)"
ebs\_optimized: "The EC2 EBS optimized configuration."
enable\_ephemeral\_runners: "Enable ephemeral runners, runners will only be used once."
enable\_job\_queued\_check: Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners."
enable\_on\_demand\_failover\_for\_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later."
scale\_errors: "List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors"
enable\_organization\_runners: "Register runners to organization, instead of repo level"
enable\_runner\_binaries\_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
enable\_ssm\_on\_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances."
enable\_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI."
instance\_allocation\_strategy: "The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`."
instance\_max\_spot\_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet."
instance\_target\_capacity\_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`."
instance\_types: "List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)."
job\_queue\_retention\_in\_seconds: "The number of seconds the job is held in the queue before it is purged"
minimum\_running\_time\_in\_minutes: "The time an ec2 action runner should be running at minimum before terminated if not busy."
pool\_runner\_owner: "The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported."
runner\_additional\_security\_group\_ids: "List of additional security groups IDs to apply to the runner. If added outside the multi\_runner\_config block, the additional security group(s) will be applied to all runner configs. If added inside the multi\_runner\_config, the additional security group(s) will be applied to the individual runner."
runner\_as\_root: "Run the action runner under the root user. Variable `runner_run_as` will be ignored."
runner\_boot\_time\_in\_minutes: "The minimum time for an EC2 runner to boot and register as a runner."
runner\_disable\_default\_labels: "Disable default labels for the runners (os, architecture and `self-hosted`). If enabled, the runner will only have the extra labels provided in `runner_extra_labels`. In case you on own start script is used, this configuration parameter needs to be parsed via SSM."
runner\_extra\_labels: "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `multi_runner_config.matcherConfig.exactMatch`. GitHub read-only labels should not be provided."
runner\_group\_name: "Name of the runner group."
runner\_name\_prefix: "Prefix for the GitHub runner name."
runner\_run\_as: "Run the GitHub actions agent as user."
runners\_maximum\_count: "The maximum number of runners that will be created. Setting the variable to `-1` disables the maximum check."
scale\_down\_schedule\_expression: "Scheduler expression to check every x for scale down."
scale\_up\_reserved\_concurrent\_executions: "Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations."
lambda\_event\_source\_mapping\_batch\_size: "(Optional) Maximum number of records per Lambda invocation for this runner flavor. Overrides the module-level `lambda_event_source_mapping_batch_size` when set."
lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds: "(Optional) Maximum seconds to gather records before invoking Lambda for this runner flavor. Overrides the module-level `lambda_event_source_mapping_maximum_batching_window_in_seconds` when set."
userdata\_template: "Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored."
enable\_jit\_config: "Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is available. In case you are upgrading from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI."
enable\_runner\_detailed\_monitoring: "Should detailed monitoring be enabled for the runner. Set this to true if you want to use detailed monitoring. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html for details."
enable\_cloudwatch\_agent: "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
cloudwatch\_config: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
userdata\_pre\_install: "Script to be ran before the GitHub Actions runner is installed on the EC2 instances"
userdata\_post\_install: "Script to be ran after the GitHub Actions runner is installed on the EC2 instances"
runner\_hook\_job\_started: "Script to be ran in the runner environment at the beginning of every job"
runner\_hook\_job\_completed: "Script to be ran in the runner environment at the end of every job"
runner\_ec2\_tags: "Map of tags that will be added to the launch template instance tag specifications."
runner\_iam\_role\_managed\_policy\_arns: "Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role"
vpc\_id: "The VPC for security groups of the action runners. If not set uses the value of `var.vpc_id`."
subnet\_ids: "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. If not set, uses the value of `var.subnet_ids`."
idle\_config: "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle."
license\_specifications: "Optional EC2 License Manager license configuration ARNs for the runner launch template. Required for macOS dedicated-host runners when the host resource group uses a Mac dedicated host license configuration."
use\_dedicated\_host: "Experimental! Can be removed / changed without trigger a major release. Whether to use EC2 dedicated hosts for the runners. Needed for macos runners Note that using dedicated hosts can increase cost significantly."
runner\_log\_files: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
block\_device\_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`."
job\_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app."
pool\_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)."
iam\_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances."
}
matcherConfig: {
labelMatchers: "The list of list of labels supported by the runner configuration. `[[self-hosted, linux, x64, example]]`"
exactMatch: "DEPRECATED: Use `bidirectionalLabelMatch` instead. If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ workflow label matches it will trigger the webhook. Note: this only checks that workflow labels are a subset of runner labels, not the reverse."
bidirectionalLabelMatch: "If set to true, the runner labels and workflow job labels must be an exact two-way match (same set, any order, no extras or missing labels). This is stricter than `exactMatch` which only checks that workflow labels are a subset of runner labels. When false, if __any__ workflow label matches it will trigger the webhook."
priority: "If set it defines the priority of the matcher, the matcher with the lowest priority will be evaluated first. Default is 999, allowed values 0-999."
enableDynamicLabels: "Experimental! When true the dispatcher allows `ghr-*` dynamic labels for jobs routed to this runner. Default false."
awsDynamicLabelsPolicy: "Optional AWS dynamic label policy evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose provider dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`."
}
redrive\_build\_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries."
} | 
map(object({
    runner_config = object({
      runner_os           = string
      runner_architecture = string
      runner_metadata_options = optional(map(any), {
        instance_metadata_tags      = "enabled"
        http_endpoint               = "enabled"
        http_tokens                 = "required"
        http_put_response_hop_limit = 1
      })
      ami = optional(object({
        filter               = optional(map(list(string)), { state = ["available"] })
        owners               = optional(list(string), ["amazon"])
        id_ssm_parameter_arn = optional(string, null)
        kms_key_arn          = optional(string, null)
      }), null)
      create_service_linked_role_spot      = optional(bool, false)
      credit_specification                 = optional(string, null)
      delay_webhook_event                  = optional(number, 30)
      disable_runner_autoupdate            = optional(bool, false)
      ebs_optimized                        = optional(bool, false)
      enable_ephemeral_runners             = optional(bool, false)
      enable_job_queued_check              = optional(bool, null)
      enable_on_demand_failover_for_errors = optional(list(string), [])
      scale_errors = optional(list(string), [
        "UnfulfillableCapacity",
        "MaxSpotInstanceCountExceeded",
        "TargetCapacityLimitExceededException",
        "RequestLimitExceeded",
        "ResourceLimitExceeded",
        "MaxSpotInstanceCountExceeded",
        "MaxSpotFleetRequestCountExceeded",
        "InsufficientInstanceCapacity",
        "InsufficientCapacityOnHost",
      ])
      enable_organization_runners                                    = optional(bool, false)
      enable_runner_binaries_syncer                                  = optional(bool, true)
      enable_ssm_on_runners                                          = optional(bool, false)
      enable_userdata                                                = optional(bool, true)
      instance_allocation_strategy                                   = optional(string, "lowest-price")
      instance_max_spot_price                                        = optional(string, null)
      instance_target_capacity_type                                  = optional(string, "spot")
      instance_types                                                 = list(string)
      job_queue_retention_in_seconds                                 = optional(number, 86400)
      minimum_running_time_in_minutes                                = optional(number, null)
      pool_runner_owner                                              = optional(string, null)
      runner_as_root                                                 = optional(bool, false)
      runner_boot_time_in_minutes                                    = optional(number, 5)
      runner_disable_default_labels                                  = optional(bool, false)
      runner_extra_labels                                            = optional(list(string), [])
      runner_group_name                                              = optional(string, "Default")
      runner_name_prefix                                             = optional(string, "")
      runner_run_as                                                  = optional(string, "ec2-user")
      runners_maximum_count                                          = number
      runner_additional_security_group_ids                           = optional(list(string), [])
      scale_down_schedule_expression                                 = optional(string, "cron(*/5 * * * ? *)")
      scale_up_reserved_concurrent_executions                        = optional(number, 1)
      lambda_event_source_mapping_batch_size                         = optional(number, null)
      lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
      userdata_template                                              = optional(string, null)
      userdata_content                                               = optional(string, null)
      enable_jit_config                                              = optional(bool, null)
      enable_runner_detailed_monitoring                              = optional(bool, false)
      enable_cloudwatch_agent                                        = optional(bool, true)
      cloudwatch_config                                              = optional(string, null)
      userdata_pre_install                                           = optional(string, "")
      userdata_post_install                                          = optional(string, "")
      runner_hook_job_started                                        = optional(string, "")
      runner_hook_job_completed                                      = optional(string, "")
      runner_ec2_tags                                                = optional(map(string), {})
      runner_iam_role_managed_policy_arns                            = optional(list(string), [])
      vpc_id                                                         = optional(string, null)
      subnet_ids                                                     = optional(list(string), null)
      idle_config = optional(list(object({
        type             = optional(string, "ec2")
        cron             = string
        timeZone         = string
        idleCount        = number
        evictionStrategy = optional(string, "oldest_first")
      })), [])
      cpu_options = optional(object({
        core_count            = optional(number)
        threads_per_core      = optional(number)
        amd_sev_snp           = optional(string)
        nested_virtualization = optional(string)
      }), null)
      placement = optional(object({
        affinity                = optional(string)
        availability_zone       = optional(string)
        group_id                = optional(string)
        group_name              = optional(string)
        host_id                 = optional(string)
        host_resource_group_arn = optional(string)
        spread_domain           = optional(string)
        tenancy                 = optional(string)
        partition_number        = optional(number)
      }), null)
      license_specifications = optional(list(object({
        license_configuration_arn = string
      })), [])
      use_dedicated_host = optional(bool, false)
      runner_log_files = optional(list(object({
        log_group_name   = string
        prefix_log_group = bool
        file_path        = string
        log_stream_name  = string
        log_class        = optional(string, "STANDARD")
      })), null)
      block_device_mappings = optional(list(object({
        delete_on_termination      = optional(bool, true)
        device_name                = optional(string, "/dev/xvda")
        encrypted                  = optional(bool, true)
        iops                       = optional(number)
        kms_key_id                 = optional(string)
        snapshot_id                = optional(string)
        throughput                 = optional(number)
        volume_initialization_rate = optional(number)
        volume_size                = number
        volume_type                = optional(string, "gp3")
        })), [{
        volume_size = 30
      }])
      pool_config = optional(list(object({
        schedule_expression          = string
        schedule_expression_timezone = optional(string)
        size                         = number
      })), [])
      job_retry = optional(object({
        enable             = optional(bool, false)
        delay_in_seconds   = optional(number, 300)
        delay_backoff      = optional(number, 2)
        lambda_memory_size = optional(number, 256)
        lambda_timeout     = optional(number, 30)
        max_attempts       = optional(number, 1)
      }), {})
      iam_overrides = optional(object({
        override_instance_profile = optional(bool, null)
        instance_profile_name     = optional(string, null)
        override_runner_role      = optional(bool, null)
        runner_role_arn           = optional(string, null)
        }), {
        override_instance_profile = false
        instance_profile_name     = null
        override_runner_role      = false
        runner_role_arn           = null
      })
    })
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = optional(bool, false)
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy  = optional(any, null)
    })
    redrive_build_queue = optional(object({
      enabled         = bool
      maxReceiveCount = number
      }), {
      enabled         = false
      maxReceiveCount = null
    })
  }))
| n/a | yes | | [parameter\_store\_tags](#input\_parameter\_store\_tags) | Map of tags that will be added to all the SSM Parameter Store parameters created by the Lambda function. | `map(string)` | `{}` | no | | [pool\_lambda\_reserved\_concurrent\_executions](#input\_pool\_lambda\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no | | [pool\_lambda\_timeout](#input\_pool\_lambda\_timeout) | Time out for the pool lambda in seconds. | `number` | `60` | no | diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf index 0ff62b0083..51a7420a72 100644 --- a/modules/multi-runner/variables.tf +++ b/modules/multi-runner/variables.tf @@ -130,6 +130,7 @@ variable "multi_runner_config" { vpc_id = optional(string, null) subnet_ids = optional(list(string), null) idle_config = optional(list(object({ + type = optional(string, "ec2") cron = string timeZone = string idleCount = number @@ -208,7 +209,7 @@ variable "multi_runner_config" { bidirectionalLabelMatch = optional(bool, false) priority = optional(number, 999) enableDynamicLabels = optional(bool, false) - ec2DynamicLabelsPolicy = optional(any, null) + awsDynamicLabelsPolicy = optional(any, null) }) redrive_build_queue = optional(object({ enabled = bool @@ -233,7 +234,7 @@ variable "multi_runner_config" { enable_ephemeral_runners: "Enable ephemeral runners, runners will only be used once." enable_job_queued_check: Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners." enable_on_demand_failover_for_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later." - scale_errors: "List of aws error codes that should trigger retry during scale up. This list will replace the default errors defined in the variable `defaultScaleErrors` in https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/lambdas/functions/control-plane/src/aws/runners.ts" + scale_errors: "List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors" enable_organization_runners: "Register runners to organization, instead of repo level" enable_runner_binaries_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI." enable_ssm_on_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances." @@ -287,7 +288,7 @@ variable "multi_runner_config" { bidirectionalLabelMatch: "If set to true, the runner labels and workflow job labels must be an exact two-way match (same set, any order, no extras or missing labels). This is stricter than `exactMatch` which only checks that workflow labels are a subset of runner labels. When false, if __any__ workflow label matches it will trigger the webhook." priority: "If set it defines the priority of the matcher, the matcher with the lowest priority will be evaluated first. Default is 999, allowed values 0-999." enableDynamicLabels: "Experimental! When true the dispatcher allows `ghr-*` dynamic labels for jobs routed to this runner. Default false." - ec2DynamicLabelsPolicy: "Optional policy for `ghr-ec2-*` labels evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose EC2 dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`." + awsDynamicLabelsPolicy: "Optional AWS dynamic label policy evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose provider dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`." } redrive_build_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries." } diff --git a/modules/runners/README.md b/modules/runners/README.md index 31e4da9239..d1f42d8173 100644 --- a/modules/runners/README.md +++ b/modules/runners/README.md @@ -164,7 +164,7 @@ yarn run dist | [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. DO NOT SET IF USING PUBLIC GITHUB..However if you are using GitHub Enterprise Cloud with data-residency (ghe.com), set the endpoint here. Example - https://companyname.ghe.com\| | `string` | `null` | no | | [github\_app\_parameters](#input\_github\_app\_parameters) | Parameter Store for GitHub App Parameters. | 
object({
    key_base64 = map(string)
    id         = map(string)
  })
| n/a | yes | | [iam\_overrides](#input\_iam\_overrides) | This map provides the possibility to override some IAM defaults. The following attributes are supported: `instance_profile_name` overrides the instance profile name used in the launch template. `runner_role_arn` overrides the IAM role ARN used for the runner instances. | 
object({
    override_instance_profile = optional(bool, null)
    instance_profile_name     = optional(string, null)
    override_runner_role      = optional(bool, null)
    runner_role_arn           = optional(string, null)
  })
| 
{
  "instance_profile_name": null,
  "override_instance_profile": false,
  "override_runner_role": false,
  "runner_role_arn": null
}
| no | -| [idle\_config](#input\_idle\_config) | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    cron             = string
    timeZone         = string
    idleCount        = number
    evictionStrategy = optional(string, "oldest_first")
  }))
| `[]` | no | +| [idle\_config](#input\_idle\_config) | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    type             = optional(string, "ec2")
    cron             = string
    timeZone         = string
    idleCount        = number
    evictionStrategy = optional(string, "oldest_first")
  }))
| `[]` | no | | [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | | [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the prefix will be used. | `string` | `null` | no | @@ -226,7 +226,7 @@ yarn run dist | [runners\_maximum\_count](#input\_runners\_maximum\_count) | The maximum number of runners that will be created. Setting the variable to `-1` desiables the maximum check. | `number` | `3` | no | | [s3\_runner\_binaries](#input\_s3\_runner\_binaries) | Bucket details for cached GitHub binary. | 
object({
    arn = string
    id  = string
    key = string
  })
| n/a | yes | | [scale\_down\_schedule\_expression](#input\_scale\_down\_schedule\_expression) | Scheduler expression to check every x for scale down. | `string` | `"cron(*/5 * * * ? *)"` | no | -| [scale\_errors](#input\_scale\_errors) | List of aws error codes that should trigger retry during scale up. This list will replace the default errors defined in the variable `defaultScaleErrors` in https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/lambdas/functions/control-plane/src/aws/runners.ts | `list(string)` | 
[
  "UnfulfillableCapacity",
  "MaxSpotInstanceCountExceeded",
  "TargetCapacityLimitExceededException",
  "RequestLimitExceeded",
  "ResourceLimitExceeded",
  "MaxSpotInstanceCountExceeded",
  "MaxSpotFleetRequestCountExceeded",
  "InsufficientInstanceCapacity",
  "InsufficientCapacityOnHost"
]
| no | +| [scale\_errors](#input\_scale\_errors) | List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors | `list(string)` | 
[
  "UnfulfillableCapacity",
  "MaxSpotInstanceCountExceeded",
  "TargetCapacityLimitExceededException",
  "RequestLimitExceeded",
  "ResourceLimitExceeded",
  "MaxSpotInstanceCountExceeded",
  "MaxSpotFleetRequestCountExceeded",
  "InsufficientInstanceCapacity",
  "InsufficientCapacityOnHost"
]
| no | | [scale\_up\_reserved\_concurrent\_executions](#input\_scale\_up\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no | | [sqs\_build\_queue](#input\_sqs\_build\_queue) | SQS queue to consume accepted build events. | 
object({
    arn = string
    url = string
  })
| n/a | yes | | [ssm\_housekeeper](#input\_ssm\_housekeeper) | Configuration for the SSM housekeeper lambda. This lambda deletes token / JIT config from SSM.

`schedule_expression`: is used to configure the schedule for the lambda.
`state`: state of the cloudwatch event rule. Valid values are `DISABLED`, `ENABLED`, and `ENABLED_WITH_ALL_CLOUDTRAIL_MANAGEMENT_EVENTS`.
`lambda_memory_size`: lambda memory size limit.
`lambda_timeout`: timeout for the lambda in seconds.
`config`: configuration for the lambda function. Token path will be read by default from the module. | 
object({
    schedule_expression = optional(string, "rate(1 day)")
    state               = optional(string, "ENABLED")
    lambda_memory_size  = optional(number, 512)
    lambda_timeout      = optional(number, 60)
    config = object({
      tokenPath      = optional(string)
      minimumDaysOld = optional(number, 1)
      dryRun         = optional(bool, false)
    })
  })
| 
{
  "config": {}
}
| no | diff --git a/modules/runners/pool/main.tf b/modules/runners/pool/main.tf index 1871437c59..29ec0023b4 100644 --- a/modules/runners/pool/main.tf +++ b/modules/runners/pool/main.tf @@ -231,6 +231,7 @@ resource "aws_scheduler_schedule" "pool" { role_arn = aws_iam_role.scheduler.arn input = jsonencode({ poolSize = each.value.size + type = "ec2" }) } } diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf index cbc9beaa64..b8abf01d14 100644 --- a/modules/runners/scale-up.tf +++ b/modules/runners/scale-up.tf @@ -54,6 +54,7 @@ resource "aws_lambda_function" "scale_up" { RUNNER_LABELS = lower(join(",", var.runner_labels)) RUNNER_GROUP_NAME = var.runner_group_name RUNNER_NAME_PREFIX = var.runner_name_prefix + RUNNER_PROVIDER_TYPE = "ec2" RUNNERS_MAXIMUM_COUNT = var.runners_maximum_count POWERTOOLS_SERVICE_NAME = "${var.prefix}-scale-up" SSM_TOKEN_PATH = local.token_path diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index d4f265c062..c534081c95 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -349,6 +349,7 @@ variable "runner_architecture" { variable "idle_config" { description = "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle." type = list(object({ + type = optional(string, "ec2") cron = string timeZone = string idleCount = number @@ -772,7 +773,7 @@ variable "enable_on_demand_failover_for_errors" { } variable "scale_errors" { - description = "List of aws error codes that should trigger retry during scale up. This list will replace the default errors defined in the variable `defaultScaleErrors` in https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/lambdas/functions/control-plane/src/aws/runners.ts" + description = "List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors" type = list(string) default = [ "UnfulfillableCapacity", diff --git a/modules/webhook/README.md b/modules/webhook/README.md index 27834cb207..1c68c2007f 100644 --- a/modules/webhook/README.md +++ b/modules/webhook/README.md @@ -89,7 +89,7 @@ yarn run dist | [repository\_white\_list](#input\_repository\_white\_list) | List of github repository full names (owner/repo\_name) that will be allowed to use the github app. Leave empty for no filtering. | `list(string)` | `[]` | no | | [role\_path](#input\_role\_path) | The path that will be added to the role; if not set, the environment name will be used. | `string` | `null` | no | | [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no | -| [runner\_matcher\_config](#input\_runner\_matcher\_config) | SQS queue to publish accepted build events based on the runner type. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.ec2DynamicLabelsPolicy` are evaluated by the dispatcher to gate `ghr-ec2-*` labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the `ghr-ec2-*` suffix form, for example `instance-type`. | 
map(object({
    arn = string
    id  = string
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = bool
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      ec2DynamicLabelsPolicy  = optional(any, null)
    })
  }))
| n/a | yes | +| [runner\_matcher\_config](#input\_runner\_matcher\_config) | SQS queue to publish accepted build events based on the runner type. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type`. | 
map(object({
    arn = string
    id  = string
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = bool
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy  = optional(any, null)
    })
  }))
| n/a | yes | | [ssm\_paths](#input\_ssm\_paths) | The root path used in SSM to store configuration and secrets. | 
object({
    root    = string
    webhook = string
  })
| n/a | yes | | [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | | [tracing\_config](#input\_tracing\_config) | Configuration for lambda tracing. | 
object({
    mode                  = optional(string, null)
    capture_http_requests = optional(bool, false)
    capture_error         = optional(bool, false)
  })
| `{}` | no | diff --git a/modules/webhook/variables.tf b/modules/webhook/variables.tf index 6f4f17b429..427b713c69 100644 --- a/modules/webhook/variables.tf +++ b/modules/webhook/variables.tf @@ -23,7 +23,7 @@ variable "tags" { } variable "runner_matcher_config" { - description = "SQS queue to publish accepted build events based on the runner type. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.ec2DynamicLabelsPolicy` are evaluated by the dispatcher to gate `ghr-ec2-*` labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the `ghr-ec2-*` suffix form, for example `instance-type`." + description = "SQS queue to publish accepted build events based on the runner type. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type`." type = map(object({ arn = string id = string @@ -33,7 +33,7 @@ variable "runner_matcher_config" { bidirectionalLabelMatch = optional(bool, false) priority = optional(number, 999) enableDynamicLabels = optional(bool, false) - ec2DynamicLabelsPolicy = optional(any, null) + awsDynamicLabelsPolicy = optional(any, null) }) })) validation { diff --git a/variables.tf b/variables.tf index 71e4d7df06..dd9325a5c6 100644 --- a/variables.tf +++ b/variables.tf @@ -311,7 +311,7 @@ variable "enable_runner_on_demand_failover_for_errors" { } variable "scale_errors" { - description = "List of aws error codes that should trigger retry during scale up. This list will replace the default errors defined in the variable `defaultScaleErrors` in https://github.com/github-aws-runners/terraform-aws-github-runner/blob/main/lambdas/functions/control-plane/src/aws/runners.ts" + description = "List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors" type = list(string) default = [ "UnfulfillableCapacity", @@ -371,6 +371,7 @@ variable "runner_hook_job_completed" { variable "idle_config" { description = "List of time periods, defined as a cron expression, to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle." type = list(object({ + type = optional(string, "ec2") cron = string timeZone = string idleCount = number @@ -729,14 +730,15 @@ variable "enable_dynamic_labels" { default = false } -variable "ec2_dynamic_labels_policy" { +variable "aws_dynamic_labels_policy" { description = <<-EOT Experimental! Can be removed / changed without trigger a major release. - Optional policy for dynamic EC2 override labels evaluated by the webhook - dispatcher. Only effective when `enable_dynamic_labels = true`. + Optional AWS dynamic label policy evaluated by the webhook dispatcher. + Only effective when `enable_dynamic_labels = true`. - Jobs whose EC2 dynamic labels violate the policy are rejected with a 202 and a - warning is logged. + Jobs whose provider-specific dynamic labels violate the policy are rejected + with a 202 and a warning is logged. Currently this policy applies to EC2 + override labels using the `ghr-ec2-*` prefix. Evaluation: 1. Keys in `blocked_keys` are always rejected. @@ -748,8 +750,8 @@ variable "ec2_dynamic_labels_policy" { - `restricted_keys`: map of key to value rule: `{ allowed = [globs], denied = [globs], max = number|string }`. - Keys use the `ghr-ec2-*` dynamic label suffix, not the full label. For example, use - `instance-type` for `ghr-ec2-instance-type`. + Keys use the provider dynamic label suffix, not the full label. For example, + use `instance-type` for `ghr-ec2-instance-type`. EOT type = any default = null From 9bf1f614ca35422fa5eeebe6230669a4d5e2a8f1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 20:16:14 +0000 Subject: [PATCH 07/46] docs: auto update terraform docs --- README.md | 2 +- modules/termination-watcher/notification/README.md | 4 ++++ modules/termination-watcher/termination/README.md | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9420ccd26f..cfe5e26768 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh) | [ami\_housekeeper\_lambda\_timeout](#input\_ami\_housekeeper\_lambda\_timeout) | Time out of the lambda in seconds. | `number` | `300` | no | | [ami\_housekeeper\_lambda\_zip](#input\_ami\_housekeeper\_lambda\_zip) | File location of the lambda zip file. | `string` | `null` | no | | [associate\_public\_ipv4\_address](#input\_associate\_public\_ipv4\_address) | Associate public IPv4 with the runner. Only tested with IPv4 | `bool` | `false` | no | +| [aws\_dynamic\_labels\_policy](#input\_aws\_dynamic\_labels\_policy) | Experimental! Can be removed / changed without trigger a major release.
Optional AWS dynamic label policy evaluated by the webhook dispatcher.
Only effective when `enable_dynamic_labels = true`.

Jobs whose provider-specific dynamic labels violate the policy are rejected
with a 202 and a warning is logged. Currently this policy applies to EC2
override labels using the `ghr-ec2-*` prefix.

Evaluation:
1. Keys in `blocked_keys` are always rejected.
2. Keys in `restricted_keys` are allowed only when their value passes the rule.
3. Keys not listed in `blocked_keys` or `restricted_keys` are allowed.

Schema:
- `blocked_keys`: keys to reject outright.
- `restricted_keys`: map of key to value rule:
`{ allowed = [globs], denied = [globs], max = number|string }`.

Keys use the provider dynamic label suffix, not the full label. For example,
use `instance-type` for `ghr-ec2-instance-type`. | `any` | `null` | no | | [aws\_partition](#input\_aws\_partition) | (optiona) partition in the arn namespace to use if not 'aws' | `string` | `"aws"` | no | | [aws\_region](#input\_aws\_region) | AWS region. | `string` | n/a | yes | | [block\_device\_mappings](#input\_block\_device\_mappings) | The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`. | 
list(object({
    delete_on_termination      = optional(bool, true)
    device_name                = optional(string, "/dev/xvda")
    encrypted                  = optional(bool, true)
    iops                       = optional(number)
    kms_key_id                 = optional(string)
    snapshot_id                = optional(string)
    throughput                 = optional(number)
    volume_initialization_rate = optional(number)
    volume_size                = number
    volume_type                = optional(string, "gp3")
  }))
| 
[
  {
    "volume_size": 30
  }
]
| no | @@ -119,7 +120,6 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh) | [create\_service\_linked\_role\_spot](#input\_create\_service\_linked\_role\_spot) | (optional) create the service linked role for spot instances that is required by the scale-up lambda. | `bool` | `false` | no | | [delay\_webhook\_event](#input\_delay\_webhook\_event) | The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event. | `number` | `30` | no | | [disable\_runner\_autoupdate](#input\_disable\_runner\_autoupdate) | Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/) | `bool` | `false` | no | -| [aws\_dynamic\_labels\_policy](#input\_aws\_dynamic\_labels\_policy) | Experimental! Can be removed / changed without trigger a major release.
Optional AWS dynamic label policy evaluated by the webhook dispatcher.
Only effective when `enable_dynamic_labels = true`.

Jobs whose provider-specific dynamic labels violate the policy are rejected
with a 202 and a warning is logged. Currently this policy applies to EC2
override labels using the `ghr-ec2-*` prefix.

Evaluation:
1. Keys in `blocked_keys` are always rejected.
2. Keys in `restricted_keys` are allowed only when their value passes the rule.
3. Keys not listed in `blocked_keys` or `restricted_keys` are allowed.

Schema:
- `blocked_keys`: keys to reject outright.
- `restricted_keys`: map of key to value rule:
`{ allowed = [globs], denied = [globs], max = number|string }`.

Keys use the provider dynamic label suffix, not the full label. For example,
use `instance-type` for `ghr-ec2-instance-type`. | `any` | `null` | no | | [enable\_ami\_housekeeper](#input\_enable\_ami\_housekeeper) | Option to disable the lambda to clean up old AMIs. | `bool` | `false` | no | | [enable\_cloudwatch\_agent](#input\_enable\_cloudwatch\_agent) | Enables the cloudwatch agent on the ec2 runner instances. The runner uses a default config that can be overridden via `cloudwatch_config`. | `bool` | `true` | no | | [enable\_dynamic\_labels](#input\_enable\_dynamic\_labels) | Experimental! Can be removed / changed without trigger a major release. Enable dynamic EC2 configs based on workflow job labels. When enabled, jobs can request specific configs via the 'ghr-ec2-:' label (e.g., 'ghr-ec2-instance-type:t3.large'). When enabled, labels starting with `ghr-` are ignored during webhook label matching. | `bool` | `false` | no | diff --git a/modules/termination-watcher/notification/README.md b/modules/termination-watcher/notification/README.md index 1a26de3bce..31df74586f 100644 --- a/modules/termination-watcher/notification/README.md +++ b/modules/termination-watcher/notification/README.md @@ -22,10 +22,14 @@ | Name | Type | |------|------| +| [aws_cloudwatch_event_rule.ec2_instance_state_change](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | | [aws_cloudwatch_event_rule.spot_instance_termination_warning](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | | [aws_cloudwatch_event_target.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | +| [aws_cloudwatch_event_target.state_change](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | | [aws_iam_role_policy.lambda_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.ssm_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_lambda_permission.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | +| [aws_lambda_permission.state_change](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | ## Inputs diff --git a/modules/termination-watcher/termination/README.md b/modules/termination-watcher/termination/README.md index 22912b9646..32b32aa54e 100644 --- a/modules/termination-watcher/termination/README.md +++ b/modules/termination-watcher/termination/README.md @@ -25,6 +25,7 @@ | [aws_cloudwatch_event_rule.spot_instance_termination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | | [aws_cloudwatch_event_target.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | | [aws_iam_role_policy.lambda_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.ssm_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | | [aws_lambda_permission.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource | ## Inputs From 83b917af3981163fba5787efd089e5bf63fc012f Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 7 Jul 2026 23:36:54 +0200 Subject: [PATCH 08/46] refactor(tests): keep provider tests in existing files --- .../{ec2-runners.test.ts => runners.test.ts} | 0 .../control-plane/src/pool/ec2-pool.test.ts | 54 - .../control-plane/src/pool/pool.test.ts | 44 +- .../src/scale-runners/ec2-labels.test.ts | 708 ---- .../src/scale-runners/ec2-scale-down.test.ts | 890 ---- .../src/scale-runners/ec2-scale-up.test.ts | 2769 ------------- .../src/scale-runners/scale-down.test.ts | 998 ++++- .../src/scale-runners/scale-up-config.test.ts | 13 - .../src/scale-runners/scale-up.test.ts | 3570 ++++++++++++++++- .../src/runners/aws-dynamic-labels.test.ts | 28 - .../webhook/src/runners/dispatch.test.ts | 145 + ....test.ts => dynamic-labels-policy.test.ts} | 0 .../webhook/src/runners/labels.test.ts | 121 - 13 files changed, 4466 insertions(+), 4874 deletions(-) rename lambdas/functions/control-plane/src/aws/{ec2-runners.test.ts => runners.test.ts} (100%) delete mode 100644 lambdas/functions/control-plane/src/pool/ec2-pool.test.ts delete mode 100644 lambdas/functions/control-plane/src/scale-runners/ec2-labels.test.ts delete mode 100644 lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.test.ts delete mode 100644 lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.test.ts delete mode 100644 lambdas/functions/control-plane/src/scale-runners/scale-up-config.test.ts delete mode 100644 lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts rename lambdas/functions/webhook/src/runners/{ec2-dynamic-labels-policy.test.ts => dynamic-labels-policy.test.ts} (100%) delete mode 100644 lambdas/functions/webhook/src/runners/labels.test.ts diff --git a/lambdas/functions/control-plane/src/aws/ec2-runners.test.ts b/lambdas/functions/control-plane/src/aws/runners.test.ts similarity index 100% rename from lambdas/functions/control-plane/src/aws/ec2-runners.test.ts rename to lambdas/functions/control-plane/src/aws/runners.test.ts diff --git a/lambdas/functions/control-plane/src/pool/ec2-pool.test.ts b/lambdas/functions/control-plane/src/pool/ec2-pool.test.ts deleted file mode 100644 index 64797408ea..0000000000 --- a/lambdas/functions/control-plane/src/pool/ec2-pool.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { bootTimeExceeded } from '../aws/ec2-runners'; -import type { RunnerList } from '../aws/ec2-runners.d'; -import { calculateEc2PoolSize } from './ec2-pool'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; - -vi.mock('../aws/ec2-runners', async () => ({ - bootTimeExceeded: vi.fn(), - listEC2Runners: vi.fn(), -})); - -vi.mock('../scale-runners/ec2', async () => ({ - createRunners: vi.fn(), -})); - -const mockBootTimeExceeded = vi.mocked(bootTimeExceeded); - -describe('calculateEc2PoolSize', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('counts registered online idle runners', () => { - const runners: RunnerList[] = [{ instanceId: 'i-idle' }]; - const runnerStatus = new Map([['i-idle', { busy: false, status: 'online' }]]); - - expect(calculateEc2PoolSize(runners, runnerStatus)).toBe(1); - expect(mockBootTimeExceeded).not.toHaveBeenCalled(); - }); - - it('does not count registered busy or offline runners', () => { - const runners: RunnerList[] = [{ instanceId: 'i-busy' }, { instanceId: 'i-offline' }]; - const runnerStatus = new Map([ - ['i-busy', { busy: true, status: 'online' }], - ['i-offline', { busy: false, status: 'offline' }], - ]); - - expect(calculateEc2PoolSize(runners, runnerStatus)).toBe(0); - expect(mockBootTimeExceeded).not.toHaveBeenCalled(); - }); - - it('counts unregistered runners that are still booting', () => { - const runners: RunnerList[] = [{ instanceId: 'i-booting' }]; - mockBootTimeExceeded.mockReturnValue(false); - - expect(calculateEc2PoolSize(runners, new Map())).toBe(1); - }); - - it('does not count unregistered runners whose boot time expired', () => { - const runners: RunnerList[] = [{ instanceId: 'i-expired' }]; - mockBootTimeExceeded.mockReturnValue(true); - - expect(calculateEc2PoolSize(runners, new Map())).toBe(0); - }); -}); diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index 63cecbc346..795bec21ea 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -2,11 +2,13 @@ import { Octokit } from '@octokit/rest'; import moment from 'moment-timezone'; import * as nock from 'nock'; -import { listEC2Runners } from '../aws/ec2-runners'; +import { bootTimeExceeded, listEC2Runners } from '../aws/ec2-runners'; +import type { RunnerList } from '../aws/ec2-runners.d'; import * as ghAuth from '../github/auth'; import { createRunners } from '../scale-runners/ec2'; import { getGitHubEnterpriseApiUrl } from '../scale-runners/github-runner'; import { adjust } from './pool'; +import { calculateEc2PoolSize } from './ec2-pool'; import { describe, it, expect, beforeEach, vi, MockedClass } from 'vitest'; const mockOctokit = { @@ -54,6 +56,7 @@ const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); const mockedInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); const mockCreateClient = vi.mocked(ghAuth.createOctokitClient); const mockListRunners = vi.mocked(listEC2Runners); +const mockBootTimeExceeded = vi.mocked(bootTimeExceeded); const cleanEnv = process.env; @@ -463,3 +466,42 @@ describe('Test simple pool.', () => { }); }); }); + +describe('calculateEc2PoolSize', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('counts registered online idle runners', () => { + const runners: RunnerList[] = [{ instanceId: 'i-idle' }]; + const runnerStatus = new Map([['i-idle', { busy: false, status: 'online' }]]); + + expect(calculateEc2PoolSize(runners, runnerStatus)).toBe(1); + expect(mockBootTimeExceeded).not.toHaveBeenCalled(); + }); + + it('does not count registered busy or offline runners', () => { + const runners: RunnerList[] = [{ instanceId: 'i-busy' }, { instanceId: 'i-offline' }]; + const runnerStatus = new Map([ + ['i-busy', { busy: true, status: 'online' }], + ['i-offline', { busy: false, status: 'offline' }], + ]); + + expect(calculateEc2PoolSize(runners, runnerStatus)).toBe(0); + expect(mockBootTimeExceeded).not.toHaveBeenCalled(); + }); + + it('counts unregistered runners that are still booting', () => { + const runners: RunnerList[] = [{ instanceId: 'i-booting' }]; + mockBootTimeExceeded.mockReturnValue(false); + + expect(calculateEc2PoolSize(runners, new Map())).toBe(1); + }); + + it('does not count unregistered runners whose boot time expired', () => { + const runners: RunnerList[] = [{ instanceId: 'i-expired' }]; + mockBootTimeExceeded.mockReturnValue(true); + + expect(calculateEc2PoolSize(runners, new Map())).toBe(0); + }); +}); diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-labels.test.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-labels.test.ts deleted file mode 100644 index c2203d2c67..0000000000 --- a/lambdas/functions/control-plane/src/scale-runners/ec2-labels.test.ts +++ /dev/null @@ -1,708 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { parseEc2OverrideConfig } from './ec2-labels'; - -describe('parseEc2OverrideConfig', () => { - describe('Basic Fleet Overrides', () => { - it('should parse instance-type label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-instance-type:c5.xlarge']); - expect(result?.InstanceType).toBe('c5.xlarge'); - }); - - it('should parse subnet-id label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-subnet-id:subnet-123456']); - expect(result?.SubnetId).toBe('subnet-123456'); - }); - - it('should parse availability-zone label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-availability-zone:us-east-1a']); - expect(result?.AvailabilityZone).toBe('us-east-1a'); - }); - - it('should parse availability-zone-id label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-availability-zone-id:use1-az1']); - expect(result?.AvailabilityZoneId).toBe('use1-az1'); - }); - - it('should parse max-price label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-max-price:0.50']); - expect(result?.MaxPrice).toBe('0.50'); - }); - - it('should parse priority label as number', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-priority:1']); - expect(result?.Priority).toBe(1); - }); - - it('should parse weighted-capacity label as number', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-weighted-capacity:2']); - expect(result?.WeightedCapacity).toBe(2); - }); - - it('should parse image-id label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-image-id:ami-12345678']); - expect(result?.ImageId).toBe('ami-12345678'); - }); - - it('should parse multiple basic fleet overrides', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-instance-type:r5.2xlarge', - 'ghr-ec2-max-price:1.00', - 'ghr-ec2-priority:2', - ]); - expect(result?.InstanceType).toBe('r5.2xlarge'); - expect(result?.MaxPrice).toBe('1.00'); - expect(result?.Priority).toBe(2); - }); - }); - - describe('Placement', () => { - it('should parse placement-group-name label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-placement-group-name:my-placement-group']); - expect(result?.Placement?.GroupName).toBe('my-placement-group'); - }); - - it('should parse placement-group-id label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-placement-group-id:pg-1234567890abcdef0']); - expect(result?.Placement?.GroupId).toBe('pg-1234567890abcdef0'); - }); - - it('should parse placement-tenancy label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-placement-tenancy:dedicated']); - expect(result?.Placement?.Tenancy).toBe('dedicated'); - }); - - it('should parse placement-host-id label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-placement-host-id:h-1234567890abcdef']); - expect(result?.Placement?.HostId).toBe('h-1234567890abcdef'); - }); - - it('should parse placement-affinity label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-placement-affinity:host']); - expect(result?.Placement?.Affinity).toBe('host'); - }); - - it('should parse placement-partition-number label as number', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-placement-partition-number:3']); - expect(result?.Placement?.PartitionNumber).toBe(3); - }); - - it('should parse placement-availability-zone label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-placement-availability-zone:us-west-2b']); - expect(result?.Placement?.AvailabilityZone).toBe('us-west-2b'); - }); - - it('should parse placement-availability-zone-id label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-placement-availability-zone-id:use1-az1']); - expect(result?.Placement?.AvailabilityZoneId).toBe('use1-az1'); - }); - - it('should parse placement-spread-domain label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-placement-spread-domain:my-spread-domain']); - expect(result?.Placement?.SpreadDomain).toBe('my-spread-domain'); - }); - - it('should parse placement-host-resource-group-arn label', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-placement-host-resource-group-arn:arn:aws:ec2:us-east-1:123456789012:host-resource-group/hrg-1234', - ]); - expect(result?.Placement?.HostResourceGroupArn).toBe( - 'arn:aws:ec2:us-east-1:123456789012:host-resource-group/hrg-1234', - ); - }); - - it('should parse multiple placement labels', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-placement-group-name:group-1', - 'ghr-ec2-placement-tenancy:dedicated', - 'ghr-ec2-placement-availability-zone:us-east-1b', - ]); - expect(result?.Placement?.GroupName).toBe('group-1'); - expect(result?.Placement?.Tenancy).toBe('dedicated'); - expect(result?.Placement?.AvailabilityZone).toBe('us-east-1b'); - }); - }); - - describe('Block Device Mappings', () => { - it('should parse block-device-name label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-block-device-name:/dev/sdg']); - expect(result?.BlockDeviceMappings?.[0]?.DeviceName).toBe('/dev/sdg'); - }); - - it('should use default block device name when provided', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-ebs-volume-size:100'], '/dev/sda1'); - expect(result?.BlockDeviceMappings?.[0]?.DeviceName).toBe('/dev/sda1'); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(100); - }); - - it('should parse ebs-volume-size label as number', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-ebs-volume-size:100']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(100); - }); - - it('should parse ebs-volume-type label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-ebs-volume-type:gp3']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeType).toBe('gp3'); - }); - - it('should parse ebs-iops label as number', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-ebs-iops:3000']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Iops).toBe(3000); - }); - - it('should parse ebs-throughput label as number', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-ebs-throughput:250']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Throughput).toBe(250); - }); - - it('should parse ebs-encrypted label as boolean true', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-ebs-encrypted:true']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); - }); - - it('should parse ebs-encrypted label as boolean false', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-ebs-encrypted:false']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(false); - }); - - it('should parse ebs-kms-key-id label', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-ebs-kms-key-id:arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012', - ]); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.KmsKeyId).toBe( - 'arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012', - ); - }); - - it('should parse ebs-delete-on-termination label as boolean true', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-ebs-delete-on-termination:true']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.DeleteOnTermination).toBe(true); - }); - - it('should parse ebs-delete-on-termination label as boolean false', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-ebs-delete-on-termination:false']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.DeleteOnTermination).toBe(false); - }); - - it('should parse ebs-snapshot-id label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-ebs-snapshot-id:snap-1234567890abcdef']); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.SnapshotId).toBe('snap-1234567890abcdef'); - }); - - it('should parse block-device-virtual-name label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-block-device-virtual-name:ephemeral0']); - expect(result?.BlockDeviceMappings?.[0]?.VirtualName).toBe('ephemeral0'); - }); - - it('should parse block-device-no-device label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-block-device-no-device:true']); - expect(result?.BlockDeviceMappings?.[0]?.NoDevice).toBe('true'); - }); - - it('should parse multiple block device mapping labels', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-ebs-volume-size:200', - 'ghr-ec2-ebs-volume-type:gp3', - 'ghr-ec2-ebs-iops:5000', - 'ghr-ec2-ebs-encrypted:true', - ]); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(200); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeType).toBe('gp3'); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Iops).toBe(5000); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); - }); - - it('should initialize BlockDeviceMappings when not present', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-ebs-volume-size:50']); - expect(result?.BlockDeviceMappings).toBeDefined(); - }); - }); - - describe('Instance Requirements - vCPU and Memory', () => { - it('should parse vcpu-count-min label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-vcpu-count-min:4']); - expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(4); - }); - - it('should parse vcpu-count-max label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-vcpu-count-max:16']); - expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(16); - }); - - it('should parse both vcpu-count-min and vcpu-count-max labels', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-vcpu-count-min:2', 'ghr-ec2-vcpu-count-max:8']); - expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(2); - expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(8); - }); - - it('should parse memory-mib-min label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-memory-mib-min:8192']); - expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(8192); - }); - - it('should parse memory-mib-max label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-memory-mib-max:32768']); - expect(result?.InstanceRequirements?.MemoryMiB?.Max).toBe(32768); - }); - - it('should parse both memory-mib-min and memory-mib-max labels', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-memory-mib-min:16384', 'ghr-ec2-memory-mib-max:65536']); - expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(16384); - expect(result?.InstanceRequirements?.MemoryMiB?.Max).toBe(65536); - }); - - it('should parse memory-gib-per-vcpu-min label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-memory-gib-per-vcpu-min:2']); - expect(result?.InstanceRequirements?.MemoryGiBPerVCpu?.Min).toBe(2); - }); - - it('should parse memory-gib-per-vcpu-max label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-memory-gib-per-vcpu-max:8']); - expect(result?.InstanceRequirements?.MemoryGiBPerVCpu?.Max).toBe(8); - }); - - it('should parse combined vCPU and memory requirements', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-vcpu-count-min:8', - 'ghr-ec2-vcpu-count-max:32', - 'ghr-ec2-memory-mib-min:32768', - 'ghr-ec2-memory-mib-max:131072', - ]); - expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(8); - expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(32); - expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(32768); - expect(result?.InstanceRequirements?.MemoryMiB?.Max).toBe(131072); - }); - }); - - describe('Instance Requirements - CPU and Performance', () => { - it('should parse cpu-manufacturers as single value', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-cpu-manufacturers:intel']); - expect(result?.InstanceRequirements?.CpuManufacturers).toEqual(['intel']); - }); - - it('should parse cpu-manufacturers as semicolon-separated list', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-cpu-manufacturers:intel;amd']); - expect(result?.InstanceRequirements?.CpuManufacturers).toEqual(['intel', 'amd']); - }); - - it('should parse instance-generations as single value', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-instance-generations:current']); - expect(result?.InstanceRequirements?.InstanceGenerations).toEqual(['current']); - }); - - it('should parse instance-generations as semicolon-separated list', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-instance-generations:current;previous']); - expect(result?.InstanceRequirements?.InstanceGenerations).toEqual(['current', 'previous']); - }); - - it('should parse excluded-instance-types as single value', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-excluded-instance-types:t2.micro']); - expect(result?.InstanceRequirements?.ExcludedInstanceTypes).toEqual(['t2.micro']); - }); - - it('should parse excluded-instance-types as semicolon-separated list', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-excluded-instance-types:t2.micro;t2.small']); - expect(result?.InstanceRequirements?.ExcludedInstanceTypes).toEqual(['t2.micro', 't2.small']); - }); - - it('should parse allowed-instance-types as single value', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-allowed-instance-types:c5.xlarge']); - expect(result?.InstanceRequirements?.AllowedInstanceTypes).toEqual(['c5.xlarge']); - }); - - it('should parse allowed-instance-types as semicolon-separated list', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-allowed-instance-types:c5.xlarge;c5.2xlarge']); - expect(result?.InstanceRequirements?.AllowedInstanceTypes).toEqual(['c5.xlarge', 'c5.2xlarge']); - }); - - it('should parse burstable-performance label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-burstable-performance:included']); - expect(result?.InstanceRequirements?.BurstablePerformance).toBe('included'); - }); - - it('should parse bare-metal label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-bare-metal:excluded']); - expect(result?.InstanceRequirements?.BareMetal).toBe('excluded'); - }); - }); - - describe('Instance Requirements - Accelerators', () => { - it('should parse accelerator-count-min label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-count-min:1']); - expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); - }); - - it('should parse accelerator-count-max label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-count-max:4']); - expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(4); - }); - - it('should parse both accelerator-count-min and accelerator-count-max', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-count-min:1', 'ghr-ec2-accelerator-count-max:2']); - expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); - expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(2); - }); - - it('should parse accelerator-types as single value', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-types:gpu']); - expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu']); - }); - - it('should parse accelerator-types as semicolon-separated list', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-types:gpu;fpga']); - expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu', 'fpga']); - }); - - it('should parse accelerator-manufacturers as single value', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-manufacturers:nvidia']); - expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia']); - }); - - it('should parse accelerator-manufacturers as semicolon-separated list', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-manufacturers:nvidia;amd']); - expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia', 'amd']); - }); - - it('should parse accelerator-names as single value', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-names:a100']); - expect(result?.InstanceRequirements?.AcceleratorNames).toEqual(['a100']); - }); - - it('should parse accelerator-names as semicolon-separated list', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-names:a100;v100']); - expect(result?.InstanceRequirements?.AcceleratorNames).toEqual(['a100', 'v100']); - }); - - it('should parse accelerator-total-memory-mib-min label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-total-memory-mib-min:8192']); - expect(result?.InstanceRequirements?.AcceleratorTotalMemoryMiB?.Min).toBe(8192); - }); - - it('should parse accelerator-total-memory-mib-max label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-total-memory-mib-max:40960']); - expect(result?.InstanceRequirements?.AcceleratorTotalMemoryMiB?.Max).toBe(40960); - }); - - it('should parse combined accelerator requirements', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-accelerator-count-min:1', - 'ghr-ec2-accelerator-count-max:2', - 'ghr-ec2-accelerator-types:gpu', - 'ghr-ec2-accelerator-manufacturers:nvidia', - ]); - expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); - expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(2); - expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu']); - expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia']); - }); - }); - - describe('Instance Requirements - Network and Storage', () => { - it('should parse network-interface-count-min label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-network-interface-count-min:2']); - expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Min).toBe(2); - }); - - it('should parse network-interface-count-max label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-network-interface-count-max:4']); - expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Max).toBe(4); - }); - - it('should parse network-bandwidth-gbps-min label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-network-bandwidth-gbps-min:5']); - expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Min).toBe(5); - }); - - it('should parse network-bandwidth-gbps-max label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-network-bandwidth-gbps-max:25']); - expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Max).toBe(25); - }); - - it('should parse local-storage label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-local-storage:included']); - expect(result?.InstanceRequirements?.LocalStorage).toBe('included'); - }); - - it('should parse local-storage-types as single value', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-local-storage-types:ssd']); - expect(result?.InstanceRequirements?.LocalStorageTypes).toEqual(['ssd']); - }); - - it('should parse local-storage-types as semicolon-separated list', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-local-storage-types:hdd;ssd']); - expect(result?.InstanceRequirements?.LocalStorageTypes).toEqual(['hdd', 'ssd']); - }); - - it('should parse total-local-storage-gb-min label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-total-local-storage-gb-min:100']); - expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Min).toBe(100); - }); - - it('should parse total-local-storage-gb-max label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-total-local-storage-gb-max:1000']); - expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Max).toBe(1000); - }); - - it('should parse baseline-ebs-bandwidth-mbps-min label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-baseline-ebs-bandwidth-mbps-min:500']); - expect(result?.InstanceRequirements?.BaselineEbsBandwidthMbps?.Min).toBe(500); - }); - - it('should parse baseline-ebs-bandwidth-mbps-max label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-baseline-ebs-bandwidth-mbps-max:2000']); - expect(result?.InstanceRequirements?.BaselineEbsBandwidthMbps?.Max).toBe(2000); - }); - }); - - describe('Instance Requirements - Pricing and Other', () => { - it('should parse spot-max-price-percentage-over-lowest-price label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-spot-max-price-percentage-over-lowest-price:50']); - expect(result?.InstanceRequirements?.SpotMaxPricePercentageOverLowestPrice).toBe(50); - }); - - it('should parse on-demand-max-price-percentage-over-lowest-price label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-on-demand-max-price-percentage-over-lowest-price:75']); - expect(result?.InstanceRequirements?.OnDemandMaxPricePercentageOverLowestPrice).toBe(75); - }); - - it('should parse max-spot-price-as-percentage-of-optimal-on-demand-price label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-max-spot-price-as-percentage-of-optimal-on-demand-price:60']); - expect(result?.InstanceRequirements?.MaxSpotPriceAsPercentageOfOptimalOnDemandPrice).toBe(60); - }); - - it('should parse require-hibernate-support label as boolean true', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-require-hibernate-support:true']); - expect(result?.InstanceRequirements?.RequireHibernateSupport).toBe(true); - }); - - it('should parse require-hibernate-support label as boolean false', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-require-hibernate-support:false']); - expect(result?.InstanceRequirements?.RequireHibernateSupport).toBe(false); - }); - - it('should parse require-encryption-in-transit label as boolean true', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-require-encryption-in-transit:true']); - expect(result?.InstanceRequirements?.RequireEncryptionInTransit).toBe(true); - }); - - it('should parse require-encryption-in-transit label as boolean false', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-require-encryption-in-transit:false']); - expect(result?.InstanceRequirements?.RequireEncryptionInTransit).toBe(false); - }); - - it('should parse baseline-performance-factors-cpu-reference-families label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-baseline-performance-factors-cpu-reference-families:intel']); - expect(result?.InstanceRequirements?.BaselinePerformanceFactors?.Cpu?.References?.[0]?.InstanceFamily).toBe( - 'intel', - ); - }); - it('should parse baseline-performance-factors-cpu-reference-families list label', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-baseline-performance-factors-cpu-reference-families:intel;amd']); - expect(result?.InstanceRequirements?.BaselinePerformanceFactors?.Cpu?.References?.[0]?.InstanceFamily).toBe( - 'intel', - ); - expect(result?.InstanceRequirements?.BaselinePerformanceFactors?.Cpu?.References?.[1]?.InstanceFamily).toBe( - 'amd', - ); - }); - }); - - describe('Edge Cases', () => { - it('should return undefined when empty array is provided', () => { - const result = parseEc2OverrideConfig([]); - expect(result).toBeUndefined(); - }); - - it('should return undefined when no ghr-ec2 labels are provided', () => { - const result = parseEc2OverrideConfig(['self-hosted', 'linux', 'x64']); - expect(result).toBeUndefined(); - }); - - it('should ignore non-ghr-ec2 labels and only parse ghr-ec2 labels', () => { - const result = parseEc2OverrideConfig([ - 'self-hosted', - 'ghr-ec2-instance-type:m5.large', - 'linux', - 'ghr-ec2-max-price:0.30', - ]); - expect(result?.InstanceType).toBe('m5.large'); - expect(result?.MaxPrice).toBe('0.30'); - }); - - it('should handle labels with colons in values (ARNs)', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-ebs-kms-key-id:arn:aws:kms:us-east-1:123456789012:key/abc-def-ghi', - ]); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.KmsKeyId).toBe( - 'arn:aws:kms:us-east-1:123456789012:key/abc-def-ghi', - ); - }); - - it('should handle labels with colons in placement ARNs', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-placement-host-resource-group-arn:arn:aws:ec2:us-west-2:123456789012:host-resource-group/hrg-abc123', - ]); - expect(result?.Placement?.HostResourceGroupArn).toBe( - 'arn:aws:ec2:us-west-2:123456789012:host-resource-group/hrg-abc123', - ); - }); - - it('should handle labels without values gracefully', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-instance-type:', 'ghr-ec2-max-price:0.50']); - expect(result?.InstanceType).toBeUndefined(); - expect(result?.MaxPrice).toBe('0.50'); - }); - - it('should handle malformed labels (no colon) gracefully', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-instance-type-m5-large', 'ghr-ec2-max-price:0.50']); - expect(result?.MaxPrice).toBe('0.50'); - expect(result?.InstanceType).toBeUndefined(); - }); - - it('should handle numeric strings correctly for number fields', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-priority:5', - 'ghr-ec2-weighted-capacity:10', - 'ghr-ec2-vcpu-count-min:4', - ]); - expect(result?.Priority).toBe(5); - expect(result?.WeightedCapacity).toBe(10); - expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(4); - }); - - it('should handle boolean strings correctly for boolean fields', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-ebs-encrypted:true', - 'ghr-ec2-ebs-delete-on-termination:false', - 'ghr-ec2-require-hibernate-support:true', - ]); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.DeleteOnTermination).toBe(false); - expect(result?.InstanceRequirements?.RequireHibernateSupport).toBe(true); - }); - - it('should handle floating point numbers in max-price', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-max-price:0.12345']); - expect(result?.MaxPrice).toBe('0.12345'); - }); - - it('should handle whitespace in semicolon-separated lists', () => { - const result = parseEc2OverrideConfig(['ghr-ec2-cpu-manufacturers: intel ; amd ']); - expect(result?.InstanceRequirements?.CpuManufacturers).toEqual([' intel ', ' amd ']); - }); - - it('should return config with all parsed labels', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-instance-type:c5.xlarge', - 'ghr-ec2-vcpu-count-min:4', - 'ghr-ec2-memory-mib-min:8192', - 'ghr-ec2-placement-tenancy:dedicated', - 'ghr-ec2-ebs-volume-size:100', - ]); - expect(result?.InstanceType).toBe('c5.xlarge'); - expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(4); - expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(8192); - expect(result?.Placement?.Tenancy).toBe('dedicated'); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(100); - }); - }); - - describe('Complex Scenarios', () => { - it('should handle comprehensive EC2 configuration with all categories', () => { - const result = parseEc2OverrideConfig([ - // Basic Fleet - 'ghr-ec2-instance-type:r5.2xlarge', - 'ghr-ec2-max-price:0.75', - 'ghr-ec2-priority:1', - // Placement - 'ghr-ec2-placement-group-name:my-group', - 'ghr-ec2-placement-tenancy:dedicated', - // Block Device - 'ghr-ec2-ebs-volume-size:200', - 'ghr-ec2-ebs-volume-type:gp3', - 'ghr-ec2-ebs-encrypted:true', - // Instance Requirements - 'ghr-ec2-vcpu-count-min:8', - 'ghr-ec2-vcpu-count-max:32', - 'ghr-ec2-memory-mib-min:32768', - 'ghr-ec2-cpu-manufacturers:intel;amd', - 'ghr-ec2-instance-generations:current', - ]); - - expect(result?.InstanceType).toBe('r5.2xlarge'); - expect(result?.MaxPrice).toBe('0.75'); - expect(result?.Priority).toBe(1); - expect(result?.Placement?.GroupName).toBe('my-group'); - expect(result?.Placement?.Tenancy).toBe('dedicated'); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(200); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeType).toBe('gp3'); - expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); - expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(8); - expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(32); - expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(32768); - expect(result?.InstanceRequirements?.CpuManufacturers).toEqual(['intel', 'amd']); - expect(result?.InstanceRequirements?.InstanceGenerations).toEqual(['current']); - }); - - it('should handle GPU instance configuration', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-accelerator-count-min:1', - 'ghr-ec2-accelerator-count-max:4', - 'ghr-ec2-accelerator-types:gpu', - 'ghr-ec2-accelerator-manufacturers:nvidia', - 'ghr-ec2-accelerator-names:a100;v100', - 'ghr-ec2-accelerator-total-memory-mib-min:16384', - ]); - - expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); - expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(4); - expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu']); - expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia']); - expect(result?.InstanceRequirements?.AcceleratorNames).toEqual(['a100', 'v100']); - expect(result?.InstanceRequirements?.AcceleratorTotalMemoryMiB?.Min).toBe(16384); - }); - - it('should handle network-optimized instance configuration', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-network-interface-count-min:2', - 'ghr-ec2-network-interface-count-max:8', - 'ghr-ec2-network-bandwidth-gbps-min:10', - 'ghr-ec2-network-bandwidth-gbps-max:100', - 'ghr-ec2-baseline-ebs-bandwidth-mbps-min:1000', - ]); - - expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Min).toBe(2); - expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Max).toBe(8); - expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Min).toBe(10); - expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Max).toBe(100); - expect(result?.InstanceRequirements?.BaselineEbsBandwidthMbps?.Min).toBe(1000); - }); - - it('should handle storage-optimized instance configuration', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-local-storage:included', - 'ghr-ec2-local-storage-types:ssd', - 'ghr-ec2-total-local-storage-gb-min:500', - 'ghr-ec2-total-local-storage-gb-max:2000', - ]); - - expect(result?.InstanceRequirements?.LocalStorage).toBe('included'); - expect(result?.InstanceRequirements?.LocalStorageTypes).toEqual(['ssd']); - expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Min).toBe(500); - expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Max).toBe(2000); - }); - - it('should handle spot instance configuration with pricing', () => { - const result = parseEc2OverrideConfig([ - 'ghr-ec2-max-price:0.50', - 'ghr-ec2-spot-max-price-percentage-over-lowest-price:100', - 'ghr-ec2-on-demand-max-price-percentage-over-lowest-price:150', - ]); - - expect(result?.MaxPrice).toBe('0.50'); - expect(result?.InstanceRequirements?.SpotMaxPricePercentageOverLowestPrice).toBe(100); - expect(result?.InstanceRequirements?.OnDemandMaxPricePercentageOverLowestPrice).toBe(150); - }); - }); -}); diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.test.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.test.ts deleted file mode 100644 index beda1af4e7..0000000000 --- a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.test.ts +++ /dev/null @@ -1,890 +0,0 @@ -import { Octokit } from '@octokit/rest'; -import { RequestError } from '@octokit/request-error'; -import moment from 'moment'; -import nock from 'nock'; - -import { RunnerInfo, RunnerList } from '../aws/ec2-runners.d'; -import * as ghAuth from '../github/auth'; -import { listEC2Runners, terminateRunner, tag, untag } from './../aws/ec2-runners'; -import { githubCache } from './cache'; -import { newestFirstStrategy, oldestFirstStrategy, scaleDown } from './scale-down'; -import { describe, it, expect, beforeEach, vi } from 'vitest'; - -const mockOctokit = { - apps: { - getOrgInstallation: vi.fn(), - getRepoInstallation: vi.fn(), - }, - actions: { - listSelfHostedRunnersForRepo: vi.fn(), - listSelfHostedRunnersForOrg: vi.fn(), - deleteSelfHostedRunnerFromOrg: vi.fn(), - deleteSelfHostedRunnerFromRepo: vi.fn(), - getSelfHostedRunnerForOrg: vi.fn(), - getSelfHostedRunnerForRepo: vi.fn(), - }, - paginate: vi.fn(), -}; -vi.mock('@octokit/rest', () => ({ - Octokit: vi.fn().mockImplementation(function () { - return mockOctokit; - }), -})); - -vi.mock('./../aws/ec2-runners', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - tag: vi.fn(), - untag: vi.fn(), - terminateRunner: vi.fn(), - listEC2Runners: vi.fn(), - }; -}); -vi.mock('./../github/auth', async () => ({ - createGithubAppAuth: vi.fn(), - createGithubInstallationAuth: vi.fn(), - createOctokitClient: vi.fn(), -})); - -vi.mock('./cache', async () => ({ - githubCache: { - getRunner: vi.fn(), - addRunner: vi.fn(), - clients: new Map(), - runners: new Map(), - reset: vi.fn().mockImplementation(() => { - githubCache.clients.clear(); - githubCache.runners.clear(); - }), - }, -})); - -const mocktokit = Octokit as vi.MockedClass; -const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); -const mockedInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); -const mockCreateClient = vi.mocked(ghAuth.createOctokitClient); -const mockListRunners = vi.mocked(listEC2Runners); -const mockTagRunners = vi.mocked(tag); -const mockUntagRunners = vi.mocked(untag); -const mockTerminateRunners = vi.mocked(terminateRunner); - -export interface TestData { - repositoryName: string; - repositoryOwner: string; -} - -const cleanEnv = process.env; - -const ENVIRONMENT = 'unit-test-environment'; -const MINIMUM_TIME_RUNNING_IN_MINUTES = 30; -const MINIMUM_BOOT_TIME = 5; -const TEST_DATA: TestData = { - repositoryName: 'hello-world', - repositoryOwner: 'Codertocat', -}; - -interface RunnerTestItem extends RunnerList { - registered: boolean; - orphan: boolean; - shouldBeTerminated: boolean; -} - -describe('Scale down runners', () => { - beforeEach(() => { - process.env = { ...cleanEnv }; - process.env.GITHUB_APP_KEY_BASE64 = 'TEST_CERTIFICATE_DATA'; - process.env.GITHUB_APP_ID = '1337'; - process.env.GITHUB_APP_CLIENT_ID = 'TEST_CLIENT_ID'; - process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET'; - process.env.RUNNERS_MAXIMUM_COUNT = '3'; - process.env.SCALE_DOWN_CONFIG = '[]'; - process.env.ENVIRONMENT = ENVIRONMENT; - process.env.MINIMUM_RUNNING_TIME_IN_MINUTES = MINIMUM_TIME_RUNNING_IN_MINUTES.toString(); - process.env.RUNNER_BOOT_TIME_IN_MINUTES = MINIMUM_BOOT_TIME.toString(); - - nock.disableNetConnect(); - vi.clearAllMocks(); - vi.resetModules(); - githubCache.clients.clear(); - githubCache.runners.clear(); - mockOctokit.apps.getOrgInstallation.mockImplementation(() => ({ - data: { - id: 'ORG', - }, - })); - mockOctokit.apps.getRepoInstallation.mockImplementation(() => ({ - data: { - id: 'REPO', - }, - })); - - mockOctokit.paginate.mockResolvedValue([]); - mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation((repo) => { - // check if repo.runner_id contains the word "busy". If yes, throw an error else return 204 - if (repo.runner_id.includes('busy')) { - throw Error(); - } else { - return { status: 204 }; - } - }); - - mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation((repo) => { - // check if repo.runner_id contains the word "busy". If yes, throw an error else return 204 - if (repo.runner_id.includes('busy')) { - throw Error(); - } else { - return { status: 204 }; - } - }); - - mockOctokit.actions.getSelfHostedRunnerForRepo.mockImplementation((repo) => { - if (repo.runner_id.includes('busy')) { - return { - data: { busy: true }, - }; - } else { - return { - data: { busy: false }, - }; - } - }); - mockOctokit.actions.getSelfHostedRunnerForOrg.mockImplementation((repo) => { - if (repo.runner_id.includes('busy')) { - return { - data: { busy: true }, - }; - } else { - return { - data: { busy: false }, - }; - } - }); - - mockTerminateRunners.mockImplementation(async () => { - return; - }); - mockedAppAuth.mockResolvedValue({ - type: 'app', - token: 'token', - appId: 1, - expiresAt: 'some-date', - }); - mockedInstallationAuth.mockResolvedValue({ - type: 'token', - tokenType: 'installation', - token: 'token', - createdAt: 'some-date', - expiresAt: 'some-date', - permissions: {}, - repositorySelection: 'all', - installationId: 0, - }); - mockCreateClient.mockResolvedValue(new mocktokit()); - }); - - const endpoints = ['https://api.github.com', 'https://github.enterprise.something', 'https://companyname.ghe.com']; - - describe.each(endpoints)('for %s', (endpoint) => { - beforeEach(() => { - if (endpoint.includes('enterprise') || endpoint.endsWith('.ghe.com')) { - process.env.GHES_URL = endpoint; - } - }); - - type RunnerType = 'Repo' | 'Org'; - const runnerTypes: RunnerType[] = ['Org', 'Repo']; - describe.each(runnerTypes)('For %s runners.', (type) => { - it('Should not call terminate when no runners online.', async () => { - // setup - mockAwsRunners([]); - - // act - await scaleDown(); - - // assert - expect(listEC2Runners).toHaveBeenCalledWith({ - environment: ENVIRONMENT, - }); - expect(terminateRunner).not.toHaveBeenCalled(); - expect(mockOctokit.apps.getRepoInstallation).not.toHaveBeenCalled(); - expect(mockOctokit.apps.getRepoInstallation).not.toHaveBeenCalled(); - }); - - it(`Should terminate runner without idle config ${type} runners.`, async () => { - // setup - const runners = [ - createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES - 1, true, false, false), - createRunnerTestData('idle-2', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 4, true, false, true), - createRunnerTestData('busy-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 3, true, false, false), - createRunnerTestData('booting-1', type, MINIMUM_BOOT_TIME - 1, false, false, false), - ]; - - mockGitHubRunners(runners); - mockListRunners.mockResolvedValue(runners); - mockAwsRunners(runners); - - await scaleDown(); - - // assert - expect(listEC2Runners).toHaveBeenCalledWith({ - environment: ENVIRONMENT, - }); - - if (type === 'Repo') { - expect(mockOctokit.apps.getRepoInstallation).toHaveBeenCalled(); - } else { - expect(mockOctokit.apps.getOrgInstallation).toHaveBeenCalled(); - } - - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it(`Should respect idle runner with minimum running time not exceeded.`, async () => { - // setup - const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES - 1, true, false, false)]; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await scaleDown(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it(`Should respect booting runner.`, async () => { - // setup - const runners = [createRunnerTestData('booting-1', type, MINIMUM_BOOT_TIME - 1, false, false, false)]; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await scaleDown(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it(`Should respect busy runner.`, async () => { - // setup - const runners = [createRunnerTestData('busy-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false)]; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await scaleDown(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it(`Should not terminate runner with bypass-removal tag set.`, async () => { - // setup - const runners = [ - createRunnerTestData('idle-with-bypass', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 10, true, false, false), - ]; - // Set bypass-removal tag - runners[0].bypassRemoval = true; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await scaleDown(); - - // assert - expect(terminateRunner).not.toHaveBeenCalled(); - checkNonTerminated(runners); - }); - - it(`Should not terminate orphaned runner with bypass-removal tag set.`, async () => { - // setup - orphan runner with bypass-removal tag - const orphanRunner = createRunnerTestData('orphan-bypass', type, MINIMUM_BOOT_TIME + 1, false, false, false); - orphanRunner.bypassRemoval = true; - - const idleRunner = createRunnerTestData('idle-1', type, MINIMUM_BOOT_TIME + 1, true, false, false); - const runners = [orphanRunner, idleRunner]; - - mockGitHubRunners([idleRunner]); - mockAwsRunners(runners); - - // act - first cycle marks orphan - await scaleDown(); - - // mark as orphan for next cycle - orphanRunner.orphan = true; - - // act - second cycle should skip termination due to bypass-removal - await scaleDown(); - - // assert - orphan runner should NOT be terminated - expect(terminateRunner).not.toHaveBeenCalledWith(orphanRunner.instanceId); - }); - - it(`Should not terminate a runner that became busy just before deregister runner.`, async () => { - // setup - const runners = [ - createRunnerTestData( - 'job-just-start-at-deregister-1', - type, - MINIMUM_TIME_RUNNING_IN_MINUTES + 1, - true, - false, - false, - ), - ]; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { - return { status: 500 }; - }); - - mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { - return { status: 500 }; - }); - - // act and ensure no exception is thrown - await expect(scaleDown()).resolves.not.toThrow(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it(`Should terminate orphan (Non JIT)`, async () => { - // setup - const orphanRunner = createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, false, false); - const idleRunner = createRunnerTestData('idle-1', type, MINIMUM_BOOT_TIME + 1, true, false, false); - const runners = [orphanRunner, idleRunner]; - - mockGitHubRunners([idleRunner]); - mockAwsRunners(runners); - - // act - await scaleDown(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - - expect(mockTagRunners).toHaveBeenCalledWith(orphanRunner.instanceId, [ - { - Key: 'ghr:orphan', - Value: 'true', - }, - ]); - - expect(mockTagRunners).not.toHaveBeenCalledWith(idleRunner.instanceId, expect.anything()); - - // next cycle, update test data set orphan to true and terminate should be true - orphanRunner.orphan = true; - orphanRunner.shouldBeTerminated = true; - - // act - await scaleDown(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it('Should test if orphaned runner, untag if online and busy, else terminate (JIT)', async () => { - // arrange - const orphanRunner = createRunnerTestData( - 'orphan-jit', - type, - MINIMUM_BOOT_TIME + 1, - false, - true, - false, - undefined, - 1234567890, - ); - const runners = [orphanRunner]; - - mockGitHubRunners([]); - mockAwsRunners(runners); - - if (type === 'Repo') { - mockOctokit.actions.getSelfHostedRunnerForRepo.mockResolvedValueOnce({ - data: { id: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' }, - }); - } else { - mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ - data: { id: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' }, - }); - } - - // act - await scaleDown(); - - // assert - expect(mockUntagRunners).toHaveBeenCalledWith(orphanRunner.instanceId, [{ Key: 'ghr:orphan', Value: 'true' }]); - expect(mockTerminateRunners).not.toHaveBeenCalledWith(orphanRunner.instanceId); - - // arrange - if (type === 'Repo') { - mockOctokit.actions.getSelfHostedRunnerForRepo.mockResolvedValueOnce({ - data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'offline' }, - }); - } else { - mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ - data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'offline' }, - }); - } - - // act - await scaleDown(); - - // assert - expect(mockTerminateRunners).toHaveBeenCalledWith(orphanRunner.instanceId); - }); - - it('Should handle 404 error when checking orphaned runner (JIT) - treat as orphaned', async () => { - // arrange - const orphanRunner = createRunnerTestData( - 'orphan-jit-404', - type, - MINIMUM_BOOT_TIME + 1, - false, - true, - true, // should be terminated when 404 - undefined, - 1234567890, - ); - const runners = [orphanRunner]; - - mockGitHubRunners([]); - mockAwsRunners(runners); - - // Mock 404 error response - const error404 = new RequestError('Runner not found', 404, { - request: { - method: 'GET', - url: 'https://api.github.com/test', - headers: {}, - }, - }); - - if (type === 'Repo') { - mockOctokit.actions.getSelfHostedRunnerForRepo.mockRejectedValueOnce(error404); - } else { - mockOctokit.actions.getSelfHostedRunnerForOrg.mockRejectedValueOnce(error404); - } - - // act - await scaleDown(); - - // assert - should terminate since 404 means runner doesn't exist on GitHub - expect(mockTerminateRunners).toHaveBeenCalledWith(orphanRunner.instanceId); - }); - - it('Should handle 404 error when checking runner busy state - treat as not busy', async () => { - // arrange - const runner = createRunnerTestData( - 'runner-404', - type, - MINIMUM_TIME_RUNNING_IN_MINUTES + 1, - true, - false, - true, // should be terminated since not busy due to 404 - ); - const runners = [runner]; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // Mock 404 error response for busy state check - const error404 = new RequestError('Runner not found', 404, { - request: { - method: 'GET', - url: 'https://api.github.com/test', - headers: {}, - }, - }); - - if (type === 'Repo') { - mockOctokit.actions.getSelfHostedRunnerForRepo.mockRejectedValueOnce(error404); - } else { - mockOctokit.actions.getSelfHostedRunnerForOrg.mockRejectedValueOnce(error404); - } - - // act - await scaleDown(); - - // assert - should terminate since 404 means runner is not busy - checkTerminated(runners); - }); - - it('Should re-throw non-404 errors when checking runner state', async () => { - // arrange - const orphanRunner = createRunnerTestData( - 'orphan-error', - type, - MINIMUM_BOOT_TIME + 1, - false, - true, - false, - undefined, - 1234567890, - ); - const runners = [orphanRunner]; - - mockGitHubRunners([]); - mockAwsRunners(runners); - - // Mock non-404 error response - const error500 = new RequestError('Internal server error', 500, { - request: { - method: 'GET', - url: 'https://api.github.com/test', - headers: {}, - }, - }); - - if (type === 'Repo') { - mockOctokit.actions.getSelfHostedRunnerForRepo.mockRejectedValueOnce(error500); - } else { - mockOctokit.actions.getSelfHostedRunnerForOrg.mockRejectedValueOnce(error500); - } - - // act & assert - should not throw because error handling is in terminateOrphan - await expect(scaleDown()).resolves.not.toThrow(); - - // Should not terminate since the error was not a 404 - expect(terminateRunner).not.toHaveBeenCalledWith(orphanRunner.instanceId); - }); - - it(`Should ignore errors when termination orphan fails.`, async () => { - // setup - const orphanRunner = createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, true, true); - const runners = [orphanRunner]; - - mockGitHubRunners([]); - mockAwsRunners(runners); - mockTerminateRunners.mockImplementation(() => { - throw new Error('Failed to terminate'); - }); - - // act - await scaleDown(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - describe('When orphan termination fails', () => { - it(`Should not throw in case of list runner exception.`, async () => { - // setup - const runners = [createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, true, true)]; - - mockGitHubRunners([]); - mockListRunners.mockRejectedValueOnce(new Error('Failed to list runners')); - mockAwsRunners(runners); - - // ac - await scaleDown(); - - // assert - checkNonTerminated(runners); - }); - - it(`Should not throw in case of terminate runner exception.`, async () => { - // setup - const runners = [createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, true, true)]; - - mockGitHubRunners([]); - mockAwsRunners(runners); - mockTerminateRunners.mockRejectedValue(new Error('Failed to terminate')); - - // act and ensure no exception is thrown - await scaleDown(); - - // assert - checkNonTerminated(runners); - }); - }); - - it(`Should not terminate instance in case de-register fails.`, async () => { - // setup - const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false)]; - - mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { - return { status: 500 }; - }); - mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { - return { status: 500 }; - }); - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act and should resolve - await expect(scaleDown()).resolves.not.toThrow(); - - // assert - checkTerminated(runners); - checkNonTerminated(runners); - }); - - it(`Should not throw an exception in case of failure during removing a runner.`, async () => { - // setup - const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, true, false)]; - - mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { - throw new Error('Failed to delete runner'); - }); - mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { - throw new Error('Failed to delete runner'); - }); - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await expect(scaleDown()).resolves.not.toThrow(); - }); - - it(`Should not terminate instance when de-registration throws an error.`, async () => { - // setup - runner should NOT be terminated because de-registration fails - const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false)]; - - const error502 = new RequestError('Server Error', 502, { - request: { - method: 'DELETE', - url: 'https://api.github.com/test', - headers: {}, - }, - }); - - mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { - throw error502; - }); - mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { - throw error502; - }); - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await expect(scaleDown()).resolves.not.toThrow(); - - // assert - should NOT terminate since de-registration failed - expect(terminateRunner).not.toHaveBeenCalled(); - }); - - const evictionStrategies = ['oldest_first', 'newest_first']; - describe.each(evictionStrategies)('When idle config defined', (evictionStrategy) => { - const defaultConfig = { - idleCount: 1, - cron: '* * * * * *', - timeZone: 'Europe/Amsterdam', - evictionStrategy, - }; - - beforeEach(() => { - process.env.SCALE_DOWN_CONFIG = JSON.stringify([defaultConfig]); - }); - - it(`Should terminate based on the the idle config with ${evictionStrategy} eviction strategy`, async () => { - // setup - const runnerToTerminateTime = - evictionStrategy === 'oldest_first' - ? MINIMUM_TIME_RUNNING_IN_MINUTES + 5 - : MINIMUM_TIME_RUNNING_IN_MINUTES + 1; - const runners = [ - createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 4, true, false, false), - createRunnerTestData('idle-to-terminate', type, runnerToTerminateTime, true, false, true), - ]; - - mockGitHubRunners(runners); - mockAwsRunners(runners); - - // act - await scaleDown(); - - // assert - const runnersToTerminate = runners.filter((r) => r.shouldBeTerminated); - for (const toTerminate of runnersToTerminate) { - expect(terminateRunner).toHaveBeenCalledWith(toTerminate.instanceId); - } - - const runnersNotToTerminate = runners.filter((r) => !r.shouldBeTerminated); - for (const notTerminated of runnersNotToTerminate) { - expect(terminateRunner).not.toHaveBeenCalledWith(notTerminated.instanceId); - } - }); - }); - }); - }); - - describe('When runners are sorted', () => { - const runners: RunnerInfo[] = [ - { - instanceId: '1', - launchTime: moment(new Date()).subtract(1, 'minute').toDate(), - owner: 'owner', - type: 'type', - }, - { - instanceId: '3', - launchTime: moment(new Date()).subtract(3, 'minute').toDate(), - owner: 'owner', - type: 'type', - }, - { - instanceId: '2', - launchTime: moment(new Date()).subtract(2, 'minute').toDate(), - owner: 'owner', - type: 'type', - }, - { - instanceId: '0', - launchTime: moment(new Date()).subtract(0, 'minute').toDate(), - owner: 'owner', - type: 'type', - }, - ]; - - it('Should sort runners descending for eviction strategy oldest first te keep the youngest.', () => { - runners.sort(oldestFirstStrategy); - expect(runners[0].instanceId).toEqual('0'); - expect(runners[1].instanceId).toEqual('1'); - expect(runners[2].instanceId).toEqual('2'); - expect(runners[3].instanceId).toEqual('3'); - }); - - it('Should sort runners ascending for eviction strategy newest first te keep oldest.', () => { - runners.sort(newestFirstStrategy); - expect(runners[0].instanceId).toEqual('3'); - expect(runners[1].instanceId).toEqual('2'); - expect(runners[2].instanceId).toEqual('1'); - expect(runners[3].instanceId).toEqual('0'); - }); - - it('Should sort runners with equal launch time.', () => { - const runnersTest = [...runners]; - const same = moment(new Date()).subtract(4, 'minute').toDate(); - runnersTest.push({ - instanceId: '4', - launchTime: same, - owner: 'owner', - type: 'type', - }); - runnersTest.push({ - instanceId: '5', - launchTime: same, - owner: 'owner', - type: 'type', - }); - runnersTest.sort(oldestFirstStrategy); - expect(runnersTest[3].launchTime).not.toEqual(same); - expect(runnersTest[4].launchTime).toEqual(same); - expect(runnersTest[5].launchTime).toEqual(same); - - runnersTest.sort(newestFirstStrategy); - expect(runnersTest[3].launchTime).not.toEqual(same); - expect(runnersTest[1].launchTime).toEqual(same); - expect(runnersTest[0].launchTime).toEqual(same); - }); - - it('Should sort runners even when launch time is undefined.', () => { - const runnersTest = [ - { - instanceId: '0', - launchTime: undefined, - owner: 'owner', - type: 'type', - }, - { - instanceId: '1', - launchTime: moment(new Date()).subtract(3, 'minute').toDate(), - owner: 'owner', - type: 'type', - }, - { - instanceId: '0', - launchTime: undefined, - owner: 'owner', - type: 'type', - }, - ]; - runnersTest.sort(oldestFirstStrategy); - expect(runnersTest[0].launchTime).toBeUndefined(); - expect(runnersTest[1].launchTime).toBeDefined(); - expect(runnersTest[2].launchTime).not.toBeDefined(); - }); - }); -}); - -function mockAwsRunners(runners: RunnerTestItem[]) { - mockListRunners.mockImplementation(async (filter) => { - return runners.filter((r) => !filter?.orphan || filter?.orphan === r.orphan); - }); -} - -function checkNonTerminated(runners: RunnerTestItem[]) { - const notTerminated = runners.filter((r) => !r.shouldBeTerminated); - for (const toTerminate of notTerminated) { - expect(terminateRunner).not.toHaveBeenCalledWith(toTerminate.instanceId); - } -} - -function checkTerminated(runners: RunnerTestItem[]) { - const runnersToTerminate = runners.filter((r) => r.shouldBeTerminated); - expect(terminateRunner).toHaveBeenCalledTimes(runnersToTerminate.length); - for (const toTerminate of runnersToTerminate) { - expect(terminateRunner).toHaveBeenCalledWith(toTerminate.instanceId); - } -} - -function mockGitHubRunners(runners: RunnerTestItem[]) { - mockOctokit.paginate.mockResolvedValue( - runners - .filter((r) => r.registered) - .map((r) => { - return { - id: r.instanceId, - name: r.instanceId, - }; - }), - ); -} - -function createRunnerTestData( - name: string, - type: 'Org' | 'Repo', - minutesLaunchedAgo: number, - registered: boolean, - orphan: boolean, - shouldBeTerminated: boolean, - owner?: string, - runnerId?: number, -): RunnerTestItem { - return { - instanceId: `i-${name}-${type.toLowerCase()}`, - launchTime: moment(new Date()).subtract(minutesLaunchedAgo, 'minutes').toDate(), - type, - owner: owner - ? owner - : type === 'Repo' - ? `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}` - : `${TEST_DATA.repositoryOwner}`, - registered, - orphan, - shouldBeTerminated, - runnerId: runnerId !== undefined ? String(runnerId) : undefined, - bypassRemoval: false, - }; -} diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.test.ts deleted file mode 100644 index 50f0e6776f..0000000000 --- a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.test.ts +++ /dev/null @@ -1,2769 +0,0 @@ -import { DescribeLaunchTemplateVersionsCommand, EC2Client } from '@aws-sdk/client-ec2'; -import { PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; -import { mockClient } from 'aws-sdk-client-mock'; -import 'aws-sdk-client-mock-jest/vitest'; -// Using vi.mocked instead of jest-mock -import nock from 'nock'; -import { performance } from 'perf_hooks'; - -import * as ghAuth from '../github/auth'; -import { createRunner, listEC2Runners, tag } from './../aws/ec2-runners'; -import { RunnerInputParameters } from './../aws/ec2-runners.d'; -import * as scaleUpModule from './scale-up'; -import { EC2_TAG_VALUE_MAX_LENGTH, RUNNER_LABELS_TAG_MAX_COUNT } from './ec2'; -import { getParameter } from '@aws-github-runner/aws-ssm-util'; -import { publishRetryMessage } from './job-retry'; -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import type { Octokit } from '@octokit/rest'; -import type { ActionRequestMessageSQS } from './types'; - -const mockOctokit = { - paginate: vi.fn(), - checks: { get: vi.fn() }, - actions: { - createRegistrationTokenForOrg: vi.fn(), - createRegistrationTokenForRepo: vi.fn(), - getJobForWorkflowRun: vi.fn(), - generateRunnerJitconfigForOrg: vi.fn(), - generateRunnerJitconfigForRepo: vi.fn(), - }, - apps: { - getOrgInstallation: vi.fn(), - getRepoInstallation: vi.fn(), - }, -}; - -const mockCreateRunner = vi.mocked(createRunner); -const mockListRunners = vi.mocked(listEC2Runners); -const mockTag = vi.mocked(tag); -const mockEC2Client = mockClient(EC2Client); -const mockSSMClient = mockClient(SSMClient); -const mockSSMgetParameter = vi.mocked(getParameter); -const mockPublishRetryMessage = vi.mocked(publishRetryMessage); - -vi.mock('@octokit/rest', () => ({ - Octokit: vi.fn().mockImplementation(function () { - return mockOctokit; - }), -})); - -vi.mock('./../aws/ec2-runners', async () => ({ - createRunner: vi.fn(), - listEC2Runners: vi.fn(), - tag: vi.fn(), -})); - -vi.mock('./../github/auth', async () => ({ - createGithubAppAuth: vi.fn(), - createGithubInstallationAuth: vi.fn(), - createOctokitClient: vi.fn(), -})); - -vi.mock('@aws-github-runner/aws-ssm-util', async () => { - const actual = (await vi.importActual( - '@aws-github-runner/aws-ssm-util', - )) as typeof import('@aws-github-runner/aws-ssm-util'); - - return { - ...actual, - getParameter: vi.fn(), - }; -}); - -vi.mock('./job-retry', () => ({ - publishRetryMessage: vi.fn(), - checkAndRetryJob: vi.fn(), -})); - -export type RunnerType = 'ephemeral' | 'non-ephemeral'; - -// for ephemeral and non-ephemeral runners -const RUNNER_TYPES: RunnerType[] = ['ephemeral', 'non-ephemeral']; - -const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); -const mockedInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); -const mockCreateClient = vi.mocked(ghAuth.createOctokitClient); - -const TEST_DATA_SINGLE: ActionRequestMessageSQS = { - id: 1, - eventType: 'workflow_job', - repositoryName: 'hello-world', - repositoryOwner: 'Codertocat', - installationId: 2, - repoOwnerType: 'Organization', - messageId: 'foobar', -}; - -const TEST_DATA: ActionRequestMessageSQS[] = [ - { - ...TEST_DATA_SINGLE, - messageId: 'foobar', - }, -]; - -const cleanEnv = process.env; - -const EXPECTED_RUNNER_PARAMS: RunnerInputParameters = { - environment: 'unit-test-environment', - runnerType: 'Org', - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - numberOfRunners: 1, - launchTemplateName: 'lt-1', - ec2instanceCriteria: { - instanceTypes: ['m5.large'], - targetCapacityType: 'spot', - instanceAllocationStrategy: 'lowest-price', - }, - subnets: ['subnet-123'], - tracingEnabled: false, - onDemandFailoverOnError: [], - scaleErrors: ['UnfulfillableCapacity', 'MaxSpotInstanceCountExceeded', 'TargetCapacityLimitExceededException'], - source: 'scale-up-lambda', - useDedicatedHost: false, -}; -let expectedRunnerParams: RunnerInputParameters; - -function setDefaults() { - process.env = { ...cleanEnv }; - process.env.PARAMETER_GITHUB_APP_ID_NAME = 'github-app-id'; - process.env.GITHUB_APP_KEY_BASE64 = 'TEST_CERTIFICATE_DATA'; - process.env.GITHUB_APP_ID = '1337'; - process.env.GITHUB_APP_CLIENT_ID = 'TEST_CLIENT_ID'; - process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET'; - process.env.RUNNERS_MAXIMUM_COUNT = '3'; - process.env.ENVIRONMENT = EXPECTED_RUNNER_PARAMS.environment; - process.env.LAUNCH_TEMPLATE_NAME = 'lt-1'; - process.env.SUBNET_IDS = 'subnet-123'; - process.env.INSTANCE_TYPES = 'm5.large'; - process.env.INSTANCE_TARGET_CAPACITY_TYPE = 'spot'; - process.env.ENABLE_ON_DEMAND_FAILOVER = undefined; - process.env.SCALE_ERRORS = - '["UnfulfillableCapacity","MaxSpotInstanceCountExceeded","TargetCapacityLimitExceededException"]'; -} - -beforeEach(() => { - nock.disableNetConnect(); - vi.resetModules(); - vi.clearAllMocks(); - setDefaults(); - - mockEC2Client.reset(); - mockEC2Client.on(DescribeLaunchTemplateVersionsCommand).resolves({ - LaunchTemplateVersions: [ - { - LaunchTemplateData: { - BlockDeviceMappings: [ - { - DeviceName: '/dev/sda1', - Ebs: {}, - }, - ], - }, - }, - ], - }); - - defaultSSMGetParameterMockImpl(); - defaultOctokitMockImpl(); - - mockCreateRunner.mockImplementation(async () => { - return ['i-12345']; - }); - mockListRunners.mockImplementation(async () => [ - { - instanceId: 'i-1234', - launchTime: new Date(), - type: 'Org', - owner: TEST_DATA_SINGLE.repositoryOwner, - }, - ]); - - mockedAppAuth.mockResolvedValue({ - type: 'app', - token: 'token', - appId: TEST_DATA_SINGLE.installationId, - expiresAt: 'some-date', - }); - mockedInstallationAuth.mockResolvedValue({ - type: 'token', - tokenType: 'installation', - token: 'token', - createdAt: 'some-date', - expiresAt: 'some-date', - permissions: {}, - repositorySelection: 'all', - installationId: 0, - }); - - mockCreateClient.mockResolvedValue(mockOctokit as unknown as Octokit); -}); - -describe('scaleUp with GHES', () => { - beforeEach(() => { - process.env.GHES_URL = 'https://github.enterprise.something'; - }); - - it('checks queued workflows', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalledWith({ - job_id: TEST_DATA_SINGLE.id, - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('does not list runners when no workflows are queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ - data: { total_count: 0 }, - })); - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).not.toBeCalled(); - }); - - describe('on org level', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.RUNNER_NAME_PREFIX = 'unit-test-'; - process.env.RUNNER_GROUP_NAME = 'Default'; - process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; - process.env.RUNNER_LABELS = 'label1,label2'; - - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - mockSSMClient.reset(); - }); - - it('gets the current org level runners', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).toBeCalledWith({ - environment: 'unit-test-environment', - runnerType: 'Org', - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }); - }); - - it('does not create a token when maximum runners has been reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('does not create runners when current runners exceed maximum (race condition)', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '5'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - // Simulate race condition where pool lambda created more runners than max - mockListRunners.mockImplementation(async () => - Array.from({ length: 10 }, (_, i) => ({ - instanceId: `i-${i}`, - launchTime: new Date(), - type: 'Org', - owner: TEST_DATA_SINGLE.repositoryOwner, - })), - ); - await scaleUpModule.scaleUp(TEST_DATA); - // Should not attempt to create runners (would be negative without fix) - expect(createRunner).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - }); - - it('does create a runner if maximum is set to -1', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '-1'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).not.toHaveBeenCalled(); - expect(createRunner).toHaveBeenCalled(); - }); - - it('creates a token when maximum runners has not been reached', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - }); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a runner with correct config', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('tags the EC2 runner with the GitHub runner metadata returned by JIT config', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockTag).toHaveBeenCalledWith('i-12345', [ - { Key: 'ghr:github_runner_id', Value: '9876543210' }, - { Key: 'ghr:runner_labels', Value: 'label1,label2' }, - ]); - }); - - it('chunks comma-joined GitHub runner labels by the EC2 tag value max length', async () => { - const runnerLabels = ['a'.repeat(EC2_TAG_VALUE_MAX_LENGTH), 'b'].join(','); - process.env.RUNNER_LABELS = runnerLabels; - - await scaleUpModule.scaleUp(TEST_DATA); - - expect(mockTag).toHaveBeenCalledWith('i-12345', [ - { Key: 'ghr:github_runner_id', Value: '9876543210' }, - { Key: 'ghr:runner_labels', Value: runnerLabels.slice(0, EC2_TAG_VALUE_MAX_LENGTH) }, - { - Key: 'ghr:runner_labels:2', - Value: runnerLabels.slice(EC2_TAG_VALUE_MAX_LENGTH), - }, - ]); - }); - - it('limits GitHub runner label metadata to five EC2 tags', async () => { - const runnerLabels = Array.from({ length: RUNNER_LABELS_TAG_MAX_COUNT + 1 }, (_, index) => - String.fromCharCode('a'.charCodeAt(0) + index).repeat(EC2_TAG_VALUE_MAX_LENGTH), - ).join(','); - process.env.RUNNER_LABELS = runnerLabels; - - await scaleUpModule.scaleUp(TEST_DATA); - - expect(mockTag).toHaveBeenCalledWith('i-12345', [ - { Key: 'ghr:github_runner_id', Value: '9876543210' }, - ...Array.from({ length: RUNNER_LABELS_TAG_MAX_COUNT }, (_, index) => ({ - Key: index === 0 ? 'ghr:runner_labels' : `ghr:runner_labels:${index + 1}`, - Value: runnerLabels.slice(index * EC2_TAG_VALUE_MAX_LENGTH, (index + 1) * EC2_TAG_VALUE_MAX_LENGTH), - })), - ]); - }); - - it('uses a reserved separator when packing GitHub runner labels into EC2 tags', async () => { - process.env.RUNNER_LABELS = ['label:with/slash', 'label+with+plus'].join(','); - const testDataWithSemicolonDynamicLabel = [ - { - ...TEST_DATA_SINGLE, - labels: ['ghr-ec2-cpu-manufacturers:intel;amd'], - messageId: 'test-semicolon-dynamic-label', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithSemicolonDynamicLabel); - - expect(mockTag).toHaveBeenCalledWith('i-12345', [ - { Key: 'ghr:github_runner_id', Value: '9876543210' }, - { - Key: 'ghr:runner_labels', - Value: 'label:with/slash,label+with+plus,ghr-ec2-cpu-manufacturers:intel;amd', - }, - ]); - }); - - it('creates a runner with labels in a specific group', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner with ami id override from ssm parameter', async () => { - process.env.AMI_ID_SSM_PARAMETER_NAME = 'my-ami-id-param'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith({ ...expectedRunnerParams, amiIdSsmParameterName: 'my-ami-id-param' }); - }); - - it('Throws an error if runner group does not exist for ephemeral runners', async () => { - process.env.RUNNER_GROUP_NAME = 'test-runner-group'; - mockSSMgetParameter.mockImplementation(async () => { - throw new Error('ParameterNotFound'); - }); - await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toBeInstanceOf(Error); - expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); - }); - - it('Discards event if it is a User repo and org level runners is enabled', async () => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - const USER_REPO_TEST_DATA = structuredClone(TEST_DATA); - USER_REPO_TEST_DATA[0].repoOwnerType = 'User'; - await scaleUpModule.scaleUp(USER_REPO_TEST_DATA); - expect(createRunner).not.toHaveBeenCalled(); - }); - - it('create SSM parameter for runner group id if it does not exist', async () => { - mockSSMgetParameter.mockImplementation(async () => { - throw new Error('ParameterNotFound'); - }); - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 2); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`, - Value: '1', - Type: 'String', - }); - }); - - it('Does not create SSM parameter for runner group id if it exists', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.paginate).toHaveBeenCalledTimes(0); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 1); - }); - - it('create start runner config for ephemeral runners ', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '2'; - - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toBeCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - name: 'unit-test-i-12345', - runner_group_id: 1, - labels: ['label1', 'label2'], - }); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: 'TEST_JIT_CONFIG_ORG', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('create start runner config for non-ephemeral runners ', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - process.env.RUNNERS_MAXIMUM_COUNT = '2'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalled(); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: - '--url https://github.enterprise.something/Codertocat --token 1234abcd ' + - '--labels label1,label2 --runnergroup Default', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('quotes runner labels with semicolon separators in non-ephemeral runner config', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - process.env.RUNNERS_MAXIMUM_COUNT = '2'; - - await scaleUpModule.scaleUp([ - { - ...TEST_DATA_SINGLE, - labels: ['ghr-ec2-cpu-manufacturers:intel;amd'], - messageId: 'test-semicolon-labels', - }, - ]); - - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: - '--url https://github.enterprise.something/Codertocat --token 1234abcd ' + - "--labels 'label1,label2,ghr-ec2-cpu-manufacturers:intel;amd' --runnergroup Default", - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('should create JIT config for all remaining instances even when GitHub API fails for one instance', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '5'; - mockCreateRunner.mockImplementation(async () => { - return ['i-instance-1', 'i-instance-2', 'i-instance-3']; - }); - mockListRunners.mockImplementation(async () => { - return []; - }); - - mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ name }) => { - if (name === 'unit-test-i-instance-2') { - // Simulate a 503 Service Unavailable error from GitHub - const error = new Error('Service Unavailable') as Error & { - status: number; - response: { status: number; data: { message: string } }; - }; - error.status = 503; - error.response = { - status: 503, - data: { message: 'Service temporarily unavailable' }, - }; - throw error; - } - return { - data: { - runner: { id: 9876543210 }, - encoded_jit_config: `TEST_JIT_CONFIG_${name}`, - }, - headers: {}, - }; - }); - - await scaleUpModule.scaleUp(TEST_DATA); - - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toHaveBeenCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - name: 'unit-test-i-instance-1', - runner_group_id: 1, - labels: ['label1', 'label2'], - }); - - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toHaveBeenCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - name: 'unit-test-i-instance-2', - runner_group_id: 1, - labels: ['label1', 'label2'], - }); - - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toHaveBeenCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - name: 'unit-test-i-instance-3', - runner_group_id: 1, - labels: ['label1', 'label2'], - }); - - expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-1', - Value: 'TEST_JIT_CONFIG_unit-test-i-instance-1', - Type: 'SecureString', - Tags: [{ Key: 'InstanceId', Value: 'i-instance-1' }], - }); - - expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-3', - Value: 'TEST_JIT_CONFIG_unit-test-i-instance-3', - Type: 'SecureString', - Tags: [{ Key: 'InstanceId', Value: 'i-instance-3' }], - }); - - expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-2', - }); - }); - - it('should handle retryable errors with error handling logic', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '5'; - mockCreateRunner.mockImplementation(async () => { - return ['i-instance-1', 'i-instance-2']; - }); - mockListRunners.mockImplementation(async () => { - return []; - }); - - mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ name }) => { - if (name === 'unit-test-i-instance-1') { - const error = new Error('Internal Server Error') as Error & { - status: number; - response: { status: number; data: { message: string } }; - }; - error.status = 500; - error.response = { - status: 500, - data: { message: 'Internal server error' }, - }; - throw error; - } - return { - data: { - runner: { id: 9876543210 }, - encoded_jit_config: `TEST_JIT_CONFIG_${name}`, - }, - headers: {}, - }; - }); - - await scaleUpModule.scaleUp(TEST_DATA); - - expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-2', - Value: 'TEST_JIT_CONFIG_unit-test-i-instance-2', - Type: 'SecureString', - Tags: [{ Key: 'InstanceId', Value: 'i-instance-2' }], - }); - - expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-1', - }); - }); - - it('should handle non-retryable 4xx errors gracefully', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '5'; - mockCreateRunner.mockImplementation(async () => { - return ['i-instance-1', 'i-instance-2']; - }); - mockListRunners.mockImplementation(async () => { - return []; - }); - - mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ name }) => { - if (name === 'unit-test-i-instance-1') { - // 404 is not retryable - will fail immediately - const error = new Error('Not Found') as Error & { - status: number; - response: { status: number; data: { message: string } }; - }; - error.status = 404; - error.response = { - status: 404, - data: { message: 'Resource not found' }, - }; - throw error; - } - return { - data: { - runner: { id: 9876543210 }, - encoded_jit_config: `TEST_JIT_CONFIG_${name}`, - }, - headers: {}, - }; - }); - - await scaleUpModule.scaleUp(TEST_DATA); - - expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-2', - Value: 'TEST_JIT_CONFIG_unit-test-i-instance-2', - Type: 'SecureString', - Tags: [{ Key: 'InstanceId', Value: 'i-instance-2' }], - }); - - expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-1', - }); - }); - - it.each(RUNNER_TYPES)( - 'calls create start runner config of 40' + ' instances (ssm rate limit condition) to test time delay ', - async (type: RunnerType) => { - process.env.ENABLE_EPHEMERAL_RUNNERS = type === 'ephemeral' ? 'true' : 'false'; - process.env.RUNNERS_MAXIMUM_COUNT = '40'; - mockCreateRunner.mockImplementation(async () => { - return instances; - }); - mockListRunners.mockImplementation(async () => { - return []; - }); - const startTime = performance.now(); - const instances = [ - 'i-1234', - 'i-5678', - 'i-5567', - 'i-5569', - 'i-5561', - 'i-5560', - 'i-5566', - 'i-5536', - 'i-5526', - 'i-5516', - 'i-122', - 'i-123', - 'i-124', - 'i-125', - 'i-126', - 'i-127', - 'i-128', - 'i-129', - 'i-130', - 'i-131', - 'i-132', - 'i-133', - 'i-134', - 'i-135', - 'i-136', - 'i-137', - 'i-138', - 'i-139', - 'i-140', - 'i-141', - 'i-142', - 'i-143', - 'i-144', - 'i-145', - 'i-146', - 'i-147', - 'i-148', - 'i-149', - 'i-150', - 'i-151', - ]; - await scaleUpModule.scaleUp(TEST_DATA); - const endTime = performance.now(); - expect(endTime - startTime).toBeGreaterThan(1000); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 40); - }, - 10000, - ); - }); - - describe('Dynamic EC2 Configuration', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - process.env.RUNNER_LABELS = 'base-label'; - process.env.INSTANCE_TYPES = 't3.medium,t3.large'; - process.env.RUNNER_NAME_PREFIX = 'unit-test'; - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - mockSSMClient.reset(); - }); - - it('appends EC2 labels to existing runner labels when EC2 labels are present', async () => { - const testDataWithEc2Labels = [ - { - ...TEST_DATA_SINGLE, - labels: ['ghr-ec2-instance-type:c5.2xlarge', 'ghr-ec2-custom:value'], - messageId: 'test-1', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithEc2Labels); - - // Verify createRunner was called with EC2 instance type in override config - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2instanceCriteria: expect.objectContaining({ - instanceTypes: ['t3.medium', 't3.large'], - }), - ec2OverrideConfig: expect.objectContaining({ - InstanceType: 'c5.2xlarge', - }), - }), - ); - expect(mockTag).toHaveBeenCalledWith('i-12345', [ - { - Key: 'ghr:github_runner_id', - Value: '9876543210', - }, - { - Key: 'ghr:runner_labels', - Value: 'base-label,ghr-ec2-instance-type:c5.2xlarge,ghr-ec2-custom:value', - }, - ]); - }); - - it('uses default instance types when no instance type EC2 label is provided', async () => { - const testDataWithEc2Labels = [ - { - ...TEST_DATA_SINGLE, - labels: ['ghr-ec2-custom:value'], - messageId: 'test-3', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithEc2Labels); - - // Should use the default INSTANCE_TYPES from environment - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2instanceCriteria: expect.objectContaining({ - instanceTypes: ['t3.medium', 't3.large'], - }), - }), - ); - }); - - it('loads the launch template block device name for dynamic EBS labels without DeviceName', async () => { - mockEC2Client.on(DescribeLaunchTemplateVersionsCommand).resolves({ - LaunchTemplateVersions: [ - { - LaunchTemplateData: { - BlockDeviceMappings: [ - { - DeviceName: '/dev/sdb', - VirtualName: 'ephemeral0', - }, - { - DeviceName: '/dev/sdf', - Ebs: {}, - }, - ], - }, - }, - ], - }); - - const testDataWithEbsLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['ghr-ec2-ebs-volume-size:100', 'ghr-ec2-ebs-volume-type:gp3'], - messageId: 'test-ebs-device-name', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithEbsLabels); - - expect(mockEC2Client).toHaveReceivedCommandWith(DescribeLaunchTemplateVersionsCommand, { - LaunchTemplateName: 'lt-1', - Versions: ['$Default'], - }); - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - BlockDeviceMappings: [ - { - DeviceName: '/dev/sdf', - Ebs: { - VolumeSize: 100, - VolumeType: 'gp3', - }, - }, - ], - }), - }), - ); - }); - - it('does not load launch template block device name when DeviceName is provided by labels', async () => { - const testDataWithEbsLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['ghr-ec2-block-device-name:/dev/sdg', 'ghr-ec2-ebs-volume-size:100', 'ghr-ec2-ebs-volume-type:gp3'], - messageId: 'test-explicit-ebs-device-name', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithEbsLabels); - - expect(mockEC2Client).not.toHaveReceivedCommand(DescribeLaunchTemplateVersionsCommand); - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - BlockDeviceMappings: [ - { - DeviceName: '/dev/sdg', - Ebs: { - VolumeSize: 100, - VolumeType: 'gp3', - }, - }, - ], - }), - }), - ); - }); - - it('handles messages with no labels gracefully', async () => { - const testDataWithNoLabels = [ - { - ...TEST_DATA_SINGLE, - labels: undefined, - messageId: 'test-5', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithNoLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2instanceCriteria: expect.objectContaining({ - instanceTypes: ['t3.medium', 't3.large'], - }), - }), - ); - }); - - it('handles empty labels array', async () => { - const testDataWithEmptyLabels = [ - { - ...TEST_DATA_SINGLE, - labels: [], - messageId: 'test-6', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithEmptyLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2instanceCriteria: expect.objectContaining({ - instanceTypes: ['t3.medium', 't3.large'], - }), - }), - ); - }); - - it('handles multiple EC2 labels correctly', async () => { - const testDataWithMultipleEc2Labels = [ - { - ...TEST_DATA_SINGLE, - labels: ['regular-label', 'ghr-ec2-instance-type:r5.2xlarge', 'ghr-ec2-ami:custom-ami', 'ghr-ec2-disk:200'], - messageId: 'test-8', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithMultipleEc2Labels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2instanceCriteria: expect.objectContaining({ - instanceTypes: ['t3.medium', 't3.large'], - }), - ec2OverrideConfig: expect.objectContaining({ - InstanceType: 'r5.2xlarge', - }), - }), - ); - }); - - it('includes ec2OverrideConfig with VCpuCount requirements when specified', async () => { - const testDataWithVCpuLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-vcpu-count-min:4', 'ghr-ec2-vcpu-count-max:16'], - messageId: 'test-9', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithVCpuLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - InstanceRequirements: expect.objectContaining({ - VCpuCount: { - Min: 4, - Max: 16, - }, - }), - }), - }), - ); - }); - - it('includes ec2OverrideConfig with MemoryMiB requirements when specified', async () => { - const testDataWithMemoryLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-memory-mib-min:8192', 'ghr-ec2-memory-mib-max:32768'], - messageId: 'test-10', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithMemoryLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - InstanceRequirements: expect.objectContaining({ - MemoryMiB: { - Min: 8192, - Max: 32768, - }, - }), - }), - }), - ); - }); - - it('includes ec2OverrideConfig with CPU manufacturers when specified', async () => { - const testDataWithCpuLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-cpu-manufacturers:intel;amd'], - messageId: 'test-11', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithCpuLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - InstanceRequirements: expect.objectContaining({ - CpuManufacturers: ['intel', 'amd'], - }), - }), - }), - ); - }); - - it('includes ec2OverrideConfig with instance generations when specified', async () => { - const testDataWithGenerationLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-instance-generations:current'], - messageId: 'test-12', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithGenerationLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - InstanceRequirements: expect.objectContaining({ - InstanceGenerations: ['current'], - }), - }), - }), - ); - }); - - it('includes ec2OverrideConfig with accelerator requirements when specified', async () => { - const testDataWithAcceleratorLabels = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-accelerator-count-min:1', 'ghr-ec2-accelerator-types:gpu'], - messageId: 'test-13', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithAcceleratorLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - InstanceRequirements: expect.objectContaining({ - AcceleratorCount: { - Min: 1, - }, - AcceleratorTypes: ['gpu'], - }), - }), - }), - ); - }); - - it('includes ec2OverrideConfig with max price when specified', async () => { - const testDataWithMaxPrice = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-max-price:0.50'], - messageId: 'test-14', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithMaxPrice); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - MaxPrice: '0.50', - }), - }), - ); - }); - - it('includes ec2OverrideConfig with priority and weighted capacity when specified', async () => { - const testDataWithPriorityWeight = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-priority:1', 'ghr-ec2-weighted-capacity:2'], - messageId: 'test-15', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithPriorityWeight); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - Priority: 1, - WeightedCapacity: 2, - }), - }), - ); - }); - - it('includes ec2OverrideConfig with combined requirements', async () => { - const testDataWithCombinedLabels = [ - { - ...TEST_DATA_SINGLE, - labels: [ - 'self-hosted', - 'linux', - 'ghr-ec2-vcpu-count-min:8', - 'ghr-ec2-memory-mib-min:16384', - 'ghr-ec2-cpu-manufacturers:intel', - 'ghr-ec2-instance-generations:current', - 'ghr-ec2-max-price:1.00', - ], - messageId: 'test-16', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithCombinedLabels); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2OverrideConfig: expect.objectContaining({ - InstanceRequirements: expect.objectContaining({ - VCpuCount: { Min: 8 }, - MemoryMiB: { Min: 16384 }, - CpuManufacturers: ['intel'], - InstanceGenerations: ['current'], - }), - MaxPrice: '1.00', - }), - }), - ); - }); - - it('includes both instance type and ec2OverrideConfig when both specified', async () => { - const testDataWithBoth = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-instance-type:c5.xlarge', 'ghr-ec2-vcpu-count-min:4'], - messageId: 'test-18', - }, - ]; - - await scaleUpModule.scaleUp(testDataWithBoth); - - expect(createRunner).toBeCalledWith( - expect.objectContaining({ - ec2instanceCriteria: expect.objectContaining({ - instanceTypes: ['t3.medium', 't3.large'], - }), - ec2OverrideConfig: expect.objectContaining({ - InstanceType: 'c5.xlarge', - InstanceRequirements: expect.objectContaining({ - VCpuCount: { Min: 4 }, - }), - }), - }), - ); - }); - - it('does not accumulate labels across groups when multiple messages have different dynamic labels', async () => { - const testDataMultipleGroups = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:m7a.large', 'ghr-job-id:run-1-inst-0'], - messageId: 'msg-1', - }, - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:m7i.xlarge', 'ghr-job-id:run-1-inst-1'], - messageId: 'msg-2', - }, - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:c7a.large', 'ghr-job-id:run-1-inst-2'], - messageId: 'msg-3', - }, - ]; - - await scaleUpModule.scaleUp(testDataMultipleGroups); - - expect(createRunner).toBeCalledTimes(3); - - const jitCalls = mockOctokit.actions.generateRunnerJitconfigForOrg.mock.calls; - expect(jitCalls).toHaveLength(3); - - for (const call of jitCalls) { - const labels = call[0].labels as string[]; - - if (labels.includes('ghr-ec2-instance-type:m7a.large')) { - expect(labels).toContain('ghr-job-id:run-1-inst-0'); - expect(labels).not.toContain('ghr-job-id:run-1-inst-1'); - expect(labels).not.toContain('ghr-job-id:run-1-inst-2'); - expect(labels).not.toContain('ghr-ec2-instance-type:m7i.xlarge'); - expect(labels).not.toContain('ghr-ec2-instance-type:c7a.large'); - } else if (labels.includes('ghr-ec2-instance-type:m7i.xlarge')) { - expect(labels).toContain('ghr-job-id:run-1-inst-1'); - expect(labels).not.toContain('ghr-job-id:run-1-inst-0'); - expect(labels).not.toContain('ghr-job-id:run-1-inst-2'); - expect(labels).not.toContain('ghr-ec2-instance-type:m7a.large'); - expect(labels).not.toContain('ghr-ec2-instance-type:c7a.large'); - } else if (labels.includes('ghr-ec2-instance-type:c7a.large')) { - expect(labels).toContain('ghr-job-id:run-1-inst-2'); - expect(labels).not.toContain('ghr-job-id:run-1-inst-0'); - expect(labels).not.toContain('ghr-job-id:run-1-inst-1'); - expect(labels).not.toContain('ghr-ec2-instance-type:m7a.large'); - expect(labels).not.toContain('ghr-ec2-instance-type:m7i.xlarge'); - } else { - throw new Error(`Unexpected labels combination: ${labels.join(',')}`); - } - } - }); - - it('preserves base RUNNER_LABELS for each group without mutation', async () => { - process.env.RUNNER_LABELS = 'ubuntu-2404,x64'; - - const testDataTwoGroups = [ - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-instance-type:m7a.large', 'ghr-team:alpha'], - messageId: 'msg-a', - }, - { - ...TEST_DATA_SINGLE, - labels: ['self-hosted', 'ghr-ec2-instance-type:c7i.large', 'ghr-team:beta'], - messageId: 'msg-b', - }, - ]; - - await scaleUpModule.scaleUp(testDataTwoGroups); - - expect(createRunner).toBeCalledTimes(2); - - const jitCalls = mockOctokit.actions.generateRunnerJitconfigForOrg.mock.calls; - expect(jitCalls).toHaveLength(2); - - for (const call of jitCalls) { - const labels = call[0].labels as string[]; - - expect(labels).toContain('ubuntu-2404'); - expect(labels).toContain('x64'); - - if (labels.includes('ghr-team:alpha')) { - expect(labels).not.toContain('ghr-team:beta'); - expect(labels).not.toContain('ghr-ec2-instance-type:c7i.large'); - } else if (labels.includes('ghr-team:beta')) { - expect(labels).not.toContain('ghr-team:alpha'); - expect(labels).not.toContain('ghr-ec2-instance-type:m7a.large'); - } else { - throw new Error(`Unexpected labels combination: ${labels.join(',')}`); - } - } - }); - }); - - describe('on repo level', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; - process.env.RUNNER_NAME_PREFIX = 'unit-test'; - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - expectedRunnerParams.runnerType = 'Repo'; - expectedRunnerParams.runnerOwner = `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`; - // `--url https://github.enterprise.something/${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, - // `--token 1234abcd`, - // ]; - }); - - it('gets the current repo level runners', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).toBeCalledWith({ - environment: 'unit-test-environment', - runnerType: 'Repo', - runnerOwner: `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, - }); - }); - - it('does not create a token when maximum runners has been reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a token when maximum runners has not been reached', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('uses the default runner max count', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = undefined; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('creates a runner with correct config and labels', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner and ensure the group argument is ignored', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('Check error is thrown', async () => { - const mockCreateRunners = vi.mocked(createRunner); - mockCreateRunners.mockRejectedValue(new Error('no retry')); - await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow('no retry'); - mockCreateRunners.mockReset(); - }); - }); - - describe('Batch processing', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.RUNNERS_MAXIMUM_COUNT = '10'; - }); - - const createTestMessages = ( - count: number, - overrides: Partial[] = [], - ): ActionRequestMessageSQS[] => { - return Array.from({ length: count }, (_, i) => ({ - ...TEST_DATA_SINGLE, - id: i + 1, - messageId: `message-${i}`, - ...overrides[i], - })); - }; - - it('Should handle multiple messages for the same organization', async () => { - const messages = createTestMessages(3); - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(1); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 3, - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }), - ); - }); - - it('Should handle multiple messages for different organizations', async () => { - const messages = createTestMessages(3, [ - { repositoryOwner: 'org1' }, - { repositoryOwner: 'org2' }, - { repositoryOwner: 'org1' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(2); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, - runnerOwner: 'org1', - }), - ); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, - runnerOwner: 'org2', - }), - ); - }); - - it('Should handle multiple messages for different repositories when org-level is disabled', async () => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; - const messages = createTestMessages(3, [ - { repositoryOwner: 'owner1', repositoryName: 'repo1' }, - { repositoryOwner: 'owner1', repositoryName: 'repo2' }, - { repositoryOwner: 'owner1', repositoryName: 'repo1' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(2); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, - runnerOwner: 'owner1/repo1', - }), - ); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, - runnerOwner: 'owner1/repo2', - }), - ); - }); - - it('Should reject messages when maximum runners limit is reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; // Set to 1 so with 1 existing, no new ones can be created - mockListRunners.mockImplementation(async () => [ - { - instanceId: 'i-existing', - launchTime: new Date(), - type: 'Org', - owner: TEST_DATA_SINGLE.repositoryOwner, - }, - ]); - - const messages = createTestMessages(3); - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).not.toHaveBeenCalled(); // No runners should be created - expect(rejectedMessages).toHaveLength(3); // All 3 messages should be rejected - }); - - it('Should handle partial EC2 instance creation failures', async () => { - mockCreateRunner.mockImplementation(async () => ['i-12345']); // Only creates 1 instead of requested 3 - - const messages = createTestMessages(3); - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(rejectedMessages).toHaveLength(2); // 3 requested - 1 created = 2 failed - expect(rejectedMessages).toEqual(['message-0', 'message-1']); - }); - - it('Should filter out invalid event types for ephemeral runners', async () => { - const messages = createTestMessages(3, [ - { eventType: 'workflow_job' }, - { eventType: 'check_run' }, - { eventType: 'workflow_job' }, - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only workflow_job events processed - }), - ); - expect(rejectedMessages).toContain('message-1'); // check_run event rejected - }); - - it('Should skip invalid repo owner types but not reject them', async () => { - const messages = createTestMessages(3, [ - { repoOwnerType: 'Organization' }, - { repoOwnerType: 'User' }, // Invalid for org-level runners - { repoOwnerType: 'Organization' }, - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only Organization events processed - }), - ); - expect(rejectedMessages).not.toContain('message-1'); // User repo not rejected, just skipped - }); - - it('Should skip messages when jobs are not queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { - const isQueued = params.job_id === 1 || params.job_id === 3; // Only jobs 1 and 3 are queued - return { - data: { - status: isQueued ? 'queued' : 'completed', - }, - }; - }); - - const messages = createTestMessages(3); - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only queued jobs processed - }), - ); - }); - - it('Should create separate GitHub clients for different installations', async () => { - // Override the default mock to return different installation IDs - mockOctokit.apps.getOrgInstallation.mockReset(); - mockOctokit.apps.getOrgInstallation.mockImplementation((params) => ({ - data: { - id: params.org === 'org1' ? 100 : 200, - }, - })); - - const messages = createTestMessages(2, [ - { repositoryOwner: 'org1', installationId: 0 }, - { repositoryOwner: 'org2', installationId: 0 }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients - expect(mockedInstallationAuth).toHaveBeenCalledWith(100, 'https://github.enterprise.something/api/v3'); - expect(mockedInstallationAuth).toHaveBeenCalledWith(200, 'https://github.enterprise.something/api/v3'); - }); - - it('Should reuse GitHub clients for same installation', async () => { - const messages = createTestMessages(3, [ - { repositoryOwner: 'same-org' }, - { repositoryOwner: 'same-org' }, - { repositoryOwner: 'same-org' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(mockCreateClient).toHaveBeenCalledTimes(2); // 1 app client, 1 installation client - expect(mockedInstallationAuth).toHaveBeenCalledTimes(1); - }); - - it('Should return empty array when no valid messages to process', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - const messages = createTestMessages(2, [ - { eventType: 'check_run' }, // Invalid for ephemeral - { eventType: 'check_run' }, // Invalid for ephemeral - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).not.toHaveBeenCalled(); - expect(rejectedMessages).toEqual(['message-0', 'message-1']); - }); - - it('Should handle unlimited runners configuration', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '-1'; - const messages = createTestMessages(10); - - await scaleUpModule.scaleUp(messages); - - expect(listEC2Runners).not.toHaveBeenCalled(); // No need to check current runners - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 10, // All messages processed - }), - ); - }); - }); -}); - -describe('scaleUp with public GH', () => { - it('checks queued workflows', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalledWith({ - job_id: TEST_DATA_SINGLE.id, - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('not checking queued workflows', async () => { - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); - }); - - it('does not list runners when no workflows are queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ - data: { status: 'completed' }, - })); - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).not.toBeCalled(); - }); - - describe('on org level', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.RUNNER_NAME_PREFIX = 'unit-test'; - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - }); - - it('gets the current org level runners', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).toBeCalledWith({ - environment: 'unit-test-environment', - runnerType: 'Org', - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }); - }); - - it('does not create a token when maximum runners has been reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a token when maximum runners has not been reached', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - }); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a runner with correct config', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner with labels in s specific group', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - }); - - describe('on repo level', () => { - beforeEach(() => { - mockSSMClient.reset(); - - process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; - process.env.RUNNER_NAME_PREFIX = 'unit-test'; - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - expectedRunnerParams.runnerType = 'Repo'; - expectedRunnerParams.runnerOwner = `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`; - }); - - it('gets the current repo level runners', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).toBeCalledWith({ - environment: 'unit-test-environment', - runnerType: 'Repo', - runnerOwner: `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, - }); - }); - - it('does not create a token when maximum runners has been reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a token when maximum runners has not been reached', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('creates a runner with correct config and labels', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner with correct config and labels and on demand failover enabled.', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = JSON.stringify(['InsufficientInstanceCapacity']); - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith({ - ...expectedRunnerParams, - onDemandFailoverOnError: ['InsufficientInstanceCapacity'], - }); - }); - - it('creates a runner with correct config and labels and custom scale errors enabled.', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.SCALE_ERRORS = JSON.stringify(['RequestLimitExceeded']); - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith({ - ...expectedRunnerParams, - scaleErrors: ['RequestLimitExceeded'], - }); - }); - - it('creates a runner and ensure the group argument is ignored', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('ephemeral runners only run with workflow_job event, others should fail.', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - - const USER_REPO_TEST_DATA = structuredClone(TEST_DATA); - USER_REPO_TEST_DATA[0].eventType = 'check_run'; - - await expect(scaleUpModule.scaleUp(USER_REPO_TEST_DATA)).resolves.toEqual(['foobar']); - }); - - it('creates a ephemeral runner with JIT config.', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: 'TEST_JIT_CONFIG_REPO', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('creates a ephemeral runner with registration token.', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.ENABLE_JIT_CONFIG = 'false'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --ephemeral', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('JIT config is ignored for non-ephemeral runners.', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - process.env.ENABLE_JIT_CONFIG = 'true'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - process.env.RUNNER_LABELS = 'jit'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --labels jit', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('creates a ephemeral runner after checking job is queued.', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'true'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalled(); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('disable auto update on the runner.', async () => { - process.env.DISABLE_RUNNER_AUTOUPDATE = 'true'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('Scaling error should return failed message IDs so retry can be triggered.', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - await expect(scaleUpModule.scaleUp(TEST_DATA)).resolves.toEqual(['foobar']); - }); - }); - - describe('Batch processing', () => { - const createTestMessages = ( - count: number, - overrides: Partial[] = [], - ): ActionRequestMessageSQS[] => { - return Array.from({ length: count }, (_, i) => ({ - ...TEST_DATA_SINGLE, - id: i + 1, - messageId: `message-${i}`, - ...overrides[i], - })); - }; - - beforeEach(() => { - setDefaults(); - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.RUNNERS_MAXIMUM_COUNT = '10'; - }); - - it('Should handle multiple messages for the same organization', async () => { - const messages = createTestMessages(3); - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(1); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 3, - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }), - ); - }); - - it('Should handle multiple messages for different organizations', async () => { - const messages = createTestMessages(3, [ - { repositoryOwner: 'org1' }, - { repositoryOwner: 'org2' }, - { repositoryOwner: 'org1' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(2); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, - runnerOwner: 'org1', - }), - ); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, - runnerOwner: 'org2', - }), - ); - }); - - it('Should handle multiple messages for different repositories when org-level is disabled', async () => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; - const messages = createTestMessages(3, [ - { repositoryOwner: 'owner1', repositoryName: 'repo1' }, - { repositoryOwner: 'owner1', repositoryName: 'repo2' }, - { repositoryOwner: 'owner1', repositoryName: 'repo1' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(2); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, - runnerOwner: 'owner1/repo1', - }), - ); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, - runnerOwner: 'owner1/repo2', - }), - ); - }); - - it('Should reject messages when maximum runners limit is reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; // Set to 1 so with 1 existing, no new ones can be created - mockListRunners.mockImplementation(async () => [ - { - instanceId: 'i-existing', - launchTime: new Date(), - type: 'Org', - owner: TEST_DATA_SINGLE.repositoryOwner, - }, - ]); - - const messages = createTestMessages(3); - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).not.toHaveBeenCalled(); // No runners should be created - expect(rejectedMessages).toHaveLength(3); // All 3 messages should be rejected - }); - - it('Should handle partial EC2 instance creation failures', async () => { - mockCreateRunner.mockImplementation(async () => ['i-12345']); // Only creates 1 instead of requested 3 - - const messages = createTestMessages(3); - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(rejectedMessages).toHaveLength(2); // 3 requested - 1 created = 2 failed - expect(rejectedMessages).toEqual(['message-0', 'message-1']); - }); - - it('Should filter out invalid event types for ephemeral runners', async () => { - const messages = createTestMessages(3, [ - { eventType: 'workflow_job' }, - { eventType: 'check_run' }, - { eventType: 'workflow_job' }, - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only workflow_job events processed - }), - ); - expect(rejectedMessages).toContain('message-1'); // check_run event rejected - }); - - it('Should skip invalid repo owner types but not reject them', async () => { - const messages = createTestMessages(3, [ - { repoOwnerType: 'Organization' }, - { repoOwnerType: 'User' }, // Invalid for org-level runners - { repoOwnerType: 'Organization' }, - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only Organization events processed - }), - ); - expect(rejectedMessages).not.toContain('message-1'); // User repo not rejected, just skipped - }); - - it('Should skip messages when jobs are not queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { - const isQueued = params.job_id === 1 || params.job_id === 3; // Only jobs 1 and 3 are queued - return { - data: { - status: isQueued ? 'queued' : 'completed', - }, - }; - }); - - const messages = createTestMessages(3); - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only queued jobs processed - }), - ); - }); - - it('Should create separate GitHub clients for different installations', async () => { - // Override the default mock to return different installation IDs - mockOctokit.apps.getOrgInstallation.mockReset(); - mockOctokit.apps.getOrgInstallation.mockImplementation((params) => ({ - data: { - id: params.org === 'org1' ? 100 : 200, - }, - })); - - const messages = createTestMessages(2, [ - { repositoryOwner: 'org1', installationId: 0 }, - { repositoryOwner: 'org2', installationId: 0 }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients - expect(mockedInstallationAuth).toHaveBeenCalledWith(100, ''); - expect(mockedInstallationAuth).toHaveBeenCalledWith(200, ''); - }); - - it('Should reuse GitHub clients for same installation', async () => { - const messages = createTestMessages(3, [ - { repositoryOwner: 'same-org' }, - { repositoryOwner: 'same-org' }, - { repositoryOwner: 'same-org' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(mockCreateClient).toHaveBeenCalledTimes(2); // 1 app client, 1 installation client - expect(mockedInstallationAuth).toHaveBeenCalledTimes(1); - }); - - it('Should return empty array when no valid messages to process', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - const messages = createTestMessages(2, [ - { eventType: 'check_run' }, // Invalid for ephemeral - { eventType: 'check_run' }, // Invalid for ephemeral - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).not.toHaveBeenCalled(); - expect(rejectedMessages).toEqual(['message-0', 'message-1']); - }); - - it('Should handle unlimited runners configuration', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '-1'; - const messages = createTestMessages(10); - - await scaleUpModule.scaleUp(messages); - - expect(listEC2Runners).not.toHaveBeenCalled(); // No need to check current runners - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 10, // All messages processed - }), - ); - }); - }); -}); - -describe('scaleUp with Github Data Residency', () => { - beforeEach(() => { - process.env.GHES_URL = 'https://companyname.ghe.com'; - }); - - it('checks queued workflows', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalledWith({ - job_id: TEST_DATA_SINGLE.id, - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('does not list runners when no workflows are queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ - data: { total_count: 0 }, - })); - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).not.toBeCalled(); - }); - - describe('on org level', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.RUNNER_NAME_PREFIX = 'unit-test-'; - process.env.RUNNER_GROUP_NAME = 'Default'; - process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; - process.env.RUNNER_LABELS = 'label1,label2'; - - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - mockSSMClient.reset(); - }); - - it('gets the current org level runners', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).toBeCalledWith({ - environment: 'unit-test-environment', - runnerType: 'Org', - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }); - }); - - it('does not create a token when maximum runners has been reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('does create a runner if maximum is set to -1', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '-1'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).not.toHaveBeenCalled(); - expect(createRunner).toHaveBeenCalled(); - }); - - it('creates a token when maximum runners has not been reached', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - }); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a runner with correct config', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner with labels in a specific group', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner with ami id override from ssm parameter', async () => { - process.env.AMI_ID_SSM_PARAMETER_NAME = 'my-ami-id-param'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith({ ...expectedRunnerParams, amiIdSsmParameterName: 'my-ami-id-param' }); - }); - - it('Throws an error if runner group does not exist for ephemeral runners', async () => { - process.env.RUNNER_GROUP_NAME = 'test-runner-group'; - mockSSMgetParameter.mockImplementation(async () => { - throw new Error('ParameterNotFound'); - }); - await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toBeInstanceOf(Error); - expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); - }); - - it('Discards event if it is a User repo and org level runners is enabled', async () => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - const USER_REPO_TEST_DATA = structuredClone(TEST_DATA); - USER_REPO_TEST_DATA[0].repoOwnerType = 'User'; - await scaleUpModule.scaleUp(USER_REPO_TEST_DATA); - expect(createRunner).not.toHaveBeenCalled(); - }); - - it('create SSM parameter for runner group id if it does not exist', async () => { - mockSSMgetParameter.mockImplementation(async () => { - throw new Error('ParameterNotFound'); - }); - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 2); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`, - Value: '1', - Type: 'String', - }); - }); - - it('Does not create SSM parameter for runner group id if it exists', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.paginate).toHaveBeenCalledTimes(0); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 1); - }); - - it('create start runner config for ephemeral runners ', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '2'; - - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toBeCalledWith({ - org: TEST_DATA_SINGLE.repositoryOwner, - name: 'unit-test-i-12345', - runner_group_id: 1, - labels: ['label1', 'label2'], - }); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: 'TEST_JIT_CONFIG_ORG', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - - it('create start runner config for non-ephemeral runners ', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - process.env.RUNNERS_MAXIMUM_COUNT = '2'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.generateRunnerJitconfigForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalled(); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: - '--url https://companyname.ghe.com/Codertocat --token 1234abcd ' + - '--labels label1,label2 --runnergroup Default', - Type: 'SecureString', - Tags: [ - { - Key: 'InstanceId', - Value: 'i-12345', - }, - ], - }); - }); - it.each(RUNNER_TYPES)( - 'calls create start runner config of 40' + ' instances (ssm rate limit condition) to test time delay ', - async (type: RunnerType) => { - process.env.ENABLE_EPHEMERAL_RUNNERS = type === 'ephemeral' ? 'true' : 'false'; - process.env.RUNNERS_MAXIMUM_COUNT = '40'; - mockCreateRunner.mockImplementation(async () => { - return instances; - }); - mockListRunners.mockImplementation(async () => { - return []; - }); - const startTime = performance.now(); - const instances = [ - 'i-1234', - 'i-5678', - 'i-5567', - 'i-5569', - 'i-5561', - 'i-5560', - 'i-5566', - 'i-5536', - 'i-5526', - 'i-5516', - 'i-122', - 'i-123', - 'i-124', - 'i-125', - 'i-126', - 'i-127', - 'i-128', - 'i-129', - 'i-130', - 'i-131', - 'i-132', - 'i-133', - 'i-134', - 'i-135', - 'i-136', - 'i-137', - 'i-138', - 'i-139', - 'i-140', - 'i-141', - 'i-142', - 'i-143', - 'i-144', - 'i-145', - 'i-146', - 'i-147', - 'i-148', - 'i-149', - 'i-150', - 'i-151', - ]; - await scaleUpModule.scaleUp(TEST_DATA); - const endTime = performance.now(); - expect(endTime - startTime).toBeGreaterThan(1000); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 40); - }, - 10000, - ); - }); - describe('on repo level', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; - process.env.RUNNER_NAME_PREFIX = 'unit-test'; - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - expectedRunnerParams.runnerType = 'Repo'; - expectedRunnerParams.runnerOwner = `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`; - // `--url https://companyname.ghe.com${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, - // `--token 1234abcd`, - // ]; - }); - - it('gets the current repo level runners', async () => { - await scaleUpModule.scaleUp(TEST_DATA); - expect(listEC2Runners).toBeCalledWith({ - environment: 'unit-test-environment', - runnerType: 'Repo', - runnerOwner: `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, - }); - }); - - it('does not create a token when maximum runners has been reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '1'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); - }); - - it('creates a token when maximum runners has not been reached', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); - expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('uses the default runner max count', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = undefined; - await scaleUpModule.scaleUp(TEST_DATA); - expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ - owner: TEST_DATA_SINGLE.repositoryOwner, - repo: TEST_DATA_SINGLE.repositoryName, - }); - }); - - it('creates a runner with correct config and labels', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('creates a runner and ensure the group argument is ignored', async () => { - process.env.RUNNER_LABELS = 'label1,label2'; - process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toBeCalledWith(expectedRunnerParams); - }); - - it('Check error is thrown', async () => { - const mockCreateRunners = vi.mocked(createRunner); - mockCreateRunners.mockRejectedValue(new Error('no retry')); - await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow('no retry'); - mockCreateRunners.mockReset(); - }); - }); - - describe('Batch processing', () => { - const createTestMessages = ( - count: number, - overrides: Partial[] = [], - ): ActionRequestMessageSQS[] => { - return Array.from({ length: count }, (_, i) => ({ - ...TEST_DATA_SINGLE, - id: i + 1, - messageId: `message-${i}`, - ...overrides[i], - })); - }; - - beforeEach(() => { - setDefaults(); - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.RUNNERS_MAXIMUM_COUNT = '10'; - }); - - it('Should handle multiple messages for the same organization', async () => { - const messages = createTestMessages(3); - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(1); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 3, - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }), - ); - }); - - it('Should handle multiple messages for different organizations', async () => { - const messages = createTestMessages(3, [ - { repositoryOwner: 'org1' }, - { repositoryOwner: 'org2' }, - { repositoryOwner: 'org1' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(2); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, - runnerOwner: 'org1', - }), - ); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, - runnerOwner: 'org2', - }), - ); - }); - - it('Should handle multiple messages for different repositories when org-level is disabled', async () => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; - const messages = createTestMessages(3, [ - { repositoryOwner: 'owner1', repositoryName: 'repo1' }, - { repositoryOwner: 'owner1', repositoryName: 'repo2' }, - { repositoryOwner: 'owner1', repositoryName: 'repo1' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledTimes(2); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, - runnerOwner: 'owner1/repo1', - }), - ); - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, - runnerOwner: 'owner1/repo2', - }), - ); - }); - - it('Should reject messages when maximum runners limit is reached', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '2'; - mockListRunners.mockImplementation(async () => [ - { - instanceId: 'i-existing', - launchTime: new Date(), - type: 'Org', - owner: TEST_DATA_SINGLE.repositoryOwner, - }, - ]); - - const messages = createTestMessages(5); - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 1, // 2 max - 1 existing = 1 new - }), - ); - expect(rejectedMessages).toHaveLength(4); // 5 requested - 1 created = 4 rejected - }); - - it('Should handle partial EC2 instance creation failures', async () => { - mockCreateRunner.mockImplementation(async () => ['i-12345']); // Only creates 1 instead of requested 3 - - const messages = createTestMessages(3); - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(rejectedMessages).toHaveLength(2); // 3 requested - 1 created = 2 failed - expect(rejectedMessages).toEqual(['message-0', 'message-1']); - }); - - it('Should filter out invalid event types for ephemeral runners', async () => { - const messages = createTestMessages(3, [ - { eventType: 'workflow_job' }, - { eventType: 'check_run' }, - { eventType: 'workflow_job' }, - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only workflow_job events processed - }), - ); - expect(rejectedMessages).toContain('message-1'); // check_run event rejected - }); - - it('Should skip invalid repo owner types but not reject them', async () => { - const messages = createTestMessages(3, [ - { repoOwnerType: 'Organization' }, - { repoOwnerType: 'User' }, // Invalid for org-level runners - { repoOwnerType: 'Organization' }, - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only Organization events processed - }), - ); - expect(rejectedMessages).not.toContain('message-1'); // User repo not rejected, just skipped - }); - - it('Should skip messages when jobs are not queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { - const isQueued = params.job_id === 1 || params.job_id === 3; // Only jobs 1 and 3 are queued - return { - data: { - status: isQueued ? 'queued' : 'completed', - }, - }; - }); - - const messages = createTestMessages(3); - await scaleUpModule.scaleUp(messages); - - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 2, // Only queued jobs processed - }), - ); - }); - - it('Should create separate GitHub clients for different installations', async () => { - mockOctokit.apps.getOrgInstallation.mockImplementation((params) => ({ - data: { - id: params.org === 'org1' ? 100 : 200, - }, - })); - - const messages = createTestMessages(2, [ - { repositoryOwner: 'org1', installationId: 0 }, - { repositoryOwner: 'org2', installationId: 0 }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients - expect(mockedInstallationAuth).toHaveBeenCalledWith(100, ''); - expect(mockedInstallationAuth).toHaveBeenCalledWith(200, ''); - }); - - it('Should reuse GitHub clients for same installation', async () => { - const messages = createTestMessages(3, [ - { repositoryOwner: 'same-org' }, - { repositoryOwner: 'same-org' }, - { repositoryOwner: 'same-org' }, - ]); - - await scaleUpModule.scaleUp(messages); - - expect(mockCreateClient).toHaveBeenCalledTimes(2); // 1 app client, 1 installation client - expect(mockedInstallationAuth).toHaveBeenCalledTimes(1); - }); - - it('Should return empty array when no valid messages to process', async () => { - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - const messages = createTestMessages(2, [ - { eventType: 'check_run' }, // Invalid for ephemeral - { eventType: 'check_run' }, // Invalid for ephemeral - ]); - - const rejectedMessages = await scaleUpModule.scaleUp(messages); - - expect(createRunner).not.toHaveBeenCalled(); - expect(rejectedMessages).toEqual(['message-0', 'message-1']); - }); - - it('Should handle unlimited runners configuration', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '-1'; - const messages = createTestMessages(10); - - await scaleUpModule.scaleUp(messages); - - expect(listEC2Runners).not.toHaveBeenCalled(); // No need to check current runners - expect(createRunner).toHaveBeenCalledWith( - expect.objectContaining({ - numberOfRunners: 10, // All messages processed - }), - ); - }); - }); -}); - -describe('Retry mechanism tests', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.ENABLE_JOB_QUEUED_CHECK = 'true'; - process.env.RUNNERS_MAXIMUM_COUNT = '10'; - expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - mockSSMClient.reset(); - }); - - const createTestMessages = ( - count: number, - overrides: Partial[] = [], - ): ActionRequestMessageSQS[] => { - return Array.from({ length: count }, (_, i) => ({ - ...TEST_DATA_SINGLE, - id: i + 1, - messageId: `message-${i + 1}`, - ...overrides[i], - })); - }; - - it('calls publishRetryMessage for each valid message when job is queued', async () => { - const messages = createTestMessages(3); - mockCreateRunner.mockResolvedValue(['i-12345', 'i-67890', 'i-abcdef']); // Create all requested runners - - await scaleUpModule.scaleUp(messages); - - expect(mockPublishRetryMessage).toHaveBeenCalledTimes(3); - expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ - id: 1, - messageId: 'message-1', - }), - ); - expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ - id: 2, - messageId: 'message-2', - }), - ); - expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( - 3, - expect.objectContaining({ - id: 3, - messageId: 'message-3', - }), - ); - }); - - it('does not call publishRetryMessage when job is not queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { - const isQueued = params.job_id === 1; // Only job 1 is queued - return { - data: { - status: isQueued ? 'queued' : 'completed', - }, - }; - }); - - const messages = createTestMessages(3); - - await scaleUpModule.scaleUp(messages); - - // Only message with id 1 should trigger retry - expect(mockPublishRetryMessage).toHaveBeenCalledTimes(1); - expect(mockPublishRetryMessage).toHaveBeenCalledWith( - expect.objectContaining({ - id: 1, - messageId: 'message-1', - }), - ); - }); - - it('does not call publishRetryMessage when maximum runners is reached and messages are marked invalid', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '0'; // No runners can be created - - const messages = createTestMessages(2); - - await scaleUpModule.scaleUp(messages); - - // Verify listEC2Runners is called to check current runner count - expect(listEC2Runners).toHaveBeenCalledWith({ - environment: 'unit-test-environment', - runnerType: 'Org', - runnerOwner: TEST_DATA_SINGLE.repositoryOwner, - }); - - // publishRetryMessage should NOT be called because messages are marked as invalid - // Invalid messages go back to the SQS queue and will be retried there - expect(mockPublishRetryMessage).not.toHaveBeenCalled(); - expect(createRunner).not.toHaveBeenCalled(); - }); - - it('calls publishRetryMessage with correct message structure including retry counter', async () => { - const message = { - ...TEST_DATA_SINGLE, - messageId: 'test-message-id', - retryCounter: 2, - }; - - await scaleUpModule.scaleUp([message]); - - expect(mockPublishRetryMessage).toHaveBeenCalledWith( - expect.objectContaining({ - id: message.id, - messageId: 'test-message-id', - retryCounter: 2, - }), - ); - }); - - it('calls publishRetryMessage when ENABLE_JOB_QUEUED_CHECK is false', async () => { - process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - mockCreateRunner.mockResolvedValue(['i-12345', 'i-67890']); // Create all requested runners - - const messages = createTestMessages(2); - - await scaleUpModule.scaleUp(messages); - - // Should always call publishRetryMessage when queue check is disabled - expect(mockPublishRetryMessage).toHaveBeenCalledTimes(2); - expect(mockOctokit.actions.getJobForWorkflowRun).not.toHaveBeenCalled(); - }); - - it('calls publishRetryMessage for each message in a multi-runner scenario', async () => { - mockCreateRunner.mockResolvedValue(['i-12345', 'i-67890', 'i-abcdef', 'i-11111', 'i-22222']); // Create all requested runners - const messages = createTestMessages(5); - - await scaleUpModule.scaleUp(messages); - - expect(mockPublishRetryMessage).toHaveBeenCalledTimes(5); - messages.forEach((msg, index) => { - expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( - index + 1, - expect.objectContaining({ - id: msg.id, - messageId: msg.messageId, - }), - ); - }); - }); - - it('calls publishRetryMessage after runner creation', async () => { - const messages = createTestMessages(1); - mockCreateRunner.mockResolvedValue(['i-12345']); // Create the requested runner - - const callOrder: string[] = []; - mockPublishRetryMessage.mockImplementation(() => { - callOrder.push('publishRetryMessage'); - return Promise.resolve(); - }); - mockCreateRunner.mockImplementation(async () => { - callOrder.push('createRunner'); - return ['i-12345']; - }); - - await scaleUpModule.scaleUp(messages); - - expect(callOrder).toEqual(['createRunner', 'publishRetryMessage']); - }); -}); - -describe('useDedicatedHost', () => { - beforeEach(() => { - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; - process.env.RUNNER_NAME_PREFIX = 'unit-test-'; - process.env.RUNNER_GROUP_NAME = 'Default'; - process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; - process.env.RUNNER_LABELS = 'label1,label2'; - }); - - it('defaults to false when USE_DEDICATED_HOST env var is not set', async () => { - delete process.env.USE_DEDICATED_HOST; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toHaveBeenCalledWith(expect.objectContaining({ useDedicatedHost: false })); - }); - - it('is true when USE_DEDICATED_HOST is "true"', async () => { - process.env.USE_DEDICATED_HOST = 'true'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toHaveBeenCalledWith(expect.objectContaining({ useDedicatedHost: true })); - }); - - it('is false when USE_DEDICATED_HOST is "false"', async () => { - process.env.USE_DEDICATED_HOST = 'false'; - await scaleUpModule.scaleUp(TEST_DATA); - expect(createRunner).toHaveBeenCalledWith(expect.objectContaining({ useDedicatedHost: false })); - }); -}); - -function defaultOctokitMockImpl() { - mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ - data: { - status: 'queued', - }, - })); - mockOctokit.paginate.mockImplementation(() => [ - { - id: 1, - name: 'Default', - }, - ]); - mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ labels }: { labels: string[] }) => ({ - data: { - runner: { id: 9876543210, labels: labels.map((name: string) => ({ name })) }, - encoded_jit_config: 'TEST_JIT_CONFIG_ORG', - }, - })); - mockOctokit.actions.generateRunnerJitconfigForRepo.mockImplementation(({ labels }: { labels: string[] }) => ({ - data: { - runner: { id: 9876543210, labels: labels.map((name: string) => ({ name })) }, - encoded_jit_config: 'TEST_JIT_CONFIG_REPO', - }, - })); - mockOctokit.checks.get.mockImplementation(() => ({ - data: { - status: 'queued', - }, - })); - - const mockTokenReturnValue = { - data: { - token: '1234abcd', - }, - }; - const mockInstallationIdReturnValueOrgs = { - data: { - id: TEST_DATA_SINGLE.installationId, - }, - }; - const mockInstallationIdReturnValueRepos = { - data: { - id: TEST_DATA_SINGLE.installationId, - }, - }; - - mockOctokit.actions.createRegistrationTokenForOrg.mockImplementation(() => mockTokenReturnValue); - mockOctokit.actions.createRegistrationTokenForRepo.mockImplementation(() => mockTokenReturnValue); - mockOctokit.apps.getOrgInstallation.mockImplementation(() => mockInstallationIdReturnValueOrgs); - mockOctokit.apps.getRepoInstallation.mockImplementation(() => mockInstallationIdReturnValueRepos); -} - -function defaultSSMGetParameterMockImpl() { - mockSSMgetParameter.mockImplementation(async (name: string) => { - if (name === `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`) { - return '1'; - } else if (name === `${process.env.PARAMETER_GITHUB_APP_ID_NAME}`) { - return `${process.env.GITHUB_APP_ID}`; - } else { - throw new Error(`ParameterNotFound: ${name}`); - } - }); -} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts index a82f971b1a..beda1af4e7 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts @@ -1,21 +1,14 @@ +import { Octokit } from '@octokit/rest'; +import { RequestError } from '@octokit/request-error'; import moment from 'moment'; -import { describe, it, expect, beforeEach, vi } from 'vitest'; +import nock from 'nock'; +import { RunnerInfo, RunnerList } from '../aws/ec2-runners.d'; import * as ghAuth from '../github/auth'; +import { listEC2Runners, terminateRunner, tag, untag } from './../aws/ec2-runners'; import { githubCache } from './cache'; import { newestFirstStrategy, oldestFirstStrategy, scaleDown } from './scale-down'; -import { createScaleDownRunnerProvider, getDefaultScaleDownRunnerProviderType } from './scale-down-provider-registry'; -import type { Octokit } from '@octokit/rest'; -import type { RunnerInfo, RunnerList } from './scale-down-provider'; - -const testProvider = vi.hoisted(() => ({ - name: 'TestProvider', - list: vi.fn(), - bootTimeExceeded: vi.fn(), - markOrphan: vi.fn(), - unmarkOrphan: vi.fn(), - terminate: vi.fn(), -})); +import { describe, it, expect, beforeEach, vi } from 'vitest'; const mockOctokit = { apps: { @@ -23,178 +16,875 @@ const mockOctokit = { getRepoInstallation: vi.fn(), }, actions: { - listSelfHostedRunnersForOrg: vi.fn(), listSelfHostedRunnersForRepo: vi.fn(), + listSelfHostedRunnersForOrg: vi.fn(), + deleteSelfHostedRunnerFromOrg: vi.fn(), + deleteSelfHostedRunnerFromRepo: vi.fn(), getSelfHostedRunnerForOrg: vi.fn(), getSelfHostedRunnerForRepo: vi.fn(), }, paginate: vi.fn(), }; +vi.mock('@octokit/rest', () => ({ + Octokit: vi.fn().mockImplementation(function () { + return mockOctokit; + }), +})); -vi.mock('../github/auth', async () => ({ +vi.mock('./../aws/ec2-runners', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + tag: vi.fn(), + untag: vi.fn(), + terminateRunner: vi.fn(), + listEC2Runners: vi.fn(), + }; +}); +vi.mock('./../github/auth', async () => ({ createGithubAppAuth: vi.fn(), createGithubInstallationAuth: vi.fn(), createOctokitClient: vi.fn(), })); -vi.mock('./scale-down-provider-registry', () => ({ - createScaleDownRunnerProvider: vi.fn(() => testProvider), - getDefaultScaleDownRunnerProviderType: vi.fn(() => 'test-provider'), +vi.mock('./cache', async () => ({ + githubCache: { + getRunner: vi.fn(), + addRunner: vi.fn(), + clients: new Map(), + runners: new Map(), + reset: vi.fn().mockImplementation(() => { + githubCache.clients.clear(); + githubCache.runners.clear(); + }), + }, })); +const mocktokit = Octokit as vi.MockedClass; +const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); +const mockedInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); +const mockCreateClient = vi.mocked(ghAuth.createOctokitClient); +const mockListRunners = vi.mocked(listEC2Runners); +const mockTagRunners = vi.mocked(tag); +const mockUntagRunners = vi.mocked(untag); +const mockTerminateRunners = vi.mocked(terminateRunner); + +export interface TestData { + repositoryName: string; + repositoryOwner: string; +} + const cleanEnv = process.env; -const mockedCreateGithubAppAuth = vi.mocked(ghAuth.createGithubAppAuth); -const mockedCreateGithubInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); -const mockedCreateOctokitClient = vi.mocked(ghAuth.createOctokitClient); -const mockedCreateScaleDownRunnerProvider = vi.mocked(createScaleDownRunnerProvider); -const mockedGetDefaultScaleDownRunnerProviderType = vi.mocked(getDefaultScaleDownRunnerProviderType); - -function setDefaults() { - process.env = { ...cleanEnv }; - process.env.GITHUB_APP_KEY_BASE64 = 'TEST_CERTIFICATE_DATA'; - process.env.GITHUB_APP_ID = '1337'; - process.env.GITHUB_APP_CLIENT_ID = 'TEST_CLIENT_ID'; - process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET'; - process.env.SCALE_DOWN_CONFIG = '[]'; - process.env.ENVIRONMENT = 'unit-test-environment'; - process.env.MINIMUM_RUNNING_TIME_IN_MINUTES = '30'; - process.env.RUNNER_BOOT_TIME_IN_MINUTES = '5'; + +const ENVIRONMENT = 'unit-test-environment'; +const MINIMUM_TIME_RUNNING_IN_MINUTES = 30; +const MINIMUM_BOOT_TIME = 5; +const TEST_DATA: TestData = { + repositoryName: 'hello-world', + repositoryOwner: 'Codertocat', +}; + +interface RunnerTestItem extends RunnerList { + registered: boolean; + orphan: boolean; + shouldBeTerminated: boolean; } -beforeEach(() => { - vi.clearAllMocks(); - setDefaults(); - githubCache.clients.clear(); - githubCache.runners.clear(); - - testProvider.list.mockResolvedValue([]); - testProvider.bootTimeExceeded.mockReturnValue(false); - testProvider.markOrphan.mockResolvedValue(undefined); - testProvider.unmarkOrphan.mockResolvedValue(undefined); - testProvider.terminate.mockResolvedValue(undefined); - - mockedCreateGithubAppAuth.mockResolvedValue({ - type: 'app', - token: 'app-token', - appId: 1, - expiresAt: 'some-date', - }); - mockedCreateGithubInstallationAuth.mockResolvedValue({ - type: 'token', - tokenType: 'installation', - token: 'installation-token', - createdAt: 'some-date', - expiresAt: 'some-date', - permissions: {}, - repositorySelection: 'all', - installationId: 2, +describe('Scale down runners', () => { + beforeEach(() => { + process.env = { ...cleanEnv }; + process.env.GITHUB_APP_KEY_BASE64 = 'TEST_CERTIFICATE_DATA'; + process.env.GITHUB_APP_ID = '1337'; + process.env.GITHUB_APP_CLIENT_ID = 'TEST_CLIENT_ID'; + process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET'; + process.env.RUNNERS_MAXIMUM_COUNT = '3'; + process.env.SCALE_DOWN_CONFIG = '[]'; + process.env.ENVIRONMENT = ENVIRONMENT; + process.env.MINIMUM_RUNNING_TIME_IN_MINUTES = MINIMUM_TIME_RUNNING_IN_MINUTES.toString(); + process.env.RUNNER_BOOT_TIME_IN_MINUTES = MINIMUM_BOOT_TIME.toString(); + + nock.disableNetConnect(); + vi.clearAllMocks(); + vi.resetModules(); + githubCache.clients.clear(); + githubCache.runners.clear(); + mockOctokit.apps.getOrgInstallation.mockImplementation(() => ({ + data: { + id: 'ORG', + }, + })); + mockOctokit.apps.getRepoInstallation.mockImplementation(() => ({ + data: { + id: 'REPO', + }, + })); + + mockOctokit.paginate.mockResolvedValue([]); + mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation((repo) => { + // check if repo.runner_id contains the word "busy". If yes, throw an error else return 204 + if (repo.runner_id.includes('busy')) { + throw Error(); + } else { + return { status: 204 }; + } + }); + + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation((repo) => { + // check if repo.runner_id contains the word "busy". If yes, throw an error else return 204 + if (repo.runner_id.includes('busy')) { + throw Error(); + } else { + return { status: 204 }; + } + }); + + mockOctokit.actions.getSelfHostedRunnerForRepo.mockImplementation((repo) => { + if (repo.runner_id.includes('busy')) { + return { + data: { busy: true }, + }; + } else { + return { + data: { busy: false }, + }; + } + }); + mockOctokit.actions.getSelfHostedRunnerForOrg.mockImplementation((repo) => { + if (repo.runner_id.includes('busy')) { + return { + data: { busy: true }, + }; + } else { + return { + data: { busy: false }, + }; + } + }); + + mockTerminateRunners.mockImplementation(async () => { + return; + }); + mockedAppAuth.mockResolvedValue({ + type: 'app', + token: 'token', + appId: 1, + expiresAt: 'some-date', + }); + mockedInstallationAuth.mockResolvedValue({ + type: 'token', + tokenType: 'installation', + token: 'token', + createdAt: 'some-date', + expiresAt: 'some-date', + permissions: {}, + repositorySelection: 'all', + installationId: 0, + }); + mockCreateClient.mockResolvedValue(new mocktokit()); }); - mockedCreateOctokitClient.mockResolvedValue(mockOctokit as unknown as Octokit); - mockOctokit.apps.getOrgInstallation.mockResolvedValue({ data: { id: 2 } }); - mockOctokit.apps.getRepoInstallation.mockResolvedValue({ data: { id: 2 } }); - mockOctokit.paginate.mockResolvedValue([]); -}); -describe('scaleDown runner provider orchestration', () => { - it('creates the default provider and lists orphan and active runners', async () => { - await scaleDown(); + const endpoints = ['https://api.github.com', 'https://github.enterprise.something', 'https://companyname.ghe.com']; + + describe.each(endpoints)('for %s', (endpoint) => { + beforeEach(() => { + if (endpoint.includes('enterprise') || endpoint.endsWith('.ghe.com')) { + process.env.GHES_URL = endpoint; + } + }); + + type RunnerType = 'Repo' | 'Org'; + const runnerTypes: RunnerType[] = ['Org', 'Repo']; + describe.each(runnerTypes)('For %s runners.', (type) => { + it('Should not call terminate when no runners online.', async () => { + // setup + mockAwsRunners([]); + + // act + await scaleDown(); + + // assert + expect(listEC2Runners).toHaveBeenCalledWith({ + environment: ENVIRONMENT, + }); + expect(terminateRunner).not.toHaveBeenCalled(); + expect(mockOctokit.apps.getRepoInstallation).not.toHaveBeenCalled(); + expect(mockOctokit.apps.getRepoInstallation).not.toHaveBeenCalled(); + }); + + it(`Should terminate runner without idle config ${type} runners.`, async () => { + // setup + const runners = [ + createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES - 1, true, false, false), + createRunnerTestData('idle-2', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 4, true, false, true), + createRunnerTestData('busy-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 3, true, false, false), + createRunnerTestData('booting-1', type, MINIMUM_BOOT_TIME - 1, false, false, false), + ]; + + mockGitHubRunners(runners); + mockListRunners.mockResolvedValue(runners); + mockAwsRunners(runners); + + await scaleDown(); + + // assert + expect(listEC2Runners).toHaveBeenCalledWith({ + environment: ENVIRONMENT, + }); + + if (type === 'Repo') { + expect(mockOctokit.apps.getRepoInstallation).toHaveBeenCalled(); + } else { + expect(mockOctokit.apps.getOrgInstallation).toHaveBeenCalled(); + } + + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it(`Should respect idle runner with minimum running time not exceeded.`, async () => { + // setup + const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES - 1, true, false, false)]; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await scaleDown(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it(`Should respect booting runner.`, async () => { + // setup + const runners = [createRunnerTestData('booting-1', type, MINIMUM_BOOT_TIME - 1, false, false, false)]; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await scaleDown(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it(`Should respect busy runner.`, async () => { + // setup + const runners = [createRunnerTestData('busy-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false)]; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await scaleDown(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it(`Should not terminate runner with bypass-removal tag set.`, async () => { + // setup + const runners = [ + createRunnerTestData('idle-with-bypass', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 10, true, false, false), + ]; + // Set bypass-removal tag + runners[0].bypassRemoval = true; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await scaleDown(); + + // assert + expect(terminateRunner).not.toHaveBeenCalled(); + checkNonTerminated(runners); + }); + + it(`Should not terminate orphaned runner with bypass-removal tag set.`, async () => { + // setup - orphan runner with bypass-removal tag + const orphanRunner = createRunnerTestData('orphan-bypass', type, MINIMUM_BOOT_TIME + 1, false, false, false); + orphanRunner.bypassRemoval = true; + + const idleRunner = createRunnerTestData('idle-1', type, MINIMUM_BOOT_TIME + 1, true, false, false); + const runners = [orphanRunner, idleRunner]; + + mockGitHubRunners([idleRunner]); + mockAwsRunners(runners); + + // act - first cycle marks orphan + await scaleDown(); + + // mark as orphan for next cycle + orphanRunner.orphan = true; + + // act - second cycle should skip termination due to bypass-removal + await scaleDown(); + + // assert - orphan runner should NOT be terminated + expect(terminateRunner).not.toHaveBeenCalledWith(orphanRunner.instanceId); + }); + + it(`Should not terminate a runner that became busy just before deregister runner.`, async () => { + // setup + const runners = [ + createRunnerTestData( + 'job-just-start-at-deregister-1', + type, + MINIMUM_TIME_RUNNING_IN_MINUTES + 1, + true, + false, + false, + ), + ]; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { + return { status: 500 }; + }); + + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { + return { status: 500 }; + }); + + // act and ensure no exception is thrown + await expect(scaleDown()).resolves.not.toThrow(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it(`Should terminate orphan (Non JIT)`, async () => { + // setup + const orphanRunner = createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, false, false); + const idleRunner = createRunnerTestData('idle-1', type, MINIMUM_BOOT_TIME + 1, true, false, false); + const runners = [orphanRunner, idleRunner]; + + mockGitHubRunners([idleRunner]); + mockAwsRunners(runners); + + // act + await scaleDown(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + + expect(mockTagRunners).toHaveBeenCalledWith(orphanRunner.instanceId, [ + { + Key: 'ghr:orphan', + Value: 'true', + }, + ]); + + expect(mockTagRunners).not.toHaveBeenCalledWith(idleRunner.instanceId, expect.anything()); + + // next cycle, update test data set orphan to true and terminate should be true + orphanRunner.orphan = true; + orphanRunner.shouldBeTerminated = true; + + // act + await scaleDown(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it('Should test if orphaned runner, untag if online and busy, else terminate (JIT)', async () => { + // arrange + const orphanRunner = createRunnerTestData( + 'orphan-jit', + type, + MINIMUM_BOOT_TIME + 1, + false, + true, + false, + undefined, + 1234567890, + ); + const runners = [orphanRunner]; + + mockGitHubRunners([]); + mockAwsRunners(runners); + + if (type === 'Repo') { + mockOctokit.actions.getSelfHostedRunnerForRepo.mockResolvedValueOnce({ + data: { id: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' }, + }); + } else { + mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ + data: { id: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'online' }, + }); + } + + // act + await scaleDown(); - expect(mockedGetDefaultScaleDownRunnerProviderType).toHaveBeenCalled(); - expect(mockedCreateScaleDownRunnerProvider).toHaveBeenCalledWith('test-provider'); - expect(testProvider.list).toHaveBeenNthCalledWith(1, 'unit-test-environment', true); - expect(testProvider.list).toHaveBeenNthCalledWith(2, 'unit-test-environment'); - expect(testProvider.terminate).not.toHaveBeenCalled(); + // assert + expect(mockUntagRunners).toHaveBeenCalledWith(orphanRunner.instanceId, [{ Key: 'ghr:orphan', Value: 'true' }]); + expect(mockTerminateRunners).not.toHaveBeenCalledWith(orphanRunner.instanceId); + + // arrange + if (type === 'Repo') { + mockOctokit.actions.getSelfHostedRunnerForRepo.mockResolvedValueOnce({ + data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'offline' }, + }); + } else { + mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ + data: { runnerId: 1234567890, name: orphanRunner.instanceId, busy: true, status: 'offline' }, + }); + } + + // act + await scaleDown(); + + // assert + expect(mockTerminateRunners).toHaveBeenCalledWith(orphanRunner.instanceId); + }); + + it('Should handle 404 error when checking orphaned runner (JIT) - treat as orphaned', async () => { + // arrange + const orphanRunner = createRunnerTestData( + 'orphan-jit-404', + type, + MINIMUM_BOOT_TIME + 1, + false, + true, + true, // should be terminated when 404 + undefined, + 1234567890, + ); + const runners = [orphanRunner]; + + mockGitHubRunners([]); + mockAwsRunners(runners); + + // Mock 404 error response + const error404 = new RequestError('Runner not found', 404, { + request: { + method: 'GET', + url: 'https://api.github.com/test', + headers: {}, + }, + }); + + if (type === 'Repo') { + mockOctokit.actions.getSelfHostedRunnerForRepo.mockRejectedValueOnce(error404); + } else { + mockOctokit.actions.getSelfHostedRunnerForOrg.mockRejectedValueOnce(error404); + } + + // act + await scaleDown(); + + // assert - should terminate since 404 means runner doesn't exist on GitHub + expect(mockTerminateRunners).toHaveBeenCalledWith(orphanRunner.instanceId); + }); + + it('Should handle 404 error when checking runner busy state - treat as not busy', async () => { + // arrange + const runner = createRunnerTestData( + 'runner-404', + type, + MINIMUM_TIME_RUNNING_IN_MINUTES + 1, + true, + false, + true, // should be terminated since not busy due to 404 + ); + const runners = [runner]; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // Mock 404 error response for busy state check + const error404 = new RequestError('Runner not found', 404, { + request: { + method: 'GET', + url: 'https://api.github.com/test', + headers: {}, + }, + }); + + if (type === 'Repo') { + mockOctokit.actions.getSelfHostedRunnerForRepo.mockRejectedValueOnce(error404); + } else { + mockOctokit.actions.getSelfHostedRunnerForOrg.mockRejectedValueOnce(error404); + } + + // act + await scaleDown(); + + // assert - should terminate since 404 means runner is not busy + checkTerminated(runners); + }); + + it('Should re-throw non-404 errors when checking runner state', async () => { + // arrange + const orphanRunner = createRunnerTestData( + 'orphan-error', + type, + MINIMUM_BOOT_TIME + 1, + false, + true, + false, + undefined, + 1234567890, + ); + const runners = [orphanRunner]; + + mockGitHubRunners([]); + mockAwsRunners(runners); + + // Mock non-404 error response + const error500 = new RequestError('Internal server error', 500, { + request: { + method: 'GET', + url: 'https://api.github.com/test', + headers: {}, + }, + }); + + if (type === 'Repo') { + mockOctokit.actions.getSelfHostedRunnerForRepo.mockRejectedValueOnce(error500); + } else { + mockOctokit.actions.getSelfHostedRunnerForOrg.mockRejectedValueOnce(error500); + } + + // act & assert - should not throw because error handling is in terminateOrphan + await expect(scaleDown()).resolves.not.toThrow(); + + // Should not terminate since the error was not a 404 + expect(terminateRunner).not.toHaveBeenCalledWith(orphanRunner.instanceId); + }); + + it(`Should ignore errors when termination orphan fails.`, async () => { + // setup + const orphanRunner = createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, true, true); + const runners = [orphanRunner]; + + mockGitHubRunners([]); + mockAwsRunners(runners); + mockTerminateRunners.mockImplementation(() => { + throw new Error('Failed to terminate'); + }); + + // act + await scaleDown(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + describe('When orphan termination fails', () => { + it(`Should not throw in case of list runner exception.`, async () => { + // setup + const runners = [createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, true, true)]; + + mockGitHubRunners([]); + mockListRunners.mockRejectedValueOnce(new Error('Failed to list runners')); + mockAwsRunners(runners); + + // ac + await scaleDown(); + + // assert + checkNonTerminated(runners); + }); + + it(`Should not throw in case of terminate runner exception.`, async () => { + // setup + const runners = [createRunnerTestData('orphan-1', type, MINIMUM_BOOT_TIME + 1, false, true, true)]; + + mockGitHubRunners([]); + mockAwsRunners(runners); + mockTerminateRunners.mockRejectedValue(new Error('Failed to terminate')); + + // act and ensure no exception is thrown + await scaleDown(); + + // assert + checkNonTerminated(runners); + }); + }); + + it(`Should not terminate instance in case de-register fails.`, async () => { + // setup + const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false)]; + + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { + return { status: 500 }; + }); + mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { + return { status: 500 }; + }); + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act and should resolve + await expect(scaleDown()).resolves.not.toThrow(); + + // assert + checkTerminated(runners); + checkNonTerminated(runners); + }); + + it(`Should not throw an exception in case of failure during removing a runner.`, async () => { + // setup + const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, true, false)]; + + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { + throw new Error('Failed to delete runner'); + }); + mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { + throw new Error('Failed to delete runner'); + }); + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await expect(scaleDown()).resolves.not.toThrow(); + }); + + it(`Should not terminate instance when de-registration throws an error.`, async () => { + // setup - runner should NOT be terminated because de-registration fails + const runners = [createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 1, true, false, false)]; + + const error502 = new RequestError('Server Error', 502, { + request: { + method: 'DELETE', + url: 'https://api.github.com/test', + headers: {}, + }, + }); + + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockImplementation(() => { + throw error502; + }); + mockOctokit.actions.deleteSelfHostedRunnerFromRepo.mockImplementation(() => { + throw error502; + }); + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await expect(scaleDown()).resolves.not.toThrow(); + + // assert - should NOT terminate since de-registration failed + expect(terminateRunner).not.toHaveBeenCalled(); + }); + + const evictionStrategies = ['oldest_first', 'newest_first']; + describe.each(evictionStrategies)('When idle config defined', (evictionStrategy) => { + const defaultConfig = { + idleCount: 1, + cron: '* * * * * *', + timeZone: 'Europe/Amsterdam', + evictionStrategy, + }; + + beforeEach(() => { + process.env.SCALE_DOWN_CONFIG = JSON.stringify([defaultConfig]); + }); + + it(`Should terminate based on the the idle config with ${evictionStrategy} eviction strategy`, async () => { + // setup + const runnerToTerminateTime = + evictionStrategy === 'oldest_first' + ? MINIMUM_TIME_RUNNING_IN_MINUTES + 5 + : MINIMUM_TIME_RUNNING_IN_MINUTES + 1; + const runners = [ + createRunnerTestData('idle-1', type, MINIMUM_TIME_RUNNING_IN_MINUTES + 4, true, false, false), + createRunnerTestData('idle-to-terminate', type, runnerToTerminateTime, true, false, true), + ]; + + mockGitHubRunners(runners); + mockAwsRunners(runners); + + // act + await scaleDown(); + + // assert + const runnersToTerminate = runners.filter((r) => r.shouldBeTerminated); + for (const toTerminate of runnersToTerminate) { + expect(terminateRunner).toHaveBeenCalledWith(toTerminate.instanceId); + } + + const runnersNotToTerminate = runners.filter((r) => !r.shouldBeTerminated); + for (const notTerminated of runnersNotToTerminate) { + expect(terminateRunner).not.toHaveBeenCalledWith(notTerminated.instanceId); + } + }); + }); + }); }); - it('creates the provider type configured on idle config', async () => { - process.env.SCALE_DOWN_CONFIG = JSON.stringify([ + describe('When runners are sorted', () => { + const runners: RunnerInfo[] = [ { - idleCount: 0, - cron: '* * * * *', - timeZone: 'UTC', - type: 'custom-provider', + instanceId: '1', + launchTime: moment(new Date()).subtract(1, 'minute').toDate(), + owner: 'owner', + type: 'type', }, - ]); + { + instanceId: '3', + launchTime: moment(new Date()).subtract(3, 'minute').toDate(), + owner: 'owner', + type: 'type', + }, + { + instanceId: '2', + launchTime: moment(new Date()).subtract(2, 'minute').toDate(), + owner: 'owner', + type: 'type', + }, + { + instanceId: '0', + launchTime: moment(new Date()).subtract(0, 'minute').toDate(), + owner: 'owner', + type: 'type', + }, + ]; - await scaleDown(); + it('Should sort runners descending for eviction strategy oldest first te keep the youngest.', () => { + runners.sort(oldestFirstStrategy); + expect(runners[0].instanceId).toEqual('0'); + expect(runners[1].instanceId).toEqual('1'); + expect(runners[2].instanceId).toEqual('2'); + expect(runners[3].instanceId).toEqual('3'); + }); - expect(mockedCreateScaleDownRunnerProvider).toHaveBeenCalledWith('custom-provider'); - }); + it('Should sort runners ascending for eviction strategy newest first te keep oldest.', () => { + runners.sort(newestFirstStrategy); + expect(runners[0].instanceId).toEqual('3'); + expect(runners[1].instanceId).toEqual('2'); + expect(runners[2].instanceId).toEqual('1'); + expect(runners[3].instanceId).toEqual('0'); + }); - it('marks a provider runner as orphan when it is not registered and boot time is exceeded', async () => { - const runner: RunnerInfo = { - id: 'runner-1', - launchTime: moment(new Date()).subtract(10, 'minutes').toDate(), - owner: 'Codertocat', - type: 'Org', - }; - testProvider.list.mockImplementation(async (_environment: string, orphan?: boolean) => (orphan ? [] : [runner])); - testProvider.bootTimeExceeded.mockReturnValue(true); - - await scaleDown(); - - expect(testProvider.bootTimeExceeded).toHaveBeenCalledWith(runner); - expect(testProvider.markOrphan).toHaveBeenCalledWith('runner-1'); - expect(testProvider.terminate).not.toHaveBeenCalledWith('runner-1'); - }); + it('Should sort runners with equal launch time.', () => { + const runnersTest = [...runners]; + const same = moment(new Date()).subtract(4, 'minute').toDate(); + runnersTest.push({ + instanceId: '4', + launchTime: same, + owner: 'owner', + type: 'type', + }); + runnersTest.push({ + instanceId: '5', + launchTime: same, + owner: 'owner', + type: 'type', + }); + runnersTest.sort(oldestFirstStrategy); + expect(runnersTest[3].launchTime).not.toEqual(same); + expect(runnersTest[4].launchTime).toEqual(same); + expect(runnersTest[5].launchTime).toEqual(same); + + runnersTest.sort(newestFirstStrategy); + expect(runnersTest[3].launchTime).not.toEqual(same); + expect(runnersTest[1].launchTime).toEqual(same); + expect(runnersTest[0].launchTime).toEqual(same); + }); - it('terminates orphan runners through the selected provider', async () => { - const orphanRunner: RunnerList = { - id: 'orphan-1', - launchTime: moment(new Date()).subtract(10, 'minutes').toDate(), - owner: 'Codertocat', - type: 'Org', - orphan: true, - }; - testProvider.list.mockImplementation(async (_environment: string, orphan?: boolean) => - orphan ? [orphanRunner] : [], - ); - - await scaleDown(); - - expect(testProvider.terminate).toHaveBeenCalledWith('orphan-1'); + it('Should sort runners even when launch time is undefined.', () => { + const runnersTest = [ + { + instanceId: '0', + launchTime: undefined, + owner: 'owner', + type: 'type', + }, + { + instanceId: '1', + launchTime: moment(new Date()).subtract(3, 'minute').toDate(), + owner: 'owner', + type: 'type', + }, + { + instanceId: '0', + launchTime: undefined, + owner: 'owner', + type: 'type', + }, + ]; + runnersTest.sort(oldestFirstStrategy); + expect(runnersTest[0].launchTime).toBeUndefined(); + expect(runnersTest[1].launchTime).toBeDefined(); + expect(runnersTest[2].launchTime).not.toBeDefined(); + }); }); }); -describe('scaleDown runner sort strategies', () => { - const createRunners = (): RunnerInfo[] => [ - { - id: '1', - launchTime: moment(new Date()).subtract(1, 'minute').toDate(), - owner: 'owner', - type: 'type', - }, - { - id: '3', - launchTime: moment(new Date()).subtract(3, 'minute').toDate(), - owner: 'owner', - type: 'type', - }, - { - id: '2', - launchTime: moment(new Date()).subtract(2, 'minute').toDate(), - owner: 'owner', - type: 'type', - }, - { - id: '0', - launchTime: moment(new Date()).subtract(0, 'minute').toDate(), - owner: 'owner', - type: 'type', - }, - ]; - - it('sorts runners descending for oldest first to keep the youngest', () => { - const runners = createRunners(); - - runners.sort(oldestFirstStrategy); - expect(runners.map((runner) => runner.id)).toEqual(['0', '1', '2', '3']); +function mockAwsRunners(runners: RunnerTestItem[]) { + mockListRunners.mockImplementation(async (filter) => { + return runners.filter((r) => !filter?.orphan || filter?.orphan === r.orphan); }); +} - it('sorts runners ascending for newest first to keep the oldest', () => { - const runners = createRunners(); +function checkNonTerminated(runners: RunnerTestItem[]) { + const notTerminated = runners.filter((r) => !r.shouldBeTerminated); + for (const toTerminate of notTerminated) { + expect(terminateRunner).not.toHaveBeenCalledWith(toTerminate.instanceId); + } +} - runners.sort(newestFirstStrategy); - expect(runners.map((runner) => runner.id)).toEqual(['3', '2', '1', '0']); - }); -}); +function checkTerminated(runners: RunnerTestItem[]) { + const runnersToTerminate = runners.filter((r) => r.shouldBeTerminated); + expect(terminateRunner).toHaveBeenCalledTimes(runnersToTerminate.length); + for (const toTerminate of runnersToTerminate) { + expect(terminateRunner).toHaveBeenCalledWith(toTerminate.instanceId); + } +} + +function mockGitHubRunners(runners: RunnerTestItem[]) { + mockOctokit.paginate.mockResolvedValue( + runners + .filter((r) => r.registered) + .map((r) => { + return { + id: r.instanceId, + name: r.instanceId, + }; + }), + ); +} + +function createRunnerTestData( + name: string, + type: 'Org' | 'Repo', + minutesLaunchedAgo: number, + registered: boolean, + orphan: boolean, + shouldBeTerminated: boolean, + owner?: string, + runnerId?: number, +): RunnerTestItem { + return { + instanceId: `i-${name}-${type.toLowerCase()}`, + launchTime: moment(new Date()).subtract(minutesLaunchedAgo, 'minutes').toDate(), + type, + owner: owner + ? owner + : type === 'Repo' + ? `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}` + : `${TEST_DATA.repositoryOwner}`, + registered, + orphan, + shouldBeTerminated, + runnerId: runnerId !== undefined ? String(runnerId) : undefined, + bypassRemoval: false, + }; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-config.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-config.test.ts deleted file mode 100644 index 83fdb914b1..0000000000 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-config.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { getScaleUpRunnerProviderType } from './scale-up-config'; - -describe('getScaleUpRunnerProviderType', () => { - it('defaults to ec2 when no type is defined', () => { - expect(getScaleUpRunnerProviderType(undefined, 'ec2')).toEqual('ec2'); - }); - - it('uses configured ec2 type', () => { - expect(getScaleUpRunnerProviderType('ec2', 'microvm')).toEqual('ec2'); - }); -}); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index f97f8e8f6c..fb921806e6 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -1,22 +1,33 @@ -import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { DescribeLaunchTemplateVersionsCommand, EC2Client } from '@aws-sdk/client-ec2'; +import { PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +// Using vi.mocked instead of jest-mock +import nock from 'nock'; +import { performance } from 'perf_hooks'; import * as ghAuth from '../github/auth'; +import { createRunner, listEC2Runners, tag } from './../aws/ec2-runners'; +import { RunnerInputParameters } from './../aws/ec2-runners.d'; +import * as scaleUpModule from './scale-up'; +import { parseEc2OverrideConfig } from './ec2-labels'; +import { getScaleUpRunnerProviderType } from './scale-up-config'; +import { EC2_TAG_VALUE_MAX_LENGTH, RUNNER_LABELS_TAG_MAX_COUNT } from './ec2'; +import { getParameter } from '@aws-github-runner/aws-ssm-util'; import { publishRetryMessage } from './job-retry'; -import { scaleUp } from './scale-up'; -import { createScaleUpRunnerProviderFromEnv, getDefaultScaleUpRunnerProviderType } from './scale-up-provider-registry'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; import type { Octokit } from '@octokit/rest'; import type { ActionRequestMessageSQS } from './types'; -const testProvider = vi.hoisted(() => ({ - type: 'test-provider', - prepareGroup: vi.fn(), - getCurrentRunners: vi.fn(), - createRunners: vi.fn(), -})); - const mockOctokit = { + paginate: vi.fn(), + checks: { get: vi.fn() }, actions: { + createRegistrationTokenForOrg: vi.fn(), + createRegistrationTokenForRepo: vi.fn(), getJobForWorkflowRun: vi.fn(), + generateRunnerJitconfigForOrg: vi.fn(), + generateRunnerJitconfigForRepo: vi.fn(), }, apps: { getOrgInstallation: vi.fn(), @@ -24,195 +35,3482 @@ const mockOctokit = { }, }; -vi.mock('../github/auth', async () => ({ +const mockCreateRunner = vi.mocked(createRunner); +const mockListRunners = vi.mocked(listEC2Runners); +const mockTag = vi.mocked(tag); +const mockEC2Client = mockClient(EC2Client); +const mockSSMClient = mockClient(SSMClient); +const mockSSMgetParameter = vi.mocked(getParameter); +const mockPublishRetryMessage = vi.mocked(publishRetryMessage); + +vi.mock('@octokit/rest', () => ({ + Octokit: vi.fn().mockImplementation(function () { + return mockOctokit; + }), +})); + +vi.mock('./../aws/ec2-runners', async () => ({ + createRunner: vi.fn(), + listEC2Runners: vi.fn(), + tag: vi.fn(), +})); + +vi.mock('./../github/auth', async () => ({ createGithubAppAuth: vi.fn(), createGithubInstallationAuth: vi.fn(), createOctokitClient: vi.fn(), })); +vi.mock('@aws-github-runner/aws-ssm-util', async () => { + const actual = (await vi.importActual( + '@aws-github-runner/aws-ssm-util', + )) as typeof import('@aws-github-runner/aws-ssm-util'); + + return { + ...actual, + getParameter: vi.fn(), + }; +}); + vi.mock('./job-retry', () => ({ publishRetryMessage: vi.fn(), + checkAndRetryJob: vi.fn(), })); -vi.mock('./scale-up-provider-registry', () => ({ - createScaleUpRunnerProviderFromEnv: vi.fn(() => testProvider), - getDefaultScaleUpRunnerProviderType: vi.fn(() => 'test-provider'), -})); +export type RunnerType = 'ephemeral' | 'non-ephemeral'; -const cleanEnv = process.env; -const mockedCreateGithubAppAuth = vi.mocked(ghAuth.createGithubAppAuth); -const mockedCreateGithubInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); -const mockedCreateOctokitClient = vi.mocked(ghAuth.createOctokitClient); -const mockedCreateScaleUpRunnerProviderFromEnv = vi.mocked(createScaleUpRunnerProviderFromEnv); -const mockedGetDefaultScaleUpRunnerProviderType = vi.mocked(getDefaultScaleUpRunnerProviderType); -const mockedPublishRetryMessage = vi.mocked(publishRetryMessage); - -const TEST_MESSAGE: ActionRequestMessageSQS = { +// for ephemeral and non-ephemeral runners +const RUNNER_TYPES: RunnerType[] = ['ephemeral', 'non-ephemeral']; + +const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); +const mockedInstallationAuth = vi.mocked(ghAuth.createGithubInstallationAuth); +const mockCreateClient = vi.mocked(ghAuth.createOctokitClient); + +const TEST_DATA_SINGLE: ActionRequestMessageSQS = { id: 1, eventType: 'workflow_job', repositoryName: 'hello-world', repositoryOwner: 'Codertocat', installationId: 2, repoOwnerType: 'Organization', - messageId: 'message-1', - labels: ['self-hosted', 'ghr-provider-size:large'], + messageId: 'foobar', }; +const TEST_DATA: ActionRequestMessageSQS[] = [ + { + ...TEST_DATA_SINGLE, + messageId: 'foobar', + }, +]; + +const cleanEnv = process.env; + +const EXPECTED_RUNNER_PARAMS: RunnerInputParameters = { + environment: 'unit-test-environment', + runnerType: 'Org', + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + numberOfRunners: 1, + launchTemplateName: 'lt-1', + ec2instanceCriteria: { + instanceTypes: ['m5.large'], + targetCapacityType: 'spot', + instanceAllocationStrategy: 'lowest-price', + }, + subnets: ['subnet-123'], + tracingEnabled: false, + onDemandFailoverOnError: [], + scaleErrors: ['UnfulfillableCapacity', 'MaxSpotInstanceCountExceeded', 'TargetCapacityLimitExceededException'], + source: 'scale-up-lambda', + useDedicatedHost: false, +}; +let expectedRunnerParams: RunnerInputParameters; + function setDefaults() { process.env = { ...cleanEnv }; + process.env.PARAMETER_GITHUB_APP_ID_NAME = 'github-app-id'; process.env.GITHUB_APP_KEY_BASE64 = 'TEST_CERTIFICATE_DATA'; process.env.GITHUB_APP_ID = '1337'; process.env.GITHUB_APP_CLIENT_ID = 'TEST_CLIENT_ID'; process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET'; process.env.RUNNERS_MAXIMUM_COUNT = '3'; - process.env.ENVIRONMENT = 'unit-test-environment'; - process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; - process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; - process.env.RUNNER_NAME_PREFIX = 'unit-test-'; - process.env.RUNNER_GROUP_NAME = 'Default'; - process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; - process.env.RUNNER_LABELS = 'base-label'; + process.env.ENVIRONMENT = EXPECTED_RUNNER_PARAMS.environment; + process.env.LAUNCH_TEMPLATE_NAME = 'lt-1'; + process.env.SUBNET_IDS = 'subnet-123'; + process.env.INSTANCE_TYPES = 'm5.large'; + process.env.INSTANCE_TARGET_CAPACITY_TYPE = 'spot'; + process.env.ENABLE_ON_DEMAND_FAILOVER = undefined; process.env.SCALE_ERRORS = '["UnfulfillableCapacity","MaxSpotInstanceCountExceeded","TargetCapacityLimitExceededException"]'; } beforeEach(() => { + nock.disableNetConnect(); + vi.resetModules(); vi.clearAllMocks(); setDefaults(); - testProvider.prepareGroup.mockResolvedValue({ - runnerLabels: ['ghr-provider-size:large'], - state: { prepared: true }, + mockEC2Client.reset(); + mockEC2Client.on(DescribeLaunchTemplateVersionsCommand).resolves({ + LaunchTemplateVersions: [ + { + LaunchTemplateData: { + BlockDeviceMappings: [ + { + DeviceName: '/dev/sda1', + Ebs: {}, + }, + ], + }, + }, + ], + }); + + defaultSSMGetParameterMockImpl(); + defaultOctokitMockImpl(); + + mockCreateRunner.mockImplementation(async () => { + return ['i-12345']; }); - testProvider.getCurrentRunners.mockResolvedValue(0); - testProvider.createRunners.mockResolvedValue(['runner-1']); + mockListRunners.mockImplementation(async () => [ + { + instanceId: 'i-1234', + launchTime: new Date(), + type: 'Org', + owner: TEST_DATA_SINGLE.repositoryOwner, + }, + ]); - mockedCreateGithubAppAuth.mockResolvedValue({ + mockedAppAuth.mockResolvedValue({ type: 'app', - token: 'app-token', - appId: 1, + token: 'token', + appId: TEST_DATA_SINGLE.installationId, expiresAt: 'some-date', }); - mockedCreateGithubInstallationAuth.mockResolvedValue({ + mockedInstallationAuth.mockResolvedValue({ type: 'token', tokenType: 'installation', - token: 'installation-token', + token: 'token', createdAt: 'some-date', expiresAt: 'some-date', permissions: {}, repositorySelection: 'all', - installationId: 2, - }); - mockedCreateOctokitClient.mockResolvedValue(mockOctokit as unknown as Octokit); - mockOctokit.actions.getJobForWorkflowRun.mockResolvedValue({ - data: { status: 'queued' }, - headers: {}, + installationId: 0, }); + + mockCreateClient.mockResolvedValue(mockOctokit as unknown as Octokit); }); -describe('scaleUp runner provider orchestration', () => { - it('creates the configured provider and forwards provider state into runner creation', async () => { - process.env.RUNNER_PROVIDER_TYPE = 'test-provider'; - - const rejectedMessages = await scaleUp([TEST_MESSAGE]); - - expect(rejectedMessages).toEqual([]); - expect(mockedGetDefaultScaleUpRunnerProviderType).toHaveBeenCalled(); - expect(mockedCreateScaleUpRunnerProviderFromEnv).toHaveBeenCalledWith('test-provider', 'unit-test-environment', [ - 'UnfulfillableCapacity', - 'MaxSpotInstanceCountExceeded', - 'TargetCapacityLimitExceededException', - ]); - expect(testProvider.prepareGroup).toHaveBeenCalledWith(TEST_MESSAGE.labels); - expect(testProvider.getCurrentRunners).toHaveBeenCalledWith( - { prepared: true }, - { - runnerOwner: TEST_MESSAGE.repositoryOwner, +describe('scaleUp with GHES', () => { + beforeEach(() => { + process.env.GHES_URL = 'https://github.enterprise.something'; + }); + + it('checks queued workflows', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalledWith({ + job_id: TEST_DATA_SINGLE.id, + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('does not list runners when no workflows are queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ + data: { total_count: 0 }, + })); + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).not.toBeCalled(); + }); + + describe('on org level', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.RUNNER_NAME_PREFIX = 'unit-test-'; + process.env.RUNNER_GROUP_NAME = 'Default'; + process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; + process.env.RUNNER_LABELS = 'label1,label2'; + + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + mockSSMClient.reset(); + }); + + it('gets the current org level runners', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).toBeCalledWith({ + environment: 'unit-test-environment', runnerType: 'Org', + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }); + }); + + it('does not create a token when maximum runners has been reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('does not create runners when current runners exceed maximum (race condition)', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '5'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + // Simulate race condition where pool lambda created more runners than max + mockListRunners.mockImplementation(async () => + Array.from({ length: 10 }, (_, i) => ({ + instanceId: `i-${i}`, + launchTime: new Date(), + type: 'Org', + owner: TEST_DATA_SINGLE.repositoryOwner, + })), + ); + await scaleUpModule.scaleUp(TEST_DATA); + // Should not attempt to create runners (would be negative without fix) + expect(createRunner).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + }); + + it('does create a runner if maximum is set to -1', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '-1'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).not.toHaveBeenCalled(); + expect(createRunner).toHaveBeenCalled(); + }); + + it('creates a token when maximum runners has not been reached', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + }); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a runner with correct config', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('tags the EC2 runner with the GitHub runner metadata returned by JIT config', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockTag).toHaveBeenCalledWith('i-12345', [ + { Key: 'ghr:github_runner_id', Value: '9876543210' }, + { Key: 'ghr:runner_labels', Value: 'label1,label2' }, + ]); + }); + + it('chunks comma-joined GitHub runner labels by the EC2 tag value max length', async () => { + const runnerLabels = ['a'.repeat(EC2_TAG_VALUE_MAX_LENGTH), 'b'].join(','); + process.env.RUNNER_LABELS = runnerLabels; + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockTag).toHaveBeenCalledWith('i-12345', [ + { Key: 'ghr:github_runner_id', Value: '9876543210' }, + { Key: 'ghr:runner_labels', Value: runnerLabels.slice(0, EC2_TAG_VALUE_MAX_LENGTH) }, + { + Key: 'ghr:runner_labels:2', + Value: runnerLabels.slice(EC2_TAG_VALUE_MAX_LENGTH), + }, + ]); + }); + + it('limits GitHub runner label metadata to five EC2 tags', async () => { + const runnerLabels = Array.from({ length: RUNNER_LABELS_TAG_MAX_COUNT + 1 }, (_, index) => + String.fromCharCode('a'.charCodeAt(0) + index).repeat(EC2_TAG_VALUE_MAX_LENGTH), + ).join(','); + process.env.RUNNER_LABELS = runnerLabels; + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockTag).toHaveBeenCalledWith('i-12345', [ + { Key: 'ghr:github_runner_id', Value: '9876543210' }, + ...Array.from({ length: RUNNER_LABELS_TAG_MAX_COUNT }, (_, index) => ({ + Key: index === 0 ? 'ghr:runner_labels' : `ghr:runner_labels:${index + 1}`, + Value: runnerLabels.slice(index * EC2_TAG_VALUE_MAX_LENGTH, (index + 1) * EC2_TAG_VALUE_MAX_LENGTH), + })), + ]); + }); + + it('uses a reserved separator when packing GitHub runner labels into EC2 tags', async () => { + process.env.RUNNER_LABELS = ['label:with/slash', 'label+with+plus'].join(','); + const testDataWithSemicolonDynamicLabel = [ + { + ...TEST_DATA_SINGLE, + labels: ['ghr-ec2-cpu-manufacturers:intel;amd'], + messageId: 'test-semicolon-dynamic-label', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithSemicolonDynamicLabel); + + expect(mockTag).toHaveBeenCalledWith('i-12345', [ + { Key: 'ghr:github_runner_id', Value: '9876543210' }, + { + Key: 'ghr:runner_labels', + Value: 'label:with/slash,label+with+plus,ghr-ec2-cpu-manufacturers:intel;amd', + }, + ]); + }); + + it('creates a runner with labels in a specific group', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner with ami id override from ssm parameter', async () => { + process.env.AMI_ID_SSM_PARAMETER_NAME = 'my-ami-id-param'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith({ ...expectedRunnerParams, amiIdSsmParameterName: 'my-ami-id-param' }); + }); + + it('Throws an error if runner group does not exist for ephemeral runners', async () => { + process.env.RUNNER_GROUP_NAME = 'test-runner-group'; + mockSSMgetParameter.mockImplementation(async () => { + throw new Error('ParameterNotFound'); + }); + await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toBeInstanceOf(Error); + expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); + }); + + it('Discards event if it is a User repo and org level runners is enabled', async () => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + const USER_REPO_TEST_DATA = structuredClone(TEST_DATA); + USER_REPO_TEST_DATA[0].repoOwnerType = 'User'; + await scaleUpModule.scaleUp(USER_REPO_TEST_DATA); + expect(createRunner).not.toHaveBeenCalled(); + }); + + it('create SSM parameter for runner group id if it does not exist', async () => { + mockSSMgetParameter.mockImplementation(async () => { + throw new Error('ParameterNotFound'); + }); + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); + expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 2); + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`, + Value: '1', + Type: 'String', + }); + }); + + it('Does not create SSM parameter for runner group id if it exists', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.paginate).toHaveBeenCalledTimes(0); + expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 1); + }); + + it('create start runner config for ephemeral runners ', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toBeCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + name: 'unit-test-i-12345', + runner_group_id: 1, + labels: ['label1', 'label2'], + }); + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: 'TEST_JIT_CONFIG_ORG', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('create start runner config for non-ephemeral runners ', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalled(); + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: + '--url https://github.enterprise.something/Codertocat --token 1234abcd ' + + '--labels label1,label2 --runnergroup Default', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('quotes runner labels with semicolon separators in non-ephemeral runner config', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + + await scaleUpModule.scaleUp([ + { + ...TEST_DATA_SINGLE, + labels: ['ghr-ec2-cpu-manufacturers:intel;amd'], + messageId: 'test-semicolon-labels', + }, + ]); + + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: + '--url https://github.enterprise.something/Codertocat --token 1234abcd ' + + "--labels 'label1,label2,ghr-ec2-cpu-manufacturers:intel;amd' --runnergroup Default", + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('should create JIT config for all remaining instances even when GitHub API fails for one instance', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '5'; + mockCreateRunner.mockImplementation(async () => { + return ['i-instance-1', 'i-instance-2', 'i-instance-3']; + }); + mockListRunners.mockImplementation(async () => { + return []; + }); + + mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ name }) => { + if (name === 'unit-test-i-instance-2') { + // Simulate a 503 Service Unavailable error from GitHub + const error = new Error('Service Unavailable') as Error & { + status: number; + response: { status: number; data: { message: string } }; + }; + error.status = 503; + error.response = { + status: 503, + data: { message: 'Service temporarily unavailable' }, + }; + throw error; + } + return { + data: { + runner: { id: 9876543210 }, + encoded_jit_config: `TEST_JIT_CONFIG_${name}`, + }, + headers: {}, + }; + }); + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toHaveBeenCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + name: 'unit-test-i-instance-1', + runner_group_id: 1, + labels: ['label1', 'label2'], + }); + + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toHaveBeenCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + name: 'unit-test-i-instance-2', + runner_group_id: 1, + labels: ['label1', 'label2'], + }); + + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toHaveBeenCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + name: 'unit-test-i-instance-3', + runner_group_id: 1, + labels: ['label1', 'label2'], + }); + + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-1', + Value: 'TEST_JIT_CONFIG_unit-test-i-instance-1', + Type: 'SecureString', + Tags: [{ Key: 'InstanceId', Value: 'i-instance-1' }], + }); + + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-3', + Value: 'TEST_JIT_CONFIG_unit-test-i-instance-3', + Type: 'SecureString', + Tags: [{ Key: 'InstanceId', Value: 'i-instance-3' }], + }); + + expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-2', + }); + }); + + it('should handle retryable errors with error handling logic', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '5'; + mockCreateRunner.mockImplementation(async () => { + return ['i-instance-1', 'i-instance-2']; + }); + mockListRunners.mockImplementation(async () => { + return []; + }); + + mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ name }) => { + if (name === 'unit-test-i-instance-1') { + const error = new Error('Internal Server Error') as Error & { + status: number; + response: { status: number; data: { message: string } }; + }; + error.status = 500; + error.response = { + status: 500, + data: { message: 'Internal server error' }, + }; + throw error; + } + return { + data: { + runner: { id: 9876543210 }, + encoded_jit_config: `TEST_JIT_CONFIG_${name}`, + }, + headers: {}, + }; + }); + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-2', + Value: 'TEST_JIT_CONFIG_unit-test-i-instance-2', + Type: 'SecureString', + Tags: [{ Key: 'InstanceId', Value: 'i-instance-2' }], + }); + + expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-1', + }); + }); + + it('should handle non-retryable 4xx errors gracefully', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '5'; + mockCreateRunner.mockImplementation(async () => { + return ['i-instance-1', 'i-instance-2']; + }); + mockListRunners.mockImplementation(async () => { + return []; + }); + + mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ name }) => { + if (name === 'unit-test-i-instance-1') { + // 404 is not retryable - will fail immediately + const error = new Error('Not Found') as Error & { + status: number; + response: { status: number; data: { message: string } }; + }; + error.status = 404; + error.response = { + status: 404, + data: { message: 'Resource not found' }, + }; + throw error; + } + return { + data: { + runner: { id: 9876543210 }, + encoded_jit_config: `TEST_JIT_CONFIG_${name}`, + }, + headers: {}, + }; + }); + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-2', + Value: 'TEST_JIT_CONFIG_unit-test-i-instance-2', + Type: 'SecureString', + Tags: [{ Key: 'InstanceId', Value: 'i-instance-2' }], + }); + + expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-instance-1', + }); + }); + + it.each(RUNNER_TYPES)( + 'calls create start runner config of 40' + ' instances (ssm rate limit condition) to test time delay ', + async (type: RunnerType) => { + process.env.ENABLE_EPHEMERAL_RUNNERS = type === 'ephemeral' ? 'true' : 'false'; + process.env.RUNNERS_MAXIMUM_COUNT = '40'; + mockCreateRunner.mockImplementation(async () => { + return instances; + }); + mockListRunners.mockImplementation(async () => { + return []; + }); + const startTime = performance.now(); + const instances = [ + 'i-1234', + 'i-5678', + 'i-5567', + 'i-5569', + 'i-5561', + 'i-5560', + 'i-5566', + 'i-5536', + 'i-5526', + 'i-5516', + 'i-122', + 'i-123', + 'i-124', + 'i-125', + 'i-126', + 'i-127', + 'i-128', + 'i-129', + 'i-130', + 'i-131', + 'i-132', + 'i-133', + 'i-134', + 'i-135', + 'i-136', + 'i-137', + 'i-138', + 'i-139', + 'i-140', + 'i-141', + 'i-142', + 'i-143', + 'i-144', + 'i-145', + 'i-146', + 'i-147', + 'i-148', + 'i-149', + 'i-150', + 'i-151', + ]; + await scaleUpModule.scaleUp(TEST_DATA); + const endTime = performance.now(); + expect(endTime - startTime).toBeGreaterThan(1000); + expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 40); }, + 10000, ); - expect(testProvider.createRunners).toHaveBeenCalledWith( - expect.objectContaining({ - githubInstallationClient: mockOctokit, - messages: [TEST_MESSAGE], - numberOfRunners: 1, - state: { prepared: true }, - githubRunnerConfig: expect.objectContaining({ - runnerLabels: 'base-label,ghr-provider-size:large', - runnerOwner: TEST_MESSAGE.repositoryOwner, - runnerType: 'Org', - }), - }), - ); - expect(mockedPublishRetryMessage).toHaveBeenCalledWith(expect.objectContaining({ messageId: 'message-1' })); }); - it('uses the default provider type when no provider type is configured', async () => { - await scaleUp([TEST_MESSAGE]); + describe('Dynamic EC2 Configuration', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + process.env.RUNNER_LABELS = 'base-label'; + process.env.INSTANCE_TYPES = 't3.medium,t3.large'; + process.env.RUNNER_NAME_PREFIX = 'unit-test'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + mockSSMClient.reset(); + }); + + it('appends EC2 labels to existing runner labels when EC2 labels are present', async () => { + const testDataWithEc2Labels = [ + { + ...TEST_DATA_SINGLE, + labels: ['ghr-ec2-instance-type:c5.2xlarge', 'ghr-ec2-custom:value'], + messageId: 'test-1', + }, + ]; - expect(mockedCreateScaleUpRunnerProviderFromEnv).toHaveBeenCalledWith( - 'test-provider', - 'unit-test-environment', - expect.any(Array), - ); + await scaleUpModule.scaleUp(testDataWithEc2Labels); + + // Verify createRunner was called with EC2 instance type in override config + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ + instanceTypes: ['t3.medium', 't3.large'], + }), + ec2OverrideConfig: expect.objectContaining({ + InstanceType: 'c5.2xlarge', + }), + }), + ); + expect(mockTag).toHaveBeenCalledWith('i-12345', [ + { + Key: 'ghr:github_runner_id', + Value: '9876543210', + }, + { + Key: 'ghr:runner_labels', + Value: 'base-label,ghr-ec2-instance-type:c5.2xlarge,ghr-ec2-custom:value', + }, + ]); + }); + + it('uses default instance types when no instance type EC2 label is provided', async () => { + const testDataWithEc2Labels = [ + { + ...TEST_DATA_SINGLE, + labels: ['ghr-ec2-custom:value'], + messageId: 'test-3', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithEc2Labels); + + // Should use the default INSTANCE_TYPES from environment + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ + instanceTypes: ['t3.medium', 't3.large'], + }), + }), + ); + }); + + it('loads the launch template block device name for dynamic EBS labels without DeviceName', async () => { + mockEC2Client.on(DescribeLaunchTemplateVersionsCommand).resolves({ + LaunchTemplateVersions: [ + { + LaunchTemplateData: { + BlockDeviceMappings: [ + { + DeviceName: '/dev/sdb', + VirtualName: 'ephemeral0', + }, + { + DeviceName: '/dev/sdf', + Ebs: {}, + }, + ], + }, + }, + ], + }); + + const testDataWithEbsLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['ghr-ec2-ebs-volume-size:100', 'ghr-ec2-ebs-volume-type:gp3'], + messageId: 'test-ebs-device-name', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithEbsLabels); + + expect(mockEC2Client).toHaveReceivedCommandWith(DescribeLaunchTemplateVersionsCommand, { + LaunchTemplateName: 'lt-1', + Versions: ['$Default'], + }); + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + BlockDeviceMappings: [ + { + DeviceName: '/dev/sdf', + Ebs: { + VolumeSize: 100, + VolumeType: 'gp3', + }, + }, + ], + }), + }), + ); + }); + + it('does not load launch template block device name when DeviceName is provided by labels', async () => { + const testDataWithEbsLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['ghr-ec2-block-device-name:/dev/sdg', 'ghr-ec2-ebs-volume-size:100', 'ghr-ec2-ebs-volume-type:gp3'], + messageId: 'test-explicit-ebs-device-name', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithEbsLabels); + + expect(mockEC2Client).not.toHaveReceivedCommand(DescribeLaunchTemplateVersionsCommand); + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + BlockDeviceMappings: [ + { + DeviceName: '/dev/sdg', + Ebs: { + VolumeSize: 100, + VolumeType: 'gp3', + }, + }, + ], + }), + }), + ); + }); + + it('handles messages with no labels gracefully', async () => { + const testDataWithNoLabels = [ + { + ...TEST_DATA_SINGLE, + labels: undefined, + messageId: 'test-5', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithNoLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ + instanceTypes: ['t3.medium', 't3.large'], + }), + }), + ); + }); + + it('handles empty labels array', async () => { + const testDataWithEmptyLabels = [ + { + ...TEST_DATA_SINGLE, + labels: [], + messageId: 'test-6', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithEmptyLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ + instanceTypes: ['t3.medium', 't3.large'], + }), + }), + ); + }); + + it('handles multiple EC2 labels correctly', async () => { + const testDataWithMultipleEc2Labels = [ + { + ...TEST_DATA_SINGLE, + labels: ['regular-label', 'ghr-ec2-instance-type:r5.2xlarge', 'ghr-ec2-ami:custom-ami', 'ghr-ec2-disk:200'], + messageId: 'test-8', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithMultipleEc2Labels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ + instanceTypes: ['t3.medium', 't3.large'], + }), + ec2OverrideConfig: expect.objectContaining({ + InstanceType: 'r5.2xlarge', + }), + }), + ); + }); + + it('includes ec2OverrideConfig with VCpuCount requirements when specified', async () => { + const testDataWithVCpuLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-vcpu-count-min:4', 'ghr-ec2-vcpu-count-max:16'], + messageId: 'test-9', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithVCpuLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + InstanceRequirements: expect.objectContaining({ + VCpuCount: { + Min: 4, + Max: 16, + }, + }), + }), + }), + ); + }); + + it('includes ec2OverrideConfig with MemoryMiB requirements when specified', async () => { + const testDataWithMemoryLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-memory-mib-min:8192', 'ghr-ec2-memory-mib-max:32768'], + messageId: 'test-10', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithMemoryLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + InstanceRequirements: expect.objectContaining({ + MemoryMiB: { + Min: 8192, + Max: 32768, + }, + }), + }), + }), + ); + }); + + it('includes ec2OverrideConfig with CPU manufacturers when specified', async () => { + const testDataWithCpuLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-cpu-manufacturers:intel;amd'], + messageId: 'test-11', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithCpuLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + InstanceRequirements: expect.objectContaining({ + CpuManufacturers: ['intel', 'amd'], + }), + }), + }), + ); + }); + + it('includes ec2OverrideConfig with instance generations when specified', async () => { + const testDataWithGenerationLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-instance-generations:current'], + messageId: 'test-12', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithGenerationLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + InstanceRequirements: expect.objectContaining({ + InstanceGenerations: ['current'], + }), + }), + }), + ); + }); + + it('includes ec2OverrideConfig with accelerator requirements when specified', async () => { + const testDataWithAcceleratorLabels = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-accelerator-count-min:1', 'ghr-ec2-accelerator-types:gpu'], + messageId: 'test-13', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithAcceleratorLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + InstanceRequirements: expect.objectContaining({ + AcceleratorCount: { + Min: 1, + }, + AcceleratorTypes: ['gpu'], + }), + }), + }), + ); + }); + + it('includes ec2OverrideConfig with max price when specified', async () => { + const testDataWithMaxPrice = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-max-price:0.50'], + messageId: 'test-14', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithMaxPrice); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + MaxPrice: '0.50', + }), + }), + ); + }); + + it('includes ec2OverrideConfig with priority and weighted capacity when specified', async () => { + const testDataWithPriorityWeight = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-priority:1', 'ghr-ec2-weighted-capacity:2'], + messageId: 'test-15', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithPriorityWeight); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + Priority: 1, + WeightedCapacity: 2, + }), + }), + ); + }); + + it('includes ec2OverrideConfig with combined requirements', async () => { + const testDataWithCombinedLabels = [ + { + ...TEST_DATA_SINGLE, + labels: [ + 'self-hosted', + 'linux', + 'ghr-ec2-vcpu-count-min:8', + 'ghr-ec2-memory-mib-min:16384', + 'ghr-ec2-cpu-manufacturers:intel', + 'ghr-ec2-instance-generations:current', + 'ghr-ec2-max-price:1.00', + ], + messageId: 'test-16', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithCombinedLabels); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2OverrideConfig: expect.objectContaining({ + InstanceRequirements: expect.objectContaining({ + VCpuCount: { Min: 8 }, + MemoryMiB: { Min: 16384 }, + CpuManufacturers: ['intel'], + InstanceGenerations: ['current'], + }), + MaxPrice: '1.00', + }), + }), + ); + }); + + it('includes both instance type and ec2OverrideConfig when both specified', async () => { + const testDataWithBoth = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-instance-type:c5.xlarge', 'ghr-ec2-vcpu-count-min:4'], + messageId: 'test-18', + }, + ]; + + await scaleUpModule.scaleUp(testDataWithBoth); + + expect(createRunner).toBeCalledWith( + expect.objectContaining({ + ec2instanceCriteria: expect.objectContaining({ + instanceTypes: ['t3.medium', 't3.large'], + }), + ec2OverrideConfig: expect.objectContaining({ + InstanceType: 'c5.xlarge', + InstanceRequirements: expect.objectContaining({ + VCpuCount: { Min: 4 }, + }), + }), + }), + ); + }); + + it('does not accumulate labels across groups when multiple messages have different dynamic labels', async () => { + const testDataMultipleGroups = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:m7a.large', 'ghr-job-id:run-1-inst-0'], + messageId: 'msg-1', + }, + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:m7i.xlarge', 'ghr-job-id:run-1-inst-1'], + messageId: 'msg-2', + }, + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:c7a.large', 'ghr-job-id:run-1-inst-2'], + messageId: 'msg-3', + }, + ]; + + await scaleUpModule.scaleUp(testDataMultipleGroups); + + expect(createRunner).toBeCalledTimes(3); + + const jitCalls = mockOctokit.actions.generateRunnerJitconfigForOrg.mock.calls; + expect(jitCalls).toHaveLength(3); + + for (const call of jitCalls) { + const labels = call[0].labels as string[]; + + if (labels.includes('ghr-ec2-instance-type:m7a.large')) { + expect(labels).toContain('ghr-job-id:run-1-inst-0'); + expect(labels).not.toContain('ghr-job-id:run-1-inst-1'); + expect(labels).not.toContain('ghr-job-id:run-1-inst-2'); + expect(labels).not.toContain('ghr-ec2-instance-type:m7i.xlarge'); + expect(labels).not.toContain('ghr-ec2-instance-type:c7a.large'); + } else if (labels.includes('ghr-ec2-instance-type:m7i.xlarge')) { + expect(labels).toContain('ghr-job-id:run-1-inst-1'); + expect(labels).not.toContain('ghr-job-id:run-1-inst-0'); + expect(labels).not.toContain('ghr-job-id:run-1-inst-2'); + expect(labels).not.toContain('ghr-ec2-instance-type:m7a.large'); + expect(labels).not.toContain('ghr-ec2-instance-type:c7a.large'); + } else if (labels.includes('ghr-ec2-instance-type:c7a.large')) { + expect(labels).toContain('ghr-job-id:run-1-inst-2'); + expect(labels).not.toContain('ghr-job-id:run-1-inst-0'); + expect(labels).not.toContain('ghr-job-id:run-1-inst-1'); + expect(labels).not.toContain('ghr-ec2-instance-type:m7a.large'); + expect(labels).not.toContain('ghr-ec2-instance-type:m7i.xlarge'); + } else { + throw new Error(`Unexpected labels combination: ${labels.join(',')}`); + } + } + }); + + it('preserves base RUNNER_LABELS for each group without mutation', async () => { + process.env.RUNNER_LABELS = 'ubuntu-2404,x64'; + + const testDataTwoGroups = [ + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-instance-type:m7a.large', 'ghr-team:alpha'], + messageId: 'msg-a', + }, + { + ...TEST_DATA_SINGLE, + labels: ['self-hosted', 'ghr-ec2-instance-type:c7i.large', 'ghr-team:beta'], + messageId: 'msg-b', + }, + ]; + + await scaleUpModule.scaleUp(testDataTwoGroups); + + expect(createRunner).toBeCalledTimes(2); + + const jitCalls = mockOctokit.actions.generateRunnerJitconfigForOrg.mock.calls; + expect(jitCalls).toHaveLength(2); + + for (const call of jitCalls) { + const labels = call[0].labels as string[]; + + expect(labels).toContain('ubuntu-2404'); + expect(labels).toContain('x64'); + + if (labels.includes('ghr-team:alpha')) { + expect(labels).not.toContain('ghr-team:beta'); + expect(labels).not.toContain('ghr-ec2-instance-type:c7i.large'); + } else if (labels.includes('ghr-team:beta')) { + expect(labels).not.toContain('ghr-team:alpha'); + expect(labels).not.toContain('ghr-ec2-instance-type:m7a.large'); + } else { + throw new Error(`Unexpected labels combination: ${labels.join(',')}`); + } + } + }); }); - it('resolves installation again when the event installation belongs to another app', async () => { - process.env.GHES_URL = 'https://github.enterprise.something'; - mockOctokit.apps.getOrgInstallation.mockResolvedValue({ data: { id: 123 } }); - mockedCreateGithubInstallationAuth.mockRejectedValueOnce({ status: 404 }).mockResolvedValueOnce({ - type: 'token', - tokenType: 'installation', - token: 'installation-token', - createdAt: 'some-date', - expiresAt: 'some-date', - permissions: {}, - repositorySelection: 'all', - installationId: 123, - }); - - await scaleUp([TEST_MESSAGE]); - - expect(mockOctokit.apps.getOrgInstallation).toHaveBeenCalledWith({ org: TEST_MESSAGE.repositoryOwner }); - expect(mockedCreateGithubInstallationAuth).toHaveBeenNthCalledWith( - 1, - TEST_MESSAGE.installationId, - 'https://github.enterprise.something/api/v3', - ); - expect(mockedCreateGithubInstallationAuth).toHaveBeenNthCalledWith( - 2, - 123, - 'https://github.enterprise.something/api/v3', - ); - expect(testProvider.createRunners).toHaveBeenCalledTimes(1); + describe('on repo level', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + process.env.RUNNER_NAME_PREFIX = 'unit-test'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + expectedRunnerParams.runnerType = 'Repo'; + expectedRunnerParams.runnerOwner = `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`; + // `--url https://github.enterprise.something/${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, + // `--token 1234abcd`, + // ]; + }); + + it('gets the current repo level runners', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).toBeCalledWith({ + environment: 'unit-test-environment', + runnerType: 'Repo', + runnerOwner: `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, + }); + }); + + it('does not create a token when maximum runners has been reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a token when maximum runners has not been reached', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('uses the default runner max count', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = undefined; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('creates a runner with correct config and labels', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner and ensure the group argument is ignored', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('Check error is thrown', async () => { + const mockCreateRunners = vi.mocked(createRunner); + mockCreateRunners.mockRejectedValue(new Error('no retry')); + await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow('no retry'); + mockCreateRunners.mockReset(); + }); }); - it('does not query current provider runners when the maximum runner count is unlimited', async () => { - process.env.RUNNERS_MAXIMUM_COUNT = '-1'; - testProvider.createRunners.mockResolvedValue(['runner-1', 'runner-2']); - const secondMessage: ActionRequestMessageSQS = { - ...TEST_MESSAGE, - id: 2, - messageId: 'message-2', + describe('Batch processing', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.RUNNERS_MAXIMUM_COUNT = '10'; + }); + + const createTestMessages = ( + count: number, + overrides: Partial[] = [], + ): ActionRequestMessageSQS[] => { + return Array.from({ length: count }, (_, i) => ({ + ...TEST_DATA_SINGLE, + id: i + 1, + messageId: `message-${i}`, + ...overrides[i], + })); }; - await scaleUp([TEST_MESSAGE, secondMessage]); + it('Should handle multiple messages for the same organization', async () => { + const messages = createTestMessages(3); + await scaleUpModule.scaleUp(messages); - expect(testProvider.getCurrentRunners).not.toHaveBeenCalled(); - expect(testProvider.createRunners).toHaveBeenCalledWith(expect.objectContaining({ numberOfRunners: 2 })); - }); + expect(createRunner).toHaveBeenCalledTimes(1); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 3, + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }), + ); + }); - it('does not ask the provider to create runners when no jobs are queued', async () => { - mockOctokit.actions.getJobForWorkflowRun.mockResolvedValue({ - data: { status: 'completed' }, - headers: {}, + it('Should handle multiple messages for different organizations', async () => { + const messages = createTestMessages(3, [ + { repositoryOwner: 'org1' }, + { repositoryOwner: 'org2' }, + { repositoryOwner: 'org1' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(2); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, + runnerOwner: 'org1', + }), + ); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, + runnerOwner: 'org2', + }), + ); + }); + + it('Should handle multiple messages for different repositories when org-level is disabled', async () => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + const messages = createTestMessages(3, [ + { repositoryOwner: 'owner1', repositoryName: 'repo1' }, + { repositoryOwner: 'owner1', repositoryName: 'repo2' }, + { repositoryOwner: 'owner1', repositoryName: 'repo1' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(2); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, + runnerOwner: 'owner1/repo1', + }), + ); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, + runnerOwner: 'owner1/repo2', + }), + ); + }); + + it('Should reject messages when maximum runners limit is reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; // Set to 1 so with 1 existing, no new ones can be created + mockListRunners.mockImplementation(async () => [ + { + instanceId: 'i-existing', + launchTime: new Date(), + type: 'Org', + owner: TEST_DATA_SINGLE.repositoryOwner, + }, + ]); + + const messages = createTestMessages(3); + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).not.toHaveBeenCalled(); // No runners should be created + expect(rejectedMessages).toHaveLength(3); // All 3 messages should be rejected }); - await scaleUp([TEST_MESSAGE]); + it('Should handle partial EC2 instance creation failures', async () => { + mockCreateRunner.mockImplementation(async () => ['i-12345']); // Only creates 1 instead of requested 3 + + const messages = createTestMessages(3); + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(rejectedMessages).toHaveLength(2); // 3 requested - 1 created = 2 failed + expect(rejectedMessages).toEqual(['message-0', 'message-1']); + }); + + it('Should filter out invalid event types for ephemeral runners', async () => { + const messages = createTestMessages(3, [ + { eventType: 'workflow_job' }, + { eventType: 'check_run' }, + { eventType: 'workflow_job' }, + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only workflow_job events processed + }), + ); + expect(rejectedMessages).toContain('message-1'); // check_run event rejected + }); + + it('Should skip invalid repo owner types but not reject them', async () => { + const messages = createTestMessages(3, [ + { repoOwnerType: 'Organization' }, + { repoOwnerType: 'User' }, // Invalid for org-level runners + { repoOwnerType: 'Organization' }, + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only Organization events processed + }), + ); + expect(rejectedMessages).not.toContain('message-1'); // User repo not rejected, just skipped + }); + + it('Should skip messages when jobs are not queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { + const isQueued = params.job_id === 1 || params.job_id === 3; // Only jobs 1 and 3 are queued + return { + data: { + status: isQueued ? 'queued' : 'completed', + }, + }; + }); + + const messages = createTestMessages(3); + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only queued jobs processed + }), + ); + }); + + it('Should create separate GitHub clients for different installations', async () => { + // Override the default mock to return different installation IDs + mockOctokit.apps.getOrgInstallation.mockReset(); + mockOctokit.apps.getOrgInstallation.mockImplementation((params) => ({ + data: { + id: params.org === 'org1' ? 100 : 200, + }, + })); + + const messages = createTestMessages(2, [ + { repositoryOwner: 'org1', installationId: 0 }, + { repositoryOwner: 'org2', installationId: 0 }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients + expect(mockedInstallationAuth).toHaveBeenCalledWith(100, 'https://github.enterprise.something/api/v3'); + expect(mockedInstallationAuth).toHaveBeenCalledWith(200, 'https://github.enterprise.something/api/v3'); + }); + + it('Should resolve installation again when event installation belongs to another app', async () => { + mockOctokit.apps.getOrgInstallation.mockReset(); + mockOctokit.apps.getOrgInstallation.mockImplementation(() => ({ + data: { + id: 123, + }, + })); + + mockedInstallationAuth.mockRejectedValueOnce({ status: 404 }).mockResolvedValueOnce({ + type: 'token', + tokenType: 'installation', + token: 'token', + createdAt: 'some-date', + expiresAt: 'some-date', + permissions: {}, + repositorySelection: 'all', + installationId: 123, + }); + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockOctokit.apps.getOrgInstallation).toHaveBeenCalledWith({ org: TEST_DATA_SINGLE.repositoryOwner }); + expect(mockedInstallationAuth).toHaveBeenNthCalledWith( + 1, + TEST_DATA_SINGLE.installationId, + 'https://github.enterprise.something/api/v3', + ); + expect(mockedInstallationAuth).toHaveBeenNthCalledWith(2, 123, 'https://github.enterprise.something/api/v3'); + }); + + it('Should reuse GitHub clients for same installation', async () => { + const messages = createTestMessages(3, [ + { repositoryOwner: 'same-org' }, + { repositoryOwner: 'same-org' }, + { repositoryOwner: 'same-org' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(mockCreateClient).toHaveBeenCalledTimes(2); // 1 app client, 1 installation client + expect(mockedInstallationAuth).toHaveBeenCalledTimes(1); + }); + + it('Should return empty array when no valid messages to process', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + const messages = createTestMessages(2, [ + { eventType: 'check_run' }, // Invalid for ephemeral + { eventType: 'check_run' }, // Invalid for ephemeral + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).not.toHaveBeenCalled(); + expect(rejectedMessages).toEqual(['message-0', 'message-1']); + }); + + it('Should handle unlimited runners configuration', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '-1'; + const messages = createTestMessages(10); + + await scaleUpModule.scaleUp(messages); + + expect(listEC2Runners).not.toHaveBeenCalled(); // No need to check current runners + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 10, // All messages processed + }), + ); + }); + }); +}); + +describe('scaleUp with public GH', () => { + it('checks queued workflows', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalledWith({ + job_id: TEST_DATA_SINGLE.id, + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('not checking queued workflows', async () => { + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); + }); + + it('does not list runners when no workflows are queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ + data: { status: 'completed' }, + })); + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).not.toBeCalled(); + }); + + describe('on org level', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.RUNNER_NAME_PREFIX = 'unit-test'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + }); + + it('gets the current org level runners', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).toBeCalledWith({ + environment: 'unit-test-environment', + runnerType: 'Org', + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }); + }); + + it('does not create a token when maximum runners has been reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a token when maximum runners has not been reached', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + }); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a runner with correct config', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner with labels in s specific group', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + }); + + describe('on repo level', () => { + beforeEach(() => { + mockSSMClient.reset(); + + process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + process.env.RUNNER_NAME_PREFIX = 'unit-test'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + expectedRunnerParams.runnerType = 'Repo'; + expectedRunnerParams.runnerOwner = `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`; + }); + + it('gets the current repo level runners', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).toBeCalledWith({ + environment: 'unit-test-environment', + runnerType: 'Repo', + runnerOwner: `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, + }); + }); + + it('does not create a token when maximum runners has been reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a token when maximum runners has not been reached', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('creates a runner with correct config and labels', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner with correct config and labels and on demand failover enabled.', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = JSON.stringify(['InsufficientInstanceCapacity']); + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith({ + ...expectedRunnerParams, + onDemandFailoverOnError: ['InsufficientInstanceCapacity'], + }); + }); + + it('creates a runner with correct config and labels and custom scale errors enabled.', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.SCALE_ERRORS = JSON.stringify(['RequestLimitExceeded']); + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith({ + ...expectedRunnerParams, + scaleErrors: ['RequestLimitExceeded'], + }); + }); + + it('creates a runner and ensure the group argument is ignored', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('ephemeral runners only run with workflow_job event, others should fail.', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + + const USER_REPO_TEST_DATA = structuredClone(TEST_DATA); + USER_REPO_TEST_DATA[0].eventType = 'check_run'; + + await expect(scaleUpModule.scaleUp(USER_REPO_TEST_DATA)).resolves.toEqual(['foobar']); + }); + + it('creates a ephemeral runner with JIT config.', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: 'TEST_JIT_CONFIG_REPO', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('creates a ephemeral runner with registration token.', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.ENABLE_JIT_CONFIG = 'false'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --ephemeral', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('JIT config is ignored for non-ephemeral runners.', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + process.env.ENABLE_JIT_CONFIG = 'true'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + process.env.RUNNER_LABELS = 'jit'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --labels jit', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('creates a ephemeral runner after checking job is queued.', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'true'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalled(); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('disable auto update on the runner.', async () => { + process.env.DISABLE_RUNNER_AUTOUPDATE = 'true'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('Scaling error should return failed message IDs so retry can be triggered.', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + await expect(scaleUpModule.scaleUp(TEST_DATA)).resolves.toEqual(['foobar']); + }); + }); + + describe('Batch processing', () => { + const createTestMessages = ( + count: number, + overrides: Partial[] = [], + ): ActionRequestMessageSQS[] => { + return Array.from({ length: count }, (_, i) => ({ + ...TEST_DATA_SINGLE, + id: i + 1, + messageId: `message-${i}`, + ...overrides[i], + })); + }; + + beforeEach(() => { + setDefaults(); + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.RUNNERS_MAXIMUM_COUNT = '10'; + }); + + it('Should handle multiple messages for the same organization', async () => { + const messages = createTestMessages(3); + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(1); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 3, + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }), + ); + }); + + it('Should handle multiple messages for different organizations', async () => { + const messages = createTestMessages(3, [ + { repositoryOwner: 'org1' }, + { repositoryOwner: 'org2' }, + { repositoryOwner: 'org1' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(2); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, + runnerOwner: 'org1', + }), + ); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, + runnerOwner: 'org2', + }), + ); + }); + + it('Should handle multiple messages for different repositories when org-level is disabled', async () => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + const messages = createTestMessages(3, [ + { repositoryOwner: 'owner1', repositoryName: 'repo1' }, + { repositoryOwner: 'owner1', repositoryName: 'repo2' }, + { repositoryOwner: 'owner1', repositoryName: 'repo1' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(2); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, + runnerOwner: 'owner1/repo1', + }), + ); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, + runnerOwner: 'owner1/repo2', + }), + ); + }); + + it('Should reject messages when maximum runners limit is reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; // Set to 1 so with 1 existing, no new ones can be created + mockListRunners.mockImplementation(async () => [ + { + instanceId: 'i-existing', + launchTime: new Date(), + type: 'Org', + owner: TEST_DATA_SINGLE.repositoryOwner, + }, + ]); + + const messages = createTestMessages(3); + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).not.toHaveBeenCalled(); // No runners should be created + expect(rejectedMessages).toHaveLength(3); // All 3 messages should be rejected + }); + + it('Should handle partial EC2 instance creation failures', async () => { + mockCreateRunner.mockImplementation(async () => ['i-12345']); // Only creates 1 instead of requested 3 + + const messages = createTestMessages(3); + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(rejectedMessages).toHaveLength(2); // 3 requested - 1 created = 2 failed + expect(rejectedMessages).toEqual(['message-0', 'message-1']); + }); + + it('Should filter out invalid event types for ephemeral runners', async () => { + const messages = createTestMessages(3, [ + { eventType: 'workflow_job' }, + { eventType: 'check_run' }, + { eventType: 'workflow_job' }, + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only workflow_job events processed + }), + ); + expect(rejectedMessages).toContain('message-1'); // check_run event rejected + }); + + it('Should skip invalid repo owner types but not reject them', async () => { + const messages = createTestMessages(3, [ + { repoOwnerType: 'Organization' }, + { repoOwnerType: 'User' }, // Invalid for org-level runners + { repoOwnerType: 'Organization' }, + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only Organization events processed + }), + ); + expect(rejectedMessages).not.toContain('message-1'); // User repo not rejected, just skipped + }); + + it('Should skip messages when jobs are not queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { + const isQueued = params.job_id === 1 || params.job_id === 3; // Only jobs 1 and 3 are queued + return { + data: { + status: isQueued ? 'queued' : 'completed', + }, + }; + }); + + const messages = createTestMessages(3); + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only queued jobs processed + }), + ); + }); + + it('Should create separate GitHub clients for different installations', async () => { + // Override the default mock to return different installation IDs + mockOctokit.apps.getOrgInstallation.mockReset(); + mockOctokit.apps.getOrgInstallation.mockImplementation((params) => ({ + data: { + id: params.org === 'org1' ? 100 : 200, + }, + })); + + const messages = createTestMessages(2, [ + { repositoryOwner: 'org1', installationId: 0 }, + { repositoryOwner: 'org2', installationId: 0 }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients + expect(mockedInstallationAuth).toHaveBeenCalledWith(100, ''); + expect(mockedInstallationAuth).toHaveBeenCalledWith(200, ''); + }); + + it('Should reuse GitHub clients for same installation', async () => { + const messages = createTestMessages(3, [ + { repositoryOwner: 'same-org' }, + { repositoryOwner: 'same-org' }, + { repositoryOwner: 'same-org' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(mockCreateClient).toHaveBeenCalledTimes(2); // 1 app client, 1 installation client + expect(mockedInstallationAuth).toHaveBeenCalledTimes(1); + }); + + it('Should return empty array when no valid messages to process', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + const messages = createTestMessages(2, [ + { eventType: 'check_run' }, // Invalid for ephemeral + { eventType: 'check_run' }, // Invalid for ephemeral + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).not.toHaveBeenCalled(); + expect(rejectedMessages).toEqual(['message-0', 'message-1']); + }); + + it('Should handle unlimited runners configuration', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '-1'; + const messages = createTestMessages(10); + + await scaleUpModule.scaleUp(messages); + + expect(listEC2Runners).not.toHaveBeenCalled(); // No need to check current runners + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 10, // All messages processed + }), + ); + }); + }); +}); + +describe('scaleUp with Github Data Residency', () => { + beforeEach(() => { + process.env.GHES_URL = 'https://companyname.ghe.com'; + }); + + it('checks queued workflows', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.getJobForWorkflowRun).toBeCalledWith({ + job_id: TEST_DATA_SINGLE.id, + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('does not list runners when no workflows are queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ + data: { total_count: 0 }, + })); + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).not.toBeCalled(); + }); + + describe('on org level', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.RUNNER_NAME_PREFIX = 'unit-test-'; + process.env.RUNNER_GROUP_NAME = 'Default'; + process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; + process.env.RUNNER_LABELS = 'label1,label2'; + + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + mockSSMClient.reset(); + }); + + it('gets the current org level runners', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).toBeCalledWith({ + environment: 'unit-test-environment', + runnerType: 'Org', + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }); + }); + + it('does not create a token when maximum runners has been reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('does create a runner if maximum is set to -1', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '-1'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).not.toHaveBeenCalled(); + expect(createRunner).toHaveBeenCalled(); + }); + + it('creates a token when maximum runners has not been reached', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + }); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a runner with correct config', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner with labels in a specific group', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner with ami id override from ssm parameter', async () => { + process.env.AMI_ID_SSM_PARAMETER_NAME = 'my-ami-id-param'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith({ ...expectedRunnerParams, amiIdSsmParameterName: 'my-ami-id-param' }); + }); + + it('Throws an error if runner group does not exist for ephemeral runners', async () => { + process.env.RUNNER_GROUP_NAME = 'test-runner-group'; + mockSSMgetParameter.mockImplementation(async () => { + throw new Error('ParameterNotFound'); + }); + await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toBeInstanceOf(Error); + expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); + }); + + it('Discards event if it is a User repo and org level runners is enabled', async () => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + const USER_REPO_TEST_DATA = structuredClone(TEST_DATA); + USER_REPO_TEST_DATA[0].repoOwnerType = 'User'; + await scaleUpModule.scaleUp(USER_REPO_TEST_DATA); + expect(createRunner).not.toHaveBeenCalled(); + }); + + it('create SSM parameter for runner group id if it does not exist', async () => { + mockSSMgetParameter.mockImplementation(async () => { + throw new Error('ParameterNotFound'); + }); + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); + expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 2); + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`, + Value: '1', + Type: 'String', + }); + }); + + it('Does not create SSM parameter for runner group id if it exists', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.paginate).toHaveBeenCalledTimes(0); + expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 1); + }); + + it('create start runner config for ephemeral runners ', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).toBeCalledWith({ + org: TEST_DATA_SINGLE.repositoryOwner, + name: 'unit-test-i-12345', + runner_group_id: 1, + labels: ['label1', 'label2'], + }); + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: 'TEST_JIT_CONFIG_ORG', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + + it('create start runner config for non-ephemeral runners ', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.generateRunnerJitconfigForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalled(); + expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { + Name: '/github-action-runners/default/runners/config/i-12345', + Value: + '--url https://companyname.ghe.com/Codertocat --token 1234abcd ' + + '--labels label1,label2 --runnergroup Default', + Type: 'SecureString', + Tags: [ + { + Key: 'InstanceId', + Value: 'i-12345', + }, + ], + }); + }); + it.each(RUNNER_TYPES)( + 'calls create start runner config of 40' + ' instances (ssm rate limit condition) to test time delay ', + async (type: RunnerType) => { + process.env.ENABLE_EPHEMERAL_RUNNERS = type === 'ephemeral' ? 'true' : 'false'; + process.env.RUNNERS_MAXIMUM_COUNT = '40'; + mockCreateRunner.mockImplementation(async () => { + return instances; + }); + mockListRunners.mockImplementation(async () => { + return []; + }); + const startTime = performance.now(); + const instances = [ + 'i-1234', + 'i-5678', + 'i-5567', + 'i-5569', + 'i-5561', + 'i-5560', + 'i-5566', + 'i-5536', + 'i-5526', + 'i-5516', + 'i-122', + 'i-123', + 'i-124', + 'i-125', + 'i-126', + 'i-127', + 'i-128', + 'i-129', + 'i-130', + 'i-131', + 'i-132', + 'i-133', + 'i-134', + 'i-135', + 'i-136', + 'i-137', + 'i-138', + 'i-139', + 'i-140', + 'i-141', + 'i-142', + 'i-143', + 'i-144', + 'i-145', + 'i-146', + 'i-147', + 'i-148', + 'i-149', + 'i-150', + 'i-151', + ]; + await scaleUpModule.scaleUp(TEST_DATA); + const endTime = performance.now(); + expect(endTime - startTime).toBeGreaterThan(1000); + expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 40); + }, + 10000, + ); + }); + describe('on repo level', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + process.env.RUNNER_NAME_PREFIX = 'unit-test'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + expectedRunnerParams.runnerType = 'Repo'; + expectedRunnerParams.runnerOwner = `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`; + // `--url https://companyname.ghe.com${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, + // `--token 1234abcd`, + // ]; + }); + + it('gets the current repo level runners', async () => { + await scaleUpModule.scaleUp(TEST_DATA); + expect(listEC2Runners).toBeCalledWith({ + environment: 'unit-test-environment', + runnerType: 'Repo', + runnerOwner: `${TEST_DATA_SINGLE.repositoryOwner}/${TEST_DATA_SINGLE.repositoryName}`, + }); + }); + + it('does not create a token when maximum runners has been reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '1'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); + }); + + it('creates a token when maximum runners has not been reached', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); + expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('uses the default runner max count', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = undefined; + await scaleUpModule.scaleUp(TEST_DATA); + expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ + owner: TEST_DATA_SINGLE.repositoryOwner, + repo: TEST_DATA_SINGLE.repositoryName, + }); + }); + + it('creates a runner with correct config and labels', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('creates a runner and ensure the group argument is ignored', async () => { + process.env.RUNNER_LABELS = 'label1,label2'; + process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + + it('Check error is thrown', async () => { + const mockCreateRunners = vi.mocked(createRunner); + mockCreateRunners.mockRejectedValue(new Error('no retry')); + await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow('no retry'); + mockCreateRunners.mockReset(); + }); + }); + + describe('Batch processing', () => { + const createTestMessages = ( + count: number, + overrides: Partial[] = [], + ): ActionRequestMessageSQS[] => { + return Array.from({ length: count }, (_, i) => ({ + ...TEST_DATA_SINGLE, + id: i + 1, + messageId: `message-${i}`, + ...overrides[i], + })); + }; + + beforeEach(() => { + setDefaults(); + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.RUNNERS_MAXIMUM_COUNT = '10'; + }); + + it('Should handle multiple messages for the same organization', async () => { + const messages = createTestMessages(3); + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(1); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 3, + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }), + ); + }); + + it('Should handle multiple messages for different organizations', async () => { + const messages = createTestMessages(3, [ + { repositoryOwner: 'org1' }, + { repositoryOwner: 'org2' }, + { repositoryOwner: 'org1' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(2); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, + runnerOwner: 'org1', + }), + ); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, + runnerOwner: 'org2', + }), + ); + }); + + it('Should handle multiple messages for different repositories when org-level is disabled', async () => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + const messages = createTestMessages(3, [ + { repositoryOwner: 'owner1', repositoryName: 'repo1' }, + { repositoryOwner: 'owner1', repositoryName: 'repo2' }, + { repositoryOwner: 'owner1', repositoryName: 'repo1' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledTimes(2); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, + runnerOwner: 'owner1/repo1', + }), + ); + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, + runnerOwner: 'owner1/repo2', + }), + ); + }); + + it('Should reject messages when maximum runners limit is reached', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + mockListRunners.mockImplementation(async () => [ + { + instanceId: 'i-existing', + launchTime: new Date(), + type: 'Org', + owner: TEST_DATA_SINGLE.repositoryOwner, + }, + ]); + + const messages = createTestMessages(5); + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 1, // 2 max - 1 existing = 1 new + }), + ); + expect(rejectedMessages).toHaveLength(4); // 5 requested - 1 created = 4 rejected + }); + + it('Should handle partial EC2 instance creation failures', async () => { + mockCreateRunner.mockImplementation(async () => ['i-12345']); // Only creates 1 instead of requested 3 + + const messages = createTestMessages(3); + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(rejectedMessages).toHaveLength(2); // 3 requested - 1 created = 2 failed + expect(rejectedMessages).toEqual(['message-0', 'message-1']); + }); + + it('Should filter out invalid event types for ephemeral runners', async () => { + const messages = createTestMessages(3, [ + { eventType: 'workflow_job' }, + { eventType: 'check_run' }, + { eventType: 'workflow_job' }, + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only workflow_job events processed + }), + ); + expect(rejectedMessages).toContain('message-1'); // check_run event rejected + }); + + it('Should skip invalid repo owner types but not reject them', async () => { + const messages = createTestMessages(3, [ + { repoOwnerType: 'Organization' }, + { repoOwnerType: 'User' }, // Invalid for org-level runners + { repoOwnerType: 'Organization' }, + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only Organization events processed + }), + ); + expect(rejectedMessages).not.toContain('message-1'); // User repo not rejected, just skipped + }); + + it('Should skip messages when jobs are not queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { + const isQueued = params.job_id === 1 || params.job_id === 3; // Only jobs 1 and 3 are queued + return { + data: { + status: isQueued ? 'queued' : 'completed', + }, + }; + }); + + const messages = createTestMessages(3); + await scaleUpModule.scaleUp(messages); + + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 2, // Only queued jobs processed + }), + ); + }); + + it('Should create separate GitHub clients for different installations', async () => { + mockOctokit.apps.getOrgInstallation.mockImplementation((params) => ({ + data: { + id: params.org === 'org1' ? 100 : 200, + }, + })); + + const messages = createTestMessages(2, [ + { repositoryOwner: 'org1', installationId: 0 }, + { repositoryOwner: 'org2', installationId: 0 }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(mockCreateClient).toHaveBeenCalledTimes(3); // 1 app client, 2 repo installation clients + expect(mockedInstallationAuth).toHaveBeenCalledWith(100, ''); + expect(mockedInstallationAuth).toHaveBeenCalledWith(200, ''); + }); + + it('Should reuse GitHub clients for same installation', async () => { + const messages = createTestMessages(3, [ + { repositoryOwner: 'same-org' }, + { repositoryOwner: 'same-org' }, + { repositoryOwner: 'same-org' }, + ]); + + await scaleUpModule.scaleUp(messages); + + expect(mockCreateClient).toHaveBeenCalledTimes(2); // 1 app client, 1 installation client + expect(mockedInstallationAuth).toHaveBeenCalledTimes(1); + }); + + it('Should return empty array when no valid messages to process', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + const messages = createTestMessages(2, [ + { eventType: 'check_run' }, // Invalid for ephemeral + { eventType: 'check_run' }, // Invalid for ephemeral + ]); + + const rejectedMessages = await scaleUpModule.scaleUp(messages); + + expect(createRunner).not.toHaveBeenCalled(); + expect(rejectedMessages).toEqual(['message-0', 'message-1']); + }); + + it('Should handle unlimited runners configuration', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '-1'; + const messages = createTestMessages(10); + + await scaleUpModule.scaleUp(messages); + + expect(listEC2Runners).not.toHaveBeenCalled(); // No need to check current runners + expect(createRunner).toHaveBeenCalledWith( + expect.objectContaining({ + numberOfRunners: 10, // All messages processed + }), + ); + }); + }); +}); + +describe('Retry mechanism tests', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.ENABLE_JOB_QUEUED_CHECK = 'true'; + process.env.RUNNERS_MAXIMUM_COUNT = '10'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + mockSSMClient.reset(); + }); + + const createTestMessages = ( + count: number, + overrides: Partial[] = [], + ): ActionRequestMessageSQS[] => { + return Array.from({ length: count }, (_, i) => ({ + ...TEST_DATA_SINGLE, + id: i + 1, + messageId: `message-${i + 1}`, + ...overrides[i], + })); + }; + + it('calls publishRetryMessage for each valid message when job is queued', async () => { + const messages = createTestMessages(3); + mockCreateRunner.mockResolvedValue(['i-12345', 'i-67890', 'i-abcdef']); // Create all requested runners + + await scaleUpModule.scaleUp(messages); + + expect(mockPublishRetryMessage).toHaveBeenCalledTimes(3); + expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + id: 1, + messageId: 'message-1', + }), + ); + expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + id: 2, + messageId: 'message-2', + }), + ); + expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( + 3, + expect.objectContaining({ + id: 3, + messageId: 'message-3', + }), + ); + }); + + it('does not call publishRetryMessage when job is not queued', async () => { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation((params) => { + const isQueued = params.job_id === 1; // Only job 1 is queued + return { + data: { + status: isQueued ? 'queued' : 'completed', + }, + }; + }); + + const messages = createTestMessages(3); + + await scaleUpModule.scaleUp(messages); + + // Only message with id 1 should trigger retry + expect(mockPublishRetryMessage).toHaveBeenCalledTimes(1); + expect(mockPublishRetryMessage).toHaveBeenCalledWith( + expect.objectContaining({ + id: 1, + messageId: 'message-1', + }), + ); + }); + + it('does not call publishRetryMessage when maximum runners is reached and messages are marked invalid', async () => { + process.env.RUNNERS_MAXIMUM_COUNT = '0'; // No runners can be created + + const messages = createTestMessages(2); + + await scaleUpModule.scaleUp(messages); + + // Verify listEC2Runners is called to check current runner count + expect(listEC2Runners).toHaveBeenCalledWith({ + environment: 'unit-test-environment', + runnerType: 'Org', + runnerOwner: TEST_DATA_SINGLE.repositoryOwner, + }); + + // publishRetryMessage should NOT be called because messages are marked as invalid + // Invalid messages go back to the SQS queue and will be retried there + expect(mockPublishRetryMessage).not.toHaveBeenCalled(); + expect(createRunner).not.toHaveBeenCalled(); + }); + + it('calls publishRetryMessage with correct message structure including retry counter', async () => { + const message = { + ...TEST_DATA_SINGLE, + messageId: 'test-message-id', + retryCounter: 2, + }; + + await scaleUpModule.scaleUp([message]); + + expect(mockPublishRetryMessage).toHaveBeenCalledWith( + expect.objectContaining({ + id: message.id, + messageId: 'test-message-id', + retryCounter: 2, + }), + ); + }); + + it('calls publishRetryMessage when ENABLE_JOB_QUEUED_CHECK is false', async () => { + process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; + mockCreateRunner.mockResolvedValue(['i-12345', 'i-67890']); // Create all requested runners + + const messages = createTestMessages(2); + + await scaleUpModule.scaleUp(messages); + + // Should always call publishRetryMessage when queue check is disabled + expect(mockPublishRetryMessage).toHaveBeenCalledTimes(2); + expect(mockOctokit.actions.getJobForWorkflowRun).not.toHaveBeenCalled(); + }); + + it('calls publishRetryMessage for each message in a multi-runner scenario', async () => { + mockCreateRunner.mockResolvedValue(['i-12345', 'i-67890', 'i-abcdef', 'i-11111', 'i-22222']); // Create all requested runners + const messages = createTestMessages(5); + + await scaleUpModule.scaleUp(messages); + + expect(mockPublishRetryMessage).toHaveBeenCalledTimes(5); + messages.forEach((msg, index) => { + expect(mockPublishRetryMessage).toHaveBeenNthCalledWith( + index + 1, + expect.objectContaining({ + id: msg.id, + messageId: msg.messageId, + }), + ); + }); + }); + + it('calls publishRetryMessage after runner creation', async () => { + const messages = createTestMessages(1); + mockCreateRunner.mockResolvedValue(['i-12345']); // Create the requested runner + + const callOrder: string[] = []; + mockPublishRetryMessage.mockImplementation(() => { + callOrder.push('publishRetryMessage'); + return Promise.resolve(); + }); + mockCreateRunner.mockImplementation(async () => { + callOrder.push('createRunner'); + return ['i-12345']; + }); + + await scaleUpModule.scaleUp(messages); + + expect(callOrder).toEqual(['createRunner', 'publishRetryMessage']); + }); +}); + +describe('useDedicatedHost', () => { + beforeEach(() => { + process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; + process.env.RUNNER_NAME_PREFIX = 'unit-test-'; + process.env.RUNNER_GROUP_NAME = 'Default'; + process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; + process.env.RUNNER_LABELS = 'label1,label2'; + }); + + it('defaults to false when USE_DEDICATED_HOST env var is not set', async () => { + delete process.env.USE_DEDICATED_HOST; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toHaveBeenCalledWith(expect.objectContaining({ useDedicatedHost: false })); + }); + + it('is true when USE_DEDICATED_HOST is "true"', async () => { + process.env.USE_DEDICATED_HOST = 'true'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toHaveBeenCalledWith(expect.objectContaining({ useDedicatedHost: true })); + }); + + it('is false when USE_DEDICATED_HOST is "false"', async () => { + process.env.USE_DEDICATED_HOST = 'false'; + await scaleUpModule.scaleUp(TEST_DATA); + expect(createRunner).toHaveBeenCalledWith(expect.objectContaining({ useDedicatedHost: false })); + }); +}); + +function defaultOctokitMockImpl() { + mockOctokit.actions.getJobForWorkflowRun.mockImplementation(() => ({ + data: { + status: 'queued', + }, + })); + mockOctokit.paginate.mockImplementation(() => [ + { + id: 1, + name: 'Default', + }, + ]); + mockOctokit.actions.generateRunnerJitconfigForOrg.mockImplementation(({ labels }: { labels: string[] }) => ({ + data: { + runner: { id: 9876543210, labels: labels.map((name: string) => ({ name })) }, + encoded_jit_config: 'TEST_JIT_CONFIG_ORG', + }, + })); + mockOctokit.actions.generateRunnerJitconfigForRepo.mockImplementation(({ labels }: { labels: string[] }) => ({ + data: { + runner: { id: 9876543210, labels: labels.map((name: string) => ({ name })) }, + encoded_jit_config: 'TEST_JIT_CONFIG_REPO', + }, + })); + mockOctokit.checks.get.mockImplementation(() => ({ + data: { + status: 'queued', + }, + })); + + const mockTokenReturnValue = { + data: { + token: '1234abcd', + }, + }; + const mockInstallationIdReturnValueOrgs = { + data: { + id: TEST_DATA_SINGLE.installationId, + }, + }; + const mockInstallationIdReturnValueRepos = { + data: { + id: TEST_DATA_SINGLE.installationId, + }, + }; + + mockOctokit.actions.createRegistrationTokenForOrg.mockImplementation(() => mockTokenReturnValue); + mockOctokit.actions.createRegistrationTokenForRepo.mockImplementation(() => mockTokenReturnValue); + mockOctokit.apps.getOrgInstallation.mockImplementation(() => mockInstallationIdReturnValueOrgs); + mockOctokit.apps.getRepoInstallation.mockImplementation(() => mockInstallationIdReturnValueRepos); +} + +function defaultSSMGetParameterMockImpl() { + mockSSMgetParameter.mockImplementation(async (name: string) => { + if (name === `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`) { + return '1'; + } else if (name === `${process.env.PARAMETER_GITHUB_APP_ID_NAME}`) { + return `${process.env.GITHUB_APP_ID}`; + } else { + throw new Error(`ParameterNotFound: ${name}`); + } + }); +} + +describe('parseEc2OverrideConfig', () => { + describe('Basic Fleet Overrides', () => { + it('should parse instance-type label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-instance-type:c5.xlarge']); + expect(result?.InstanceType).toBe('c5.xlarge'); + }); + + it('should parse subnet-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-subnet-id:subnet-123456']); + expect(result?.SubnetId).toBe('subnet-123456'); + }); + + it('should parse availability-zone label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-availability-zone:us-east-1a']); + expect(result?.AvailabilityZone).toBe('us-east-1a'); + }); + + it('should parse availability-zone-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-availability-zone-id:use1-az1']); + expect(result?.AvailabilityZoneId).toBe('use1-az1'); + }); + + it('should parse max-price label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-max-price:0.50']); + expect(result?.MaxPrice).toBe('0.50'); + }); + + it('should parse priority label as number', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-priority:1']); + expect(result?.Priority).toBe(1); + }); + + it('should parse weighted-capacity label as number', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-weighted-capacity:2']); + expect(result?.WeightedCapacity).toBe(2); + }); + + it('should parse image-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-image-id:ami-12345678']); + expect(result?.ImageId).toBe('ami-12345678'); + }); + + it('should parse multiple basic fleet overrides', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-instance-type:r5.2xlarge', + 'ghr-ec2-max-price:1.00', + 'ghr-ec2-priority:2', + ]); + expect(result?.InstanceType).toBe('r5.2xlarge'); + expect(result?.MaxPrice).toBe('1.00'); + expect(result?.Priority).toBe(2); + }); + }); + + describe('Placement', () => { + it('should parse placement-group-name label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-group-name:my-placement-group']); + expect(result?.Placement?.GroupName).toBe('my-placement-group'); + }); + + it('should parse placement-group-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-group-id:pg-1234567890abcdef0']); + expect(result?.Placement?.GroupId).toBe('pg-1234567890abcdef0'); + }); + + it('should parse placement-tenancy label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-tenancy:dedicated']); + expect(result?.Placement?.Tenancy).toBe('dedicated'); + }); + + it('should parse placement-host-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-host-id:h-1234567890abcdef']); + expect(result?.Placement?.HostId).toBe('h-1234567890abcdef'); + }); + + it('should parse placement-affinity label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-affinity:host']); + expect(result?.Placement?.Affinity).toBe('host'); + }); + + it('should parse placement-partition-number label as number', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-partition-number:3']); + expect(result?.Placement?.PartitionNumber).toBe(3); + }); + + it('should parse placement-availability-zone label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-availability-zone:us-west-2b']); + expect(result?.Placement?.AvailabilityZone).toBe('us-west-2b'); + }); + + it('should parse placement-availability-zone-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-availability-zone-id:use1-az1']); + expect(result?.Placement?.AvailabilityZoneId).toBe('use1-az1'); + }); + + it('should parse placement-spread-domain label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-placement-spread-domain:my-spread-domain']); + expect(result?.Placement?.SpreadDomain).toBe('my-spread-domain'); + }); + + it('should parse placement-host-resource-group-arn label', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-placement-host-resource-group-arn:arn:aws:ec2:us-east-1:123456789012:host-resource-group/hrg-1234', + ]); + expect(result?.Placement?.HostResourceGroupArn).toBe( + 'arn:aws:ec2:us-east-1:123456789012:host-resource-group/hrg-1234', + ); + }); + + it('should parse multiple placement labels', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-placement-group-name:group-1', + 'ghr-ec2-placement-tenancy:dedicated', + 'ghr-ec2-placement-availability-zone:us-east-1b', + ]); + expect(result?.Placement?.GroupName).toBe('group-1'); + expect(result?.Placement?.Tenancy).toBe('dedicated'); + expect(result?.Placement?.AvailabilityZone).toBe('us-east-1b'); + }); + }); + + describe('Block Device Mappings', () => { + it('should parse block-device-name label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-block-device-name:/dev/sdg']); + expect(result?.BlockDeviceMappings?.[0]?.DeviceName).toBe('/dev/sdg'); + }); + + it('should use default block device name when provided', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-volume-size:100'], '/dev/sda1'); + expect(result?.BlockDeviceMappings?.[0]?.DeviceName).toBe('/dev/sda1'); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(100); + }); + + it('should parse ebs-volume-size label as number', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-volume-size:100']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(100); + }); + + it('should parse ebs-volume-type label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-volume-type:gp3']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeType).toBe('gp3'); + }); + + it('should parse ebs-iops label as number', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-iops:3000']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Iops).toBe(3000); + }); + + it('should parse ebs-throughput label as number', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-throughput:250']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Throughput).toBe(250); + }); + + it('should parse ebs-encrypted label as boolean true', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-encrypted:true']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); + }); + + it('should parse ebs-encrypted label as boolean false', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-encrypted:false']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(false); + }); + + it('should parse ebs-kms-key-id label', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-ebs-kms-key-id:arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012', + ]); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.KmsKeyId).toBe( + 'arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012', + ); + }); + + it('should parse ebs-delete-on-termination label as boolean true', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-delete-on-termination:true']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.DeleteOnTermination).toBe(true); + }); + + it('should parse ebs-delete-on-termination label as boolean false', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-delete-on-termination:false']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.DeleteOnTermination).toBe(false); + }); + + it('should parse ebs-snapshot-id label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-snapshot-id:snap-1234567890abcdef']); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.SnapshotId).toBe('snap-1234567890abcdef'); + }); + + it('should parse block-device-virtual-name label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-block-device-virtual-name:ephemeral0']); + expect(result?.BlockDeviceMappings?.[0]?.VirtualName).toBe('ephemeral0'); + }); + + it('should parse block-device-no-device label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-block-device-no-device:true']); + expect(result?.BlockDeviceMappings?.[0]?.NoDevice).toBe('true'); + }); + + it('should parse multiple block device mapping labels', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-ebs-volume-size:200', + 'ghr-ec2-ebs-volume-type:gp3', + 'ghr-ec2-ebs-iops:5000', + 'ghr-ec2-ebs-encrypted:true', + ]); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(200); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeType).toBe('gp3'); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Iops).toBe(5000); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); + }); + + it('should initialize BlockDeviceMappings when not present', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-ebs-volume-size:50']); + expect(result?.BlockDeviceMappings).toBeDefined(); + }); + }); + + describe('Instance Requirements - vCPU and Memory', () => { + it('should parse vcpu-count-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-vcpu-count-min:4']); + expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(4); + }); + + it('should parse vcpu-count-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-vcpu-count-max:16']); + expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(16); + }); + + it('should parse both vcpu-count-min and vcpu-count-max labels', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-vcpu-count-min:2', 'ghr-ec2-vcpu-count-max:8']); + expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(2); + expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(8); + }); + + it('should parse memory-mib-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-memory-mib-min:8192']); + expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(8192); + }); + + it('should parse memory-mib-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-memory-mib-max:32768']); + expect(result?.InstanceRequirements?.MemoryMiB?.Max).toBe(32768); + }); + + it('should parse both memory-mib-min and memory-mib-max labels', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-memory-mib-min:16384', 'ghr-ec2-memory-mib-max:65536']); + expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(16384); + expect(result?.InstanceRequirements?.MemoryMiB?.Max).toBe(65536); + }); + + it('should parse memory-gib-per-vcpu-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-memory-gib-per-vcpu-min:2']); + expect(result?.InstanceRequirements?.MemoryGiBPerVCpu?.Min).toBe(2); + }); + + it('should parse memory-gib-per-vcpu-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-memory-gib-per-vcpu-max:8']); + expect(result?.InstanceRequirements?.MemoryGiBPerVCpu?.Max).toBe(8); + }); + + it('should parse combined vCPU and memory requirements', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-vcpu-count-min:8', + 'ghr-ec2-vcpu-count-max:32', + 'ghr-ec2-memory-mib-min:32768', + 'ghr-ec2-memory-mib-max:131072', + ]); + expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(8); + expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(32); + expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(32768); + expect(result?.InstanceRequirements?.MemoryMiB?.Max).toBe(131072); + }); + }); + + describe('Instance Requirements - CPU and Performance', () => { + it('should parse cpu-manufacturers as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-cpu-manufacturers:intel']); + expect(result?.InstanceRequirements?.CpuManufacturers).toEqual(['intel']); + }); + + it('should parse cpu-manufacturers as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-cpu-manufacturers:intel;amd']); + expect(result?.InstanceRequirements?.CpuManufacturers).toEqual(['intel', 'amd']); + }); + + it('should parse instance-generations as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-instance-generations:current']); + expect(result?.InstanceRequirements?.InstanceGenerations).toEqual(['current']); + }); + + it('should parse instance-generations as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-instance-generations:current;previous']); + expect(result?.InstanceRequirements?.InstanceGenerations).toEqual(['current', 'previous']); + }); + + it('should parse excluded-instance-types as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-excluded-instance-types:t2.micro']); + expect(result?.InstanceRequirements?.ExcludedInstanceTypes).toEqual(['t2.micro']); + }); + + it('should parse excluded-instance-types as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-excluded-instance-types:t2.micro;t2.small']); + expect(result?.InstanceRequirements?.ExcludedInstanceTypes).toEqual(['t2.micro', 't2.small']); + }); + + it('should parse allowed-instance-types as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-allowed-instance-types:c5.xlarge']); + expect(result?.InstanceRequirements?.AllowedInstanceTypes).toEqual(['c5.xlarge']); + }); + + it('should parse allowed-instance-types as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-allowed-instance-types:c5.xlarge;c5.2xlarge']); + expect(result?.InstanceRequirements?.AllowedInstanceTypes).toEqual(['c5.xlarge', 'c5.2xlarge']); + }); + + it('should parse burstable-performance label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-burstable-performance:included']); + expect(result?.InstanceRequirements?.BurstablePerformance).toBe('included'); + }); + + it('should parse bare-metal label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-bare-metal:excluded']); + expect(result?.InstanceRequirements?.BareMetal).toBe('excluded'); + }); + }); + + describe('Instance Requirements - Accelerators', () => { + it('should parse accelerator-count-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-count-min:1']); + expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); + }); + + it('should parse accelerator-count-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-count-max:4']); + expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(4); + }); + + it('should parse both accelerator-count-min and accelerator-count-max', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-count-min:1', 'ghr-ec2-accelerator-count-max:2']); + expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); + expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(2); + }); + + it('should parse accelerator-types as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-types:gpu']); + expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu']); + }); + + it('should parse accelerator-types as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-types:gpu;fpga']); + expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu', 'fpga']); + }); + + it('should parse accelerator-manufacturers as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-manufacturers:nvidia']); + expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia']); + }); + + it('should parse accelerator-manufacturers as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-manufacturers:nvidia;amd']); + expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia', 'amd']); + }); + + it('should parse accelerator-names as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-names:a100']); + expect(result?.InstanceRequirements?.AcceleratorNames).toEqual(['a100']); + }); + + it('should parse accelerator-names as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-names:a100;v100']); + expect(result?.InstanceRequirements?.AcceleratorNames).toEqual(['a100', 'v100']); + }); + + it('should parse accelerator-total-memory-mib-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-total-memory-mib-min:8192']); + expect(result?.InstanceRequirements?.AcceleratorTotalMemoryMiB?.Min).toBe(8192); + }); + + it('should parse accelerator-total-memory-mib-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-accelerator-total-memory-mib-max:40960']); + expect(result?.InstanceRequirements?.AcceleratorTotalMemoryMiB?.Max).toBe(40960); + }); + + it('should parse combined accelerator requirements', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-accelerator-count-min:1', + 'ghr-ec2-accelerator-count-max:2', + 'ghr-ec2-accelerator-types:gpu', + 'ghr-ec2-accelerator-manufacturers:nvidia', + ]); + expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); + expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(2); + expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu']); + expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia']); + }); + }); + + describe('Instance Requirements - Network and Storage', () => { + it('should parse network-interface-count-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-network-interface-count-min:2']); + expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Min).toBe(2); + }); + + it('should parse network-interface-count-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-network-interface-count-max:4']); + expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Max).toBe(4); + }); + + it('should parse network-bandwidth-gbps-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-network-bandwidth-gbps-min:5']); + expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Min).toBe(5); + }); + + it('should parse network-bandwidth-gbps-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-network-bandwidth-gbps-max:25']); + expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Max).toBe(25); + }); + + it('should parse local-storage label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-local-storage:included']); + expect(result?.InstanceRequirements?.LocalStorage).toBe('included'); + }); + + it('should parse local-storage-types as single value', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-local-storage-types:ssd']); + expect(result?.InstanceRequirements?.LocalStorageTypes).toEqual(['ssd']); + }); + + it('should parse local-storage-types as semicolon-separated list', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-local-storage-types:hdd;ssd']); + expect(result?.InstanceRequirements?.LocalStorageTypes).toEqual(['hdd', 'ssd']); + }); + + it('should parse total-local-storage-gb-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-total-local-storage-gb-min:100']); + expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Min).toBe(100); + }); + + it('should parse total-local-storage-gb-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-total-local-storage-gb-max:1000']); + expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Max).toBe(1000); + }); + + it('should parse baseline-ebs-bandwidth-mbps-min label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-baseline-ebs-bandwidth-mbps-min:500']); + expect(result?.InstanceRequirements?.BaselineEbsBandwidthMbps?.Min).toBe(500); + }); + + it('should parse baseline-ebs-bandwidth-mbps-max label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-baseline-ebs-bandwidth-mbps-max:2000']); + expect(result?.InstanceRequirements?.BaselineEbsBandwidthMbps?.Max).toBe(2000); + }); + }); + + describe('Instance Requirements - Pricing and Other', () => { + it('should parse spot-max-price-percentage-over-lowest-price label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-spot-max-price-percentage-over-lowest-price:50']); + expect(result?.InstanceRequirements?.SpotMaxPricePercentageOverLowestPrice).toBe(50); + }); + + it('should parse on-demand-max-price-percentage-over-lowest-price label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-on-demand-max-price-percentage-over-lowest-price:75']); + expect(result?.InstanceRequirements?.OnDemandMaxPricePercentageOverLowestPrice).toBe(75); + }); + + it('should parse max-spot-price-as-percentage-of-optimal-on-demand-price label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-max-spot-price-as-percentage-of-optimal-on-demand-price:60']); + expect(result?.InstanceRequirements?.MaxSpotPriceAsPercentageOfOptimalOnDemandPrice).toBe(60); + }); + + it('should parse require-hibernate-support label as boolean true', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-require-hibernate-support:true']); + expect(result?.InstanceRequirements?.RequireHibernateSupport).toBe(true); + }); + + it('should parse require-hibernate-support label as boolean false', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-require-hibernate-support:false']); + expect(result?.InstanceRequirements?.RequireHibernateSupport).toBe(false); + }); + + it('should parse require-encryption-in-transit label as boolean true', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-require-encryption-in-transit:true']); + expect(result?.InstanceRequirements?.RequireEncryptionInTransit).toBe(true); + }); + + it('should parse require-encryption-in-transit label as boolean false', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-require-encryption-in-transit:false']); + expect(result?.InstanceRequirements?.RequireEncryptionInTransit).toBe(false); + }); + + it('should parse baseline-performance-factors-cpu-reference-families label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-baseline-performance-factors-cpu-reference-families:intel']); + expect(result?.InstanceRequirements?.BaselinePerformanceFactors?.Cpu?.References?.[0]?.InstanceFamily).toBe( + 'intel', + ); + }); + it('should parse baseline-performance-factors-cpu-reference-families list label', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-baseline-performance-factors-cpu-reference-families:intel;amd']); + expect(result?.InstanceRequirements?.BaselinePerformanceFactors?.Cpu?.References?.[0]?.InstanceFamily).toBe( + 'intel', + ); + expect(result?.InstanceRequirements?.BaselinePerformanceFactors?.Cpu?.References?.[1]?.InstanceFamily).toBe( + 'amd', + ); + }); + }); + + describe('Edge Cases', () => { + it('should return undefined when empty array is provided', () => { + const result = parseEc2OverrideConfig([]); + expect(result).toBeUndefined(); + }); + + it('should return undefined when no ghr-ec2 labels are provided', () => { + const result = parseEc2OverrideConfig(['self-hosted', 'linux', 'x64']); + expect(result).toBeUndefined(); + }); + + it('should ignore non-ghr-ec2 labels and only parse ghr-ec2 labels', () => { + const result = parseEc2OverrideConfig([ + 'self-hosted', + 'ghr-ec2-instance-type:m5.large', + 'linux', + 'ghr-ec2-max-price:0.30', + ]); + expect(result?.InstanceType).toBe('m5.large'); + expect(result?.MaxPrice).toBe('0.30'); + }); + + it('should handle labels with colons in values (ARNs)', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-ebs-kms-key-id:arn:aws:kms:us-east-1:123456789012:key/abc-def-ghi', + ]); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.KmsKeyId).toBe( + 'arn:aws:kms:us-east-1:123456789012:key/abc-def-ghi', + ); + }); + + it('should handle labels with colons in placement ARNs', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-placement-host-resource-group-arn:arn:aws:ec2:us-west-2:123456789012:host-resource-group/hrg-abc123', + ]); + expect(result?.Placement?.HostResourceGroupArn).toBe( + 'arn:aws:ec2:us-west-2:123456789012:host-resource-group/hrg-abc123', + ); + }); + + it('should handle labels without values gracefully', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-instance-type:', 'ghr-ec2-max-price:0.50']); + expect(result?.InstanceType).toBeUndefined(); + expect(result?.MaxPrice).toBe('0.50'); + }); + + it('should handle malformed labels (no colon) gracefully', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-instance-type-m5-large', 'ghr-ec2-max-price:0.50']); + expect(result?.MaxPrice).toBe('0.50'); + expect(result?.InstanceType).toBeUndefined(); + }); + + it('should handle numeric strings correctly for number fields', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-priority:5', + 'ghr-ec2-weighted-capacity:10', + 'ghr-ec2-vcpu-count-min:4', + ]); + expect(result?.Priority).toBe(5); + expect(result?.WeightedCapacity).toBe(10); + expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(4); + }); + + it('should handle boolean strings correctly for boolean fields', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-ebs-encrypted:true', + 'ghr-ec2-ebs-delete-on-termination:false', + 'ghr-ec2-require-hibernate-support:true', + ]); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.DeleteOnTermination).toBe(false); + expect(result?.InstanceRequirements?.RequireHibernateSupport).toBe(true); + }); + + it('should handle floating point numbers in max-price', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-max-price:0.12345']); + expect(result?.MaxPrice).toBe('0.12345'); + }); + + it('should handle whitespace in semicolon-separated lists', () => { + const result = parseEc2OverrideConfig(['ghr-ec2-cpu-manufacturers: intel ; amd ']); + expect(result?.InstanceRequirements?.CpuManufacturers).toEqual([' intel ', ' amd ']); + }); + + it('should return config with all parsed labels', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-instance-type:c5.xlarge', + 'ghr-ec2-vcpu-count-min:4', + 'ghr-ec2-memory-mib-min:8192', + 'ghr-ec2-placement-tenancy:dedicated', + 'ghr-ec2-ebs-volume-size:100', + ]); + expect(result?.InstanceType).toBe('c5.xlarge'); + expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(4); + expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(8192); + expect(result?.Placement?.Tenancy).toBe('dedicated'); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(100); + }); + }); + + describe('Complex Scenarios', () => { + it('should handle comprehensive EC2 configuration with all categories', () => { + const result = parseEc2OverrideConfig([ + // Basic Fleet + 'ghr-ec2-instance-type:r5.2xlarge', + 'ghr-ec2-max-price:0.75', + 'ghr-ec2-priority:1', + // Placement + 'ghr-ec2-placement-group-name:my-group', + 'ghr-ec2-placement-tenancy:dedicated', + // Block Device + 'ghr-ec2-ebs-volume-size:200', + 'ghr-ec2-ebs-volume-type:gp3', + 'ghr-ec2-ebs-encrypted:true', + // Instance Requirements + 'ghr-ec2-vcpu-count-min:8', + 'ghr-ec2-vcpu-count-max:32', + 'ghr-ec2-memory-mib-min:32768', + 'ghr-ec2-cpu-manufacturers:intel;amd', + 'ghr-ec2-instance-generations:current', + ]); + + expect(result?.InstanceType).toBe('r5.2xlarge'); + expect(result?.MaxPrice).toBe('0.75'); + expect(result?.Priority).toBe(1); + expect(result?.Placement?.GroupName).toBe('my-group'); + expect(result?.Placement?.Tenancy).toBe('dedicated'); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeSize).toBe(200); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.VolumeType).toBe('gp3'); + expect(result?.BlockDeviceMappings?.[0]?.Ebs?.Encrypted).toBe(true); + expect(result?.InstanceRequirements?.VCpuCount?.Min).toBe(8); + expect(result?.InstanceRequirements?.VCpuCount?.Max).toBe(32); + expect(result?.InstanceRequirements?.MemoryMiB?.Min).toBe(32768); + expect(result?.InstanceRequirements?.CpuManufacturers).toEqual(['intel', 'amd']); + expect(result?.InstanceRequirements?.InstanceGenerations).toEqual(['current']); + }); + + it('should handle GPU instance configuration', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-accelerator-count-min:1', + 'ghr-ec2-accelerator-count-max:4', + 'ghr-ec2-accelerator-types:gpu', + 'ghr-ec2-accelerator-manufacturers:nvidia', + 'ghr-ec2-accelerator-names:a100;v100', + 'ghr-ec2-accelerator-total-memory-mib-min:16384', + ]); + + expect(result?.InstanceRequirements?.AcceleratorCount?.Min).toBe(1); + expect(result?.InstanceRequirements?.AcceleratorCount?.Max).toBe(4); + expect(result?.InstanceRequirements?.AcceleratorTypes).toEqual(['gpu']); + expect(result?.InstanceRequirements?.AcceleratorManufacturers).toEqual(['nvidia']); + expect(result?.InstanceRequirements?.AcceleratorNames).toEqual(['a100', 'v100']); + expect(result?.InstanceRequirements?.AcceleratorTotalMemoryMiB?.Min).toBe(16384); + }); + + it('should handle network-optimized instance configuration', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-network-interface-count-min:2', + 'ghr-ec2-network-interface-count-max:8', + 'ghr-ec2-network-bandwidth-gbps-min:10', + 'ghr-ec2-network-bandwidth-gbps-max:100', + 'ghr-ec2-baseline-ebs-bandwidth-mbps-min:1000', + ]); + + expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Min).toBe(2); + expect(result?.InstanceRequirements?.NetworkInterfaceCount?.Max).toBe(8); + expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Min).toBe(10); + expect(result?.InstanceRequirements?.NetworkBandwidthGbps?.Max).toBe(100); + expect(result?.InstanceRequirements?.BaselineEbsBandwidthMbps?.Min).toBe(1000); + }); + + it('should handle storage-optimized instance configuration', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-local-storage:included', + 'ghr-ec2-local-storage-types:ssd', + 'ghr-ec2-total-local-storage-gb-min:500', + 'ghr-ec2-total-local-storage-gb-max:2000', + ]); + + expect(result?.InstanceRequirements?.LocalStorage).toBe('included'); + expect(result?.InstanceRequirements?.LocalStorageTypes).toEqual(['ssd']); + expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Min).toBe(500); + expect(result?.InstanceRequirements?.TotalLocalStorageGB?.Max).toBe(2000); + }); + + it('should handle spot instance configuration with pricing', () => { + const result = parseEc2OverrideConfig([ + 'ghr-ec2-max-price:0.50', + 'ghr-ec2-spot-max-price-percentage-over-lowest-price:100', + 'ghr-ec2-on-demand-max-price-percentage-over-lowest-price:150', + ]); + + expect(result?.MaxPrice).toBe('0.50'); + expect(result?.InstanceRequirements?.SpotMaxPricePercentageOverLowestPrice).toBe(100); + expect(result?.InstanceRequirements?.OnDemandMaxPricePercentageOverLowestPrice).toBe(150); + }); + }); +}); + +describe('getScaleUpRunnerProviderType', () => { + it('defaults to ec2 when no type is defined', () => { + expect(getScaleUpRunnerProviderType(undefined, 'ec2')).toEqual('ec2'); + }); - expect(testProvider.getCurrentRunners).not.toHaveBeenCalled(); - expect(testProvider.createRunners).not.toHaveBeenCalled(); - expect(mockedPublishRetryMessage).not.toHaveBeenCalled(); + it('uses configured ec2 type', () => { + expect(getScaleUpRunnerProviderType('ec2', 'microvm')).toEqual('ec2'); }); }); diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts deleted file mode 100644 index 51b7e13020..0000000000 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import type { RunnerMatcherConfig, RunnerProvider } from '../sqs'; -import { selectAwsDynamicLabelQueue } from './aws-dynamic-labels'; - -describe('selectAwsDynamicLabelQueue', () => { - it('defaults queues without a provider to EC2 dynamic label handling', () => { - const queue = runnerQueue('default-ec2'); - - expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ - queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); - }); -}); - -function runnerQueue(id: string, runnerProvider?: RunnerProvider): RunnerMatcherConfig { - return { - id, - arn: `arn:${id}`, - runnerProvider, - matcherConfig: { - labelMatchers: [['self-hosted', 'linux']], - exactMatch: true, - enableDynamicLabels: true, - }, - }; -} diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index ae571da9d8..acb36c0c68 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -7,7 +7,10 @@ import workFlowJobEvent from '../../test/resources/github_workflowjob_event.json import runnerConfig from '../../test/resources/multi_runner_configurations.json'; import { RunnerConfig, sendActionRequest } from '../sqs'; +import type { RunnerMatcherConfig, RunnerProvider } from '../sqs'; import { dispatch } from './dispatch'; +import { selectAwsDynamicLabelQueue } from './aws-dynamic-labels'; +import { canRunJob } from './labels'; import { ConfigDispatcher } from '../ConfigLoader'; import { logger } from '@aws-github-runner/aws-powertools-util'; import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; @@ -459,3 +462,145 @@ async function createConfig(repositoryAllowList?: string[], runnerConfig?: Runne mockSSMResponse(runnerConfig); return await ConfigDispatcher.load(); } + +describe('decides can run job based on label and config (canRunJob)', () => { + it('should accept job with an exact match and identical labels.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should accept job with an exact match and identical labels, ignoring cases.', () => { + const workflowLabels = ['self-Hosted', 'Linux', 'X64', 'ubuntu-Latest']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should accept job with an exact match and runner supports requested capabilities.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should NOT accept job with an exact match and runner not matching requested capabilities.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(false); + }); + + it('should accept job with for a non exact match. Any label that matches will accept the job.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; + const runnerLabels = [['gpu']]; + expect(canRunJob(workflowLabels, runnerLabels, false)).toBe(true); + }); + + it('should NOT accept job with for an exact match. Not all requested capabilities are supported.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; + const runnerLabels = [['gpu']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(false); + }); + + it('should match when runner has more labels than workflow requests with exactMatch=true (unidirectional).', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404', 'on-demand']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should match when labels are exactly identical with exactMatch=true.', () => { + const workflowLabels = ['self-hosted', 'linux', 'on-demand']; + const runnerLabels = [['self-hosted', 'linux', 'on-demand']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should match with exactMatch=true when labels are in different order.', () => { + const workflowLabels = ['linux', 'self-hosted', 'x64']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should match with exactMatch=true when labels are completely shuffled.', () => { + const workflowLabels = ['x64', 'ubuntu-latest', 'self-hosted', 'linux']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should match with exactMatch=false when labels are in different order.', () => { + const workflowLabels = ['gpu', 'self-hosted']; + const runnerLabels = [['self-hosted', 'gpu']]; + expect(canRunJob(workflowLabels, runnerLabels, false)).toBe(true); + }); + + // bidirectionalLabelMatch tests + it('should NOT match when runner has more labels than workflow requests (bidirectionalLabelMatch=true).', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404', 'on-demand']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); + }); + + it('should NOT match when workflow has more labels than runner (bidirectionalLabelMatch=true).', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); + }); + + it('should match when labels are exactly identical with bidirectionalLabelMatch=true.', () => { + const workflowLabels = ['self-hosted', 'linux', 'on-demand']; + const runnerLabels = [['self-hosted', 'linux', 'on-demand']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); + }); + + it('should match with bidirectionalLabelMatch=true when labels are in different order.', () => { + const workflowLabels = ['linux', 'self-hosted', 'x64']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); + }); + + it('should match with bidirectionalLabelMatch=true when labels are completely shuffled.', () => { + const workflowLabels = ['x64', 'ubuntu-latest', 'self-hosted', 'linux']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); + }); + + it('should match with bidirectionalLabelMatch=true ignoring case.', () => { + const workflowLabels = ['Self-Hosted', 'Linux', 'X64']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); + }); + + it('should NOT match empty workflow labels with bidirectionalLabelMatch=true.', () => { + const workflowLabels: string[] = []; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); + }); + + it('bidirectionalLabelMatch takes precedence over exactMatch when both are true.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + // exactMatch alone would accept this (runner has extra labels), but bidirectional should reject + expect(canRunJob(workflowLabels, runnerLabels, true, true)).toBe(false); + }); +}); + +describe('selectAwsDynamicLabelQueue', () => { + it('defaults queues without a provider to EC2 dynamic label handling', () => { + const queue = runnerQueue('default-ec2'); + + expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ + queue, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + }); + }); +}); + +function runnerQueue(id: string, runnerProvider?: RunnerProvider): RunnerMatcherConfig { + return { + id, + arn: `arn:${id}`, + runnerProvider, + matcherConfig: { + labelMatchers: [['self-hosted', 'linux']], + exactMatch: true, + enableDynamicLabels: true, + }, + }; +} diff --git a/lambdas/functions/webhook/src/runners/ec2-dynamic-labels-policy.test.ts b/lambdas/functions/webhook/src/runners/dynamic-labels-policy.test.ts similarity index 100% rename from lambdas/functions/webhook/src/runners/ec2-dynamic-labels-policy.test.ts rename to lambdas/functions/webhook/src/runners/dynamic-labels-policy.test.ts diff --git a/lambdas/functions/webhook/src/runners/labels.test.ts b/lambdas/functions/webhook/src/runners/labels.test.ts deleted file mode 100644 index 5263766063..0000000000 --- a/lambdas/functions/webhook/src/runners/labels.test.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { canRunJob } from './labels'; - -describe('decides can run job based on label and config (canRunJob)', () => { - it('should accept job with an exact match and identical labels.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should accept job with an exact match and identical labels, ignoring cases.', () => { - const workflowLabels = ['self-Hosted', 'Linux', 'X64', 'ubuntu-Latest']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should accept job with an exact match and runner supports requested capabilities.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should NOT accept job with an exact match and runner not matching requested capabilities.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(false); - }); - - it('should accept job with for a non exact match. Any label that matches will accept the job.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; - const runnerLabels = [['gpu']]; - expect(canRunJob(workflowLabels, runnerLabels, false)).toBe(true); - }); - - it('should NOT accept job with for an exact match. Not all requested capabilities are supported.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; - const runnerLabels = [['gpu']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(false); - }); - - it('should match when runner has more labels than workflow requests with exactMatch=true (unidirectional).', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404', 'on-demand']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should match when labels are exactly identical with exactMatch=true.', () => { - const workflowLabels = ['self-hosted', 'linux', 'on-demand']; - const runnerLabels = [['self-hosted', 'linux', 'on-demand']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should match with exactMatch=true when labels are in different order.', () => { - const workflowLabels = ['linux', 'self-hosted', 'x64']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should match with exactMatch=true when labels are completely shuffled.', () => { - const workflowLabels = ['x64', 'ubuntu-latest', 'self-hosted', 'linux']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should match with exactMatch=false when labels are in different order.', () => { - const workflowLabels = ['gpu', 'self-hosted']; - const runnerLabels = [['self-hosted', 'gpu']]; - expect(canRunJob(workflowLabels, runnerLabels, false)).toBe(true); - }); - - // bidirectionalLabelMatch tests - it('should NOT match when runner has more labels than workflow requests (bidirectionalLabelMatch=true).', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404', 'on-demand']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); - }); - - it('should NOT match when workflow has more labels than runner (bidirectionalLabelMatch=true).', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); - }); - - it('should match when labels are exactly identical with bidirectionalLabelMatch=true.', () => { - const workflowLabels = ['self-hosted', 'linux', 'on-demand']; - const runnerLabels = [['self-hosted', 'linux', 'on-demand']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); - }); - - it('should match with bidirectionalLabelMatch=true when labels are in different order.', () => { - const workflowLabels = ['linux', 'self-hosted', 'x64']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); - }); - - it('should match with bidirectionalLabelMatch=true when labels are completely shuffled.', () => { - const workflowLabels = ['x64', 'ubuntu-latest', 'self-hosted', 'linux']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); - }); - - it('should match with bidirectionalLabelMatch=true ignoring case.', () => { - const workflowLabels = ['Self-Hosted', 'Linux', 'X64']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); - }); - - it('should NOT match empty workflow labels with bidirectionalLabelMatch=true.', () => { - const workflowLabels: string[] = []; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); - }); - - it('bidirectionalLabelMatch takes precedence over exactMatch when both are true.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - // exactMatch alone would accept this (runner has extra labels), but bidirectional should reject - expect(canRunJob(workflowLabels, runnerLabels, true, true)).toBe(false); - }); -}); From cf33d93527fb75c4cca1e3bc7ddcaeecf3f5e967 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 21:37:52 +0000 Subject: [PATCH 09/46] docs: auto update terraform docs --- README.md | 3 ++- modules/multi-runner/README.md | 2 +- modules/runners/README.md | 3 ++- modules/runners/pool/README.md | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cfe5e26768..3e866b0475 100644 --- a/README.md +++ b/README.md @@ -142,11 +142,12 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh) | [github\_app](#input\_github\_app) | GitHub app parameters, see your github app.
You can optionally create the SSM parameters yourself and provide the ARN and name here, through the `*_ssm` attributes.
If you chose to provide the configuration values directly here,
please ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`).
Note: the provided SSM parameters arn and name have a precedence over the actual value (i.e `key_base64_ssm` has a precedence over `key_base64` etc). | 
object({
    key_base64 = optional(string)
    key_base64_ssm = optional(object({
      arn  = string
      name = string
    }))
    id = optional(string)
    id_ssm = optional(object({
      arn  = string
      name = string
    }))
    webhook_secret = optional(string)
    webhook_secret_ssm = optional(object({
      arn  = string
      name = string
    }))
  })
| n/a | yes | | [iam\_overrides](#input\_iam\_overrides) | This map provides the possibility to override some IAM defaults. Note that when using this variable, you are responsible for ensuring the role has necessary permissions to access required resources. `override_instance_profile`: When set to true, uses the instance profile name specified in `instance_profile_name` instead of creating a new instance profile. `override_runner_role`: When set to true, uses the role ARN specified in `runner_role_arn` instead of creating a new IAM role. | 
object({
    override_instance_profile = optional(bool, null)
    instance_profile_name     = optional(string, null)
    override_runner_role      = optional(bool, null)
    runner_role_arn           = optional(string, null)
  })
| 
{
  "instance_profile_name": null,
  "override_instance_profile": false,
  "override_runner_role": false,
  "runner_role_arn": null
}
| no | | [idle\_config](#input\_idle\_config) | List of time periods, defined as a cron expression, to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    type             = optional(string, "ec2")
    cron             = string
    timeZone         = string
    idleCount        = number
    evictionStrategy = optional(string, "oldest_first")
  }))
| `[]` | no | -| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for spot instances. AWS recommends using `price-capacity-optimized` however the AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | +| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | | [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no | | [instance\_target\_capacity\_type](#input\_instance\_target\_capacity\_type) | Default lifecycle used for runner instances, can be either `spot` or `on-demand`. | `string` | `"spot"` | no | | [instance\_termination\_watcher](#input\_instance\_termination\_watcher) | Configuration for the instance termination watcher. This feature is Beta, changes will not trigger a major release as long in beta.

`enable`: Enable or disable the spot termination watcher.
'features': Enable or disable features of the termination watcher.
`enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated.
`memory_size`: Memory size limit in MB of the lambda.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`timeout`: Time out of the lambda in seconds.
`zip`: File location of the lambda zip file. | 
object({
    enable = optional(bool, false)
    features = optional(object({
      enable_spot_termination_handler              = optional(bool, true)
      enable_spot_termination_notification_watcher = optional(bool, true)
    }), {})
    enable_runner_deregistration = optional(bool, true)
    memory_size                  = optional(number, null)
    s3_key                       = optional(string, null)
    s3_object_version            = optional(string, null)
    timeout                      = optional(number, null)
    zip                          = optional(string, null)
  })
| `{}` | no | +| [instance\_type\_priorities](#input\_instance\_type\_priorities) | A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`. | `map(number)` | `null` | no | | [instance\_types](#input\_instance\_types) | List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win). | `list(string)` | 
[
  "m5.large",
  "c5.large"
]
| no | | [job\_queue\_retention\_in\_seconds](#input\_job\_queue\_retention\_in\_seconds) | The number of seconds the job is held in the queue before it is purged. | `number` | `86400` | no | | [job\_retry](#input\_job\_retry) | Experimental! Can be removed / changed without trigger a major release.Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app.

`enable`: Enable or disable the job retry feature.
`delay_in_seconds`: The delay in seconds before the job retry check lambda will check the job status.
`delay_backoff`: The backoff factor for the delay.
`lambda_memory_size`: Memory size limit in MB for the job retry check lambda.
`lambda_timeout`: Time out of the job retry check lambda in seconds.
`max_attempts`: The maximum number of attempts to retry the job. | 
object({
    enable             = optional(bool, false)
    delay_in_seconds   = optional(number, 300)
    delay_backoff      = optional(number, 2)
    lambda_memory_size = optional(number, 256)
    lambda_timeout     = optional(number, 30)
    max_attempts       = optional(number, 1)
  })
| `{}` | no | diff --git a/modules/multi-runner/README.md b/modules/multi-runner/README.md index e658ef7062..dbe328ec3c 100644 --- a/modules/multi-runner/README.md +++ b/modules/multi-runner/README.md @@ -152,7 +152,7 @@ module "multi-runner" { | [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no | | [matcher\_config\_parameter\_store\_tier](#input\_matcher\_config\_parameter\_store\_tier) | The tier of the parameter store for the matcher configuration. Valid values are `Standard`, and `Advanced`. | `string` | `"Standard"` | no | | [metrics](#input\_metrics) | Configuration for metrics created by the module, by default metrics are disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | 
object({
    enable    = optional(bool, false)
    namespace = optional(string, "GitHub Runners")
    metric = optional(object({
      enable_github_app_rate_limit    = optional(bool, true)
      enable_job_retry                = optional(bool, true)
      enable_spot_termination_warning = optional(bool, true)
    }), {})
  })
| `{}` | no | -| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {
runner\_config: {
runner\_os: "The EC2 Operating System type to use for action runner instances (linux, osx, windows)."
runner\_architecture: "The platform architecture of the runner instance\_type."
runner\_metadata\_options: "(Optional) Metadata options for the ec2 runner instances."
ami: "(Optional) AMI configuration for the action runner instances. This object allows you to specify all AMI-related settings in one place."
create\_service\_linked\_role\_spot: (Optional) create the serviced linked role for spot instances that is required by the scale-up lambda.
credit\_specification: "(Optional) The credit specification of the runner instance\_type. Can be unset, `standard` or `unlimited`.
delay\_webhook\_event: "The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event."
disable\_runner\_autoupdate: "Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)"
ebs\_optimized: "The EC2 EBS optimized configuration."
enable\_ephemeral\_runners: "Enable ephemeral runners, runners will only be used once."
enable\_job\_queued\_check: Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners."
enable\_on\_demand\_failover\_for\_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later."
scale\_errors: "List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors"
enable\_organization\_runners: "Register runners to organization, instead of repo level"
enable\_runner\_binaries\_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
enable\_ssm\_on\_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances."
enable\_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI."
instance\_allocation\_strategy: "The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`."
instance\_max\_spot\_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet."
instance\_target\_capacity\_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`."
instance\_types: "List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)."
job\_queue\_retention\_in\_seconds: "The number of seconds the job is held in the queue before it is purged"
minimum\_running\_time\_in\_minutes: "The time an ec2 action runner should be running at minimum before terminated if not busy."
pool\_runner\_owner: "The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported."
runner\_additional\_security\_group\_ids: "List of additional security groups IDs to apply to the runner. If added outside the multi\_runner\_config block, the additional security group(s) will be applied to all runner configs. If added inside the multi\_runner\_config, the additional security group(s) will be applied to the individual runner."
runner\_as\_root: "Run the action runner under the root user. Variable `runner_run_as` will be ignored."
runner\_boot\_time\_in\_minutes: "The minimum time for an EC2 runner to boot and register as a runner."
runner\_disable\_default\_labels: "Disable default labels for the runners (os, architecture and `self-hosted`). If enabled, the runner will only have the extra labels provided in `runner_extra_labels`. In case you on own start script is used, this configuration parameter needs to be parsed via SSM."
runner\_extra\_labels: "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `multi_runner_config.matcherConfig.exactMatch`. GitHub read-only labels should not be provided."
runner\_group\_name: "Name of the runner group."
runner\_name\_prefix: "Prefix for the GitHub runner name."
runner\_run\_as: "Run the GitHub actions agent as user."
runners\_maximum\_count: "The maximum number of runners that will be created. Setting the variable to `-1` disables the maximum check."
scale\_down\_schedule\_expression: "Scheduler expression to check every x for scale down."
scale\_up\_reserved\_concurrent\_executions: "Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations."
lambda\_event\_source\_mapping\_batch\_size: "(Optional) Maximum number of records per Lambda invocation for this runner flavor. Overrides the module-level `lambda_event_source_mapping_batch_size` when set."
lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds: "(Optional) Maximum seconds to gather records before invoking Lambda for this runner flavor. Overrides the module-level `lambda_event_source_mapping_maximum_batching_window_in_seconds` when set."
userdata\_template: "Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored."
enable\_jit\_config: "Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is available. In case you are upgrading from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI."
enable\_runner\_detailed\_monitoring: "Should detailed monitoring be enabled for the runner. Set this to true if you want to use detailed monitoring. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html for details."
enable\_cloudwatch\_agent: "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
cloudwatch\_config: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
userdata\_pre\_install: "Script to be ran before the GitHub Actions runner is installed on the EC2 instances"
userdata\_post\_install: "Script to be ran after the GitHub Actions runner is installed on the EC2 instances"
runner\_hook\_job\_started: "Script to be ran in the runner environment at the beginning of every job"
runner\_hook\_job\_completed: "Script to be ran in the runner environment at the end of every job"
runner\_ec2\_tags: "Map of tags that will be added to the launch template instance tag specifications."
runner\_iam\_role\_managed\_policy\_arns: "Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role"
vpc\_id: "The VPC for security groups of the action runners. If not set uses the value of `var.vpc_id`."
subnet\_ids: "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. If not set, uses the value of `var.subnet_ids`."
idle\_config: "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle."
license\_specifications: "Optional EC2 License Manager license configuration ARNs for the runner launch template. Required for macOS dedicated-host runners when the host resource group uses a Mac dedicated host license configuration."
use\_dedicated\_host: "Experimental! Can be removed / changed without trigger a major release. Whether to use EC2 dedicated hosts for the runners. Needed for macos runners Note that using dedicated hosts can increase cost significantly."
runner\_log\_files: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
block\_device\_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`."
job\_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app."
pool\_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)."
iam\_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances."
}
matcherConfig: {
labelMatchers: "The list of list of labels supported by the runner configuration. `[[self-hosted, linux, x64, example]]`"
exactMatch: "DEPRECATED: Use `bidirectionalLabelMatch` instead. If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ workflow label matches it will trigger the webhook. Note: this only checks that workflow labels are a subset of runner labels, not the reverse."
bidirectionalLabelMatch: "If set to true, the runner labels and workflow job labels must be an exact two-way match (same set, any order, no extras or missing labels). This is stricter than `exactMatch` which only checks that workflow labels are a subset of runner labels. When false, if __any__ workflow label matches it will trigger the webhook."
priority: "If set it defines the priority of the matcher, the matcher with the lowest priority will be evaluated first. Default is 999, allowed values 0-999."
enableDynamicLabels: "Experimental! When true the dispatcher allows `ghr-*` dynamic labels for jobs routed to this runner. Default false."
awsDynamicLabelsPolicy: "Optional AWS dynamic label policy evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose provider dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`."
}
redrive\_build\_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries."
} | 
map(object({
    runner_config = object({
      runner_os           = string
      runner_architecture = string
      runner_metadata_options = optional(map(any), {
        instance_metadata_tags      = "enabled"
        http_endpoint               = "enabled"
        http_tokens                 = "required"
        http_put_response_hop_limit = 1
      })
      ami = optional(object({
        filter               = optional(map(list(string)), { state = ["available"] })
        owners               = optional(list(string), ["amazon"])
        id_ssm_parameter_arn = optional(string, null)
        kms_key_arn          = optional(string, null)
      }), null)
      create_service_linked_role_spot      = optional(bool, false)
      credit_specification                 = optional(string, null)
      delay_webhook_event                  = optional(number, 30)
      disable_runner_autoupdate            = optional(bool, false)
      ebs_optimized                        = optional(bool, false)
      enable_ephemeral_runners             = optional(bool, false)
      enable_job_queued_check              = optional(bool, null)
      enable_on_demand_failover_for_errors = optional(list(string), [])
      scale_errors = optional(list(string), [
        "UnfulfillableCapacity",
        "MaxSpotInstanceCountExceeded",
        "TargetCapacityLimitExceededException",
        "RequestLimitExceeded",
        "ResourceLimitExceeded",
        "MaxSpotInstanceCountExceeded",
        "MaxSpotFleetRequestCountExceeded",
        "InsufficientInstanceCapacity",
        "InsufficientCapacityOnHost",
      ])
      enable_organization_runners                                    = optional(bool, false)
      enable_runner_binaries_syncer                                  = optional(bool, true)
      enable_ssm_on_runners                                          = optional(bool, false)
      enable_userdata                                                = optional(bool, true)
      instance_allocation_strategy                                   = optional(string, "lowest-price")
      instance_max_spot_price                                        = optional(string, null)
      instance_target_capacity_type                                  = optional(string, "spot")
      instance_types                                                 = list(string)
      job_queue_retention_in_seconds                                 = optional(number, 86400)
      minimum_running_time_in_minutes                                = optional(number, null)
      pool_runner_owner                                              = optional(string, null)
      runner_as_root                                                 = optional(bool, false)
      runner_boot_time_in_minutes                                    = optional(number, 5)
      runner_disable_default_labels                                  = optional(bool, false)
      runner_extra_labels                                            = optional(list(string), [])
      runner_group_name                                              = optional(string, "Default")
      runner_name_prefix                                             = optional(string, "")
      runner_run_as                                                  = optional(string, "ec2-user")
      runners_maximum_count                                          = number
      runner_additional_security_group_ids                           = optional(list(string), [])
      scale_down_schedule_expression                                 = optional(string, "cron(*/5 * * * ? *)")
      scale_up_reserved_concurrent_executions                        = optional(number, 1)
      lambda_event_source_mapping_batch_size                         = optional(number, null)
      lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
      userdata_template                                              = optional(string, null)
      userdata_content                                               = optional(string, null)
      enable_jit_config                                              = optional(bool, null)
      enable_runner_detailed_monitoring                              = optional(bool, false)
      enable_cloudwatch_agent                                        = optional(bool, true)
      cloudwatch_config                                              = optional(string, null)
      userdata_pre_install                                           = optional(string, "")
      userdata_post_install                                          = optional(string, "")
      runner_hook_job_started                                        = optional(string, "")
      runner_hook_job_completed                                      = optional(string, "")
      runner_ec2_tags                                                = optional(map(string), {})
      runner_iam_role_managed_policy_arns                            = optional(list(string), [])
      vpc_id                                                         = optional(string, null)
      subnet_ids                                                     = optional(list(string), null)
      idle_config = optional(list(object({
        type             = optional(string, "ec2")
        cron             = string
        timeZone         = string
        idleCount        = number
        evictionStrategy = optional(string, "oldest_first")
      })), [])
      cpu_options = optional(object({
        core_count            = optional(number)
        threads_per_core      = optional(number)
        amd_sev_snp           = optional(string)
        nested_virtualization = optional(string)
      }), null)
      placement = optional(object({
        affinity                = optional(string)
        availability_zone       = optional(string)
        group_id                = optional(string)
        group_name              = optional(string)
        host_id                 = optional(string)
        host_resource_group_arn = optional(string)
        spread_domain           = optional(string)
        tenancy                 = optional(string)
        partition_number        = optional(number)
      }), null)
      license_specifications = optional(list(object({
        license_configuration_arn = string
      })), [])
      use_dedicated_host = optional(bool, false)
      runner_log_files = optional(list(object({
        log_group_name   = string
        prefix_log_group = bool
        file_path        = string
        log_stream_name  = string
        log_class        = optional(string, "STANDARD")
      })), null)
      block_device_mappings = optional(list(object({
        delete_on_termination      = optional(bool, true)
        device_name                = optional(string, "/dev/xvda")
        encrypted                  = optional(bool, true)
        iops                       = optional(number)
        kms_key_id                 = optional(string)
        snapshot_id                = optional(string)
        throughput                 = optional(number)
        volume_initialization_rate = optional(number)
        volume_size                = number
        volume_type                = optional(string, "gp3")
        })), [{
        volume_size = 30
      }])
      pool_config = optional(list(object({
        schedule_expression          = string
        schedule_expression_timezone = optional(string)
        size                         = number
      })), [])
      job_retry = optional(object({
        enable             = optional(bool, false)
        delay_in_seconds   = optional(number, 300)
        delay_backoff      = optional(number, 2)
        lambda_memory_size = optional(number, 256)
        lambda_timeout     = optional(number, 30)
        max_attempts       = optional(number, 1)
      }), {})
      iam_overrides = optional(object({
        override_instance_profile = optional(bool, null)
        instance_profile_name     = optional(string, null)
        override_runner_role      = optional(bool, null)
        runner_role_arn           = optional(string, null)
        }), {
        override_instance_profile = false
        instance_profile_name     = null
        override_runner_role      = false
        runner_role_arn           = null
      })
    })
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = optional(bool, false)
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy  = optional(any, null)
    })
    redrive_build_queue = optional(object({
      enabled         = bool
      maxReceiveCount = number
      }), {
      enabled         = false
      maxReceiveCount = null
    })
  }))
| n/a | yes | +| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {
runner\_config: {
runner\_os: "The EC2 Operating System type to use for action runner instances (linux, osx, windows)."
runner\_architecture: "The platform architecture of the runner instance\_type."
runner\_metadata\_options: "(Optional) Metadata options for the ec2 runner instances."
ami: "(Optional) AMI configuration for the action runner instances. This object allows you to specify all AMI-related settings in one place."
create\_service\_linked\_role\_spot: (Optional) create the serviced linked role for spot instances that is required by the scale-up lambda.
credit\_specification: "(Optional) The credit specification of the runner instance\_type. Can be unset, `standard` or `unlimited`.
delay\_webhook\_event: "The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event."
disable\_runner\_autoupdate: "Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)"
ebs\_optimized: "The EC2 EBS optimized configuration."
enable\_ephemeral\_runners: "Enable ephemeral runners, runners will only be used once."
enable\_job\_queued\_check: Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners."
enable\_on\_demand\_failover\_for\_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later."
scale\_errors: "List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors"
enable\_organization\_runners: "Register runners to organization, instead of repo level"
enable\_runner\_binaries\_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
enable\_ssm\_on\_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances."
enable\_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI."
instance\_allocation\_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`."
instance\_type\_priorities: "A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`."
instance\_max\_spot\_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet."
instance\_target\_capacity\_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`."
instance\_types: "List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)."
job\_queue\_retention\_in\_seconds: "The number of seconds the job is held in the queue before it is purged"
minimum\_running\_time\_in\_minutes: "The time an ec2 action runner should be running at minimum before terminated if not busy."
pool\_runner\_owner: "The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported."
runner\_additional\_security\_group\_ids: "List of additional security groups IDs to apply to the runner. If added outside the multi\_runner\_config block, the additional security group(s) will be applied to all runner configs. If added inside the multi\_runner\_config, the additional security group(s) will be applied to the individual runner."
runner\_as\_root: "Run the action runner under the root user. Variable `runner_run_as` will be ignored."
runner\_boot\_time\_in\_minutes: "The minimum time for an EC2 runner to boot and register as a runner."
runner\_disable\_default\_labels: "Disable default labels for the runners (os, architecture and `self-hosted`). If enabled, the runner will only have the extra labels provided in `runner_extra_labels`. In case you on own start script is used, this configuration parameter needs to be parsed via SSM."
runner\_extra\_labels: "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `multi_runner_config.matcherConfig.exactMatch`. GitHub read-only labels should not be provided."
runner\_group\_name: "Name of the runner group."
runner\_name\_prefix: "Prefix for the GitHub runner name."
runner\_run\_as: "Run the GitHub actions agent as user."
runners\_maximum\_count: "The maximum number of runners that will be created. Setting the variable to `-1` disables the maximum check."
scale\_down\_schedule\_expression: "Scheduler expression to check every x for scale down."
scale\_up\_reserved\_concurrent\_executions: "Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations."
lambda\_event\_source\_mapping\_batch\_size: "(Optional) Maximum number of records per Lambda invocation for this runner flavor. Overrides the module-level `lambda_event_source_mapping_batch_size` when set."
lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds: "(Optional) Maximum seconds to gather records before invoking Lambda for this runner flavor. Overrides the module-level `lambda_event_source_mapping_maximum_batching_window_in_seconds` when set."
userdata\_template: "Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored."
enable\_jit\_config: "Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is available. In case you are upgrading from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI."
enable\_runner\_detailed\_monitoring: "Should detailed monitoring be enabled for the runner. Set this to true if you want to use detailed monitoring. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html for details."
enable\_cloudwatch\_agent: "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
cloudwatch\_config: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
userdata\_pre\_install: "Script to be ran before the GitHub Actions runner is installed on the EC2 instances"
userdata\_post\_install: "Script to be ran after the GitHub Actions runner is installed on the EC2 instances"
runner\_hook\_job\_started: "Script to be ran in the runner environment at the beginning of every job"
runner\_hook\_job\_completed: "Script to be ran in the runner environment at the end of every job"
runner\_ec2\_tags: "Map of tags that will be added to the launch template instance tag specifications."
runner\_iam\_role\_managed\_policy\_arns: "Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role"
vpc\_id: "The VPC for security groups of the action runners. If not set uses the value of `var.vpc_id`."
subnet\_ids: "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. If not set, uses the value of `var.subnet_ids`."
idle\_config: "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle."
license\_specifications: "Optional EC2 License Manager license configuration ARNs for the runner launch template. Required for macOS dedicated-host runners when the host resource group uses a Mac dedicated host license configuration."
use\_dedicated\_host: "Experimental! Can be removed / changed without trigger a major release. Whether to use EC2 dedicated hosts for the runners. Needed for macos runners Note that using dedicated hosts can increase cost significantly."
runner\_log\_files: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
block\_device\_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`."
job\_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app."
pool\_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)."
iam\_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances."
}
matcherConfig: {
labelMatchers: "The list of list of labels supported by the runner configuration. `[[self-hosted, linux, x64, example]]`"
exactMatch: "DEPRECATED: Use `bidirectionalLabelMatch` instead. If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ workflow label matches it will trigger the webhook. Note: this only checks that workflow labels are a subset of runner labels, not the reverse."
bidirectionalLabelMatch: "If set to true, the runner labels and workflow job labels must be an exact two-way match (same set, any order, no extras or missing labels). This is stricter than `exactMatch` which only checks that workflow labels are a subset of runner labels. When false, if __any__ workflow label matches it will trigger the webhook."
priority: "If set it defines the priority of the matcher, the matcher with the lowest priority will be evaluated first. Default is 999, allowed values 0-999."
enableDynamicLabels: "Experimental! When true the dispatcher allows `ghr-*` dynamic labels for jobs routed to this runner. Default false."
awsDynamicLabelsPolicy: "Optional AWS dynamic label policy evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose provider dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`."
}
redrive\_build\_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries."
} | 
map(object({
    runner_config = object({
      runner_os           = string
      runner_architecture = string
      runner_metadata_options = optional(map(any), {
        instance_metadata_tags      = "enabled"
        http_endpoint               = "enabled"
        http_tokens                 = "required"
        http_put_response_hop_limit = 1
      })
      ami = optional(object({
        filter               = optional(map(list(string)), { state = ["available"] })
        owners               = optional(list(string), ["amazon"])
        id_ssm_parameter_arn = optional(string, null)
        kms_key_arn          = optional(string, null)
      }), null)
      create_service_linked_role_spot      = optional(bool, false)
      credit_specification                 = optional(string, null)
      delay_webhook_event                  = optional(number, 30)
      disable_runner_autoupdate            = optional(bool, false)
      ebs_optimized                        = optional(bool, false)
      enable_ephemeral_runners             = optional(bool, false)
      enable_job_queued_check              = optional(bool, null)
      enable_on_demand_failover_for_errors = optional(list(string), [])
      scale_errors = optional(list(string), [
        "UnfulfillableCapacity",
        "MaxSpotInstanceCountExceeded",
        "TargetCapacityLimitExceededException",
        "RequestLimitExceeded",
        "ResourceLimitExceeded",
        "MaxSpotInstanceCountExceeded",
        "MaxSpotFleetRequestCountExceeded",
        "InsufficientInstanceCapacity",
        "InsufficientCapacityOnHost",
      ])
      enable_organization_runners                                    = optional(bool, false)
      enable_runner_binaries_syncer                                  = optional(bool, true)
      enable_ssm_on_runners                                          = optional(bool, false)
      enable_userdata                                                = optional(bool, true)
      instance_allocation_strategy                                   = optional(string, "lowest-price")
      instance_type_priorities                                       = optional(map(number), null)
      instance_max_spot_price                                        = optional(string, null)
      instance_target_capacity_type                                  = optional(string, "spot")
      instance_types                                                 = list(string)
      job_queue_retention_in_seconds                                 = optional(number, 86400)
      minimum_running_time_in_minutes                                = optional(number, null)
      pool_runner_owner                                              = optional(string, null)
      runner_as_root                                                 = optional(bool, false)
      runner_boot_time_in_minutes                                    = optional(number, 5)
      runner_disable_default_labels                                  = optional(bool, false)
      runner_extra_labels                                            = optional(list(string), [])
      runner_group_name                                              = optional(string, "Default")
      runner_name_prefix                                             = optional(string, "")
      runner_run_as                                                  = optional(string, "ec2-user")
      runners_maximum_count                                          = number
      runner_additional_security_group_ids                           = optional(list(string), [])
      scale_down_schedule_expression                                 = optional(string, "cron(*/5 * * * ? *)")
      scale_up_reserved_concurrent_executions                        = optional(number, 1)
      lambda_event_source_mapping_batch_size                         = optional(number, null)
      lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
      userdata_template                                              = optional(string, null)
      userdata_content                                               = optional(string, null)
      enable_jit_config                                              = optional(bool, null)
      enable_runner_detailed_monitoring                              = optional(bool, false)
      enable_cloudwatch_agent                                        = optional(bool, true)
      cloudwatch_config                                              = optional(string, null)
      userdata_pre_install                                           = optional(string, "")
      userdata_post_install                                          = optional(string, "")
      runner_hook_job_started                                        = optional(string, "")
      runner_hook_job_completed                                      = optional(string, "")
      runner_ec2_tags                                                = optional(map(string), {})
      runner_iam_role_managed_policy_arns                            = optional(list(string), [])
      vpc_id                                                         = optional(string, null)
      subnet_ids                                                     = optional(list(string), null)
      idle_config = optional(list(object({
        type             = optional(string, "ec2")
        cron             = string
        timeZone         = string
        idleCount        = number
        evictionStrategy = optional(string, "oldest_first")
      })), [])
      cpu_options = optional(object({
        core_count            = optional(number)
        threads_per_core      = optional(number)
        amd_sev_snp           = optional(string)
        nested_virtualization = optional(string)
      }), null)
      placement = optional(object({
        affinity                = optional(string)
        availability_zone       = optional(string)
        group_id                = optional(string)
        group_name              = optional(string)
        host_id                 = optional(string)
        host_resource_group_arn = optional(string)
        spread_domain           = optional(string)
        tenancy                 = optional(string)
        partition_number        = optional(number)
      }), null)
      license_specifications = optional(list(object({
        license_configuration_arn = string
      })), [])
      use_dedicated_host = optional(bool, false)
      runner_log_files = optional(list(object({
        log_group_name   = string
        prefix_log_group = bool
        file_path        = string
        log_stream_name  = string
        log_class        = optional(string, "STANDARD")
      })), null)
      block_device_mappings = optional(list(object({
        delete_on_termination      = optional(bool, true)
        device_name                = optional(string, "/dev/xvda")
        encrypted                  = optional(bool, true)
        iops                       = optional(number)
        kms_key_id                 = optional(string)
        snapshot_id                = optional(string)
        throughput                 = optional(number)
        volume_initialization_rate = optional(number)
        volume_size                = number
        volume_type                = optional(string, "gp3")
        })), [{
        volume_size = 30
      }])
      pool_config = optional(list(object({
        schedule_expression          = string
        schedule_expression_timezone = optional(string)
        size                         = number
      })), [])
      job_retry = optional(object({
        enable             = optional(bool, false)
        delay_in_seconds   = optional(number, 300)
        delay_backoff      = optional(number, 2)
        lambda_memory_size = optional(number, 256)
        lambda_timeout     = optional(number, 30)
        max_attempts       = optional(number, 1)
      }), {})
      iam_overrides = optional(object({
        override_instance_profile = optional(bool, null)
        instance_profile_name     = optional(string, null)
        override_runner_role      = optional(bool, null)
        runner_role_arn           = optional(string, null)
        }), {
        override_instance_profile = false
        instance_profile_name     = null
        override_runner_role      = false
        runner_role_arn           = null
      })
    })
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = optional(bool, false)
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy  = optional(any, null)
    })
    redrive_build_queue = optional(object({
      enabled         = bool
      maxReceiveCount = number
      }), {
      enabled         = false
      maxReceiveCount = null
    })
  }))
| n/a | yes | | [parameter\_store\_tags](#input\_parameter\_store\_tags) | Map of tags that will be added to all the SSM Parameter Store parameters created by the Lambda function. | `map(string)` | `{}` | no | | [pool\_lambda\_reserved\_concurrent\_executions](#input\_pool\_lambda\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no | | [pool\_lambda\_timeout](#input\_pool\_lambda\_timeout) | Time out for the pool lambda in seconds. | `number` | `60` | no | diff --git a/modules/runners/README.md b/modules/runners/README.md index d1f42d8173..8b1cad13a5 100644 --- a/modules/runners/README.md +++ b/modules/runners/README.md @@ -165,10 +165,11 @@ yarn run dist | [github\_app\_parameters](#input\_github\_app\_parameters) | Parameter Store for GitHub App Parameters. | 
object({
    key_base64 = map(string)
    id         = map(string)
  })
| n/a | yes | | [iam\_overrides](#input\_iam\_overrides) | This map provides the possibility to override some IAM defaults. The following attributes are supported: `instance_profile_name` overrides the instance profile name used in the launch template. `runner_role_arn` overrides the IAM role ARN used for the runner instances. | 
object({
    override_instance_profile = optional(bool, null)
    instance_profile_name     = optional(string, null)
    override_runner_role      = optional(bool, null)
    runner_role_arn           = optional(string, null)
  })
| 
{
  "instance_profile_name": null,
  "override_instance_profile": false,
  "override_runner_role": false,
  "runner_role_arn": null
}
| no | | [idle\_config](#input\_idle\_config) | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    type             = optional(string, "ec2")
    cron             = string
    timeZone         = string
    idleCount        = number
    evictionStrategy = optional(string, "oldest_first")
  }))
| `[]` | no | -| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for spot instances. AWS recommends to use `capacity-optimized` however the AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | +| [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | | [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the prefix will be used. | `string` | `null` | no | | [instance\_target\_capacity\_type](#input\_instance\_target\_capacity\_type) | Default lifecycle used runner instances, can be either `spot` or `on-demand`. | `string` | `"spot"` | no | +| [instance\_type\_priorities](#input\_instance\_type\_priorities) | A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`. | `map(number)` | `null` | no | | [instance\_types](#input\_instance\_types) | List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win). | `list(string)` | `null` | no | | [job\_retry](#input\_job\_retry) | Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app.

`enable`: Enable or disable the job retry feature.
`delay_in_seconds`: The delay in seconds before the job retry check lambda will check the job status.
`delay_backoff`: The backoff factor for the delay.
`lambda_memory_size`: Memory size limit in MB for the job retry check lambda.
'lambda\_reserved\_concurrent\_executions': Amount of reserved concurrent executions for the job retry check lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations.
`lambda_timeout`: Time out of the job retry check lambda in seconds.
`max_attempts`: The maximum number of attempts to retry the job. | 
object({
    enable                                = optional(bool, false)
    delay_in_seconds                      = optional(number, 300)
    delay_backoff                         = optional(number, 2)
    lambda_memory_size                    = optional(number, 256)
    lambda_reserved_concurrent_executions = optional(number, 1)

    lambda_timeout = optional(number, 30)

    max_attempts = optional(number, 1)
  })
| `{}` | no | | [key\_name](#input\_key\_name) | Key pair name | `string` | `null` | no | diff --git a/modules/runners/pool/README.md b/modules/runners/pool/README.md index 401c7f2cc8..b16f3ec353 100644 --- a/modules/runners/pool/README.md +++ b/modules/runners/pool/README.md @@ -49,7 +49,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [aws\_partition](#input\_aws\_partition) | (optional) partition for the arn if not 'aws' | `string` | `"aws"` | no | -| [config](#input\_config) | Lookup details in parent module. | 
object({
    lambda = object({
      log_level                      = string
      logging_retention_in_days      = number
      logging_kms_key_id             = string
      log_class                      = string
      reserved_concurrent_executions = number
      s3_bucket                      = string
      s3_key                         = string
      s3_object_version              = string
      security_group_ids             = list(string)
      runtime                        = string
      architecture                   = string
      memory_size                    = number
      timeout                        = number
      zip                            = string
      subnet_ids                     = list(string)
      parameter_store_tags           = string
    })
    tags = map(string)
    ghes = object({
      url        = string
      ssl_verify = string
    })
    github_app_parameters = object({
      key_base64 = map(string)
      id         = map(string)
    })
    subnet_ids = list(string)
    runner = object({
      disable_runner_autoupdate            = bool
      ephemeral                            = bool
      enable_jit_config                    = bool
      enable_on_demand_failover_for_errors = list(string)
      scale_errors                         = list(string)
      boot_time_in_minutes                 = number
      labels                               = list(string)
      launch_template = object({
        name = string
      })
      group_name  = string
      name_prefix = string
      pool_owner  = string
      role = object({
        arn = string
      })
      use_dedicated_host = bool
    })
    runners_maximum_count         = number
    instance_types                = list(string)
    instance_target_capacity_type = string
    instance_allocation_strategy  = string
    instance_max_spot_price       = string
    prefix                        = string
    pool = list(object({
      schedule_expression          = string
      schedule_expression_timezone = string
      size                         = number
    }))
    role_permissions_boundary            = string
    kms_key_arn                          = string
    ami_kms_key_arn                      = string
    ami_id_ssm_parameter_arn             = string
    role_path                            = string
    ssm_token_path                       = string
    ssm_config_path                      = string
    ami_id_ssm_parameter_name            = string
    ami_id_ssm_parameter_read_policy_arn = string
    arn_ssm_parameters_path_config       = string
    lambda_tags                          = map(string)
    user_agent                           = string
  })
| n/a | yes | +| [config](#input\_config) | Lookup details in parent module. | 
object({
    lambda = object({
      log_level                      = string
      logging_retention_in_days      = number
      logging_kms_key_id             = string
      log_class                      = string
      reserved_concurrent_executions = number
      s3_bucket                      = string
      s3_key                         = string
      s3_object_version              = string
      security_group_ids             = list(string)
      runtime                        = string
      architecture                   = string
      memory_size                    = number
      timeout                        = number
      zip                            = string
      subnet_ids                     = list(string)
      parameter_store_tags           = string
    })
    tags = map(string)
    ghes = object({
      url        = string
      ssl_verify = string
    })
    github_app_parameters = object({
      key_base64 = map(string)
      id         = map(string)
    })
    subnet_ids = list(string)
    runner = object({
      disable_runner_autoupdate            = bool
      ephemeral                            = bool
      enable_jit_config                    = bool
      enable_on_demand_failover_for_errors = list(string)
      scale_errors                         = list(string)
      boot_time_in_minutes                 = number
      labels                               = list(string)
      launch_template = object({
        name = string
      })
      group_name  = string
      name_prefix = string
      pool_owner  = string
      role = object({
        arn = string
      })
      use_dedicated_host = bool
    })
    runners_maximum_count         = number
    instance_types                = list(string)
    instance_type_priorities      = optional(map(number))
    instance_target_capacity_type = string
    instance_allocation_strategy  = string
    instance_max_spot_price       = string
    prefix                        = string
    pool = list(object({
      schedule_expression          = string
      schedule_expression_timezone = string
      size                         = number
    }))
    role_permissions_boundary            = string
    kms_key_arn                          = string
    ami_kms_key_arn                      = string
    ami_id_ssm_parameter_arn             = string
    role_path                            = string
    ssm_token_path                       = string
    ssm_config_path                      = string
    ami_id_ssm_parameter_name            = string
    ami_id_ssm_parameter_read_policy_arn = string
    arn_ssm_parameters_path_config       = string
    lambda_tags                          = map(string)
    user_agent                           = string
  })
| n/a | yes | | [tracing\_config](#input\_tracing\_config) | Configuration for lambda tracing. | 
object({
    mode                  = optional(string, null)
    capture_http_requests = optional(bool, false)
    capture_error         = optional(bool, false)
  })
| `{}` | no | ## Outputs From 4f71c2ad8de80b0aa5e09c2742bd16a0e1348177 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 7 Jul 2026 23:45:57 +0200 Subject: [PATCH 10/46] refactor(tests): keep dispatch label tests in place --- .../webhook/src/runners/dispatch.test.ts | 236 +++++++++--------- 1 file changed, 118 insertions(+), 118 deletions(-) diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index acb36c0c68..48d92be9af 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -246,6 +246,124 @@ describe('Dispatcher', () => { }); }); + describe('decides can run job based on label and config (canRunJob)', () => { + it('should accept job with an exact match and identical labels.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should accept job with an exact match and identical labels, ignoring cases.', () => { + const workflowLabels = ['self-Hosted', 'Linux', 'X64', 'ubuntu-Latest']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should accept job with an exact match and runner supports requested capabilities.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should NOT accept job with an exact match and runner not matching requested capabilities.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(false); + }); + + it('should accept job with for a non exact match. Any label that matches will accept the job.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; + const runnerLabels = [['gpu']]; + expect(canRunJob(workflowLabels, runnerLabels, false)).toBe(true); + }); + + it('should NOT accept job with for an exact match. Not all requested capabilities are supported.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; + const runnerLabels = [['gpu']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(false); + }); + + it('should match when runner has more labels than workflow requests with exactMatch=true (unidirectional).', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404', 'on-demand']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should match when labels are exactly identical with exactMatch=true.', () => { + const workflowLabels = ['self-hosted', 'linux', 'on-demand']; + const runnerLabels = [['self-hosted', 'linux', 'on-demand']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should match with exactMatch=true when labels are in different order.', () => { + const workflowLabels = ['linux', 'self-hosted', 'x64']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should match with exactMatch=true when labels are completely shuffled.', () => { + const workflowLabels = ['x64', 'ubuntu-latest', 'self-hosted', 'linux']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); + }); + + it('should match with exactMatch=false when labels are in different order.', () => { + const workflowLabels = ['gpu', 'self-hosted']; + const runnerLabels = [['self-hosted', 'gpu']]; + expect(canRunJob(workflowLabels, runnerLabels, false)).toBe(true); + }); + + // bidirectionalLabelMatch tests + it('should NOT match when runner has more labels than workflow requests (bidirectionalLabelMatch=true).', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404', 'on-demand']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); + }); + + it('should NOT match when workflow has more labels than runner (bidirectionalLabelMatch=true).', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); + }); + + it('should match when labels are exactly identical with bidirectionalLabelMatch=true.', () => { + const workflowLabels = ['self-hosted', 'linux', 'on-demand']; + const runnerLabels = [['self-hosted', 'linux', 'on-demand']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); + }); + + it('should match with bidirectionalLabelMatch=true when labels are in different order.', () => { + const workflowLabels = ['linux', 'self-hosted', 'x64']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); + }); + + it('should match with bidirectionalLabelMatch=true when labels are completely shuffled.', () => { + const workflowLabels = ['x64', 'ubuntu-latest', 'self-hosted', 'linux']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); + }); + + it('should match with bidirectionalLabelMatch=true ignoring case.', () => { + const workflowLabels = ['Self-Hosted', 'Linux', 'X64']; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); + }); + + it('should NOT match empty workflow labels with bidirectionalLabelMatch=true.', () => { + const workflowLabels: string[] = []; + const runnerLabels = [['self-hosted', 'linux', 'x64']]; + expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); + }); + + it('bidirectionalLabelMatch takes precedence over exactMatch when both are true.', () => { + const workflowLabels = ['self-hosted', 'linux', 'x64']; + const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; + // exactMatch alone would accept this (runner has extra labels), but bidirectional should reject + expect(canRunJob(workflowLabels, runnerLabels, true, true)).toBe(false); + }); + }); + describe('per-matcher dynamic labels handling', () => { const baseRunner = runnerConfig[0]; @@ -463,124 +581,6 @@ async function createConfig(repositoryAllowList?: string[], runnerConfig?: Runne return await ConfigDispatcher.load(); } -describe('decides can run job based on label and config (canRunJob)', () => { - it('should accept job with an exact match and identical labels.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should accept job with an exact match and identical labels, ignoring cases.', () => { - const workflowLabels = ['self-Hosted', 'Linux', 'X64', 'ubuntu-Latest']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should accept job with an exact match and runner supports requested capabilities.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should NOT accept job with an exact match and runner not matching requested capabilities.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(false); - }); - - it('should accept job with for a non exact match. Any label that matches will accept the job.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; - const runnerLabels = [['gpu']]; - expect(canRunJob(workflowLabels, runnerLabels, false)).toBe(true); - }); - - it('should NOT accept job with for an exact match. Not all requested capabilities are supported.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; - const runnerLabels = [['gpu']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(false); - }); - - it('should match when runner has more labels than workflow requests with exactMatch=true (unidirectional).', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404', 'on-demand']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should match when labels are exactly identical with exactMatch=true.', () => { - const workflowLabels = ['self-hosted', 'linux', 'on-demand']; - const runnerLabels = [['self-hosted', 'linux', 'on-demand']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should match with exactMatch=true when labels are in different order.', () => { - const workflowLabels = ['linux', 'self-hosted', 'x64']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should match with exactMatch=true when labels are completely shuffled.', () => { - const workflowLabels = ['x64', 'ubuntu-latest', 'self-hosted', 'linux']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, true)).toBe(true); - }); - - it('should match with exactMatch=false when labels are in different order.', () => { - const workflowLabels = ['gpu', 'self-hosted']; - const runnerLabels = [['self-hosted', 'gpu']]; - expect(canRunJob(workflowLabels, runnerLabels, false)).toBe(true); - }); - - // bidirectionalLabelMatch tests - it('should NOT match when runner has more labels than workflow requests (bidirectionalLabelMatch=true).', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'staging', 'ubuntu-2404', 'on-demand']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); - }); - - it('should NOT match when workflow has more labels than runner (bidirectionalLabelMatch=true).', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64', 'ubuntu-latest', 'gpu']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); - }); - - it('should match when labels are exactly identical with bidirectionalLabelMatch=true.', () => { - const workflowLabels = ['self-hosted', 'linux', 'on-demand']; - const runnerLabels = [['self-hosted', 'linux', 'on-demand']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); - }); - - it('should match with bidirectionalLabelMatch=true when labels are in different order.', () => { - const workflowLabels = ['linux', 'self-hosted', 'x64']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); - }); - - it('should match with bidirectionalLabelMatch=true when labels are completely shuffled.', () => { - const workflowLabels = ['x64', 'ubuntu-latest', 'self-hosted', 'linux']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); - }); - - it('should match with bidirectionalLabelMatch=true ignoring case.', () => { - const workflowLabels = ['Self-Hosted', 'Linux', 'X64']; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(true); - }); - - it('should NOT match empty workflow labels with bidirectionalLabelMatch=true.', () => { - const workflowLabels: string[] = []; - const runnerLabels = [['self-hosted', 'linux', 'x64']]; - expect(canRunJob(workflowLabels, runnerLabels, false, true)).toBe(false); - }); - - it('bidirectionalLabelMatch takes precedence over exactMatch when both are true.', () => { - const workflowLabels = ['self-hosted', 'linux', 'x64']; - const runnerLabels = [['self-hosted', 'linux', 'x64', 'ubuntu-latest']]; - // exactMatch alone would accept this (runner has extra labels), but bidirectional should reject - expect(canRunJob(workflowLabels, runnerLabels, true, true)).toBe(false); - }); -}); - describe('selectAwsDynamicLabelQueue', () => { it('defaults queues without a provider to EC2 dynamic label handling', () => { const queue = runnerQueue('default-ec2'); From 606913053737269e5be699ddfe92be825e36a5eb Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 8 Jul 2026 00:35:42 +0200 Subject: [PATCH 11/46] refactor(terraform): remove idle config provider type --- README.md | 2 +- modules/multi-runner/README.md | 2 +- modules/multi-runner/variables.tf | 1 - modules/runners/README.md | 2 +- modules/runners/variables.tf | 1 - variables.tf | 1 - 6 files changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3e866b0475..c92dc28877 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh) | [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. Example: https://github.internal.co - DO NOT SET IF USING PUBLIC GITHUB. However if you are using GitHub Enterprise Cloud with data-residency (ghe.com), set the endpoint here. Example - https://companyname.ghe.com | `string` | `null` | no | | [github\_app](#input\_github\_app) | GitHub app parameters, see your github app.
You can optionally create the SSM parameters yourself and provide the ARN and name here, through the `*_ssm` attributes.
If you chose to provide the configuration values directly here,
please ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`).
Note: the provided SSM parameters arn and name have a precedence over the actual value (i.e `key_base64_ssm` has a precedence over `key_base64` etc). | 
object({
    key_base64 = optional(string)
    key_base64_ssm = optional(object({
      arn  = string
      name = string
    }))
    id = optional(string)
    id_ssm = optional(object({
      arn  = string
      name = string
    }))
    webhook_secret = optional(string)
    webhook_secret_ssm = optional(object({
      arn  = string
      name = string
    }))
  })
| n/a | yes | | [iam\_overrides](#input\_iam\_overrides) | This map provides the possibility to override some IAM defaults. Note that when using this variable, you are responsible for ensuring the role has necessary permissions to access required resources. `override_instance_profile`: When set to true, uses the instance profile name specified in `instance_profile_name` instead of creating a new instance profile. `override_runner_role`: When set to true, uses the role ARN specified in `runner_role_arn` instead of creating a new IAM role. | 
object({
    override_instance_profile = optional(bool, null)
    instance_profile_name     = optional(string, null)
    override_runner_role      = optional(bool, null)
    runner_role_arn           = optional(string, null)
  })
| 
{
  "instance_profile_name": null,
  "override_instance_profile": false,
  "override_runner_role": false,
  "runner_role_arn": null
}
| no | -| [idle\_config](#input\_idle\_config) | List of time periods, defined as a cron expression, to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    type             = optional(string, "ec2")
    cron             = string
    timeZone         = string
    idleCount        = number
    evictionStrategy = optional(string, "oldest_first")
  }))
| `[]` | no | +| [idle\_config](#input\_idle\_config) | List of time periods, defined as a cron expression, to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    cron             = string
    timeZone         = string
    idleCount        = number
    evictionStrategy = optional(string, "oldest_first")
  }))
| `[]` | no | | [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | | [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no | diff --git a/modules/multi-runner/README.md b/modules/multi-runner/README.md index dbe328ec3c..612df47652 100644 --- a/modules/multi-runner/README.md +++ b/modules/multi-runner/README.md @@ -152,7 +152,7 @@ module "multi-runner" { | [logging\_retention\_in\_days](#input\_logging\_retention\_in\_days) | Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. | `number` | `180` | no | | [matcher\_config\_parameter\_store\_tier](#input\_matcher\_config\_parameter\_store\_tier) | The tier of the parameter store for the matcher configuration. Valid values are `Standard`, and `Advanced`. | `string` | `"Standard"` | no | | [metrics](#input\_metrics) | Configuration for metrics created by the module, by default metrics are disabled to avoid additional costs. When metrics are enable all metrics are created unless explicit configured otherwise. | 
object({
    enable    = optional(bool, false)
    namespace = optional(string, "GitHub Runners")
    metric = optional(object({
      enable_github_app_rate_limit    = optional(bool, true)
      enable_job_retry                = optional(bool, true)
      enable_spot_termination_warning = optional(bool, true)
    }), {})
  })
| `{}` | no | -| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {
runner\_config: {
runner\_os: "The EC2 Operating System type to use for action runner instances (linux, osx, windows)."
runner\_architecture: "The platform architecture of the runner instance\_type."
runner\_metadata\_options: "(Optional) Metadata options for the ec2 runner instances."
ami: "(Optional) AMI configuration for the action runner instances. This object allows you to specify all AMI-related settings in one place."
create\_service\_linked\_role\_spot: (Optional) create the serviced linked role for spot instances that is required by the scale-up lambda.
credit\_specification: "(Optional) The credit specification of the runner instance\_type. Can be unset, `standard` or `unlimited`.
delay\_webhook\_event: "The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event."
disable\_runner\_autoupdate: "Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)"
ebs\_optimized: "The EC2 EBS optimized configuration."
enable\_ephemeral\_runners: "Enable ephemeral runners, runners will only be used once."
enable\_job\_queued\_check: Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners."
enable\_on\_demand\_failover\_for\_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later."
scale\_errors: "List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors"
enable\_organization\_runners: "Register runners to organization, instead of repo level"
enable\_runner\_binaries\_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
enable\_ssm\_on\_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances."
enable\_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI."
instance\_allocation\_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`."
instance\_type\_priorities: "A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`."
instance\_max\_spot\_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet."
instance\_target\_capacity\_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`."
instance\_types: "List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)."
job\_queue\_retention\_in\_seconds: "The number of seconds the job is held in the queue before it is purged"
minimum\_running\_time\_in\_minutes: "The time an ec2 action runner should be running at minimum before terminated if not busy."
pool\_runner\_owner: "The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported."
runner\_additional\_security\_group\_ids: "List of additional security groups IDs to apply to the runner. If added outside the multi\_runner\_config block, the additional security group(s) will be applied to all runner configs. If added inside the multi\_runner\_config, the additional security group(s) will be applied to the individual runner."
runner\_as\_root: "Run the action runner under the root user. Variable `runner_run_as` will be ignored."
runner\_boot\_time\_in\_minutes: "The minimum time for an EC2 runner to boot and register as a runner."
runner\_disable\_default\_labels: "Disable default labels for the runners (os, architecture and `self-hosted`). If enabled, the runner will only have the extra labels provided in `runner_extra_labels`. In case you on own start script is used, this configuration parameter needs to be parsed via SSM."
runner\_extra\_labels: "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `multi_runner_config.matcherConfig.exactMatch`. GitHub read-only labels should not be provided."
runner\_group\_name: "Name of the runner group."
runner\_name\_prefix: "Prefix for the GitHub runner name."
runner\_run\_as: "Run the GitHub actions agent as user."
runners\_maximum\_count: "The maximum number of runners that will be created. Setting the variable to `-1` disables the maximum check."
scale\_down\_schedule\_expression: "Scheduler expression to check every x for scale down."
scale\_up\_reserved\_concurrent\_executions: "Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations."
lambda\_event\_source\_mapping\_batch\_size: "(Optional) Maximum number of records per Lambda invocation for this runner flavor. Overrides the module-level `lambda_event_source_mapping_batch_size` when set."
lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds: "(Optional) Maximum seconds to gather records before invoking Lambda for this runner flavor. Overrides the module-level `lambda_event_source_mapping_maximum_batching_window_in_seconds` when set."
userdata\_template: "Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored."
enable\_jit\_config: "Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is available. In case you are upgrading from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI."
enable\_runner\_detailed\_monitoring: "Should detailed monitoring be enabled for the runner. Set this to true if you want to use detailed monitoring. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html for details."
enable\_cloudwatch\_agent: "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
cloudwatch\_config: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
userdata\_pre\_install: "Script to be ran before the GitHub Actions runner is installed on the EC2 instances"
userdata\_post\_install: "Script to be ran after the GitHub Actions runner is installed on the EC2 instances"
runner\_hook\_job\_started: "Script to be ran in the runner environment at the beginning of every job"
runner\_hook\_job\_completed: "Script to be ran in the runner environment at the end of every job"
runner\_ec2\_tags: "Map of tags that will be added to the launch template instance tag specifications."
runner\_iam\_role\_managed\_policy\_arns: "Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role"
vpc\_id: "The VPC for security groups of the action runners. If not set uses the value of `var.vpc_id`."
subnet\_ids: "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. If not set, uses the value of `var.subnet_ids`."
idle\_config: "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle."
license\_specifications: "Optional EC2 License Manager license configuration ARNs for the runner launch template. Required for macOS dedicated-host runners when the host resource group uses a Mac dedicated host license configuration."
use\_dedicated\_host: "Experimental! Can be removed / changed without trigger a major release. Whether to use EC2 dedicated hosts for the runners. Needed for macos runners Note that using dedicated hosts can increase cost significantly."
runner\_log\_files: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
block\_device\_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`."
job\_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app."
pool\_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)."
iam\_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances."
}
matcherConfig: {
labelMatchers: "The list of list of labels supported by the runner configuration. `[[self-hosted, linux, x64, example]]`"
exactMatch: "DEPRECATED: Use `bidirectionalLabelMatch` instead. If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ workflow label matches it will trigger the webhook. Note: this only checks that workflow labels are a subset of runner labels, not the reverse."
bidirectionalLabelMatch: "If set to true, the runner labels and workflow job labels must be an exact two-way match (same set, any order, no extras or missing labels). This is stricter than `exactMatch` which only checks that workflow labels are a subset of runner labels. When false, if __any__ workflow label matches it will trigger the webhook."
priority: "If set it defines the priority of the matcher, the matcher with the lowest priority will be evaluated first. Default is 999, allowed values 0-999."
enableDynamicLabels: "Experimental! When true the dispatcher allows `ghr-*` dynamic labels for jobs routed to this runner. Default false."
awsDynamicLabelsPolicy: "Optional AWS dynamic label policy evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose provider dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`."
}
redrive\_build\_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries."
} | 
map(object({
    runner_config = object({
      runner_os           = string
      runner_architecture = string
      runner_metadata_options = optional(map(any), {
        instance_metadata_tags      = "enabled"
        http_endpoint               = "enabled"
        http_tokens                 = "required"
        http_put_response_hop_limit = 1
      })
      ami = optional(object({
        filter               = optional(map(list(string)), { state = ["available"] })
        owners               = optional(list(string), ["amazon"])
        id_ssm_parameter_arn = optional(string, null)
        kms_key_arn          = optional(string, null)
      }), null)
      create_service_linked_role_spot      = optional(bool, false)
      credit_specification                 = optional(string, null)
      delay_webhook_event                  = optional(number, 30)
      disable_runner_autoupdate            = optional(bool, false)
      ebs_optimized                        = optional(bool, false)
      enable_ephemeral_runners             = optional(bool, false)
      enable_job_queued_check              = optional(bool, null)
      enable_on_demand_failover_for_errors = optional(list(string), [])
      scale_errors = optional(list(string), [
        "UnfulfillableCapacity",
        "MaxSpotInstanceCountExceeded",
        "TargetCapacityLimitExceededException",
        "RequestLimitExceeded",
        "ResourceLimitExceeded",
        "MaxSpotInstanceCountExceeded",
        "MaxSpotFleetRequestCountExceeded",
        "InsufficientInstanceCapacity",
        "InsufficientCapacityOnHost",
      ])
      enable_organization_runners                                    = optional(bool, false)
      enable_runner_binaries_syncer                                  = optional(bool, true)
      enable_ssm_on_runners                                          = optional(bool, false)
      enable_userdata                                                = optional(bool, true)
      instance_allocation_strategy                                   = optional(string, "lowest-price")
      instance_type_priorities                                       = optional(map(number), null)
      instance_max_spot_price                                        = optional(string, null)
      instance_target_capacity_type                                  = optional(string, "spot")
      instance_types                                                 = list(string)
      job_queue_retention_in_seconds                                 = optional(number, 86400)
      minimum_running_time_in_minutes                                = optional(number, null)
      pool_runner_owner                                              = optional(string, null)
      runner_as_root                                                 = optional(bool, false)
      runner_boot_time_in_minutes                                    = optional(number, 5)
      runner_disable_default_labels                                  = optional(bool, false)
      runner_extra_labels                                            = optional(list(string), [])
      runner_group_name                                              = optional(string, "Default")
      runner_name_prefix                                             = optional(string, "")
      runner_run_as                                                  = optional(string, "ec2-user")
      runners_maximum_count                                          = number
      runner_additional_security_group_ids                           = optional(list(string), [])
      scale_down_schedule_expression                                 = optional(string, "cron(*/5 * * * ? *)")
      scale_up_reserved_concurrent_executions                        = optional(number, 1)
      lambda_event_source_mapping_batch_size                         = optional(number, null)
      lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
      userdata_template                                              = optional(string, null)
      userdata_content                                               = optional(string, null)
      enable_jit_config                                              = optional(bool, null)
      enable_runner_detailed_monitoring                              = optional(bool, false)
      enable_cloudwatch_agent                                        = optional(bool, true)
      cloudwatch_config                                              = optional(string, null)
      userdata_pre_install                                           = optional(string, "")
      userdata_post_install                                          = optional(string, "")
      runner_hook_job_started                                        = optional(string, "")
      runner_hook_job_completed                                      = optional(string, "")
      runner_ec2_tags                                                = optional(map(string), {})
      runner_iam_role_managed_policy_arns                            = optional(list(string), [])
      vpc_id                                                         = optional(string, null)
      subnet_ids                                                     = optional(list(string), null)
      idle_config = optional(list(object({
        type             = optional(string, "ec2")
        cron             = string
        timeZone         = string
        idleCount        = number
        evictionStrategy = optional(string, "oldest_first")
      })), [])
      cpu_options = optional(object({
        core_count            = optional(number)
        threads_per_core      = optional(number)
        amd_sev_snp           = optional(string)
        nested_virtualization = optional(string)
      }), null)
      placement = optional(object({
        affinity                = optional(string)
        availability_zone       = optional(string)
        group_id                = optional(string)
        group_name              = optional(string)
        host_id                 = optional(string)
        host_resource_group_arn = optional(string)
        spread_domain           = optional(string)
        tenancy                 = optional(string)
        partition_number        = optional(number)
      }), null)
      license_specifications = optional(list(object({
        license_configuration_arn = string
      })), [])
      use_dedicated_host = optional(bool, false)
      runner_log_files = optional(list(object({
        log_group_name   = string
        prefix_log_group = bool
        file_path        = string
        log_stream_name  = string
        log_class        = optional(string, "STANDARD")
      })), null)
      block_device_mappings = optional(list(object({
        delete_on_termination      = optional(bool, true)
        device_name                = optional(string, "/dev/xvda")
        encrypted                  = optional(bool, true)
        iops                       = optional(number)
        kms_key_id                 = optional(string)
        snapshot_id                = optional(string)
        throughput                 = optional(number)
        volume_initialization_rate = optional(number)
        volume_size                = number
        volume_type                = optional(string, "gp3")
        })), [{
        volume_size = 30
      }])
      pool_config = optional(list(object({
        schedule_expression          = string
        schedule_expression_timezone = optional(string)
        size                         = number
      })), [])
      job_retry = optional(object({
        enable             = optional(bool, false)
        delay_in_seconds   = optional(number, 300)
        delay_backoff      = optional(number, 2)
        lambda_memory_size = optional(number, 256)
        lambda_timeout     = optional(number, 30)
        max_attempts       = optional(number, 1)
      }), {})
      iam_overrides = optional(object({
        override_instance_profile = optional(bool, null)
        instance_profile_name     = optional(string, null)
        override_runner_role      = optional(bool, null)
        runner_role_arn           = optional(string, null)
        }), {
        override_instance_profile = false
        instance_profile_name     = null
        override_runner_role      = false
        runner_role_arn           = null
      })
    })
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = optional(bool, false)
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy  = optional(any, null)
    })
    redrive_build_queue = optional(object({
      enabled         = bool
      maxReceiveCount = number
      }), {
      enabled         = false
      maxReceiveCount = null
    })
  }))
| n/a | yes | +| [multi\_runner\_config](#input\_multi\_runner\_config) | multi\_runner\_config = {
runner\_config: {
runner\_os: "The EC2 Operating System type to use for action runner instances (linux, osx, windows)."
runner\_architecture: "The platform architecture of the runner instance\_type."
runner\_metadata\_options: "(Optional) Metadata options for the ec2 runner instances."
ami: "(Optional) AMI configuration for the action runner instances. This object allows you to specify all AMI-related settings in one place."
create\_service\_linked\_role\_spot: (Optional) create the serviced linked role for spot instances that is required by the scale-up lambda.
credit\_specification: "(Optional) The credit specification of the runner instance\_type. Can be unset, `standard` or `unlimited`.
delay\_webhook\_event: "The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event."
disable\_runner\_autoupdate: "Disable the auto update of the github runner agent. Be aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)"
ebs\_optimized: "The EC2 EBS optimized configuration."
enable\_ephemeral\_runners: "Enable ephemeral runners, runners will only be used once."
enable\_job\_queued\_check: Enables JIT configuration for creating runners instead of registration token based registraton. JIT configuration will only be applied for ephemeral runners. By default JIT configuration is enabled for ephemeral runners an can be disabled via this override. When running on GHES without support for JIT configuration this variable should be set to true for ephemeral runners."
enable\_on\_demand\_failover\_for\_errors: "Enable on-demand failover. For example to fall back to on demand when no spot capacity is available the variable can be set to `InsufficientInstanceCapacity`. When not defined the default behavior is to retry later."
scale\_errors: "List of AWS error codes that should trigger retry during scale up. This list replaces the module default scale-up retry errors"
enable\_organization\_runners: "Register runners to organization, instead of repo level"
enable\_runner\_binaries\_syncer: "Option to disable the lambda to sync GitHub runner distribution, useful when using a pre-build AMI."
enable\_ssm\_on\_runners: "Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances."
enable\_userdata: "Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI."
instance\_allocation\_strategy: "The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`."
instance\_type\_priorities: "A map of instance type to priority for the `prioritized` and `capacity-optimized-prioritized` allocation strategies. Lower numbers mean higher priority. If not provided, priorities are assigned based on the order of `instance_types`."
instance\_max\_spot\_price: "Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet."
instance\_target\_capacity\_type: "Default lifecycle used for runner instances, can be either `spot` or `on-demand`."
instance\_types: "List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win)."
job\_queue\_retention\_in\_seconds: "The number of seconds the job is held in the queue before it is purged"
minimum\_running\_time\_in\_minutes: "The time an ec2 action runner should be running at minimum before terminated if not busy."
pool\_runner\_owner: "The pool will deploy runners to the GitHub org ID, set this value to the org to which you want the runners deployed. Repo level is not supported."
runner\_additional\_security\_group\_ids: "List of additional security groups IDs to apply to the runner. If added outside the multi\_runner\_config block, the additional security group(s) will be applied to all runner configs. If added inside the multi\_runner\_config, the additional security group(s) will be applied to the individual runner."
runner\_as\_root: "Run the action runner under the root user. Variable `runner_run_as` will be ignored."
runner\_boot\_time\_in\_minutes: "The minimum time for an EC2 runner to boot and register as a runner."
runner\_disable\_default\_labels: "Disable default labels for the runners (os, architecture and `self-hosted`). If enabled, the runner will only have the extra labels provided in `runner_extra_labels`. In case you on own start script is used, this configuration parameter needs to be parsed via SSM."
runner\_extra\_labels: "Extra (custom) labels for the runners (GitHub). Separate each label by a comma. Labels checks on the webhook can be enforced by setting `multi_runner_config.matcherConfig.exactMatch`. GitHub read-only labels should not be provided."
runner\_group\_name: "Name of the runner group."
runner\_name\_prefix: "Prefix for the GitHub runner name."
runner\_run\_as: "Run the GitHub actions agent as user."
runners\_maximum\_count: "The maximum number of runners that will be created. Setting the variable to `-1` disables the maximum check."
scale\_down\_schedule\_expression: "Scheduler expression to check every x for scale down."
scale\_up\_reserved\_concurrent\_executions: "Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations."
lambda\_event\_source\_mapping\_batch\_size: "(Optional) Maximum number of records per Lambda invocation for this runner flavor. Overrides the module-level `lambda_event_source_mapping_batch_size` when set."
lambda\_event\_source\_mapping\_maximum\_batching\_window\_in\_seconds: "(Optional) Maximum seconds to gather records before invoking Lambda for this runner flavor. Overrides the module-level `lambda_event_source_mapping_maximum_batching_window_in_seconds` when set."
userdata\_template: "Alternative user-data template, replacing the default template. By providing your own user\_data you have to take care of installing all required software, including the action runner. Variables userdata\_pre/post\_install are ignored."
enable\_jit\_config: "Overwrite the default behavior for JIT configuration. By default JIT configuration is enabled for ephemeral runners and disabled for non-ephemeral runners. In case of GHES check first if the JIT config API is available. In case you are upgrading from 3.x to 4.x you can set `enable_jit_config` to `false` to avoid a breaking change when having your own AMI."
enable\_runner\_detailed\_monitoring: "Should detailed monitoring be enabled for the runner. Set this to true if you want to use detailed monitoring. See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html for details."
enable\_cloudwatch\_agent: "Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`."
cloudwatch\_config: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
userdata\_pre\_install: "Script to be ran before the GitHub Actions runner is installed on the EC2 instances"
userdata\_post\_install: "Script to be ran after the GitHub Actions runner is installed on the EC2 instances"
runner\_hook\_job\_started: "Script to be ran in the runner environment at the beginning of every job"
runner\_hook\_job\_completed: "Script to be ran in the runner environment at the end of every job"
runner\_ec2\_tags: "Map of tags that will be added to the launch template instance tag specifications."
runner\_iam\_role\_managed\_policy\_arns: "Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role"
vpc\_id: "The VPC for security groups of the action runners. If not set uses the value of `var.vpc_id`."
subnet\_ids: "List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`. If not set, uses the value of `var.subnet_ids`."
idle\_config: "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle."
license\_specifications: "Optional EC2 License Manager license configuration ARNs for the runner launch template. Required for macOS dedicated-host runners when the host resource group uses a Mac dedicated host license configuration."
use\_dedicated\_host: "Experimental! Can be removed / changed without trigger a major release. Whether to use EC2 dedicated hosts for the runners. Needed for macos runners Note that using dedicated hosts can increase cost significantly."
runner\_log\_files: "(optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details."
block\_device\_mappings: "The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops`, `throughput`, `kms_key_id`, `snapshot_id`, `volume_initialization_rate`."
job\_retry: "Experimental! Can be removed / changed without trigger a major release. Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app."
pool\_config: "The configuration for updating the pool. The `pool_size` to adjust to by the events triggered by the `schedule_expression`. For example you can configure a cron expression for week days to adjust the pool to 10 and another expression for the weekend to adjust the pool to 1. Use `schedule_expression_timezone` to override the schedule time zone (defaults to UTC)."
iam\_overrides: "Allows to (optionally) override the instance profile and runner role created by the module. Set `override_instance_profile` to true and provide the `instance_profile_name` to use an existing instance profile. Set `override_runner_role` to true and provide the `runner_role_arn` to use an existing role for the runner instances."
}
matcherConfig: {
labelMatchers: "The list of list of labels supported by the runner configuration. `[[self-hosted, linux, x64, example]]`"
exactMatch: "DEPRECATED: Use `bidirectionalLabelMatch` instead. If set to true all labels in the workflow job must match the GitHub labels (os, architecture and `self-hosted`). When false if __any__ workflow label matches it will trigger the webhook. Note: this only checks that workflow labels are a subset of runner labels, not the reverse."
bidirectionalLabelMatch: "If set to true, the runner labels and workflow job labels must be an exact two-way match (same set, any order, no extras or missing labels). This is stricter than `exactMatch` which only checks that workflow labels are a subset of runner labels. When false, if __any__ workflow label matches it will trigger the webhook."
priority: "If set it defines the priority of the matcher, the matcher with the lowest priority will be evaluated first. Default is 999, allowed values 0-999."
enableDynamicLabels: "Experimental! When true the dispatcher allows `ghr-*` dynamic labels for jobs routed to this runner. Default false."
awsDynamicLabelsPolicy: "Optional AWS dynamic label policy evaluated by the dispatcher. Only effective when `enableDynamicLabels = true`. Jobs whose provider dynamic labels violate every matching runner's policy are rejected with a 202 (a warning is logged). Evaluation: keys in `blocked_keys` are always rejected; keys in `restricted_keys` are allowed only when their value passes the rule; unlisted keys are allowed. Schema: `{ blocked_keys = [], restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } } }`. Keys use the dynamic label suffix, e.g. `instance-type` for `ghr-ec2-instance-type`."
}
redrive\_build\_queue: "Set options to attach (optional) a dead letter queue to the build queue, the queue between the webhook and the scale up lambda. You have the following options. 1. Disable by setting `enabled` to false. 2. Enable by setting `enabled` to `true`, `maxReceiveCount` to a number of max retries."
} | 
map(object({
    runner_config = object({
      runner_os           = string
      runner_architecture = string
      runner_metadata_options = optional(map(any), {
        instance_metadata_tags      = "enabled"
        http_endpoint               = "enabled"
        http_tokens                 = "required"
        http_put_response_hop_limit = 1
      })
      ami = optional(object({
        filter               = optional(map(list(string)), { state = ["available"] })
        owners               = optional(list(string), ["amazon"])
        id_ssm_parameter_arn = optional(string, null)
        kms_key_arn          = optional(string, null)
      }), null)
      create_service_linked_role_spot      = optional(bool, false)
      credit_specification                 = optional(string, null)
      delay_webhook_event                  = optional(number, 30)
      disable_runner_autoupdate            = optional(bool, false)
      ebs_optimized                        = optional(bool, false)
      enable_ephemeral_runners             = optional(bool, false)
      enable_job_queued_check              = optional(bool, null)
      enable_on_demand_failover_for_errors = optional(list(string), [])
      scale_errors = optional(list(string), [
        "UnfulfillableCapacity",
        "MaxSpotInstanceCountExceeded",
        "TargetCapacityLimitExceededException",
        "RequestLimitExceeded",
        "ResourceLimitExceeded",
        "MaxSpotInstanceCountExceeded",
        "MaxSpotFleetRequestCountExceeded",
        "InsufficientInstanceCapacity",
        "InsufficientCapacityOnHost",
      ])
      enable_organization_runners                                    = optional(bool, false)
      enable_runner_binaries_syncer                                  = optional(bool, true)
      enable_ssm_on_runners                                          = optional(bool, false)
      enable_userdata                                                = optional(bool, true)
      instance_allocation_strategy                                   = optional(string, "lowest-price")
      instance_type_priorities                                       = optional(map(number), null)
      instance_max_spot_price                                        = optional(string, null)
      instance_target_capacity_type                                  = optional(string, "spot")
      instance_types                                                 = list(string)
      job_queue_retention_in_seconds                                 = optional(number, 86400)
      minimum_running_time_in_minutes                                = optional(number, null)
      pool_runner_owner                                              = optional(string, null)
      runner_as_root                                                 = optional(bool, false)
      runner_boot_time_in_minutes                                    = optional(number, 5)
      runner_disable_default_labels                                  = optional(bool, false)
      runner_extra_labels                                            = optional(list(string), [])
      runner_group_name                                              = optional(string, "Default")
      runner_name_prefix                                             = optional(string, "")
      runner_run_as                                                  = optional(string, "ec2-user")
      runners_maximum_count                                          = number
      runner_additional_security_group_ids                           = optional(list(string), [])
      scale_down_schedule_expression                                 = optional(string, "cron(*/5 * * * ? *)")
      scale_up_reserved_concurrent_executions                        = optional(number, 1)
      lambda_event_source_mapping_batch_size                         = optional(number, null)
      lambda_event_source_mapping_maximum_batching_window_in_seconds = optional(number, null)
      userdata_template                                              = optional(string, null)
      userdata_content                                               = optional(string, null)
      enable_jit_config                                              = optional(bool, null)
      enable_runner_detailed_monitoring                              = optional(bool, false)
      enable_cloudwatch_agent                                        = optional(bool, true)
      cloudwatch_config                                              = optional(string, null)
      userdata_pre_install                                           = optional(string, "")
      userdata_post_install                                          = optional(string, "")
      runner_hook_job_started                                        = optional(string, "")
      runner_hook_job_completed                                      = optional(string, "")
      runner_ec2_tags                                                = optional(map(string), {})
      runner_iam_role_managed_policy_arns                            = optional(list(string), [])
      vpc_id                                                         = optional(string, null)
      subnet_ids                                                     = optional(list(string), null)
      idle_config = optional(list(object({
        cron             = string
        timeZone         = string
        idleCount        = number
        evictionStrategy = optional(string, "oldest_first")
      })), [])
      cpu_options = optional(object({
        core_count            = optional(number)
        threads_per_core      = optional(number)
        amd_sev_snp           = optional(string)
        nested_virtualization = optional(string)
      }), null)
      placement = optional(object({
        affinity                = optional(string)
        availability_zone       = optional(string)
        group_id                = optional(string)
        group_name              = optional(string)
        host_id                 = optional(string)
        host_resource_group_arn = optional(string)
        spread_domain           = optional(string)
        tenancy                 = optional(string)
        partition_number        = optional(number)
      }), null)
      license_specifications = optional(list(object({
        license_configuration_arn = string
      })), [])
      use_dedicated_host = optional(bool, false)
      runner_log_files = optional(list(object({
        log_group_name   = string
        prefix_log_group = bool
        file_path        = string
        log_stream_name  = string
        log_class        = optional(string, "STANDARD")
      })), null)
      block_device_mappings = optional(list(object({
        delete_on_termination      = optional(bool, true)
        device_name                = optional(string, "/dev/xvda")
        encrypted                  = optional(bool, true)
        iops                       = optional(number)
        kms_key_id                 = optional(string)
        snapshot_id                = optional(string)
        throughput                 = optional(number)
        volume_initialization_rate = optional(number)
        volume_size                = number
        volume_type                = optional(string, "gp3")
        })), [{
        volume_size = 30
      }])
      pool_config = optional(list(object({
        schedule_expression          = string
        schedule_expression_timezone = optional(string)
        size                         = number
      })), [])
      job_retry = optional(object({
        enable             = optional(bool, false)
        delay_in_seconds   = optional(number, 300)
        delay_backoff      = optional(number, 2)
        lambda_memory_size = optional(number, 256)
        lambda_timeout     = optional(number, 30)
        max_attempts       = optional(number, 1)
      }), {})
      iam_overrides = optional(object({
        override_instance_profile = optional(bool, null)
        instance_profile_name     = optional(string, null)
        override_runner_role      = optional(bool, null)
        runner_role_arn           = optional(string, null)
        }), {
        override_instance_profile = false
        instance_profile_name     = null
        override_runner_role      = false
        runner_role_arn           = null
      })
    })
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = optional(bool, false)
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy  = optional(any, null)
    })
    redrive_build_queue = optional(object({
      enabled         = bool
      maxReceiveCount = number
      }), {
      enabled         = false
      maxReceiveCount = null
    })
  }))
| n/a | yes | | [parameter\_store\_tags](#input\_parameter\_store\_tags) | Map of tags that will be added to all the SSM Parameter Store parameters created by the Lambda function. | `map(string)` | `{}` | no | | [pool\_lambda\_reserved\_concurrent\_executions](#input\_pool\_lambda\_reserved\_concurrent\_executions) | Amount of reserved concurrent executions for the scale-up lambda function. A value of 0 disables lambda from being triggered and -1 removes any concurrency limitations. | `number` | `1` | no | | [pool\_lambda\_timeout](#input\_pool\_lambda\_timeout) | Time out for the pool lambda in seconds. | `number` | `60` | no | diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf index 51a7420a72..df6fb77473 100644 --- a/modules/multi-runner/variables.tf +++ b/modules/multi-runner/variables.tf @@ -130,7 +130,6 @@ variable "multi_runner_config" { vpc_id = optional(string, null) subnet_ids = optional(list(string), null) idle_config = optional(list(object({ - type = optional(string, "ec2") cron = string timeZone = string idleCount = number diff --git a/modules/runners/README.md b/modules/runners/README.md index 8b1cad13a5..0a1c0c53cd 100644 --- a/modules/runners/README.md +++ b/modules/runners/README.md @@ -164,7 +164,7 @@ yarn run dist | [ghes\_url](#input\_ghes\_url) | GitHub Enterprise Server URL. DO NOT SET IF USING PUBLIC GITHUB..However if you are using GitHub Enterprise Cloud with data-residency (ghe.com), set the endpoint here. Example - https://companyname.ghe.com\| | `string` | `null` | no | | [github\_app\_parameters](#input\_github\_app\_parameters) | Parameter Store for GitHub App Parameters. | 
object({
    key_base64 = map(string)
    id         = map(string)
  })
| n/a | yes | | [iam\_overrides](#input\_iam\_overrides) | This map provides the possibility to override some IAM defaults. The following attributes are supported: `instance_profile_name` overrides the instance profile name used in the launch template. `runner_role_arn` overrides the IAM role ARN used for the runner instances. | 
object({
    override_instance_profile = optional(bool, null)
    instance_profile_name     = optional(string, null)
    override_runner_role      = optional(bool, null)
    runner_role_arn           = optional(string, null)
  })
| 
{
  "instance_profile_name": null,
  "override_instance_profile": false,
  "override_runner_role": false,
  "runner_role_arn": null
}
| no | -| [idle\_config](#input\_idle\_config) | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    type             = optional(string, "ec2")
    cron             = string
    timeZone         = string
    idleCount        = number
    evictionStrategy = optional(string, "oldest_first")
  }))
| `[]` | no | +| [idle\_config](#input\_idle\_config) | List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle. | 
list(object({
    cron             = string
    timeZone         = string
    idleCount        = number
    evictionStrategy = optional(string, "oldest_first")
  }))
| `[]` | no | | [instance\_allocation\_strategy](#input\_instance\_allocation\_strategy) | The allocation strategy for creating instances. For spot, AWS recommends `price-capacity-optimized`; for on-demand, use `lowest-price` or `prioritized`. The AWS default is `lowest-price`. | `string` | `"lowest-price"` | no | | [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the prefix will be used. | `string` | `null` | no | diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index c534081c95..08283ce65c 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -349,7 +349,6 @@ variable "runner_architecture" { variable "idle_config" { description = "List of time period that can be defined as cron expression to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle." type = list(object({ - type = optional(string, "ec2") cron = string timeZone = string idleCount = number diff --git a/variables.tf b/variables.tf index dd9325a5c6..4af2ab4cd1 100644 --- a/variables.tf +++ b/variables.tf @@ -371,7 +371,6 @@ variable "runner_hook_job_completed" { variable "idle_config" { description = "List of time periods, defined as a cron expression, to keep a minimum amount of runners active instead of scaling down to 0. By defining this list you can ensure that in time periods that match the cron expression within 5 seconds a runner is kept idle." type = list(object({ - type = optional(string, "ec2") cron = string timeZone = string idleCount = number From a960c3182494bd8d6b46e75b954cec951713b4fd Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 8 Jul 2026 00:45:07 +0200 Subject: [PATCH 12/46] docs(config): remove idle config provider type --- docs/configuration.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 62e9154bc5..ab68d41905 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -118,8 +118,6 @@ By default, the oldest instances are evicted. This helps keep your environment u ```hcl idle_config = [{ - # Defaults to 'ec2' - type = "ec2" cron = "* * 9-17 * * 1-5" timeZone = "Europe/Amsterdam" idleCount = 2 From 161c26b6299cfc9f7386158cb21dbe0487493801 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 8 Jul 2026 00:46:53 +0200 Subject: [PATCH 13/46] test(pool): remove redundant ec2 provider assertion --- lambdas/functions/control-plane/src/pool/pool.test.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index 795bec21ea..e5904cacc0 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -207,17 +207,6 @@ describe('Test simple pool.', () => { ); }); - it('uses the EC2 pool provider when the event type is ec2.', async () => { - await adjust({ poolSize: 3, type: 'ec2' }); - expect(createRunners).toHaveBeenCalledTimes(1); - expect(mockListRunners).toHaveBeenCalledWith({ - environment: 'unit-test-environment', - runnerOwner: ORG, - runnerType: 'Org', - statuses: ['running'], - }); - }); - it('Should not top up if pool size is reached.', async () => { await adjust({ poolSize: 1, type: 'ec2' }); expect(createRunners).not.toHaveBeenCalled(); From 0f6ed81e73128fd2372ae942a5679f3a9d4d7a2f Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 8 Jul 2026 00:51:21 +0200 Subject: [PATCH 14/46] fix(scale-up): default blank runner provider type --- .../control-plane/src/scale-runners/scale-up-config.ts | 3 ++- .../control-plane/src/scale-runners/scale-up.test.ts | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts index 58fe22b646..5ca1458aa2 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts @@ -4,5 +4,6 @@ export function getScaleUpRunnerProviderType( type: string | undefined, defaultType: ScaleUpRunnerProviderType, ): ScaleUpRunnerProviderType { - return type ?? defaultType; + const normalizedType = type?.trim(); + return normalizedType || defaultType; } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index fb921806e6..a13decdd50 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -3510,6 +3510,11 @@ describe('getScaleUpRunnerProviderType', () => { expect(getScaleUpRunnerProviderType(undefined, 'ec2')).toEqual('ec2'); }); + it('defaults to ec2 when the configured type is blank', () => { + expect(getScaleUpRunnerProviderType('', 'ec2')).toEqual('ec2'); + expect(getScaleUpRunnerProviderType(' ', 'ec2')).toEqual('ec2'); + }); + it('uses configured ec2 type', () => { expect(getScaleUpRunnerProviderType('ec2', 'microvm')).toEqual('ec2'); }); From e6d42f5083b10317a89a10bde455b30696b6412d Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 13:03:17 +0200 Subject: [PATCH 15/46] chore(pool): format available runner count --- lambdas/functions/control-plane/src/pool/pool.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 626fd8567a..0fb976434b 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -57,11 +57,7 @@ export async function adjust(event: PoolEvent): Promise { runnerType: 'Org', }); - const numberOfRunnersInPool = runnerProvider.countAvailableRunners( - poolRunners, - runnerStatusses, - includeBusyRunners, - ); + const numberOfRunnersInPool = runnerProvider.countAvailableRunners(poolRunners, runnerStatusses, includeBusyRunners); let topUp = event.poolSize - numberOfRunnersInPool; // The pool must never push the total number of runners (busy + idle) past the configured maximum. From 7c71a4d1d5c7c4a11dd17c3e954c913b3e30b279 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 11:07:45 +0000 Subject: [PATCH 16/46] docs: auto update terraform docs --- modules/runners/pool/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/runners/pool/README.md b/modules/runners/pool/README.md index b16f3ec353..24f9de5811 100644 --- a/modules/runners/pool/README.md +++ b/modules/runners/pool/README.md @@ -49,7 +49,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [aws\_partition](#input\_aws\_partition) | (optional) partition for the arn if not 'aws' | `string` | `"aws"` | no | -| [config](#input\_config) | Lookup details in parent module. | 
object({
    lambda = object({
      log_level                      = string
      logging_retention_in_days      = number
      logging_kms_key_id             = string
      log_class                      = string
      reserved_concurrent_executions = number
      s3_bucket                      = string
      s3_key                         = string
      s3_object_version              = string
      security_group_ids             = list(string)
      runtime                        = string
      architecture                   = string
      memory_size                    = number
      timeout                        = number
      zip                            = string
      subnet_ids                     = list(string)
      parameter_store_tags           = string
    })
    tags = map(string)
    ghes = object({
      url        = string
      ssl_verify = string
    })
    github_app_parameters = object({
      key_base64 = map(string)
      id         = map(string)
    })
    subnet_ids = list(string)
    runner = object({
      disable_runner_autoupdate            = bool
      ephemeral                            = bool
      enable_jit_config                    = bool
      enable_on_demand_failover_for_errors = list(string)
      scale_errors                         = list(string)
      boot_time_in_minutes                 = number
      labels                               = list(string)
      launch_template = object({
        name = string
      })
      group_name  = string
      name_prefix = string
      pool_owner  = string
      role = object({
        arn = string
      })
      use_dedicated_host = bool
    })
    runners_maximum_count         = number
    instance_types                = list(string)
    instance_type_priorities      = optional(map(number))
    instance_target_capacity_type = string
    instance_allocation_strategy  = string
    instance_max_spot_price       = string
    prefix                        = string
    pool = list(object({
      schedule_expression          = string
      schedule_expression_timezone = string
      size                         = number
    }))
    role_permissions_boundary            = string
    kms_key_arn                          = string
    ami_kms_key_arn                      = string
    ami_id_ssm_parameter_arn             = string
    role_path                            = string
    ssm_token_path                       = string
    ssm_config_path                      = string
    ami_id_ssm_parameter_name            = string
    ami_id_ssm_parameter_read_policy_arn = string
    arn_ssm_parameters_path_config       = string
    lambda_tags                          = map(string)
    user_agent                           = string
  })
| n/a | yes | +| [config](#input\_config) | Lookup details in parent module. | 
object({
    lambda = object({
      log_level                      = string
      logging_retention_in_days      = number
      logging_kms_key_id             = string
      log_class                      = string
      reserved_concurrent_executions = number
      s3_bucket                      = string
      s3_key                         = string
      s3_object_version              = string
      security_group_ids             = list(string)
      runtime                        = string
      architecture                   = string
      memory_size                    = number
      timeout                        = number
      zip                            = string
      subnet_ids                     = list(string)
      parameter_store_tags           = string
    })
    tags = map(string)
    ghes = object({
      url        = string
      ssl_verify = string
    })
    github_app_parameters = object({
      key_base64 = map(string)
      id         = map(string)
    })
    subnet_ids = list(string)
    runner = object({
      disable_runner_autoupdate            = bool
      ephemeral                            = bool
      enable_jit_config                    = bool
      enable_on_demand_failover_for_errors = list(string)
      scale_errors                         = list(string)
      boot_time_in_minutes                 = number
      labels                               = list(string)
      launch_template = object({
        name = string
      })
      group_name  = string
      name_prefix = string
      pool_owner  = string
      role = object({
        arn = string
      })
      use_dedicated_host = bool
    })
    runners_maximum_count         = number
    instance_types                = list(string)
    instance_type_priorities      = optional(map(number))
    instance_target_capacity_type = string
    instance_allocation_strategy  = string
    instance_max_spot_price       = string
    prefix                        = string
    pool = list(object({
      schedule_expression          = string
      schedule_expression_timezone = string
      size                         = number
    }))
    include_busy_runners                 = bool
    role_permissions_boundary            = string
    kms_key_arn                          = string
    ami_kms_key_arn                      = string
    ami_id_ssm_parameter_arn             = string
    role_path                            = string
    ssm_token_path                       = string
    ssm_config_path                      = string
    ami_id_ssm_parameter_name            = string
    ami_id_ssm_parameter_read_policy_arn = string
    arn_ssm_parameters_path_config       = string
    lambda_tags                          = map(string)
    user_agent                           = string
  })
| n/a | yes | | [tracing\_config](#input\_tracing\_config) | Configuration for lambda tracing. | 
object({
    mode                  = optional(string, null)
    capture_http_requests = optional(bool, false)
    capture_error         = optional(bool, false)
  })
| `{}` | no | ## Outputs From 742aa8fd5dda8a02573fdd10c771c91fc8114381 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 21:17:58 +0200 Subject: [PATCH 17/46] fix(runners): preserve provider compatibility --- .../src/pool/pool-provider-registry.ts | 10 +++- .../control-plane/src/pool/pool-provider.ts | 11 ++--- .../control-plane/src/pool/pool.test.ts | 28 +++++++++++ .../functions/control-plane/src/pool/pool.ts | 8 +-- .../control-plane/src/runner-provider.test.ts | 40 +++++++++++++++ .../control-plane/src/runner-provider.ts | 14 ++++++ .../src/scale-runners/ec2-scale-down.ts | 3 +- .../src/scale-runners/ec2-scale-up.ts | 20 ++++---- .../src/scale-runners/github-runner.ts | 14 +++--- .../scale-runners/scale-down-config.test.ts | 9 ++++ .../src/scale-runners/scale-down-config.ts | 13 +++-- .../scale-down-provider-registry.ts | 10 ++-- .../src/scale-runners/scale-down-provider.ts | 11 ++--- .../src/scale-runners/scale-down.ts | 7 ++- .../src/scale-runners/scale-up-config.ts | 8 +-- .../scale-up-provider-registry.ts | 8 +-- .../src/scale-runners/scale-up-provider.ts | 28 +++++------ .../src/scale-runners/scale-up.ts | 2 +- .../webhook/src/runners/aws-dynamic-labels.ts | 20 ++++++-- .../webhook/src/runners/dispatch.test.ts | 49 +++++++++++++++++++ .../webhook/src/runners/ec2-dynamic-labels.ts | 19 ++++++- lambdas/functions/webhook/src/sqs/index.ts | 5 +- modules/webhook/README.md | 2 +- modules/webhook/variables.tf | 14 ++++-- modules/webhook/webhook.tf | 7 ++- 25 files changed, 278 insertions(+), 82 deletions(-) create mode 100644 lambdas/functions/control-plane/src/runner-provider.test.ts create mode 100644 lambdas/functions/control-plane/src/runner-provider.ts diff --git a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts index d745c670ac..ff0e2b9e1e 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts @@ -1,10 +1,16 @@ +import { normalizeRunnerProviderType } from '../runner-provider'; import { ec2PoolRunnerProviderStrategy } from './ec2-pool'; import type { PoolRunnerProvider, PoolRunnerProviderStrategy, PoolRunnerProviderType } from './pool-provider'; const poolRunnerProviderStrategies: PoolRunnerProviderStrategy[] = [ec2PoolRunnerProviderStrategy]; -export function createPoolRunnerProviderFromEnv(type: PoolRunnerProviderType): PoolRunnerProvider { - const strategy = poolRunnerProviderStrategies.find((strategy) => strategy.type === type); +export function getDefaultPoolRunnerProviderType(): PoolRunnerProviderType { + return poolRunnerProviderStrategies[0].type; +} + +export function createPoolRunnerProviderFromEnv(type: string): PoolRunnerProvider { + const normalizedType = normalizeRunnerProviderType(type); + const strategy = poolRunnerProviderStrategies.find((strategy) => strategy.type === normalizedType); if (!strategy) { throw new Error(`Unsupported pool runner provider type '${type}'`); diff --git a/lambdas/functions/control-plane/src/pool/pool-provider.ts b/lambdas/functions/control-plane/src/pool/pool-provider.ts index 83ec25a93c..7af55628ac 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider.ts @@ -1,8 +1,9 @@ import type { Octokit } from '@octokit/rest'; +import type { RunnerProvider, RunnerProviderStrategy, RunnerProviderType } from '../runner-provider'; import type { CreateGitHubRunnerConfig, GitHubRunnerType } from '../scale-runners/types'; -export type PoolRunnerProviderType = string; +export type PoolRunnerProviderType = RunnerProviderType; export interface RunnerStatus { busy: boolean; @@ -21,8 +22,7 @@ export interface CreatePoolRunnersInput { githubInstallationClient: Octokit; } -export interface PoolRunnerProvider { - type: PoolRunnerProviderType; +export interface PoolRunnerProvider extends RunnerProvider { listRunners(input: ListPoolRunnersInput): Promise; countAvailableRunners( runners: TRunner[], @@ -32,7 +32,4 @@ export interface PoolRunnerProvider { createRunners(input: CreatePoolRunnersInput): Promise; } -export interface PoolRunnerProviderStrategy { - type: PoolRunnerProviderType; - createFromEnv(): PoolRunnerProvider; -} +export type PoolRunnerProviderStrategy = RunnerProviderStrategy; diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index e5904cacc0..49886e91cc 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -207,6 +207,34 @@ describe('Test simple pool.', () => { ); }); + it('Defaults legacy pool events without a provider type to EC2.', async () => { + await adjust({ poolSize: 10 }); + expect(mockListRunners).toHaveBeenCalledWith({ + environment: 'unit-test-environment', + runnerOwner: ORG, + runnerType: 'Org', + statuses: ['running'], + }); + expect(createRunners).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + 8, + expect.anything(), + 'pool-lambda', + ); + }); + + it('Selects the EC2 pool provider case-insensitively.', async () => { + await adjust({ poolSize: 10, type: ' EC2 ' }); + expect(createRunners).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + 8, + expect.anything(), + 'pool-lambda', + ); + }); + it('Should not top up if pool size is reached.', async () => { await adjust({ poolSize: 1, type: 'ec2' }); expect(createRunners).not.toHaveBeenCalled(); diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 0fb976434b..145ae7a431 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -4,18 +4,18 @@ import yn from 'yn'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; import { getGitHubEnterpriseApiUrl, validateSsmParameterStoreTags } from '../scale-runners/github-runner'; -import { createPoolRunnerProviderFromEnv } from './pool-provider-registry'; -import type { PoolRunnerProviderType, RunnerStatus } from './pool-provider'; +import { createPoolRunnerProviderFromEnv, getDefaultPoolRunnerProviderType } from './pool-provider-registry'; +import type { RunnerStatus } from './pool-provider'; const logger = createChildLogger('pool'); export interface PoolEvent { poolSize: number; - type: PoolRunnerProviderType; + type?: string; } export async function adjust(event: PoolEvent): Promise { - const runnerProviderType = event.type; + const runnerProviderType = event.type?.trim() ? event.type : getDefaultPoolRunnerProviderType(); logger.info(`Checking current ${runnerProviderType} pool size against pool of size: ${event.poolSize}`); const runnerLabels = process.env.RUNNER_LABELS || ''; const runnerGroup = process.env.RUNNER_GROUP_NAME || ''; diff --git a/lambdas/functions/control-plane/src/runner-provider.test.ts b/lambdas/functions/control-plane/src/runner-provider.test.ts new file mode 100644 index 0000000000..16962bf7d7 --- /dev/null +++ b/lambdas/functions/control-plane/src/runner-provider.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest'; + +import { createPoolRunnerProviderFromEnv } from './pool/pool-provider-registry'; +import { normalizeRunnerProviderType } from './runner-provider'; +import { createScaleDownRunnerProviderFromEnv } from './scale-runners/scale-down-provider-registry'; +import { createScaleUpRunnerProviderFromEnv } from './scale-runners/scale-up-provider-registry'; + +describe('runner provider selection', () => { + it.each([ + [' EC2 ', 'ec2'], + ['MicroVM', 'microvm'], + ])('normalizes provider type %j to %j', (type, expected) => { + expect(normalizeRunnerProviderType(type)).toBe(expected); + }); + + it('selects the EC2 scale-down provider case-insensitively', () => { + expect(createScaleDownRunnerProviderFromEnv(' EC2 ')).toMatchObject({ + type: 'ec2', + name: 'EC2', + }); + }); + + it('quotes the original unsupported pool provider type', () => { + expect(() => createPoolRunnerProviderFromEnv(' Unknown ')).toThrow( + "Unsupported pool runner provider type ' Unknown '", + ); + }); + + it('quotes the original unsupported scale-up provider type', () => { + expect(() => createScaleUpRunnerProviderFromEnv(' Unknown ', 'test', [])).toThrow( + "Unsupported scale-up runner provider type ' Unknown '", + ); + }); + + it('quotes the original unsupported scale-down provider type', () => { + expect(() => createScaleDownRunnerProviderFromEnv(' Unknown ')).toThrow( + "Unsupported scale-down runner provider type ' Unknown '", + ); + }); +}); diff --git a/lambdas/functions/control-plane/src/runner-provider.ts b/lambdas/functions/control-plane/src/runner-provider.ts new file mode 100644 index 0000000000..af4876bdf9 --- /dev/null +++ b/lambdas/functions/control-plane/src/runner-provider.ts @@ -0,0 +1,14 @@ +export type RunnerProviderType = 'ec2' | 'microvm'; + +export function normalizeRunnerProviderType(type: string): string { + return type.trim().toLowerCase(); +} + +export interface RunnerProvider { + type: RunnerProviderType; +} + +export interface RunnerProviderStrategy { + type: TProvider['type']; + createFromEnv(...args: TCreateFromEnvArgs): TProvider; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts index 2747d7b149..be859eacaf 100644 --- a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts @@ -8,6 +8,7 @@ import type { export function createEc2ScaleDownProvider(): ScaleDownRunnerProvider { return { + type: 'ec2', name: 'EC2', list: async (environment, orphan) => (await listEC2Runners({ environment, orphan })).map(toScaleDownRunner), bootTimeExceeded, @@ -19,7 +20,7 @@ export function createEc2ScaleDownProvider(): ScaleDownRunnerProvider { export const ec2ScaleDownRunnerProviderStrategy: ScaleDownRunnerProviderStrategy = { type: 'ec2', - create: createEc2ScaleDownProvider, + createFromEnv: createEc2ScaleDownProvider, }; function toScaleDownRunner(runner: RunnerList): ScaleDownRunnerList { diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts index e6a62e958e..678d6d40d0 100644 --- a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts @@ -20,7 +20,7 @@ interface Ec2ScaleUpState { ec2OverrideConfig?: Ec2OverrideConfig; } -export function createEc2ScaleUpProvider(config: Ec2ScaleUpProviderConfig): ScaleUpRunnerProvider { +export function createEc2ScaleUpProvider(config: Ec2ScaleUpProviderConfig): ScaleUpRunnerProvider { return { type: 'ec2', prepareGroup: async (messageLabels) => { @@ -47,24 +47,24 @@ export function createEc2ScaleUpProvider(config: Ec2ScaleUpProviderConfig): Scal }, getCurrentRunners: async (_state, { runnerType, runnerOwner }) => (await listEC2Runners({ environment: config.environment, runnerType, runnerOwner })).length, - createRunners: async ({ githubRunnerConfig, numberOfRunners, githubInstallationClient, state }) => { - const ec2State = state as Ec2ScaleUpState; - - return await createRunners( + createRunners: async ({ githubRunnerConfig, numberOfRunners, githubInstallationClient, state }) => + await createRunners( githubRunnerConfig, { ...config, - ec2OverrideConfig: ec2State.ec2OverrideConfig, + ec2OverrideConfig: state.ec2OverrideConfig, }, numberOfRunners, githubInstallationClient, 'scale-up-lambda', - ); - }, + ), }; } -export function createEc2ScaleUpProviderFromEnv(environment: string, scaleErrors: string[]): ScaleUpRunnerProvider { +export function createEc2ScaleUpProviderFromEnv( + environment: string, + scaleErrors: string[], +): ScaleUpRunnerProvider { return createEc2ScaleUpProvider({ ec2instanceCriteria: { instanceTypes: process.env.INSTANCE_TYPES.split(','), @@ -88,7 +88,7 @@ export function createEc2ScaleUpProviderFromEnv(environment: string, scaleErrors }); } -export const ec2ScaleUpRunnerProviderStrategy: ScaleUpRunnerProviderStrategy = { +export const ec2ScaleUpRunnerProviderStrategy: ScaleUpRunnerProviderStrategy = { type: 'ec2', createFromEnv: ({ environment, scaleErrors }) => createEc2ScaleUpProviderFromEnv(environment, scaleErrors), }; diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts index 773da3a269..0b9ea2af51 100644 --- a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts @@ -323,7 +323,7 @@ async function createJitConfig( // store jit config in ssm parameter store logger.debug('Runner JIT config for ephemeral runner generated.', { - runnerId, + instance: runnerId, }); await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerConfig.data.encoded_jit_config, true, { tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags], @@ -334,18 +334,18 @@ async function createJitConfig( } } catch (error) { failedRunnerIds.push(runnerId); - logger.warn('Failed to create JIT config for runner, continuing with remaining runners', { - runnerId, + logger.warn('Failed to create JIT config for instance, continuing with remaining instances', { + instance: runnerId, error: error instanceof Error ? error.message : String(error), }); } } if (failedRunnerIds.length > 0) { - logger.error('Failed to create JIT config for some runners', { - failedRunnerIds, - totalRunners: runnerIds.length, - successfulRunners: runnerIds.length - failedRunnerIds.length, + logger.error('Failed to create JIT config for some instances', { + failedInstances: failedRunnerIds, + totalInstances: runnerIds.length, + successfulInstances: runnerIds.length - failedRunnerIds.length, }); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts index aa71d3173a..8923ee6e94 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts @@ -75,5 +75,14 @@ describe('scaleDownConfig', () => { const scaleDownConfig = getConfig(['* * * * * *']).map((config) => ({ ...config, type: 'ec2' as const })); expect(getScaleDownRunnerProviderType(scaleDownConfig, 'microvm')).toEqual('ec2'); }); + + it('Treats provider types case-insensitively when checking for multiple types', async () => { + const scaleDownConfig = getConfig(['* * * * * *', '* * * * * *']).map((config, index) => ({ + ...config, + type: index === 0 ? ' EC2 ' : 'ec2', + })); + + expect(() => getScaleDownRunnerProviderType(scaleDownConfig, 'microvm')).not.toThrow(); + }); }); }); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.ts index c3c51a6432..362b190f3d 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.ts @@ -2,12 +2,13 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import parser from 'cron-parser'; import moment from 'moment'; +import { normalizeRunnerProviderType } from '../runner-provider'; import type { ScaleDownRunnerProviderType } from './scale-down-provider'; export type ScalingDownConfigList = ScalingDownConfig[]; export type EvictionStrategy = 'newest_first' | 'oldest_first'; export interface ScalingDownConfig { - type?: ScaleDownRunnerProviderType; + type?: string; cron: string; idleCount: number; timeZone: string; @@ -48,10 +49,12 @@ export function getEvictionStrategy(scalingDownConfigs: ScalingDownConfigList): export function getScaleDownRunnerProviderType( scalingDownConfigs: ScalingDownConfigList, defaultType: ScaleDownRunnerProviderType, -): ScaleDownRunnerProviderType { - const configuredTypes = new Set( - scalingDownConfigs.map((scalingDownConfig) => scalingDownConfig.type ?? defaultType), - ); +): string { + const configuredTypes = new Map(); + for (const scalingDownConfig of scalingDownConfigs) { + const configuredType = scalingDownConfig.type?.trim() ? scalingDownConfig.type : defaultType; + configuredTypes.set(normalizeRunnerProviderType(configuredType), configuredType); + } if (configuredTypes.size > 1) { throw new Error(`Multiple scale-down runner provider types are not supported in a single scale-down config.`); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts index ed6501ff70..8479c5ca42 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts @@ -1,3 +1,4 @@ +import { normalizeRunnerProviderType } from '../runner-provider'; import { ec2ScaleDownRunnerProviderStrategy } from './ec2-scale-down'; import type { ScaleDownRunnerProvider, @@ -11,12 +12,13 @@ export function getDefaultScaleDownRunnerProviderType(): ScaleDownRunnerProvider return scaleDownRunnerProviderStrategies[0].type; } -export function createScaleDownRunnerProvider(type: ScaleDownRunnerProviderType): ScaleDownRunnerProvider { - const strategy = scaleDownRunnerProviderStrategies.find((strategy) => strategy.type === type); +export function createScaleDownRunnerProviderFromEnv(type: string): ScaleDownRunnerProvider { + const normalizedType = normalizeRunnerProviderType(type); + const strategy = scaleDownRunnerProviderStrategies.find((strategy) => strategy.type === normalizedType); if (!strategy) { - throw new Error(`Unsupported scale-down runner provider type: ${type}`); + throw new Error(`Unsupported scale-down runner provider type '${type}'`); } - return strategy.create(); + return strategy.createFromEnv(); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts index 712b281635..e38782d021 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts @@ -1,3 +1,5 @@ +import type { RunnerProvider, RunnerProviderStrategy, RunnerProviderType } from '../runner-provider'; + export interface RunnerList { id: string; launchTime?: Date; @@ -15,9 +17,9 @@ export interface RunnerInfo extends RunnerList { type: string; } -export type ScaleDownRunnerProviderType = string; +export type ScaleDownRunnerProviderType = RunnerProviderType; -export interface ScaleDownRunnerProvider { +export interface ScaleDownRunnerProvider extends RunnerProvider { name: string; list(environment: string, orphan?: boolean): Promise; bootTimeExceeded(runner: RunnerInfo): boolean; @@ -26,7 +28,4 @@ export interface ScaleDownRunnerProvider { terminate(id: string): Promise; } -export interface ScaleDownRunnerProviderStrategy { - type: ScaleDownRunnerProviderType; - create(): ScaleDownRunnerProvider; -} +export type ScaleDownRunnerProviderStrategy = RunnerProviderStrategy; diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index a759289058..3cfadc93ea 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -12,7 +12,10 @@ import { getIdleRunnerCount, getScaleDownRunnerProviderType, } from './scale-down-config'; -import { createScaleDownRunnerProvider, getDefaultScaleDownRunnerProviderType } from './scale-down-provider-registry'; +import { + createScaleDownRunnerProviderFromEnv, + getDefaultScaleDownRunnerProviderType, +} from './scale-down-provider-registry'; import { metricGitHubAppRateLimit } from '../github/rate-limit'; import { getGitHubEnterpriseApiUrl } from './github-runner'; import type { RunnerInfo, RunnerList, ScaleDownRunnerProvider } from './scale-down-provider'; @@ -354,7 +357,7 @@ export async function scaleDown(): Promise { const environment = process.env.ENVIRONMENT; const scaleDownConfigs = JSON.parse(process.env.SCALE_DOWN_CONFIG) as [ScalingDownConfig]; const runnerProviderType = getScaleDownRunnerProviderType(scaleDownConfigs, getDefaultScaleDownRunnerProviderType()); - const runnerProvider = createScaleDownRunnerProvider(runnerProviderType); + const runnerProvider = createScaleDownRunnerProviderFromEnv(runnerProviderType); // first runners marked to be orphan. await terminateOrphan(environment, runnerProvider); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts index 5ca1458aa2..ab3dccb441 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts @@ -1,9 +1,5 @@ import type { ScaleUpRunnerProviderType } from './scale-up-provider'; -export function getScaleUpRunnerProviderType( - type: string | undefined, - defaultType: ScaleUpRunnerProviderType, -): ScaleUpRunnerProviderType { - const normalizedType = type?.trim(); - return normalizedType || defaultType; +export function getScaleUpRunnerProviderType(type: string | undefined, defaultType: ScaleUpRunnerProviderType): string { + return type?.trim() ? type : defaultType; } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts index 1931e0f383..ae3febd1eb 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts @@ -1,3 +1,4 @@ +import { normalizeRunnerProviderType } from '../runner-provider'; import { ec2ScaleUpRunnerProviderStrategy } from './ec2-scale-up'; import type { ScaleUpRunnerProvider, @@ -12,14 +13,15 @@ export function getDefaultScaleUpRunnerProviderType(): ScaleUpRunnerProviderType } export function createScaleUpRunnerProviderFromEnv( - type: ScaleUpRunnerProviderType, + type: string, environment: string, scaleErrors: string[], ): ScaleUpRunnerProvider { - const strategy = scaleUpRunnerProviderStrategies.find((strategy) => strategy.type === type); + const normalizedType = normalizeRunnerProviderType(type); + const strategy = scaleUpRunnerProviderStrategies.find((strategy) => strategy.type === normalizedType); if (!strategy) { - throw new Error(`Unsupported scale-up runner provider type: ${type}`); + throw new Error(`Unsupported scale-up runner provider type '${type}'`); } return strategy.createFromEnv({ environment, scaleErrors }); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts index 4442f0e6cc..24805bd530 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts @@ -1,5 +1,6 @@ import type { Octokit } from '@octokit/rest'; +import type { RunnerProvider, RunnerProviderStrategy, RunnerProviderType } from '../runner-provider'; import type { ActionRequestMessageSQS, CreateGitHubRunnerConfig, GitHubRunnerType } from './types'; export interface CurrentRunnersInput { @@ -7,26 +8,25 @@ export interface CurrentRunnersInput { runnerOwner: string; } -export interface CreateScaleUpRunnersInput { +export interface CreateScaleUpRunnersInput { githubRunnerConfig: CreateGitHubRunnerConfig; numberOfRunners: number; githubInstallationClient: Octokit; messages: ActionRequestMessageSQS[]; - state: unknown; + state: TState; } -export interface PreparedScaleUpRunnerGroup { +export interface PreparedScaleUpRunnerGroup { runnerLabels: string[]; - state: unknown; + state: TState; } -export type ScaleUpRunnerProviderType = string; +export type ScaleUpRunnerProviderType = RunnerProviderType; -export interface ScaleUpRunnerProvider { - type: ScaleUpRunnerProviderType; - prepareGroup(messageLabels: string[]): Promise; - getCurrentRunners(state: unknown, input: CurrentRunnersInput): Promise; - createRunners(input: CreateScaleUpRunnersInput): Promise; +export interface ScaleUpRunnerProvider extends RunnerProvider { + prepareGroup(messageLabels: string[]): Promise>; + getCurrentRunners(state: TState, input: CurrentRunnersInput): Promise; + createRunners(input: CreateScaleUpRunnersInput): Promise; } export interface CreateScaleUpRunnerProviderInput { @@ -34,7 +34,7 @@ export interface CreateScaleUpRunnerProviderInput { scaleErrors: string[]; } -export interface ScaleUpRunnerProviderStrategy { - type: ScaleUpRunnerProviderType; - createFromEnv(input: CreateScaleUpRunnerProviderInput): ScaleUpRunnerProvider; -} +export type ScaleUpRunnerProviderStrategy = RunnerProviderStrategy< + ScaleUpRunnerProvider, + [input: CreateScaleUpRunnerProviderInput] +>; diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 81831390ab..2bf5f8e81c 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -313,7 +313,7 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise strategy.type === provider); + const provider = normalizeRunnerProvider(queue.runnerProvider); + const strategy = provider + ? awsDynamicLabelProviderStrategies.find((strategy) => strategy.type === provider) + : undefined; if (!strategy) { - logger.warn(`Queue ${queue.id} has unsupported runner provider '${provider}'`); + logger.warn(`Queue ${queue.id} has unsupported runner provider '${provider ?? String(queue.runnerProvider)}'`); continue; } diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index 48d92be9af..5140032547 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -590,6 +590,55 @@ describe('selectAwsDynamicLabelQueue', () => { labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], }); }); + + it('normalizes runner provider casing and surrounding whitespace', () => { + const queue = runnerQueue('normalized-ec2'); + (queue as unknown as { runnerProvider: string }).runnerProvider = ' EC2 '; + + expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ + queue, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + }); + }); + + it('does not select a provider strategy that is planned but not implemented', () => { + const queue = runnerQueue('microvm', 'microvm'); + + expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-microvm-cpu-count:2'])).toBeUndefined(); + }); + + it('rejects a malformed non-string runner provider without throwing', () => { + const queue = runnerQueue('malformed-provider'); + (queue as unknown as { runnerProvider: number }).runnerProvider = 42; + + expect( + selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), + ).toBeUndefined(); + }); + + it('enforces a legacy EC2 dynamic labels policy when the new key is absent', () => { + const queue = runnerQueue('legacy-ec2-policy'); + queue.matcherConfig.ec2DynamicLabelsPolicy = { + blocked_keys: ['instance-type'], + }; + + expect( + selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), + ).toBeUndefined(); + }); + + it('prefers the new AWS dynamic labels policy when both keys are present', () => { + const queue = runnerQueue('new-policy-precedence'); + queue.matcherConfig.ec2DynamicLabelsPolicy = { + blocked_keys: ['instance-type'], + }; + queue.matcherConfig.awsDynamicLabelsPolicy = null; + + expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ + queue, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + }); + }); }); function runnerQueue(id: string, runnerProvider?: RunnerProvider): RunnerMatcherConfig { diff --git a/lambdas/functions/webhook/src/runners/ec2-dynamic-labels.ts b/lambdas/functions/webhook/src/runners/ec2-dynamic-labels.ts index fa27d4c559..44025ca8ad 100644 --- a/lambdas/functions/webhook/src/runners/ec2-dynamic-labels.ts +++ b/lambdas/functions/webhook/src/runners/ec2-dynamic-labels.ts @@ -8,6 +8,23 @@ const logger = createChildLogger('handler'); export type Ec2DynamicLabelDispatchTarget = AwsDynamicLabelDispatchTarget; +function resolveEc2DynamicLabelsPolicy(queue: RunnerMatcherConfig) { + const hasAwsDynamicLabelsPolicy = Object.prototype.hasOwnProperty.call(queue.matcherConfig, 'awsDynamicLabelsPolicy'); + const hasLegacyEc2DynamicLabelsPolicy = Object.prototype.hasOwnProperty.call( + queue.matcherConfig, + 'ec2DynamicLabelsPolicy', + ); + + if (!hasAwsDynamicLabelsPolicy && hasLegacyEc2DynamicLabelsPolicy) { + logger.warn( + `Queue ${queue.id}: using deprecated matcherConfig.ec2DynamicLabelsPolicy; migrate to matcherConfig.awsDynamicLabelsPolicy`, + ); + return queue.matcherConfig.ec2DynamicLabelsPolicy; + } + + return queue.matcherConfig.awsDynamicLabelsPolicy; +} + export function selectEc2DynamicLabelQueue( matches: RunnerMatcherConfig[], nonGhrLabels: string[], @@ -19,7 +36,7 @@ export function selectEc2DynamicLabelQueue( continue; } - const violations = violationsAgainstPolicy(sanitizedGhrLabels, queue.matcherConfig.awsDynamicLabelsPolicy); + const violations = violationsAgainstPolicy(sanitizedGhrLabels, resolveEc2DynamicLabelsPolicy(queue)); if (violations.length === 0) { return { queue, diff --git a/lambdas/functions/webhook/src/sqs/index.ts b/lambdas/functions/webhook/src/sqs/index.ts index 9d14fb7474..e4a6fcad9a 100644 --- a/lambdas/functions/webhook/src/sqs/index.ts +++ b/lambdas/functions/webhook/src/sqs/index.ts @@ -8,7 +8,7 @@ const logger = createChildLogger('sqs'); const sqsClientsByRegion = new Map(); -export type RunnerProvider = string; +export type RunnerProvider = 'ec2' | 'microvm'; export interface ActionRequestMessage { id: number; @@ -27,6 +27,9 @@ export interface MatcherConfig { bidirectionalLabelMatch?: boolean; enableDynamicLabels?: boolean; awsDynamicLabelsPolicy?: AwsDynamicLabelsPolicy | null; + // TODO: Remove this legacy compatibility field and fallback in the next release. + /** @deprecated Use awsDynamicLabelsPolicy. Retained while existing SSM configurations migrate. */ + ec2DynamicLabelsPolicy?: AwsDynamicLabelsPolicy | null; } export type RunnerConfig = RunnerMatcherConfig[]; diff --git a/modules/webhook/README.md b/modules/webhook/README.md index 1c68c2007f..e63a8c01b7 100644 --- a/modules/webhook/README.md +++ b/modules/webhook/README.md @@ -89,7 +89,7 @@ yarn run dist | [repository\_white\_list](#input\_repository\_white\_list) | List of github repository full names (owner/repo\_name) that will be allowed to use the github app. Leave empty for no filtering. | `list(string)` | `[]` | no | | [role\_path](#input\_role\_path) | The path that will be added to the role; if not set, the environment name will be used. | `string` | `null` | no | | [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no | -| [runner\_matcher\_config](#input\_runner\_matcher\_config) | SQS queue to publish accepted build events based on the runner type. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type`. | 
map(object({
    arn = string
    id  = string
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = bool
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy  = optional(any, null)
    })
  }))
| n/a | yes | +| [runner\_matcher\_config](#input\_runner\_matcher\_config) | SQS queue to publish accepted build events based on the runner type. `runnerProvider` defaults to `ec2`; EC2 is the only provider currently implemented. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type`. | 
map(object({
    arn            = string
    id             = string
    runnerProvider = optional(string, "ec2")
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = bool
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy  = optional(any, null)
    })
  }))
| n/a | yes | | [ssm\_paths](#input\_ssm\_paths) | The root path used in SSM to store configuration and secrets. | 
object({
    root    = string
    webhook = string
  })
| n/a | yes | | [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | | [tracing\_config](#input\_tracing\_config) | Configuration for lambda tracing. | 
object({
    mode                  = optional(string, null)
    capture_http_requests = optional(bool, false)
    capture_error         = optional(bool, false)
  })
| `{}` | no | diff --git a/modules/webhook/variables.tf b/modules/webhook/variables.tf index 427b713c69..fcdb7a7dcf 100644 --- a/modules/webhook/variables.tf +++ b/modules/webhook/variables.tf @@ -23,10 +23,11 @@ variable "tags" { } variable "runner_matcher_config" { - description = "SQS queue to publish accepted build events based on the runner type. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type`." + description = "SQS queue to publish accepted build events based on the runner type. `runnerProvider` defaults to `ec2`; EC2 is the only provider currently implemented. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type`." type = map(object({ - arn = string - id = string + arn = string + id = string + runnerProvider = optional(string, "ec2") matcherConfig = object({ labelMatchers = list(list(string)) exactMatch = bool @@ -40,6 +41,13 @@ variable "runner_matcher_config" { condition = try(var.runner_matcher_config.matcherConfig.priority, 999) >= 0 && try(var.runner_matcher_config.matcherConfig.priority, 999) < 1000 error_message = "The priority of the matcher must be between 0 and 999." } + validation { + condition = alltrue([ + for config in values(var.runner_matcher_config) : + lower(trimspace(config.runnerProvider)) == "ec2" + ]) + error_message = "runnerProvider must be ec2." + } } variable "lambda_zip" { diff --git a/modules/webhook/webhook.tf b/modules/webhook/webhook.tf index de48192ea3..84a89bbc93 100644 --- a/modules/webhook/webhook.tf +++ b/modules/webhook/webhook.tf @@ -1,6 +1,11 @@ locals { # config with combined key and order - runner_matcher_config = { for k, v in var.runner_matcher_config : format("%03d-%s", v.matcherConfig.priority, k) => merge(v, { key = k }) } + runner_matcher_config = { + for k, v in var.runner_matcher_config : format("%03d-%s", v.matcherConfig.priority, k) => merge(v, { + key = k + runnerProvider = lower(trimspace(v.runnerProvider)) + }) + } # sorted list runner_matcher_config_sorted = [for k in sort(keys(local.runner_matcher_config)) : local.runner_matcher_config[k]] From f77678853740af39fded4f1a3523a79533c7bfde Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 21:39:51 +0200 Subject: [PATCH 18/46] docs(runners): clarify future microvm support --- lambdas/functions/webhook/src/sqs/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lambdas/functions/webhook/src/sqs/index.ts b/lambdas/functions/webhook/src/sqs/index.ts index e4a6fcad9a..b5a23a6dd0 100644 --- a/lambdas/functions/webhook/src/sqs/index.ts +++ b/lambdas/functions/webhook/src/sqs/index.ts @@ -8,6 +8,7 @@ const logger = createChildLogger('sqs'); const sqsClientsByRegion = new Map(); +// EC2 is the only implemented provider; MicroVM is reserved for future support. export type RunnerProvider = 'ec2' | 'microvm'; export interface ActionRequestMessage { From e593a3ff44944c56d8ff007a6991c05e3ebc368c Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 21:43:03 +0200 Subject: [PATCH 19/46] test(runners): limit provider normalization to ec2 --- lambdas/functions/control-plane/src/runner-provider.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lambdas/functions/control-plane/src/runner-provider.test.ts b/lambdas/functions/control-plane/src/runner-provider.test.ts index 16962bf7d7..9075613429 100644 --- a/lambdas/functions/control-plane/src/runner-provider.test.ts +++ b/lambdas/functions/control-plane/src/runner-provider.test.ts @@ -6,10 +6,7 @@ import { createScaleDownRunnerProviderFromEnv } from './scale-runners/scale-down import { createScaleUpRunnerProviderFromEnv } from './scale-runners/scale-up-provider-registry'; describe('runner provider selection', () => { - it.each([ - [' EC2 ', 'ec2'], - ['MicroVM', 'microvm'], - ])('normalizes provider type %j to %j', (type, expected) => { + it.each([[' EC2 ', 'ec2']])('normalizes provider type %j to %j', (type, expected) => { expect(normalizeRunnerProviderType(type)).toBe(expected); }); From 3805873042e0dfe4b828dd95f7080702f7709de3 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 21:47:27 +0200 Subject: [PATCH 20/46] fix(runners): align scale-down provider contract --- .../control-plane/src/runner-provider.test.ts | 1 - .../src/scale-runners/ec2-scale-down.ts | 1 - .../src/scale-runners/scale-down-provider.ts | 1 - .../control-plane/src/scale-runners/scale-down.ts | 14 ++++++++++---- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lambdas/functions/control-plane/src/runner-provider.test.ts b/lambdas/functions/control-plane/src/runner-provider.test.ts index 9075613429..28756ed8b9 100644 --- a/lambdas/functions/control-plane/src/runner-provider.test.ts +++ b/lambdas/functions/control-plane/src/runner-provider.test.ts @@ -13,7 +13,6 @@ describe('runner provider selection', () => { it('selects the EC2 scale-down provider case-insensitively', () => { expect(createScaleDownRunnerProviderFromEnv(' EC2 ')).toMatchObject({ type: 'ec2', - name: 'EC2', }); }); diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts index be859eacaf..eec0e0f2f0 100644 --- a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts @@ -9,7 +9,6 @@ import type { export function createEc2ScaleDownProvider(): ScaleDownRunnerProvider { return { type: 'ec2', - name: 'EC2', list: async (environment, orphan) => (await listEC2Runners({ environment, orphan })).map(toScaleDownRunner), bootTimeExceeded, markOrphan: async (id) => await tag(id, [{ Key: 'ghr:orphan', Value: 'true' }]), diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts index e38782d021..8a4d0b0b88 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts @@ -20,7 +20,6 @@ export interface RunnerInfo extends RunnerList { export type ScaleDownRunnerProviderType = RunnerProviderType; export interface ScaleDownRunnerProvider extends RunnerProvider { - name: string; list(environment: string, orphan?: boolean): Promise; bootTimeExceeded(runner: RunnerInfo): boolean; markOrphan(id: string): Promise; diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index 3cfadc93ea..30052077e5 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -198,7 +198,9 @@ async function removeRunner( if (allSucceeded) { await runnerProvider.terminate(runner.id); - logger.info(`${runnerProvider.name} runner '${runner.id}' is terminated and GitHub runner is de-registered.`); + logger.info( + `${runnerProvider.type.toUpperCase()} runner '${runner.id}' is terminated and GitHub runner is de-registered.`, + ); } else { // Only terminate the provider runner if it was successfully de-registered from GitHub. logger.error( @@ -365,8 +367,10 @@ export async function scaleDown(): Promise { // next scale down idle runners with respect to config and mark potential orphans const providerRunners = await listRunners(environment, runnerProvider); const activeProviderRunnersCount = providerRunners.length; - logger.info(`Found: '${activeProviderRunnersCount}' active ${runnerProvider.name} runners before clean-up.`); - logger.debug(`Active ${runnerProvider.name} runners: ${JSON.stringify(providerRunners)}`); + logger.info( + `Found: '${activeProviderRunnersCount}' active ${runnerProvider.type.toUpperCase()} runners before clean-up.`, + ); + logger.debug(`Active ${runnerProvider.type.toUpperCase()} runners: ${JSON.stringify(providerRunners)}`); if (activeProviderRunnersCount === 0) { logger.debug(`No active runners found for environment: '${environment}'`); @@ -377,5 +381,7 @@ export async function scaleDown(): Promise { await evaluateAndRemoveRunners(runners, scaleDownConfigs, runnerProvider); const activeProviderRunnersCountAfter = (await listRunners(environment, runnerProvider)).length; - logger.info(`Found: '${activeProviderRunnersCountAfter}' active ${runnerProvider.name} runners after clean-up.`); + logger.info( + `Found: '${activeProviderRunnersCountAfter}' active ${runnerProvider.type.toUpperCase()} runners after clean-up.`, + ); } From 443b08c266d5290430b13e1d63c0ca27a7ba53b3 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 21:59:04 +0200 Subject: [PATCH 21/46] fix(webhook): restrict runner provider to ec2 --- lambdas/functions/webhook/src/runner-provider.ts | 14 ++++++++++++++ .../src/runners/aws-dynamic-labels-provider.ts | 3 ++- .../webhook/src/runners/aws-dynamic-labels.ts | 15 ++------------- .../webhook/src/runners/dispatch.test.ts | 12 ++++++++---- lambdas/functions/webhook/src/sqs/index.ts | 4 ++-- 5 files changed, 28 insertions(+), 20 deletions(-) create mode 100644 lambdas/functions/webhook/src/runner-provider.ts diff --git a/lambdas/functions/webhook/src/runner-provider.ts b/lambdas/functions/webhook/src/runner-provider.ts new file mode 100644 index 0000000000..f938006a2c --- /dev/null +++ b/lambdas/functions/webhook/src/runner-provider.ts @@ -0,0 +1,14 @@ +// TODO: Add MicroVM when its webhook provider strategy is implemented. +export type RunnerProvider = 'ec2'; + +const defaultRunnerProvider: RunnerProvider = 'ec2'; + +export function normalizeRunnerProvider(provider: unknown): RunnerProvider | undefined { + if (provider === undefined) return defaultRunnerProvider; + if (typeof provider !== 'string') return undefined; + + const normalizedProvider = provider.trim().toLowerCase(); + if (!normalizedProvider) return defaultRunnerProvider; + + return normalizedProvider === 'ec2' ? normalizedProvider : undefined; +} diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels-provider.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels-provider.ts index ae9fe26f3d..b16e38b781 100644 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels-provider.ts +++ b/lambdas/functions/webhook/src/runners/aws-dynamic-labels-provider.ts @@ -1,4 +1,5 @@ -import type { RunnerMatcherConfig, RunnerProvider } from '../sqs'; +import type { RunnerProvider } from '../runner-provider'; +import type { RunnerMatcherConfig } from '../sqs'; export interface AwsDynamicLabelDispatchTarget { queue: RunnerMatcherConfig; diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts index a5daccf208..94b4bf25b4 100644 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts +++ b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts @@ -1,24 +1,13 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import { RunnerMatcherConfig, RunnerProvider } from '../sqs'; +import { normalizeRunnerProvider } from '../runner-provider'; +import type { RunnerMatcherConfig } from '../sqs'; import { AwsDynamicLabelDispatchTarget, AwsDynamicLabelProviderStrategy } from './aws-dynamic-labels-provider'; import { ec2DynamicLabelProviderStrategy } from './ec2-dynamic-labels'; const logger = createChildLogger('handler'); const awsDynamicLabelProviderStrategies: AwsDynamicLabelProviderStrategy[] = [ec2DynamicLabelProviderStrategy]; -const defaultRunnerProvider: RunnerProvider = 'ec2'; - -function normalizeRunnerProvider(provider: unknown): RunnerProvider | undefined { - if (provider === undefined) return defaultRunnerProvider; - if (typeof provider !== 'string') return undefined; - - const normalizedProvider = provider.trim().toLowerCase(); - if (!normalizedProvider) return defaultRunnerProvider; - if (normalizedProvider === 'ec2' || normalizedProvider === 'microvm') return normalizedProvider; - - return undefined; -} export function selectAwsDynamicLabelQueue( matches: RunnerMatcherConfig[], diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index 5140032547..b194f2f032 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -6,8 +6,9 @@ import { WorkflowJobEvent } from '@octokit/webhooks-types'; import workFlowJobEvent from '../../test/resources/github_workflowjob_event.json'; import runnerConfig from '../../test/resources/multi_runner_configurations.json'; +import type { RunnerProvider } from '../runner-provider'; import { RunnerConfig, sendActionRequest } from '../sqs'; -import type { RunnerMatcherConfig, RunnerProvider } from '../sqs'; +import type { RunnerMatcherConfig } from '../sqs'; import { dispatch } from './dispatch'; import { selectAwsDynamicLabelQueue } from './aws-dynamic-labels'; import { canRunJob } from './labels'; @@ -601,10 +602,13 @@ describe('selectAwsDynamicLabelQueue', () => { }); }); - it('does not select a provider strategy that is planned but not implemented', () => { - const queue = runnerQueue('microvm', 'microvm'); + it('does not select an unsupported provider strategy', () => { + const queue = runnerQueue('unsupported-provider'); + (queue as unknown as { runnerProvider: string }).runnerProvider = 'unsupported'; - expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-microvm-cpu-count:2'])).toBeUndefined(); + expect( + selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), + ).toBeUndefined(); }); it('rejects a malformed non-string runner provider without throwing', () => { diff --git a/lambdas/functions/webhook/src/sqs/index.ts b/lambdas/functions/webhook/src/sqs/index.ts index b5a23a6dd0..f4027a8e73 100644 --- a/lambdas/functions/webhook/src/sqs/index.ts +++ b/lambdas/functions/webhook/src/sqs/index.ts @@ -3,13 +3,13 @@ import { WorkflowJobEvent } from '@octokit/webhooks-types'; import { createChildLogger, getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; import type { AwsDynamicLabelsPolicy } from '../runners/aws-dynamic-labels-policy'; +import type { RunnerProvider } from '../runner-provider'; const logger = createChildLogger('sqs'); const sqsClientsByRegion = new Map(); -// EC2 is the only implemented provider; MicroVM is reserved for future support. -export type RunnerProvider = 'ec2' | 'microvm'; +export type { RunnerProvider }; export interface ActionRequestMessage { id: number; From 384861a042f954cfe93da3f3d1003752d460e811 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 22:03:11 +0200 Subject: [PATCH 22/46] fix(runners): restrict provider type to ec2 --- lambdas/functions/control-plane/src/runner-provider.ts | 3 ++- .../src/scale-runners/scale-down-config.test.ts | 7 +------ .../control-plane/src/scale-runners/scale-up.test.ts | 4 ---- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/lambdas/functions/control-plane/src/runner-provider.ts b/lambdas/functions/control-plane/src/runner-provider.ts index af4876bdf9..8e68b1150a 100644 --- a/lambdas/functions/control-plane/src/runner-provider.ts +++ b/lambdas/functions/control-plane/src/runner-provider.ts @@ -1,4 +1,5 @@ -export type RunnerProviderType = 'ec2' | 'microvm'; +// TODO: Add MicroVM when its control-plane provider strategies are implemented. +export type RunnerProviderType = 'ec2'; export function normalizeRunnerProviderType(type: string): string { return type.trim().toLowerCase(); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts index 8923ee6e94..48509c073d 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts @@ -71,18 +71,13 @@ describe('scaleDownConfig', () => { expect(getScaleDownRunnerProviderType(scaleDownConfig, 'ec2')).toEqual('ec2'); }); - it('Uses configured ec2 type', async () => { - const scaleDownConfig = getConfig(['* * * * * *']).map((config) => ({ ...config, type: 'ec2' as const })); - expect(getScaleDownRunnerProviderType(scaleDownConfig, 'microvm')).toEqual('ec2'); - }); - it('Treats provider types case-insensitively when checking for multiple types', async () => { const scaleDownConfig = getConfig(['* * * * * *', '* * * * * *']).map((config, index) => ({ ...config, type: index === 0 ? ' EC2 ' : 'ec2', })); - expect(() => getScaleDownRunnerProviderType(scaleDownConfig, 'microvm')).not.toThrow(); + expect(() => getScaleDownRunnerProviderType(scaleDownConfig, 'ec2')).not.toThrow(); }); }); }); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index a13decdd50..10804b4dbc 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -3514,8 +3514,4 @@ describe('getScaleUpRunnerProviderType', () => { expect(getScaleUpRunnerProviderType('', 'ec2')).toEqual('ec2'); expect(getScaleUpRunnerProviderType(' ', 'ec2')).toEqual('ec2'); }); - - it('uses configured ec2 type', () => { - expect(getScaleUpRunnerProviderType('ec2', 'microvm')).toEqual('ec2'); - }); }); From 78e59bc741f5286dcb9a73bfd0b57dae732758fd Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 22:10:19 +0200 Subject: [PATCH 23/46] refactor(runners): simplify provider selection --- .../control-plane/src/pool/ec2-pool.ts | 7 +------ .../src/pool/pool-provider-registry.ts | 15 +++++---------- .../control-plane/src/pool/pool-provider.ts | 4 +--- .../control-plane/src/runner-provider.ts | 7 +------ .../src/scale-runners/ec2-scale-down.ts | 11 +---------- .../src/scale-runners/ec2-scale-up.ts | 7 +------ .../scale-down-provider-registry.ts | 19 +++++-------------- .../src/scale-runners/scale-down-provider.ts | 4 +--- .../scale-up-provider-registry.ts | 19 +++++-------------- .../src/scale-runners/scale-up-provider.ts | 12 +----------- 10 files changed, 22 insertions(+), 83 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/ec2-pool.ts b/lambdas/functions/control-plane/src/pool/ec2-pool.ts index d1c222d935..83a5f18561 100644 --- a/lambdas/functions/control-plane/src/pool/ec2-pool.ts +++ b/lambdas/functions/control-plane/src/pool/ec2-pool.ts @@ -5,7 +5,7 @@ import { bootTimeExceeded, listEC2Runners } from '../aws/ec2-runners'; import type { RunnerList } from '../aws/ec2-runners.d'; import { createRunners } from '../scale-runners/ec2'; import type { CreateEC2RunnerConfig } from '../scale-runners/ec2'; -import type { PoolRunnerProvider, PoolRunnerProviderStrategy, RunnerStatus } from './pool-provider'; +import type { PoolRunnerProvider, RunnerStatus } from './pool-provider'; const logger = createChildLogger('pool'); @@ -45,11 +45,6 @@ export function createEc2PoolProviderFromEnv(): PoolRunnerProvider { }); } -export const ec2PoolRunnerProviderStrategy: PoolRunnerProviderStrategy = { - type: 'ec2', - createFromEnv: createEc2PoolProviderFromEnv, -}; - function createEc2PoolProvider(config: Ec2PoolProviderConfig): PoolRunnerProvider { return { type: 'ec2', diff --git a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts index ff0e2b9e1e..b24e78275a 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts @@ -1,20 +1,15 @@ import { normalizeRunnerProviderType } from '../runner-provider'; -import { ec2PoolRunnerProviderStrategy } from './ec2-pool'; -import type { PoolRunnerProvider, PoolRunnerProviderStrategy, PoolRunnerProviderType } from './pool-provider'; - -const poolRunnerProviderStrategies: PoolRunnerProviderStrategy[] = [ec2PoolRunnerProviderStrategy]; +import { createEc2PoolProviderFromEnv } from './ec2-pool'; +import type { PoolRunnerProvider, PoolRunnerProviderType } from './pool-provider'; export function getDefaultPoolRunnerProviderType(): PoolRunnerProviderType { - return poolRunnerProviderStrategies[0].type; + return 'ec2'; } export function createPoolRunnerProviderFromEnv(type: string): PoolRunnerProvider { - const normalizedType = normalizeRunnerProviderType(type); - const strategy = poolRunnerProviderStrategies.find((strategy) => strategy.type === normalizedType); - - if (!strategy) { + if (normalizeRunnerProviderType(type) !== 'ec2') { throw new Error(`Unsupported pool runner provider type '${type}'`); } - return strategy.createFromEnv(); + return createEc2PoolProviderFromEnv(); } diff --git a/lambdas/functions/control-plane/src/pool/pool-provider.ts b/lambdas/functions/control-plane/src/pool/pool-provider.ts index 7af55628ac..ff17f3eea6 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider.ts @@ -1,6 +1,6 @@ import type { Octokit } from '@octokit/rest'; -import type { RunnerProvider, RunnerProviderStrategy, RunnerProviderType } from '../runner-provider'; +import type { RunnerProvider, RunnerProviderType } from '../runner-provider'; import type { CreateGitHubRunnerConfig, GitHubRunnerType } from '../scale-runners/types'; export type PoolRunnerProviderType = RunnerProviderType; @@ -31,5 +31,3 @@ export interface PoolRunnerProvider extends RunnerProvider { ): number; createRunners(input: CreatePoolRunnersInput): Promise; } - -export type PoolRunnerProviderStrategy = RunnerProviderStrategy; diff --git a/lambdas/functions/control-plane/src/runner-provider.ts b/lambdas/functions/control-plane/src/runner-provider.ts index 8e68b1150a..c8de555b54 100644 --- a/lambdas/functions/control-plane/src/runner-provider.ts +++ b/lambdas/functions/control-plane/src/runner-provider.ts @@ -1,4 +1,4 @@ -// TODO: Add MicroVM when its control-plane provider strategies are implemented. +// TODO: Add MicroVM when its control-plane provider implementations are available. export type RunnerProviderType = 'ec2'; export function normalizeRunnerProviderType(type: string): string { @@ -8,8 +8,3 @@ export function normalizeRunnerProviderType(type: string): string { export interface RunnerProvider { type: RunnerProviderType; } - -export interface RunnerProviderStrategy { - type: TProvider['type']; - createFromEnv(...args: TCreateFromEnvArgs): TProvider; -} diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts index eec0e0f2f0..bc914e84b2 100644 --- a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts @@ -1,10 +1,6 @@ import { bootTimeExceeded, listEC2Runners, tag, terminateRunner, untag } from './../aws/ec2-runners'; import type { RunnerList } from './../aws/ec2-runners.d'; -import type { - RunnerList as ScaleDownRunnerList, - ScaleDownRunnerProvider, - ScaleDownRunnerProviderStrategy, -} from './scale-down-provider'; +import type { RunnerList as ScaleDownRunnerList, ScaleDownRunnerProvider } from './scale-down-provider'; export function createEc2ScaleDownProvider(): ScaleDownRunnerProvider { return { @@ -17,11 +13,6 @@ export function createEc2ScaleDownProvider(): ScaleDownRunnerProvider { }; } -export const ec2ScaleDownRunnerProviderStrategy: ScaleDownRunnerProviderStrategy = { - type: 'ec2', - createFromEnv: createEc2ScaleDownProvider, -}; - function toScaleDownRunner(runner: RunnerList): ScaleDownRunnerList { return { id: runner.instanceId, diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts index 678d6d40d0..c10158678d 100644 --- a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts @@ -10,7 +10,7 @@ import { } from './ec2-labels'; import { createRunners } from './ec2'; import type { CreateEC2RunnerConfig } from './ec2'; -import type { ScaleUpRunnerProvider, ScaleUpRunnerProviderStrategy } from './scale-up-provider'; +import type { ScaleUpRunnerProvider } from './scale-up-provider'; const logger = createChildLogger('ec2-scale-up'); @@ -87,8 +87,3 @@ export function createEc2ScaleUpProviderFromEnv( useDedicatedHost: yn(process.env.USE_DEDICATED_HOST, { default: false }), }); } - -export const ec2ScaleUpRunnerProviderStrategy: ScaleUpRunnerProviderStrategy = { - type: 'ec2', - createFromEnv: ({ environment, scaleErrors }) => createEc2ScaleUpProviderFromEnv(environment, scaleErrors), -}; diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts index 8479c5ca42..d75e7311bf 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts @@ -1,24 +1,15 @@ import { normalizeRunnerProviderType } from '../runner-provider'; -import { ec2ScaleDownRunnerProviderStrategy } from './ec2-scale-down'; -import type { - ScaleDownRunnerProvider, - ScaleDownRunnerProviderStrategy, - ScaleDownRunnerProviderType, -} from './scale-down-provider'; - -const scaleDownRunnerProviderStrategies: ScaleDownRunnerProviderStrategy[] = [ec2ScaleDownRunnerProviderStrategy]; +import { createEc2ScaleDownProvider } from './ec2-scale-down'; +import type { ScaleDownRunnerProvider, ScaleDownRunnerProviderType } from './scale-down-provider'; export function getDefaultScaleDownRunnerProviderType(): ScaleDownRunnerProviderType { - return scaleDownRunnerProviderStrategies[0].type; + return 'ec2'; } export function createScaleDownRunnerProviderFromEnv(type: string): ScaleDownRunnerProvider { - const normalizedType = normalizeRunnerProviderType(type); - const strategy = scaleDownRunnerProviderStrategies.find((strategy) => strategy.type === normalizedType); - - if (!strategy) { + if (normalizeRunnerProviderType(type) !== 'ec2') { throw new Error(`Unsupported scale-down runner provider type '${type}'`); } - return strategy.createFromEnv(); + return createEc2ScaleDownProvider(); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts index 8a4d0b0b88..848e1e1392 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts @@ -1,4 +1,4 @@ -import type { RunnerProvider, RunnerProviderStrategy, RunnerProviderType } from '../runner-provider'; +import type { RunnerProvider, RunnerProviderType } from '../runner-provider'; export interface RunnerList { id: string; @@ -26,5 +26,3 @@ export interface ScaleDownRunnerProvider extends RunnerProvider { unmarkOrphan(id: string): Promise; terminate(id: string): Promise; } - -export type ScaleDownRunnerProviderStrategy = RunnerProviderStrategy; diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts index ae3febd1eb..659cc43023 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts @@ -1,15 +1,9 @@ import { normalizeRunnerProviderType } from '../runner-provider'; -import { ec2ScaleUpRunnerProviderStrategy } from './ec2-scale-up'; -import type { - ScaleUpRunnerProvider, - ScaleUpRunnerProviderStrategy, - ScaleUpRunnerProviderType, -} from './scale-up-provider'; - -const scaleUpRunnerProviderStrategies: ScaleUpRunnerProviderStrategy[] = [ec2ScaleUpRunnerProviderStrategy]; +import { createEc2ScaleUpProviderFromEnv } from './ec2-scale-up'; +import type { ScaleUpRunnerProvider, ScaleUpRunnerProviderType } from './scale-up-provider'; export function getDefaultScaleUpRunnerProviderType(): ScaleUpRunnerProviderType { - return scaleUpRunnerProviderStrategies[0].type; + return 'ec2'; } export function createScaleUpRunnerProviderFromEnv( @@ -17,12 +11,9 @@ export function createScaleUpRunnerProviderFromEnv( environment: string, scaleErrors: string[], ): ScaleUpRunnerProvider { - const normalizedType = normalizeRunnerProviderType(type); - const strategy = scaleUpRunnerProviderStrategies.find((strategy) => strategy.type === normalizedType); - - if (!strategy) { + if (normalizeRunnerProviderType(type) !== 'ec2') { throw new Error(`Unsupported scale-up runner provider type '${type}'`); } - return strategy.createFromEnv({ environment, scaleErrors }); + return createEc2ScaleUpProviderFromEnv(environment, scaleErrors); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts index 24805bd530..04c6beedc3 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts @@ -1,6 +1,6 @@ import type { Octokit } from '@octokit/rest'; -import type { RunnerProvider, RunnerProviderStrategy, RunnerProviderType } from '../runner-provider'; +import type { RunnerProvider, RunnerProviderType } from '../runner-provider'; import type { ActionRequestMessageSQS, CreateGitHubRunnerConfig, GitHubRunnerType } from './types'; export interface CurrentRunnersInput { @@ -28,13 +28,3 @@ export interface ScaleUpRunnerProvider extends RunnerProvider getCurrentRunners(state: TState, input: CurrentRunnersInput): Promise; createRunners(input: CreateScaleUpRunnersInput): Promise; } - -export interface CreateScaleUpRunnerProviderInput { - environment: string; - scaleErrors: string[]; -} - -export type ScaleUpRunnerProviderStrategy = RunnerProviderStrategy< - ScaleUpRunnerProvider, - [input: CreateScaleUpRunnerProviderInput] ->; From 4129c9e62450b1f250b0aa9a07743b0330075b71 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 22:14:24 +0200 Subject: [PATCH 24/46] refactor(runners): centralize default provider --- .../control-plane/src/pool/pool-provider-registry.ts | 6 +----- lambdas/functions/control-plane/src/pool/pool.ts | 5 +++-- lambdas/functions/control-plane/src/runner-provider.ts | 2 ++ .../src/scale-runners/scale-down-provider-registry.ts | 6 +----- .../control-plane/src/scale-runners/scale-down.ts | 8 +++----- .../src/scale-runners/scale-up-provider-registry.ts | 6 +----- .../functions/control-plane/src/scale-runners/scale-up.ts | 8 +++----- 7 files changed, 14 insertions(+), 27 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts index b24e78275a..f0d01ba62e 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts @@ -1,10 +1,6 @@ import { normalizeRunnerProviderType } from '../runner-provider'; import { createEc2PoolProviderFromEnv } from './ec2-pool'; -import type { PoolRunnerProvider, PoolRunnerProviderType } from './pool-provider'; - -export function getDefaultPoolRunnerProviderType(): PoolRunnerProviderType { - return 'ec2'; -} +import type { PoolRunnerProvider } from './pool-provider'; export function createPoolRunnerProviderFromEnv(type: string): PoolRunnerProvider { if (normalizeRunnerProviderType(type) !== 'ec2') { diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 145ae7a431..6d98720846 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -3,8 +3,9 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import yn from 'yn'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; +import { defaultRunnerProvider } from '../runner-provider'; import { getGitHubEnterpriseApiUrl, validateSsmParameterStoreTags } from '../scale-runners/github-runner'; -import { createPoolRunnerProviderFromEnv, getDefaultPoolRunnerProviderType } from './pool-provider-registry'; +import { createPoolRunnerProviderFromEnv } from './pool-provider-registry'; import type { RunnerStatus } from './pool-provider'; const logger = createChildLogger('pool'); @@ -15,7 +16,7 @@ export interface PoolEvent { } export async function adjust(event: PoolEvent): Promise { - const runnerProviderType = event.type?.trim() ? event.type : getDefaultPoolRunnerProviderType(); + const runnerProviderType = event.type?.trim() ? event.type : defaultRunnerProvider; logger.info(`Checking current ${runnerProviderType} pool size against pool of size: ${event.poolSize}`); const runnerLabels = process.env.RUNNER_LABELS || ''; const runnerGroup = process.env.RUNNER_GROUP_NAME || ''; diff --git a/lambdas/functions/control-plane/src/runner-provider.ts b/lambdas/functions/control-plane/src/runner-provider.ts index c8de555b54..aef5277bd8 100644 --- a/lambdas/functions/control-plane/src/runner-provider.ts +++ b/lambdas/functions/control-plane/src/runner-provider.ts @@ -1,6 +1,8 @@ // TODO: Add MicroVM when its control-plane provider implementations are available. export type RunnerProviderType = 'ec2'; +export const defaultRunnerProvider: RunnerProviderType = 'ec2'; + export function normalizeRunnerProviderType(type: string): string { return type.trim().toLowerCase(); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts index d75e7311bf..262200fe90 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts @@ -1,10 +1,6 @@ import { normalizeRunnerProviderType } from '../runner-provider'; import { createEc2ScaleDownProvider } from './ec2-scale-down'; -import type { ScaleDownRunnerProvider, ScaleDownRunnerProviderType } from './scale-down-provider'; - -export function getDefaultScaleDownRunnerProviderType(): ScaleDownRunnerProviderType { - return 'ec2'; -} +import type { ScaleDownRunnerProvider } from './scale-down-provider'; export function createScaleDownRunnerProviderFromEnv(type: string): ScaleDownRunnerProvider { if (normalizeRunnerProviderType(type) !== 'ec2') { diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index 30052077e5..3bafe04df7 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -5,6 +5,7 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import moment from 'moment'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; +import { defaultRunnerProvider } from '../runner-provider'; import { GhRunners, githubCache } from './cache'; import { ScalingDownConfig, @@ -12,10 +13,7 @@ import { getIdleRunnerCount, getScaleDownRunnerProviderType, } from './scale-down-config'; -import { - createScaleDownRunnerProviderFromEnv, - getDefaultScaleDownRunnerProviderType, -} from './scale-down-provider-registry'; +import { createScaleDownRunnerProviderFromEnv } from './scale-down-provider-registry'; import { metricGitHubAppRateLimit } from '../github/rate-limit'; import { getGitHubEnterpriseApiUrl } from './github-runner'; import type { RunnerInfo, RunnerList, ScaleDownRunnerProvider } from './scale-down-provider'; @@ -358,7 +356,7 @@ export async function scaleDown(): Promise { githubCache.reset(); const environment = process.env.ENVIRONMENT; const scaleDownConfigs = JSON.parse(process.env.SCALE_DOWN_CONFIG) as [ScalingDownConfig]; - const runnerProviderType = getScaleDownRunnerProviderType(scaleDownConfigs, getDefaultScaleDownRunnerProviderType()); + const runnerProviderType = getScaleDownRunnerProviderType(scaleDownConfigs, defaultRunnerProvider); const runnerProvider = createScaleDownRunnerProviderFromEnv(runnerProviderType); // first runners marked to be orphan. diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts index 659cc43023..c853e008ff 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts @@ -1,10 +1,6 @@ import { normalizeRunnerProviderType } from '../runner-provider'; import { createEc2ScaleUpProviderFromEnv } from './ec2-scale-up'; -import type { ScaleUpRunnerProvider, ScaleUpRunnerProviderType } from './scale-up-provider'; - -export function getDefaultScaleUpRunnerProviderType(): ScaleUpRunnerProviderType { - return 'ec2'; -} +import type { ScaleUpRunnerProvider } from './scale-up-provider'; export function createScaleUpRunnerProviderFromEnv( type: string, diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 2bf5f8e81c..581359ba7b 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -3,6 +3,7 @@ import { Octokit } from '@octokit/rest'; import yn from 'yn'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; +import { defaultRunnerProvider } from '../runner-provider'; import { getGitHubEnterpriseApiUrl, getInstallationId, @@ -12,7 +13,7 @@ import { } from './github-runner'; import { publishRetryMessage } from './job-retry'; import { getScaleUpRunnerProviderType } from './scale-up-config'; -import { createScaleUpRunnerProviderFromEnv, getDefaultScaleUpRunnerProviderType } from './scale-up-provider-registry'; +import { createScaleUpRunnerProviderFromEnv } from './scale-up-provider-registry'; import type { ActionRequestMessage, ActionRequestMessageRetry, @@ -86,10 +87,7 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise Date: Tue, 14 Jul 2026 22:18:24 +0200 Subject: [PATCH 25/46] fix(runners): align scale-down config contract --- .../scale-runners/scale-down-config.test.ts | 28 +------------------ .../src/scale-runners/scale-down-config.ts | 21 -------------- .../src/scale-runners/scale-down.ts | 14 +++------- 3 files changed, 5 insertions(+), 58 deletions(-) diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts index 48509c073d..ff2325128a 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.test.ts @@ -1,12 +1,6 @@ import moment from 'moment-timezone'; -import { - EvictionStrategy, - ScalingDownConfigList, - getEvictionStrategy, - getIdleRunnerCount, - getScaleDownRunnerProviderType, -} from './scale-down-config'; +import { EvictionStrategy, ScalingDownConfigList, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config'; import { describe, it, expect } from 'vitest'; const DEFAULT_TIMEZONE = 'America/Los_Angeles'; @@ -60,24 +54,4 @@ describe('scaleDownConfig', () => { expect(getEvictionStrategy(scaleDownConfig)).toEqual(DEFAULT_EVICTION_STRATEGY); }); }); - - describe('Determine runner provider type.', () => { - it('Defaults to ec2 when no idle config is defined', async () => { - expect(getScaleDownRunnerProviderType([], 'ec2')).toEqual('ec2'); - }); - - it('Defaults to ec2 when config has no type', async () => { - const scaleDownConfig = getConfig(['* * * * * *']); - expect(getScaleDownRunnerProviderType(scaleDownConfig, 'ec2')).toEqual('ec2'); - }); - - it('Treats provider types case-insensitively when checking for multiple types', async () => { - const scaleDownConfig = getConfig(['* * * * * *', '* * * * * *']).map((config, index) => ({ - ...config, - type: index === 0 ? ' EC2 ' : 'ec2', - })); - - expect(() => getScaleDownRunnerProviderType(scaleDownConfig, 'ec2')).not.toThrow(); - }); - }); }); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.ts index 362b190f3d..2d28c38cfc 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-config.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-config.ts @@ -2,13 +2,9 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import parser from 'cron-parser'; import moment from 'moment'; -import { normalizeRunnerProviderType } from '../runner-provider'; -import type { ScaleDownRunnerProviderType } from './scale-down-provider'; - export type ScalingDownConfigList = ScalingDownConfig[]; export type EvictionStrategy = 'newest_first' | 'oldest_first'; export interface ScalingDownConfig { - type?: string; cron: string; idleCount: number; timeZone: string; @@ -45,20 +41,3 @@ export function getEvictionStrategy(scalingDownConfigs: ScalingDownConfigList): } return 'oldest_first'; } - -export function getScaleDownRunnerProviderType( - scalingDownConfigs: ScalingDownConfigList, - defaultType: ScaleDownRunnerProviderType, -): string { - const configuredTypes = new Map(); - for (const scalingDownConfig of scalingDownConfigs) { - const configuredType = scalingDownConfig.type?.trim() ? scalingDownConfig.type : defaultType; - configuredTypes.set(normalizeRunnerProviderType(configuredType), configuredType); - } - - if (configuredTypes.size > 1) { - throw new Error(`Multiple scale-down runner provider types are not supported in a single scale-down config.`); - } - - return configuredTypes.values().next().value ?? defaultType; -} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index 3bafe04df7..ae67b47238 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -7,12 +7,7 @@ import moment from 'moment'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; import { defaultRunnerProvider } from '../runner-provider'; import { GhRunners, githubCache } from './cache'; -import { - ScalingDownConfig, - getEvictionStrategy, - getIdleRunnerCount, - getScaleDownRunnerProviderType, -} from './scale-down-config'; +import { ScalingDownConfigList, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config'; import { createScaleDownRunnerProviderFromEnv } from './scale-down-provider-registry'; import { metricGitHubAppRateLimit } from '../github/rate-limit'; import { getGitHubEnterpriseApiUrl } from './github-runner'; @@ -220,7 +215,7 @@ async function removeRunner( async function evaluateAndRemoveRunners( runners: RunnerInfo[], - scaleDownConfigs: ScalingDownConfig[], + scaleDownConfigs: ScalingDownConfigList, runnerProvider: ScaleDownRunnerProvider, ): Promise { let idleCounter = getIdleRunnerCount(scaleDownConfigs); @@ -355,9 +350,8 @@ function filterRunners(runners: RunnerList[]): RunnerInfo[] { export async function scaleDown(): Promise { githubCache.reset(); const environment = process.env.ENVIRONMENT; - const scaleDownConfigs = JSON.parse(process.env.SCALE_DOWN_CONFIG) as [ScalingDownConfig]; - const runnerProviderType = getScaleDownRunnerProviderType(scaleDownConfigs, defaultRunnerProvider); - const runnerProvider = createScaleDownRunnerProviderFromEnv(runnerProviderType); + const scaleDownConfigs = JSON.parse(process.env.SCALE_DOWN_CONFIG) as ScalingDownConfigList; + const runnerProvider = createScaleDownRunnerProviderFromEnv(defaultRunnerProvider); // first runners marked to be orphan. await terminateOrphan(environment, runnerProvider); From b58312b77883c67273b832ac0ca1388b671721d5 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 22:27:26 +0200 Subject: [PATCH 26/46] refactor(runners): centralize provider validation --- .../src/pool/pool-provider-registry.ts | 7 +--- .../control-plane/src/pool/pool.test.ts | 7 ++++ .../functions/control-plane/src/pool/pool.ts | 6 +-- .../control-plane/src/runner-provider.test.ts | 38 +++++-------------- .../control-plane/src/runner-provider.ts | 17 +++++++-- .../scale-down-provider-registry.ts | 7 +--- .../src/scale-runners/scale-down.ts | 3 +- .../src/scale-runners/scale-up-config.ts | 5 --- .../scale-up-provider-registry.ts | 11 +----- .../src/scale-runners/scale-up.test.ts | 13 +++---- .../src/scale-runners/scale-up.ts | 7 ++-- 11 files changed, 46 insertions(+), 75 deletions(-) delete mode 100644 lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts diff --git a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts index f0d01ba62e..4cc05ac6b4 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts @@ -1,11 +1,6 @@ -import { normalizeRunnerProviderType } from '../runner-provider'; import { createEc2PoolProviderFromEnv } from './ec2-pool'; import type { PoolRunnerProvider } from './pool-provider'; -export function createPoolRunnerProviderFromEnv(type: string): PoolRunnerProvider { - if (normalizeRunnerProviderType(type) !== 'ec2') { - throw new Error(`Unsupported pool runner provider type '${type}'`); - } - +export function createPoolRunnerProviderFromEnv(): PoolRunnerProvider { return createEc2PoolProviderFromEnv(); } diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index 49886e91cc..86ccdc163e 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -235,6 +235,13 @@ describe('Test simple pool.', () => { ); }); + it('Rejects unsupported pool provider types.', async () => { + await expect(adjust({ poolSize: 10, type: 'microvm' })).rejects.toThrow( + "Unsupported runner provider type 'microvm'", + ); + expect(mockListRunners).not.toHaveBeenCalled(); + }); + it('Should not top up if pool size is reached.', async () => { await adjust({ poolSize: 1, type: 'ec2' }); expect(createRunners).not.toHaveBeenCalled(); diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 6d98720846..415cf25124 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -3,7 +3,7 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import yn from 'yn'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; -import { defaultRunnerProvider } from '../runner-provider'; +import { resolveRunnerProviderType } from '../runner-provider'; import { getGitHubEnterpriseApiUrl, validateSsmParameterStoreTags } from '../scale-runners/github-runner'; import { createPoolRunnerProviderFromEnv } from './pool-provider-registry'; import type { RunnerStatus } from './pool-provider'; @@ -16,7 +16,7 @@ export interface PoolEvent { } export async function adjust(event: PoolEvent): Promise { - const runnerProviderType = event.type?.trim() ? event.type : defaultRunnerProvider; + const runnerProviderType = resolveRunnerProviderType(event.type); logger.info(`Checking current ${runnerProviderType} pool size against pool of size: ${event.poolSize}`); const runnerLabels = process.env.RUNNER_LABELS || ''; const runnerGroup = process.env.RUNNER_GROUP_NAME || ''; @@ -36,7 +36,7 @@ export async function adjust(event: PoolEvent): Promise { // when unset so the pool keeps its previous behavior on stacks that do not provide the variable. const maximumRunners = parseInt(process.env.RUNNERS_MAXIMUM_COUNT || '-1'); const includeBusyRunners = yn(process.env.INCLUDE_BUSY_RUNNERS, { default: false }); - const runnerProvider = createPoolRunnerProviderFromEnv(runnerProviderType); + const runnerProvider = createPoolRunnerProviderFromEnv(); const { ghesApiUrl, ghesBaseUrl } = getGitHubEnterpriseApiUrl(); diff --git a/lambdas/functions/control-plane/src/runner-provider.test.ts b/lambdas/functions/control-plane/src/runner-provider.test.ts index 28756ed8b9..ba7eec1908 100644 --- a/lambdas/functions/control-plane/src/runner-provider.test.ts +++ b/lambdas/functions/control-plane/src/runner-provider.test.ts @@ -1,36 +1,18 @@ import { describe, expect, it } from 'vitest'; -import { createPoolRunnerProviderFromEnv } from './pool/pool-provider-registry'; -import { normalizeRunnerProviderType } from './runner-provider'; -import { createScaleDownRunnerProviderFromEnv } from './scale-runners/scale-down-provider-registry'; -import { createScaleUpRunnerProviderFromEnv } from './scale-runners/scale-up-provider-registry'; +import { resolveRunnerProviderType } from './runner-provider'; describe('runner provider selection', () => { - it.each([[' EC2 ', 'ec2']])('normalizes provider type %j to %j', (type, expected) => { - expect(normalizeRunnerProviderType(type)).toBe(expected); + it.each([ + [undefined, 'ec2'], + ['', 'ec2'], + [' ', 'ec2'], + [' EC2 ', 'ec2'], + ])('resolves provider type %j to %j', (type, expected) => { + expect(resolveRunnerProviderType(type)).toBe(expected); }); - it('selects the EC2 scale-down provider case-insensitively', () => { - expect(createScaleDownRunnerProviderFromEnv(' EC2 ')).toMatchObject({ - type: 'ec2', - }); - }); - - it('quotes the original unsupported pool provider type', () => { - expect(() => createPoolRunnerProviderFromEnv(' Unknown ')).toThrow( - "Unsupported pool runner provider type ' Unknown '", - ); - }); - - it('quotes the original unsupported scale-up provider type', () => { - expect(() => createScaleUpRunnerProviderFromEnv(' Unknown ', 'test', [])).toThrow( - "Unsupported scale-up runner provider type ' Unknown '", - ); - }); - - it('quotes the original unsupported scale-down provider type', () => { - expect(() => createScaleDownRunnerProviderFromEnv(' Unknown ')).toThrow( - "Unsupported scale-down runner provider type ' Unknown '", - ); + it.each([[' Unknown '], ['microvm'], [null], [1]])('rejects unsupported provider type %j', (type) => { + expect(() => resolveRunnerProviderType(type)).toThrow(`Unsupported runner provider type '${String(type)}'`); }); }); diff --git a/lambdas/functions/control-plane/src/runner-provider.ts b/lambdas/functions/control-plane/src/runner-provider.ts index aef5277bd8..fb784a54c9 100644 --- a/lambdas/functions/control-plane/src/runner-provider.ts +++ b/lambdas/functions/control-plane/src/runner-provider.ts @@ -1,10 +1,21 @@ // TODO: Add MicroVM when its control-plane provider implementations are available. export type RunnerProviderType = 'ec2'; -export const defaultRunnerProvider: RunnerProviderType = 'ec2'; +const defaultRunnerProvider: RunnerProviderType = 'ec2'; -export function normalizeRunnerProviderType(type: string): string { - return type.trim().toLowerCase(); +export function resolveRunnerProviderType(type: unknown): RunnerProviderType { + if (type === undefined) return defaultRunnerProvider; + if (typeof type !== 'string') { + throw new Error(`Unsupported runner provider type '${String(type)}'`); + } + + const normalizedType = type.trim().toLowerCase(); + if (!normalizedType) return defaultRunnerProvider; + if (normalizedType !== 'ec2') { + throw new Error(`Unsupported runner provider type '${type}'`); + } + + return normalizedType; } export interface RunnerProvider { diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts index 262200fe90..859951f1c9 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts @@ -1,11 +1,6 @@ -import { normalizeRunnerProviderType } from '../runner-provider'; import { createEc2ScaleDownProvider } from './ec2-scale-down'; import type { ScaleDownRunnerProvider } from './scale-down-provider'; -export function createScaleDownRunnerProviderFromEnv(type: string): ScaleDownRunnerProvider { - if (normalizeRunnerProviderType(type) !== 'ec2') { - throw new Error(`Unsupported scale-down runner provider type '${type}'`); - } - +export function createScaleDownRunnerProviderFromEnv(): ScaleDownRunnerProvider { return createEc2ScaleDownProvider(); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index ae67b47238..3471a2a675 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -5,7 +5,6 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import moment from 'moment'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; -import { defaultRunnerProvider } from '../runner-provider'; import { GhRunners, githubCache } from './cache'; import { ScalingDownConfigList, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config'; import { createScaleDownRunnerProviderFromEnv } from './scale-down-provider-registry'; @@ -351,7 +350,7 @@ export async function scaleDown(): Promise { githubCache.reset(); const environment = process.env.ENVIRONMENT; const scaleDownConfigs = JSON.parse(process.env.SCALE_DOWN_CONFIG) as ScalingDownConfigList; - const runnerProvider = createScaleDownRunnerProviderFromEnv(defaultRunnerProvider); + const runnerProvider = createScaleDownRunnerProviderFromEnv(); // first runners marked to be orphan. await terminateOrphan(environment, runnerProvider); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts deleted file mode 100644 index ab3dccb441..0000000000 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-config.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { ScaleUpRunnerProviderType } from './scale-up-provider'; - -export function getScaleUpRunnerProviderType(type: string | undefined, defaultType: ScaleUpRunnerProviderType): string { - return type?.trim() ? type : defaultType; -} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts index c853e008ff..c73cb9fe94 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts @@ -1,15 +1,6 @@ -import { normalizeRunnerProviderType } from '../runner-provider'; import { createEc2ScaleUpProviderFromEnv } from './ec2-scale-up'; import type { ScaleUpRunnerProvider } from './scale-up-provider'; -export function createScaleUpRunnerProviderFromEnv( - type: string, - environment: string, - scaleErrors: string[], -): ScaleUpRunnerProvider { - if (normalizeRunnerProviderType(type) !== 'ec2') { - throw new Error(`Unsupported scale-up runner provider type '${type}'`); - } - +export function createScaleUpRunnerProviderFromEnv(environment: string, scaleErrors: string[]): ScaleUpRunnerProvider { return createEc2ScaleUpProviderFromEnv(environment, scaleErrors); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index 10804b4dbc..bf27b92c6a 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -11,7 +11,6 @@ import { createRunner, listEC2Runners, tag } from './../aws/ec2-runners'; import { RunnerInputParameters } from './../aws/ec2-runners.d'; import * as scaleUpModule from './scale-up'; import { parseEc2OverrideConfig } from './ec2-labels'; -import { getScaleUpRunnerProviderType } from './scale-up-config'; import { EC2_TAG_VALUE_MAX_LENGTH, RUNNER_LABELS_TAG_MAX_COUNT } from './ec2'; import { getParameter } from '@aws-github-runner/aws-ssm-util'; import { publishRetryMessage } from './job-retry'; @@ -3505,13 +3504,11 @@ describe('parseEc2OverrideConfig', () => { }); }); -describe('getScaleUpRunnerProviderType', () => { - it('defaults to ec2 when no type is defined', () => { - expect(getScaleUpRunnerProviderType(undefined, 'ec2')).toEqual('ec2'); - }); +describe('runner provider selection', () => { + it('rejects unsupported scale-up provider types', async () => { + process.env.RUNNER_PROVIDER_TYPE = 'microvm'; - it('defaults to ec2 when the configured type is blank', () => { - expect(getScaleUpRunnerProviderType('', 'ec2')).toEqual('ec2'); - expect(getScaleUpRunnerProviderType(' ', 'ec2')).toEqual('ec2'); + await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow("Unsupported runner provider type 'microvm'"); + expect(mockedAppAuth).not.toHaveBeenCalled(); }); }); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 581359ba7b..3f87d8fda9 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -3,7 +3,7 @@ import { Octokit } from '@octokit/rest'; import yn from 'yn'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; -import { defaultRunnerProvider } from '../runner-provider'; +import { resolveRunnerProviderType } from '../runner-provider'; import { getGitHubEnterpriseApiUrl, getInstallationId, @@ -12,7 +12,6 @@ import { validateSsmParameterStoreTags, } from './github-runner'; import { publishRetryMessage } from './job-retry'; -import { getScaleUpRunnerProviderType } from './scale-up-config'; import { createScaleUpRunnerProviderFromEnv } from './scale-up-provider-registry'; import type { ActionRequestMessage, @@ -87,8 +86,8 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise Date: Tue, 14 Jul 2026 22:30:30 +0200 Subject: [PATCH 27/46] refactor(pool): name provider functions --- .../control-plane/src/pool/ec2-pool.ts | 64 +++++++++++-------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/ec2-pool.ts b/lambdas/functions/control-plane/src/pool/ec2-pool.ts index 83a5f18561..a51fea1079 100644 --- a/lambdas/functions/control-plane/src/pool/ec2-pool.ts +++ b/lambdas/functions/control-plane/src/pool/ec2-pool.ts @@ -5,7 +5,7 @@ import { bootTimeExceeded, listEC2Runners } from '../aws/ec2-runners'; import type { RunnerList } from '../aws/ec2-runners.d'; import { createRunners } from '../scale-runners/ec2'; import type { CreateEC2RunnerConfig } from '../scale-runners/ec2'; -import type { PoolRunnerProvider, RunnerStatus } from './pool-provider'; +import type { CreatePoolRunnersInput, ListPoolRunnersInput, PoolRunnerProvider, RunnerStatus } from './pool-provider'; const logger = createChildLogger('pool'); @@ -46,33 +46,47 @@ export function createEc2PoolProviderFromEnv(): PoolRunnerProvider { } function createEc2PoolProvider(config: Ec2PoolProviderConfig): PoolRunnerProvider { + async function listEc2PoolRunners({ + environment, + runnerOwner, + runnerType, + }: ListPoolRunnersInput): Promise { + return await listEC2Runners({ + environment, + runnerOwner, + runnerType, + statuses: ['running'], + }); + } + + async function createEc2PoolRunners({ + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + }: CreatePoolRunnersInput): Promise { + return await createRunners( + githubRunnerConfig, + { + ec2instanceCriteria: config.ec2instanceCriteria, + environment: config.environment, + launchTemplateName: config.launchTemplateName, + subnets: config.subnets, + amiIdSsmParameterName: config.amiIdSsmParameterName, + tracingEnabled: config.tracingEnabled, + onDemandFailoverOnError: config.onDemandFailoverOnError, + scaleErrors: config.scaleErrors, + }, + numberOfRunners, + githubInstallationClient, + 'pool-lambda', + ); + } + return { type: 'ec2', - listRunners: async ({ environment, runnerOwner, runnerType }) => - await listEC2Runners({ - environment, - runnerOwner, - runnerType, - statuses: ['running'], - }), + listRunners: listEc2PoolRunners, countAvailableRunners: calculateEc2PoolSize, - createRunners: async ({ githubRunnerConfig, numberOfRunners, githubInstallationClient }) => - await createRunners( - githubRunnerConfig, - { - ec2instanceCriteria: config.ec2instanceCriteria, - environment: config.environment, - launchTemplateName: config.launchTemplateName, - subnets: config.subnets, - amiIdSsmParameterName: config.amiIdSsmParameterName, - tracingEnabled: config.tracingEnabled, - onDemandFailoverOnError: config.onDemandFailoverOnError, - scaleErrors: config.scaleErrors, - }, - numberOfRunners, - githubInstallationClient, - 'pool-lambda', - ), + createRunners: createEc2PoolRunners, }; } From 853762b45ff5739b6c7403ab47edb6ff88e6a6d4 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 22:33:01 +0200 Subject: [PATCH 28/46] refactor(pool): simplify provider creation --- lambdas/functions/control-plane/src/pool/ec2-pool.ts | 10 ++++++---- .../control-plane/src/pool/pool-provider-registry.ts | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/ec2-pool.ts b/lambdas/functions/control-plane/src/pool/ec2-pool.ts index a51fea1079..077ceaa0b5 100644 --- a/lambdas/functions/control-plane/src/pool/ec2-pool.ts +++ b/lambdas/functions/control-plane/src/pool/ec2-pool.ts @@ -20,10 +20,10 @@ interface Ec2PoolProviderConfig { scaleErrors: string[]; } -export function createEc2PoolProviderFromEnv(): PoolRunnerProvider { +function loadEc2PoolProviderConfig(): Ec2PoolProviderConfig { const scaleErrors = JSON.parse(process.env.SCALE_ERRORS) as [string]; - return createEc2PoolProvider({ + return { environment: process.env.ENVIRONMENT, subnets: process.env.SUBNET_IDS.split(','), launchTemplateName: process.env.LAUNCH_TEMPLATE_NAME, @@ -42,10 +42,12 @@ export function createEc2PoolProviderFromEnv(): PoolRunnerProvider { ? (JSON.parse(process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS) as [string]) : [], scaleErrors, - }); + }; } -function createEc2PoolProvider(config: Ec2PoolProviderConfig): PoolRunnerProvider { +export function createEc2PoolProvider(): PoolRunnerProvider { + const config = loadEc2PoolProviderConfig(); + async function listEc2PoolRunners({ environment, runnerOwner, diff --git a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts index 4cc05ac6b4..08c659dc13 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts @@ -1,6 +1,6 @@ -import { createEc2PoolProviderFromEnv } from './ec2-pool'; +import { createEc2PoolProvider } from './ec2-pool'; import type { PoolRunnerProvider } from './pool-provider'; export function createPoolRunnerProviderFromEnv(): PoolRunnerProvider { - return createEc2PoolProviderFromEnv(); + return createEc2PoolProvider(); } From 7496329b8546fdcc7391779f3a0c6783355b86a9 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 22:35:56 +0200 Subject: [PATCH 29/46] refactor(pool): map provider factories --- .../control-plane/src/pool/pool-provider-registry.ts | 12 +++++++++--- lambdas/functions/control-plane/src/pool/pool.ts | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts index 08c659dc13..adae6c3cf8 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts @@ -1,6 +1,12 @@ import { createEc2PoolProvider } from './ec2-pool'; -import type { PoolRunnerProvider } from './pool-provider'; +import type { PoolRunnerProvider, PoolRunnerProviderType } from './pool-provider'; -export function createPoolRunnerProviderFromEnv(): PoolRunnerProvider { - return createEc2PoolProvider(); +type PoolRunnerProviderFactory = () => PoolRunnerProvider; + +const poolRunnerProviderFactories: Record = { + ec2: createEc2PoolProvider, +}; + +export function createPoolRunnerProviderFromEnv(type: PoolRunnerProviderType): PoolRunnerProvider { + return poolRunnerProviderFactories[type](); } diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 415cf25124..cedc1b2065 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -36,7 +36,7 @@ export async function adjust(event: PoolEvent): Promise { // when unset so the pool keeps its previous behavior on stacks that do not provide the variable. const maximumRunners = parseInt(process.env.RUNNERS_MAXIMUM_COUNT || '-1'); const includeBusyRunners = yn(process.env.INCLUDE_BUSY_RUNNERS, { default: false }); - const runnerProvider = createPoolRunnerProviderFromEnv(); + const runnerProvider = createPoolRunnerProviderFromEnv(runnerProviderType); const { ghesApiUrl, ghesBaseUrl } = getGitHubEnterpriseApiUrl(); From c2f5fcafaa9dd8917326f7f6895f2963e0940164 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 22:38:07 +0200 Subject: [PATCH 30/46] refactor(pool): remove provider type alias --- .../control-plane/src/pool/pool-provider-registry.ts | 7 ++++--- lambdas/functions/control-plane/src/pool/pool-provider.ts | 4 +--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts index adae6c3cf8..fd1ebb5594 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts @@ -1,12 +1,13 @@ +import type { RunnerProviderType } from '../runner-provider'; import { createEc2PoolProvider } from './ec2-pool'; -import type { PoolRunnerProvider, PoolRunnerProviderType } from './pool-provider'; +import type { PoolRunnerProvider } from './pool-provider'; type PoolRunnerProviderFactory = () => PoolRunnerProvider; -const poolRunnerProviderFactories: Record = { +const poolRunnerProviderFactories: Record = { ec2: createEc2PoolProvider, }; -export function createPoolRunnerProviderFromEnv(type: PoolRunnerProviderType): PoolRunnerProvider { +export function createPoolRunnerProviderFromEnv(type: RunnerProviderType): PoolRunnerProvider { return poolRunnerProviderFactories[type](); } diff --git a/lambdas/functions/control-plane/src/pool/pool-provider.ts b/lambdas/functions/control-plane/src/pool/pool-provider.ts index ff17f3eea6..1ca2c975ee 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider.ts @@ -1,10 +1,8 @@ import type { Octokit } from '@octokit/rest'; -import type { RunnerProvider, RunnerProviderType } from '../runner-provider'; +import type { RunnerProvider } from '../runner-provider'; import type { CreateGitHubRunnerConfig, GitHubRunnerType } from '../scale-runners/types'; -export type PoolRunnerProviderType = RunnerProviderType; - export interface RunnerStatus { busy: boolean; status: string; From 88f1791540816edae556bef024f6ac038cdbcf55 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 22:40:48 +0200 Subject: [PATCH 31/46] refactor(runners): derive provider type values --- lambdas/functions/control-plane/src/runner-provider.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lambdas/functions/control-plane/src/runner-provider.ts b/lambdas/functions/control-plane/src/runner-provider.ts index fb784a54c9..e121e6ff8f 100644 --- a/lambdas/functions/control-plane/src/runner-provider.ts +++ b/lambdas/functions/control-plane/src/runner-provider.ts @@ -1,8 +1,13 @@ // TODO: Add MicroVM when its control-plane provider implementations are available. -export type RunnerProviderType = 'ec2'; +const runnerProviderTypes = ['ec2'] as const; +export type RunnerProviderType = (typeof runnerProviderTypes)[number]; const defaultRunnerProvider: RunnerProviderType = 'ec2'; +function isRunnerProviderType(type: string): type is RunnerProviderType { + return runnerProviderTypes.some((runnerProviderType) => runnerProviderType === type); +} + export function resolveRunnerProviderType(type: unknown): RunnerProviderType { if (type === undefined) return defaultRunnerProvider; if (typeof type !== 'string') { @@ -11,7 +16,7 @@ export function resolveRunnerProviderType(type: unknown): RunnerProviderType { const normalizedType = type.trim().toLowerCase(); if (!normalizedType) return defaultRunnerProvider; - if (normalizedType !== 'ec2') { + if (!isRunnerProviderType(normalizedType)) { throw new Error(`Unsupported runner provider type '${type}'`); } From 09f43b6a7fa8dee0bc6b4032158f9b2acc97f5db Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 22:43:28 +0200 Subject: [PATCH 32/46] refactor(pool): rename provider factory --- .../control-plane/src/pool/pool-provider-registry.ts | 2 +- lambdas/functions/control-plane/src/pool/pool.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts index fd1ebb5594..f3d0c2bb24 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts @@ -8,6 +8,6 @@ const poolRunnerProviderFactories: Record { const runnerProviderType = resolveRunnerProviderType(event.type); + const runnerProvider = createPoolRunnerProvider(runnerProviderType); logger.info(`Checking current ${runnerProviderType} pool size against pool of size: ${event.poolSize}`); const runnerLabels = process.env.RUNNER_LABELS || ''; const runnerGroup = process.env.RUNNER_GROUP_NAME || ''; @@ -36,7 +37,6 @@ export async function adjust(event: PoolEvent): Promise { // when unset so the pool keeps its previous behavior on stacks that do not provide the variable. const maximumRunners = parseInt(process.env.RUNNERS_MAXIMUM_COUNT || '-1'); const includeBusyRunners = yn(process.env.INCLUDE_BUSY_RUNNERS, { default: false }); - const runnerProvider = createPoolRunnerProviderFromEnv(runnerProviderType); const { ghesApiUrl, ghesBaseUrl } = getGitHubEnterpriseApiUrl(); From 506be5547c38ecd2a684c71d924da642eb03518f Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 22:47:07 +0200 Subject: [PATCH 33/46] refactor(pool): remove redundant provider type --- lambdas/functions/control-plane/src/pool/ec2-pool.ts | 1 - lambdas/functions/control-plane/src/pool/pool-provider.ts | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/ec2-pool.ts b/lambdas/functions/control-plane/src/pool/ec2-pool.ts index 077ceaa0b5..0a35c71bd6 100644 --- a/lambdas/functions/control-plane/src/pool/ec2-pool.ts +++ b/lambdas/functions/control-plane/src/pool/ec2-pool.ts @@ -85,7 +85,6 @@ export function createEc2PoolProvider(): PoolRunnerProvider { } return { - type: 'ec2', listRunners: listEc2PoolRunners, countAvailableRunners: calculateEc2PoolSize, createRunners: createEc2PoolRunners, diff --git a/lambdas/functions/control-plane/src/pool/pool-provider.ts b/lambdas/functions/control-plane/src/pool/pool-provider.ts index 1ca2c975ee..ebe3f2272d 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider.ts @@ -1,6 +1,5 @@ import type { Octokit } from '@octokit/rest'; -import type { RunnerProvider } from '../runner-provider'; import type { CreateGitHubRunnerConfig, GitHubRunnerType } from '../scale-runners/types'; export interface RunnerStatus { @@ -20,7 +19,7 @@ export interface CreatePoolRunnersInput { githubInstallationClient: Octokit; } -export interface PoolRunnerProvider extends RunnerProvider { +export interface PoolRunnerProvider { listRunners(input: ListPoolRunnersInput): Promise; countAvailableRunners( runners: TRunner[], From 5299d513df78daa8188ac456adf32eaed2647d50 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 22:48:56 +0200 Subject: [PATCH 34/46] refactor(pool): inline scale errors config --- lambdas/functions/control-plane/src/pool/ec2-pool.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/ec2-pool.ts b/lambdas/functions/control-plane/src/pool/ec2-pool.ts index 0a35c71bd6..a87d392ca4 100644 --- a/lambdas/functions/control-plane/src/pool/ec2-pool.ts +++ b/lambdas/functions/control-plane/src/pool/ec2-pool.ts @@ -21,8 +21,6 @@ interface Ec2PoolProviderConfig { } function loadEc2PoolProviderConfig(): Ec2PoolProviderConfig { - const scaleErrors = JSON.parse(process.env.SCALE_ERRORS) as [string]; - return { environment: process.env.ENVIRONMENT, subnets: process.env.SUBNET_IDS.split(','), @@ -41,7 +39,7 @@ function loadEc2PoolProviderConfig(): Ec2PoolProviderConfig { onDemandFailoverOnError: process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS ? (JSON.parse(process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS) as [string]) : [], - scaleErrors, + scaleErrors: JSON.parse(process.env.SCALE_ERRORS) as [string], }; } From 870ce90d8b1b4528d646b2df115c6f4214932da7 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 22:53:00 +0200 Subject: [PATCH 35/46] refactor(pool): move provider functions to module scope --- .../control-plane/src/pool/ec2-pool.ts | 63 +++++++++---------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/ec2-pool.ts b/lambdas/functions/control-plane/src/pool/ec2-pool.ts index a87d392ca4..6c4883773d 100644 --- a/lambdas/functions/control-plane/src/pool/ec2-pool.ts +++ b/lambdas/functions/control-plane/src/pool/ec2-pool.ts @@ -43,49 +43,48 @@ function loadEc2PoolProviderConfig(): Ec2PoolProviderConfig { }; } -export function createEc2PoolProvider(): PoolRunnerProvider { - const config = loadEc2PoolProviderConfig(); - - async function listEc2PoolRunners({ +async function listEc2PoolRunners({ + environment, + runnerOwner, + runnerType, +}: ListPoolRunnersInput): Promise { + return await listEC2Runners({ environment, runnerOwner, runnerType, - }: ListPoolRunnersInput): Promise { - return await listEC2Runners({ - environment, - runnerOwner, - runnerType, - statuses: ['running'], - }); - } + statuses: ['running'], + }); +} - async function createEc2PoolRunners({ +async function createEc2PoolRunners( + config: Ec2PoolProviderConfig, + { githubRunnerConfig, numberOfRunners, githubInstallationClient }: CreatePoolRunnersInput, +): Promise { + return await createRunners( githubRunnerConfig, + { + ec2instanceCriteria: config.ec2instanceCriteria, + environment: config.environment, + launchTemplateName: config.launchTemplateName, + subnets: config.subnets, + amiIdSsmParameterName: config.amiIdSsmParameterName, + tracingEnabled: config.tracingEnabled, + onDemandFailoverOnError: config.onDemandFailoverOnError, + scaleErrors: config.scaleErrors, + }, numberOfRunners, githubInstallationClient, - }: CreatePoolRunnersInput): Promise { - return await createRunners( - githubRunnerConfig, - { - ec2instanceCriteria: config.ec2instanceCriteria, - environment: config.environment, - launchTemplateName: config.launchTemplateName, - subnets: config.subnets, - amiIdSsmParameterName: config.amiIdSsmParameterName, - tracingEnabled: config.tracingEnabled, - onDemandFailoverOnError: config.onDemandFailoverOnError, - scaleErrors: config.scaleErrors, - }, - numberOfRunners, - githubInstallationClient, - 'pool-lambda', - ); - } + 'pool-lambda', + ); +} + +export function createEc2PoolProvider(): PoolRunnerProvider { + const config = loadEc2PoolProviderConfig(); return { listRunners: listEc2PoolRunners, countAvailableRunners: calculateEc2PoolSize, - createRunners: createEc2PoolRunners, + createRunners: createEc2PoolRunners.bind(undefined, config), }; } From f6fc43de9feefc3993018670668b525122d059e0 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 22:56:16 +0200 Subject: [PATCH 36/46] refactor(pool): expose provider identity --- lambdas/functions/control-plane/src/pool/ec2-pool.ts | 1 + lambdas/functions/control-plane/src/pool/pool-provider.ts | 3 ++- lambdas/functions/control-plane/src/pool/pool.ts | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/ec2-pool.ts b/lambdas/functions/control-plane/src/pool/ec2-pool.ts index 6c4883773d..15042d5a1b 100644 --- a/lambdas/functions/control-plane/src/pool/ec2-pool.ts +++ b/lambdas/functions/control-plane/src/pool/ec2-pool.ts @@ -82,6 +82,7 @@ export function createEc2PoolProvider(): PoolRunnerProvider { const config = loadEc2PoolProviderConfig(); return { + type: 'ec2', listRunners: listEc2PoolRunners, countAvailableRunners: calculateEc2PoolSize, createRunners: createEc2PoolRunners.bind(undefined, config), diff --git a/lambdas/functions/control-plane/src/pool/pool-provider.ts b/lambdas/functions/control-plane/src/pool/pool-provider.ts index ebe3f2272d..1ca2c975ee 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider.ts @@ -1,5 +1,6 @@ import type { Octokit } from '@octokit/rest'; +import type { RunnerProvider } from '../runner-provider'; import type { CreateGitHubRunnerConfig, GitHubRunnerType } from '../scale-runners/types'; export interface RunnerStatus { @@ -19,7 +20,7 @@ export interface CreatePoolRunnersInput { githubInstallationClient: Octokit; } -export interface PoolRunnerProvider { +export interface PoolRunnerProvider extends RunnerProvider { listRunners(input: ListPoolRunnersInput): Promise; countAvailableRunners( runners: TRunner[], diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 2bd9e7425b..3788aaa0d1 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -18,7 +18,7 @@ export interface PoolEvent { export async function adjust(event: PoolEvent): Promise { const runnerProviderType = resolveRunnerProviderType(event.type); const runnerProvider = createPoolRunnerProvider(runnerProviderType); - logger.info(`Checking current ${runnerProviderType} pool size against pool of size: ${event.poolSize}`); + logger.info(`Checking current ${runnerProvider.type} pool size against pool of size: ${event.poolSize}`); const runnerLabels = process.env.RUNNER_LABELS || ''; const runnerGroup = process.env.RUNNER_GROUP_NAME || ''; const runnerNamePrefix = process.env.RUNNER_NAME_PREFIX || ''; From 46be26ce6437839abc2985533659ab8375cecb8e Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 23:00:49 +0200 Subject: [PATCH 37/46] refactor(pool): load config when creating runners --- .../functions/control-plane/src/pool/ec2-pool.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/ec2-pool.ts b/lambdas/functions/control-plane/src/pool/ec2-pool.ts index 15042d5a1b..0d574e1e08 100644 --- a/lambdas/functions/control-plane/src/pool/ec2-pool.ts +++ b/lambdas/functions/control-plane/src/pool/ec2-pool.ts @@ -56,10 +56,13 @@ async function listEc2PoolRunners({ }); } -async function createEc2PoolRunners( - config: Ec2PoolProviderConfig, - { githubRunnerConfig, numberOfRunners, githubInstallationClient }: CreatePoolRunnersInput, -): Promise { +async function createEc2PoolRunners({ + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, +}: CreatePoolRunnersInput): Promise { + const config = loadEc2PoolProviderConfig(); + return await createRunners( githubRunnerConfig, { @@ -79,13 +82,11 @@ async function createEc2PoolRunners( } export function createEc2PoolProvider(): PoolRunnerProvider { - const config = loadEc2PoolProviderConfig(); - return { type: 'ec2', listRunners: listEc2PoolRunners, countAvailableRunners: calculateEc2PoolSize, - createRunners: createEc2PoolRunners.bind(undefined, config), + createRunners: createEc2PoolRunners, }; } From 0698b34a6fa1e9c3ccfa0eeb61e1eda3057596f6 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 23:08:57 +0200 Subject: [PATCH 38/46] refactor(scale-runners): simplify ec2 providers --- .../src/scale-runners/ec2-scale-down.ts | 18 ++- .../src/scale-runners/ec2-scale-up.ts | 118 ++++++++++-------- .../scale-down-provider-registry.ts | 11 +- .../src/scale-runners/scale-down-provider.ts | 4 +- .../src/scale-runners/scale-down.ts | 6 +- .../scale-up-provider-registry.ts | 13 +- .../src/scale-runners/scale-up-provider.ts | 4 +- .../src/scale-runners/scale-up.ts | 8 +- 8 files changed, 112 insertions(+), 70 deletions(-) diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts index bc914e84b2..11c85cd218 100644 --- a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts @@ -2,13 +2,25 @@ import { bootTimeExceeded, listEC2Runners, tag, terminateRunner, untag } from '. import type { RunnerList } from './../aws/ec2-runners.d'; import type { RunnerList as ScaleDownRunnerList, ScaleDownRunnerProvider } from './scale-down-provider'; +async function listEc2ScaleDownRunners(environment: string, orphan?: boolean): Promise { + return (await listEC2Runners({ environment, orphan })).map(toScaleDownRunner); +} + +async function markEc2RunnerOrphan(id: string): Promise { + await tag(id, [{ Key: 'ghr:orphan', Value: 'true' }]); +} + +async function unmarkEc2RunnerOrphan(id: string): Promise { + await untag(id, [{ Key: 'ghr:orphan', Value: 'true' }]); +} + export function createEc2ScaleDownProvider(): ScaleDownRunnerProvider { return { type: 'ec2', - list: async (environment, orphan) => (await listEC2Runners({ environment, orphan })).map(toScaleDownRunner), + list: listEc2ScaleDownRunners, bootTimeExceeded, - markOrphan: async (id) => await tag(id, [{ Key: 'ghr:orphan', Value: 'true' }]), - unmarkOrphan: async (id) => await untag(id, [{ Key: 'ghr:orphan', Value: 'true' }]), + markOrphan: markEc2RunnerOrphan, + unmarkOrphan: unmarkEc2RunnerOrphan, terminate: terminateRunner, }; } diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts index c10158678d..9e503d2d95 100644 --- a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts @@ -10,7 +10,12 @@ import { } from './ec2-labels'; import { createRunners } from './ec2'; import type { CreateEC2RunnerConfig } from './ec2'; -import type { ScaleUpRunnerProvider } from './scale-up-provider'; +import type { + CreateScaleUpRunnersInput, + CurrentRunnersInput, + PreparedScaleUpRunnerGroup, + ScaleUpRunnerProvider, +} from './scale-up-provider'; const logger = createChildLogger('ec2-scale-up'); @@ -20,52 +25,8 @@ interface Ec2ScaleUpState { ec2OverrideConfig?: Ec2OverrideConfig; } -export function createEc2ScaleUpProvider(config: Ec2ScaleUpProviderConfig): ScaleUpRunnerProvider { +function loadEc2ScaleUpProviderConfig(): Ec2ScaleUpProviderConfig { return { - type: 'ec2', - prepareGroup: async (messageLabels) => { - const trimmedLabels = messageLabels.map((label) => label.trim()); - const dynamicEC2Labels = trimmedLabels.filter((label) => label.startsWith('ghr-ec2-')); - const nonEc2DynamicLabels = trimmedLabels.filter( - (label) => label.startsWith('ghr-') && !label.startsWith('ghr-ec2-'), - ); - const runnerLabels = [...nonEc2DynamicLabels, ...dynamicEC2Labels]; - let ec2OverrideConfig: Ec2OverrideConfig | undefined; - - if (dynamicEC2Labels.length > 0) { - const defaultBlockDeviceName = shouldLoadLaunchTemplateBlockDeviceName(dynamicEC2Labels) - ? await getDefaultBlockDeviceNameFromLaunchTemplate(config.launchTemplateName) - : undefined; - - ec2OverrideConfig = parseEc2OverrideConfig(dynamicEC2Labels, defaultBlockDeviceName); - if (ec2OverrideConfig) { - logger.debug('EC2 override config parsed from labels', { ec2OverrideConfig }); - } - } - - return { runnerLabels, state: { ec2OverrideConfig } }; - }, - getCurrentRunners: async (_state, { runnerType, runnerOwner }) => - (await listEC2Runners({ environment: config.environment, runnerType, runnerOwner })).length, - createRunners: async ({ githubRunnerConfig, numberOfRunners, githubInstallationClient, state }) => - await createRunners( - githubRunnerConfig, - { - ...config, - ec2OverrideConfig: state.ec2OverrideConfig, - }, - numberOfRunners, - githubInstallationClient, - 'scale-up-lambda', - ), - }; -} - -export function createEc2ScaleUpProviderFromEnv( - environment: string, - scaleErrors: string[], -): ScaleUpRunnerProvider { - return createEc2ScaleUpProvider({ ec2instanceCriteria: { instanceTypes: process.env.INSTANCE_TYPES.split(','), instanceTypePriorities: process.env.INSTANCE_TYPE_PRIORITIES @@ -75,7 +36,7 @@ export function createEc2ScaleUpProviderFromEnv( maxSpotPrice: process.env.INSTANCE_MAX_SPOT_PRICE, instanceAllocationStrategy: process.env.INSTANCE_ALLOCATION_STRATEGY || 'lowest-price', }, - environment, + environment: process.env.ENVIRONMENT, launchTemplateName: process.env.LAUNCH_TEMPLATE_NAME, subnets: process.env.SUBNET_IDS.split(','), amiIdSsmParameterName: process.env.AMI_ID_SSM_PARAMETER_NAME, @@ -83,7 +44,66 @@ export function createEc2ScaleUpProviderFromEnv( onDemandFailoverOnError: process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS ? (JSON.parse(process.env.ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS) as [string]) : [], - scaleErrors, + scaleErrors: JSON.parse(process.env.SCALE_ERRORS) as [string], useDedicatedHost: yn(process.env.USE_DEDICATED_HOST, { default: false }), - }); + }; +} + +async function prepareEc2ScaleUpGroup(messageLabels: string[]): Promise> { + const trimmedLabels = messageLabels.map((label) => label.trim()); + const dynamicEC2Labels = trimmedLabels.filter((label) => label.startsWith('ghr-ec2-')); + const nonEc2DynamicLabels = trimmedLabels.filter( + (label) => label.startsWith('ghr-') && !label.startsWith('ghr-ec2-'), + ); + const runnerLabels = [...nonEc2DynamicLabels, ...dynamicEC2Labels]; + let ec2OverrideConfig: Ec2OverrideConfig | undefined; + + if (dynamicEC2Labels.length > 0) { + const defaultBlockDeviceName = shouldLoadLaunchTemplateBlockDeviceName(dynamicEC2Labels) + ? await getDefaultBlockDeviceNameFromLaunchTemplate(process.env.LAUNCH_TEMPLATE_NAME) + : undefined; + + ec2OverrideConfig = parseEc2OverrideConfig(dynamicEC2Labels, defaultBlockDeviceName); + if (ec2OverrideConfig) { + logger.debug('EC2 override config parsed from labels', { ec2OverrideConfig }); + } + } + + return { runnerLabels, state: { ec2OverrideConfig } }; +} + +async function getCurrentEc2Runners( + _state: Ec2ScaleUpState, + { runnerType, runnerOwner }: CurrentRunnersInput, +): Promise { + return (await listEC2Runners({ environment: process.env.ENVIRONMENT, runnerType, runnerOwner })).length; +} + +async function createEc2ScaleUpRunners({ + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + state, +}: CreateScaleUpRunnersInput): Promise { + const config = loadEc2ScaleUpProviderConfig(); + + return await createRunners( + githubRunnerConfig, + { + ...config, + ec2OverrideConfig: state.ec2OverrideConfig, + }, + numberOfRunners, + githubInstallationClient, + 'scale-up-lambda', + ); +} + +export function createEc2ScaleUpProvider(): ScaleUpRunnerProvider { + return { + type: 'ec2', + prepareGroup: prepareEc2ScaleUpGroup, + getCurrentRunners: getCurrentEc2Runners, + createRunners: createEc2ScaleUpRunners, + }; } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts index 859951f1c9..67140b0d39 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts @@ -1,6 +1,13 @@ +import type { RunnerProviderType } from '../runner-provider'; import { createEc2ScaleDownProvider } from './ec2-scale-down'; import type { ScaleDownRunnerProvider } from './scale-down-provider'; -export function createScaleDownRunnerProviderFromEnv(): ScaleDownRunnerProvider { - return createEc2ScaleDownProvider(); +type ScaleDownRunnerProviderFactory = () => ScaleDownRunnerProvider; + +const scaleDownRunnerProviderFactories: Record = { + ec2: createEc2ScaleDownProvider, +}; + +export function createScaleDownRunnerProvider(type: RunnerProviderType): ScaleDownRunnerProvider { + return scaleDownRunnerProviderFactories[type](); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts index 848e1e1392..c614ffcff5 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts @@ -1,4 +1,4 @@ -import type { RunnerProvider, RunnerProviderType } from '../runner-provider'; +import type { RunnerProvider } from '../runner-provider'; export interface RunnerList { id: string; @@ -17,8 +17,6 @@ export interface RunnerInfo extends RunnerList { type: string; } -export type ScaleDownRunnerProviderType = RunnerProviderType; - export interface ScaleDownRunnerProvider extends RunnerProvider { list(environment: string, orphan?: boolean): Promise; bootTimeExceeded(runner: RunnerInfo): boolean; diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index 3471a2a675..fa70329e3d 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -5,9 +5,10 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import moment from 'moment'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; +import { resolveRunnerProviderType } from '../runner-provider'; import { GhRunners, githubCache } from './cache'; import { ScalingDownConfigList, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config'; -import { createScaleDownRunnerProviderFromEnv } from './scale-down-provider-registry'; +import { createScaleDownRunnerProvider } from './scale-down-provider-registry'; import { metricGitHubAppRateLimit } from '../github/rate-limit'; import { getGitHubEnterpriseApiUrl } from './github-runner'; import type { RunnerInfo, RunnerList, ScaleDownRunnerProvider } from './scale-down-provider'; @@ -350,7 +351,8 @@ export async function scaleDown(): Promise { githubCache.reset(); const environment = process.env.ENVIRONMENT; const scaleDownConfigs = JSON.parse(process.env.SCALE_DOWN_CONFIG) as ScalingDownConfigList; - const runnerProvider = createScaleDownRunnerProviderFromEnv(); + const runnerProviderType = resolveRunnerProviderType(process.env.RUNNER_PROVIDER_TYPE); + const runnerProvider = createScaleDownRunnerProvider(runnerProviderType); // first runners marked to be orphan. await terminateOrphan(environment, runnerProvider); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts index c73cb9fe94..4a70775a7c 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts @@ -1,6 +1,13 @@ -import { createEc2ScaleUpProviderFromEnv } from './ec2-scale-up'; +import type { RunnerProviderType } from '../runner-provider'; +import { createEc2ScaleUpProvider } from './ec2-scale-up'; import type { ScaleUpRunnerProvider } from './scale-up-provider'; -export function createScaleUpRunnerProviderFromEnv(environment: string, scaleErrors: string[]): ScaleUpRunnerProvider { - return createEc2ScaleUpProviderFromEnv(environment, scaleErrors); +type ScaleUpRunnerProviderFactory = () => ScaleUpRunnerProvider; + +const scaleUpRunnerProviderFactories: Record = { + ec2: createEc2ScaleUpProvider, +}; + +export function createScaleUpRunnerProvider(type: RunnerProviderType): ScaleUpRunnerProvider { + return scaleUpRunnerProviderFactories[type](); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts index 04c6beedc3..1956151892 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts @@ -1,6 +1,6 @@ import type { Octokit } from '@octokit/rest'; -import type { RunnerProvider, RunnerProviderType } from '../runner-provider'; +import type { RunnerProvider } from '../runner-provider'; import type { ActionRequestMessageSQS, CreateGitHubRunnerConfig, GitHubRunnerType } from './types'; export interface CurrentRunnersInput { @@ -21,8 +21,6 @@ export interface PreparedScaleUpRunnerGroup { state: TState; } -export type ScaleUpRunnerProviderType = RunnerProviderType; - export interface ScaleUpRunnerProvider extends RunnerProvider { prepareGroup(messageLabels: string[]): Promise>; getCurrentRunners(state: TState, input: CurrentRunnersInput): Promise; diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 3f87d8fda9..0dddcc406d 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -12,7 +12,7 @@ import { validateSsmParameterStoreTags, } from './github-runner'; import { publishRetryMessage } from './job-retry'; -import { createScaleUpRunnerProviderFromEnv } from './scale-up-provider-registry'; +import { createScaleUpRunnerProvider } from './scale-up-provider-registry'; import type { ActionRequestMessage, ActionRequestMessageRetry, @@ -73,7 +73,6 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise Date: Tue, 14 Jul 2026 23:12:52 +0200 Subject: [PATCH 39/46] refactor(runners): assign provider identity in registries --- lambdas/functions/control-plane/src/pool/ec2-pool.ts | 3 +-- .../control-plane/src/pool/pool-provider-registry.ts | 4 ++-- .../control-plane/src/scale-runners/ec2-scale-down.ts | 3 +-- .../functions/control-plane/src/scale-runners/ec2-scale-up.ts | 3 +-- .../src/scale-runners/scale-down-provider-registry.ts | 4 ++-- .../src/scale-runners/scale-up-provider-registry.ts | 4 ++-- 6 files changed, 9 insertions(+), 12 deletions(-) diff --git a/lambdas/functions/control-plane/src/pool/ec2-pool.ts b/lambdas/functions/control-plane/src/pool/ec2-pool.ts index 0d574e1e08..84cde7aa9f 100644 --- a/lambdas/functions/control-plane/src/pool/ec2-pool.ts +++ b/lambdas/functions/control-plane/src/pool/ec2-pool.ts @@ -81,9 +81,8 @@ async function createEc2PoolRunners({ ); } -export function createEc2PoolProvider(): PoolRunnerProvider { +export function createEc2PoolProvider(): Omit { return { - type: 'ec2', listRunners: listEc2PoolRunners, countAvailableRunners: calculateEc2PoolSize, createRunners: createEc2PoolRunners, diff --git a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts index f3d0c2bb24..4ec10c2bd9 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts @@ -2,12 +2,12 @@ import type { RunnerProviderType } from '../runner-provider'; import { createEc2PoolProvider } from './ec2-pool'; import type { PoolRunnerProvider } from './pool-provider'; -type PoolRunnerProviderFactory = () => PoolRunnerProvider; +type PoolRunnerProviderFactory = () => Omit; const poolRunnerProviderFactories: Record = { ec2: createEc2PoolProvider, }; export function createPoolRunnerProvider(type: RunnerProviderType): PoolRunnerProvider { - return poolRunnerProviderFactories[type](); + return { ...poolRunnerProviderFactories[type](), type }; } diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts index 11c85cd218..554ea375b7 100644 --- a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-down.ts @@ -14,9 +14,8 @@ async function unmarkEc2RunnerOrphan(id: string): Promise { await untag(id, [{ Key: 'ghr:orphan', Value: 'true' }]); } -export function createEc2ScaleDownProvider(): ScaleDownRunnerProvider { +export function createEc2ScaleDownProvider(): Omit { return { - type: 'ec2', list: listEc2ScaleDownRunners, bootTimeExceeded, markOrphan: markEc2RunnerOrphan, diff --git a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts index 9e503d2d95..b79761f2a2 100644 --- a/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/ec2-scale-up.ts @@ -99,9 +99,8 @@ async function createEc2ScaleUpRunners({ ); } -export function createEc2ScaleUpProvider(): ScaleUpRunnerProvider { +export function createEc2ScaleUpProvider(): Omit, 'type'> { return { - type: 'ec2', prepareGroup: prepareEc2ScaleUpGroup, getCurrentRunners: getCurrentEc2Runners, createRunners: createEc2ScaleUpRunners, diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts index 67140b0d39..cdeae740db 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts @@ -2,12 +2,12 @@ import type { RunnerProviderType } from '../runner-provider'; import { createEc2ScaleDownProvider } from './ec2-scale-down'; import type { ScaleDownRunnerProvider } from './scale-down-provider'; -type ScaleDownRunnerProviderFactory = () => ScaleDownRunnerProvider; +type ScaleDownRunnerProviderFactory = () => Omit; const scaleDownRunnerProviderFactories: Record = { ec2: createEc2ScaleDownProvider, }; export function createScaleDownRunnerProvider(type: RunnerProviderType): ScaleDownRunnerProvider { - return scaleDownRunnerProviderFactories[type](); + return { ...scaleDownRunnerProviderFactories[type](), type }; } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts index 4a70775a7c..2277accc01 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts @@ -2,12 +2,12 @@ import type { RunnerProviderType } from '../runner-provider'; import { createEc2ScaleUpProvider } from './ec2-scale-up'; import type { ScaleUpRunnerProvider } from './scale-up-provider'; -type ScaleUpRunnerProviderFactory = () => ScaleUpRunnerProvider; +type ScaleUpRunnerProviderFactory = () => Omit; const scaleUpRunnerProviderFactories: Record = { ec2: createEc2ScaleUpProvider, }; export function createScaleUpRunnerProvider(type: RunnerProviderType): ScaleUpRunnerProvider { - return scaleUpRunnerProviderFactories[type](); + return { ...scaleUpRunnerProviderFactories[type](), type }; } From ef544f84e6fdff90d0f8f5c20e801baa36bf3bb0 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 23:22:23 +0200 Subject: [PATCH 40/46] refactor(runners): consolidate provider registry --- .../src/pool/pool-provider-registry.ts | 13 -------- .../functions/control-plane/src/pool/pool.ts | 2 +- .../src/runner-provider-registry.ts | 33 +++++++++++++++++++ .../scale-down-provider-registry.ts | 13 -------- .../src/scale-runners/scale-down.ts | 2 +- .../scale-up-provider-registry.ts | 13 -------- .../src/scale-runners/scale-up.ts | 2 +- 7 files changed, 36 insertions(+), 42 deletions(-) delete mode 100644 lambdas/functions/control-plane/src/pool/pool-provider-registry.ts create mode 100644 lambdas/functions/control-plane/src/runner-provider-registry.ts delete mode 100644 lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts delete mode 100644 lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts diff --git a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts b/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts deleted file mode 100644 index 4ec10c2bd9..0000000000 --- a/lambdas/functions/control-plane/src/pool/pool-provider-registry.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { RunnerProviderType } from '../runner-provider'; -import { createEc2PoolProvider } from './ec2-pool'; -import type { PoolRunnerProvider } from './pool-provider'; - -type PoolRunnerProviderFactory = () => Omit; - -const poolRunnerProviderFactories: Record = { - ec2: createEc2PoolProvider, -}; - -export function createPoolRunnerProvider(type: RunnerProviderType): PoolRunnerProvider { - return { ...poolRunnerProviderFactories[type](), type }; -} diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 3788aaa0d1..f59d861b74 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -3,9 +3,9 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import yn from 'yn'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; +import { createPoolRunnerProvider } from '../runner-provider-registry'; import { resolveRunnerProviderType } from '../runner-provider'; import { getGitHubEnterpriseApiUrl, validateSsmParameterStoreTags } from '../scale-runners/github-runner'; -import { createPoolRunnerProvider } from './pool-provider-registry'; import type { RunnerStatus } from './pool-provider'; const logger = createChildLogger('pool'); diff --git a/lambdas/functions/control-plane/src/runner-provider-registry.ts b/lambdas/functions/control-plane/src/runner-provider-registry.ts new file mode 100644 index 0000000000..0013861378 --- /dev/null +++ b/lambdas/functions/control-plane/src/runner-provider-registry.ts @@ -0,0 +1,33 @@ +import { createEc2PoolProvider } from './pool/ec2-pool'; +import type { PoolRunnerProvider } from './pool/pool-provider'; +import type { RunnerProviderType } from './runner-provider'; +import { createEc2ScaleDownProvider } from './scale-runners/ec2-scale-down'; +import { createEc2ScaleUpProvider } from './scale-runners/ec2-scale-up'; +import type { ScaleDownRunnerProvider } from './scale-runners/scale-down-provider'; +import type { ScaleUpRunnerProvider } from './scale-runners/scale-up-provider'; + +interface RunnerProviderFactory { + pool: () => Omit; + scaleUp: () => Omit; + scaleDown: () => Omit; +} + +const runnerProviderFactories: Record = { + ec2: { + pool: createEc2PoolProvider, + scaleUp: createEc2ScaleUpProvider, + scaleDown: createEc2ScaleDownProvider, + }, +}; + +export function createPoolRunnerProvider(type: RunnerProviderType): PoolRunnerProvider { + return { ...runnerProviderFactories[type].pool(), type }; +} + +export function createScaleUpRunnerProvider(type: RunnerProviderType): ScaleUpRunnerProvider { + return { ...runnerProviderFactories[type].scaleUp(), type }; +} + +export function createScaleDownRunnerProvider(type: RunnerProviderType): ScaleDownRunnerProvider { + return { ...runnerProviderFactories[type].scaleDown(), type }; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts deleted file mode 100644 index cdeae740db..0000000000 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider-registry.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { RunnerProviderType } from '../runner-provider'; -import { createEc2ScaleDownProvider } from './ec2-scale-down'; -import type { ScaleDownRunnerProvider } from './scale-down-provider'; - -type ScaleDownRunnerProviderFactory = () => Omit; - -const scaleDownRunnerProviderFactories: Record = { - ec2: createEc2ScaleDownProvider, -}; - -export function createScaleDownRunnerProvider(type: RunnerProviderType): ScaleDownRunnerProvider { - return { ...scaleDownRunnerProviderFactories[type](), type }; -} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index fa70329e3d..2f0f7e8f41 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -5,10 +5,10 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import moment from 'moment'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; +import { createScaleDownRunnerProvider } from '../runner-provider-registry'; import { resolveRunnerProviderType } from '../runner-provider'; import { GhRunners, githubCache } from './cache'; import { ScalingDownConfigList, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config'; -import { createScaleDownRunnerProvider } from './scale-down-provider-registry'; import { metricGitHubAppRateLimit } from '../github/rate-limit'; import { getGitHubEnterpriseApiUrl } from './github-runner'; import type { RunnerInfo, RunnerList, ScaleDownRunnerProvider } from './scale-down-provider'; diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts deleted file mode 100644 index 2277accc01..0000000000 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider-registry.ts +++ /dev/null @@ -1,13 +0,0 @@ -import type { RunnerProviderType } from '../runner-provider'; -import { createEc2ScaleUpProvider } from './ec2-scale-up'; -import type { ScaleUpRunnerProvider } from './scale-up-provider'; - -type ScaleUpRunnerProviderFactory = () => Omit; - -const scaleUpRunnerProviderFactories: Record = { - ec2: createEc2ScaleUpProvider, -}; - -export function createScaleUpRunnerProvider(type: RunnerProviderType): ScaleUpRunnerProvider { - return { ...scaleUpRunnerProviderFactories[type](), type }; -} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 0dddcc406d..29fbbc9912 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -3,6 +3,7 @@ import { Octokit } from '@octokit/rest'; import yn from 'yn'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; +import { createScaleUpRunnerProvider } from '../runner-provider-registry'; import { resolveRunnerProviderType } from '../runner-provider'; import { getGitHubEnterpriseApiUrl, @@ -12,7 +13,6 @@ import { validateSsmParameterStoreTags, } from './github-runner'; import { publishRetryMessage } from './job-retry'; -import { createScaleUpRunnerProvider } from './scale-up-provider-registry'; import type { ActionRequestMessage, ActionRequestMessageRetry, From 556c2bb3ccec490dc6e605318e6996882bd02df6 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 23:35:52 +0200 Subject: [PATCH 41/46] test(runners): cover provider registry --- .../src/runner-provider-registry.test.ts | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 lambdas/functions/control-plane/src/runner-provider-registry.test.ts diff --git a/lambdas/functions/control-plane/src/runner-provider-registry.test.ts b/lambdas/functions/control-plane/src/runner-provider-registry.test.ts new file mode 100644 index 0000000000..f22de57530 --- /dev/null +++ b/lambdas/functions/control-plane/src/runner-provider-registry.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { createEc2PoolProvider } from './pool/ec2-pool'; +import type { PoolRunnerProvider } from './pool/pool-provider'; +import { + createPoolRunnerProvider, + createScaleDownRunnerProvider, + createScaleUpRunnerProvider, +} from './runner-provider-registry'; +import { createEc2ScaleDownProvider } from './scale-runners/ec2-scale-down'; +import { createEc2ScaleUpProvider } from './scale-runners/ec2-scale-up'; +import type { ScaleDownRunnerProvider } from './scale-runners/scale-down-provider'; +import type { ScaleUpRunnerProvider } from './scale-runners/scale-up-provider'; + +vi.mock('./pool/ec2-pool', () => ({ createEc2PoolProvider: vi.fn() })); +vi.mock('./scale-runners/ec2-scale-down', () => ({ createEc2ScaleDownProvider: vi.fn() })); +vi.mock('./scale-runners/ec2-scale-up', () => ({ createEc2ScaleUpProvider: vi.fn() })); + +const poolImplementation = { + listRunners: vi.fn(async () => []), + countAvailableRunners: vi.fn(() => 0), + createRunners: vi.fn(async () => []), +} satisfies Omit; + +const scaleUpImplementation = { + prepareGroup: vi.fn(async () => ({ runnerLabels: [], state: undefined })), + getCurrentRunners: vi.fn(async () => 0), + createRunners: vi.fn(async () => []), +} satisfies Omit; + +const scaleDownImplementation = { + list: vi.fn(async () => []), + bootTimeExceeded: vi.fn(() => false), + markOrphan: vi.fn(async () => undefined), + unmarkOrphan: vi.fn(async () => undefined), + terminate: vi.fn(async () => undefined), +} satisfies Omit; + +describe('runner provider registry', () => { + it('routes EC2 capabilities and injects the provider type', () => { + vi.mocked(createEc2PoolProvider).mockReturnValue(poolImplementation); + vi.mocked(createEc2ScaleUpProvider).mockReturnValue(scaleUpImplementation); + vi.mocked(createEc2ScaleDownProvider).mockReturnValue(scaleDownImplementation); + + expect(createPoolRunnerProvider('ec2')).toStrictEqual({ ...poolImplementation, type: 'ec2' }); + expect(createScaleUpRunnerProvider('ec2')).toStrictEqual({ ...scaleUpImplementation, type: 'ec2' }); + expect(createScaleDownRunnerProvider('ec2')).toStrictEqual({ ...scaleDownImplementation, type: 'ec2' }); + + expect(createEc2PoolProvider).toHaveBeenCalledTimes(1); + expect(createEc2ScaleUpProvider).toHaveBeenCalledTimes(1); + expect(createEc2ScaleDownProvider).toHaveBeenCalledTimes(1); + }); +}); From 61df834ec675204eb8cfa7ebd330ea343358b16e Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 23:42:30 +0200 Subject: [PATCH 42/46] refactor(runners): share provider type contract --- lambdas/functions/control-plane/package.json | 1 + .../control-plane/src/runner-provider.ts | 21 ++++------------- lambdas/functions/webhook/package.json | 1 + .../functions/webhook/src/runner-provider.ts | 14 ----------- .../runners/aws-dynamic-labels-provider.ts | 5 ++-- .../webhook/src/runners/aws-dynamic-labels.ts | 4 ++-- .../webhook/src/runners/dispatch.test.ts | 23 ++++++++++++------- lambdas/functions/webhook/src/sqs/index.ts | 6 ++--- lambdas/libs/runner-provider/package.json | 23 +++++++++++++++++++ .../libs/runner-provider/src/index.test.ts | 18 +++++++++++++++ lambdas/libs/runner-provider/src/index.ts | 15 ++++++++++++ lambdas/libs/runner-provider/tsconfig.json | 5 ++++ lambdas/libs/runner-provider/vitest.config.ts | 17 ++++++++++++++ lambdas/yarn.lock | 8 +++++++ 14 files changed, 116 insertions(+), 45 deletions(-) delete mode 100644 lambdas/functions/webhook/src/runner-provider.ts create mode 100644 lambdas/libs/runner-provider/package.json create mode 100644 lambdas/libs/runner-provider/src/index.test.ts create mode 100644 lambdas/libs/runner-provider/src/index.ts create mode 100644 lambdas/libs/runner-provider/tsconfig.json create mode 100644 lambdas/libs/runner-provider/vitest.config.ts diff --git a/lambdas/functions/control-plane/package.json b/lambdas/functions/control-plane/package.json index 8f978c0d17..f3a3481b50 100644 --- a/lambdas/functions/control-plane/package.json +++ b/lambdas/functions/control-plane/package.json @@ -32,6 +32,7 @@ "dependencies": { "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", + "@aws-github-runner/runner-provider": "*", "@aws-lambda-powertools/parameters": "^2.31.0", "@aws-sdk/client-ec2": "^3.1009.0", "@aws-sdk/client-sqs": "^3.1009.0", diff --git a/lambdas/functions/control-plane/src/runner-provider.ts b/lambdas/functions/control-plane/src/runner-provider.ts index e121e6ff8f..21da701804 100644 --- a/lambdas/functions/control-plane/src/runner-provider.ts +++ b/lambdas/functions/control-plane/src/runner-provider.ts @@ -1,25 +1,14 @@ -// TODO: Add MicroVM when its control-plane provider implementations are available. -const runnerProviderTypes = ['ec2'] as const; -export type RunnerProviderType = (typeof runnerProviderTypes)[number]; +import { normalizeRunnerProviderType } from '@aws-github-runner/runner-provider'; +import type { RunnerProviderType } from '@aws-github-runner/runner-provider'; -const defaultRunnerProvider: RunnerProviderType = 'ec2'; - -function isRunnerProviderType(type: string): type is RunnerProviderType { - return runnerProviderTypes.some((runnerProviderType) => runnerProviderType === type); -} +export type { RunnerProviderType } from '@aws-github-runner/runner-provider'; export function resolveRunnerProviderType(type: unknown): RunnerProviderType { - if (type === undefined) return defaultRunnerProvider; - if (typeof type !== 'string') { + const normalizedType = normalizeRunnerProviderType(type); + if (!normalizedType) { throw new Error(`Unsupported runner provider type '${String(type)}'`); } - const normalizedType = type.trim().toLowerCase(); - if (!normalizedType) return defaultRunnerProvider; - if (!isRunnerProviderType(normalizedType)) { - throw new Error(`Unsupported runner provider type '${type}'`); - } - return normalizedType; } diff --git a/lambdas/functions/webhook/package.json b/lambdas/functions/webhook/package.json index 6e30328b35..767b97532b 100644 --- a/lambdas/functions/webhook/package.json +++ b/lambdas/functions/webhook/package.json @@ -30,6 +30,7 @@ "dependencies": { "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", + "@aws-github-runner/runner-provider": "*", "@aws-sdk/client-sqs": "^3.1009.0", "@middy/core": "^6.4.5", "@octokit/rest": "22.0.1", diff --git a/lambdas/functions/webhook/src/runner-provider.ts b/lambdas/functions/webhook/src/runner-provider.ts deleted file mode 100644 index f938006a2c..0000000000 --- a/lambdas/functions/webhook/src/runner-provider.ts +++ /dev/null @@ -1,14 +0,0 @@ -// TODO: Add MicroVM when its webhook provider strategy is implemented. -export type RunnerProvider = 'ec2'; - -const defaultRunnerProvider: RunnerProvider = 'ec2'; - -export function normalizeRunnerProvider(provider: unknown): RunnerProvider | undefined { - if (provider === undefined) return defaultRunnerProvider; - if (typeof provider !== 'string') return undefined; - - const normalizedProvider = provider.trim().toLowerCase(); - if (!normalizedProvider) return defaultRunnerProvider; - - return normalizedProvider === 'ec2' ? normalizedProvider : undefined; -} diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels-provider.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels-provider.ts index b16e38b781..bd2d8faa7a 100644 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels-provider.ts +++ b/lambdas/functions/webhook/src/runners/aws-dynamic-labels-provider.ts @@ -1,4 +1,5 @@ -import type { RunnerProvider } from '../runner-provider'; +import type { RunnerProviderType } from '@aws-github-runner/runner-provider'; + import type { RunnerMatcherConfig } from '../sqs'; export interface AwsDynamicLabelDispatchTarget { @@ -13,6 +14,6 @@ export interface SelectAwsDynamicLabelQueueInput { } export interface AwsDynamicLabelProviderStrategy { - type: RunnerProvider; + type: RunnerProviderType; selectQueue(input: SelectAwsDynamicLabelQueueInput): AwsDynamicLabelDispatchTarget | undefined; } diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts index 94b4bf25b4..ccfb2e7462 100644 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts +++ b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts @@ -1,6 +1,6 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { normalizeRunnerProviderType } from '@aws-github-runner/runner-provider'; -import { normalizeRunnerProvider } from '../runner-provider'; import type { RunnerMatcherConfig } from '../sqs'; import { AwsDynamicLabelDispatchTarget, AwsDynamicLabelProviderStrategy } from './aws-dynamic-labels-provider'; import { ec2DynamicLabelProviderStrategy } from './ec2-dynamic-labels'; @@ -15,7 +15,7 @@ export function selectAwsDynamicLabelQueue( sanitizedGhrLabels: string[], ): AwsDynamicLabelDispatchTarget | undefined { for (const queue of matches) { - const provider = normalizeRunnerProvider(queue.runnerProvider); + const provider = normalizeRunnerProviderType(queue.runnerProvider); const strategy = provider ? awsDynamicLabelProviderStrategies.find((strategy) => strategy.type === provider) : undefined; diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index b194f2f032..13e0a467e9 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -6,9 +6,8 @@ import { WorkflowJobEvent } from '@octokit/webhooks-types'; import workFlowJobEvent from '../../test/resources/github_workflowjob_event.json'; import runnerConfig from '../../test/resources/multi_runner_configurations.json'; -import type { RunnerProvider } from '../runner-provider'; import { RunnerConfig, sendActionRequest } from '../sqs'; -import type { RunnerMatcherConfig } from '../sqs'; +import type { RunnerMatcherConfig, RunnerProviderType } from '../sqs'; import { dispatch } from './dispatch'; import { selectAwsDynamicLabelQueue } from './aws-dynamic-labels'; import { canRunJob } from './labels'; @@ -602,13 +601,21 @@ describe('selectAwsDynamicLabelQueue', () => { }); }); - it('does not select an unsupported provider strategy', () => { - const queue = runnerQueue('unsupported-provider'); - (queue as unknown as { runnerProvider: string }).runnerProvider = 'unsupported'; + it('skips an unsupported provider strategy and selects the next supported queue', () => { + const unsupportedQueue = runnerQueue('unsupported-provider'); + (unsupportedQueue as unknown as { runnerProvider: string }).runnerProvider = 'unsupported'; + const ec2Queue = runnerQueue('ec2'); expect( - selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), - ).toBeUndefined(); + selectAwsDynamicLabelQueue( + [unsupportedQueue, ec2Queue], + ['self-hosted', 'linux'], + ['ghr-ec2-instance-type:t3.large'], + ), + ).toEqual({ + queue: ec2Queue, + labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + }); }); it('rejects a malformed non-string runner provider without throwing', () => { @@ -645,7 +652,7 @@ describe('selectAwsDynamicLabelQueue', () => { }); }); -function runnerQueue(id: string, runnerProvider?: RunnerProvider): RunnerMatcherConfig { +function runnerQueue(id: string, runnerProvider?: RunnerProviderType): RunnerMatcherConfig { return { id, arn: `arn:${id}`, diff --git a/lambdas/functions/webhook/src/sqs/index.ts b/lambdas/functions/webhook/src/sqs/index.ts index f4027a8e73..466d0ebf89 100644 --- a/lambdas/functions/webhook/src/sqs/index.ts +++ b/lambdas/functions/webhook/src/sqs/index.ts @@ -1,15 +1,15 @@ import { SQS, SendMessageCommandInput } from '@aws-sdk/client-sqs'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; import { createChildLogger, getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; +import type { RunnerProviderType } from '@aws-github-runner/runner-provider'; import type { AwsDynamicLabelsPolicy } from '../runners/aws-dynamic-labels-policy'; -import type { RunnerProvider } from '../runner-provider'; const logger = createChildLogger('sqs'); const sqsClientsByRegion = new Map(); -export type { RunnerProvider }; +export type { RunnerProviderType } from '@aws-github-runner/runner-provider'; export interface ActionRequestMessage { id: number; @@ -37,7 +37,7 @@ export type RunnerConfig = RunnerMatcherConfig[]; export interface RunnerMatcherConfig { matcherConfig: MatcherConfig; - runnerProvider?: RunnerProvider; + runnerProvider?: RunnerProviderType; id: string; arn: string; } diff --git a/lambdas/libs/runner-provider/package.json b/lambdas/libs/runner-provider/package.json new file mode 100644 index 0000000000..6ccf3983d8 --- /dev/null +++ b/lambdas/libs/runner-provider/package.json @@ -0,0 +1,23 @@ +{ + "name": "@aws-github-runner/runner-provider", + "version": "1.0.0", + "main": "src/index.ts", + "type": "module", + "license": "MIT", + "scripts": { + "test": "NODE_ENV=test nx test", + "test:watch": "NODE_ENV=test nx test --watch", + "lint": "eslint src", + "format": "prettier --write \"**/*.ts\"", + "format-check": "prettier --check \"**/*.ts\"", + "all": "yarn format && yarn lint && yarn test" + }, + "nx": { + "includedScripts": [ + "format", + "format-check", + "lint", + "all" + ] + } +} diff --git a/lambdas/libs/runner-provider/src/index.test.ts b/lambdas/libs/runner-provider/src/index.test.ts new file mode 100644 index 0000000000..e51030fd0d --- /dev/null +++ b/lambdas/libs/runner-provider/src/index.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest'; + +import { normalizeRunnerProviderType } from './index'; + +describe('runner provider normalization', () => { + it.each([ + [undefined, 'ec2'], + ['', 'ec2'], + [' ', 'ec2'], + [' EC2 ', 'ec2'], + ])('normalizes provider type %j to %j', (type, expected) => { + expect(normalizeRunnerProviderType(type)).toBe(expected); + }); + + it.each([[' Unknown '], ['microvm'], [null], [1]])('returns undefined for unsupported provider type %j', (type) => { + expect(normalizeRunnerProviderType(type)).toBeUndefined(); + }); +}); diff --git a/lambdas/libs/runner-provider/src/index.ts b/lambdas/libs/runner-provider/src/index.ts new file mode 100644 index 0000000000..866b5f75de --- /dev/null +++ b/lambdas/libs/runner-provider/src/index.ts @@ -0,0 +1,15 @@ +// TODO: Add MicroVM when its webhook and control-plane provider implementations are available. +const runnerProviderTypes = ['ec2'] as const; +export type RunnerProviderType = (typeof runnerProviderTypes)[number]; + +const defaultRunnerProvider: RunnerProviderType = 'ec2'; + +export function normalizeRunnerProviderType(type: unknown): RunnerProviderType | undefined { + if (type === undefined) return defaultRunnerProvider; + if (typeof type !== 'string') return undefined; + + const normalizedType = type.trim().toLowerCase(); + if (!normalizedType) return defaultRunnerProvider; + + return runnerProviderTypes.find((runnerProviderType) => runnerProviderType === normalizedType); +} diff --git a/lambdas/libs/runner-provider/tsconfig.json b/lambdas/libs/runner-provider/tsconfig.json new file mode 100644 index 0000000000..9d616eca0f --- /dev/null +++ b/lambdas/libs/runner-provider/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/lambdas/libs/runner-provider/vitest.config.ts b/lambdas/libs/runner-provider/vitest.config.ts new file mode 100644 index 0000000000..5179569242 --- /dev/null +++ b/lambdas/libs/runner-provider/vitest.config.ts @@ -0,0 +1,17 @@ +import { mergeConfig } from 'vitest/config'; +import defaultConfig from '../../vitest.base.config'; + +export default mergeConfig(defaultConfig, { + test: { + coverage: { + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts', 'src/**/*.d.ts'], + thresholds: { + statements: 100, + branches: 100, + functions: 100, + lines: 100, + }, + }, + }, +}); diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index bfe69a07b9..df81fdf245 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -147,6 +147,7 @@ __metadata: dependencies: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" + "@aws-github-runner/runner-provider": "npm:*" "@aws-lambda-powertools/parameters": "npm:^2.31.0" "@aws-sdk/client-ec2": "npm:^3.1009.0" "@aws-sdk/client-sqs": "npm:^3.1009.0" @@ -192,6 +193,12 @@ __metadata: languageName: unknown linkType: soft +"@aws-github-runner/runner-provider@npm:*, @aws-github-runner/runner-provider@workspace:libs/runner-provider": + version: 0.0.0-use.local + resolution: "@aws-github-runner/runner-provider@workspace:libs/runner-provider" + languageName: unknown + linkType: soft + "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher": version: 0.0.0-use.local resolution: "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher" @@ -221,6 +228,7 @@ __metadata: dependencies: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" + "@aws-github-runner/runner-provider": "npm:*" "@aws-sdk/client-eventbridge": "npm:^3.1009.0" "@aws-sdk/client-sqs": "npm:^3.1009.0" "@middy/core": "npm:^6.4.5" From f02d68db2312f49f7eeb5cb64120fb388e46b8f0 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 23:51:15 +0200 Subject: [PATCH 43/46] refactor(runners): centralize provider contract --- .../control-plane/src/pool/pool-provider.ts | 2 +- .../functions/control-plane/src/pool/pool.ts | 2 +- .../src/runner-provider-registry.ts | 3 ++- .../control-plane/src/runner-provider.test.ts | 18 ------------------ .../control-plane/src/runner-provider.ts | 17 ----------------- .../src/scale-runners/scale-down-provider.ts | 2 +- .../src/scale-runners/scale-down.ts | 2 +- .../src/scale-runners/scale-up-provider.ts | 2 +- .../src/scale-runners/scale-up.ts | 2 +- lambdas/libs/runner-provider/src/index.test.ts | 17 ++++++++++++++++- lambdas/libs/runner-provider/src/index.ts | 13 +++++++++++++ 11 files changed, 37 insertions(+), 43 deletions(-) delete mode 100644 lambdas/functions/control-plane/src/runner-provider.test.ts delete mode 100644 lambdas/functions/control-plane/src/runner-provider.ts diff --git a/lambdas/functions/control-plane/src/pool/pool-provider.ts b/lambdas/functions/control-plane/src/pool/pool-provider.ts index 1ca2c975ee..fa12b43a5e 100644 --- a/lambdas/functions/control-plane/src/pool/pool-provider.ts +++ b/lambdas/functions/control-plane/src/pool/pool-provider.ts @@ -1,6 +1,6 @@ import type { Octokit } from '@octokit/rest'; +import type { RunnerProvider } from '@aws-github-runner/runner-provider'; -import type { RunnerProvider } from '../runner-provider'; import type { CreateGitHubRunnerConfig, GitHubRunnerType } from '../scale-runners/types'; export interface RunnerStatus { diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index f59d861b74..5ef5fa897f 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -1,10 +1,10 @@ import { Octokit } from '@octokit/rest'; import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { resolveRunnerProviderType } from '@aws-github-runner/runner-provider'; import yn from 'yn'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; import { createPoolRunnerProvider } from '../runner-provider-registry'; -import { resolveRunnerProviderType } from '../runner-provider'; import { getGitHubEnterpriseApiUrl, validateSsmParameterStoreTags } from '../scale-runners/github-runner'; import type { RunnerStatus } from './pool-provider'; diff --git a/lambdas/functions/control-plane/src/runner-provider-registry.ts b/lambdas/functions/control-plane/src/runner-provider-registry.ts index 0013861378..95c342ebdf 100644 --- a/lambdas/functions/control-plane/src/runner-provider-registry.ts +++ b/lambdas/functions/control-plane/src/runner-provider-registry.ts @@ -1,6 +1,7 @@ +import type { RunnerProviderType } from '@aws-github-runner/runner-provider'; + import { createEc2PoolProvider } from './pool/ec2-pool'; import type { PoolRunnerProvider } from './pool/pool-provider'; -import type { RunnerProviderType } from './runner-provider'; import { createEc2ScaleDownProvider } from './scale-runners/ec2-scale-down'; import { createEc2ScaleUpProvider } from './scale-runners/ec2-scale-up'; import type { ScaleDownRunnerProvider } from './scale-runners/scale-down-provider'; diff --git a/lambdas/functions/control-plane/src/runner-provider.test.ts b/lambdas/functions/control-plane/src/runner-provider.test.ts deleted file mode 100644 index ba7eec1908..0000000000 --- a/lambdas/functions/control-plane/src/runner-provider.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { resolveRunnerProviderType } from './runner-provider'; - -describe('runner provider selection', () => { - it.each([ - [undefined, 'ec2'], - ['', 'ec2'], - [' ', 'ec2'], - [' EC2 ', 'ec2'], - ])('resolves provider type %j to %j', (type, expected) => { - expect(resolveRunnerProviderType(type)).toBe(expected); - }); - - it.each([[' Unknown '], ['microvm'], [null], [1]])('rejects unsupported provider type %j', (type) => { - expect(() => resolveRunnerProviderType(type)).toThrow(`Unsupported runner provider type '${String(type)}'`); - }); -}); diff --git a/lambdas/functions/control-plane/src/runner-provider.ts b/lambdas/functions/control-plane/src/runner-provider.ts deleted file mode 100644 index 21da701804..0000000000 --- a/lambdas/functions/control-plane/src/runner-provider.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { normalizeRunnerProviderType } from '@aws-github-runner/runner-provider'; -import type { RunnerProviderType } from '@aws-github-runner/runner-provider'; - -export type { RunnerProviderType } from '@aws-github-runner/runner-provider'; - -export function resolveRunnerProviderType(type: unknown): RunnerProviderType { - const normalizedType = normalizeRunnerProviderType(type); - if (!normalizedType) { - throw new Error(`Unsupported runner provider type '${String(type)}'`); - } - - return normalizedType; -} - -export interface RunnerProvider { - type: RunnerProviderType; -} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts index c614ffcff5..c4966201aa 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-provider.ts @@ -1,4 +1,4 @@ -import type { RunnerProvider } from '../runner-provider'; +import type { RunnerProvider } from '@aws-github-runner/runner-provider'; export interface RunnerList { id: string; diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index 2f0f7e8f41..c517f89b9c 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -2,11 +2,11 @@ import { Octokit } from '@octokit/rest'; import { Endpoints } from '@octokit/types'; import { RequestError } from '@octokit/request-error'; import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { resolveRunnerProviderType } from '@aws-github-runner/runner-provider'; import moment from 'moment'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; import { createScaleDownRunnerProvider } from '../runner-provider-registry'; -import { resolveRunnerProviderType } from '../runner-provider'; import { GhRunners, githubCache } from './cache'; import { ScalingDownConfigList, getEvictionStrategy, getIdleRunnerCount } from './scale-down-config'; import { metricGitHubAppRateLimit } from '../github/rate-limit'; diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts index 1956151892..fc34d7f243 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-provider.ts @@ -1,6 +1,6 @@ import type { Octokit } from '@octokit/rest'; +import type { RunnerProvider } from '@aws-github-runner/runner-provider'; -import type { RunnerProvider } from '../runner-provider'; import type { ActionRequestMessageSQS, CreateGitHubRunnerConfig, GitHubRunnerType } from './types'; export interface CurrentRunnersInput { diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 29fbbc9912..f6d039cb34 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -1,10 +1,10 @@ import { addPersistentContextToChildLogger, createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { resolveRunnerProviderType } from '@aws-github-runner/runner-provider'; import { Octokit } from '@octokit/rest'; import yn from 'yn'; import { createGithubAppAuth, createGithubInstallationAuth, createOctokitClient } from '../github/auth'; import { createScaleUpRunnerProvider } from '../runner-provider-registry'; -import { resolveRunnerProviderType } from '../runner-provider'; import { getGitHubEnterpriseApiUrl, getInstallationId, diff --git a/lambdas/libs/runner-provider/src/index.test.ts b/lambdas/libs/runner-provider/src/index.test.ts index e51030fd0d..616ed40d37 100644 --- a/lambdas/libs/runner-provider/src/index.test.ts +++ b/lambdas/libs/runner-provider/src/index.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { normalizeRunnerProviderType } from './index'; +import { normalizeRunnerProviderType, resolveRunnerProviderType } from './index'; describe('runner provider normalization', () => { it.each([ @@ -16,3 +16,18 @@ describe('runner provider normalization', () => { expect(normalizeRunnerProviderType(type)).toBeUndefined(); }); }); + +describe('runner provider resolution', () => { + it.each([ + [undefined, 'ec2'], + ['', 'ec2'], + [' ', 'ec2'], + [' EC2 ', 'ec2'], + ])('resolves provider type %j to %j', (type, expected) => { + expect(resolveRunnerProviderType(type)).toBe(expected); + }); + + it.each([[' Unknown '], ['microvm'], [null], [1]])('rejects unsupported provider type %j', (type) => { + expect(() => resolveRunnerProviderType(type)).toThrow(`Unsupported runner provider type '${String(type)}'`); + }); +}); diff --git a/lambdas/libs/runner-provider/src/index.ts b/lambdas/libs/runner-provider/src/index.ts index 866b5f75de..bb73cfd983 100644 --- a/lambdas/libs/runner-provider/src/index.ts +++ b/lambdas/libs/runner-provider/src/index.ts @@ -13,3 +13,16 @@ export function normalizeRunnerProviderType(type: unknown): RunnerProviderType | return runnerProviderTypes.find((runnerProviderType) => runnerProviderType === normalizedType); } + +export function resolveRunnerProviderType(type: unknown): RunnerProviderType { + const normalizedType = normalizeRunnerProviderType(type); + if (!normalizedType) { + throw new Error(`Unsupported runner provider type '${String(type)}'`); + } + + return normalizedType; +} + +export interface RunnerProvider { + type: RunnerProviderType; +} From a4692b887fd52c12305ba7c225add5889c1bbea8 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 14 Jul 2026 23:57:08 +0200 Subject: [PATCH 44/46] fix(webhook): require matcher runner provider --- .../webhook/src/ConfigLoader.test.ts | 49 ++++++++++++++----- .../webhook/src/runners/dispatch.test.ts | 5 +- lambdas/functions/webhook/src/sqs/index.ts | 2 +- .../webhook/src/webhook/index.test.ts | 1 + .../multi_runner_configurations.json | 16 ++---- 5 files changed, 46 insertions(+), 27 deletions(-) diff --git a/lambdas/functions/webhook/src/ConfigLoader.test.ts b/lambdas/functions/webhook/src/ConfigLoader.test.ts index 9f4e5e5864..820c8ed928 100644 --- a/lambdas/functions/webhook/src/ConfigLoader.test.ts +++ b/lambdas/functions/webhook/src/ConfigLoader.test.ts @@ -30,12 +30,13 @@ describe('ConfigLoader Tests', () => { { id: '1', arn: 'arn:aws:sqs:us-east-1:123456789012:queue1', + runnerProvider: 'ec2', matcherConfig: { labelMatchers: [['label1', 'label2']], exactMatch: true, }, }, - ]; + ] satisfies RunnerMatcherConfig[]; vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { if (paramPath === '/path/to/matcher/config') { return JSON.stringify(matcherConfig); @@ -100,12 +101,13 @@ describe('ConfigLoader Tests', () => { { id: '1', arn: 'arn:aws:sqs:us-east-1:123456789012:queue1', + runnerProvider: 'ec2', matcherConfig: { labelMatchers: [['label1', 'label2']], exactMatch: true, }, }, - ]; + ] satisfies RunnerMatcherConfig[]; vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { if (paramPath === '/path/to/matcher/config') { return JSON.stringify(matcherConfig); @@ -130,12 +132,13 @@ describe('ConfigLoader Tests', () => { { id: '1', arn: 'arn:aws:sqs:us-east-1:123456789012:queue1', + runnerProvider: 'ec2', matcherConfig: { labelMatchers: [['label1', 'label2']], exactMatch: true, }, }, - ]; + ] satisfies RunnerMatcherConfig[]; vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { if (paramPath === '/path/to/matcher/config') { return JSON.stringify(matcherConfig); @@ -174,14 +177,24 @@ describe('ConfigLoader Tests', () => { process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const partialMatcher1 = - '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}'; + '[{"id":"1","arn":"arn:aws:sqs:queue1","runnerProvider":"ec2","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}'; const partialMatcher2 = - ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}]'; + ',{"id":"2","arn":"arn:aws:sqs:queue2","runnerProvider":"ec2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}]'; const combinedMatcherConfig = [ - { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['a']], exactMatch: true } }, - { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['b']], exactMatch: true } }, - ]; + { + id: '1', + arn: 'arn:aws:sqs:queue1', + runnerProvider: 'ec2', + matcherConfig: { labelMatchers: [['a']], exactMatch: true }, + }, + { + id: '2', + arn: 'arn:aws:sqs:queue2', + runnerProvider: 'ec2', + matcherConfig: { labelMatchers: [['b']], exactMatch: true }, + }, + ] satisfies RunnerMatcherConfig[]; // Mock getParameters for batch fetching multiple paths vi.mocked(getParameters).mockResolvedValue( @@ -270,6 +283,7 @@ describe('ConfigLoader Tests', () => { { arn: 'arn:aws:sqs:eu-central-1:123456:npalm-default-queued-builds', id: 'https://sqs.eu-central-1.amazonaws.com/123456/npalm-default-queued-builds', + runnerProvider: 'ec2', matcherConfig: { exactMatch: true, labelMatchers: [['default', 'example', 'linux', 'self-hosted', 'x64']], @@ -294,13 +308,23 @@ describe('ConfigLoader Tests', () => { process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2'; const partial1 = - '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["x"]],"exactMatch":true}}'; + '[{"id":"1","arn":"arn:aws:sqs:queue1","runnerProvider":"ec2","matcherConfig":{"labelMatchers":[["x"]],"exactMatch":true}}'; const partial2 = - ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["y"]],"exactMatch":true}}]'; + ',{"id":"2","arn":"arn:aws:sqs:queue2","runnerProvider":"ec2","matcherConfig":{"labelMatchers":[["y"]],"exactMatch":true}}]'; const combined: RunnerMatcherConfig[] = [ - { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['x']], exactMatch: true } }, - { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['y']], exactMatch: true } }, + { + id: '1', + arn: 'arn:aws:sqs:queue1', + runnerProvider: 'ec2', + matcherConfig: { labelMatchers: [['x']], exactMatch: true }, + }, + { + id: '2', + arn: 'arn:aws:sqs:queue2', + runnerProvider: 'ec2', + matcherConfig: { labelMatchers: [['y']], exactMatch: true }, + }, ]; // Mock getParameters for batch fetching multiple paths @@ -334,6 +358,7 @@ describe('ConfigLoader Tests', () => { { arn: 'arn:aws:sqs:eu-central-1:123456:npalm-default-queued-builds', id: 'https://sqs.eu-central-1.amazonaws.com/123456/npalm-default-queued-builds', + runnerProvider: 'ec2', matcherConfig: { exactMatch: true, labelMatchers: [['default', 'example', 'linux', 'self-hosted', 'x64']], diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index 13e0a467e9..8ec75d5f21 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -582,8 +582,9 @@ async function createConfig(repositoryAllowList?: string[], runnerConfig?: Runne } describe('selectAwsDynamicLabelQueue', () => { - it('defaults queues without a provider to EC2 dynamic label handling', () => { + it('defensively defaults queues with a missing provider to EC2 dynamic label handling', () => { const queue = runnerQueue('default-ec2'); + delete (queue as Partial).runnerProvider; expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ queue, @@ -652,7 +653,7 @@ describe('selectAwsDynamicLabelQueue', () => { }); }); -function runnerQueue(id: string, runnerProvider?: RunnerProviderType): RunnerMatcherConfig { +function runnerQueue(id: string, runnerProvider: RunnerProviderType = 'ec2'): RunnerMatcherConfig { return { id, arn: `arn:${id}`, diff --git a/lambdas/functions/webhook/src/sqs/index.ts b/lambdas/functions/webhook/src/sqs/index.ts index 466d0ebf89..ca8e18db1d 100644 --- a/lambdas/functions/webhook/src/sqs/index.ts +++ b/lambdas/functions/webhook/src/sqs/index.ts @@ -37,7 +37,7 @@ export type RunnerConfig = RunnerMatcherConfig[]; export interface RunnerMatcherConfig { matcherConfig: MatcherConfig; - runnerProvider?: RunnerProviderType; + runnerProvider: RunnerProviderType; id: string; arn: string; } diff --git a/lambdas/functions/webhook/src/webhook/index.test.ts b/lambdas/functions/webhook/src/webhook/index.test.ts index aa4fbbc506..31539812ed 100644 --- a/lambdas/functions/webhook/src/webhook/index.test.ts +++ b/lambdas/functions/webhook/src/webhook/index.test.ts @@ -291,6 +291,7 @@ function mockSSMResponse() { { id: '1', arn: 'arn:aws:sqs:us-east-1:123456789012:queue1', + runnerProvider: 'ec2', matcherConfig: { labelMatchers: [['label1', 'label2']], exactMatch: true, diff --git a/lambdas/functions/webhook/test/resources/multi_runner_configurations.json b/lambdas/functions/webhook/test/resources/multi_runner_configurations.json index b90378fb16..02b0062d1c 100644 --- a/lambdas/functions/webhook/test/resources/multi_runner_configurations.json +++ b/lambdas/functions/webhook/test/resources/multi_runner_configurations.json @@ -3,13 +3,9 @@ "id": "ubuntu-queue-id", "arn": "queueARN-ubuntu", "fifo": false, + "runnerProvider": "ec2", "matcherConfig": { - "labelMatchers": [[ - "self-hosted", - "linux", - "x64", - "ubuntu" - ]], + "labelMatchers": [["self-hosted", "linux", "x64", "ubuntu"]], "exactMatch": true } }, @@ -17,13 +13,9 @@ "id": "latest-queue-id", "arn": "queueARN-latest", "fifo": false, + "runnerProvider": "ec2", "matcherConfig": { - "labelMatchers": [[ - "self-hosted", - "linux", - "x64", - "latest" - ]], + "labelMatchers": [["self-hosted", "linux", "x64", "latest"]], "exactMatch": false } } From 688e578d85f7abf9d2aa5a17ab77690ff171185c Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 15 Jul 2026 00:05:03 +0200 Subject: [PATCH 45/46] fix(webhook): keep matcher runner provider optional --- .../webhook/src/ConfigLoader.test.ts | 49 +++++-------------- .../webhook/src/runners/dispatch.test.ts | 5 +- lambdas/functions/webhook/src/sqs/index.ts | 2 +- .../webhook/src/webhook/index.test.ts | 1 - .../multi_runner_configurations.json | 16 ++++-- 5 files changed, 27 insertions(+), 46 deletions(-) diff --git a/lambdas/functions/webhook/src/ConfigLoader.test.ts b/lambdas/functions/webhook/src/ConfigLoader.test.ts index 820c8ed928..9f4e5e5864 100644 --- a/lambdas/functions/webhook/src/ConfigLoader.test.ts +++ b/lambdas/functions/webhook/src/ConfigLoader.test.ts @@ -30,13 +30,12 @@ describe('ConfigLoader Tests', () => { { id: '1', arn: 'arn:aws:sqs:us-east-1:123456789012:queue1', - runnerProvider: 'ec2', matcherConfig: { labelMatchers: [['label1', 'label2']], exactMatch: true, }, }, - ] satisfies RunnerMatcherConfig[]; + ]; vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { if (paramPath === '/path/to/matcher/config') { return JSON.stringify(matcherConfig); @@ -101,13 +100,12 @@ describe('ConfigLoader Tests', () => { { id: '1', arn: 'arn:aws:sqs:us-east-1:123456789012:queue1', - runnerProvider: 'ec2', matcherConfig: { labelMatchers: [['label1', 'label2']], exactMatch: true, }, }, - ] satisfies RunnerMatcherConfig[]; + ]; vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { if (paramPath === '/path/to/matcher/config') { return JSON.stringify(matcherConfig); @@ -132,13 +130,12 @@ describe('ConfigLoader Tests', () => { { id: '1', arn: 'arn:aws:sqs:us-east-1:123456789012:queue1', - runnerProvider: 'ec2', matcherConfig: { labelMatchers: [['label1', 'label2']], exactMatch: true, }, }, - ] satisfies RunnerMatcherConfig[]; + ]; vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { if (paramPath === '/path/to/matcher/config') { return JSON.stringify(matcherConfig); @@ -177,24 +174,14 @@ describe('ConfigLoader Tests', () => { process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const partialMatcher1 = - '[{"id":"1","arn":"arn:aws:sqs:queue1","runnerProvider":"ec2","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}'; + '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}'; const partialMatcher2 = - ',{"id":"2","arn":"arn:aws:sqs:queue2","runnerProvider":"ec2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}]'; + ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}]'; const combinedMatcherConfig = [ - { - id: '1', - arn: 'arn:aws:sqs:queue1', - runnerProvider: 'ec2', - matcherConfig: { labelMatchers: [['a']], exactMatch: true }, - }, - { - id: '2', - arn: 'arn:aws:sqs:queue2', - runnerProvider: 'ec2', - matcherConfig: { labelMatchers: [['b']], exactMatch: true }, - }, - ] satisfies RunnerMatcherConfig[]; + { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['a']], exactMatch: true } }, + { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['b']], exactMatch: true } }, + ]; // Mock getParameters for batch fetching multiple paths vi.mocked(getParameters).mockResolvedValue( @@ -283,7 +270,6 @@ describe('ConfigLoader Tests', () => { { arn: 'arn:aws:sqs:eu-central-1:123456:npalm-default-queued-builds', id: 'https://sqs.eu-central-1.amazonaws.com/123456/npalm-default-queued-builds', - runnerProvider: 'ec2', matcherConfig: { exactMatch: true, labelMatchers: [['default', 'example', 'linux', 'self-hosted', 'x64']], @@ -308,23 +294,13 @@ describe('ConfigLoader Tests', () => { process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2'; const partial1 = - '[{"id":"1","arn":"arn:aws:sqs:queue1","runnerProvider":"ec2","matcherConfig":{"labelMatchers":[["x"]],"exactMatch":true}}'; + '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["x"]],"exactMatch":true}}'; const partial2 = - ',{"id":"2","arn":"arn:aws:sqs:queue2","runnerProvider":"ec2","matcherConfig":{"labelMatchers":[["y"]],"exactMatch":true}}]'; + ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["y"]],"exactMatch":true}}]'; const combined: RunnerMatcherConfig[] = [ - { - id: '1', - arn: 'arn:aws:sqs:queue1', - runnerProvider: 'ec2', - matcherConfig: { labelMatchers: [['x']], exactMatch: true }, - }, - { - id: '2', - arn: 'arn:aws:sqs:queue2', - runnerProvider: 'ec2', - matcherConfig: { labelMatchers: [['y']], exactMatch: true }, - }, + { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['x']], exactMatch: true } }, + { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['y']], exactMatch: true } }, ]; // Mock getParameters for batch fetching multiple paths @@ -358,7 +334,6 @@ describe('ConfigLoader Tests', () => { { arn: 'arn:aws:sqs:eu-central-1:123456:npalm-default-queued-builds', id: 'https://sqs.eu-central-1.amazonaws.com/123456/npalm-default-queued-builds', - runnerProvider: 'ec2', matcherConfig: { exactMatch: true, labelMatchers: [['default', 'example', 'linux', 'self-hosted', 'x64']], diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index 8ec75d5f21..13e0a467e9 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -582,9 +582,8 @@ async function createConfig(repositoryAllowList?: string[], runnerConfig?: Runne } describe('selectAwsDynamicLabelQueue', () => { - it('defensively defaults queues with a missing provider to EC2 dynamic label handling', () => { + it('defaults queues without a provider to EC2 dynamic label handling', () => { const queue = runnerQueue('default-ec2'); - delete (queue as Partial).runnerProvider; expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ queue, @@ -653,7 +652,7 @@ describe('selectAwsDynamicLabelQueue', () => { }); }); -function runnerQueue(id: string, runnerProvider: RunnerProviderType = 'ec2'): RunnerMatcherConfig { +function runnerQueue(id: string, runnerProvider?: RunnerProviderType): RunnerMatcherConfig { return { id, arn: `arn:${id}`, diff --git a/lambdas/functions/webhook/src/sqs/index.ts b/lambdas/functions/webhook/src/sqs/index.ts index ca8e18db1d..466d0ebf89 100644 --- a/lambdas/functions/webhook/src/sqs/index.ts +++ b/lambdas/functions/webhook/src/sqs/index.ts @@ -37,7 +37,7 @@ export type RunnerConfig = RunnerMatcherConfig[]; export interface RunnerMatcherConfig { matcherConfig: MatcherConfig; - runnerProvider: RunnerProviderType; + runnerProvider?: RunnerProviderType; id: string; arn: string; } diff --git a/lambdas/functions/webhook/src/webhook/index.test.ts b/lambdas/functions/webhook/src/webhook/index.test.ts index 31539812ed..aa4fbbc506 100644 --- a/lambdas/functions/webhook/src/webhook/index.test.ts +++ b/lambdas/functions/webhook/src/webhook/index.test.ts @@ -291,7 +291,6 @@ function mockSSMResponse() { { id: '1', arn: 'arn:aws:sqs:us-east-1:123456789012:queue1', - runnerProvider: 'ec2', matcherConfig: { labelMatchers: [['label1', 'label2']], exactMatch: true, diff --git a/lambdas/functions/webhook/test/resources/multi_runner_configurations.json b/lambdas/functions/webhook/test/resources/multi_runner_configurations.json index 02b0062d1c..b90378fb16 100644 --- a/lambdas/functions/webhook/test/resources/multi_runner_configurations.json +++ b/lambdas/functions/webhook/test/resources/multi_runner_configurations.json @@ -3,9 +3,13 @@ "id": "ubuntu-queue-id", "arn": "queueARN-ubuntu", "fifo": false, - "runnerProvider": "ec2", "matcherConfig": { - "labelMatchers": [["self-hosted", "linux", "x64", "ubuntu"]], + "labelMatchers": [[ + "self-hosted", + "linux", + "x64", + "ubuntu" + ]], "exactMatch": true } }, @@ -13,9 +17,13 @@ "id": "latest-queue-id", "arn": "queueARN-latest", "fifo": false, - "runnerProvider": "ec2", "matcherConfig": { - "labelMatchers": [["self-hosted", "linux", "x64", "latest"]], + "labelMatchers": [[ + "self-hosted", + "linux", + "x64", + "latest" + ]], "exactMatch": false } } From 4f27502dca39637006e26ee91561d32dcd816368 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 15 Jul 2026 00:09:13 +0200 Subject: [PATCH 46/46] refactor(webhook): import provider type directly --- lambdas/functions/webhook/src/runners/dispatch.test.ts | 3 ++- lambdas/functions/webhook/src/sqs/index.ts | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index 13e0a467e9..c2b9116680 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -1,4 +1,5 @@ import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import type { RunnerProviderType } from '@aws-github-runner/runner-provider'; import nock from 'nock'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; @@ -7,7 +8,7 @@ import workFlowJobEvent from '../../test/resources/github_workflowjob_event.json import runnerConfig from '../../test/resources/multi_runner_configurations.json'; import { RunnerConfig, sendActionRequest } from '../sqs'; -import type { RunnerMatcherConfig, RunnerProviderType } from '../sqs'; +import type { RunnerMatcherConfig } from '../sqs'; import { dispatch } from './dispatch'; import { selectAwsDynamicLabelQueue } from './aws-dynamic-labels'; import { canRunJob } from './labels'; diff --git a/lambdas/functions/webhook/src/sqs/index.ts b/lambdas/functions/webhook/src/sqs/index.ts index 466d0ebf89..ba805ce88c 100644 --- a/lambdas/functions/webhook/src/sqs/index.ts +++ b/lambdas/functions/webhook/src/sqs/index.ts @@ -9,8 +9,6 @@ const logger = createChildLogger('sqs'); const sqsClientsByRegion = new Map(); -export type { RunnerProviderType } from '@aws-github-runner/runner-provider'; - export interface ActionRequestMessage { id: number; eventType: string;