diff --git a/main.tf b/main.tf index 56681177..4d14d75c 100644 --- a/main.tf +++ b/main.tf @@ -68,6 +68,7 @@ module "runners" { s3_location_runner_binaries = local.s3_action_runner_url instance_type = var.instance_type + instance_types = var.instance_types market_options = var.market_options block_device_mappings = var.block_device_mappings diff --git a/modules/runners/lambdas/runners/src/scale-runners/runners.test.ts b/modules/runners/lambdas/runners/src/scale-runners/runners.test.ts index 0c3cab19..4436a1b6 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/runners.test.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/runners.test.ts @@ -7,6 +7,7 @@ jest.mock('aws-sdk', () => ({ SSM: jest.fn().mockImplementation(() => mockSSM), })); +const LAUNCH_TEMPLATE = 'lt-1'; const ORG_NAME = 'SomeAwesomeCoder'; const REPO_NAME = `${ORG_NAME}/some-amazing-library`; const ENVIRONMENT = 'unit-test-environment'; @@ -115,22 +116,20 @@ describe('create runner', () => { ], }); mockSSM.putParameter.mockImplementation(() => mockPutParameter); - process.env.LAUNCH_TEMPLATE_NAME = 'launch-template-name'; - process.env.LAUNCH_TEMPLATE_VERSION = '1'; process.env.SUBNET_IDS = 'sub-1234'; }); it('calls run instances with the correct config for repo', async () => { await createRunner({ - runnerConfig: 'bla', + runnerServiceConfig: 'bla', environment: ENVIRONMENT, runnerType: 'Repo', runnerOwner: REPO_NAME - }); + }, LAUNCH_TEMPLATE); expect(mockEC2.runInstances).toBeCalledWith({ MaxCount: 1, MinCount: 1, - LaunchTemplate: { LaunchTemplateName: 'launch-template-name', Version: '1' }, + LaunchTemplate: { LaunchTemplateName: LAUNCH_TEMPLATE, Version: '$Default' }, SubnetId: 'sub-1234', TagSpecifications: [ { @@ -146,15 +145,15 @@ describe('create runner', () => { it('calls run instances with the correct config for org', async () => { await createRunner({ - runnerConfig: 'bla', + runnerServiceConfig: 'bla', environment: ENVIRONMENT, runnerType: 'Org', runnerOwner: ORG_NAME, - }); + }, LAUNCH_TEMPLATE); expect(mockEC2.runInstances).toBeCalledWith({ MaxCount: 1, MinCount: 1, - LaunchTemplate: { LaunchTemplateName: 'launch-template-name', Version: '1' }, + LaunchTemplate: { LaunchTemplateName: LAUNCH_TEMPLATE, Version: '$Default' }, SubnetId: 'sub-1234', TagSpecifications: [ { @@ -170,11 +169,11 @@ describe('create runner', () => { it('creates ssm parameters for each created instance', async () => { await createRunner({ - runnerConfig: 'bla', + runnerServiceConfig: 'bla', environment: ENVIRONMENT, runnerType: 'Org', runnerOwner: ORG_NAME, - }); + }, LAUNCH_TEMPLATE); expect(mockSSM.putParameter).toBeCalledWith({ Name: `${ENVIRONMENT}-i-1234`, Value: 'bla', @@ -187,11 +186,11 @@ describe('create runner', () => { Instances: [], }); await createRunner({ - runnerConfig: 'bla', + runnerServiceConfig: 'bla', environment: ENVIRONMENT, runnerType: 'Org', runnerOwner: ORG_NAME, - }); + }, LAUNCH_TEMPLATE); expect(mockSSM.putParameter).not.toBeCalled(); }); }); diff --git a/modules/runners/lambdas/runners/src/scale-runners/runners.ts b/modules/runners/lambdas/runners/src/scale-runners/runners.ts index e5e9b3f5..11b10589 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/runners.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/runners.ts @@ -13,6 +13,13 @@ export interface ListRunnerFilters { environment: string | undefined; } +export interface RunnerInputParameters { + runnerServiceConfig: string; + environment: string; + runnerType: 'Org' | 'Repo'; + runnerOwner: string; +} + export async function listRunners(filters: ListRunnerFilters | undefined = undefined): Promise { const ec2 = new EC2(); const ec2Filters = [ @@ -46,13 +53,6 @@ export async function listRunners(filters: ListRunnerFilters | undefined = undef return runners; } -export interface RunnerInputParameters { - runnerConfig: string; - environment: string; - runnerType: 'Org' | 'Repo'; - runnerOwner: string; -} - export async function terminateRunner(runner: RunnerInfo): Promise { const ec2 = new EC2(); await ec2 @@ -63,47 +63,53 @@ export async function terminateRunner(runner: RunnerInfo): Promise { console.debug('Runner terminated.' + runner.instanceId); } -export async function createRunner(runnerParameters: RunnerInputParameters): Promise { - const launchTemplateName = process.env.LAUNCH_TEMPLATE_NAME as string; - const launchTemplateVersion = process.env.LAUNCH_TEMPLATE_VERSION as string; - - const subnets = (process.env.SUBNET_IDS as string).split(','); - const randomSubnet = subnets[Math.floor(Math.random() * subnets.length)]; +export async function createRunner(runnerParameters: RunnerInputParameters, launchTemplateName: string): Promise { console.debug('Runner configuration: ' + JSON.stringify(runnerParameters)); const ec2 = new EC2(); const runInstancesResponse = await ec2 - .runInstances({ - MaxCount: 1, - MinCount: 1, - LaunchTemplate: { - LaunchTemplateName: launchTemplateName, - Version: launchTemplateVersion, - }, - SubnetId: randomSubnet, - TagSpecifications: [ - { - ResourceType: 'instance', - Tags: [ - { Key: 'Application', Value: 'github-action-runner' }, - { - Key: runnerParameters.runnerType, - Value: runnerParameters.runnerOwner - }, - ], - }, - ], - }) + .runInstances(getInstanceParams(launchTemplateName, runnerParameters)) .promise(); console.info('Created instance(s): ', runInstancesResponse.Instances?.map((i) => i.InstanceId).join(',')); - const ssm = new SSM(); runInstancesResponse.Instances?.forEach(async (i: EC2.Instance) => { await ssm .putParameter({ Name: runnerParameters.environment + '-' + (i.InstanceId as string), - Value: runnerParameters.runnerConfig, + Value: runnerParameters.runnerServiceConfig, Type: 'SecureString', }) .promise(); }); } + +function getInstanceParams( + launchTemplateName: string, + runnerParameters: RunnerInputParameters +): EC2.RunInstancesRequest { + return { + MaxCount: 1, + MinCount: 1, + LaunchTemplate: { + LaunchTemplateName: launchTemplateName, + Version: '$Default', + }, + SubnetId: getSubnet(), + TagSpecifications: [ + { + ResourceType: 'instance', + Tags: [ + { Key: 'Application', Value: 'github-action-runner' }, + { + Key: runnerParameters.runnerType, + Value: runnerParameters.runnerOwner + }, + ], + }, + ], + }; +} + +function getSubnet(): string { + const subnets = (process.env.SUBNET_IDS as string).split(','); + return subnets[Math.floor(Math.random() * subnets.length)]; +} diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts index e2f6640a..8e157f90 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts @@ -1,6 +1,6 @@ import { mocked } from 'ts-jest/utils'; -import { ActionRequestMessage, scaleUp } from './scale-up'; -import { listRunners, createRunner } from './runners'; +import * as scaleUpModule from './scale-up'; +import { listRunners, createRunner, RunnerInputParameters } from './runners'; import * as ghAuth from './gh-auth'; import nock from 'nock'; @@ -24,7 +24,7 @@ jest.mock('@octokit/rest', () => ({ jest.mock('./runners'); -const TEST_DATA: ActionRequestMessage = { +const TEST_DATA: scaleUpModule.ActionRequestMessage = { id: 1, eventType: 'check_run', repositoryName: 'hello-world', @@ -32,7 +32,7 @@ const TEST_DATA: ActionRequestMessage = { installationId: 2, }; -const TEST_DATA_WITHOUT_INSTALL_ID: ActionRequestMessage = { +const TEST_DATA_WITHOUT_INSTALL_ID: scaleUpModule.ActionRequestMessage = { id: 3, eventType: 'check_run', repositoryName: 'hello-world', @@ -40,8 +40,18 @@ const TEST_DATA_WITHOUT_INSTALL_ID: ActionRequestMessage = { installationId: 0, }; +const LAUNCH_TEMPLATE = 'lt-1'; + const cleanEnv = process.env; +const EXPECTED_RUNNER_PARAMS: RunnerInputParameters = { + environment: 'unit-test-environment', + runnerServiceConfig: `--url https://github.enterprise.something/${TEST_DATA.repositoryOwner} --token 1234abcd `, + runnerType: 'Org', + runnerOwner: TEST_DATA.repositoryOwner +}; +let expectedRunnerParams: RunnerInputParameters; + beforeEach(() => { nock.disableNetConnect(); jest.resetModules(); @@ -53,6 +63,7 @@ beforeEach(() => { process.env.GITHUB_APP_CLIENT_SECRET = 'TEST_CLIENT_SECRET'; process.env.RUNNERS_MAXIMUM_COUNT = '3'; process.env.ENVIRONMENT = 'unit-test-environment'; + process.env.LAUNCH_TEMPLATE_NAME = 'lt-1,lt-2'; mockOctokit.checks.get.mockImplementation(() => ({ data: { @@ -97,11 +108,11 @@ describe('scaleUp with GHES', () => { it('ignores non-sqs events', async () => { expect.assertions(1); - expect(scaleUp('aws:s3', TEST_DATA)).rejects.toEqual(Error('Cannot handle non-SQS events!')); + expect(scaleUpModule.scaleUp('aws:s3', TEST_DATA)).rejects.toEqual(Error('Cannot handle non-SQS events!')); }); it('checks queued workflows', async () => { - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.checks.get).toBeCalledWith({ check_run_id: TEST_DATA.id, owner: TEST_DATA.repositoryOwner, @@ -113,17 +124,18 @@ describe('scaleUp with GHES', () => { mockOctokit.checks.get.mockImplementation(() => ({ data: { total_count: 0, runners: [] }, })); - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(listRunners).not.toBeCalled(); }); describe('on org level', () => { beforeEach(() => { process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; }); it('gets the current org level runners', async () => { - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(listRunners).toBeCalledWith({ environment: 'unit-test-environment', runnerType: 'Org', @@ -133,12 +145,12 @@ describe('scaleUp with GHES', () => { it('does not create a token when maximum runners has been reached', async () => { process.env.RUNNERS_MAXIMUM_COUNT = '1'; - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); }); it('creates a token when maximum runners has not been reached', async () => { - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalled(); expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ org: TEST_DATA.repositoryOwner, @@ -147,7 +159,7 @@ describe('scaleUp with GHES', () => { it('does not retrieve installation id if already set', async () => { const spy = jest.spyOn(ghAuth, 'createGithubAuth'); - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.apps.getOrgInstallation).not.toBeCalled(); expect(mockOctokit.apps.getRepoInstallation).not.toBeCalled(); expect(spy).toBeCalledWith( @@ -159,7 +171,7 @@ describe('scaleUp with GHES', () => { it('retrieves installation id if not set', async () => { const spy = jest.spyOn(ghAuth, 'createGithubAuth'); - await scaleUp('aws:sqs', TEST_DATA_WITHOUT_INSTALL_ID); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA_WITHOUT_INSTALL_ID); expect(mockOctokit.apps.getRepoInstallation).not.toBeCalled(); expect(spy).toHaveBeenNthCalledWith(1, undefined, 'app', "https://github.enterprise.something/api/v3"); expect(spy).toHaveBeenNthCalledWith( @@ -171,36 +183,53 @@ describe('scaleUp with GHES', () => { }); it('creates a runner with correct config', async () => { - await scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledWith({ - environment: 'unit-test-environment', - runnerConfig: `--url https://github.enterprise.something/${TEST_DATA.repositoryOwner} --token 1234abcd `, - runnerType: 'Org', - runnerOwner: TEST_DATA.repositoryOwner, - }); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams, 'lt-1'); }); - it('creates a runner with labels in s specific group', async () => { + it('creates a runner with labels in a specific group', async () => { process.env.RUNNER_EXTRA_LABELS = 'label1,label2'; process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; - await scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledWith({ - environment: 'unit-test-environment', - runnerConfig: `--url https://github.enterprise.something/${TEST_DATA.repositoryOwner} ` + - `--token 1234abcd --labels label1,label2 --runnergroup TEST_GROUP`, - runnerType: 'Org', - runnerOwner: TEST_DATA.repositoryOwner, - }); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); + expectedRunnerParams.runnerServiceConfig = expectedRunnerParams.runnerServiceConfig + + `--labels label1,label2 --runnergroup TEST_GROUP`; + expect(createRunner).toBeCalledWith(expectedRunnerParams, 'lt-1'); + }); + + it('attempts next launch template if first fails', async () => { + const mockCreateRunners = mocked(createRunner); + mockCreateRunners.mockRejectedValueOnce(new Error('no capactiy')); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); + expect(createRunner).toBeCalledTimes(2); + expect(createRunner).toHaveBeenNthCalledWith(1, expectedRunnerParams, 'lt-1'); + expect(createRunner).toHaveBeenNthCalledWith(2, expectedRunnerParams, 'lt-2'); + mockCreateRunners.mockReset(); + }); + + it('all launch templates fail', async () => { + const mockCreateRunners = mocked(createRunner); + mockCreateRunners.mockRejectedValue(new Error('All launch templates failed')); + await expect(scaleUpModule.scaleUp('aws:sqs', TEST_DATA)).rejects.toThrow('All launch templates failed'); + expect(createRunner).toBeCalledTimes(2); + expect(createRunner).toHaveBeenNthCalledWith(1, expectedRunnerParams, 'lt-1'); + expect(createRunner).toHaveBeenNthCalledWith(2, expectedRunnerParams, 'lt-2'); + mockCreateRunners.mockReset(); }); }); describe('on repo level', () => { beforeEach(() => { process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + expectedRunnerParams.runnerType = 'Repo'; + expectedRunnerParams.runnerOwner = `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`; + expectedRunnerParams.runnerServiceConfig = `--url ` + + `https://github.enterprise.something/${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName} ` + + `--token 1234abcd `; }); it('gets the current repo level runners', async () => { - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(listRunners).toBeCalledWith({ environment: 'unit-test-environment', runnerType: 'Repo', @@ -210,12 +239,12 @@ describe('scaleUp with GHES', () => { it('does not create a token when maximum runners has been reached', async () => { process.env.RUNNERS_MAXIMUM_COUNT = '1'; - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); }); it('creates a token when maximum runners has not been reached', async () => { - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ owner: TEST_DATA.repositoryOwner, repo: TEST_DATA.repositoryName, @@ -224,7 +253,7 @@ describe('scaleUp with GHES', () => { it('does not retrieve installation id if already set', async () => { const spy = jest.spyOn(ghAuth, 'createGithubAuth'); - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.apps.getOrgInstallation).not.toBeCalled(); expect(mockOctokit.apps.getRepoInstallation).not.toBeCalled(); expect(spy).toBeCalledWith( @@ -236,7 +265,7 @@ describe('scaleUp with GHES', () => { it('retrieves installation id if not set', async () => { const spy = jest.spyOn(ghAuth, 'createGithubAuth'); - await scaleUp('aws:sqs', TEST_DATA_WITHOUT_INSTALL_ID); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA_WITHOUT_INSTALL_ID); expect(mockOctokit.apps.getOrgInstallation).not.toBeCalled(); expect(spy).toHaveBeenNthCalledWith(1, undefined, 'app', "https://github.enterprise.something/api/v3"); expect(spy).toHaveBeenNthCalledWith( @@ -249,29 +278,31 @@ describe('scaleUp with GHES', () => { it('creates a runner with correct config and labels', async () => { process.env.RUNNER_EXTRA_LABELS = 'label1,label2'; - await scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledWith({ - environment: 'unit-test-environment', - runnerConfig: `--url ` + - `https://github.enterprise.something/${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName} ` + - `--token 1234abcd --labels label1,label2`, - runnerType: 'Repo', - runnerOwner: `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`, - }); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); + expectedRunnerParams.runnerServiceConfig = expectedRunnerParams.runnerServiceConfig + `--labels label1,label2`; + expectedRunnerParams.runnerType = 'Repo'; + expectedRunnerParams.runnerOwner = `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`; + expect(createRunner).toBeCalledWith(expectedRunnerParams, 'lt-1'); }); it('creates a runner and ensure the group argument is ignored', async () => { process.env.RUNNER_EXTRA_LABELS = 'label1,label2'; process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; - await scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledWith({ - environment: 'unit-test-environment', - runnerConfig: `--url ` + - `https://github.enterprise.something/${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName} ` + - `--token 1234abcd --labels label1,label2`, - runnerType: 'Repo', - runnerOwner: `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`, - }); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); + expectedRunnerParams.runnerServiceConfig = `--url ` + + `https://github.enterprise.something/${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName} ` + + `--token 1234abcd --labels label1,label2`; + expectedRunnerParams.runnerOwner = `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`; + expect(createRunner).toBeCalledWith(expectedRunnerParams, 'lt-1'); + }); + + it('attempts next launch template if first fails', async () => { + const mockCreateRunners = mocked(createRunner); + mockCreateRunners.mockRejectedValueOnce(new Error('no capactiy')); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); + expect(createRunner).toBeCalledTimes(2); + expect(createRunner).toHaveBeenNthCalledWith(1, expectedRunnerParams, 'lt-1'); + expect(createRunner).toHaveBeenNthCalledWith(2, expectedRunnerParams, 'lt-2'); }); }); }); @@ -279,11 +310,11 @@ describe('scaleUp with GHES', () => { describe('scaleUp with public GH', () => { it('ignores non-sqs events', async () => { expect.assertions(1); - expect(scaleUp('aws:s3', TEST_DATA)).rejects.toEqual(Error('Cannot handle non-SQS events!')); + expect(scaleUpModule.scaleUp('aws:s3', TEST_DATA)).rejects.toEqual(Error('Cannot handle non-SQS events!')); }); it('checks queued workflows', async () => { - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.checks.get).toBeCalledWith({ check_run_id: TEST_DATA.id, owner: TEST_DATA.repositoryOwner, @@ -293,7 +324,7 @@ describe('scaleUp with public GH', () => { it('does not retrieve installation id if already set', async () => { const spy = jest.spyOn(ghAuth, 'createGithubAuth'); - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.apps.getOrgInstallation).not.toBeCalled(); expect(mockOctokit.apps.getRepoInstallation).not.toBeCalled(); expect(spy).toBeCalledWith(TEST_DATA.installationId, 'installation', ""); @@ -301,7 +332,7 @@ describe('scaleUp with public GH', () => { it('retrieves installation id if not set', async () => { const spy = jest.spyOn(ghAuth, 'createGithubAuth'); - await scaleUp('aws:sqs', TEST_DATA_WITHOUT_INSTALL_ID); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA_WITHOUT_INSTALL_ID); expect(mockOctokit.apps.getRepoInstallation).not.toBeCalled(); expect(spy).toHaveBeenNthCalledWith(1, undefined, 'app', ""); expect(spy).toHaveBeenNthCalledWith(2, TEST_DATA.installationId, 'installation', ""); @@ -311,17 +342,20 @@ describe('scaleUp with public GH', () => { mockOctokit.checks.get.mockImplementation(() => ({ data: { status: 'completed' }, })); - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(listRunners).not.toBeCalled(); }); describe('on org level', () => { beforeEach(() => { process.env.ENABLE_ORGANIZATION_RUNNERS = 'true'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + expectedRunnerParams.runnerServiceConfig = + `--url https://github.com/${TEST_DATA.repositoryOwner} --token 1234abcd `; }); it('gets the current org level runners', async () => { - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(listRunners).toBeCalledWith({ environment: 'unit-test-environment', runnerType: 'Org', @@ -331,12 +365,12 @@ describe('scaleUp with public GH', () => { it('does not create a token when maximum runners has been reached', async () => { process.env.RUNNERS_MAXIMUM_COUNT = '1'; - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.actions.createRegistrationTokenForOrg).not.toBeCalled(); }); it('creates a token when maximum runners has not been reached', async () => { - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalled(); expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalledWith({ org: TEST_DATA.repositoryOwner, @@ -344,36 +378,42 @@ describe('scaleUp with public GH', () => { }); it('creates a runner with correct config', async () => { - await scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledWith({ - environment: 'unit-test-environment', - runnerConfig: `--url https://github.com/${TEST_DATA.repositoryOwner} --token 1234abcd `, - runnerType: 'Org', - runnerOwner: TEST_DATA.repositoryOwner - }); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); + expect(createRunner).toBeCalledWith(expectedRunnerParams, LAUNCH_TEMPLATE); }); it('creates a runner with labels in s specific group', async () => { process.env.RUNNER_EXTRA_LABELS = 'label1,label2'; process.env.RUNNER_GROUP_NAME = 'TEST_GROUP'; - await scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledWith({ - environment: 'unit-test-environment', - runnerConfig: `--url https://github.com/${TEST_DATA.repositoryOwner} ` + - `--token 1234abcd --labels label1,label2 --runnergroup TEST_GROUP`, - runnerType: 'Org', - runnerOwner: TEST_DATA.repositoryOwner - }); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); + expectedRunnerParams.runnerServiceConfig = + expectedRunnerParams.runnerServiceConfig + `--labels label1,label2 --runnergroup TEST_GROUP`; + expect(createRunner).toBeCalledWith(expectedRunnerParams, LAUNCH_TEMPLATE); + }); + + it('attempts next launch template if first fails', async () => { + const mockCreateRunners = mocked(createRunner); + mockCreateRunners.mockRejectedValueOnce(new Error('no capactiy')); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); + expect(createRunner).toBeCalledTimes(2); + expect(createRunner).toHaveBeenNthCalledWith(1, expectedRunnerParams, 'lt-1'); + expect(createRunner).toHaveBeenNthCalledWith(2, expectedRunnerParams, 'lt-2'); }); }); describe('on repo level', () => { beforeEach(() => { process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; + expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; + expectedRunnerParams.runnerType = 'Repo'; + expectedRunnerParams.runnerOwner = `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`; + expectedRunnerParams.runnerServiceConfig = `--url ` + + `https://github.com/${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName} ` + + `--token 1234abcd `; }); it('gets the current repo level runners', async () => { - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(listRunners).toBeCalledWith({ environment: 'unit-test-environment', runnerType: 'Repo', @@ -383,12 +423,12 @@ describe('scaleUp with public GH', () => { it('does not create a token when maximum runners has been reached', async () => { process.env.RUNNERS_MAXIMUM_COUNT = '1'; - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.actions.createRegistrationTokenForRepo).not.toBeCalled(); }); it('creates a token when maximum runners has not been reached', async () => { - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.actions.createRegistrationTokenForRepo).toBeCalledWith({ owner: TEST_DATA.repositoryOwner, repo: TEST_DATA.repositoryName, @@ -397,7 +437,7 @@ describe('scaleUp with public GH', () => { it('does not retrieve installation id if already set', async () => { const spy = jest.spyOn(ghAuth, 'createGithubAuth'); - await scaleUp('aws:sqs', TEST_DATA); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); expect(mockOctokit.apps.getOrgInstallation).not.toBeCalled(); expect(mockOctokit.apps.getRepoInstallation).not.toBeCalled(); expect(spy).toBeCalledWith(TEST_DATA.installationId, 'installation', ""); @@ -405,7 +445,7 @@ describe('scaleUp with public GH', () => { it('retrieves installation id if not set', async () => { const spy = jest.spyOn(ghAuth, 'createGithubAuth'); - await scaleUp('aws:sqs', TEST_DATA_WITHOUT_INSTALL_ID); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA_WITHOUT_INSTALL_ID); expect(mockOctokit.apps.getOrgInstallation).not.toBeCalled(); expect(spy).toHaveBeenNthCalledWith(1, undefined, 'app', ""); expect(spy).toHaveBeenNthCalledWith(2, TEST_DATA.installationId, 'installation', ""); @@ -413,27 +453,26 @@ describe('scaleUp with public GH', () => { it('creates a runner with correct config and labels', async () => { process.env.RUNNER_EXTRA_LABELS = 'label1,label2'; - await scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledWith({ - environment: 'unit-test-environment', - runnerConfig: `--url https://github.com/${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName} ` + - `--token 1234abcd --labels label1,label2`, - runnerType: 'Repo', - runnerOwner: `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`, - }); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); + expectedRunnerParams.runnerServiceConfig = expectedRunnerParams.runnerServiceConfig + `--labels label1,label2`; + expect(createRunner).toBeCalledWith(expectedRunnerParams, LAUNCH_TEMPLATE); }); it('creates a runner and ensure the group argument is ignored', async () => { process.env.RUNNER_EXTRA_LABELS = 'label1,label2'; process.env.RUNNER_GROUP_NAME = 'TEST_GROUP_IGNORED'; - await scaleUp('aws:sqs', TEST_DATA); - expect(createRunner).toBeCalledWith({ - environment: 'unit-test-environment', - runnerConfig: `--url https://github.com/${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName} ` + - `--token 1234abcd --labels label1,label2`, - runnerType: 'Repo', - runnerOwner: `${TEST_DATA.repositoryOwner}/${TEST_DATA.repositoryName}`, - }); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); + expectedRunnerParams.runnerServiceConfig = expectedRunnerParams.runnerServiceConfig + `--labels label1,label2`; + expect(createRunner).toBeCalledWith(expectedRunnerParams, LAUNCH_TEMPLATE); + }); + + it('attempts next launch template if first fails', async () => { + const mockCreateRunners = mocked(createRunner); + mockCreateRunners.mockRejectedValueOnce(new Error('no capactiy')); + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); + expect(createRunner).toBeCalledTimes(2); + expect(createRunner).toHaveBeenNthCalledWith(1, expectedRunnerParams, 'lt-1'); + expect(createRunner).toHaveBeenNthCalledWith(2, expectedRunnerParams, 'lt-2'); }); }); }); diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts index dfbbf5d1..07f9f050 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts @@ -1,4 +1,4 @@ -import { listRunners, createRunner } from './runners'; +import { listRunners, createRunner, RunnerInputParameters } from './runners'; import { createOctoClient, createGithubAuth } from './gh-auth'; import yn from 'yn'; @@ -77,16 +77,37 @@ export const scaleUp = async (eventSource: string, payload: ActionRequestMessage const labelsArgument = runnerExtraLabels !== undefined ? `--labels ${runnerExtraLabels}` : ''; const runnerGroupArgument = runnerGroup !== undefined ? ` --runnergroup ${runnerGroup}` : ''; const configBaseUrl = ghesBaseUrl ? ghesBaseUrl : 'https://github.com'; - await createRunner({ - environment: environment, - runnerConfig: enableOrgLevel - ? `--url ${configBaseUrl}/${runnerOwner} --token ${token} ${labelsArgument}${runnerGroupArgument}` - : `--url ${configBaseUrl}/${runnerOwner} --token ${token} ${labelsArgument}`, - runnerType, + + await createRunnerLoop({ + environment, + runnerServiceConfig: enableOrgLevel + ? `--url ${configBaseUrl}/${payload.repositoryOwner} --token ${token} ${labelsArgument}${runnerGroupArgument}` + : `--url ${configBaseUrl}/${payload.repositoryOwner}/${payload.repositoryName} ` + + `--token ${token} ${labelsArgument}`, runnerOwner, + runnerType + }); } else { console.info('No runner will be created, maximum number of runners reached.'); } } }; + +export async function createRunnerLoop(runnerParameters: RunnerInputParameters): Promise { + const launchTemplateNames = process.env.LAUNCH_TEMPLATE_NAME?.split(',') as string[]; + let launched = false; + for (const launchTemplateName of launchTemplateNames) { + console.info(`Attempting to launch instance using ${launchTemplateName}.`); + try { + await createRunner(runnerParameters, launchTemplateName); + launched = true; + break; + } catch (error) { + console.error(error); + } + } + if (launched == false) { + throw Error('All launch templates failed'); + } +} diff --git a/modules/runners/main.tf b/modules/runners/main.tf index 35387014..ac742d8d 100644 --- a/modules/runners/main.tf +++ b/modules/runners/main.tf @@ -17,6 +17,8 @@ locals { userdata_template = var.userdata_template == null ? "${path.module}/templates/user-data.sh" : var.userdata_template userdata_arm_patch = "${path.module}/templates/arm-runner-patch.tpl" userdata_install_config_runner = "${path.module}/templates/install-config-runner.sh" + + instance_types = var.instance_types == null ? [var.instance_type] : var.instance_types } data "aws_ami" "runner" { @@ -34,7 +36,9 @@ data "aws_ami" "runner" { } resource "aws_launch_template" "runner" { - name = "${var.environment}-action-runner" + for_each = local.instance_types + + name = "${var.environment}-action-runner-${each.value}" dynamic "block_device_mappings" { for_each = [var.block_device_mappings] @@ -62,7 +66,7 @@ resource "aws_launch_template" "runner" { } image_id = data.aws_ami.runner.id - instance_type = var.instance_type + instance_type = each.value key_name = var.key_name vpc_security_group_ids = compact(concat( @@ -102,6 +106,8 @@ resource "aws_launch_template" "runner" { })) tags = local.tags + + update_default_version = true } locals { diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf index 54188806..37c40d7d 100644 --- a/modules/runners/scale-up.tf +++ b/modules/runners/scale-up.tf @@ -39,8 +39,7 @@ resource "aws_lambda_function" "scale_up" { RUNNER_EXTRA_LABELS = var.runner_extra_labels RUNNER_GROUP_NAME = var.runner_group_name RUNNERS_MAXIMUM_COUNT = var.runners_maximum_count - LAUNCH_TEMPLATE_NAME = aws_launch_template.runner.name - LAUNCH_TEMPLATE_VERSION = aws_launch_template.runner.latest_version + LAUNCH_TEMPLATE_NAME = join(",", [for template in aws_launch_template.runner : template.name]) SUBNET_IDS = join(",", var.subnet_ids) } } diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index f1ca68aa..0516f0b1 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -58,11 +58,17 @@ variable "market_options" { } variable "instance_type" { - description = "Default instance type for the action runner." + description = "[DEPRECATED] See instance_types." type = string default = "m5.large" } +variable "instance_types" { + description = "List of instance types for the action runner." + type = set(string) + default = null +} + variable "ami_filter" { description = "List of maps used to create the AMI filter for the action runner AMI." type = map(list(string)) diff --git a/outputs.tf b/outputs.tf index 7b39a8d8..3cb4684a 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,8 +1,8 @@ output "runners" { value = { - launch_template_name = module.runners.launch_template.name - launch_template_id = module.runners.launch_template.id - launch_template_version = module.runners.launch_template.latest_version + launch_template_name = [for template in module.runners.launch_template : template.name] + launch_template_id = [for template in module.runners.launch_template : template.id] + launch_template_version = [for template in module.runners.launch_template : template.latest_version] lambda_up = module.runners.lambda_scale_up lambda_down = module.runners.lambda_scale_down role_runner = module.runners.role_runner diff --git a/variables.tf b/variables.tf index 4f455467..21773e9f 100644 --- a/variables.tf +++ b/variables.tf @@ -126,7 +126,7 @@ variable "instance_profile_path" { } variable "instance_type" { - description = "Instance type for the action runner." + description = "[DEPRECATED] See instance_types." type = string default = "m5.large" } @@ -354,3 +354,9 @@ variable "volume_size" { type = number default = 30 } + +variable "instance_types" { + description = "List of instance types for the action runner." + type = set(string) + default = null +}