mirror of
https://gitea.com/mcereda/oam.git
synced 2026-02-09 05:44:23 +00:00
chore(examples/pulumi): gitlab runner autoscaler using docker-autoscaler in aws
This commit is contained in:
Michele Cereda
3469
examples/gitlab/gitlab.with-defaults.rb
Normal file
3469
examples/gitlab/gitlab.with-defaults.rb
Normal file
File diff suppressed because it is too large
Load Diff
259
examples/gitlab/runner.config.toml
Normal file
259
examples/gitlab/runner.config.toml
Normal file
@@ -0,0 +1,259 @@
|
|||||||
|
###
|
||||||
|
# Gitlab runner configuration file
|
||||||
|
# --------------------------------------
|
||||||
|
# Default locations:
|
||||||
|
# - /etc/gitlab-runner/config.toml
|
||||||
|
# - $HOME/.gitlab-runner/config.toml
|
||||||
|
#
|
||||||
|
# Refer 'https://docs.gitlab.com/runner/configuration/advanced-configuration.html'
|
||||||
|
###
|
||||||
|
|
||||||
|
concurrent = 32 # global through *all* runners
|
||||||
|
listen_address = "0.0.0.0:9100"
|
||||||
|
|
||||||
|
|
||||||
|
###
|
||||||
|
# Docker autoscaler executor
|
||||||
|
# ------------------
|
||||||
|
###
|
||||||
|
|
||||||
|
[[runners]]
|
||||||
|
name = "docker autoscaler on AWS"
|
||||||
|
|
||||||
|
# instances using docker-autoscaler executor
|
||||||
|
# ----------------
|
||||||
|
# up to 10 instances at a time, min 1 idle during working hours
|
||||||
|
# use each instance for up to 250 jobs
|
||||||
|
# 1 job per instance at a time
|
||||||
|
|
||||||
|
url = "https://gitlab.example.org"
|
||||||
|
token = "glrt-abc…"
|
||||||
|
|
||||||
|
environment = [ "AWS_REGION=eu-west-1" ]
|
||||||
|
executor = "docker-autoscaler"
|
||||||
|
|
||||||
|
[runners.docker]
|
||||||
|
privileged = false
|
||||||
|
|
||||||
|
image = "busybox:latest"
|
||||||
|
pull_policy = [
|
||||||
|
"if-not-present",
|
||||||
|
"always"
|
||||||
|
]
|
||||||
|
allowed_pull_policies = [
|
||||||
|
"if-not-present",
|
||||||
|
"always",
|
||||||
|
"never"
|
||||||
|
]
|
||||||
|
|
||||||
|
[runners.autoscaler]
|
||||||
|
plugin = "aws"
|
||||||
|
max_instances = 10
|
||||||
|
max_use_count = 250
|
||||||
|
capacity_per_instance = 1
|
||||||
|
|
||||||
|
[runners.autoscaler.plugin_config]
|
||||||
|
name = "GitlabRunners" # autoscaling group name
|
||||||
|
|
||||||
|
[[runners.autoscaler.policy]]
|
||||||
|
periods = [ "* 7-19 * * mon-fri" ]
|
||||||
|
timezone = "Europe/Amsterdam"
|
||||||
|
idle_count = 1
|
||||||
|
idle_time = "20m0s"
|
||||||
|
|
||||||
|
### Docker autoscaler executor - end
|
||||||
|
|
||||||
|
|
||||||
|
###
|
||||||
|
# Docker machine executor
|
||||||
|
# ------------------
|
||||||
|
# DEPRECATED - use 'docker-autoscaler' or 'instance' executors instead
|
||||||
|
###
|
||||||
|
|
||||||
|
[[runners]]
|
||||||
|
name = "docker machine ondemand on AWS"
|
||||||
|
|
||||||
|
# ondemand instance using docker+machine executor
|
||||||
|
# ----------------
|
||||||
|
# Up to 1, min 1 during working hours
|
||||||
|
|
||||||
|
url = "https://gitlab.example.org"
|
||||||
|
token = "glrt-abc…"
|
||||||
|
|
||||||
|
environment = [ "AWS_REGION=eu-west-1" ]
|
||||||
|
executor = "docker+machine"
|
||||||
|
|
||||||
|
# Number of jobs that can be run concurrently by the VMs created by *this* runner
|
||||||
|
# Defines the *upper limit* of how many VMs can be created by *this* runner, since it is 1 task per VM at a time
|
||||||
|
limit = 1
|
||||||
|
|
||||||
|
[runners.cache]
|
||||||
|
Type = "s3"
|
||||||
|
Path = "cache/"
|
||||||
|
Shared = true
|
||||||
|
MaxUploadedArchiveSize = 0
|
||||||
|
|
||||||
|
[runners.cache.s3]
|
||||||
|
ServerAddress = "s3.amazonaws.com"
|
||||||
|
BucketName = "exampleorg-gitlab-cache"
|
||||||
|
BucketLocation = "eu-west-1"
|
||||||
|
|
||||||
|
[runners.docker]
|
||||||
|
tls_verify = false
|
||||||
|
image = "busybox:latest"
|
||||||
|
privileged = false
|
||||||
|
disable_entrypoint_overwrite = false
|
||||||
|
oom_kill_disable = false
|
||||||
|
disable_cache = false
|
||||||
|
volumes = [
|
||||||
|
"/cache",
|
||||||
|
|
||||||
|
# docker-in-docker
|
||||||
|
"/var/run/docker.sock:/var/run/docker.sock"
|
||||||
|
]
|
||||||
|
shm_size = 0
|
||||||
|
network_mtu = 0
|
||||||
|
pull_policy = [
|
||||||
|
"if-not-present",
|
||||||
|
"always"
|
||||||
|
]
|
||||||
|
allowed_pull_policies = [
|
||||||
|
"if-not-present",
|
||||||
|
"always",
|
||||||
|
"never"
|
||||||
|
]
|
||||||
|
|
||||||
|
[runners.docker.services_tmpfs]
|
||||||
|
# speed up i/o for 'postgresql' services
|
||||||
|
"/var/lib/postgresql/data" = "rw,noexec"
|
||||||
|
|
||||||
|
[runners.machine]
|
||||||
|
# Static number of VMs that need to be idle at all times
|
||||||
|
IdleCount = 0
|
||||||
|
|
||||||
|
# Remove VMs after 250 jobs
|
||||||
|
# Keeps instances "fresh"
|
||||||
|
MaxBuilds = 250
|
||||||
|
|
||||||
|
# Maximum number of VMs that can be added to this runner in parallel
|
||||||
|
# Defaults to 0 (no limit)
|
||||||
|
MaxGrowthRate = 1
|
||||||
|
|
||||||
|
MachineDriver = "amazonec2"
|
||||||
|
MachineName = "autoscaled-b-ondemand-%s"
|
||||||
|
MachineOptions = [
|
||||||
|
"amazonec2-region=eu-west-1",
|
||||||
|
"amazonec2-vpc-id=vpc-01234567",
|
||||||
|
"amazonec2-zone=b",
|
||||||
|
"amazonec2-subnet-id=subnet-0123456789abcdef0",
|
||||||
|
"amazonec2-use-private-address=true",
|
||||||
|
"amazonec2-private-address-only=true",
|
||||||
|
"amazonec2-security-group=GitlabRunners",
|
||||||
|
"amazonec2-instance-type=m7i.xlarge",
|
||||||
|
"amazonec2-root-size=50",
|
||||||
|
"amazonec2-volume-type=gp3",
|
||||||
|
"amazonec2-iam-instance-profile=GitlabRunner",
|
||||||
|
"amazonec2-tags=Team,Infra,Application,GitlabRunner,SpotInstance,False",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[runners.machine.autoscaling]]
|
||||||
|
Periods = [ "* * 9-5 * * mon-fri *" ]
|
||||||
|
Timezone = "Europe/Amsterdam"
|
||||||
|
IdleCount = 1
|
||||||
|
|
||||||
|
|
||||||
|
[[runners]]
|
||||||
|
name = "docker machine spot on AWS"
|
||||||
|
|
||||||
|
# spot instances using docker+machine executor
|
||||||
|
# ----------------
|
||||||
|
# Up to 10, min 2 during working hours
|
||||||
|
|
||||||
|
url = "https://gitlab.example.org"
|
||||||
|
token = "glrt-abc…"
|
||||||
|
|
||||||
|
environment = [ "AWS_REGION=eu-west-1" ]
|
||||||
|
executor = "docker+machine"
|
||||||
|
limit = 10
|
||||||
|
request_concurrency = 4
|
||||||
|
|
||||||
|
[runners.cache]
|
||||||
|
Type = "s3"
|
||||||
|
Path = "cache/"
|
||||||
|
Shared = true
|
||||||
|
MaxUploadedArchiveSize = 0
|
||||||
|
|
||||||
|
[runners.cache.s3]
|
||||||
|
ServerAddress = "s3.amazonaws.com"
|
||||||
|
BucketName = "exampleorg-gitlab-cache"
|
||||||
|
BucketLocation = "eu-west-1"
|
||||||
|
|
||||||
|
[runners.docker]
|
||||||
|
tls_verify = false
|
||||||
|
image = "busybox:latest"
|
||||||
|
privileged = false
|
||||||
|
disable_entrypoint_overwrite = false
|
||||||
|
oom_kill_disable = false
|
||||||
|
disable_cache = false
|
||||||
|
volumes = [
|
||||||
|
"/cache",
|
||||||
|
|
||||||
|
# docker-in-docker
|
||||||
|
"/var/run/docker.sock:/var/run/docker.sock"
|
||||||
|
]
|
||||||
|
shm_size = 0
|
||||||
|
network_mtu = 0
|
||||||
|
services_limit = -1
|
||||||
|
pull_policy = [
|
||||||
|
"if-not-present",
|
||||||
|
"always"
|
||||||
|
]
|
||||||
|
allowed_pull_policies = [
|
||||||
|
"if-not-present",
|
||||||
|
"always",
|
||||||
|
"never"
|
||||||
|
]
|
||||||
|
|
||||||
|
[runners.docker.services_tmpfs]
|
||||||
|
# speed up i/o for 'postgresql' services
|
||||||
|
"/var/lib/postgresql/data" = "rw,noexec"
|
||||||
|
|
||||||
|
[runners.machine]
|
||||||
|
IdleCount = 0
|
||||||
|
|
||||||
|
# Remove VMs after 5m in the idle state
|
||||||
|
IdleTime = 300
|
||||||
|
|
||||||
|
MaxBuilds = 50
|
||||||
|
MaxGrowthRate = 4
|
||||||
|
|
||||||
|
MachineDriver = "amazonec2"
|
||||||
|
MachineName = "autoscaled-a-spot-%s"
|
||||||
|
MachineOptions = [
|
||||||
|
"amazonec2-region=eu-west-1",
|
||||||
|
"amazonec2-vpc-id=vpc-01234567",
|
||||||
|
"amazonec2-zone=a",
|
||||||
|
"amazonec2-subnet-id=subnet-0123456789abcdef0",
|
||||||
|
"amazonec2-use-private-address=true",
|
||||||
|
"amazonec2-private-address-only=true",
|
||||||
|
"amazonec2-security-group=GitlabRunners",
|
||||||
|
|
||||||
|
"amazonec2-instance-type=m7i.xlarge",
|
||||||
|
"amazonec2-root-size=50",
|
||||||
|
"amazonec2-volume-type=gp3",
|
||||||
|
"amazonec2-iam-instance-profile=GitlabRunner",
|
||||||
|
"amazonec2-tags=Team,Infra,Application,GitlabRunner,SpotInstance,True",
|
||||||
|
|
||||||
|
"amazonec2-request-spot-instance=true",
|
||||||
|
"amazonec2-spot-price=0.3",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[runners.machine.autoscaling]]
|
||||||
|
Periods = ["* 7-19 * * mon-fri *"]
|
||||||
|
Timezone = "Europe/Amsterdam"
|
||||||
|
IdleCount = 10
|
||||||
|
IdleCountMin = 2
|
||||||
|
IdleScaleFactor = 1.5
|
||||||
|
IdleTime = 900
|
||||||
|
|
||||||
|
### Docker machine executor - end
|
||||||
@@ -9,3 +9,5 @@ config:
|
|||||||
pulumi:tags:
|
pulumi:tags:
|
||||||
value:
|
value:
|
||||||
pulumi:template: aws-typescript
|
pulumi:template: aws-typescript
|
||||||
|
backend:
|
||||||
|
url: file://.
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
set -x PULUMI_BACKEND_URL 'file://.'
|
||||||
|
set -x PULUMI_CONFIG_PASSPHRASE 'test123'
|
||||||
2
examples/pulumi/aws/gitlab-runner using docker-autoscaler/.gitignore
vendored
Normal file
2
examples/pulumi/aws/gitlab-runner using docker-autoscaler/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
/bin/
|
||||||
|
/node_modules/
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
encryptionsalt: v1:p/R9FMstlOM=:v1:0Co3Kfu2CitxpVg6:UdYMMz8ZcasE7GPn1WjcTwK3R6+mmg==
|
||||||
|
config:
|
||||||
|
aws:region: eu-west-1
|
||||||
|
aws:defaultTags:
|
||||||
|
tags:
|
||||||
|
ManagedByPulumi: "true" # or it will not visualize
|
||||||
|
Owner: "somebody@example.com"
|
||||||
|
aws_gitlab-runner-autoscaler_docker-autoscaler:gitlab-url: https://gitlab.example.org
|
||||||
|
aws_gitlab-runner-autoscaler_docker-autoscaler:gitlab-runner-token:
|
||||||
|
secure: v1:osMr46blYbAd9dpP:IV7ld3oAnGiG51AbPS/pu/0/75NWxB9rb3/PvLPmu6JqhrCfd46+/uM=
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
name: aws_gitlab-runner-autoscaler_docker-autoscaler
|
||||||
|
description: >-
|
||||||
|
AWS example: Gitlab runner autoscaler using docker-autoscaler
|
||||||
|
runtime:
|
||||||
|
name: nodejs
|
||||||
|
options:
|
||||||
|
packagemanager: npm
|
||||||
|
config:
|
||||||
|
pulumi:tags:
|
||||||
|
value:
|
||||||
|
pulumi:template: aws-typescript
|
||||||
|
backend:
|
||||||
|
url: file://.
|
||||||
@@ -0,0 +1,453 @@
|
|||||||
|
/**
|
||||||
|
* Gitlab Runner Autoscaler using docker-autoscaler
|
||||||
|
* -------------------------------------
|
||||||
|
* This implementation uses a single EC2 instance that executes a gitlab runner leveraging the
|
||||||
|
* 'docker-autoscaler' executor and acting as runner manager.
|
||||||
|
* Both the manager and runners must have the Docker Engine installed.
|
||||||
|
* The manager connects to the runners through the instance's default user, but can be set otherwise.
|
||||||
|
* Runners must be set up so that the user the manager connects with can access the Docker Engine socket (i.e. adding)
|
||||||
|
* it to the 'docker' group).
|
||||||
|
* Runners are created and deleted by means of an AutoScaling Group that the manager controls.
|
||||||
|
* Container images are pulled by the manager and sent to the runners through Docker pipes.
|
||||||
|
*
|
||||||
|
* Requirements:
|
||||||
|
* - An EC2 instance with Docker Engine to act as manager.
|
||||||
|
* - A Launch Template:
|
||||||
|
* - referencing an AMI equipped with Docker Engine for the runners to use, or
|
||||||
|
* - using any AMI but providing userData so that the Docker Engine is installed and configured "properly".
|
||||||
|
* - An AutoScaling Group with Minimum and Desired capacity set to 0.
|
||||||
|
* - Permissions to discover and scale the ASG (manager).
|
||||||
|
* - ECR authentication (manager).
|
||||||
|
* - ECR read only access to pull images from used repositories (manager).
|
||||||
|
*
|
||||||
|
* Pulumi resources info:
|
||||||
|
* - https://www.pulumi.com/registry/packages/aws/api-docs/ec2/securitygroup/
|
||||||
|
* - https://www.pulumi.com/registry/packages/aws/api-docs/vpc/securitygroupingressrule/
|
||||||
|
* - https://www.pulumi.com/registry/packages/aws/api-docs/vpc/securitygroupegressrule/
|
||||||
|
* - https://www.pulumi.com/registry/packages/aws/api-docs/autoscaling/group/
|
||||||
|
* - https://www.pulumi.com/registry/packages/aws/api-docs/ec2/launchtemplate/
|
||||||
|
* - https://www.pulumi.com/registry/packages/aws/api-docs/ec2/instance/
|
||||||
|
**/
|
||||||
|
|
||||||
|
import * as aws from "@pulumi/aws";
|
||||||
|
import * as cloudinit from "@pulumi/cloudinit";
|
||||||
|
import * as pulumi from "@pulumi/pulumi";
|
||||||
|
import * as yaml from "yaml";
|
||||||
|
|
||||||
|
const awsRegion_output = aws.getRegionOutput();
|
||||||
|
const callerIdentity_output = aws.getCallerIdentity();
|
||||||
|
|
||||||
|
const config = new pulumi.Config();
|
||||||
|
const gitlab_url = config.require("gitlab-url");
|
||||||
|
const token = config.requireSecret("gitlab-runner-token");
|
||||||
|
|
||||||
|
const ami_amazonLinux_arm64_latest = aws.ec2.getAmiOutput({
|
||||||
|
owners: [ "amazon" ],
|
||||||
|
nameRegex: "^al2023-ami-2023.*",
|
||||||
|
filters: [
|
||||||
|
{
|
||||||
|
name: "architecture",
|
||||||
|
values: [ "arm64" ],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "state",
|
||||||
|
values: [ "available" ],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
mostRecent: true,
|
||||||
|
});
|
||||||
|
const ami_amazonLinux_x86_64_latest = aws.ec2.getAmiOutput({
|
||||||
|
owners: [ "amazon" ],
|
||||||
|
nameRegex: "^al2023-ami-2023.*",
|
||||||
|
filters: [
|
||||||
|
{
|
||||||
|
name: "architecture",
|
||||||
|
values: [ "x86_64" ],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "state",
|
||||||
|
values: [ "available" ],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
mostRecent: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const subnet_ids = aws.ec2.getSubnetsOutput({
|
||||||
|
filters: [{
|
||||||
|
name: "tag:Name",
|
||||||
|
values: [
|
||||||
|
// "private_a",
|
||||||
|
// "private_b",
|
||||||
|
// "private_c",
|
||||||
|
"private-eu-west-1a",
|
||||||
|
"private-eu-west-1b",
|
||||||
|
"private-eu-west-1c",
|
||||||
|
],
|
||||||
|
}],
|
||||||
|
}).apply(subnets => subnets.ids);
|
||||||
|
|
||||||
|
// manager's security group - start
|
||||||
|
const gitlab_runner_autoscalingManager_securityGroup = new aws.ec2.SecurityGroup(
|
||||||
|
"gitlab-runner-autoscalingManager",
|
||||||
|
{
|
||||||
|
name: "Gitlab Runner autoscaling manager",
|
||||||
|
description: "Security perimeter for the Gitlab Runner autoscaling manager",
|
||||||
|
tags: {
|
||||||
|
Name: "Gitlab Runner autoscaling manager",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
new aws.vpc.SecurityGroupIngressRule(
|
||||||
|
"gitlab-runner-autoscalingManager-fullAccess",
|
||||||
|
{
|
||||||
|
securityGroupId: gitlab_runner_autoscalingManager_securityGroup.id,
|
||||||
|
description: "Allow all",
|
||||||
|
cidrIpv4: "0.0.0.0/0",
|
||||||
|
ipProtocol: "-1",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
new aws.vpc.SecurityGroupEgressRule(
|
||||||
|
"gitlab-runner-autoscalingManager-fullAccess",
|
||||||
|
{
|
||||||
|
securityGroupId: gitlab_runner_autoscalingManager_securityGroup.id,
|
||||||
|
description: "Allow all",
|
||||||
|
cidrIpv4: "0.0.0.0/0",
|
||||||
|
ipProtocol: "-1",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
// manager's security group - end
|
||||||
|
|
||||||
|
// runners' security group - start
|
||||||
|
const gitlab_runners_securityGroup = new aws.ec2.SecurityGroup(
|
||||||
|
"gitlab-runners",
|
||||||
|
{
|
||||||
|
name: "Gitlab Runners",
|
||||||
|
description: "Security perimeter for the Gitlab Runners",
|
||||||
|
tags: {
|
||||||
|
Name: "Gitlab Runners",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
new aws.vpc.SecurityGroupIngressRule( // FIXME: reduce?
|
||||||
|
"gitlab-runners-managerAccess",
|
||||||
|
{
|
||||||
|
securityGroupId: gitlab_runners_securityGroup.id,
|
||||||
|
description: "Allow all from Gitlab Runner autoscaling manager",
|
||||||
|
referencedSecurityGroupId: gitlab_runner_autoscalingManager_securityGroup.id,
|
||||||
|
ipProtocol: "-1",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
new aws.vpc.SecurityGroupEgressRule( // FIXME: reduce?
|
||||||
|
"gitlab-runners-internetAccess",
|
||||||
|
{
|
||||||
|
securityGroupId: gitlab_runners_securityGroup.id,
|
||||||
|
description: "Allow all",
|
||||||
|
cidrIpv4: "0.0.0.0/0",
|
||||||
|
ipProtocol: "-1",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
// runners' security group - end
|
||||||
|
|
||||||
|
// runners - start
|
||||||
|
const gitlab_runners_userData = new cloudinit.Config(
|
||||||
|
"gitlab-runners",
|
||||||
|
{
|
||||||
|
base64Encode: true, // required by the launch template
|
||||||
|
parts: [{
|
||||||
|
filename: "cloud-config.docker-engine.yml",
|
||||||
|
contentType: "text/cloud-config",
|
||||||
|
content: yaml.stringify({
|
||||||
|
packages: [ "docker" ],
|
||||||
|
runcmd: [
|
||||||
|
"systemctl daemon-reload",
|
||||||
|
"systemctl enable --now docker.service",
|
||||||
|
"grep docker /etc/group -q && usermod -a -G docker ec2-user"
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const gitlab_runners_launchTemplate = new aws.ec2.LaunchTemplate(
|
||||||
|
"gitlab-runners",
|
||||||
|
{
|
||||||
|
name: "GitlabRunners",
|
||||||
|
imageId: ami_amazonLinux_x86_64_latest.apply(amis => amis.id),
|
||||||
|
vpcSecurityGroupIds: [ gitlab_runners_securityGroup.id ],
|
||||||
|
userData: gitlab_runners_userData.rendered,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const gitlab_runners_autoScalingGroup = new aws.autoscaling.Group(
|
||||||
|
"gitlab-runners",
|
||||||
|
{
|
||||||
|
name: "GitlabRunners",
|
||||||
|
tags: [
|
||||||
|
{
|
||||||
|
key: "Owner",
|
||||||
|
value: "infra@example.org",
|
||||||
|
propagateAtLaunch: true,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
vpcZoneIdentifiers: subnet_ids,
|
||||||
|
minSize: 0,
|
||||||
|
maxSize: 2,
|
||||||
|
desiredCapacity: 0,
|
||||||
|
mixedInstancesPolicy:{
|
||||||
|
instancesDistribution: {
|
||||||
|
onDemandBaseCapacity: 1,
|
||||||
|
onDemandPercentageAboveBaseCapacity: 0,
|
||||||
|
|
||||||
|
// be mindful of prices
|
||||||
|
// https://docs.aws.amazon.com/autoscaling/ec2/userguide/allocation-strategies.html#spot-allocation-strategy
|
||||||
|
spotAllocationStrategy: "price-capacity-optimized",
|
||||||
|
},
|
||||||
|
launchTemplate: {
|
||||||
|
launchTemplateSpecification: {
|
||||||
|
launchTemplateId: gitlab_runners_launchTemplate.id,
|
||||||
|
version: "$Latest",
|
||||||
|
},
|
||||||
|
overrides: [
|
||||||
|
{ instanceType: aws.ec2.InstanceType.M6a_XLarge },
|
||||||
|
{ instanceType: aws.ec2.InstanceType.M6i_XLarge },
|
||||||
|
{ instanceType: aws.ec2.InstanceType.M7a_XLarge },
|
||||||
|
{ instanceType: aws.ec2.InstanceType.M7i_XLarge },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
// runners - end
|
||||||
|
|
||||||
|
// manager - start
|
||||||
|
const gitlab_runner_autoscalingManager_role = new aws.iam.Role(
|
||||||
|
"gitlab-runner-autoscalingManager",
|
||||||
|
{
|
||||||
|
name: "GitlabRunnerAutoscalingManager",
|
||||||
|
description: "Allow Gitlab Runner autoscaling managers to scale runner instances",
|
||||||
|
assumeRolePolicy: JSON.stringify({
|
||||||
|
Version: "2012-10-17",
|
||||||
|
Statement: [{
|
||||||
|
Sid: "AllowEc2ToAssumeThisVeryRole",
|
||||||
|
Effect: "Allow",
|
||||||
|
Principal: {
|
||||||
|
Service: "ec2.amazonaws.com",
|
||||||
|
},
|
||||||
|
Action: "sts:AssumeRole",
|
||||||
|
}],
|
||||||
|
}),
|
||||||
|
managedPolicyArns: [
|
||||||
|
"arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore", // instance management via SSM
|
||||||
|
],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
pulumi.all([
|
||||||
|
gitlab_runners_autoScalingGroup.arn,
|
||||||
|
gitlab_runners_autoScalingGroup.name,
|
||||||
|
awsRegion_output,
|
||||||
|
callerIdentity_output,
|
||||||
|
]).apply(
|
||||||
|
([ asgArn, asgName, awsRegion, callerIdentity ]) => new aws.iam.RolePolicy(
|
||||||
|
"gitlab-runner-autoscalingManager-inline-allowRoleFunctions",
|
||||||
|
{
|
||||||
|
role: gitlab_runner_autoscalingManager_role,
|
||||||
|
name: "AllowRoleFunctions",
|
||||||
|
policy: JSON.stringify({
|
||||||
|
Version: "2012-10-17",
|
||||||
|
Statement: [
|
||||||
|
{
|
||||||
|
Sid: "AllowAsgDiscovering",
|
||||||
|
Effect: "Allow",
|
||||||
|
Action: [
|
||||||
|
"autoscaling:DescribeAutoScalingGroups",
|
||||||
|
"ec2:DescribeInstances",
|
||||||
|
],
|
||||||
|
Resource: "*"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Sid: "AllowAsgScaling",
|
||||||
|
Effect: "Allow",
|
||||||
|
Action: [
|
||||||
|
"autoscaling:SetDesiredCapacity",
|
||||||
|
"autoscaling:TerminateInstanceInAutoScalingGroup"
|
||||||
|
],
|
||||||
|
Resource: asgArn,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Sid: "AllowManagingAccessToAsgInstances",
|
||||||
|
Effect: "Allow",
|
||||||
|
Action: "ec2-instance-connect:SendSSHPublicKey",
|
||||||
|
Resource: `arn:aws:ec2:${awsRegion.name}:${callerIdentity.accountId}:instance/*`,
|
||||||
|
Condition: {
|
||||||
|
StringEquals: {
|
||||||
|
"ec2:ResourceTag/aws:autoscaling:groupName": asgName,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Sid: "AllowAuthenticatingToEcr",
|
||||||
|
Effect: "Allow",
|
||||||
|
Action: "ecr:GetAuthorizationToken",
|
||||||
|
Resource: "*",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Sid: "AllowPullingImagesFromEcr",
|
||||||
|
Effect: "Allow",
|
||||||
|
Action: [
|
||||||
|
"ecr:BatchGetImage",
|
||||||
|
"ecr:GetDownloadUrlForLayer",
|
||||||
|
],
|
||||||
|
Resource: "*",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
|
||||||
|
),
|
||||||
|
);
|
||||||
|
const gitlab_runner_autoscalingManager_instanceProfile = new aws.iam.InstanceProfile(
|
||||||
|
"gitlab-runner-autoscalingManager",
|
||||||
|
{
|
||||||
|
name: "GitlabRunnerAutoscalingManager",
|
||||||
|
role: gitlab_runner_autoscalingManager_role,
|
||||||
|
},
|
||||||
|
{ protect: true }
|
||||||
|
);
|
||||||
|
const gitlab_runner_autoscalingManager_userData = pulumi.all([
|
||||||
|
gitlab_runners_autoScalingGroup.name,
|
||||||
|
awsRegion_output,
|
||||||
|
callerIdentity_output,
|
||||||
|
gitlab_url,
|
||||||
|
token,
|
||||||
|
]).apply(
|
||||||
|
([ asgName, awsRegion, callerIdentity, gitlabUrl, token ]) => new cloudinit.Config(
|
||||||
|
"gitlab-runner-autoscalingManager",
|
||||||
|
{
|
||||||
|
parts: [
|
||||||
|
{
|
||||||
|
filename: "cloud-config.docker-engine.yml",
|
||||||
|
contentType: "text/cloud-config",
|
||||||
|
content: yaml.stringify({
|
||||||
|
package_upgrade: false,
|
||||||
|
packages: [
|
||||||
|
"docker",
|
||||||
|
"amazon-ecr-credential-helper",
|
||||||
|
],
|
||||||
|
write_files: [
|
||||||
|
{
|
||||||
|
path: "/root/.docker/config.json",
|
||||||
|
permissions: "0644",
|
||||||
|
content: `{ "credsStore": "ecr-login" }`,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
runcmd: [
|
||||||
|
"systemctl daemon-reload",
|
||||||
|
"systemctl enable --now docker.service",
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
filename: "cloud-config.gitlab-runner.yml",
|
||||||
|
mergeType: "dict(recurse_array,no_replace)+list(append)",
|
||||||
|
contentType: "text/cloud-config",
|
||||||
|
content: yaml.stringify({
|
||||||
|
package_upgrade: false,
|
||||||
|
yum_repos: {
|
||||||
|
"gitlab-runner": {
|
||||||
|
name: "Gitlab Runner",
|
||||||
|
baseurl: "https://packages.gitlab.com/runner/gitlab-runner/amazon/2023/$basearch",
|
||||||
|
gpgcheck: true,
|
||||||
|
gpgkey: [
|
||||||
|
"https://packages.gitlab.com/runner/gitlab-runner/gpgkey",
|
||||||
|
"https://packages.gitlab.com/runner/gitlab-runner/gpgkey/runner-gitlab-runner-4C80FB51394521E9.pub.gpg",
|
||||||
|
"https://packages.gitlab.com/runner/gitlab-runner/gpgkey/runner-gitlab-runner-49F16C5CC3A0F81F.pub.gpg",
|
||||||
|
].join("\n"),
|
||||||
|
sslverify: true,
|
||||||
|
sslcacert: "/etc/pki/tls/certs/ca-bundle.crt",
|
||||||
|
metadata_expire: 300,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
write_files: [
|
||||||
|
{
|
||||||
|
path: "/etc/gitlab-runner/config.toml",
|
||||||
|
permissions: "0600",
|
||||||
|
content: [
|
||||||
|
`concurrent = 1`,
|
||||||
|
`check_interval = 0`,
|
||||||
|
`shutdown_timeout = 0`,
|
||||||
|
``,
|
||||||
|
`[session_server]`,
|
||||||
|
` session_timeout = 1800`,
|
||||||
|
``,
|
||||||
|
`[[runners]]`,
|
||||||
|
` name = "runner autoscaler"`,
|
||||||
|
``,
|
||||||
|
` url = "${gitlabUrl}"`,
|
||||||
|
` token = "${token}"`,
|
||||||
|
``,
|
||||||
|
` executor = "docker-autoscaler"`,
|
||||||
|
` environment = [ "AWS_REGION=${awsRegion.name}" ]`,
|
||||||
|
``,
|
||||||
|
` [runners.docker]`,
|
||||||
|
` privileged = false`,
|
||||||
|
``,
|
||||||
|
` image = "${callerIdentity.accountId}.dkr.ecr.${awsRegion.name}.amazonaws.com/some-repo/busybox:latest"`,
|
||||||
|
` pull_policy = [`,
|
||||||
|
` "if-not-present",`,
|
||||||
|
` "always"`,
|
||||||
|
` ]`,
|
||||||
|
` allowed_pull_policies = [`,
|
||||||
|
` "if-not-present",`,
|
||||||
|
` "always",`,
|
||||||
|
` "never"`,
|
||||||
|
` ]`,
|
||||||
|
``,
|
||||||
|
` [runners.autoscaler]`,
|
||||||
|
` plugin = "aws"`,
|
||||||
|
``,
|
||||||
|
` [runners.autoscaler.plugin_config]`,
|
||||||
|
` name = "${asgName}"`,
|
||||||
|
``,
|
||||||
|
` [[runners.autoscaler.policy]]`,
|
||||||
|
` periods = [ "* 7-19 * * mon-fri" ]`,
|
||||||
|
` timezone = "Europe/Amsterdam"`,
|
||||||
|
` idle_count = 1`,
|
||||||
|
` idle_time = "20m0s"`,
|
||||||
|
].join("\n"), // FIXME: granted, this sucks but at least I can interpolate in it
|
||||||
|
},
|
||||||
|
{
|
||||||
|
path: "/root/.aws/config",
|
||||||
|
permissions: "0600",
|
||||||
|
content: [
|
||||||
|
`[default]`,
|
||||||
|
`region = ${awsRegion.name}`,
|
||||||
|
].join("\n"),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
packages: [ "gitlab-runner-17.4.0" ],
|
||||||
|
runcmd: [
|
||||||
|
"systemctl daemon-reload",
|
||||||
|
"systemctl enable --now 'gitlab-runner'",
|
||||||
|
"gitlab-runner fleeting install",
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
|
||||||
|
),
|
||||||
|
);
|
||||||
|
new aws.ec2.Instance(
|
||||||
|
"gitlab-runner-autoscalingManager",
|
||||||
|
{
|
||||||
|
tags: {
|
||||||
|
Name: "Gitlab Runner autoscaling manager",
|
||||||
|
},
|
||||||
|
ami: ami_amazonLinux_arm64_latest.apply(ami => ami.id),
|
||||||
|
instanceType: aws.ec2.InstanceType.T4g_Micro,
|
||||||
|
iamInstanceProfile: gitlab_runner_autoscalingManager_instanceProfile,
|
||||||
|
subnetId: subnet_ids[0],
|
||||||
|
associatePublicIpAddress: false,
|
||||||
|
vpcSecurityGroupIds: [ gitlab_runner_autoscalingManager_securityGroup.id ],
|
||||||
|
userData: gitlab_runner_autoscalingManager_userData.rendered,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
// manager - end
|
||||||
3606
examples/pulumi/aws/gitlab-runner using docker-autoscaler/package-lock.json
generated
Normal file
3606
examples/pulumi/aws/gitlab-runner using docker-autoscaler/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"name": "aws_gitlab-runner-autoscaler_docker-autoscaler",
|
||||||
|
"main": "index.ts",
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^18",
|
||||||
|
"typescript": "^5.0.0"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@pulumi/aws": "^6.55.0",
|
||||||
|
"@pulumi/cloudinit": "^1.4.7",
|
||||||
|
"@pulumi/pulumi": "^3.136.1",
|
||||||
|
"yaml": "^2.5.1"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"strict": true,
|
||||||
|
"outDir": "bin",
|
||||||
|
"target": "es2020",
|
||||||
|
"module": "commonjs",
|
||||||
|
"moduleResolution": "node",
|
||||||
|
"sourceMap": true,
|
||||||
|
"experimentalDecorators": true,
|
||||||
|
"pretty": true,
|
||||||
|
"noFallthroughCasesInSwitch": true,
|
||||||
|
"noImplicitReturns": true,
|
||||||
|
"forceConsistentCasingInFileNames": true
|
||||||
|
},
|
||||||
|
"files": [
|
||||||
|
"index.ts"
|
||||||
|
]
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user