refactor: expand gitlab's kb with its own section and the intallation of k8s runners

2026-02-09 05:44:23 +00:00 · 2024-06-02 22:38:37 +02:00
parent e71edc6260
commit 107aa0993d
10 changed files with 324 additions and 39 deletions
--- a/examples/kubernetes/values.gitlab-runner.yaml
+++ b/examples/kubernetes/values.gitlab-runner.yaml
@@ -0,0 +1,125 @@
+---
+
+##
+# Source:
+# `helm template --namespace 'gitlab' --values "values.gitlab-runner.yaml" --set global.hosts.hostSuffix='test' \
+#   'gitlab-runner' 'gitlab/gitlab-runner'`
+##
+
+useTini: false
+
+image:
+  registry: registry.gitlab.com
+  image: gitlab-org/gitlab-runner
+imagePullPolicy: IfNotPresent
+
+gitlabUrl: https://gitlab.company.com/
+
+unregisterRunners: true
+
+terminationGracePeriodSeconds: 3600
+
+concurrent: 10
+shutdown_timeout: 0
+checkInterval: 30
+
+sessionServer:
+  enabled: false
+
+rbac:
+  create: true
+  generatedServiceAccountName: ""
+  rules: []
+  clusterWideAccess: false
+  podSecurityPolicy:
+    enabled: false
+    resourceNames:
+      - gitlab-runner
+
+metrics:
+  enabled: false
+  portName: metrics
+  port: 9252
+  serviceMonitor:
+    enabled: false
+
+service:
+  enabled: false
+  type: ClusterIP
+
+runners:
+  config: |
+    [[runners]]
+
+      [runners.cache]
+        Type = "s3"
+        Path = "runner/"
+        Shared = true
+        MaxUploadedArchiveSize = 0
+
+        [runners.cache.s3]
+          ServerAddress = "s3.amazonaws.com"
+          BucketName = "company-ci"
+          BucketLocation = "eu-west-1"
+
+      [runners.kubernetes]
+        namespace = "{{.Release.Namespace}}"
+        image = "alpine"
+        pull_policy = [
+          "if-not-present",
+          "always"
+        ]
+        allowed_pull_policies = [
+          "if-not-present",
+          "always",
+          "never"
+        ]
+
+  configPath: ""
+  name: "test-runner-on-k8s"
+  secret: gitlab-runner-token
+  cache: {}
+
+securityContext:
+  allowPrivilegeEscalation: false
+  readOnlyRootFilesystem: false
+  runAsNonRoot: true
+  privileged: false
+  capabilities:
+    drop: ["ALL"]
+
+strategy: {}
+
+podSecurityContext:
+  runAsUser: 100
+  fsGroup: 65533
+
+resources: {}
+
+affinity: {}
+topologySpreadConstraints: {}
+nodeSelector: {}
+tolerations: []
+
+extraEnv: {}
+extraEnvFrom: {}
+
+hostAliases: []
+
+deploymentAnnotations: {}
+deploymentLabels:
+  team: infra
+
+podAnnotations: {}
+podLabels:
+  team: infra
+
+priorityClassName: ""
+
+secrets: []
+configMaps: {}
+
+volumeMounts: []
+volumes: []
+
+extraObjects: []
--- a/base/gitea.md
+++ b/base/gitea.md
@@ -124,7 +124,7 @@ Alternatives:
 <!-- Knowledge base -->
 [self-hosting]: self-hosting.md
 [git]: git.md
-[gitlab]: gitlab.md
+[gitlab]: gitlab/README.md

 <!-- Files -->
 [compose file]: /docker/gitea/docker-compose.yml
--- a/base/gitlab/README.md
+++ b/base/gitlab/README.md
@@ -9,13 +9,11 @@
   1. [Different owners for parts of the code base](#different-owners-for-parts-of-the-code-base)
 1. [CI/CD pipelines](#cicd-pipelines)
   1. [Specify when to run jobs](#specify-when-to-run-jobs)
-      1. [Make a job in a pipeline run only when some specific files change](#make-a-job-in-a-pipeline-run-only-when-some-specific-files-change)
+      1. [Only when some specific files change](#only-when-some-specific-files-change)
   1. [Get the version of the helper image to use for a runner](#get-the-version-of-the-helper-image-to-use-for-a-runner)
 1. [Manage kubernetes clusters](#manage-kubernetes-clusters)
 1. [Maintenance mode](#maintenance-mode)
 1. [Runners](#runners)
-   1. [Autoscaling](#autoscaling)
-      1. [Docker Machine](#docker-machine)
 1. [Troubleshooting](#troubleshooting)
   1. [Use access tokens to clone projects](#use-access-tokens-to-clone-projects)
   1. [Pipeline fails with error `You are not allowed to download code from this project`](#pipeline-fails-with-error-you-are-not-allowed-to-download-code-from-this-project)
@@ -555,7 +553,7 @@ Use the `rules` key and specify the conditions the job needs.
 Conditions are validated **in order** until one applies. The rest are ignored.<br/>
 If no condition applies, the job is skipped.

-#### Make a job in a pipeline run only when some specific files change
+#### Only when some specific files change

 ```yaml
 docker-build:
@@ -662,29 +660,7 @@ Through Rails console:

 ## Runners

-```sh
-brew install 'gitlab-runner'
-```
-
-```sh
-gitlab-runner exec docker 'job-name'
-gitlab-runner exec docker \
-  --env 'AWS_ACCESS_KEY_ID=AKIA…' --env 'AWS_SECRET_ACCESS_KEY=F…s' --env 'AWS_REGION=eu-east-1' \
-  --env 'DOCKER_AUTH_CONFIG={ "credsStore": "ecr-login" }' \
-  --docker-volumes "$HOME/.aws/credentials:/root/.aws/credentials:ro"
-  'job-requiring-ecr-access'
-```
-
-### Autoscaling
-
-#### Docker Machine
-
-[Supported cloud providers][docker machine's supported cloud providers].
-
-Pitfalls:
-
- On AWS, the driver supports only one subnet.<br/>
-  See [AWS driver does not support multiple non default subnets] and [Docker Machine's AWS driver's options].
+See [runners](runner.md).

 ## Troubleshooting

@@ -751,9 +727,7 @@ Solution: set the correct ownership with
 - [Back up GitLab Using Amazon S3]
 - [Support object storage bucket prefixes]
 - [Back up GitLab excluding specific data from the backup]
- [AWS driver does not support multiple non default subnets]
 - [Autoscaling GitLab Runner on AWS EC2]
- [Authenticating your GitLab CI runner to an AWS ECR registry using Amazon ECR Docker Credential Helper]
 - [How to restart GitLab]
 - [Customize pipeline configuration]
 - [Code owners]
@@ -806,8 +780,6 @@ Solution: set the correct ownership with
 [command-line options]: https://docs.gitlab.com/charts/installation/command-line-options.html
 [customize pipeline configuration]: https://docs.gitlab.com/ee/ci/pipelines/settings.html
 [deployment]: https://docs.gitlab.com/charts/installation/deployment.html
-[docker machine's aws driver's options]: https://gitlab.com/gitlab-org/ci-cd/docker-machine/-/blob/main/docs/drivers/aws.md#options
-[docker machine's supported cloud providers]: https://docs.gitlab.com/runner/configuration/autoscale.html#supported-cloud-providers
 [environment variables]: https://docs.gitlab.com/ee/administration/environment_variables.html
 [gitlab ha scaling runner vending machine for aws ec2 asg]: https://gitlab.com/guided-explorations/aws/gitlab-runner-autoscaling-aws-asg#gitlab-runners-on-aws-spot-best-practices
 [global settings]: https://docs.gitlab.com/charts/charts/globals.html
@@ -836,8 +808,6 @@ Solution: set the correct ownership with
 [gitlab maintenance mode]: https://docs.gitlab.com/ee/administration/maintenance_mode/

 <!-- Others -->
-[authenticating your gitlab ci runner to an aws ecr registry using amazon ecr docker credential helper]: https://faun.pub/authenticating-your-gitlab-ci-runner-to-an-aws-ecr-registry-using-amazon-ecr-docker-credential-b4604a9391eb
-[aws driver does not support multiple non default subnets]: https://github.com/docker/machine/issues/4700
 [chef infra]: https://www.chef.io/products/chef-infra
 [configuring private dns zones and upstream nameservers in kubernetes]: https://kubernetes.io/blog/2017/04/configuring-private-dns-zones-upstream-nameservers-kubernetes/
 [how to disable the two-factor authentication in gitlab?]: https://stackoverflow.com/questions/31024771/how-to-disable-the-two-factor-authentication-in-gitlab
--- a/base/gitlab/runner.md
+++ b/base/gitlab/runner.md
@@ -0,0 +1,151 @@
+# Gitlab runner
+
+TODO
+
+1. [TL;DR](#tldr)
+1. [Pull images from private AWS ECR registries](#pull-images-from-private-aws-ecr-registries)
+1. [Autoscaling](#autoscaling)
+   1. [Docker Machine](#docker-machine)
+1. [Further readings](#further-readings)
+   1. [Sources](#sources)
+
+## TL;DR
+
+<details>
+  <summary>Installation</summary>
+
+```sh
+brew install 'gitlab-runner'
+dnf install 'gitlab-runner'
+docker pull 'gitlab/gitlab-runner'
+helm --namespace 'gitlab' upgrade --install --create-namespace --version '0.64.1' --repo 'https://charts.gitlab.io' \
+  'gitlab-runner' -f 'values.gitlab-runner.yml' 'gitlab/gitlab-runner'
+```
+
+</details>
+
+<details>
+  <summary>Usage</summary>
+
+```sh
+docker run --rm --name 'runner' 'gitlab/gitlab-runner:alpine-v13.6.0' --version
+
+# `gitlab-runner exec` is deprecated and has been removed in 17.0. ┌П┐(ಠ_ಠ) Gitlab.
+# See https://docs.gitlab.com/16.11/runner/commands/#gitlab-runner-exec-deprecated.
+gitlab-runner exec docker 'job-name'
+gitlab-runner exec docker \
+  --env 'AWS_ACCESS_KEY_ID=AKIA…' --env 'AWS_SECRET_ACCESS_KEY=F…s' --env 'AWS_REGION=eu-east-1' \
+  --env 'DOCKER_AUTH_CONFIG={ "credsStore": "ecr-login" }' \
+  --docker-volumes "$HOME/.aws/credentials:/root/.aws/credentials:ro"
+  'job-requiring-ecr-access'
+```
+
+</details>
+
+## Pull images from private AWS ECR registries
+
+1. Create an IAM Role in one's AWS account and attach it the
+   `arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly` IAM policy.
+1. Create and InstanceProfile using the above IAM Role.
+1. Create an EC2 Instance.<br/>
+   Make it use the above InstanceProfile.
+1. Install the Docker Engine and the [Gitlab runner][install gitlab runner] on the EC2 Instance.
+1. Install the [Amazon ECR Docker Credential Helper].
+1. Configure an AWS Region in `/root/.aws/config`:
+
+   ```ini
+   [default]
+   region = eu-west-1
+   ```
+
+1. Create the `/root/.docker/config.json` file and add the following line to it:
+
+   ```diff
+    {
+      …
+   + "credsStore": "ecr-login"
+    }
+   ```
+
+1. Configure the runner to use the [`docker`][docker executor] or [`docker+machine`][docker machine] executor.
+
+   ```toml
+   [[runners]]
+   executor = "docker"   # or "docker+machine"
+   ```
+
+1. Configure the runner to use the ECR Credential Helper:
+
+   ```toml
+   [[runners]]
+     [runners.docker]
+     environment = [ 'DOCKER_AUTH_CONFIG={"credsStore":"ecr-login"}' ]
+   ```
+
+1. Configure jobs to use images saved in private AWS ECR registries:
+
+   ```yaml
+   phpunit:
+     stage: testing
+     image:
+       name: 123456789123.dkr.ecr.eu-west-1.amazonaws.com/php-gitlabrunner:latest
+       entrypoint: [""]
+     script:
+       - php ./vendor/bin/phpunit --coverage-text --colors=never
+   ```
+
+Now your GitLab runner should automatically authenticate to one's private ECR registry.
+
+## Autoscaling
+
+### Docker Machine
+
+Runner like any others, just configured to use the `docker+machine` executor.
+
+[Supported cloud providers][docker machine's supported cloud providers].
+
+Pitfalls:
+
+- On AWS, the driver supports only one subnet.<br/>
+  See [AWS driver does not support multiple non default subnets] and [Docker Machine's AWS driver's options].
+
+## Further readings
+
+- [Gitlab]
+- [Amazon ECR Docker Credential Helper]
+- Gitlab's [docker machine] fork
+- Gitlab's [gitlab-runner-operator] for OpenShift and Kubernetes
+
+### Sources
+
+- [Install Gitlab runner]
+- [Docker executor]
+- [Authenticating your GitLab CI runner to an AWS ECR registry using Amazon ECR Docker Credential Helper]
+- [Install and register GitLab Runner for autoscaling with Docker Machine]
+- [AWS driver does not support multiple non default subnets]
+- [GitLab Runner Helm Chart]
+
+<!--
+  Reference
+  ═╬═Time══
+  -->
+
+<!-- In-article sections -->
+<!-- Knowledge base -->
+[gitlab]: README.md
+
+<!-- Files -->
+<!-- Upstream -->
+[docker executor]: https://docs.gitlab.com/17.0/runner/executors/docker.html
+[docker machine]: https://gitlab.com/gitlab-org/ci-cd/docker-machine
+[docker machine's aws driver's options]: https://gitlab.com/gitlab-org/ci-cd/docker-machine/-/blob/main/docs/drivers/aws.md#options
+[docker machine's supported cloud providers]: https://docs.gitlab.com/runner/configuration/autoscale.html#supported-cloud-providers
+[install gitlab runner]: https://docs.gitlab.com/runner/install/
+[install and register gitlab runner for autoscaling with docker machine]: https://docs.gitlab.com/17.0/runner/executors/docker_machine.html
+[gitlab-runner-operator]: https://gitlab.com/gitlab-org/gl-openshift/gitlab-runner-operator
+[gitlab runner helm chart]: https://docs.gitlab.com/runner/install/kubernetes.html
+
+<!-- Others -->
+[authenticating your gitlab ci runner to an aws ecr registry using amazon ecr docker credential helper]: https://faun.pub/authenticating-your-gitlab-ci-runner-to-an-aws-ecr-registry-using-amazon-ecr-docker-credential-b4604a9391eb
+[aws driver does not support multiple non default subnets]: https://github.com/docker/machine/issues/4700
+[amazon ecr docker credential helper]: https://github.com/awslabs/amazon-ecr-credential-helper
--- a/base/kubernetes/helm.md
+++ b/base/kubernetes/helm.md
@@ -2,8 +2,6 @@

 Package manager for Kubernetes.

-## Table of contents <!-- omit in toc -->
-
 1. [TL;DR](#tldr)
 1. [Start managing existing resources with a specific helm chart](#start-managing-existing-resources-with-a-specific-helm-chart)
 1. [Plugins](#plugins)
--- a/base/self-hosting.md
+++ b/base/self-hosting.md
@@ -34,7 +34,7 @@ The _art_ of hosting and managing applications on one's own servers instead of c
 <!-- Knowledge base -->
 [baikal]: baikal.md
 [gitea]: gitea.md
-[gitlab]: gitlab.md
+[gitlab]: gitlab/README.md
 [nextcloud]: nextcloud.md
 [paperless-ngx]: paperless-ngx.md
 [photoprism]: photoprism.md
--- a/base/task.md
+++ b/base/task.md
@@ -83,7 +83,7 @@ curl -fsSLo "$HOME/.config/fish/completions/task.fish" 'https://raw.githubuserco
 <!-- In-article sections -->
 <!-- Knowledge base -->
 [azure devops]: cloud%20computing/azure/devops.md
-[gitlab]: gitlab.md
+[gitlab]: gitlab/README.md
 [gnu make]: gnu%20userland/make.md

 <!-- Files -->
--- a/snippets/gitlab-runner.sh
+++ b/snippets/gitlab-runner.sh
@@ -1,4 +1,9 @@
-#!sh
+#!/usr/bin/env sh
+
+helm --namespace 'gitlab' upgrade --install --create-namespace --version '0.64.1' --repo 'https://charts.gitlab.io' \
+	'gitlab-runner' -f 'values.gitlab-runner.yml' 'gitlab/gitlab-runner'
+
+gitlab-runner register --url "https://gitlab.com/" --non-interactive --executor "shell" --token "glrt-…"

 gitlab-runner exec docker \
 	--env 'AWS_ACCESS_KEY_ID=AKIA…' --env 'AWS_SECRET_ACCESS_KEY=FsN4…' --env 'AWS_REGION=eu-west-1' \
--- a/snippets/helm.sh
+++ b/snippets/helm.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env sh
+
+helm repo list
+
+helm repo add 'gitlab' 'https://charts.gitlab.io'
+
+helm repo update
+helm repo update 'keda'
+
+helm search hub --max-col-width '100' 'ingress-nginx'
+helm search repo --versions 'gitlab/gitlab-runner'
+
+helm inspect values 'gitlab/gitlab'
+helm inspect values 'gitlab/gitlab-runner' --version '0.64.1'
+
+helm pull 'ingress-nginx/ingress-nginx' --version '4.0.6' --destination '/tmp' --untar --untardir 'ingress-nginx'
+
+helm template --namespace 'gitlab' --values "values.gitlab-runner.yaml" --set global.hosts.hostSuffix='test' \
+	'gitlab-runner' 'gitlab/gitlab-runner'
+
+helm --namespace 'gitlab' upgrade --install --create-namespace --version '0.64.1' 'gitlab-runner' \
+	--values 'values.gitlab-runner.yml' 'gitlab/gitlab-runner'
+helm upgrade --install 'keda' 'keda' --repo 'https://kedacore.github.io/charts' --namespace 'keda' --create-namespace
+
+helm get manifest 'wordpress'
+
+helm plugin list
--- a/snippets/kubectl.sh
+++ b/snippets/kubectl.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+kubectl create namespace 'gitlab'
+
+kubectl apply --namespace 'gitlab' --values 'secrets.yaml'
+
+# Requires the metrics server to be running in the cluster
+kubectl top nodes
+kubectl top pods