fix(gitlab): remarks about runners and certificate

.vscode/settings.json

@@ -297,6 +297,7 @@
         "sdkman",
         "setfacl",
         "setfattr",
+        "sidekiq",
         "siem",
         "slurm",
         "snmp",

examples/ansible/role.gitlab-omnibus-on-ec2/handlers/certify/package.yml

@@ -35,7 +35,7 @@
     account_key_src: "{{ letsencrypt_privatekey_path }}"
     account_email: "{{ acme_account_email }}"
     csr: "{{ certificate_csr_path }}"
-    cert: "{{ certificate_path }}"
+    fullchain: "{{ certificate_path }}"
     remaining_days: 29
     terms_agreed: true
     data: "{{ dns_challenge }}"

examples/ansible/role.gitlab-omnibus-on-ec2/tasks/certify/package.yml

@@ -45,7 +45,7 @@
     account_key_src: "{{ letsencrypt_privatekey_path }}"
     account_email: "{{ acme_account_email }}"
     csr: "{{ certificate_csr_path }}"
-    cert: "{{ certificate_path }}"
+    fullchain: "{{ certificate_path }}"
     terms_agreed: true
     remaining_days: 29
   register: dns_challenge

knowledge base/gitlab/README.md

@@ -46,7 +46,7 @@ Default backup location: `/var/opt/gitlab/backups`.
 Refer [Install self-managed GitLab].
 
 ```sh
-sudo dnf install 'gitlab-ee'
+sudo dnf install 'gitlab-ee-16.11.6'
 sudo EXTERNAL_URL='http://gitlab.example.com' GITLAB_ROOT_PASSWORD='smthng_Strong_0r_it_llfail' apt install 'gitlab-ee'
 
 sudo gitlab-rake 'gitlab:env:info'
@@ -96,8 +96,8 @@ Backup settings for AWS buckets.</br>
 See [Back up Gitlab using Amazon S3]:
 
 ```rb
-# If using an IAM Profile, don't configure 'aws_access_key_id' and
-# 'aws_secret_access_key' but set "'use_iam_profile' => true" instead.
+# If using an IAM Profile, don't configure 'aws_access_key_id' and 'aws_secret_access_key'.
+# Set "'use_iam_profile' => true" instead.
 gitlab_rails['backup_upload_connection'] = {
   'provider' => 'AWS',
   'region' => 'eu-west-1',
@@ -121,6 +121,9 @@ given URL's schema is HTTPS.<br/>
 The Let's Encrypt account key is in OpenSSL format, while the certificate's key is in OpenSSH format. Both are **not**
 password protected.
 
+The certificate used by Gitlab's nginx should include the full chain.<br/>
+The leaf-only certificate works normally, but runners seem to require the full chain to connect properly.
+
 </details>
 
 <details>
@@ -177,7 +180,7 @@ tmux new-session -As 'gitlab-upgrade' "sudo yum update 'gitlab-ee'"
 
 # Reset the root user's password.
 sudo gitlab-rake 'gitlab:password:reset[root]'
-sudo gitlab-rails console  \
+sudo gitlab-rails console
   # --> user = User.find_by_username 'root'
   # --> user.password = 'QwerTy184'
   # --> user.password_confirmation = 'QwerTy184'
@@ -206,12 +209,20 @@ Migration procedure:
 1. Reconfigure the new instance
 1. Restore the full backup on the new instance
 
+Check the [Upgrade Path tool] before upgrading.
+
+Upgrade procedure:
+
+1. Upgrade to the latest **patch** version of the current minor first.
+1. Upgrade to the **latest** patch version of **every** mandatory step.
+1. Upgrade runners to the nearest minor version of the main instance.
+
 </details>
 
 <details>
   <summary>Removal</summary>
 
-Refer <https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/master/doc/installation/index.md#uninstall-the-linux-package-omnibus>.
+Refer [Uninstall the Linux Package (Omnibus)].
 
 ```sh
 # Remove all users and groups created by the package.
@@ -221,7 +232,7 @@ sudo gitlab-ctl stop && sudo gitlab-ctl remove-accounts
 sudo gitlab-ctl cleanse && sudo rm -r '/opt/gitlab'
 
 # Uninstall the package.
-sudo apt remove 'gitlab-ee'
+sudo apt remove 'gitlab-ce'
 sudo dnf remove 'gitlab-ee'
 ```
 
@@ -699,6 +710,7 @@ Solution: set the correct ownership with
 - [Kaniko]
 - [The GitLab Handbook]
 - [Icons]
+- [Upgrade Path tool]
 
 ### Sources
 
@@ -776,6 +788,8 @@ Solution: set the correct ownership with
 [the gitlab handbook]: https://handbook.gitlab.com/
 [tls]: https://docs.gitlab.com/charts/installation/tls.html
 [tutorial: use buildah in a rootless container with gitlab runner operator on openshift]: https://docs.gitlab.com/ee/ci/docker/buildah_rootless_tutorial.html
+[uninstall the linux package (omnibus)]: https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/master/doc/installation/index.md#uninstall-the-linux-package-omnibus
+[upgrade path tool]: https://gitlab-com.gitlab.io/support/toolbox/upgrade-path/
 [use kaniko to build docker images]: https://docs.gitlab.com/ee/ci/docker/using_kaniko.html
 
 <!-- Others -->

knowledge base/gitlab/runner.md

@@ -43,6 +43,8 @@ gitlab-runner exec docker \
 
 Each runner executor is assigned 1 task at a time.
 
+Runners seem to require the main instance to give the full certificate chain upon connection.
+
 ## Pull images from private AWS ECR registries
 
 1. Create an IAM Role in one's AWS account and attach it the

snippets/aws/commands.fish

@@ -167,3 +167,10 @@ aws iam list-users --no-cli-pager --query 'Users[].UserName' --output 'text' \
 | xargs -n1 -P (nproc) aws iam list-access-keys \
 	--query "AccessKeyMetadata[?AccessKeyId=='AKIA01234567890ABCDE'].UserName" --output 'json' --user \
 | jq -rs 'flatten|first'
+
+# Get details for access keys
+# When no user is specified, it displays only keys for the current one
+aws iam --no-cli-pager list-access-keys
+aws iam --no-cli-pager list-access-keys --user-name 'mark'
+
+watch -n '1' aws ec2 describe-instances --instance-ids 'i-0123456789abcdef0' --query 'Reservations[].Instances[].[State,StateTransitionReason]'

snippets/gitlab/gitlab-runner.sh

@@ -18,10 +18,18 @@ gitlab-runner exec docker \
 docker-machine ls
 docker-machine inspect
 
-docker-machine create --driver 'amazonec2' --amazonec2-access-key 'AKID… --amazonec2-secret-key '8T93C…' 'runner-autoscaled-01'
+docker-machine create --driver 'amazonec2' --amazonec2-access-key 'AKID…' --amazonec2-secret-key '8T93C…' 'runner-autoscaled-01'
 export AWS_ACCESS_KEY_ID='AKID…' AWS_SECRET_ACCESS_KEY='8T93C…' docker-machine create --driver 'amazonec2' 'runner-autoscaled-01'
 
 # Connect one's Docker Client to the Docker Engine running on virtual machines
 eval $(docker-machine env 'runner-hzfj7uiz-ec2-1721038998-d9d31b5a')
 
 docker-machine rm -y 'runner-r6mo9hn8-ec2-1721049931-49793fa7'
+
+# Just list configured runners
+gitlab-runner list -c '/etc/gitlab-runner/config.toml'
+
+# Check configured runners can connect to the main instance
+gitlab-runner verify -c '/etc/gitlab-runner/config.toml'
+# Also delete runners that have been removed from the main instance
+gitlab-runner verify … --delete

snippets/gitlab/package installation.sh

@@ -85,6 +85,10 @@ gitlab-rails runner '
 # Check services
 sudo gitlab-ctl status
 
+# Restart services
+sudo gitlab-ctl restart
+sudo gitlab-ctl restart nginx
+
 # Get logs
 sudo gitlab-ctl tail
 sudo gitlab-ctl tail 'prometheus'
@@ -175,3 +179,10 @@ sudo rm -rf '/etc/gitlab' '/opt/gitlab'
 sudo dnf -y remove --noautoremove 'gitlab-ee'
 
 ## Removal - end --------------------- #
+
+# Check the included PostgreSQL version
+sudo gitlab-psql -c 'SELECT version();'
+
+# Check the status of batched background migrations
+# Should the query return zero rows, all batched background migrations are complete
+sudo gitlab-psql -c "SELECT job_class_name, table_name, column_name, job_arguments FROM batched_background_migrations WHERE status NOT IN(3, 6);"