chore: create certificates for gitlab using pulumi, letsencrypt and cloud-init

2026-02-09 21:54:24 +00:00 · 2024-04-28 21:08:37 +02:00
parent 3bc2588dc2
commit c878f56563
4 changed files with 115 additions and 59 deletions
--- a/base/acme.placeholder
+++ b/base/acme.placeholder
@@ -0,0 +1,3 @@
+# Automated Certificate Management Environment
+
+[]: https://www.keyfactor.com/blog/what-is-acme-protocol-and-how-does-it-work/
--- a/abbreviations.md
+++ b/abbreviations.md
@@ -2,6 +2,7 @@

 | Acronym | Expansion                                              | Description                                                                                         |
 | ------- | ------------------------------------------------------ | --------------------------------------------------------------------------------------------------- |
+| ACME    | [Automatic Certificate Management Environment]         | Protocol to automate the issuance and renewal of certificates without human interaction             |
 | ACK     | ACKnowledgement                                        |                                                                                                     |
 | ACL     | [Access Control List][acl]                             |                                                                                                     |
 | AD      | Active Directory                                       |                                                                                                     |
@@ -139,7 +140,8 @@
 [zstd]: zstd.md

 <!-- Others -->
-[cn (canonicalname vs commonname) in active directory explained]: https://www.itechguides.com/what-is-cn-in-active-directory/
+[automatic certificate management environment]: https://en.wikipedia.org/wiki/Automatic_Certificate_Management_Environment
+[cn (canonicalName vs commonName) in active directory explained]: https://www.itechguides.com/what-is-cn-in-active-directory/
 [continuous delivery]: https://en.wikipedia.org/wiki/Continuous_delivery
 [kiss principle is not that simple]: https://artero.dev/posts/kiss-principle-is-not-that-simple/
 [sbom at a glance]: https://www.ntia.gov/sites/default/files/publications/sbom_at_a_glance_apr2021_0.pdf
--- a/base/cloud-init.md
+++ b/base/cloud-init.md
@@ -19,28 +19,42 @@ sudo cloud-init query userdata
 sudo cat /var/lib/cloud/instance/user-data.txt | gunzip

 # Assert the user data we provided is a valid cloud-config.
-# From version 22.2, drops the 'devel' command.
-sudo cloud-init devel schema --system --annotate
-sudo cloud-init devel schema --config-file '/tmp/user-data'
+# A.K.A. validate files.
+# Versions <22.2 require 'devel schema' instead of only 'schema'.
+sudo cloud-init schema -c '/var/lib/cloud/instance/user-data.txt'
+sudo cloud-init schema --system --annotate
+
+# Check the user scripts.
+ls '/var/lib/cloud/instance/scripts'

 # Check the raw logs.
-cat '/var/log/cloud-init.log'
 cat '/var/log/cloud-init-output.log'
+tail -f '/var/log/cloud-init.log' '/var/log/cloud-init-output.log'

 # Parse and organize the events in the log file by stage.
 cloud-init analyze show

-# Manually run a single cloud-config module once after the instance has booted.
-sudo cloud-init single --name 'cc_ssh' --frequency 'always'
-
 # Clean up everything so `cloud-init` can run again.
 sudo cloud-init clean

 # Re-run everything.
-sudo cloud-init init
+# 1. Clean the existing configuration.
+# 2. Detect local data sources.
+# 3. Detect any data source requiring the network and run the 'initialization' modules.
+# 4. Run the 'configuration' modules.
+# 5. Run the 'final' modules.
+sudo cloud-init clean --logs \
+&& sudo cloud-init init --local \
+&& sudo cloud-init init \
+&& sudo cloud-init modules --mode='config' \
+&& sudo cloud-init modules -m 'final'

-# Check the user scripts.
-ls '/var/lib/cloud/instance/scripts'
+# Manually run a single cloud-config module once after the instance has booted.
+# Requires one to delete the semaphores in /var/lib/cloud/instances/hostname/sem. Cloud-init will not re-run if these
+# files are present.
+sudo rm -fv '/var/lib/cloud/instances'/*/'sem/config_ssh' && sudo cloud-init single --name 'cc_ssh' --frequency 'always'
+sudo rm -fv '/var/lib/cloud/instances'/*/'sem/config_write_files_deferred' \
+&& sudo cloud-init single -n 'write_files_deferred' --frequency 'once'
 ```

 ```yaml
@@ -107,8 +121,8 @@ merge_type: 'list(append)+dict(recurse_array)+str()'

   ```ts
   new cloudinit.Config("example", {
-     gzip: false,
-     base64Encode: false,
+     gzip: true,
+     base64Encode: true,
     parts: [
       {
         contentType: "text/cloud-config",
@@ -125,6 +139,32 @@ merge_type: 'list(append)+dict(recurse_array)+str()'
         filename: "cloud-config.inline.yml",
         mergeType: "dict(recurse_array,no_replace)+list(append)",
       },
+      {
+        contentType: "text/cloud-config",
+        content: pulumi.all([
+          certificate.certificateDomain.apply(v => v),
+          certificate.certificatePem.apply(v => v),
+          certificate.privateKeyPem.apply(v => v),
+        ]).apply(([domain, certificate, privateKey]) => yaml.stringify({
+          write_files: [
+            {
+              path: `/etc/gitlab/ssl/${domain}.crt`,
+              content: certificate,
+              permissions: "0o600",
+              defer: true,
+            },
+            {
+              path: `/etc/gitlab/ssl/${domain}.key`,
+              content: btoa(privateKey),
+              permissions: "0o600",
+              encoding: "base64",
+              defer: true,
+            },
+          ],
+        })),
+        filename: "letsencrypt.certificate.yml",
+        mergeType: "dict(recurse_array,no_replace)+list(append)",
+      },
     ],
   });
   ```
@@ -205,6 +245,7 @@ merge_type: 'list(append)+dict(recurse_array)+str()'
 - [Cloud-Init configuration merging]
 - [Terraform's cloud-init provider]
 - [How to test cloud-init locally with Vagrant]
+- [How to re-run cloud-init without reboot]

 <!--
  References
@@ -226,6 +267,7 @@ merge_type: 'list(append)+dict(recurse_array)+str()'
 <!-- Others -->
 [cloud-init configuration merging]: https://jen20.dev/post/cloudinit-configuration-merging/
 [cloud-init multipart encoding issues]: https://github.com/hashicorp/terraform/issues/4794
+[how to re-run cloud-init without reboot]: https://stackoverflow.com/questions/23065673/how-to-re-run-cloud-init-without-reboot#71152408
 [how to test cloud-init locally with vagrant]: https://www.grzegorowski.com/how-to-test-cloud-init-locally-with-vagrant
 [terraform's cloud-init provider]: https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config
 [test cloud-init with a multipass container]: https://medium.com/open-devops-academy/test-cloud-init-with-a-multipass-containers-e3e3bb740604
--- a/base/pulumi.md
+++ b/base/pulumi.md
@@ -249,22 +249,22 @@ const cluster = new aws.eks.Cluster("cluster", {

 // If used in JSON documents, the function needs to cover the whole document.
 const encryptionKey = aws.kms.getKeyOutput({
-    keyId: "00001111-2222-3333-4444-555566667777",
+  keyId: "00001111-2222-3333-4444-555566667777",
 });
 const clusterServiceRole = new aws.iam.Role("clusterServiceRole", {
-    inlinePolicies: [{
-        policy: encryptionKey.arn.apply(arn => JSON.stringify({
-            Version: "2012-10-17",
-            Statement: [{
-                Effect: "Allow",
-                Action: [
-                    "kms:CreateGrant",
-                    "kms:DescribeKey",
-                ],
-                Resource: arn,
-            }],
-        })),
-    }]
+  inlinePolicies: [{
+    policy: encryptionKey.arn.apply(arn => JSON.stringify({
+      Version: "2012-10-17",
+      Statement: [{
+        Effect: "Allow",
+        Action: [
+          "kms:CreateGrant",
+          "kms:DescribeKey",
+        ],
+        Resource: arn,
+      }],
+    })),
+  }]
 });
 ```

@@ -345,36 +345,36 @@ yq -iy '. += {"backend": {"url": "s3://myBucket/prefix"}}' 'Pulumi.yaml'
 ```ts
 // Merge objects.
 tags_base = {
-    ManagedBy: "Pulumi",
-    Prod: false,
+  ManagedBy: "Pulumi",
+  Prod: false,
 };
 new aws.eks.FargateProfile("fargateProfile", {
-    tags: {
-        ...tags_base,
-        ...{
-            Description: "Fargate profile for EKS cluster EksTest",
-            EksComponent: "Fargate profile",
-            Name: "eksTest-fargateProfile",
-        },
+  tags: {
+    ...tags_base,
+    ...{
+      Description: "Fargate profile for EKS cluster EksTest",
+      EksComponent: "Fargate profile",
+      Name: "eksTest-fargateProfile",
    },
-    …
+  },
+  …
 });

 // Default tags with explicit provider.
 const provider = new aws.Provider("provider", {
-    defaultTags: {
-        tags: {
-            ManagedBy: "Pulumi",
-            Owner: "user@company.com",
-            Team: "Infra",
-        },
+  defaultTags: {
+    tags: {
+      ManagedBy: "Pulumi",
+      Owner: "user@company.com",
+      Team: "Infra",
    },
+  },
 });
 new aws.eks.FargateProfile("fargateProfile", {
-    …
+  …
 }, {
-    provider: provider,
-    …
+  provider: provider,
+  …
 });

 // Use outputs from other stacks.
@@ -382,11 +382,11 @@ const currentStack = pulumi.getStack();
 const infraStack = new pulumi.StackReference(`organization/infra/${currentStack}`);
 const subnets_private = infraStack.getOutput("subnets_private");  // list of aws.ec2.Subnets
 new aws.eks.Cluster("cluster", {
-    vpcConfig: {
-        subnetIds: subnets_private.apply((subnets: aws.ec2.Subnet[]) => subnets.map(subnet => subnet.id)),
-        …
-    },
+  vpcConfig: {
+    subnetIds: subnets_private.apply((subnets: aws.ec2.Subnet[]) => subnets.map(subnet => subnet.id)),
    …
+  },
+  …
 });

 // Debug the .apply() result of Outputs.
@@ -396,6 +396,15 @@ subnets_private.apply(
 subnets_private.apply(
  (subnets: aws.ec2.Subnet[]) => console.log(subnets.map(subnet => subnet.id)),
 );  // [ 'subnet-00001111222233334', … ]
+
+// Use multiple Outputs.
+pulumi.all([
+  aws.getRegionOutput().apply(region => region.id),
+  aws.getCallerIdentityOutput().apply(callerIdentity => callerIdentity.accountId),
+  cluster.name,
+]).apply(
+  ([regionId, accountId, clusterName]) => `arn:aws:eks:${regionId}:${accountId}:fargateprofile/${clusterName}/*`
+);
 ```

 </details>
@@ -503,19 +512,19 @@ Read [Assigning tags by default on AWS with Pulumi] first to get an idea of pros

   ```ts
   const provider = new aws.Provider("provider", {
-       defaultTags: {
-           tags: {
-               ManagedBy: "Pulumi",
-               Owner: "user@company.com",
-               Team: "Infra",
-           },
+     defaultTags: {
+       tags: {
+         ManagedBy: "Pulumi",
+         Owner: "user@company.com",
+         Team: "Infra",
       },
+     },
   });
   const fargateProfile = new aws.eks.FargateProfile("fargateProfile", {
-       …
+     …
   }, {
-       provider: provider,
-       …
+     provider: provider,
+     …
   });
   ```