From c878f565634c8ab5db60de890e441aa8babc58df Mon Sep 17 00:00:00 2001 From: Michele Cereda Date: Sun, 28 Apr 2024 21:08:37 +0200 Subject: [PATCH] chore: create certificates for gitlab using pulumi, letsencrypt and cloud-init --- knowledge base/acme.placeholder | 3 + knowledge base/acronyms and abbreviations.md | 4 +- knowledge base/cloud-init.md | 66 +++++++++--- knowledge base/pulumi.md | 101 ++++++++++--------- 4 files changed, 115 insertions(+), 59 deletions(-) create mode 100644 knowledge base/acme.placeholder diff --git a/knowledge base/acme.placeholder b/knowledge base/acme.placeholder new file mode 100644 index 0000000..5a0479a --- /dev/null +++ b/knowledge base/acme.placeholder @@ -0,0 +1,3 @@ +# Automated Certificate Management Environment + +[]: https://www.keyfactor.com/blog/what-is-acme-protocol-and-how-does-it-work/ diff --git a/knowledge base/acronyms and abbreviations.md b/knowledge base/acronyms and abbreviations.md index f94c0be..b820948 100644 --- a/knowledge base/acronyms and abbreviations.md +++ b/knowledge base/acronyms and abbreviations.md @@ -2,6 +2,7 @@ | Acronym | Expansion | Description | | ------- | ------------------------------------------------------ | --------------------------------------------------------------------------------------------------- | +| ACME | [Automatic Certificate Management Environment] | Protocol to automate the issuance and renewal of certificates without human interaction | | ACK | ACKnowledgement | | | ACL | [Access Control List][acl] | | | AD | Active Directory | | @@ -139,7 +140,8 @@ [zstd]: zstd.md -[cn (canonicalname vs commonname) in active directory explained]: https://www.itechguides.com/what-is-cn-in-active-directory/ +[automatic certificate management environment]: https://en.wikipedia.org/wiki/Automatic_Certificate_Management_Environment +[cn (canonicalName vs commonName) in active directory explained]: https://www.itechguides.com/what-is-cn-in-active-directory/ [continuous delivery]: https://en.wikipedia.org/wiki/Continuous_delivery [kiss principle is not that simple]: https://artero.dev/posts/kiss-principle-is-not-that-simple/ [sbom at a glance]: https://www.ntia.gov/sites/default/files/publications/sbom_at_a_glance_apr2021_0.pdf diff --git a/knowledge base/cloud-init.md b/knowledge base/cloud-init.md index 377d62e..ea9fb2b 100644 --- a/knowledge base/cloud-init.md +++ b/knowledge base/cloud-init.md @@ -19,28 +19,42 @@ sudo cloud-init query userdata sudo cat /var/lib/cloud/instance/user-data.txt | gunzip # Assert the user data we provided is a valid cloud-config. -# From version 22.2, drops the 'devel' command. -sudo cloud-init devel schema --system --annotate -sudo cloud-init devel schema --config-file '/tmp/user-data' +# A.K.A. validate files. +# Versions <22.2 require 'devel schema' instead of only 'schema'. +sudo cloud-init schema -c '/var/lib/cloud/instance/user-data.txt' +sudo cloud-init schema --system --annotate + +# Check the user scripts. +ls '/var/lib/cloud/instance/scripts' # Check the raw logs. -cat '/var/log/cloud-init.log' cat '/var/log/cloud-init-output.log' +tail -f '/var/log/cloud-init.log' '/var/log/cloud-init-output.log' # Parse and organize the events in the log file by stage. cloud-init analyze show -# Manually run a single cloud-config module once after the instance has booted. -sudo cloud-init single --name 'cc_ssh' --frequency 'always' - # Clean up everything so `cloud-init` can run again. sudo cloud-init clean # Re-run everything. -sudo cloud-init init +# 1. Clean the existing configuration. +# 2. Detect local data sources. +# 3. Detect any data source requiring the network and run the 'initialization' modules. +# 4. Run the 'configuration' modules. +# 5. Run the 'final' modules. +sudo cloud-init clean --logs \ +&& sudo cloud-init init --local \ +&& sudo cloud-init init \ +&& sudo cloud-init modules --mode='config' \ +&& sudo cloud-init modules -m 'final' -# Check the user scripts. -ls '/var/lib/cloud/instance/scripts' +# Manually run a single cloud-config module once after the instance has booted. +# Requires one to delete the semaphores in /var/lib/cloud/instances/hostname/sem. Cloud-init will not re-run if these +# files are present. +sudo rm -fv '/var/lib/cloud/instances'/*/'sem/config_ssh' && sudo cloud-init single --name 'cc_ssh' --frequency 'always' +sudo rm -fv '/var/lib/cloud/instances'/*/'sem/config_write_files_deferred' \ +&& sudo cloud-init single -n 'write_files_deferred' --frequency 'once' ``` ```yaml @@ -107,8 +121,8 @@ merge_type: 'list(append)+dict(recurse_array)+str()' ```ts new cloudinit.Config("example", { - gzip: false, - base64Encode: false, + gzip: true, + base64Encode: true, parts: [ { contentType: "text/cloud-config", @@ -125,6 +139,32 @@ merge_type: 'list(append)+dict(recurse_array)+str()' filename: "cloud-config.inline.yml", mergeType: "dict(recurse_array,no_replace)+list(append)", }, + { + contentType: "text/cloud-config", + content: pulumi.all([ + certificate.certificateDomain.apply(v => v), + certificate.certificatePem.apply(v => v), + certificate.privateKeyPem.apply(v => v), + ]).apply(([domain, certificate, privateKey]) => yaml.stringify({ + write_files: [ + { + path: `/etc/gitlab/ssl/${domain}.crt`, + content: certificate, + permissions: "0o600", + defer: true, + }, + { + path: `/etc/gitlab/ssl/${domain}.key`, + content: btoa(privateKey), + permissions: "0o600", + encoding: "base64", + defer: true, + }, + ], + })), + filename: "letsencrypt.certificate.yml", + mergeType: "dict(recurse_array,no_replace)+list(append)", + }, ], }); ``` @@ -205,6 +245,7 @@ merge_type: 'list(append)+dict(recurse_array)+str()' - [Cloud-Init configuration merging] - [Terraform's cloud-init provider] - [How to test cloud-init locally with Vagrant] +- [How to re-run cloud-init without reboot] [cloud-init configuration merging]: https://jen20.dev/post/cloudinit-configuration-merging/ [cloud-init multipart encoding issues]: https://github.com/hashicorp/terraform/issues/4794 +[how to re-run cloud-init without reboot]: https://stackoverflow.com/questions/23065673/how-to-re-run-cloud-init-without-reboot#71152408 [how to test cloud-init locally with vagrant]: https://www.grzegorowski.com/how-to-test-cloud-init-locally-with-vagrant [terraform's cloud-init provider]: https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/cloudinit_config [test cloud-init with a multipass container]: https://medium.com/open-devops-academy/test-cloud-init-with-a-multipass-containers-e3e3bb740604 diff --git a/knowledge base/pulumi.md b/knowledge base/pulumi.md index 7d2c4d5..4d06653 100644 --- a/knowledge base/pulumi.md +++ b/knowledge base/pulumi.md @@ -249,22 +249,22 @@ const cluster = new aws.eks.Cluster("cluster", { // If used in JSON documents, the function needs to cover the whole document. const encryptionKey = aws.kms.getKeyOutput({ - keyId: "00001111-2222-3333-4444-555566667777", + keyId: "00001111-2222-3333-4444-555566667777", }); const clusterServiceRole = new aws.iam.Role("clusterServiceRole", { - inlinePolicies: [{ - policy: encryptionKey.arn.apply(arn => JSON.stringify({ - Version: "2012-10-17", - Statement: [{ - Effect: "Allow", - Action: [ - "kms:CreateGrant", - "kms:DescribeKey", - ], - Resource: arn, - }], - })), - }] + inlinePolicies: [{ + policy: encryptionKey.arn.apply(arn => JSON.stringify({ + Version: "2012-10-17", + Statement: [{ + Effect: "Allow", + Action: [ + "kms:CreateGrant", + "kms:DescribeKey", + ], + Resource: arn, + }], + })), + }] }); ``` @@ -345,36 +345,36 @@ yq -iy '. += {"backend": {"url": "s3://myBucket/prefix"}}' 'Pulumi.yaml' ```ts // Merge objects. tags_base = { - ManagedBy: "Pulumi", - Prod: false, + ManagedBy: "Pulumi", + Prod: false, }; new aws.eks.FargateProfile("fargateProfile", { - tags: { - ...tags_base, - ...{ - Description: "Fargate profile for EKS cluster EksTest", - EksComponent: "Fargate profile", - Name: "eksTest-fargateProfile", - }, + tags: { + ...tags_base, + ...{ + Description: "Fargate profile for EKS cluster EksTest", + EksComponent: "Fargate profile", + Name: "eksTest-fargateProfile", }, - … + }, + … }); // Default tags with explicit provider. const provider = new aws.Provider("provider", { - defaultTags: { - tags: { - ManagedBy: "Pulumi", - Owner: "user@company.com", - Team: "Infra", - }, + defaultTags: { + tags: { + ManagedBy: "Pulumi", + Owner: "user@company.com", + Team: "Infra", }, + }, }); new aws.eks.FargateProfile("fargateProfile", { - … + … }, { - provider: provider, - … + provider: provider, + … }); // Use outputs from other stacks. @@ -382,11 +382,11 @@ const currentStack = pulumi.getStack(); const infraStack = new pulumi.StackReference(`organization/infra/${currentStack}`); const subnets_private = infraStack.getOutput("subnets_private"); // list of aws.ec2.Subnets new aws.eks.Cluster("cluster", { - vpcConfig: { - subnetIds: subnets_private.apply((subnets: aws.ec2.Subnet[]) => subnets.map(subnet => subnet.id)), - … - }, + vpcConfig: { + subnetIds: subnets_private.apply((subnets: aws.ec2.Subnet[]) => subnets.map(subnet => subnet.id)), … + }, + … }); // Debug the .apply() result of Outputs. @@ -396,6 +396,15 @@ subnets_private.apply( subnets_private.apply( (subnets: aws.ec2.Subnet[]) => console.log(subnets.map(subnet => subnet.id)), ); // [ 'subnet-00001111222233334', … ] + +// Use multiple Outputs. +pulumi.all([ + aws.getRegionOutput().apply(region => region.id), + aws.getCallerIdentityOutput().apply(callerIdentity => callerIdentity.accountId), + cluster.name, +]).apply( + ([regionId, accountId, clusterName]) => `arn:aws:eks:${regionId}:${accountId}:fargateprofile/${clusterName}/*` +); ``` @@ -503,19 +512,19 @@ Read [Assigning tags by default on AWS with Pulumi] first to get an idea of pros ```ts const provider = new aws.Provider("provider", { - defaultTags: { - tags: { - ManagedBy: "Pulumi", - Owner: "user@company.com", - Team: "Infra", - }, + defaultTags: { + tags: { + ManagedBy: "Pulumi", + Owner: "user@company.com", + Team: "Infra", }, + }, }); const fargateProfile = new aws.eks.FargateProfile("fargateProfile", { - … + … }, { - provider: provider, - … + provider: provider, + … }); ```