From a8d01c5ee419232ff49ccf4d5a0df6a9b305b1ec Mon Sep 17 00:00:00 2001 From: Michele Cereda Date: Fri, 31 May 2024 22:46:56 +0200 Subject: [PATCH] chore(gitlab/runner): try and fix the adversion runners apparently have to certificates --- .editorconfig | 2 +- ...te-certificate.yml => tls-certificate.yml} | 57 ++++++++------- knowledge base/cloud computing/aws/README.md | 2 + knowledge base/cloud computing/aws/ssm.md | 3 +- knowledge base/curl.md | 69 ++++++++++--------- snippets/aws.fish | 27 ++++++++ snippets/curl.sh | 11 ++- snippets/dnf.sh | 3 + snippets/openssl.sh | 19 ++--- snippets/systemd.sh | 4 ++ snippets/webserver.sh | 14 ++++ 11 files changed, 144 insertions(+), 67 deletions(-) rename examples/ansible/{letsencrypt.create-certificate.yml => tls-certificate.yml} (57%) create mode 100644 snippets/dnf.sh create mode 100644 snippets/systemd.sh create mode 100644 snippets/webserver.sh diff --git a/.editorconfig b/.editorconfig index ca0ddc7..9163dc1 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,7 +14,7 @@ insert_final_newline = true [.vscode/*.json] insert_final_newline = false -[*.{ba,,z}sh] +[*.{,ba,fi,z}sh] indent_style = tab tab_width = 4 diff --git a/examples/ansible/letsencrypt.create-certificate.yml b/examples/ansible/tls-certificate.yml similarity index 57% rename from examples/ansible/letsencrypt.create-certificate.yml rename to examples/ansible/tls-certificate.yml index 0e512b3..1b993f5 100644 --- a/examples/ansible/letsencrypt.create-certificate.yml +++ b/examples/ansible/tls-certificate.yml @@ -1,10 +1,16 @@ --- -- name: Create and validate an HTTPS certificate +- name: Create and validate a TLS certificate using LetsEncrypt with DNS-01 challenge type, and AWS Route53 hosts: all + gather_facts: false vars: + account_email: someone@example.org + acme_directory: https://acme-v02.api.letsencrypt.org/directory + # acme_directory: https://acme-staging-v02.api.letsencrypt.org/directory common_name: service.example.org + remaining_days: 7 + zone: example.org pre_tasks: @@ -15,16 +21,17 @@ size: 4096 with_items: - /tmp/{{ common_name }}.key - - /tmp/letsencrypt.account.key.pem + - /tmp/acme.account.key.pem - # - name: Generate private keys for an account and the certificate - OpenSSH alternative - # community.crypto.openssh_keypair: - # path: "{{ item }}" - # type: rsa - # size: 4096 - # with_items: - # - /tmp/{{ common_name }}.key - # - /tmp/letsencrypt.account.key.pem + - name: Generate private keys for an account and the certificate - OpenSSH alternative + when: false + community.crypto.openssh_keypair: + path: "{{ item }}" + type: rsa + size: 4096 + with_items: + - /tmp/{{ common_name }}.key + - /tmp/acme.account.key.pem tasks: @@ -38,13 +45,15 @@ community.crypto.acme_certificate: challenge: dns-01 acme_version: 2 - acme_directory: https://acme-v02.api.letsencrypt.org/directory - account_key_src: /tmp/letsencrypt.account.key.pem - account_email: someone@example.org + acme_directory: "{{ acme_directory }}" + account_key_src: /tmp/acme.account.key.pem + account_email: "{{ account_email }}" csr: /tmp/{{ common_name }}.crs - cert: /tmp/{{ common_name }}.crt + dest: /tmp/{{ common_name }}.endpointOnly.crt # endpoint only certificate + fullchain_dest: /tmp/{{ common_name }}.crt # full certificate chain terms_agreed: true - remaining_days: 21 + remaining_days: "{{ remaining_days }}" + # force: true register: dns_challenge notify: Create TXT records for challenge validation @@ -53,7 +62,7 @@ - name: Create TXT records for challenge validation when: common_name in dns_challenge.challenge_data amazon.aws.route53: - zone: example.org + zone: "{{ zone }}" record: "{{ dns_challenge.challenge_data[common_name]['dns-01'].record }}" type: TXT ttl: 60 @@ -61,7 +70,7 @@ overwrite: true wait: true value: - # Value should be enclosed in quotation marks + # shall be enclosed in quotation marks >- {{ dns_challenge.challenge_data[common_name]['dns-01'].resource_value @@ -73,14 +82,16 @@ community.crypto.acme_certificate: challenge: dns-01 acme_version: 2 - acme_directory: https://acme-v02.api.letsencrypt.org/directory - account_key_src: /tmp/letsencrypt.account.key.pem - account_email: someone@example.org + acme_directory: "{{ acme_directory }}" + account_key_src: /tmp/acme.account.key.pem + account_email: "{{ account_email }}" csr: /tmp/{{ common_name }}.crs - cert: /tmp/{{ common_name }}.crt - remaining_days: 21 + dest: /tmp/{{ common_name }}.endpointOnly.crt # endpoint only certificate + fullchain_dest: /tmp/{{ common_name }}.crt # full certificate chain + remaining_days: "{{ remaining_days }}" terms_agreed: true data: "{{ dns_challenge }}" + # force: true post_tasks: - name: Delete TXT records for challenge validation @@ -88,7 +99,7 @@ validation_record: "{{ ['_acme-challenge', common_name] | join('.') }}" when: query('community.dns.lookup', validation_record, type='TXT') != [] amazon.aws.route53: - zone: example.org + zone: "{{ zone }}" record: "{{ validation_record }}" type: TXT state: absent diff --git a/knowledge base/cloud computing/aws/README.md b/knowledge base/cloud computing/aws/README.md index d397597..54ad9ea 100644 --- a/knowledge base/cloud computing/aws/README.md +++ b/knowledge base/cloud computing/aws/README.md @@ -291,6 +291,7 @@ Examples: - [You might be clueless as to why AWS assume role isn't working, despite being correctly set up] - [Use an IAM role in the AWS CLI] - [Creating a role to delegate permissions to an IAM user] +- [How to use the PassRole permission with IAM roles] diff --git a/knowledge base/curl.md b/knowledge base/curl.md index 750d77a..26ce50e 100644 --- a/knowledge base/curl.md +++ b/knowledge base/curl.md @@ -5,20 +5,21 @@ 1. [TL;DR](#tldr) 1. [Apply settings to all connections](#apply-settings-to-all-connections) 1. [Further readings](#further-readings) -1. [Sources](#sources) + 1. [Sources](#sources) ## TL;DR ```sh -# Send a single GET request and show its output on stdout. +# Send single GET requests and show their output on stdout. curl 'http://url.of/file' +curl 'https://www.example.com' # Be quiet. -curl 'https://www.example.com' … --silent +curl … --silent curl … -s --show-error # Download files to specific paths. -curl 'http://url.of/file' --output 'path/to/file' +curl … --output 'path/to/file' curl … -o 'path/to/file' # Download files reusing their name for output. @@ -26,44 +27,44 @@ curl … --remote-name 'http://url.of/file1' -O 'http://url.of/file2' curl … -O http://url.of/file[1-24] # Resume downloads. -curl 'http://url.of/file' --continue-at - +curl … --continue-at - curl … -o 'partial_file' -C - # Limit downloads bandwidth. -curl 'http://url.of/file' --limit-rate '1000B' +curl … --limit-rate '1000B' # Follow redirects. -curl 'http://url.of/file' --location +curl … --location curl … -L # Only fetch the response's HTTP headers. # Prevents downloading the response's body. -curl 'http://example.com' --head +curl … --head curl … -I # Write specific information in output. -curl 'http://example.com' … --write-out '@template.file' +curl … … --write-out '@template.file' curl … -w 'request returned %{http_code}\nDownloaded %{size_download} bytes\n' # Send different request types. -curl 'http://example.com' --request 'PUT' +curl … --request 'PUT' curl … -X 'GET' # Specify headers. -curl 'http://example.com' -H 'Content-Type:application/json' +curl … -H 'Content-Type:application/json' curl … --header 'Content-Type:application/json' # Fail fast with no output. # Returns the HTTP error code. -curl 'http://example.com' --fail +curl … --fail curl … -f # Skip certificate validation. -curl 'https://example.com' --insecure +curl … --insecure curl … -k # Pass certificates for a resource. -curl 'https://example.com' --cert 'client.pem' --key 'key.pem' +curl … --cert 'client.pem' --key 'key.pem' curl … --cacert 'ca.pem' # Authenticate. @@ -72,25 +73,30 @@ curl 'ftp://url.of/file' -u 'username':'password' -O curl 'ftp://username:password@example.com' # Send data. -curl 'http://example.com' -X 'POST' -H "Content-Type:application/json" --data '@file.json' +curl … -X 'POST' -H "Content-Type:application/json" --data '@file.json' curl … -d '{"name": "bob"}' curl … -d 'name=bob' -# POST to a form. -curl 'http://example.com' --form 'name=user' -F 'password=test' +# POST to forms. +curl … --form 'name=user' -F 'password=test' curl … -d 'name=bob' -F 'password=@password.file' -# Use a proxy. -curl 'http://example.com' --proxy 'socks5://localhost:19999' +# Use proxies. +curl … --proxy 'socks5://localhost:19999' -# Forcefully resolve a host to a given address. -curl 'https://example.com' --resolve 'example.com:443:google.com' +# Forcefully resolve hosts to given addresses. +# The resolution *must* be an address, not an FQDN. +curl … --resolve 'super.fake.domain:8443:127.0.0.1' 'https://super.fake.domain:8443' -# Ask to use HTTP/2. -curl 'https://example.com' --http2 +# Use different names. +# Kinda like '--resolve' but to aliases and supports ports. +curl … --connect-to 'super.fake.domain:443:localhost:8443' 'https://super.fake.domain' + +# Ask kindly to use HTTP/2. +curl … --http2 # Force the use of HTTP/2. -curl 'https://example.com' --http2-prior-knowledge +curl … --http2-prior-knowledge ``` ## Apply settings to all connections @@ -108,7 +114,8 @@ The default configuration file is looked for in the following places, in this or On Non-Windows hosts, `curl` uses `getpwuid` to find the user's home directory. -On Windows, if curl finds no `.curlrc` file in the sequence described above, it will check for one in the same dir the curl executable is placed. +On Windows, if curl finds no `.curlrc` file in the sequence described above, it will check for one in the same dir the +`curl` executable is placed. ```txt # ~/.curlrc @@ -129,17 +136,17 @@ config: "~/.config/curl" - [Book] -## Sources - -All the references in the [further readings] section, plus the following: +### Sources - [cheat.sh] - [How to ignore invalid and self signed ssl connection errors with curl] - [Config file] - [HTTP2] +- [Name resolve tricks] @@ -147,9 +154,7 @@ All the references in the [further readings] section, plus the following: [config file]: https://everything.curl.dev/cmdline/configfile [http2]: https://everything.curl.dev/http/versions/http2 - -[further readings]: #further-readings - [cheat.sh]: https://cheat.sh/curl [how to ignore invalid and self signed ssl connection errors with curl]: https://www.cyberciti.biz/faq/how-to-curl-ignore-ssl-certificate-warnings-command-option/ +[name resolve tricks]: https://everything.curl.dev/usingcurl/connections/name.html diff --git a/snippets/aws.fish b/snippets/aws.fish index e79019f..0fe5724 100644 --- a/snippets/aws.fish +++ b/snippets/aws.fish @@ -16,11 +16,21 @@ function aws-assume-role-by-name && echo "Assumed role $argv[1]; Session name: '$current_caller-as-$argv[1]-stsSession'" end +function aws-ec2-instanceId-from-nameTag + aws ec2 describe-instances --output text \ + --filters "Name=tag:Name,Values=$argv[1]" \ + --query 'Reservations[].Instances[0].InstanceId' +end + function aws-iam-role-arn-from-name aws iam list-roles --output 'text' \ --query "Roles[?RoleName == '$argv[1]'].Arn" end +alias aws-ec2-running-instanceIds "aws ec2 describe-instances --output 'text' \ + --filters 'Name=instance-state-name,Values=running' \ + --query 'Reservations[].Instances[0].InstanceId' \ +| sed -E 's/\t+/\n/g'" alias aws-ssm-gitlabAutoscalingManager-ita-b "aws ec2 describe-instances --output text \ --filters \ 'Name=availability-zone,Values=eu-south-1b' \ @@ -46,3 +56,20 @@ aws ecs list-tasks --query 'taskArns' --output 'text' --cluster 'testCluster' -- | xargs -I{} curl -fs "http://{}:8080" aws ecr delete-repository --repository-name 'bananaslug' + +# Get Name and Description of all AMIs by Amazon for arm64 that are in the 'available' state +# and which name starts for 'al2023-ami-' +aws ec2 describe-images --output 'yaml' \ + --owners 'amazon' \ + --filters \ + 'Name=architecture,Values=['arm64']' \ + 'Name=state,Values=['available']' \ + --query ' + Images[] + .{"Name":@.Name,"Description":@.Description} + ' \ +| yq '.[]|select(.Name|test("^al2023-ami-"))' - + +aws iam list-instance-profiles | grep -i 'ssm' + +sudo ssm-cli get-diagnostics --output 'table' diff --git a/snippets/curl.sh b/snippets/curl.sh index 93ee0b8..48ba320 100644 --- a/snippets/curl.sh +++ b/snippets/curl.sh @@ -1,5 +1,12 @@ #!/usr/bin/env sh +# Sources: +# - https://everything.curl.dev/usingcurl/connections/name.html -# Forcefully resolve a host to a given address. -curl 'https://gitlab.mine.info' --resolve 'gitlab.mine.info:443:192.168.32.76' +# Use different names. +# Kinda like '--resolve' but to aliases and supports ports. +curl --connect-to 'super.fake.domain:443:localhost:8443' 'https://super.fake.domain' + +# Forcefully resolve hosts to given addresses. +# The resolution *must* be an address, not an FQDN. +curl --resolve 'super.fake.domain:8443:127.0.0.1' 'https://super.fake.domain:8443' diff --git a/snippets/dnf.sh b/snippets/dnf.sh new file mode 100644 index 0000000..7a89eb2 --- /dev/null +++ b/snippets/dnf.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +sudo dnf list --available --showduplicates 'gitlab-runner' diff --git a/snippets/openssl.sh b/snippets/openssl.sh index cedbb0a..4da146f 100644 --- a/snippets/openssl.sh +++ b/snippets/openssl.sh @@ -148,26 +148,27 @@ openssl pkcs12 -in 'keystore.pfx' -out 'keystore.pem' -nodes ## # Tests connections to remote servers -openssl s_client -connect 'www.google.com:443' -openssl s_client -host 'www.google.com' -port '443' +openssl s_client -connect 'www.google.com:443' < '/dev/null' +openssl s_client -host 'www.google.com' -port '443' < '/dev/null' # deprecated in favour of '-connect' # Show the full certificate chains -openssl s_client … -showcerts < '/dev/null' +openssl s_client … -showcerts # Extract certificates -openssl s_client … 2>&1 < '/dev/null' | sed -n '/-----BEGIN/,/-----END/p' > 'certificate.pem' +openssl s_client … 2>&1 | sed -n '/-----BEGIN/,/-----END/p' > 'certificate.pem' # Override SNI (Server Name Indication) extension with other server names # Allows testing multiple secure sites hosted by same IP address openssl s_client … -servername 'host.fqdn' +openssl s_client -host 'localhost' -port '8443' -servername 'testcert.com' < '/dev/null' # Test TLS connections by forcibly using specific cipher suites # Checks if servers can properly talk via different configured cipher suites -openssl s_client … -cipher 'ECDHE-RSA-AES128-GCM-SHA256' 2>&1 < '/dev/null' +openssl s_client … -cipher 'ECDHE-RSA-AES128-GCM-SHA256' 2>&1 # Measure SSL connection time without and with session reuse -openssl s_time -connect 'example.com:443' -new -openssl s_time -connect 'example.com:443' -reuse +openssl s_time … -new +openssl s_time … -reuse # Roughly examine TCP and SSL handshake times using `curl` curl -kso '/dev/null' -w "tcp:%{time_connect}, ssldone:%{time_appconnect}\n" 'https://example.com' @@ -194,7 +195,9 @@ openssl dgst -sha512 < 'input.file' cat 'input.file' | openssl sha512 # Base64 encoding and decoding -cat /dev/urandom | head -c 50 | openssl base64 | openssl base64 -d +echo 'plaintext' | openssl base64 +echo 'cGxhaW50ZXh0Cg==' | openssl base64 -d +cat '/dev/urandom' | head -c 50 | openssl base64 | openssl base64 -d # Measure speed of security algorithms openssl speed 'rsa2048' diff --git a/snippets/systemd.sh b/snippets/systemd.sh new file mode 100644 index 0000000..2201557 --- /dev/null +++ b/snippets/systemd.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env sh + +sudo systemctl enable --now 'gitlab-runner' +sudo journalctl -xefu 'gitlab-runner' diff --git a/snippets/webserver.sh b/snippets/webserver.sh new file mode 100644 index 0000000..0e299ce --- /dev/null +++ b/snippets/webserver.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env sh + +# sources: +# - https://anvileight.com/blog/posts/simple-python-http-server/ + +# Quick 'n' dirty http-only web server +# No TLS support +python -m 'http.server' +python -m 'http.server' '8080' --bind 'localhost' --directory '/files/to/serve' --protocol 'HTTP/1.1' --cgi + +# Quick 'n' dirty web server +# pip install --user 'twisted' 'pyopenssl' +twistd -no web +twistd -no web --path '/files/to/serve' --https '8443' --certificate 'server.pem' --privkey 'server.pem'