diff --git a/examples/ansible/ec2.clone-instance.yml b/examples/ansible/ec2.clone-instance.yml new file mode 100644 index 0000000..b8b86e3 --- /dev/null +++ b/examples/ansible/ec2.clone-instance.yml @@ -0,0 +1,130 @@ +--- +- name: Properly clone a running EC2 instance + hosts: localhost + gather_facts: false + vars: + original_instance_name_tag: "Use ME!!" + pre_tasks: + - name: Get information from the original instance + tags: + - gather_information + - pre_task + amazon.aws.ec2_instance_info: + filters: + "tag:Name": "{{ original_instance_name_tag }}" + "instance-state-name": [ "running" ] + register: original_instance_information + - name: Check a running instance with Name tag '{{ original_instance_name_tag }}' has been found + tags: + - gather_information + - pre_task + ansible.builtin.assert: + that: original_instance_information.instances | length > 0 + fail_msg: No running instances found with Name tag '{{ original_instance_name_tag }}' + success_msg: At least one running instance has been found with Name tag '{{ original_instance_name_tag }}' + tasks: + - name: Create a Security Group with only the connections required for testing + tags: security_group + amazon.aws.ec2_security_group: + name: Clone EC2 Instance SG + description: Temporary SG for cloning EC2 Instances + rules_egress: + - cidr_ip: 0.0.0.0/0 + ports: 443 + rule_desc: Required by SSM, but could be stricter + register: clone_instance_security_group_information + - name: Create snapshots of the instance's volumes + # Allows for more control over the snapshot, namely to avoid recreating snapshot of massive volumes and lose hours + tags: snapshot + amazon.aws.ec2_snapshot: + volume_id: "{{ item }}" + description: Temporary snapshot for cloning EC2 Instances + last_snapshot_min_age: 1440 # 1d + wait_timeout: 7200 # 2h might still be not enough for big boi volumes + loop: "{{ original_instance_information.instances[0].block_device_mappings | map(attribute='ebs.volume_id') }}" + register: original_instance_snapshots + - name: Create an AMI from the snapshot + tags: ami + amazon.aws.ec2_ami: + # no_reboot: false # set to true if one does *not* want to have the original instance shut down + wait: true + name: temp-{{ original_instance_name_tag | regex_replace(' ', '-') | lower }}-ami + description: Temporary AMI for cloning EC2 Instances + tags: + Name: Clone EC2 Instance AMI + root_device_name: "{{ original_instance_information.instances[0].root_device_name }}" + device_mapping: >- + {%- set devices_list = [] -%} + {%- for result in original_instance_snapshots.results -%} + {%- for device in original_instance_information.instances[0].block_device_mappings + | selectattr('ebs.volume_id', 'equalto', result.volume_id) -%} + {{- + devices_list.append({ + 'device_name': device.device_name, + 'snapshot_id': result.snapshots | sort(attribute='start_time') | last | json_query('snapshot_id'), + 'volume_type': 'gp3', + 'delete_on_termination': true, + }) + -}} + {%- endfor -%} + {%- endfor -%} + {{ devices_list }} + register: original_instance_ami + - name: Use the AMI to launch a clone + tags: + - clone + - instance + when: original_instance_ami.image_id is defined + amazon.aws.ec2_instance: + name: Clone EC2 Instance + vpc_subnet_id: "{{ original_instance_information.instances[0].subnet_id }}" + instance_type: "{{ original_instance_information.instances[0].instance_type }}" + image: + id: "{{ original_instance_ami.image_id }}" + security_group: "{{ clone_instance_security_group_information.group_id }}" + iam_instance_profile: "{{ original_instance_information.instances[0].iam_instance_profile.arn }}" + register: clone_instance_information + - name: Wait for the instance to be ready + tags: + - clone + - instance + - check + when: clone_instance_information.instance_ids is defined + block: + - name: Just pause enough for the instance to initialize + # Because of course there seems to be no effing way to distinguish between just running and ready, and of + # course the SSM connection plugin crashes badly instead of just erroring the task out (ノಠ益ಠ)ノ彡┻━┻ + ansible.builtin.pause: + minutes: 3 + - name: Try connecting with SSM + delegate_to: "{{ clone_instance_information.instances[0].instance_id }}" + vars: + ansible_connection: community.aws.aws_ssm + ansible_aws_ssm_bucket_name: some-bucket + ansible_aws_ssm_region: eu-west-1 + ansible_aws_ssm_timeout: 300 + ansible.builtin.ping: + - name: Ready! + ansible.builtin.debug: + msg: The clone instance is ready! + post_tasks: + - name: Remove the clone + tags: + - clone + - instance + - cleanup + - post_task + when: clone_instance_information.instance_ids is defined + amazon.aws.ec2_instance: + instance_ids: "{{ clone_instance_information.instance_ids }}" + state: absent + - name: Remove the AMI + tags: + - always # stupid ami module fails if already existing + - ami + - cleanup + - post_task + when: original_instance_ami.image_id is defined + amazon.aws.ec2_ami: + image_id: "{{ original_instance_ami.image_id }}" + state: absent diff --git a/knowledge base/cloud computing/aws/README.md b/knowledge base/cloud computing/aws/README.md index 76e6a79..e8dfaf6 100644 --- a/knowledge base/cloud computing/aws/README.md +++ b/knowledge base/cloud computing/aws/README.md @@ -70,6 +70,8 @@ One can can rapidly remapping addresses to other instances in one's account and | [Security Hub] | Aggregator for security findings | [Service icons][aws icons] are publicly available for diagrams and such. +Public service IP address ranges are [available in JSON form][aws public ip address ranges now available in json form] +at . ### Billing and Cost Management @@ -254,6 +256,7 @@ Refer [IAM]. - [Rotating AWS KMS keys] - [Image baking in AWS using Packer and Image builder] - [Using AWS KMS via the CLI with a Symmetric Key] +- [AWS Public IP Address Ranges Now Available in JSON Form] [automating dns-challenge based letsencrypt certificates with aws route 53]: https://johnrix.medium.com/automating-dns-challenge-based-letsencrypt-certificates-with-aws-route-53-8ba799dd207b diff --git a/knowledge base/cloud computing/aws/ebs.md b/knowledge base/cloud computing/aws/ebs.md index ea7d1fb..a94c1f8 100644 --- a/knowledge base/cloud computing/aws/ebs.md +++ b/knowledge base/cloud computing/aws/ebs.md @@ -28,9 +28,13 @@ Apparently, Linux machines are able to do that automatically with a reboot. ## Snapshots -When created, snapshots are **incremental**.
+The first snapshot is **complete**, with all the volume's blocks being copied. All successive snapshots of the same +volume are **incremental**, with only the changes being copied.
Incremental snapshots are stored in EBS' standard tier. +Snapshots can be unbearably slow depending on the amount of data needing to be copied.
+For comparison, the first snapshot of a 200 GiB volume took about 2h to complete. + Snapshots can be [archived][archive amazon ebs snapshots] to save money should they **not** need frequent nor fast retrieval.
When archived, incremental snapshots are converted to **full snapshots** and moved to EBS' archive tier. diff --git a/knowledge base/cloud computing/aws/ec2.md b/knowledge base/cloud computing/aws/ec2.md index 7288ba3..9bb8e63 100644 --- a/knowledge base/cloud computing/aws/ec2.md +++ b/knowledge base/cloud computing/aws/ec2.md @@ -15,6 +15,13 @@ Use an instance profile to pass an IAM role to an EC2 instance. The instance type [_can_ be changed][change the instance type]. The procedure depends on the root volume, but does require downtime. +Clone EC2 instances by: + +1. Creating an AMI from the original instance. + Mind the default behaviour of the AMI creator is to **shutdown** the instance, take a snapshot, and boot it again + [to guarantee the image's filesystem integrity][create an ami from an amazon ec2 instance]. +1. Using that AMI to launch clones identical to the original. +
Real world use cases @@ -122,6 +129,8 @@ See [EBS]. - [Retrieve instance metadata] - [Burstable performance instances] - [Change the instance type] +- [How to Clone instance EC2] +- [Create an AMI from an Amazon EC2 Instance] [best practices for handling ec2 spot instance interruptions]: https://aws.amazon.com/blogs/compute/best-practices-for-handling-ec2-spot-instance-interruptions/ [burstable performance instances]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/burstable-performance-instances.html +[change the instance type]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-resize.html [connect to your instances without requiring a public ipv4 address using ec2 instance connect endpoint]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/connect-with-ec2-instance-connect-endpoint.html +[create an ami from an amazon ec2 instance]: https://docs.aws.amazon.com/toolkit-for-visual-studio/latest/user-guide//tkv-create-ami-from-instance.html [describe-images]: https://docs.aws.amazon.com/cli/latest/reference/ec2/describe-images.html [describeimages]: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeImages.html +[how to clone instance ec2]: https://repost.aws/questions/QUOrWudF3vRL2Vqtrv0M9lfQ/how-to-clone-instance-ec2 [iam roles for amazon ec2]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html [key concepts and definitions for burstable performance instances]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/burstable-credits-baseline-concepts.html [retrieve instance metadata]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html [standard mode for burstable performance instances]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/burstable-performance-instances-standard-mode.html [unlimited mode for burstable performance instances]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/burstable-performance-instances-unlimited-mode.html [using instance profiles]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html -[change the instance type]: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-resize.html [aws ec2 instance pricing comparison]: https://ec2instances.github.io/ diff --git a/snippets/ansible/commands.sh b/snippets/ansible/commands.sh index bb63f42..2141203 100644 --- a/snippets/ansible/commands.sh +++ b/snippets/ansible/commands.sh @@ -51,7 +51,7 @@ ansible-playbook 'path/to/playbook.yml' --syntax-check # Ad-hoc commands. ansible -i 'hosts.yml' -m 'ping' 'all' ansible -i 'host-1,host-n,' 'hostRegex' -m 'ansible.builtin.shell' -a 'echo $TERM' -ansible -i 'localhost,' -c 'local' -m 'ansible.builtin.copy' -a 'src=/tmp/src' -a 'dest=/tmp/dest' 'localhost' +ansible -i 'localhost ansible_python_interpreter=venv/bin/python3,' -c 'local' -m 'ansible.builtin.copy' -a 'src=/tmp/src' -a 'dest=/tmp/dest' 'localhost' ansible-vault encrypt_string --name 'command_output' 'somethingNobodyShouldKnow' ansible-vault encrypt --output 'ssh.key' '.ssh/id_rsa' diff --git a/snippets/ansible/tasks.yml b/snippets/ansible/tasks.yml index b2f6b70..1d9b085 100644 --- a/snippets/ansible/tasks.yml +++ b/snippets/ansible/tasks.yml @@ -19,6 +19,14 @@ content: | … +- name: Show input data type + set_fact: + should_be_string: "{{ 'this' | type_debug }}" + +- name: Run locally + delegate_to: 127.0.0.1 # 'localhost' works too + command: hostname + - name: Import tasks block: - name: By using absolute paths and special variables (preferred) @@ -135,6 +143,7 @@ set_fact: random_item: "{{ ['a','b','c'] | random }}" - name: Sort dict elements in list by attribute + tags: order_by vars: snapshots: - name: sales @@ -156,6 +165,7 @@ set_fact: vpc_security_group_ids: >- {{ instance_information.vpc_security_groups | map(attribute='vpc_security_group_id') }} + volume_ids: "{{ instances_information.instances[0].block_device_mappings | map(attribute='ebs.volume_id') }}" - name: Return only elements with specific attributes matching a filter set_fact: available_rds_snapshots: snapshots_list | selectattr("status", "equalto", "available") @@ -243,6 +253,28 @@ {%- endfor -%} {%- endfor -%} {{- output -}} + - name: Get the device name and last snapshot id for all block devices in an EC2 instance + # Useful to create AMIs from instance snapshots + tags: + - aws + - ec2 + - snapshot + - ami + ansible.builtin.set_fact: + last_snap_for_device: >- + {%- set devices_list = [] -%} + {%- for result in current_instance_snapshots.results -%} + {%- for device in current_instance_information.instances[0].block_device_mappings + | selectattr('ebs.volume_id', 'equalto', result.volume_id) -%} + {{- + devices_list.append({ + 'device_name': device.device_name, + 'snapshot_id': result.snapshots | sort(attribute='start_time') | last | json_query('snapshot_id'), + }) + -}} + {%- endfor -%} + {%- endfor -%} + {{ devices_list }} - name: "Use the users' home directory for something" block: @@ -261,6 +293,7 @@ - ec2-user register: users_homedir_retrieve - name: Compute and register the results + tags: AnsibleUnsafeText_to_Dict ansible.builtin.set_fact: users_homedir: >- {{ @@ -354,7 +387,18 @@ msg: I always execute - name: AWS + tags: aws block: + - name: Get current IP ranges + # too many to be put into security group rules + set_fact: + ip_ranges: >- + lookup('url', 'https://ip-ranges.amazonaws.com/ip-ranges.json', split_lines=False) + | from_json + | json_query('prefixes') + | selectattr('region', 'equalto', 'eu-west-1') + | selectattr('service', 'equalto', 'AMAZON') + | map(attribute='ip_prefix') - name: Assume roles block: - name: Get session tokens @@ -374,6 +418,38 @@ resource: i-xyzxyz01 tags: MyNewTag: value + - name: EC2 + block: + - name: Get running instances with 'K8S' as the 'Application' tag + amazon.aws.ec2_instance_info: + filters: + "tag:Application": K8S + "instance-state-name": [ "running" ] + - name: Clone EC2 instances + vars: + source_instance_id: i-0123456789abcdef0 + block: + - name: Get instance information from the original instance + amazon.aws.ec2_instance_info: + instance_ids: + - "{{ source_instance_id }}" + register: source_instance_info + - name: Create an AMI of the original instance + amazon.aws.ec2_ami: + instance_id: "{{ source_instance_id }}" + no_reboot: true # remove if the instance rebooting upon AMI creation is no biggie + wait: true + wait_timeout: 3600 # big volumes call for bit wait times (a 200GiB volume took ) + name: ami-source + register: source_ami + - name: Use the AMI to launch clones identical to the original + when: source_ami.image_id is defined + amazon.aws.ec2_instance: + name: clone + vpc_subnet_id: "{{ source_instance_info.instances[0].subnet_id }}" + instance_type: "{{ source_instance_info.instances[0].instance_type }}" + image: + id: "{{ source_ami.image_id }}" - name: RDS block: - name: Create an instance's snapshot diff --git a/snippets/aws/commands.fish b/snippets/aws/commands.fish index fb2db4d..cc395de 100644 --- a/snippets/aws/commands.fish +++ b/snippets/aws/commands.fish @@ -49,7 +49,7 @@ aws iam list-instance-profiles | grep -i 'ssm' sudo ssm-cli get-diagnostics --output 'table' -# Check instances are available +# Check instances are available for use with SSM aws ssm get-connection-status --query "Status=='connected'" --output 'text' --target "i-0915612ff82914822" # Connect to instances if they are available