From 15369e804b74bc00f43a351c866ce271071ba65d Mon Sep 17 00:00:00 2001 From: Michele Cereda Date: Wed, 12 Jun 2024 21:46:29 +0200 Subject: [PATCH] chore(gitlab): improve notes about backups --- knowledge base/acl.md | 14 +++++--- knowledge base/cron.md | 49 +++++++++++++++++++++------- knowledge base/crontab.md | 30 +++++++++++------ knowledge base/gitlab/README.md | 2 ++ knowledge base/tar.md | 58 ++++++++++++++++++++------------- snippets/ansible.sh | 8 +++-- snippets/ansible.tasks.yml | 24 ++++++++++++++ snippets/archives.sh | 19 +++++++++++ snippets/cronjobs.sh | 11 +++++++ 9 files changed, 163 insertions(+), 52 deletions(-) create mode 100644 snippets/archives.sh create mode 100644 snippets/cronjobs.sh diff --git a/knowledge base/acl.md b/knowledge base/acl.md index a6bb238..6705f72 100644 --- a/knowledge base/acl.md +++ b/knowledge base/acl.md @@ -10,10 +10,12 @@ ## TL;DR -When **setting** permissions, the _execute_ flag can be set to the **uppercase** `X` instead of the **lowercase** `x`.
-The uppercase `X` permission allows execution only if the target is a directory or if the execute permission has already been set for the user or group. +When **setting** permissions, the _execute_ flag can be set to the **uppercase** `X` instead of the +**lowercase** `x`.
+The uppercase `X` permission allows execution only if the target is a directory or if the execute permission has already +been set for the user or group. -BSD systems use NFSv4 ACLs by default in ZFS. +BSD systems use NFSv4 ACLs by default in ZFS.
List of **NFSv4** [permission tags][syntax descriptions for setting acls] and [inheritance options][acl inheritance]. ```sh @@ -79,7 +81,8 @@ setfacl -b 'path/to/file' ## Set default permissions for files and directories -Suppose you want a folder to set the default permissions of newly created files and directories to `0664` (`-rw-rw-r--`) and `0775` (`drwxrwxr-x`) respectively.
+Suppose you want a folder to set the default permissions of newly created files and directories to `0664` (`-rw-rw-r--`) +and `0775` (`drwxrwxr-x`) respectively.
The best way to achieve this would be to set up it's ACLs accordingly. ### Posix @@ -131,7 +134,8 @@ setfacl -a '5' 'everyone@:r-x---a-R-c---:-d-----:allow' 'path/to/dir' - [`chmod`][chmod] for how to force new files to be owned by specific users or groups diff --git a/knowledge base/cron.md b/knowledge base/cron.md index f1dd612..8ef9e8c 100644 --- a/knowledge base/cron.md +++ b/knowledge base/cron.md @@ -1,29 +1,56 @@ -# Title - -## Table of contents +# Cron 1. [TL;DR](#tldr) -1. [Sources](#sources) +1. [Further readings](#further-readings) + 1. [Sources](#sources) ## TL;DR -Files in `/etc/cron.hourly` and similar need to: +Files in `/etc/cron.hourly` and similar must: -- be executable, -- match the Debian cron script namespace (`^[a-zA-Z0-9_-]+$`, so script **with an extension** won't work). +- Be **scripts**.
+ Normal crontab files will error out. +- Be **executable**.
+ If not executable, execution will silently fail. +- Match the Debian cron script namespace (`^[a-zA-Z0-9_-]+$`).
+ Files **with** an extension **won't** work. ```sh +systemctl --enable --now 'crond.service' +journalctl -xefu 'crond.service' + +# Validate crontab files. +crontab -T '/etc/cron.d/prometheus-backup' + +# List files in given directories. +# Kinda… useless? +run-parts --list '/etc/cron.daily' + # Print the names of the scripts which would be invoked. -sudo run-parts --report --test '/etc/cron.hourly' +# '--report' is *not* available everywhere. +run-parts --report --test '/etc/cron.hourly' + +# Manually run crontab files in given directories. +run-parts '/etc/cron.weekly' ``` -## Sources +## Further readings + +- [Crontab] + +### Sources - [Function of /etc/cron.hourly] +- [Use anacron for a better crontab] + +[crontab]: crontab.md + -[Function of /etc/cron.hourly]: https://askubuntu.com/questions/7676/function-of-etc-cron-hourly#607974 +[function of /etc/cron.hourly]: https://askubuntu.com/questions/7676/function-of-etc-cron-hourly#607974 +[use anacron for a better crontab]: https://opensource.com/article/21/2/linux-automation diff --git a/knowledge base/crontab.md b/knowledge base/crontab.md index 720e0d5..c249d52 100644 --- a/knowledge base/crontab.md +++ b/knowledge base/crontab.md @@ -1,28 +1,30 @@ # Crontab -## Table of contents - 1. [TL;DR](#tldr) -1. [Sources](#sources) +1. [Further readings](#further-readings) + 1. [Sources](#sources) ## TL;DR ```sh # List existing jobs. crontab -l -sudo crontab -l -u other_user +sudo crontab -l -u 'jane' # Edit crontab files. crontab -e -sudo crontab -e -u other_user +sudo crontab -e -u 'mark' # Replace the current crontab with the contents of a given file. -crontab path/to/file -sudo crontab -u other_user path/to/file +crontab 'path/to/file' +sudo crontab -u 'kelly' 'path/to/file' + +# Validate crontab files. +crontab -T '/etc/cron.d/prometheus-backup' # Remove all cron jobs. crontab -r -sudo crontab -r -u other_user +sudo crontab -r -u 'nana' ``` ```txt @@ -36,13 +38,21 @@ sudo crontab -r -u other_user 34 2 * * Fri /absolute/path/to/script.sh ``` -## Sources +## Further readings + +- [Cron] + +### Sources - [cheat.sh] + +[cron]: cron.md + [cheat.sh]: https://cheat.sh/crontab diff --git a/knowledge base/gitlab/README.md b/knowledge base/gitlab/README.md index ae99a9c..cec3e53 100644 --- a/knowledge base/gitlab/README.md +++ b/knowledge base/gitlab/README.md @@ -40,6 +40,8 @@ curl -X 'PUT' -H 'PRIVATE-TOKEN: glpat-m-…' 'https://gitlab.fqdn/api/v4/applic Previously known as 'Omnibus'. +Default backup location: `/var/opt/gitlab/backups`. +
Installation diff --git a/knowledge base/tar.md b/knowledge base/tar.md index 0854eb0..5eec456 100644 --- a/knowledge base/tar.md +++ b/knowledge base/tar.md @@ -1,44 +1,56 @@ # Tar -## Table of contents +1. [TL;DR](#tldr) +1. [Options of interest](#options-of-interest) +1. [Further readings](#further-readings) + 1. [Sources](#sources) ## TL;DR ```sh -# create an archive -tar czvf directory.tar.gz directory -tar capvf archive.tar.bz2 directory1 directory2 file +# Create archives. +tar czvf 'directory.tar.gz' 'source-directory' +tar -capvf 'archive.tar.bz2' 'directory1' 'directory2' 'file1' 'fileN' -# list the content of an archive -tar tf archive.tar -tar tf archive.tar member +# List the content of archives. +tar tf 'archive.tar' +tar -tf 'archive.tar' 'file-in-archive' -# extract an archive -tar xpf archive.tar -tar xapf archive.tar.gz -tar xjpf archive.tar.bz2 file +# Test archives by reading their contents or extracting them to stdout. +tar tf 'archive.tar' > '/dev/null' +tar tOf 'archive.tar' > '/dev/null' + +# Extract archives. +tar xpf 'archive.tar' +tar xapf 'archive.tar.gz' +tar -xjpOf 'archive.tar.bz2' 'file-in-archive' ``` -## Interesting switches +## Options of interest -short | long | description -------|-------------------|-------------------------------------------------------------------------------------------------------- -`-a` | `--auto-compress` | use archive suffix to determine the compression program -`-c` | `--create` | create a new archive; directories are archived recursively, unless the `--no-recursion` option is given -`-C` | `--directory DIR` | change to DIR before performing any operations; this option affects all options that follow -`-f` | `--file FILE` | use archive file or device FILE; if not given, tar will first examine the environment variable `TAPE` and default to the compiled-in default -`-r` | `--append` | append files to the end of an archive -`-t` | `--list` | list the contents of an archive; arguments are optional, but when given they specify the names of the members to list +| Short | Long | Description | +| ----- | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | +| `-a` | `--auto-compress` | use archive suffix to determine the compression program | +| `-c` | `--create` | create a new archive; directories are archived recursively, unless the `--no-recursion` option is given | +| `-C` | `--directory DIR` | change to DIR before performing any operations; this option affects all options that follow | +| `-f` | `--file FILE` | use archive file or device FILE; if not given, tar will first examine the environment variable `TAPE` and default to the compiled-in default | +| `-r` | `--append` | append files to the end of an archive | +| `-t` | `--list` | list the contents of an archive; arguments are optional, but when given they specify the names of the members to list | ## Further readings -- [how to compress and extract files using the tar command on linux] -- [how to create tar gz file in linux using command line] +### Sources + +- [How to compress and extract files using the tar command on linux] +- [How to create tar gz file in linux using command line] +- [How to test tar file integrity] [how to create tar gz file in linux using command line]: https://www.cyberciti.biz/faq/how-to-create-tar-gz-file-in-linux-using-command-line/ [how to compress and extract files using the tar command on linux]: https://www.howtogeek.com/248780/how-to-compress-and-extract-files-using-the-tar-command-on-linux/ +[how to test tar file integrity]: https://www.baeldung.com/linux/tar-integrity-check diff --git a/snippets/ansible.sh b/snippets/ansible.sh index bdf63bd..1cc8690 100644 --- a/snippets/ansible.sh +++ b/snippets/ansible.sh @@ -24,7 +24,9 @@ ansible-galaxy init 'gitlab' ansible-galaxy role init --type 'container' --init-path 'gitlab' 'name' # Apply changes. -ansible-playbook \ +ansible-playbook 'gitlab.yml' \ -i 'aws_ec2.yml' -e 'ansible_aws_ssm_plugin=/usr/local/sessionmanagerplugin/bin/session-manager-plugin' \ - -D --step \ - 'gitlab.yml' + -D --step +ansible-playbook 'prometheus.yml' \ + -i 'aws_ec2.yml' -e 'ansible_aws_ssm_plugin=/usr/local/sessionmanagerplugin/bin/session-manager-plugin' \ + -D -t 'cron' -l 'i-0123456789abcdef0 -C diff --git a/snippets/ansible.tasks.yml b/snippets/ansible.tasks.yml index 1e9b7f1..fa90c4a 100644 --- a/snippets/ansible.tasks.yml +++ b/snippets/ansible.tasks.yml @@ -137,3 +137,27 @@ owner: "{{ item.key }}" state: touch with_dict: "{{ users_info }}" + +- name: Cronjobs + block: + - name: At specific times + become: true + ansible.builtin.cron: + name: Prometheus manual data backup + cron_file: prometheus-manual-data-backup + + # Mind this is based on the hosts' time. + hour: 4 + minute: 0 + + user: root + job: + # - Keep '%' characters escaped or they'll be treated as newlines. + # - Archive creation returns 1 if it detects changes to read files. + # Using ';' instead of '&&' to ignore. + > + FILENAME="/tmp/prometheus-data-$(date +'\%s-\%F-\%H-\%m-\%S').tar.gz" + && tar -czf "$FILENAME" '/var/lib/prometheus/data' + ; tar -tf "$FILENAME" > '/dev/null' + && aws s3 cp "$FILENAME" 's3://backups/prometheus/' + && rm "$FILENAME" diff --git a/snippets/archives.sh b/snippets/archives.sh new file mode 100644 index 0000000..f5c24b0 --- /dev/null +++ b/snippets/archives.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env sh + +# `tar -a` guesses the compression algorithm from the archive extension + +# Create archives +tar czvf "/tmp/prometheus-data-$(date +'%s-%F-%H-%m-%S').tar.gz" '/var/lib/prometheus/data' +tar cjpvf 'docs.tar.bz2' "${HOME}/Documents" "${HOME}/Downloads" 'docs.txt' + +# List the contents of archives +tar tf "/tmp/prometheus-data-1718104097-2024-06-11-11-06-17.tar.gz" +tar tf 'kubectl.tar' 'kubectl' + +# Test archives by reading their contents or extracting them to stdout. +tar tf 'archive.tar' > '/dev/null' +tar tOf 'archive.tar' > '/dev/null' + +# Extract archives +tar xf 'portage-latest.tar.xz' -C '/mnt/gentoo/usr' +tar xpf 'stage3-amd64-'*'.tar.xz' --checkpoint '250' diff --git a/snippets/cronjobs.sh b/snippets/cronjobs.sh new file mode 100644 index 0000000..944d20e --- /dev/null +++ b/snippets/cronjobs.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env sh + +systemctl --enable --now 'crond.service' +journalctl -xefu 'crond.service' + +# Validate crontab files +crontab -T '/etc/cron.d/prometheus-backup' + +run-parts --list '/etc/cron.daily' +run-parts --test '/etc/cron.hourly' +run-parts '/etc/cron.weekly'