From 78fa817146e9e9ab937a2ad0807ea142dc4ac1a6 Mon Sep 17 00:00:00 2001 From: Michele Cereda Date: Sun, 6 Aug 2023 15:26:32 +0200 Subject: [PATCH] chore: improved rsync --- examples/rsync.backup-command.sh | 61 ++++++++++-------- knowledge base/rsync.md | 106 +++++++++++++++++++------------ knowledge base/zstd.md | 15 +++-- 3 files changed, 111 insertions(+), 71 deletions(-) diff --git a/examples/rsync.backup-command.sh b/examples/rsync.backup-command.sh index 6337fe6..7bc6d4c 100644 --- a/examples/rsync.backup-command.sh +++ b/examples/rsync.backup-command.sh @@ -1,32 +1,41 @@ #!/usr/bin/env bash -rsync /data/ nas.lan:/data/ \ - --archive --copy-links --protect-args --delete \ - --acls --xattrs --fake-super \ +# Sync directories from a Linux source to a Linux destination. +# Expand symlink at the source to their referred files. +# Assumes the same owner and group at both hosts. +rsync 'data/' 'nas.lan:data/' \ + --secluded-args --no-inc-recursive \ + --archive --copy-links --acls --xattrs --times --atimes --crtimes \ + --partial --append-verify --sparse \ + --human-readable --info='progress2' \ + --delete --backup --backup-dir "changes_$(date +'%F_%H-%M-%S')" --exclude "changes_*" +rsync 'data/' 'nas.lan:data/' \ + -abhstALNSUX --no-i-r \ --partial --append-verify \ - --compress --sparse --no-motd \ - --human-readable --no-inc-recursive --info="progress2" -vv \ - --exclude ".terraform*" --exclude "obsidian" \ - --backup --backup-dir "changes_$(date +'%F_%H-%m-%S')" --exclude "changes_*" \ -| grep -Ev -e uptodate -e "/$" + --info='progress2' \ + --delete --backup-dir "changes_$(date +'%F_%H-%M-%S')" --exclude "changes_*" -# cat '.rsync-filter' -# - .DS_Store -# - .localized -# - .obsidian -# - .terraform* -# + ** -/opt/homebrew/bin/rsync 'Data' 'nas.lan:Data' \ - -abchszAFLSUX \ - --partial --append-verify --fake-super --no-motd \ - --delete --backup-dir "changes_$(date +'%F_%H-%m-%S')" \ - --no-inc-recursive --info="progress2" +# Sync directories from a Linux source to a Synology NAS. +# The above one, just modified to be accepted from those systems. +rsync 'data/' 'synology.lan:/volume1/data/' \ + --secluded-args --no-inc-recursive \ + --archive --copy-links --acls --xattrs \ + --partial --append-verify --sparse \ + --human-readable --info='progress2' \ + --delete --backup --backup-dir "changes_$(date +'%F_%H-%M-%S')" --exclude "changes_*" \ + --no-motd --fake-super --super --chown='user:users' \ + --exclude={'@eaDir','#recycle'} +rsync 'data/' 'synology.lan:/volume1/data/' \ + -abhsALSX --no-i-r \ + --partial --append-verify \ + --info='progress2' \ + --delete --backup-dir "changes_$(date +'%F_%H-%M-%S')" --exclude "changes_*" \ + --no-motd --fake-super --super --chown='user:users' \ + --exclude={'@eaDir','#recycle'} -# .rsync-filter hides files on source, but does nothing for the ones on the remote -/opt/homebrew/bin/rsync 'Data' 'synology.lan:Data' \ - -abchszAFLSX \ - --partial --append-verify --fake-super --no-motd \ - --delete --backup-dir "changes_$(date +'%F_%H-%m-%S')" \ - --no-inc-recursive --info="progress2" \ - --exclude={'@eaDir','#recycle','changes_*'} + +# Use the '.rsync-filter' file. +# The filter file excludes files from the source, but does nothing for the ones +# on the remote side. To exclude them too, explicitly use the `--exclude` option. +$ rsync … -F diff --git a/knowledge base/rsync.md b/knowledge base/rsync.md index 047ffe3..3413939 100644 --- a/knowledge base/rsync.md +++ b/knowledge base/rsync.md @@ -4,6 +4,8 @@ 1. [TL;DR](#tldr) 1. [Explored options](#explored-options) +1. [Filters](#filters) + 1. [Filter file](#filter-file) 1. [Sources](#sources) ## TL;DR @@ -85,47 +87,69 @@ parallel -q \ ## Explored options -| Long format | Short format | Description | -| ----------------------- | ------------ | -------------------------------------------------------------------------------------------- | -| | `-P` | same as `--partial --progress` | -| `--acls` | `-A` | preserve ACLs; implies `--perms` | -| `--append-verify` | | like `--append`, but use the data already there to check the items | -| `--archive` | `-a` | archive mode, equals `-rlptgoD`; does **not** imply `-H`, `-A`, nor `-X` | -| `--backup-dir=DIR` | | use the specified directory to backup changing items | -| `--backup` | `-b` | backup items changing at the destination; see also `--suffix` and `--backup-dir` | -| `--bwlimit=RATE` | | limit the socket's I/O bandwidth to _RATE_; with no suffix, the value will be in KBPS | -| `--checksum` | `-c` | skip files basing on checksum instead of modify time and size | -| `--chown=USER:GROUP` | | simple username/groupname mapping | -| `--compress` | `-z` | compress file data during the transfer | -| `--crtimes` | | **only available on Mac OS X** | -| `--delete-during` | `--del` | set the **receiver** to delete files during the transfer | -| `--delete` | | delete items **at the destination** that **don't** exist in the source | -| `--dry-run` | `-n` | perform a trial run with no changes made | -| `--exclude=PATTERN` | | exclude files matching _PATTERN_ | -| `--executability` | `-E` | preserve executability | -| `--fake-super` | | store/recover privileged attrs using xattrs | -| `--filter=RULE` | `-f` | add a file-filtering _RULE_ | -| `--hard-links` | `-H` | preserve hard links | -| `--human-readable` | `-h` | output numbers in a human-readable format | -| `--ignore-existing` | | skip updating files that already exist at the destination | -| `--info=FLAGS` | | fine-grained informational verbosity; the `progress2` value is available since version 3.1.0 | -| `--links` | `-l` | copy symlinks as symlinks | -| `--no-inc-recursive` | `--no-i-r` | scan all directories on startup instead of incrementally | -| `--no-motd` | | suppress daemon-mode MOTD | -| `--no-OPTION` | | turn off an **implied** OPTION (e.g. `--no-D`) | -| `--partial` | | keep partially transferred files | -| `--progress` | | show progress for each file during transfer | -| `--protect-args` | `-s` | no space-splitting; wildcard chars only | -| `--prune-empty-dirs` | `-m` | prune empty directory chains from file-list | -| `--recursive` | `-r` | recurse into directories | -| `--remove-source-files` | | set the **sender** to remove synchronized files; it does **not** remove directories | -| `--rsh=COMMAND` | `-e` | specify the remote shell to use (with options, e.g. _ssh -p 1234_) | -| `--sparse` | `-S` | turn sequences of nulls into sparse blocks | -| `--stats` | | give some file-transfer stats | -| `--suffix=SUFFIX` | | suffix for backups; defaults to `~` | -| `--update` | `-u` | skip files that are newer on the receiver | -| `--verbose` | `-v` | increase verbosity once for each copy of this switch | -| `--xattrs` | `-X` | preserve extended attributes | +| Long format | Short format | Description | +| ----------------------- | ------------ | --------------------------------------------------------------------------------------------------- | +| | `-F` | same as `--filter='dir-merge /.rsync-filter'`
if repeated, same as `--filter='- .rsync-filter'` | +| | `-P` | same as `--partial --progress` | +| `--acls` | `-A` | preserve ACLs; implies `--perms` | +| `--append-verify` | | like `--append`, but use the data already there to check the items | +| `--archive` | `-a` | archive mode, equals `-rlptgoD`; does **not** imply `-H`, `-A`, nor `-X` | +| `--backup-dir=DIR` | | use the specified directory to backup changing items | +| `--backup` | `-b` | backup items changing at the destination; see also `--suffix` and `--backup-dir` | +| `--bwlimit=RATE` | | limit the socket's I/O bandwidth to _RATE_; with no suffix, the value will be in KBPS | +| `--checksum` | `-c` | skip files basing on checksum instead of modify time and size | +| `--chown=USER:GROUP` | | simple username/groupname mapping | +| `--compress` | `-z` | compress file data during the transfer | +| `--crtimes` | | **only available on Mac OS X** | +| `--delete-during` | `--del` | set the **receiver** to delete files during the transfer | +| `--delete` | | delete items **at the destination** that **don't** exist in the source | +| `--dry-run` | `-n` | perform a trial run with no changes made | +| `--exclude=PATTERN` | | exclude files matching _PATTERN_ | +| `--executability` | `-E` | preserve executability | +| `--fake-super` | | store/recover privileged attrs using xattrs | +| `--filter=RULE` | `-f` | add a file-filtering _RULE_ | +| `--hard-links` | `-H` | preserve hard links | +| `--human-readable` | `-h` | output numbers in a human-readable format | +| `--ignore-existing` | | skip updating files that already exist at the destination | +| `--info=FLAGS` | | fine-grained informational verbosity; the `progress2` value is available since version 3.1.0 | +| `--links` | `-l` | copy symlinks as symlinks | +| `--no-inc-recursive` | `--no-i-r` | scan all directories on startup instead of incrementally | +| `--no-motd` | | suppress daemon-mode MOTD | +| `--no-OPTION` | | turn off an **implied** OPTION (e.g. `--no-D`) | +| `--partial` | | keep partially transferred files | +| `--progress` | | show progress for each file during transfer | +| `--protect-args` | `-s` | no space-splitting; wildcard chars only | +| `--prune-empty-dirs` | `-m` | prune empty directory chains from file-list | +| `--recursive` | `-r` | recurse into directories | +| `--remove-source-files` | | set the **sender** to remove synchronized files; it does **not** remove directories | +| `--rsh=COMMAND` | `-e` | specify the remote shell to use (with options, e.g. _ssh -p 1234_) | +| `--sparse` | `-S` | turn sequences of nulls into sparse blocks | +| `--stats` | | give some file-transfer stats | +| `--suffix=SUFFIX` | | suffix for backups; defaults to `~` | +| `--update` | `-u` | skip files that are newer on the receiver | +| `--verbose` | `-v` | increase verbosity once for each copy of this switch | +| `--xattrs` | `-X` | preserve extended attributes | + +## Filters + +### Filter file + +Set up a `.rsync-filter` file in any directory. If `rsync` is called with the `-F` option, the filtering rules in that file will be applied from that directory to all its subfolders. + +```sh +$ cat '.rsync-filter' +- .DS_Store +- .localized +- .obsidian +- .terraform* + ++ ** + +$ rsync … -F +``` + +The filter file excludes files from the source, but does nothing for the ones on the remote.
+If one wants to exclude files from the remote, they must be set explicitly using the `--exclude` option. ## Sources diff --git a/knowledge base/zstd.md b/knowledge base/zstd.md index 83d945f..9203a5b 100644 --- a/knowledge base/zstd.md +++ b/knowledge base/zstd.md @@ -9,8 +9,15 @@ The `zstd`, `zstdmt`, `unzstd`, `zstdcat` utilities compress or decompress `.zst ## TL;DR ```sh -zstd -15 -r --compress --rsyncable "folder" -o "folder.zst" -zstd --test -zstd --list -zstd --decompress +# Compress folders into an archive. +zstd --compress -15 --rsyncable -r 'folder' -o 'archive.zst' + +# Test archives. +zstd --test 'archive.zst' + +# Print information about files in archives. +zstd --list 'archive.zst' + +# Decompress archives. +zstd --decompress 'archive.zst' ```