chore(opensearch): review and dump final findings

snippets/opensearch/api.txt

@@ -1,7 +1,11 @@
 # List indices
+GET _list/indices
+GET _list/indices/index-name-here?v
+GET _list/indices/index1,index2,index3?v
 # in cold storage
 GET _cold/indices/_search
-# in warm storage
+# in hot or warm storage (aws-managed domains only)
+GET _cat/indices/_hot
 GET _cat/indices/_warm
 
 # Search for indices in cold storage
@@ -56,6 +60,29 @@ POST _reindex?pretty
     "dest": {"index": "destinationIndex"}
 }
 
+# Close open indices
+# disables read and write operations on the impacted index
+POST /prometheus-logs-20231205/_close
+
+# Re-open closed indices
+# re-enables read and write operations on the impacted index
+POST /prometheus-logs-20231205/_open
+
+# Update indices' settings.
+# static settings can only be updated on *closed* indexes.
+PUT /prometheus-logs-20231205/_settings
+{
+    "index": {
+        "codec": "zstd_no_dict",
+        "codec.compression_level": 3,
+        "refresh_interval": "2s"
+    }
+}
+
+# Delete indices
+# one at a time
+DELETE /index-name-here
+
 # Register snapshot repositories
 # aws-managed domains
 PUT _snapshot/repository-name-here

snippets/opensearch/commands.fish

@@ -21,7 +21,8 @@ awscurl --service 'es' \
 | jq -r '.indices[].index' - \
 | tr '\n' ','
 
-# Migrate all indices from warm to hot storage
+# Migrate all indices from ultrawarm to hot storage
+# only aws-managed opensearch domains
 awscurl --service 'es' \
 	'https://search-aws-domain-abcdefghijklmnopqrstuvwxyz.eu-west-1.es.amazonaws.com/_cat/indices/_warm' \
 | grep 'app-cwl-' | sort | cut -d ' ' -f 3 \
@@ -35,6 +36,13 @@ seq 83 72 \
 	'https://search-aws-domain-abcdefghijklmnopqrstuvwxyz.eu-west-1.es.amazonaws.com/_snapshot/repo/app-logs-0000%%' \
 	-d '{"indices": "app-logs-0000%%", "include_global_state": false}'
 
+# Keep an eye on snapshots
+watch -n '5' " \
+	awscurl --service 'es' \
+		'https://search-aws-domain-abcdefghijklmnopqrstuvwxyz.eu-west-1.es.amazonaws.com/_snapshot/_status' \
+	| jq '.snapshots[]?|{\"name\":.snapshot,\"state\":.state,\"shards\":.shards_stats}' - \
+"
+
 # Delete indices that have been snapshotted
 awscurl --service 'es' \
 	'https://search-aws-domain-abcdefghijklmnopqrstuvwxyz.eu-west-1.es.amazonaws.com/_snapshot/some-repo/some-snap' \