From 6526a23cdfafe9e0b430fe0a10d479f3801ee7b4 Mon Sep 17 00:00:00 2001
From: Michele Cereda <cereda.michele@gmail.com>
Date: Fri, 24 Jan 2025 19:15:50 +0100
Subject: [PATCH] chore(logstash): improve pipeline example and commands

---
 knowledge base/logstash.md | 68 ++++++++++++++++++++++++++++++++++----
 snippets/logstash.fish     | 19 +++++++++++
 2 files changed, 81 insertions(+), 6 deletions(-)
 create mode 100644 snippets/logstash.fish

diff --git a/knowledge base/logstash.md b/knowledge base/logstash.md
index 8f5edac..a4b5757 100644
--- a/knowledge base/logstash.md	
+++ b/knowledge base/logstash.md	
@@ -10,23 +10,36 @@ Part of the Elastic Stack along with Beats, [ElasticSearch] and [Kibana].
 
 ## TL;DR
 
-<!-- Uncomment if used
 <details>
   <summary>Setup</summary>
 
 ```sh
+docker pull 'logstash:7.17.27'
 ```
 
 </details>
--->
 
 <details>
   <summary>Usage</summary>
 
 ```sh
+# Get a shell in the docker image.
+docker run --rm -ti --name 'logstash' --entrypoint 'bash' 'logstash:7.17.27'
+
 # Validate configuration files.
 logstash -tf 'config.conf'
-logstash --config.test_and_exit --path.config 'config.conf'
+logstash --config.test_and_exit --path.config 'config.conf' --api.enabled='false'
+# If given a directory, will load and check all files in it.
+logstash --config.test_and_exit --path.config 'configDir' --log.level='debug'
+docker run --rm -ti -v "$PWD:/usr/share/logstash/custom" 'docker.io/library/logstash:7.17.27' -tf 'custom'
+
+# Automatically reload configuration files on change.
+# Default interval is '3s'.
+logstash … --config.reload.automatic
+logstash … --config.reload.automatic --config.reload.interval '5s'
+
+# Force configuration files reload and restart the pipelines.
+kill -SIGHUP '14175'
 
 
 # Install plugins.
@@ -37,26 +50,67 @@ logstash-plugin list
 logstash-plugin list --verbose
 logstash-plugin list '*namefragment*'
 logstash-plugin list --group 'output'
+
+
+# Get Logstash's status.
+curl -fsS 'localhost:9600/_health_report?pretty'
+
+# Get pipelines statistics.
+curl -fsS 'localhost:9600/_node/stats/pipelines?pretty'
+curl -fsS 'localhost:9600/_node/stats/pipelines/somePipeline?pretty'
 ```
 
 ```rb
-input { … }
+input {
+  file {
+    path => "/var/log/logstash/logstash-plain.log"
+  }
+  syslog {
+    port => 9292
+    codec => "json"
+  }
+  tcp {
+    port => 9191
+    codec => "json"
+  }
+}
 
 filter {
+  grok {
+    match => { "message" => "\[%{TIMESTAMP_ISO8601:timestamp}\]\[%{LOGLEVEL:loglevel}\] .+" }
+  }
+  json {
+    skip_on_invalid_json => true
+    source => "message"
+    add_tag => ["json_body"]
+  }
   mutate {
     add_field => {
-      "cluster" => "us-central-1"
+      "cluster" => "eu-west-1"
       "job" => "logstash"
     }
     replace => { "type" => "stream"}
     remove_field => [ "src" ]
   }
+
+  if [loglevel] != "ERROR" and [loglevel] != "WARN" {
+    drop { }
+  }
 }
 
 output {
   loki {
     url => "http://loki.example.org:3100/loki/api/v1/push"
   }
+  opensearch {
+    hosts => [ "https://os.example.org:443" ]
+    auth_type => {
+      type => 'aws_iam'
+      region => 'eu-west-1'
+    }
+    index => "something-%{+YYYY.MM.dd}"
+    action => "create"
+  }
 }
 ```
 
@@ -76,6 +130,7 @@ output {
 
 - [Website]
 - [Codebase]
+- [Documentation]
 - [Beats], [ElasticSearch] and [Kibana]: the rest of the Elastic stack
 
 ### Sources
@@ -96,7 +151,8 @@ output {
 <!-- Files -->
 <!-- Upstream -->
 [codebase]: https://github.com/elastic/logstash
-[website]: https://website/
+[documentation]: https://www.elastic.co/guide/en/logstash/current/
+[website]: https://www.elastic.co/logstash
 
 <!-- Others -->
 [how to debug your logstash configuration file]: https://logz.io/blog/debug-logstash/
diff --git a/snippets/logstash.fish b/snippets/logstash.fish
new file mode 100644
index 0000000..c0ad6af
--- /dev/null
+++ b/snippets/logstash.fish
@@ -0,0 +1,19 @@
+#!/usr/bin/env fish
+
+# Validate configuration files
+logstash -tf 'config.conf'
+logstash --config.test_and_exit --path.config 'configDir' --log.level='debug'
+ls -1 *'.conf' | xargs -tn1 /usr/share/logstash/bin/logstash --api.enabled='false' --log.level='info' -tf
+docker run --rm -ti -v "$PWD:/usr/share/logstash/custom" 'docker.io/library/logstash:7.17.27' \
+	--api.enabled='false' --log.level='info' -tf 'custom'
+
+# Force configuration files reload and restart the pipelines
+kill -SIGHUP '14175'
+
+# Get Logstash's status
+curl -fsS 'localhost:9600/_health_report?pretty'
+
+# Get pipelines statistics
+curl -fsS 'localhost:9600/_node/stats/pipelines?pretty'
+curl -fsS 'localhost:9600/_node/stats/pipelines/somePipeline?pretty'
+curl -fsS 'localhost:9600/_node/stats/pipelines/serviceName' | jq '.pipelines[].plugins.outputs' -