chore(turbot-pipes): save notions found by trying to run powerpipe from a gitlab task

2026-02-09 05:44:23 +00:00 · 2024-05-23 19:06:42 +02:00
parent 07d7f3545d
commit 369781894e
5 changed files with 170 additions and 19 deletions
--- a/base/apt.md
+++ b/base/apt.md
@@ -14,6 +14,8 @@

 ## TL;DR

+Non-interactive mode: `DEBIAN_FRONTEND='noninteractive' apt`.
+
 ```sh
 # Check for broken dependencies.
 sudo apt-get check
@@ -37,6 +39,7 @@ apt depends 'ufw'

 # Install packages.
 sudo apt install 'nano' 'python3-zstd/stable' 'gparted=0.16.1-1'
+sudo apt install --no-install-recommends --assume-yes 'curl'

 # Remove packages.
 sudo apt remove 'dhclient' 'sudo'
--- a/pipes/powerpipe.md
+++ b/pipes/powerpipe.md
@@ -11,6 +11,11 @@ Quick 'n' easy dashboards for DevOps.
 Powerpipe **requires** a database to run its queries from.<br/>
 By default it uses [Steampipe]'s, but it [can be specified][selecting a database].

+_Controls_ allow to draw specific conclusion (e.g. 'OK', 'Alarm') about each row in queries.<br/>
+_Benchmarks_ group controls and other benchmarks into hierarchies of any depth.
+
+Default files directory (called _installation directory_ by Powerpipe) is `$HOME/.powerpipe`.
+
 <details>
  <summary>Installation and configuration</summary>

@@ -19,7 +24,23 @@ brew install 'turbot/tap/powerpipe'
 ```

 ```sh
+# Initialize the current directory.
+# Creates a 'mod.pp' file.
 powerpipe mod init
+
+# Disable telemetry.
+export POWERPIPE_TELEMETRY='none'
+
+# Configuration.
+# Most can be set through switch.
+# These are set to their defaults.
+export \
+  POWERPIPE_INSTALL_DIR="${HOME}/.powerpipe" \
+  POWERPIPE_LISTEN='network' \
+  POWERPIPE_MAX_PARALLEL=10 \
+  POWERPIPE_MOD_LOCATION="$PWD" \
+  POWERPIPE_PORT=9033 \
+  POWERPIPE_UPDATE_CHECK=true
 ```

 </details>
@@ -28,12 +49,49 @@ powerpipe mod init
  <summary>Usage</summary>

 ```sh
-# Install mods.
-# If none given, install all those specified in 'mod.pp' with their dependencies.
+# Install mods with their dependencies.
+# If none is given, install all those specified in the 'mod.pp' configuration file.
 powerpipe mod install
-powerpipe mod install 'github.com/turbot/steampipe-mod-aws-insights'
-powerpipe mod install --dry-run 'github.com/turbot/steampipe-mod-aws-compliance@v0.93.0'
-powerpipe mod install github.com/turbot/steampipe-mod-aws-compliance@'^1'
+powerpipe mod install 'github.com/turbot/steampipe-mod-aws-insights' 'github.com/turbot/steampipe-mod-aws-tags@v0.13'
+powerpipe mod install --dry-run 'github.com/turbot/steampipe-mod-aws-compliance@^0.92'
+
+# List installed mods.
+powerpipe mod list
+
+# Update mods.
+powerpipe mod update 'github.com/turbot/steampipe-mod-aws-compliance'
+
+# Uninstall mods.
+powerpipe mod uninstall 'github.com/turbot/steampipe-mod-aws-compliance'
+
+# List available queries.
+powerpipe query list
+
+# Show queries' information.
+powerpipe query show 'aws_insights.query.vpc_vpcs_for_vpc_subnet'
+
+# Run named queries.
+powerpipe query run 'aws_insights.query.vpc_vpcs_for_vpc_subnet'
+
+# List available controls.
+powerpipe control list
+
+# Execute controls.
+# Only one at a time.
+powerpipe control run 'aws_compliance.control.cis_v150_3_3'
+
+# List available benchmarks.
+powerpipe benchmark list
+
+# Execute benchmarks.
+powerpipe benchmark run 'aws_compliance.benchmark.cis_v300' 'aws_compliance.benchmark.gdpr'
+powerpipe benchmark run … --where "severity in ('critical', 'high')" --tag 'cis_level=1' --tag 'cis=true'
+powerpipe benchmark run … --output 'brief' --export 'output.csv' --export 'output.json' --export 'md' --export 'nunit3'
+powerpipe benchmark run … --database 'postgres://myUser:myPassword@myDbFqdn:9193/steampipe'
+
+# Run *all* benchmarks in mods.
+# This will *not* run benchmarks in the mods' dependencies.
+powerpipe benchmark run all

 # Start the dashboard.
 # Wait for server initialization before connecting.
--- a/pipes/steampipe.md
+++ b/pipes/steampipe.md
@@ -8,7 +8,7 @@ Dynamically query APIs, code and more with SQL.

 ## TL;DR

-Default config directory is `$HOME/.steampipe`.
+Default files directory (called _installation directory_ by Steampipe) is `$HOME/.steampipe`.

 <details>
  <summary>Installation and configuration</summary>
@@ -18,6 +18,18 @@ brew install 'turbot/tap/steampipe'

 steampipe completion fish | source
 steampipe completion fish > "$HOME/.config/fish/completions/steampipe.fish"
+
+# Disable telemetry.
+export STEAMPIPE_TELEMETRY='none'
+
+# Most used configuration settings.
+# Most can be set through switch.
+# These are set to their defaults.
+export \
+  STEAMPIPE_INSTALL_DIR="${HOME}/.steampipe" \
+  STEAMPIPE_MAX_PARALLEL=10 \
+  STEAMPIPE_MOD_LOCATION="$PWD" \
+  STEAMPIPE_UPDATE_CHECK=true
 ```

 </details>
@@ -27,8 +39,17 @@ steampipe completion fish > "$HOME/.config/fish/completions/steampipe.fish"

 ```sh
 # Install plugins.
-steampipe plugin install 'steampipe'
-steampipe plugin install 'aws'
+steampipe plugin install 'steampipe' 'aws@^0.130' 'theapsgroup/gitlab@v0.6.0'
+
+# List installed plugins.
+steampipe plugin list
+
+# Update plugins.
+steampipe plugin update --all
+steampipe plugin update 'steampipe' 'aws'
+
+# Uninstall plugins.
+steampipe plugin uninstall 'steampipe' 'theapsgroup/gitlab@0.6.0' 'hub.steampipe.io/plugins/turbot/aws@^0'

 # Start the service.
 steampipe service start
@@ -61,14 +82,14 @@ steampipe query 'query'
 steampipe query 'query' --output 'json'
 steampipe query 'query' --output 'csv' --separator '|'

-# Executes benchmarks and controls.
-steampipe check 'benchmark.cis_v130'
-steampipe check 'control.cis_v130_1_4' 'control.cis_v130_2_1_1'
-steampipe check 'all'
-steampipe check … --tag 'cis_level=1' --tag 'cis=true' --search-path-prefix 'aws_connection_2'
-steampipe check … --where "severity in ('critical', 'high')" --dry-run
-steampipe check … --theme 'light' --output 'brief' --export 'output.csv' --export 'output.json' --export 'md'
-steampipe check … --theme 'plain' --progress false
+# Execution of benchmarks and controls has been deprecated in favour of Powerpipe.
+#steampipe check 'benchmark.cis_v130'
+#steampipe check 'control.cis_v130_1_4' 'control.cis_v130_2_1_1'
+#steampipe check 'all'
+#steampipe check … --tag 'cis_level=1' --tag 'cis=true' --search-path-prefix 'aws_connection_2'
+#steampipe check … --where "severity in ('critical', 'high')" --dry-run
+#steampipe check … --theme 'light' --output 'brief' --export 'output.csv' --export 'output.json' --export 'md'
+#steampipe check … --theme 'plain' --progress false
 ```

 </details>
@@ -91,7 +112,7 @@ where
  and p.is_aws_managed;
 ```

-Dashboards have been deprecated from Steampipe. Use [Powerpipe] instead.
+Dashboards and Mods have been deprecated in favour of [Powerpipe]. Use it instead.

 </details>

--- a/snippets/create-users.sh
+++ b/snippets/create-users.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+
+useradd --create-home --password 'encrypted-password' --shell '/bin/bash' 'username'
+useradd -m -p 'encrypted-password' -s '/bin/bash' 'username'
+
+# Non-interactive.
+# Skip finger information.
+adduser --disabled-password --gecos '' --shell '/bin/sh' 'username'
--- a/snippets/gitlab.pipeline.tasks.yml
+++ b/snippets/gitlab.pipeline.tasks.yml
@@ -5,12 +5,73 @@ print-postgis-service-logs:
  variables:
    CI_DEBUG_SERVICES: "true"
  services:
-    - name: "postgis/postgis:15-3.3"
+    - name: postgis/postgis:15-3.4@sha256:6a6eb58d25a331da1d2532412641330b064ffec33f294aa5a7812fe26a6ed2f3
      alias: db
      variables:
        POSTGRES_DB: postgres
        POSTGRES_USER: postgres
-        POSTGRES_PASSWORD: "postgres"
+        POSTGRES_PASSWORD: postgres
        POSTGRES_HOST_AUTH_METHOD: trust
  script:
    - echo 'hello!!'
+
+docker-build-image-dind:
+  rules:
+    - when: manual
+  variables:
+    BUILDER_NAME: tmp-$CI_JOB_ID
+    DOCKER_VERSION: '26.1.2'
+    GIT_DEPTH: '1'
+    IMAGE_NAME: $CI_PROJECT_NAME
+    IMAGE_TAG: $CI_COMMIT_SHORT_SHA
+    PLATFORM: linux/amd64
+  services:
+    - docker:$DOCKER_VERSION-dind
+  image: library/docker:DOCKER_VERSION-cli-alpine3.19
+  before_script:
+    - docker info
+    - docker buildx create --driver 'docker-container' --name "$BUILDER_NAME" --use
+  script:
+    - docker buildx build --platform "$PLATFORM" --tag "$IMAGE_NAME/$IMAGE_TAG" '.'
+  after_script:
+    - docker buildx rm "$BUILDER_NAME"
+
+powerpipe-report:
+  # Strongly suggested to just create your own image for this.
+  # Initializing it from scratch alone takes me about 8 mins.
+  stage: test
+  rules:
+    - when: manual
+  variables:
+    DEBIAN_FRONTEND: noninteractive
+    POWERPIPE_MOD_LOCATION: /home/piper/powerpipe
+    POWERPIPE_TELEMETRY: none
+    POWERPIPE_UPDATE_CHECK: 'false'
+    POWERPIPE_VERSION: v0.3.0
+    STEAMPIPE_MOD_LOCATION: /home/piper/steampipe
+    STEAMPIPE_TELEMETRY: none
+    STEAMPIPE_UPDATE_CHECK: 'false'
+    STEAMPIPE_VERSION: v0.23.2
+  image: library/debian:12.5-slim@sha256:804194b909ef23fb995d9412c9378fb3505fe2427b70f3cc425339e48a828fca
+  before_script:
+    - |-
+        : "${AWS_ACCESS_KEY_ID?required}"
+        : "${AWS_SECRET_ACCESS_KEY?required}"
+    - adduser --disabled-password --gecos '' --shell '/bin/sh' 'piper'
+    - apt update
+    - apt install --assume-yes --no-install-recommends 'curl' 'ca-certificates'
+    - curl -fsSL -O 'https://steampipe.io/install/steampipe.sh' -O 'https://powerpipe.io/install/powerpipe.sh'
+    - su piper -c "mkdir -pv '$STEAMPIPE_MOD_LOCATION' '$POWERPIPE_MOD_LOCATION'"
+    - /bin/sh steampipe.sh "$STEAMPIPE_VERSION"
+    - /bin/sh powerpipe.sh "$POWERPIPE_VERSION"
+    - su piper -c "steampipe service start --database-listen 'local'"
+  script:
+    - su piper -c "steampipe plugin install 'aws'"
+    - su piper -c "powerpipe mod install 'github.com/turbot/steampipe-mod-aws-compliance'"
+    - su piper -c "powerpipe benchmark run 'aws_compliance.benchmark.gdpr' --export 'nunit3'"
+  artifacts:
+    when: always
+    expire_in: 1 week
+    reports:
+      # not a junit, so useless, but hey…
+      junit: "*.nunit3.xml"