modules/services: add loki and promtail

5 files changed, 178 insertions, 0 deletions

diff --git a/hosts/tahoe/services.nix b/hosts/tahoe/services.nix
index 87a71cf..8783e8c 100644
--- a/hosts/tahoe/services.nix
+++ b/hosts/tahoe/services.nix
@@ -26,6 +26,13 @@ in
       enable = true;
       vhostName = "dash.fcuny.xyz";
     };
+
+    loki = {
+      enable = true;
+      address = "192.168.6.40";
+    };
+
+    metrics-exporter.promtail.enable = true;
     cgit = {
       enable = true;
     };
diff --git a/modules/services/default.nix b/modules/services/default.nix
index d215173..fd42c23 100644
--- a/modules/services/default.nix
+++ b/modules/services/default.nix
@@ -11,6 +11,7 @@
     ./gitolite
     ./gnome
     ./grafana
+    ./loki
     ./metrics-exporter
     ./navidrome
     ./nginx
diff --git a/modules/services/loki/default.nix b/modules/services/loki/default.nix
new file mode 100644
index 0000000..89c2645
--- /dev/null
+++ b/modules/services/loki/default.nix
@@ -0,0 +1,103 @@
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.my.services.loki;
+in
+{
+  options.my.services.loki = with lib; {
+    enable = mkEnableOption "loki observability stack";
+    address = mkOption {
+      type = types.str;
+      example = "192.168.6.40";
+      description = "Listen address";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.loki = {
+      enable = true;
+      configuration = {
+        # no need for authentication, since we're binding on the
+        # wireguard interface, we can trust the connections.
+        auth_enabled = false;
+
+        server = {
+          http_listen_port = 3100;
+          http_listen_address = cfg.address;
+        };
+
+        ingester = {
+          lifecycler = {
+            address = cfg.address;
+            ring = {
+              kvstore = { store = "inmemory"; };
+              replication_factor = 1;
+            };
+            final_sleep = "0s";
+          };
+
+          # Any chunk not receiving new logs in this time will be flushed
+          chunk_idle_period = "1h";
+
+          # All chunks will be flushed when they hit this age, default is 1h
+          max_chunk_age = "1h";
+
+          # Loki will attempt to build chunks up to 1.5MB, flushing first if
+          # chunk_idle_period or max_chunk_age is reached first
+          chunk_target_size = 1048576;
+
+          # Must be greater than index read cache TTL if using an index cache (Default
+          # index read cache TTL is 5m)
+          chunk_retain_period = "30s";
+
+          # Chunk transfers disabled
+          max_transfer_retries = 0;
+        };
+
+        schema_config = {
+          configs = [{
+            from = "2020-10-24";
+            store = "boltdb-shipper";
+            object_store = "filesystem";
+            schema = "v11";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }];
+        };
+
+        storage_config = {
+          boltdb_shipper = {
+            active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+            cache_location = "/var/lib/loki/boltdb-shipper-cache";
+
+            # Can be increased for faster performance over longer query periods,
+            # uses more disk space
+            cache_ttl = "24h";
+
+            shared_store = "filesystem";
+          };
+
+          filesystem = { directory = "/var/lib/loki/chunks"; };
+        };
+
+        limits_config = {
+          reject_old_samples = true;
+          reject_old_samples_max_age = "168h";
+        };
+
+        chunk_store_config = { max_look_back_period = "0s"; };
+
+        table_manager = {
+          retention_deletes_enabled = false;
+          retention_period = "0s";
+        };
+
+        compactor = {
+          working_directory = "/var/lib/loki/boltdb-shipper-compactor";
+          shared_store = "filesystem";
+        };
+      };
+    };
+  };
+}
diff --git a/modules/services/metrics-exporter/default.nix b/modules/services/metrics-exporter/default.nix
index f489f78..c3c471c 100644
--- a/modules/services/metrics-exporter/default.nix
+++ b/modules/services/metrics-exporter/default.nix
@@ -2,6 +2,8 @@
 let cfg = config.my.services.metrics-exporter;
 in
 {
+  imports = [ ./promtail.nix ];
+
   options.my.services.metrics-exporter = with lib; {
     enable = mkEnableOption "Prometheus metrics exporter";
   };
diff --git a/modules/services/metrics-exporter/promtail.nix b/modules/services/metrics-exporter/promtail.nix
new file mode 100644
index 0000000..eb574d0
--- /dev/null
+++ b/modules/services/metrics-exporter/promtail.nix
@@ -0,0 +1,65 @@
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.my.services.metrics-exporter.promtail;
+in
+{
+  options.my.services.metrics-exporter.promtail = with lib; {
+    enable = mkEnableOption "promtail logs exporter";
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.promtail = {
+      enable = true;
+      configuration = {
+        clients = [{ url = "http://192.168.6.40:3100/loki/api/v1/push"; }];
+        scrape_configs = [
+          {
+            job_name = "journal";
+            journal = {
+              json = true;
+              path = "/var/log/journal";
+              max_age = "12h";
+              labels = {
+                host = "carmel";
+                job = "journal";
+                "__path__" = "/var/log/journal";
+              };
+            };
+
+            relabel_configs = [
+              {
+                source_labels = [ "__journal__systemd_unit" ];
+                target_label = "unit";
+              }
+              {
+                source_labels = [ "__journal_priority" ];
+                target_label = "priority";
+              }
+              {
+                source_labels = [ "__journal_syslog_identifier" ];
+                target_label = "syslog_id";
+              }
+            ];
+          }
+
+          {
+            job_name = "nginx";
+            static_configs = [{
+              labels = {
+                host = "carmel";
+                job = "nginx";
+                __path__ = "/var/log/nginx/*";
+              };
+            }];
+          }
+        ];
+
+        server = {
+          http_listen_port = 9832;
+          http_path_prefix = "/promtail";
+          grpc_listen_port = 0;
+        };
+      };
+    };
+  };
+}