refactor prometheus as a module

2 files changed, 189 insertions, 0 deletions

diff --git a/modules/services/default.nix b/modules/services/default.nix
index 8da870e..97a326a 100644
--- a/modules/services/default.nix
+++ b/modules/services/default.nix
@@ -4,6 +4,7 @@
   imports = [
     ./fwupd
     ./navidrome
+    ./prometheus
     ./samba
     ./ssh-server
     ./tailscale
diff --git a/modules/services/prometheus/default.nix b/modules/services/prometheus/default.nix
new file mode 100644
index 0000000..6eeee16
--- /dev/null
+++ b/modules/services/prometheus/default.nix
@@ -0,0 +1,188 @@
+{ config, pkgs, lib, ... }:
+
+let
+  cfg = config.my.services.prometheus;
+  blackboxConfig = {
+    modules = {
+      https_2xx = {
+        prober = "http";
+        timeout = "5s";
+        http = {
+          method = "GET";
+          valid_status_codes = [ ];
+          fail_if_not_ssl = true;
+          no_follow_redirects = false;
+          tls_config = { insecure_skip_verify = false; };
+          preferred_ip_protocol = "ip4";
+        };
+      };
+      icmp = {
+        prober = "icmp";
+        icmp = { preferred_ip_protocol = "ip4"; };
+        timeout = "5s";
+      };
+    };
+  };
+  relabelConfigs = [
+    {
+      source_labels = [ "__address__" ];
+      target_label = "instance";
+      replacement = "nas";
+      action = "replace";
+      regex = "192.168.6.10:(.*)";
+    }
+    {
+      source_labels = [ "__address__" ];
+      target_label = "instance";
+      replacement = "rtr";
+      action = "replace";
+      regex = "192.168.6.1:(.*)";
+    }
+    {
+      source_labels = [ "__address__" ];
+      target_label = "instance";
+      replacement = "tahoe";
+      action = "replace";
+      regex = "192.168.6.20:(.*)";
+    }
+  ];
+in {
+  options.my.services.navidrome = with lib; {
+    enable = mkEnableOption "Prometheus monitoring solution";
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.prometheus.exporters.blackbox = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      port = 9115;
+      configFile =
+        pkgs.writeText "blackbox.yml" (builtins.toJSON blackboxConfig);
+    };
+
+    services.prometheus = {
+      enable = true;
+
+      globalConfig.scrape_interval = "15s";
+
+      extraFlags = [
+        # 3 years of retention
+        "--storage.tsdb.retention=${toString (365 * 3)}d"
+        "--web.enable-admin-api"
+      ];
+
+      scrapeConfigs = [
+        {
+          job_name = "blackbox-ping";
+          metrics_path = "/probe";
+          params = { module = [ "icmp" ]; };
+          static_configs = [{
+            targets = [ "8.8.8.8" "1.1.1.1" "git.fcuny.net" "fcuny.net" ];
+          }];
+          relabel_configs = [
+            {
+              source_labels = [ "__address__" ];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = [ "__param_target" ];
+              target_label = "instance";
+            }
+            {
+              target_label = "__address__";
+              replacement = "localhost:9115";
+            }
+          ];
+        }
+        {
+          job_name = "blackbox-http";
+          metrics_path = "/probe";
+          params = { module = [ "https_2xx" ]; };
+          static_configs = [{
+            targets = [
+              "https://fcuny.net"
+              "https://git.fcuny.net"
+              "https://notes.fcuny.net"
+            ];
+          }];
+          relabel_configs = [
+            {
+              source_labels = [ "__address__" ];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = [ "__param_target" ];
+              target_label = "instance";
+            }
+            {
+              target_label = "__address__";
+              replacement = "localhost:9115";
+            }
+          ];
+        }
+        {
+          job_name = "node";
+          static_configs =
+            [{ targets = [ "192.168.6.1:9100" "192.168.6.20:9100" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "prometheus";
+          static_configs = [{ targets = [ "192.168.6.20:9090" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "traefik";
+          static_configs = [{ targets = [ "192.168.6.20:8090" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "gitea";
+          static_configs = [{ targets = [ "192.168.6.20:8002" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+
+        {
+          job_name = "dnsd";
+          static_configs = [{ targets = [ "192.168.6.1:8053" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "dnsdd";
+          static_configs = [{ targets = [ "192.168.6.1:9060" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "dhcpd";
+          static_configs = [{ targets = [ "192.168.6.1:8067" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "netd";
+          static_configs = [{ targets = [ "192.168.6.1:8055" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "unifi-poller";
+          static_configs = [{ targets = [ "192.168.6.20:9130" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+      ];
+    };
+
+    age.secrets.restic-repo-systems.file =
+      ../../../secrets/restic/repo-systems.age;
+
+    services.restic.backups = {
+      prometheus = {
+        paths = [ "/var/lib/prometheus2" ];
+        repository = "/data/slow/backups/systems";
+        passwordFile = config.age.secrets.restic-repo-systems.path;
+        initialize = true;
+        timerConfig = { OnCalendar = "00:25"; };
+        extraBackupArgs = [ "--tag prometheus" ];
+        pruneOpts = [ "--keep-daily 7" "--keep-weekly 4" ];
+      };
+    };
+  };
+}