modules/monitoring: consolidate all monitoring services together

This will help to organize and structure monitoring modules a bit
better.

1 files changed, 140 insertions, 0 deletions

diff --git a/modules/services/monitoring/prometheus.nix b/modules/services/monitoring/prometheus.nix
new file mode 100644
index 0000000..59cccb6
--- /dev/null
+++ b/modules/services/monitoring/prometheus.nix
@@ -0,0 +1,140 @@
+{ config, pkgs, lib, ... }:
+
+let
+  cfg = config.my.services.monitoring.prometheus;
+  blackboxConfig = {
+    modules = {
+      https_2xx = {
+        prober = "http";
+        timeout = "5s";
+        http = {
+          method = "GET";
+          valid_status_codes = [ ];
+          fail_if_not_ssl = true;
+          no_follow_redirects = false;
+          tls_config = { insecure_skip_verify = false; };
+          preferred_ip_protocol = "ip4";
+        };
+      };
+      icmp = {
+        prober = "icmp";
+        icmp = { preferred_ip_protocol = "ip4"; };
+        timeout = "5s";
+      };
+    };
+  };
+  relabelConfigs = [
+    {
+      source_labels = [ "__address__" ];
+      target_label = "instance";
+      replacement = "carmel";
+      action = "replace";
+      regex = "192.168.6.1:(.*)";
+    }
+    {
+      source_labels = [ "__address__" ];
+      target_label = "instance";
+      replacement = "tahoe";
+      action = "replace";
+      regex = "192.168.6.40:(.*)";
+    }
+  ];
+in
+{
+  options.my.services.monitoring.prometheus = with lib; {
+    enable = mkEnableOption "Prometheus monitoring solution";
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.prometheus.exporters.blackbox = {
+      enable = true;
+      listenAddress = "127.0.0.1";
+      port = 9115;
+      configFile =
+        pkgs.writeText "blackbox.yml" (builtins.toJSON blackboxConfig);
+    };
+
+    services.prometheus = {
+      enable = true;
+
+      globalConfig.scrape_interval = "15s";
+
+      extraFlags = [
+        # 3 years of retention
+        "--storage.tsdb.retention=${toString (365 * 3)}d"
+        "--web.enable-admin-api"
+      ];
+
+      scrapeConfigs = [
+        {
+          job_name = "blackbox-ping";
+          metrics_path = "/probe";
+          params = { module = [ "icmp" ]; };
+          static_configs = [{
+            targets = [ "8.8.8.8" "1.1.1.1" "git.fcuny.net" "fcuny.net" ];
+          }];
+          relabel_configs = [
+            {
+              source_labels = [ "__address__" ];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = [ "__param_target" ];
+              target_label = "instance";
+            }
+            {
+              target_label = "__address__";
+              replacement = "localhost:9115";
+            }
+          ];
+        }
+        {
+          job_name = "blackbox-http";
+          metrics_path = "/probe";
+          params = { module = [ "https_2xx" ]; };
+          static_configs = [{
+            targets = [
+              "https://fcuny.net"
+              "https://git.fcuny.net"
+            ];
+          }];
+          relabel_configs = [
+            {
+              source_labels = [ "__address__" ];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = [ "__param_target" ];
+              target_label = "instance";
+            }
+            {
+              target_label = "__address__";
+              replacement = "localhost:9115";
+            }
+          ];
+        }
+        {
+          job_name = "node";
+          static_configs =
+            [{ targets = [ "192.168.6.1:9100" "192.168.6.40:9100" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "prometheus";
+          static_configs = [{ targets = [ "192.168.6.40:9090" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "dnsmasq";
+          static_configs = [{ targets = [ "192.168.6.1:9153" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+        {
+          job_name = "unifi-poller";
+          static_configs = [{ targets = [ "192.168.6.40:9130" ]; }];
+          relabel_configs = relabelConfigs;
+        }
+      ];
+    };
+  };
+}