{ config, pkgs, lib, ... }: let cfg = config.my.services.prometheus; blackboxConfig = { modules = { https_2xx = { prober = "http"; timeout = "5s"; http = { method = "GET"; valid_status_codes = [ ]; fail_if_not_ssl = true; no_follow_redirects = false; tls_config = { insecure_skip_verify = false; }; preferred_ip_protocol = "ip4"; }; }; icmp = { prober = "icmp"; icmp = { preferred_ip_protocol = "ip4"; }; timeout = "5s"; }; }; }; relabelConfigs = [ { source_labels = [ "__address__" ]; target_label = "instance"; replacement = "nas"; action = "replace"; regex = "192.168.6.10:(.*)"; } { source_labels = [ "__address__" ]; target_label = "instance"; replacement = "rtr"; action = "replace"; regex = "192.168.6.1:(.*)"; } { source_labels = [ "__address__" ]; target_label = "instance"; replacement = "tahoe"; action = "replace"; regex = "192.168.6.20:(.*)"; } ]; in { options.my.services.prometheus = with lib; { enable = mkEnableOption "Prometheus monitoring solution"; }; config = lib.mkIf cfg.enable { services.prometheus.exporters.blackbox = { enable = true; listenAddress = "127.0.0.1"; port = 9115; configFile = pkgs.writeText "blackbox.yml" (builtins.toJSON blackboxConfig); }; services.prometheus = { enable = true; globalConfig.scrape_interval = "15s"; extraFlags = [ # 3 years of retention "--storage.tsdb.retention=${toString (365 * 3)}d" "--web.enable-admin-api" ]; scrapeConfigs = [ { job_name = "blackbox-ping"; metrics_path = "/probe"; params = { module = [ "icmp" ]; }; static_configs = [{ targets = [ "8.8.8.8" "1.1.1.1" "git.fcuny.net" "fcuny.net" ]; }]; relabel_configs = [ { source_labels = [ "__address__" ]; target_label = "__param_target"; } { source_labels = [ "__param_target" ]; target_label = "instance"; } { target_label = "__address__"; replacement = "localhost:9115"; } ]; } { job_name = "blackbox-http"; metrics_path = "/probe"; params = { module = [ "https_2xx" ]; }; static_configs = [{ targets = [ "https://fcuny.net" "https://git.fcuny.net" "https://notes.fcuny.net" ]; }]; relabel_configs = [ { source_labels = [ "__address__" ]; target_label = "__param_target"; } { source_labels = [ "__param_target" ]; target_label = "instance"; } { target_label = "__address__"; replacement = "localhost:9115"; } ]; } { job_name = "node"; static_configs = [{ targets = [ "192.168.6.1:9100" "192.168.6.20:9100" ]; }]; relabel_configs = relabelConfigs; } { job_name = "prometheus"; static_configs = [{ targets = [ "192.168.6.20:9090" ]; }]; relabel_configs = relabelConfigs; } { job_name = "traefik"; static_configs = [{ targets = [ "192.168.6.20:8090" ]; }]; relabel_configs = relabelConfigs; } { job_name = "gitea"; static_configs = [{ targets = [ "192.168.6.20:8002" ]; }]; relabel_configs = relabelConfigs; } { job_name = "dnsd"; static_configs = [{ targets = [ "192.168.6.1:8053" ]; }]; relabel_configs = relabelConfigs; } { job_name = "dnsdd"; static_configs = [{ targets = [ "192.168.6.1:9060" ]; }]; relabel_configs = relabelConfigs; } { job_name = "dhcpd"; static_configs = [{ targets = [ "192.168.6.1:8067" ]; }]; relabel_configs = relabelConfigs; } { job_name = "netd"; static_configs = [{ targets = [ "192.168.6.1:8055" ]; }]; relabel_configs = relabelConfigs; } { job_name = "unifi-poller"; static_configs = [{ targets = [ "192.168.6.20:9130" ]; }]; relabel_configs = relabelConfigs; } ]; }; age.secrets.restic-repo-systems.file = ../../../secrets/restic/repo-systems.age; services.restic.backups = { prometheus = { paths = [ "/var/lib/prometheus2" ]; repository = "/data/slow/backups/systems"; passwordFile = config.age.secrets.restic-repo-systems.path; initialize = true; timerConfig = { OnCalendar = "00:25"; }; extraBackupArgs = [ "--tag prometheus" ]; pruneOpts = [ "--keep-daily 7" "--keep-weekly 4" ]; }; }; }; }