{ config, pkgs, lib, ... }: let cfg = config.my.services.monitoring.prometheus; blackboxConfig = { modules = { https_2xx = { prober = "http"; timeout = "5s"; http = { method = "GET"; valid_status_codes = [ ]; fail_if_not_ssl = true; no_follow_redirects = false; tls_config = { insecure_skip_verify = false; }; preferred_ip_protocol = "ip4"; }; }; icmp = { prober = "icmp"; icmp = { preferred_ip_protocol = "ip4"; }; timeout = "5s"; }; }; }; relabelConfigs = [ { source_labels = [ "__address__" ]; target_label = "instance"; replacement = "carmel"; action = "replace"; regex = "192.168.6.1:(.*)"; } { source_labels = [ "__address__" ]; target_label = "instance"; replacement = "tahoe"; action = "replace"; regex = "192.168.6.40:(.*)"; } ]; in { options.my.services.monitoring.prometheus = with lib; { enable = mkEnableOption "Prometheus monitoring solution"; listenAddress = mkOption { type = types.str; default = "0.0.0.0"; description = lib.mdDoc '' Address to listen on. ''; }; listenPort = mkOption { type = types.port; default = 9090; description = lib.mdDoc '' Port to listen on. ''; }; }; config = lib.mkIf cfg.enable { services.prometheus.exporters.blackbox = { enable = true; listenAddress = "127.0.0.1"; port = 9115; configFile = pkgs.writeText "blackbox.yml" (builtins.toJSON blackboxConfig); }; services.prometheus = { enable = true; port = cfg.listenPort; listenAddress = cfg.listenAddress; globalConfig.scrape_interval = "15s"; extraFlags = [ # 3 years of retention "--storage.tsdb.retention=${toString (365 * 3)}d" "--web.enable-admin-api" ]; scrapeConfigs = [ { job_name = "nginx"; static_configs = [{ targets = [ "127.0.0.1:${ toString config.services.prometheus.exporters.nginx.port }" ]; labels = { instance = config.networking.hostName; }; }]; } { job_name = "blackbox-ping"; metrics_path = "/probe"; params = { module = [ "icmp" ]; }; static_configs = [{ targets = [ "8.8.8.8" "1.1.1.1" "git.fcuny.net" "fcuny.net" ]; }]; relabel_configs = [ { source_labels = [ "__address__" ]; target_label = "__param_target"; } { source_labels = [ "__param_target" ]; target_label = "instance"; } { target_label = "__address__"; replacement = "localhost:9115"; } ]; } { job_name = "blackbox-http"; metrics_path = "/probe"; params = { module = [ "https_2xx" ]; }; static_configs = [{ targets = [ "https://fcuny.net" "https://git.fcuny.net" ]; }]; relabel_configs = [ { source_labels = [ "__address__" ]; target_label = "__param_target"; } { source_labels = [ "__param_target" ]; target_label = "instance"; } { target_label = "__address__"; replacement = "localhost:9115"; } ]; } { job_name = "node"; static_configs = [{ targets = [ "192.168.6.1:9100" "192.168.6.40:9100" ]; }]; relabel_configs = relabelConfigs; } { job_name = "prometheus"; static_configs = [{ targets = [ "192.168.6.40:9090" ]; }]; relabel_configs = relabelConfigs; } { job_name = "dnsmasq"; static_configs = [{ targets = [ "192.168.6.1:9153" ]; }]; relabel_configs = relabelConfigs; } { job_name = "unifi-poller"; static_configs = [{ targets = [ "192.168.6.40:9130" ]; }]; relabel_configs = relabelConfigs; } ]; }; }; }