about summary refs log tree commit diff
path: root/hosts/common/server/prometheus.nix
blob: 0e4afff8a69491b38f5975881f9c3a02817bc223 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
{ config, pkgs, lib, ... }:

let
  blackboxConfig = {
    modules = {
      https_2xx = {
        prober = "http";
        timeout = "5s";
        http = {
          method = "GET";
          valid_status_codes = [ ];
          fail_if_not_ssl = true;
          no_follow_redirects = false;
          tls_config = { insecure_skip_verify = false; };
          preferred_ip_protocol = "ip4";
        };
      };
      icmp = {
        prober = "icmp";
        icmp = { preferred_ip_protocol = "ip4"; };
        timeout = "5s";
      };
    };
  };
  relabelConfigs = [
    {
      source_labels = [ "__address__" ];
      target_label = "instance";
      replacement = "nas";
      action = "replace";
      regex = "192.168.6.10:(.*)";
    }
    {
      source_labels = [ "__address__" ];
      target_label = "instance";
      replacement = "rtr";
      action = "replace";
      regex = "192.168.6.1:(.*)";
    }
    {
      source_labels = [ "__address__" ];
      target_label = "instance";
      replacement = "tahoe";
      action = "replace";
      regex = "192.168.6.20:(.*)";
    }
  ];
in {
  services.prometheus.exporters.blackbox = {
    enable = true;
    listenAddress = "127.0.0.1";
    port = 9115;
    configFile = pkgs.writeText "blackbox.yml" (builtins.toJSON blackboxConfig);
  };

  services.prometheus = {
    enable = true;

    globalConfig.scrape_interval = "15s";

    extraFlags = [
      # 3 years of retention
      "--storage.tsdb.retention=${toString (365 * 3)}d"
      "--web.enable-admin-api"
    ];

    scrapeConfigs = [
      {
        job_name = "blackbox-ping";
        metrics_path = "/probe";
        params = { module = [ "icmp" ]; };
        static_configs =
          [{ targets = [ "8.8.8.8" "1.1.1.1" "git.fcuny.net" "fcuny.net" ]; }];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "localhost:9115";
          }
        ];
      }
      {
        job_name = "blackbox-http";
        metrics_path = "/probe";
        params = { module = [ "https_2xx" ]; };
        static_configs = [{
          targets = [
            "https://fcuny.net"
            "https://git.fcuny.net"
            "https://notes.fcuny.net"
          ];
        }];
        relabel_configs = [
          {
            source_labels = [ "__address__" ];
            target_label = "__param_target";
          }
          {
            source_labels = [ "__param_target" ];
            target_label = "instance";
          }
          {
            target_label = "__address__";
            replacement = "localhost:9115";
          }
        ];
      }
      {
        job_name = "node";
        static_configs =
          [{ targets = [ "192.168.6.1:9100" "192.168.6.20:9100" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "prometheus";
        static_configs = [{ targets = [ "192.168.6.20:9090" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "traefik";
        static_configs = [{ targets = [ "192.168.6.20:8090" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "gitea";
        static_configs = [{ targets = [ "192.168.6.20:8002" ]; }];
        relabel_configs = relabelConfigs;
      }

      {
        job_name = "dnsd";
        static_configs = [{ targets = [ "192.168.6.1:8053" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "dnsdd";
        static_configs = [{ targets = [ "192.168.6.1:9060" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "dhcpd";
        static_configs = [{ targets = [ "192.168.6.1:8067" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "netd";
        static_configs = [{ targets = [ "192.168.6.1:8055" ]; }];
        relabel_configs = relabelConfigs;
      }
      {
        job_name = "unifi-poller";
        static_configs = [{ targets = [ "192.168.6.20:9130" ]; }];
        relabel_configs = relabelConfigs;
      }
    ];
  };

  age.secrets.restic-repo-systems.file =
    ../../../secrets/restic/repo-systems.age;

  services.restic.backups = {
    prometheus = {
      paths = [ "/var/lib/prometheus2" ];
      repository = "/data/slow/backups/systems";
      passwordFile = config.age.secrets.restic-repo-systems.path;
      initialize = true;
      timerConfig = { OnCalendar = "00:25"; };
      extraBackupArgs = [ "--tag prometheus" ];
      pruneOpts = [ "--keep-daily 7" "--keep-weekly 4" ];
    };
  };
}