diff --git a/system/monitoring/alertmanager.nix b/system/monitoring/alertmanager.nix new file mode 100644 index 00000000..966cc979 --- /dev/null +++ b/system/monitoring/alertmanager.nix @@ -0,0 +1,40 @@ +{ ... }: { + services = { + nginx = { + virtualHosts."alerts.maralorn.de" = { + enableACME = true; + forceSSL = true; + locations."/" = { proxyPass = "http://localhost:9093"; }; + }; + }; + prometheus = { + alertmanager = { + enable = true; + listenAddress = "0.0.0.0"; + extraFlags = [ "--data.retention 7d" ]; + configuration = { + "global" = { + "smtp_smarthost" = "hera.m-0.eu:587"; + "smtp_from" = "alertmanager@m-0.eu"; + "smtp_auth_username" = "alertmanager@m-0.eu"; + "smtp_auth_password" = config.m-0.private.alertmanager-mail-pw; + }; + "route" = { + "group_by" = [ "alertname" "alias" ]; + "group_wait" = "5m"; + "group_interval" = "30m"; + "repeat_interval" = "7d"; + "receiver" = "team-admins"; + }; + "receivers" = [{ + "name" = "team-admins"; + "email_configs" = [{ + "to" = "monitoring@maralorn.de"; + "send_resolved" = true; + }]; + }]; + }; + }; + }; + }; +} diff --git a/system/monitoring/default.nix b/system/monitoring/default.nix index 09b2767a..46fb0ed5 100644 --- a/system/monitoring/default.nix +++ b/system/monitoring/default.nix @@ -1,160 +1 @@ -{ config, ... }: -let - inherit (config.m-0) hosts; - inherit (config.m-0.private) monitoring-guest-pw monitoring-pw; - makeProbe = module: targets: { - job_name = "blackbox-${module}"; - metrics_path = "/probe"; - params = { module = [ module ]; }; - static_configs = [{ inherit targets; }]; - relabel_configs = [ - { - source_labels = [ "__address__" ]; - target_label = "__param_target"; - } - { - source_labels = [ "__param_target" ]; - target_label = "instance"; - } - { - target_label = "__address__"; - replacement = "localhost:9115"; - } # The blackbox exporter's real hostname:port. - ]; - }; - watchNixpkgsPackage = name: { - job_name = "nixpkgs-${name}"; - metrics_path = "/job/${name}/prometheus"; - scheme = "https"; - scrape_interval = "1h"; - static_configs = [{ targets = [ "hydra.nixos.org" ]; }]; - }; - watchNixpkgsHaskellPackage = name: [ - (watchNixpkgsPackage - "nixpkgs/haskell-updates/haskellPackages.${name}.x86_64-linux") - (watchNixpkgsPackage - "nixos/release-20.03/nixpkgs.haskellPackages.${name}.x86_64-linux") - ]; -in { - services = { - nginx = { - enable = true; - virtualHosts."monitoring.maralorn.de" = { - enableACME = true; - basicAuth.maralorn = monitoring-pw; - basicAuth.guest = monitoring-guest-pw; - forceSSL = true; - locations."/" = { proxyPass = "http://localhost:9090"; }; - }; - virtualHosts."alerts.maralorn.de" = { - enableACME = true; - basicAuth.maralorn = monitoring-pw; - forceSSL = true; - locations."/" = { proxyPass = "http://localhost:9093"; }; - }; - virtualHosts."stats.maralorn.de" = { - enableACME = true; - forceSSL = true; - locations."/".proxyPass = "http://localhost:3000/"; - }; - }; - grafana = { - enable = true; - auth.anonymous.enable = true; - users = { - allowOrgCreate = false; - allowSignUp = false; - }; - provision = { - enable = true; - datasources = [{ - access = "proxy"; - name = "prometheus"; - type = "prometheus"; - url = "http://localhost:9090"; - }]; - dashboards = [{ - name = "Static dashboards"; - options.path = ./grafana-dashboards; - }]; - }; - }; - - prometheus = { - extraFlags = - [ "--query.lookback-delta=2h" "--storage.tsdb.retention.time 720d" ]; - exporters = { - blackbox = { - enable = true; - configFile = ./blackbox_rules.yml; - }; - }; - alertmanager = { - enable = true; - listenAddress = "0.0.0.0"; - configuration = { - "global" = { - "smtp_smarthost" = "hera.m-0.eu:587"; - "smtp_from" = "alertmanager@m-0.eu"; - "smtp_auth_username" = "alertmanager@m-0.eu"; - "smtp_auth_password" = config.m-0.private.alertmanager-mail-pw; - }; - "route" = { - "group_by" = [ "alertname" "alias" ]; - "group_wait" = "5m"; - "group_interval" = "30m"; - "repeat_interval" = "7d"; - "receiver" = "team-admins"; - }; - "receivers" = [{ - "name" = "team-admins"; - "email_configs" = [{ - "to" = "monitoring@maralorn.de"; - "send_resolved" = true; - }]; - }]; - }; - }; - enable = true; - ruleFiles = [ ./rules.yml ]; - scrapeConfigs = [ - (makeProbe "tcp_connect" [ "hera.m-0.eu:25" "hera.m-0.eu:80" ]) - (makeProbe "tls_connect" [ "hera.m-0.eu:993" "hera.m-0.eu:443" ]) - (makeProbe "smtp_starttls" [ "hera.m-0.eu:587" ]) - (makeProbe "http" [ "http://localhost:9090" "http://localhost:9093" ]) - (makeProbe "https" [ - "https://blog.maralorn.de" - "https://www.mathechor.de" - "https://cloud.mathechor.de" - "https://cloud.maralorn.de" - "https://riot.maralorn.de" - "https://wiki.vocalensemble-darmstadt.de" - "https://cloud.vocalensemble-darmstadt.de" - "https://www.vocalensemble-darmstadt.de" - "https://matrix.maralorn.de" - ]) - { - job_name = "matrix"; - metrics_path = "/_synapse/metrics"; - static_configs = [{ targets = [ "localhost:9148" ]; }]; - } - { - job_name = "nodes"; - static_configs = map (entry: { - targets = [ entry.host ]; - labels = { "name" = entry.name; }; - }) config.m-0.monitoring; - } - (watchNixpkgsPackage - "nixpkgs/haskell-updates/haskellPackages.cabal-fmt.x86_64-linux") - ] ++ (watchNixpkgsHaskellPackage "ghcide") - ++ (watchNixpkgsHaskellPackage "brittany") - ++ (watchNixpkgsHaskellPackage "releaser") - ++ (watchNixpkgsHaskellPackage "hlint") - ++ (watchNixpkgsHaskellPackage "relude") - ++ (watchNixpkgsHaskellPackage "taskwarrior"); - alertmanagers = - [{ static_configs = [{ targets = [ "localhost:9093" ]; }]; }]; - }; - }; -} +{ ... }: { imports = [ ./alertmanager.nix ./grafana.nix ./prometheus.nix ]; } diff --git a/system/monitoring/grafana.nix b/system/monitoring/grafana.nix new file mode 100644 index 00000000..e6e74668 --- /dev/null +++ b/system/monitoring/grafana.nix @@ -0,0 +1,32 @@ +{ ... }: { + services = { + nginx = { + virtualHosts."stats.maralorn.de" = { + enableACME = true; + forceSSL = true; + locations."/".proxyPass = "http://localhost:3000/"; + }; + }; + grafana = { + enable = true; + auth.anonymous.enable = true; + users = { + allowOrgCreate = false; + allowSignUp = false; + }; + provision = { + enable = true; + datasources = [{ + access = "proxy"; + name = "prometheus"; + type = "prometheus"; + url = "http://localhost:9090"; + }]; + dashboards = [{ + name = "Static dashboards"; + options.path = ./grafana-dashboards; + }]; + }; + }; + }; +} diff --git a/system/monitoring/prometheus.nix b/system/monitoring/prometheus.nix new file mode 100644 index 00000000..4864e3e7 --- /dev/null +++ b/system/monitoring/prometheus.nix @@ -0,0 +1,97 @@ +{ config, ... }: +let + makeProbe = module: targets: { + job_name = "blackbox-${module}"; + metrics_path = "/probe"; + params = { module = [ module ]; }; + static_configs = [{ inherit targets; }]; + relabel_configs = [ + { + source_labels = [ "__address__" ]; + target_label = "__param_target"; + } + { + source_labels = [ "__param_target" ]; + target_label = "instance"; + } + { + target_label = "__address__"; + replacement = "localhost:9115"; + } # The blackbox exporter's real hostname:port. + ]; + }; + watchNixpkgsPackage = name: { + job_name = "nixpkgs-${name}"; + metrics_path = "/job/${name}/prometheus"; + scheme = "https"; + scrape_interval = "1h"; + static_configs = [{ targets = [ "hydra.nixos.org" ]; }]; + }; + watchNixpkgsHaskellPackage = name: [ + (watchNixpkgsPackage + "nixpkgs/haskell-updates/haskellPackages.${name}.x86_64-linux") + (watchNixpkgsPackage + "nixos/release-20.03/nixpkgs.haskellPackages.${name}.x86_64-linux") + ]; +in { + services = { + nginx = { + virtualHosts."monitoring.maralorn.de" = { + enableACME = true; + forceSSL = true; + locations."/" = { proxyPass = "http://localhost:9090"; }; + }; + }; + + prometheus = { + enable = true; + extraFlags = + [ "--query.lookback-delta 2h" "--storage.tsdb.retention.time 720d" ]; + exporters = { + blackbox = { + enable = true; + configFile = ./blackbox_rules.yml; + }; + }; + ruleFiles = [ ./rules.yml ]; + scrapeConfigs = [ + (makeProbe "tcp_connect" [ "hera.m-0.eu:25" "hera.m-0.eu:80" ]) + (makeProbe "tls_connect" [ "hera.m-0.eu:993" "hera.m-0.eu:443" ]) + (makeProbe "smtp_starttls" [ "hera.m-0.eu:587" ]) + (makeProbe "http" [ "http://localhost:9090" "http://localhost:9093" ]) + (makeProbe "https" [ + "https://blog.maralorn.de" + "https://www.mathechor.de" + "https://cloud.mathechor.de" + "https://cloud.maralorn.de" + "https://riot.maralorn.de" + "https://wiki.vocalensemble-darmstadt.de" + "https://cloud.vocalensemble-darmstadt.de" + "https://www.vocalensemble-darmstadt.de" + "https://matrix.maralorn.de" + ]) + { + job_name = "matrix"; + metrics_path = "/_synapse/metrics"; + static_configs = [{ targets = [ "localhost:9148" ]; }]; + } + { + job_name = "nodes"; + static_configs = map (entry: { + targets = [ entry.host ]; + labels = { "name" = entry.name; }; + }) config.m-0.monitoring; + } + (watchNixpkgsPackage + "nixpkgs/haskell-updates/haskellPackages.cabal-fmt.x86_64-linux") + ] ++ (watchNixpkgsHaskellPackage "ghcide") + ++ (watchNixpkgsHaskellPackage "brittany") + ++ (watchNixpkgsHaskellPackage "releaser") + ++ (watchNixpkgsHaskellPackage "hlint") + ++ (watchNixpkgsHaskellPackage "relude") + ++ (watchNixpkgsHaskellPackage "taskwarrior"); + alertmanagers = + [{ static_configs = [{ targets = [ "localhost:9093" ]; }]; }]; + }; + }; +}