diff --git a/hosts/hera/configuration.nix b/hosts/hera/configuration.nix index dcf48e1d..b38edf22 100644 --- a/hosts/hera/configuration.nix +++ b/hosts/hera/configuration.nix @@ -23,6 +23,7 @@ in { ../../system/matrix-synapse.nix ../../system/coturn.nix ../../system/serve-store.nix + ../../system/go-neb.nix ./web.nix ./mail.nix ./boot.nix diff --git a/nix/sources.json b/nix/sources.json index 598c59a6..6c934697 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -59,6 +59,18 @@ "url": "https://github.com/NixOS/nixpkgs/archive/e7752db2fb6c5e05f1de2b275535dbde07d30090.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, + "nixpkgs-master": { + "branch": "master", + "description": "Nixpkgs/NixOS branches that track the Nixpkgs/NixOS channels", + "homepage": null, + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "89fda74df52b02ea0015fbc6da75330a52079192", + "sha256": "0pivssg411nzb3f82mgvlmb8wggkpxzj9f743brg61syi0vl596q", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/89fda74df52b02ea0015fbc6da75330a52079192.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "obelisk": { "branch": "master", "description": "Obelisk provides an easy way to develop and deploy your Reflex project for web and mobile", diff --git a/overlays/10-previews.nix b/overlays/10-previews.nix index 4c31538d..2d50007e 100644 --- a/overlays/10-previews.nix +++ b/overlays/10-previews.nix @@ -12,5 +12,8 @@ in preview "20.09pre-git" (unstable: { haskellPackages = super.haskellPackages // { inherit (unstable.haskellPackages) ormolu releaser; }; - inherit (unstable) neovim vimPlugins syncthing nerdfonts cabal-install; # riot-desktop; -}) + inherit (unstable) + neovim vimPlugins syncthing nerdfonts cabal-install; # riot-desktop; +}) // { + inherit (import super.sources.nixpkgs-master { }) go-neb; +} diff --git a/system/go-neb.nix b/system/go-neb.nix new file mode 100644 index 00000000..844e5df8 --- /dev/null +++ b/system/go-neb.nix @@ -0,0 +1,48 @@ +{ pkgs, config, ... }: { + imports = [ modules/go-neb.nix ]; + + services.go-neb = { + enable = true; + baseUrl = "http://localhost"; + config = { + clients = [{ + UserId = "@marabot:maralorn.de"; + AccessToken = config.m-0.private.matrix_marabot_token; + HomeServerUrl = "https://matrix.maralorn.de"; + Sync = true; + AutoJoinRooms = true; + DisplayName = "marabot"; + }]; + realms = [ ]; + sessions = [ ]; + services = [ + { + ID = "alertmanager_service"; + Type = "alertmanager"; + UserId = "@marabot:maralorn.de"; + Config = { + webhook_url = + "http://localhost:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2UK"; + rooms = { + "!negVsngnYOmXYCLKiO:maralorn.de" = { + text_template = '' + {{range .Alerts -}} [{{ .Status }}] {{index .Labels "alertname" }}: {{index .Annotations "description"}} {{ end -}}''; + html_template = '' + {{range .Alerts -}}{{ $severity := index .Labels "severity" }}{{ if eq .Status "firing" }}{{ if eq $severity "critical"}}[FIRING - CRITICAL]{{ else if eq $severity "warning"}}[FIRING - WARNING]{{ else }}[FIRING - {{ $severity }}]{{ end }}{{ else }}[RESOLVED]{{ end }}{{ index .Annotations "description"}}source ({{ index .Labels "alertname"}})
{{end -}} + ''; + msg_type = "m.text"; # Must be either `m.text` or `m.notice` + }; + }; + }; + } + { + ID = "wikipedia_service"; + Type = "wikipedia"; + UserID = "@marabot:maralorn.de"; # requires a Syncing client + Config = { }; + } + ]; + }; + }; + +} diff --git a/system/modules/go-neb.nix b/system/modules/go-neb.nix new file mode 100644 index 00000000..1de5f906 --- /dev/null +++ b/system/modules/go-neb.nix @@ -0,0 +1,53 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.go-neb; + + configFile = pkgs.writeText "config.yml" (builtins.toJSON cfg.config); +in { + options.services.go-neb = { + enable = mkEnableOption "Extensible matrix bot written in Go"; + + bindAddress = mkOption { + type = types.str; + description = "Port (and optionally address) to listen on."; + default = ":4050"; + }; + + baseUrl = mkOption { + type = types.str; + description = "Public-facing endpoint that can receive webhooks."; + }; + + config = mkOption { + type = types.attrs; + description = '' + Your config.yaml as a Nix attribute set. + See config.sample.yaml + for possible options. + ''; + }; + }; + + config = mkIf cfg.enable { + systemd.services.go-neb = { + description = "Extensible matrix bot written in Go"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Environment = [ + "BASE_URL=${cfg.baseUrl}" + "BIND_ADDRESS=${cfg.bindAddress}" + "CONFIG_FILE=${configFile}" + ]; + ExecStart = "${pkgs.go-neb}/bin/go-neb"; + DynamicUser = true; + }; + }; + }; + + meta.maintainers = with maintainers; [ hexa maralorn ]; +} diff --git a/system/monitoring/alertmanager.nix b/system/monitoring/alertmanager.nix index 4f1fcc67..926495bb 100644 --- a/system/monitoring/alertmanager.nix +++ b/system/monitoring/alertmanager.nix @@ -8,24 +8,24 @@ listenAddress = "0.0.0.0"; extraFlags = [ "--data.retention 170h" ]; configuration = { - "global" = { - "smtp_smarthost" = "hera.m-0.eu:587"; - "smtp_from" = "alertmanager@m-0.eu"; - "smtp_auth_username" = "alertmanager@m-0.eu"; - "smtp_auth_password" = config.m-0.private.alertmanager-mail-pw; + global = { + smtp_smarthost = "hera.m-0.eu:587"; + smtp_from = "alertmanager@m-0.eu"; + smtp_auth_username = "alertmanager@m-0.eu"; + smtp_auth_password = config.m-0.private.alertmanager-mail-pw; }; - "route" = { - "group_by" = [ "alertname" "alias" ]; - "group_wait" = "5m"; - "group_interval" = "30m"; - "repeat_interval" = "168h"; - "receiver" = "team-admins"; + route = { + group_by = [ "alertname" "alias" ]; + group_wait = "5m"; + group_interval = "30m"; + repeat_interval = "168h"; + receiver = "alerts"; }; - "receivers" = [{ - "name" = "team-admins"; - "email_configs" = [{ - "to" = "monitoring@maralorn.de"; - "send_resolved" = true; + receivers = [{ + name = "alerts"; + webhook_configs = [{ + url = + "${config.services.go-neb.baseUrl}:4050/services/hooks/YWxlcnRtYW5hZ2VyX3NlcnZpY2U"; }]; }]; }; diff --git a/system/monitoring/rules.yml b/system/monitoring/rules.yml index 1eb20217..060a7c50 100644 --- a/system/monitoring/rules.yml +++ b/system/monitoring/rules.yml @@ -1,29 +1,38 @@ groups: - name: rules rules: - - alert: ProbeTimeout + - alert: probe_timeout expr: probe_success == 0 for: 60m - - alert: NixpkgsBuildFail + labels: + severity: critical + annotations: + description: '{{ $labels.instance }} probe {{ $labels.job}} failed for 60m.' + - alert: nixpkgs expr: hydra_job_failed == 1 for: 2h + labels: + severity: warning + annotations: + description: '{{ $labels.packageName }} on nixpkgs branch {{ $labels.jobset }} failed.' - alert: node_down expr: (up{name!="apollo",instance!="hydra.nixos.org:443"} == 0) for: 5m + labels: + severity: critical annotations: - description: '{{ $labels.name }} has been down for more than 5 minutes.' - summary: '{{$labels.name}}: Node is down.' + description: '{{ $labels.name }} is down for 5m.' - alert: systemd_service_failed expr: node_systemd_unit_state{state="failed"} == 1 - for: 4m + for: 5m + labels: + severity: critical annotations: - description: '{{$labels.name}} failed to (re)start service {{$labels.exported_name}}.' - summary: '{{$labels.name}}: Service {{$labels.exported_name}} failed.' + description: 'Service {{$labels.exported_name}} on {{$labels.name}} failed.' - alert: systemd_service_flapping expr: changes(node_systemd_unit_state{state="failed"}[5m]) > 5 or (changes(node_systemd_unit_state{state="failed"}[1h]) > 15 unless changes(node_systemd_unit_state{state="failed"}[30m]) < 7) annotations: description: '{{$labels.name}}: Service {{$labels.exported_name}} changed its state more than 5x/5min or 15x/1h' - summary: '{{$labels.name}}: Service {{$labels.exported_name}} is flapping.' - alert: node_filesystem_full_90percent expr: sort(node_filesystem_free{device!="ramfs"} < node_filesystem_size{device!="ramfs"} * 0.1) / 1024 ^ 3 for: 5m