commit ff508dca2b5b90486c1fcb682c1d14abfbd20414 parent 007ff91c0f08fc2676be0d4c5a79ac0b843373ec Author: mtmn <miro@haravara.org> Date: Wed, 20 May 2026 18:45:57 +0200 feat: split monitoring.nix module Diffstat:
| M | flake.nix | | | 7 | +++++-- |
| M | hosts/nixaran/configuration.nix | | | 0 | |
| M | hosts/void/overlays/config/shell/rc/void | | | 0 | |
| D | modules/services/monitoring.nix | | | 435 | ------------------------------------------------------------------------------- |
| A | modules/services/monitoring/alertmanager.nix | | | 55 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | modules/services/monitoring/default.nix | | | 133 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | modules/services/monitoring/exporters.nix | | | 55 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | modules/services/monitoring/grafana.nix | | | 124 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | modules/services/monitoring/loki.nix | | | 88 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | modules/services/monitoring/vmalert.nix | | | 50 | ++++++++++++++++++++++++++++++++++++++++++++++++++ |
10 files changed, 510 insertions(+), 437 deletions(-)
diff --git a/flake.nix b/flake.nix @@ -138,8 +138,11 @@ lib.nameValuePair (lib.removeSuffix ".nix" filename) (import ./modules/services/${filename}) - ) (lib.filterAttrs (n: t: t == "regular" && lib.hasSuffix ".nix" n) - (builtins.readDir ./modules/services)) + ) (lib.filterAttrs ( + n: t: + (t == "regular" && lib.hasSuffix ".nix" n) + || (t == "directory" && builtins.pathExists ./modules/services/${n}/default.nix) + ) (builtins.readDir ./modules/services)) // { base = import ./modules/nixos/base.nix; haravara = import ./lib/haravara.nix; diff --git a/hosts/nixaran/configuration.nix b/hosts/nixaran/configuration.nix Binary files differ. diff --git a/hosts/void/overlays/config/shell/rc/void b/hosts/void/overlays/config/shell/rc/void Binary files differ. diff --git a/modules/services/monitoring.nix b/modules/services/monitoring.nix @@ -1,435 +0,0 @@ -{ - config, - lib, - pkgs, - haravara, - ... -}: let - cfg = config.services.haravara.monitoring; - - authSnippet = haravara.mkAuthImport cfg.authLabel; - - fetchDashboard = { - name, - sha256, - url, - }: { - inherit name; - path = pkgs.runCommand name {} '' - sed 's/''${DS_PROMETHEUS}/VictoriaMetrics/g; s/''${DS_LOKI}/Loki/g' ${pkgs.fetchurl {inherit url sha256;}} > $out - ''; - }; - - dashboards = pkgs.linkFarm "grafana-dashboards" (map fetchDashboard [ - { - name = "node_exporter.json"; - url = "https://grafana.com/api/dashboards/1860/revisions/42/download"; - sha256 = "a4d827eb1819044bba2d6d257347175f6811910f2583fadeaf7e123c4df2125e"; - } - { - name = "systemd_exporter.json"; - url = "https://grafana.com/api/dashboards/22872/revisions/1/download"; - sha256 = "665bc3e86b79749b2fdae7787f4b7502e00830898bf48f73c68134471f72bf33"; - } - { - name = "alertmanager.json"; - url = "https://grafana.com/api/dashboards/9578/revisions/4/download"; - sha256 = "16jvazvjswiyn281z4r7wmwb18841d3ar24951538kma28l05izy"; - } - { - name = "seaweedfs.json"; - url = "https://raw.githubusercontent.com/chrislusf/seaweedfs/c9100a7213e27963f58a0ba783c9598a177384b6/other/metrics/grafana_seaweedfs.json"; - sha256 = "e364d113fed54127cbda10c4c2809d99a90dcc38bcd949b91846d0831e1a68e4"; - } - ] - ++ [ - { - name = "caddy.json"; - path = let - src = pkgs.fetchFromGitHub { - owner = "grafana"; - repo = "jsonnet-libs"; - rev = "2e9c7f4a276f6f6a9bc3f994f2b8804555f31bb9"; - sha256 = "12ikl9p9s6sv1qy5377dk95naz7ckwd767wgq4503xbaq0nq3k11"; - }; - in - pkgs.runCommand "caddy.json" { - nativeBuildInputs = [pkgs.go-jsonnet]; - } '' - jsonnet -J ${src} ${src}/caddy-mixin/dashboards/caddy-overview.jsonnet > $out - ''; - } - ] - ++ lib.optional config.services.haravara.corpus.enable (fetchDashboard { - name = "corpus.json"; - url = "https://raw.githubusercontent.com/mtmn/corpus/master/monitoring/grafana-dashboard.json"; - sha256 = "0b6d4kvcwarv1isy3i4x0l4an7b3zyxyy9af3acr9gbfkkrbyjv5"; - })); -in { - options.services.haravara.monitoring = { - enable = lib.mkEnableOption "Monitoring stack (VictoriaMetrics + Grafana)"; - grafanaDomain = lib.mkOption { - type = lib.types.str; - }; - victoriaDomain = lib.mkOption { - type = lib.types.str; - }; - authLabel = haravara.authLabelOption; - grafanaSecretKeyFile = lib.mkOption { - type = lib.types.path; - description = "Path to Grafana secret key file"; - }; - - alertmanager = { - enable = lib.mkEnableOption "Alertmanager for alert routing and notifications"; - domain = lib.mkOption { - type = lib.types.nullOr lib.types.str; - default = null; - description = "Domain to expose the Alertmanager UI via Caddy (optional)"; - }; - port = lib.mkOption { - type = lib.types.port; - default = 9093; - }; - configuration = lib.mkOption { - type = lib.types.attrs; - description = "Alertmanager configuration (route, receivers, etc.)"; - example = lib.literalExpression '' - { - route = { - receiver = "default"; - group_wait = "30s"; - group_interval = "5m"; - repeat_interval = "4h"; - }; - receivers = [{ - name = "default"; - webhook_configs = [{ url = "https://example.com/hook"; }]; - }]; - } - ''; - }; - }; - - vmalert = { - enable = lib.mkEnableOption "vmalert for Prometheus-compatible alert rule evaluation"; - port = lib.mkOption { - type = lib.types.port; - default = 8880; - }; - rules = lib.mkOption { - type = lib.types.listOf lib.types.path; - default = []; - description = "Paths to YAML files containing alert/recording rules"; - }; - }; - }; - - config = lib.mkIf cfg.enable { - users.groups.victoriametrics = {}; - users.users.victoriametrics = { - isSystemUser = true; - group = "victoriametrics"; - extraGroups = ["storage-users"]; - }; - - systemd = { - services = { - vmalert = lib.mkIf cfg.vmalert.enable { - description = "VictoriaMetrics alert rule evaluator"; - wantedBy = ["multi-user.target"]; - after = ["network.target" "victoriametrics.service"]; - serviceConfig = { - ExecStart = lib.concatStringsSep " " ( - let - baseArgs = [ - "${pkgs.victoriametrics}/bin/vmalert" - "-datasource.url=http://127.0.0.1:8428" - "-remoteWrite.url=http://127.0.0.1:8428" - "-httpListenAddr=127.0.0.1:${toString cfg.vmalert.port}" - "-evaluationInterval=30s" - ]; - alertArgs = lib.optionals cfg.alertmanager.enable [ - "-notifier.url=http://127.0.0.1:${toString cfg.alertmanager.port}" - ]; - ruleArgs = map (r: "-rule=${r}") cfg.vmalert.rules; - in - baseArgs ++ alertArgs ++ ruleArgs - ); - DynamicUser = true; - Restart = "on-failure"; - RestartSec = "5s"; - }; - }; - - victoriametrics = { - serviceConfig = { - User = "victoriametrics"; - Group = "victoriametrics"; - StateDirectory = lib.mkForce ""; - DynamicUser = lib.mkForce false; - ReadWritePaths = ["/var/lib/victoriametrics"]; - }; - }; - - prometheus-node-exporter.serviceConfig = { - # Relaxing sandboxing and running as root as it fails to collect metrics reliably - # due to DBus connection resets and pending reply limits. - RestrictNamespaces = lib.mkForce false; - RestrictAddressFamilies = lib.mkForce ["AF_UNIX" "AF_INET" "AF_INET6" "AF_NETLINK"]; - DynamicUser = lib.mkForce false; - User = lib.mkForce "root"; - Group = lib.mkForce "root"; - }; - - prometheus-systemd-exporter.serviceConfig = { - # Relaxing sandboxing and running as root as it fails to collect metrics reliably - # due to DBus connection resets and pending reply limits. - RestrictNamespaces = lib.mkForce false; - RestrictAddressFamilies = lib.mkForce ["AF_UNIX" "AF_INET" "AF_INET6" "AF_NETLINK"]; - DynamicUser = lib.mkForce false; - User = lib.mkForce "root"; - Group = lib.mkForce "root"; - }; - }; - }; - services = { - victoriametrics = { - enable = true; - listenAddress = "127.0.0.1:8428"; - extraOptions = - lib.optional cfg.vmalert.enable - "-vmalert.proxyURL=http://127.0.0.1:${toString cfg.vmalert.port}"; - prometheusConfig = { - global.scrape_interval = "30s"; - scrape_configs = - [ - { - job_name = "bobor_status"; - static_configs = [{targets = ["https://status.saunabobor.sk"];}]; - metrics_path = "/metrics"; - } - { - job_name = "node_exporter"; - static_configs = [{targets = ["127.0.0.1:${toString config.services.prometheus.exporters.node.port}"];}]; - } - { - job_name = "systemd_exporter"; - static_configs = [{targets = ["127.0.0.1:${toString config.services.prometheus.exporters.systemd.port}"];}]; - metrics_path = "/metrics"; - } - { - job_name = "cadvisor"; - static_configs = [{targets = ["127.0.0.1:${toString config.services.cadvisor.port}"];}]; - metrics_path = "/metrics"; - } - { - job_name = "seaweedfs"; - static_configs = [ - { - targets = [ - "127.0.0.1:9324" - "127.0.0.1:9325" - "127.0.0.1:9326" - "127.0.0.1:9327" - ]; - } - ]; - } - { - job_name = "caddy"; - static_configs = [{targets = ["127.0.0.1:2019"];}]; - } - { - job_name = "loki"; - static_configs = [{targets = ["127.0.0.1:3100"];}]; - } - ] - ++ lib.optional config.services.haravara.corpus.enable { - job_name = "corpus"; - static_configs = [{targets = ["127.0.0.1:${toString config.services.haravara.corpus.port}"];}]; - metrics_path = "/metrics"; - } - ++ lib.optional cfg.vmalert.enable { - job_name = "vmalert"; - static_configs = [{targets = ["127.0.0.1:${toString cfg.vmalert.port}"];}]; - } - ++ lib.optional cfg.alertmanager.enable { - job_name = "alertmanager"; - static_configs = [{targets = ["127.0.0.1:${toString cfg.alertmanager.port}"];}]; - }; - }; - }; - - prometheus = { - exporters.node = { - enable = true; - listenAddress = "127.0.0.1"; - enabledCollectors = ["systemd"]; - port = 9100; - }; - - exporters.systemd = { - enable = true; - listenAddress = "127.0.0.1"; - port = 9558; - extraFlags = [ - "--systemd.collector.unit-exclude=.*\\\\.(mount|slice|scope|target|device|path|timer|automount|socket|swap)" - ]; - }; - - alertmanager = lib.mkIf cfg.alertmanager.enable { - enable = true; - listenAddress = "127.0.0.1"; - inherit (cfg.alertmanager) port configuration; - }; - }; - - cadvisor = { - enable = true; - listenAddress = "127.0.0.1"; - port = 8182; - }; - - loki = { - enable = true; - configuration = { - server = { - http_listen_address = "127.0.0.1"; - http_listen_port = 3100; - grpc_listen_address = "127.0.0.1"; - }; - auth_enabled = false; - - common = { - instance_addr = "127.0.0.1"; - path_prefix = "/var/lib/loki"; - replication_factor = 1; - ring.kvstore.store = "inmemory"; - }; - - schema_config.configs = [ - { - from = "2020-10-24"; - store = "tsdb"; - object_store = "filesystem"; - schema = "v13"; - index.prefix = "index_"; - index.period = "24h"; - } - ]; - - storage_config.filesystem.directory = "/var/lib/loki/chunks"; - }; - }; - - alloy = { - enable = true; - configPath = pkgs.writeText "alloy-config.river" '' - // ── Logs ──────────────────────────────────────────────────────────── - - loki.write "default" { - endpoint { - url = "http://127.0.0.1:3100/loki/api/v1/push" - } - } - - loki.relabel "journal" { - forward_to = [] - - rule { - source_labels = ["__journal__systemd_unit"] - target_label = "unit" - } - - rule { - source_labels = ["__journal__systemd_unit"] - target_label = "unit_file" - } - - rule { - source_labels = ["__journal__systemd_unit"] - target_label = "service_name" - } - } - - loki.source.journal "systemd_logs" { - max_age = "12h" - forward_to = [loki.write.default.receiver] - relabel_rules = loki.relabel.journal.rules - labels = { - job = "systemd-journal", - host = "${config.networking.hostName}", - } - } - ''; - }; - - grafana = { - enable = true; - settings = { - server = { - http_addr = "127.0.0.1"; - http_port = 3001; - }; - security = { - secret_key = "$__file{${cfg.grafanaSecretKeyFile}}"; - }; - }; - provision = { - enable = true; - datasources.settings = { - apiVersion = 1; - datasources = - [ - { - name = "VictoriaMetrics"; - type = "prometheus"; - url = "http://127.0.0.1:8428"; - uid = "DS_PROMETHEUS"; - isDefault = true; - } - { - name = "Loki"; - type = "loki"; - url = "http://127.0.0.1:3100"; - } - ] - ++ lib.optional cfg.alertmanager.enable { - name = "Alertmanager"; - type = "alertmanager"; - url = "http://127.0.0.1:${toString cfg.alertmanager.port}"; - jsonData = { - implementation = "prometheus"; - }; - }; - }; - dashboards.settings.providers = [ - { - name = "Default"; - options.path = dashboards; - } - ]; - }; - }; - - caddy.virtualHosts = - { - "${cfg.grafanaDomain}".extraConfig = '' - ${authSnippet} - reverse_proxy 127.0.0.1:3001 - ''; - "${cfg.victoriaDomain}".extraConfig = '' - ${authSnippet} - reverse_proxy 127.0.0.1:8428 - ''; - } - // lib.optionalAttrs (cfg.alertmanager.enable && cfg.alertmanager.domain != null) { - "${cfg.alertmanager.domain}".extraConfig = '' - ${authSnippet} - reverse_proxy 127.0.0.1:${toString cfg.alertmanager.port} - ''; - }; - }; - }; -} diff --git a/modules/services/monitoring/alertmanager.nix b/modules/services/monitoring/alertmanager.nix @@ -0,0 +1,55 @@ +{ + config, + lib, + haravara, + ... +}: let + cfg = config.services.haravara.monitoring; + authSnippet = haravara.mkAuthImport cfg.authLabel; +in { + options.services.haravara.monitoring.alertmanager = { + enable = lib.mkEnableOption "Alertmanager for alert routing and notifications"; + domain = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Domain to expose the Alertmanager UI via Caddy (optional)"; + }; + port = lib.mkOption { + type = lib.types.port; + default = 9093; + }; + configuration = lib.mkOption { + type = lib.types.attrs; + description = "Alertmanager configuration (route, receivers, etc.)"; + example = lib.literalExpression '' + { + route = { + receiver = "default"; + group_wait = "30s"; + group_interval = "5m"; + repeat_interval = "4h"; + }; + receivers = [{ + name = "default"; + webhook_configs = [{ url = "https://example.com/hook"; }]; + }]; + } + ''; + }; + }; + + config = lib.mkIf (cfg.enable && cfg.alertmanager.enable) { + services.prometheus.alertmanager = { + enable = true; + listenAddress = "127.0.0.1"; + inherit (cfg.alertmanager) port configuration; + }; + + services.caddy.virtualHosts = lib.optionalAttrs (cfg.alertmanager.domain != null) { + "${cfg.alertmanager.domain}".extraConfig = '' + ${authSnippet} + reverse_proxy 127.0.0.1:${toString cfg.alertmanager.port} + ''; + }; + }; +} diff --git a/modules/services/monitoring/default.nix b/modules/services/monitoring/default.nix @@ -0,0 +1,133 @@ +{ + config, + lib, + haravara, + ... +}: let + cfg = config.services.haravara.monitoring; + authSnippet = haravara.mkAuthImport cfg.authLabel; +in { + imports = [ + ./alertmanager.nix + ./exporters.nix + ./grafana.nix + ./loki.nix + ./vmalert.nix + ]; + + options.services.haravara.monitoring = { + enable = lib.mkEnableOption "Monitoring stack (VictoriaMetrics + Grafana)"; + + grafanaDomain = lib.mkOption { + type = lib.types.str; + }; + victoriaDomain = lib.mkOption { + type = lib.types.str; + }; + authLabel = haravara.authLabelOption; + grafanaSecretKeyFile = lib.mkOption { + type = lib.types.path; + description = "Path to Grafana secret key file"; + }; + + victoriametrics = { + enable = lib.mkEnableOption "VictoriaMetrics time-series database" // {default = true;}; + extraScrapeConfigs = lib.mkOption { + type = lib.types.listOf lib.types.attrs; + default = []; + description = "Extra Prometheus scrape configs for VictoriaMetrics"; + }; + }; + }; + + config = lib.mkIf cfg.enable { + users.groups.victoriametrics = lib.mkIf cfg.victoriametrics.enable {}; + users.users.victoriametrics = lib.mkIf cfg.victoriametrics.enable { + isSystemUser = true; + group = "victoriametrics"; + extraGroups = ["storage-users"]; + }; + + systemd.services.victoriametrics = lib.mkIf cfg.victoriametrics.enable { + serviceConfig = { + User = "victoriametrics"; + Group = "victoriametrics"; + StateDirectory = lib.mkForce ""; + DynamicUser = lib.mkForce false; + ReadWritePaths = ["/var/lib/victoriametrics"]; + }; + }; + + services.victoriametrics = lib.mkIf cfg.victoriametrics.enable { + enable = true; + listenAddress = "127.0.0.1:8428"; + extraOptions = + lib.optional cfg.vmalert.enable + "-vmalert.proxyURL=http://127.0.0.1:${toString cfg.vmalert.port}"; + prometheusConfig = { + global.scrape_interval = "30s"; + scrape_configs = + (lib.optionals cfg.exporters.enable [ + { + job_name = "node_exporter"; + static_configs = [{targets = ["127.0.0.1:${toString config.services.prometheus.exporters.node.port}"];}]; + } + { + job_name = "systemd_exporter"; + static_configs = [{targets = ["127.0.0.1:${toString config.services.prometheus.exporters.systemd.port}"];}]; + metrics_path = "/metrics"; + } + { + job_name = "cadvisor"; + static_configs = [{targets = ["127.0.0.1:${toString config.services.cadvisor.port}"];}]; + metrics_path = "/metrics"; + } + ]) + ++ [ + { + job_name = "seaweedfs"; + static_configs = [ + { + targets = [ + "127.0.0.1:9324" + "127.0.0.1:9325" + "127.0.0.1:9326" + "127.0.0.1:9327" + ]; + } + ]; + } + { + job_name = "caddy"; + static_configs = [{targets = ["127.0.0.1:2019"];}]; + } + ] + ++ lib.optional cfg.loki.enable { + job_name = "loki"; + static_configs = [{targets = ["127.0.0.1:3100"];}]; + } + ++ lib.optional config.services.haravara.corpus.enable { + job_name = "corpus"; + static_configs = [{targets = ["127.0.0.1:${toString config.services.haravara.corpus.port}"];}]; + metrics_path = "/metrics"; + } + ++ lib.optional cfg.vmalert.enable { + job_name = "vmalert"; + static_configs = [{targets = ["127.0.0.1:${toString cfg.vmalert.port}"];}]; + } + ++ lib.optional cfg.alertmanager.enable { + job_name = "alertmanager"; + static_configs = [{targets = ["127.0.0.1:${toString cfg.alertmanager.port}"];}]; + } + ++ cfg.victoriametrics.extraScrapeConfigs; + }; + }; + + services.caddy.virtualHosts = lib.optionalAttrs cfg.victoriametrics.enable { + "${cfg.victoriaDomain}".extraConfig = '' + ${authSnippet} + reverse_proxy 127.0.0.1:8428 + ''; + }; + }; +} diff --git a/modules/services/monitoring/exporters.nix b/modules/services/monitoring/exporters.nix @@ -0,0 +1,55 @@ +{ + config, + lib, + ... +}: let + cfg = config.services.haravara.monitoring; +in { + options.services.haravara.monitoring.exporters = { + enable = lib.mkEnableOption "Prometheus node/systemd exporters and cAdvisor" // {default = true;}; + }; + + config = lib.mkIf (cfg.enable && cfg.exporters.enable) { + services.prometheus = { + exporters.node = { + enable = true; + listenAddress = "127.0.0.1"; + enabledCollectors = ["systemd"]; + port = 9100; + }; + + exporters.systemd = { + enable = true; + listenAddress = "127.0.0.1"; + port = 9558; + extraFlags = [ + "--systemd.collector.unit-exclude=.*\\\\.(mount|slice|scope|target|device|path|timer|automount|socket|swap)" + ]; + }; + }; + + services.cadvisor = { + enable = true; + listenAddress = "127.0.0.1"; + port = 8182; + }; + + systemd.services = { + prometheus-node-exporter.serviceConfig = { + RestrictNamespaces = lib.mkForce false; + RestrictAddressFamilies = lib.mkForce ["AF_UNIX" "AF_INET" "AF_INET6" "AF_NETLINK"]; + DynamicUser = lib.mkForce false; + User = lib.mkForce "root"; + Group = lib.mkForce "root"; + }; + + prometheus-systemd-exporter.serviceConfig = { + RestrictNamespaces = lib.mkForce false; + RestrictAddressFamilies = lib.mkForce ["AF_UNIX" "AF_INET" "AF_INET6" "AF_NETLINK"]; + DynamicUser = lib.mkForce false; + User = lib.mkForce "root"; + Group = lib.mkForce "root"; + }; + }; + }; +} diff --git a/modules/services/monitoring/grafana.nix b/modules/services/monitoring/grafana.nix @@ -0,0 +1,124 @@ +{ + config, + lib, + pkgs, + haravara, + ... +}: let + cfg = config.services.haravara.monitoring; + authSnippet = haravara.mkAuthImport cfg.authLabel; + + fetchDashboard = { + name, + sha256, + url, + }: { + inherit name; + path = pkgs.runCommand name {} '' + sed 's/''${DS_PROMETHEUS}/VictoriaMetrics/g; s/''${DS_LOKI}/Loki/g' ${pkgs.fetchurl {inherit url sha256;}} > $out + ''; + }; + + dashboards = pkgs.linkFarm "grafana-dashboards" (map fetchDashboard [ + { + name = "node_exporter.json"; + url = "https://grafana.com/api/dashboards/1860/revisions/42/download"; + sha256 = "a4d827eb1819044bba2d6d257347175f6811910f2583fadeaf7e123c4df2125e"; + } + { + name = "systemd_exporter.json"; + url = "https://grafana.com/api/dashboards/22872/revisions/1/download"; + sha256 = "665bc3e86b79749b2fdae7787f4b7502e00830898bf48f73c68134471f72bf33"; + } + { + name = "alertmanager.json"; + url = "https://grafana.com/api/dashboards/9578/revisions/4/download"; + sha256 = "16jvazvjswiyn281z4r7wmwb18841d3ar24951538kma28l05izy"; + } + { + name = "seaweedfs.json"; + url = "https://raw.githubusercontent.com/chrislusf/seaweedfs/c9100a7213e27963f58a0ba783c9598a177384b6/other/metrics/grafana_seaweedfs.json"; + sha256 = "e364d113fed54127cbda10c4c2809d99a90dcc38bcd949b91846d0831e1a68e4"; + } + ] + ++ [ + { + name = "caddy.json"; + path = let + src = pkgs.fetchFromGitHub { + owner = "grafana"; + repo = "jsonnet-libs"; + rev = "2e9c7f4a276f6f6a9bc3f994f2b8804555f31bb9"; + sha256 = "12ikl9p9s6sv1qy5377dk95naz7ckwd767wgq4503xbaq0nq3k11"; + }; + in + pkgs.runCommand "caddy.json" { + nativeBuildInputs = [pkgs.go-jsonnet]; + } '' + jsonnet -J ${src} ${src}/caddy-mixin/dashboards/caddy-overview.jsonnet > $out + ''; + } + ] + ++ lib.optional config.services.haravara.corpus.enable (fetchDashboard { + name = "corpus.json"; + url = "https://raw.githubusercontent.com/mtmn/corpus/master/monitoring/grafana-dashboard.json"; + sha256 = "0b6d4kvcwarv1isy3i4x0l4an7b3zyxyy9af3acr9gbfkkrbyjv5"; + })); +in { + options.services.haravara.monitoring.grafana = { + enable = lib.mkEnableOption "Grafana dashboard and visualization" // {default = true;}; + }; + + config = lib.mkIf (cfg.enable && cfg.grafana.enable) { + services.grafana = { + enable = true; + settings = { + server = { + http_addr = "127.0.0.1"; + http_port = 3001; + }; + security = { + secret_key = "$__file{${cfg.grafanaSecretKeyFile}}"; + }; + }; + provision = { + enable = true; + datasources.settings = { + apiVersion = 1; + datasources = + (lib.optional cfg.victoriametrics.enable { + name = "VictoriaMetrics"; + type = "prometheus"; + url = "http://127.0.0.1:8428"; + uid = "DS_PROMETHEUS"; + isDefault = true; + }) + ++ (lib.optional cfg.loki.enable { + name = "Loki"; + type = "loki"; + url = "http://127.0.0.1:3100"; + }) + ++ (lib.optional cfg.alertmanager.enable { + name = "Alertmanager"; + type = "alertmanager"; + url = "http://127.0.0.1:${toString cfg.alertmanager.port}"; + jsonData = { + implementation = "prometheus"; + }; + }); + }; + dashboards.settings.providers = [ + { + name = "Default"; + options.path = dashboards; + } + ]; + }; + }; + + services.caddy.virtualHosts."${cfg.grafanaDomain}".extraConfig = '' + ${authSnippet} + reverse_proxy 127.0.0.1:3001 + ''; + }; +} diff --git a/modules/services/monitoring/loki.nix b/modules/services/monitoring/loki.nix @@ -0,0 +1,88 @@ +{ + config, + lib, + pkgs, + ... +}: let + cfg = config.services.haravara.monitoring; +in { + options.services.haravara.monitoring.loki = { + enable = lib.mkEnableOption "Loki log aggregation and Alloy collector" // {default = true;}; + }; + + config = lib.mkIf (cfg.enable && cfg.loki.enable) { + services.loki = { + enable = true; + configuration = { + server = { + http_listen_address = "127.0.0.1"; + http_listen_port = 3100; + grpc_listen_address = "127.0.0.1"; + }; + auth_enabled = false; + + common = { + instance_addr = "127.0.0.1"; + path_prefix = "/var/lib/loki"; + replication_factor = 1; + ring.kvstore.store = "inmemory"; + }; + + schema_config.configs = [ + { + from = "2020-10-24"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index.prefix = "index_"; + index.period = "24h"; + } + ]; + + storage_config.filesystem.directory = "/var/lib/loki/chunks"; + }; + }; + + services.alloy = { + enable = true; + configPath = pkgs.writeText "alloy-config.river" '' + // ── Logs ──────────────────────────────────────────────────────────── + + loki.write "default" { + endpoint { + url = "http://127.0.0.1:3100/loki/api/v1/push" + } + } + + loki.relabel "journal" { + forward_to = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit_file" + } + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "service_name" + } + } + + loki.source.journal "systemd_logs" { + max_age = "12h" + forward_to = [loki.write.default.receiver] + relabel_rules = loki.relabel.journal.rules + labels = { + job = "systemd-journal", + host = "${config.networking.hostName}", + } + } + ''; + }; + }; +} diff --git a/modules/services/monitoring/vmalert.nix b/modules/services/monitoring/vmalert.nix @@ -0,0 +1,50 @@ +{ + config, + lib, + pkgs, + ... +}: let + cfg = config.services.haravara.monitoring; +in { + options.services.haravara.monitoring.vmalert = { + enable = lib.mkEnableOption "vmalert for Prometheus-compatible alert rule evaluation"; + port = lib.mkOption { + type = lib.types.port; + default = 8880; + }; + rules = lib.mkOption { + type = lib.types.listOf lib.types.path; + default = []; + description = "Paths to YAML files containing alert/recording rules"; + }; + }; + + config = lib.mkIf (cfg.enable && cfg.vmalert.enable) { + systemd.services.vmalert = { + description = "VictoriaMetrics alert rule evaluator"; + wantedBy = ["multi-user.target"]; + after = ["network.target" "victoriametrics.service"]; + serviceConfig = { + ExecStart = lib.concatStringsSep " " ( + let + baseArgs = [ + "${pkgs.victoriametrics}/bin/vmalert" + "-datasource.url=http://127.0.0.1:8428" + "-remoteWrite.url=http://127.0.0.1:8428" + "-httpListenAddr=127.0.0.1:${toString cfg.vmalert.port}" + "-evaluationInterval=30s" + ]; + alertArgs = lib.optionals cfg.alertmanager.enable [ + "-notifier.url=http://127.0.0.1:${toString cfg.alertmanager.port}" + ]; + ruleArgs = map (r: "-rule=${r}") cfg.vmalert.rules; + in + baseArgs ++ alertArgs ++ ruleArgs + ); + DynamicUser = true; + Restart = "on-failure"; + RestartSec = "5s"; + }; + }; + }; +}