commit 45d6b151b62c9698be48667c29666f2401db0130
parent f5cb5e7a59f1b1686beb45b9ed0f8f7c78f11b9c
Author: mtmn <miro@haravara.org>
Date: Tue, 21 Apr 2026 01:12:55 +0200
feat: implement alertmanager/vmalert
Diffstat:
6 files changed, 293 insertions(+), 553 deletions(-)
diff --git a/hosts/nixaran/configuration.nix b/hosts/nixaran/configuration.nix
Binary files differ.
diff --git a/modules/mixins/corpus/grafana-dashboard.json b/modules/mixins/corpus/grafana-dashboard.json
@@ -1,519 +0,0 @@
-{
- "title": "Corpus",
- "uid": "corpus",
- "version": 1,
- "schemaVersion": 38,
- "timezone": "browser",
- "refresh": "30s",
- "time": { "from": "now-24h", "to": "now" },
- "templating": { "list": [] },
- "annotations": { "list": [] },
- "panels": [
- {
- "id": 1,
- "type": "row",
- "title": "HTTP",
- "gridPos": { "x": 0, "y": 0, "w": 24, "h": 1 },
- "collapsed": false
- },
- {
- "id": 2,
- "type": "timeseries",
- "title": "Request Rate",
- "gridPos": { "x": 0, "y": 1, "w": 12, "h": 8 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "sum by (method, path, status) (rate(corpus_http_requests_total[5m]))",
- "legendFormat": "{{method}} {{path}} {{status}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "reqps",
- "custom": { "lineWidth": 1, "fillOpacity": 10 }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "none" },
- "legend": { "displayMode": "list", "placement": "bottom" }
- }
- },
- {
- "id": 3,
- "type": "timeseries",
- "title": "Request Duration (p50 / p90 / p99)",
- "gridPos": { "x": 12, "y": 1, "w": 12, "h": 8 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "histogram_quantile(0.50, sum by (le, path) (rate(corpus_http_request_duration_seconds_bucket[5m])))",
- "legendFormat": "p50 {{path}}",
- "refId": "A"
- },
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "histogram_quantile(0.90, sum by (le, path) (rate(corpus_http_request_duration_seconds_bucket[5m])))",
- "legendFormat": "p90 {{path}}",
- "refId": "B"
- },
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "histogram_quantile(0.99, sum by (le, path) (rate(corpus_http_request_duration_seconds_bucket[5m])))",
- "legendFormat": "p99 {{path}}",
- "refId": "C"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "s",
- "custom": { "lineWidth": 1, "fillOpacity": 10 }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "none" },
- "legend": { "displayMode": "list", "placement": "bottom" }
- }
- },
- {
- "id": 10,
- "type": "row",
- "title": "Sync",
- "gridPos": { "x": 0, "y": 9, "w": 24, "h": 1 },
- "collapsed": false
- },
- {
- "id": 11,
- "type": "timeseries",
- "title": "Sync Runs (per hour)",
- "gridPos": { "x": 0, "y": 10, "w": 8, "h": 8 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "increase(corpus_sync_runs_total[1h])",
- "legendFormat": "{{user}} {{source}} {{result}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "short",
- "custom": { "lineWidth": 1, "fillOpacity": 10 }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "none" },
- "legend": { "displayMode": "list", "placement": "bottom" }
- }
- },
- {
- "id": 12,
- "type": "timeseries",
- "title": "Scrobbles Added (per hour)",
- "gridPos": { "x": 8, "y": 10, "w": 8, "h": 8 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "increase(corpus_sync_scrobbles_added_total[1h])",
- "legendFormat": "{{user}} {{source}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "short",
- "custom": { "lineWidth": 1, "fillOpacity": 10 }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "none" },
- "legend": { "displayMode": "list", "placement": "bottom" }
- }
- },
- {
- "id": 13,
- "type": "stat",
- "title": "Time Since Last Sync",
- "gridPos": { "x": 16, "y": 10, "w": 4, "h": 4 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "time() - corpus_sync_last_success_seconds",
- "legendFormat": "{{user}} {{source}}",
- "refId": "A",
- "instant": true
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "dtdurations",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null },
- { "color": "yellow", "value": 3600 },
- { "color": "red", "value": 86400 }
- ]
- },
- "color": { "mode": "thresholds" }
- },
- "overrides": []
- },
- "options": {
- "reduceOptions": { "calcs": ["lastNotNull"] },
- "orientation": "auto",
- "textMode": "auto",
- "colorMode": "value",
- "graphMode": "none"
- }
- },
- {
- "id": 14,
- "type": "stat",
- "title": "Time Since Last Backup",
- "gridPos": { "x": 20, "y": 10, "w": 4, "h": 4 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "time() - corpus_db_backup_last_success_seconds",
- "legendFormat": "{{user}}",
- "refId": "A",
- "instant": true
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "dtdurations",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null },
- { "color": "yellow", "value": 86400 },
- { "color": "red", "value": 172800 }
- ]
- },
- "color": { "mode": "thresholds" }
- },
- "overrides": []
- },
- "options": {
- "reduceOptions": { "calcs": ["lastNotNull"] },
- "orientation": "auto",
- "textMode": "auto",
- "colorMode": "value",
- "graphMode": "none"
- }
- },
- {
- "id": 15,
- "type": "stat",
- "title": "Total Backup Runs",
- "gridPos": { "x": 16, "y": 14, "w": 4, "h": 4 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "corpus_db_backup_runs_total",
- "legendFormat": "{{user}}",
- "refId": "A",
- "instant": true
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "short",
- "color": { "mode": "fixed", "fixedColor": "blue" }
- },
- "overrides": []
- },
- "options": {
- "reduceOptions": { "calcs": ["lastNotNull"] },
- "orientation": "auto",
- "textMode": "auto",
- "colorMode": "value",
- "graphMode": "none"
- }
- },
- {
- "id": 16,
- "type": "stat",
- "title": "Total Sync Runs",
- "gridPos": { "x": 20, "y": 14, "w": 4, "h": 4 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "sum by (source) (corpus_sync_runs_total)",
- "legendFormat": "{{source}}",
- "refId": "A",
- "instant": true
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "short",
- "color": { "mode": "fixed", "fixedColor": "blue" }
- },
- "overrides": []
- },
- "options": {
- "reduceOptions": { "calcs": ["lastNotNull"] },
- "orientation": "auto",
- "textMode": "auto",
- "colorMode": "value",
- "graphMode": "none"
- }
- },
- {
- "id": 20,
- "type": "row",
- "title": "Enrichment",
- "gridPos": { "x": 0, "y": 18, "w": 24, "h": 1 },
- "collapsed": false
- },
- {
- "id": 21,
- "type": "timeseries",
- "title": "Enrichment Fetches (per hour)",
- "gridPos": { "x": 0, "y": 19, "w": 12, "h": 8 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "increase(corpus_enrichment_fetches_total[1h])",
- "legendFormat": "{{user}} {{source}} {{result}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "short",
- "custom": { "lineWidth": 1, "fillOpacity": 10 }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "none" },
- "legend": { "displayMode": "list", "placement": "bottom" }
- }
- },
- {
- "id": 22,
- "type": "timeseries",
- "title": "Enrichment Queue Size",
- "gridPos": { "x": 12, "y": 19, "w": 12, "h": 8 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "corpus_enrichment_queue_size",
- "legendFormat": "{{user}} {{type}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "short",
- "min": 0,
- "custom": { "lineWidth": 1, "fillOpacity": 10 },
- "thresholds": {
- "mode": "absolute",
- "steps": [
- { "color": "green", "value": null },
- { "color": "yellow", "value": 10 },
- { "color": "red", "value": 100 }
- ]
- }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "none" },
- "legend": { "displayMode": "list", "placement": "bottom" }
- }
- },
- {
- "id": 40,
- "type": "row",
- "title": "Cosine Club",
- "gridPos": { "x": 0, "y": 36, "w": 24, "h": 1 },
- "collapsed": false
- },
- {
- "id": 41,
- "type": "timeseries",
- "title": "Cosine Requests (per hour)",
- "gridPos": { "x": 0, "y": 37, "w": 12, "h": 8 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "increase(corpus_cosine_requests_total[1h])",
- "legendFormat": "{{user}} {{result}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "short",
- "custom": { "lineWidth": 1, "fillOpacity": 10 }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "none" },
- "legend": { "displayMode": "list", "placement": "bottom" }
- }
- },
- {
- "id": 42,
- "type": "stat",
- "title": "Cosine Requests by Result",
- "gridPos": { "x": 12, "y": 37, "w": 12, "h": 8 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "sum by (result) (corpus_cosine_requests_total)",
- "legendFormat": "{{result}}",
- "refId": "A",
- "instant": true
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "short",
- "color": { "mode": "fixed", "fixedColor": "blue" }
- },
- "overrides": []
- },
- "options": {
- "reduceOptions": { "calcs": ["lastNotNull"] },
- "orientation": "auto",
- "textMode": "auto",
- "colorMode": "value",
- "graphMode": "none"
- }
- },
- {
- "id": 30,
- "type": "row",
- "title": "Runtime",
- "gridPos": { "x": 0, "y": 45, "w": 24, "h": 1 },
- "collapsed": false
- },
- {
- "id": 31,
- "type": "timeseries",
- "title": "Event Loop Lag",
- "gridPos": { "x": 0, "y": 46, "w": 8, "h": 8 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "nodejs_eventloop_lag_p50_seconds{job=\"corpus\"}",
- "legendFormat": "p50",
- "refId": "A"
- },
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "nodejs_eventloop_lag_p90_seconds{job=\"corpus\"}",
- "legendFormat": "p90",
- "refId": "B"
- },
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "nodejs_eventloop_lag_p99_seconds{job=\"corpus\"}",
- "legendFormat": "p99",
- "refId": "C"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "s",
- "custom": { "lineWidth": 1, "fillOpacity": 10 }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "none" },
- "legend": { "displayMode": "list", "placement": "bottom" }
- }
- },
- {
- "id": 32,
- "type": "timeseries",
- "title": "Memory",
- "gridPos": { "x": 8, "y": 46, "w": 8, "h": 8 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "process_resident_memory_bytes{job=\"corpus\"}",
- "legendFormat": "RSS",
- "refId": "A"
- },
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "nodejs_heap_size_used_bytes{job=\"corpus\"}",
- "legendFormat": "Heap Used",
- "refId": "B"
- },
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "nodejs_heap_size_total_bytes{job=\"corpus\"}",
- "legendFormat": "Heap Total",
- "refId": "C"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "bytes",
- "custom": { "lineWidth": 1, "fillOpacity": 10 }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "none" },
- "legend": { "displayMode": "list", "placement": "bottom" }
- }
- },
- {
- "id": 33,
- "type": "timeseries",
- "title": "GC Duration Rate",
- "gridPos": { "x": 16, "y": 46, "w": 8, "h": 8 },
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "targets": [
- {
- "datasource": { "type": "prometheus", "uid": "DS_PROMETHEUS" },
- "expr": "rate(nodejs_gc_duration_seconds_sum{job=\"corpus\"}[5m])",
- "legendFormat": "{{kind}}",
- "refId": "A"
- }
- ],
- "fieldConfig": {
- "defaults": {
- "unit": "s",
- "custom": { "lineWidth": 1, "fillOpacity": 10 }
- },
- "overrides": []
- },
- "options": {
- "tooltip": { "mode": "multi", "sort": "none" },
- "legend": { "displayMode": "list", "placement": "bottom" }
- }
- }
- ]
-}
diff --git a/modules/mixins/monitoring/alert-rules.yaml b/modules/mixins/monitoring/alert-rules.yaml
@@ -0,0 +1,157 @@
+groups:
+ # ── Generic scrape target availability ─────────────────────────────────────
+ - name: targets
+ rules:
+ - alert: TargetDown
+ expr: up == 0
+ for: 2m
+ labels:
+ severity: critical
+ annotations:
+ summary: "Target {{ $labels.job }}/{{ $labels.instance }} is down"
+ description: >
+ Scrape target {{ $labels.instance }} (job: {{ $labels.job }}) has been
+ unreachable for more than 2 minutes.
+
+ # ── Host-level metrics (node_exporter) ─────────────────────────────────────
+ - name: node
+ rules:
+ - alert: HighCPULoad
+ expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
+ for: 10m
+ labels:
+ severity: warning
+ annotations:
+ summary: "High CPU load: {{ $value | printf \"%.0f\" }}%"
+ description: "CPU usage has been above 90% for more than 10 minutes."
+
+ - alert: HighMemoryUsage
+ expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
+ for: 5m
+ labels:
+ severity: warning
+ annotations:
+ summary: "High memory usage: {{ $value | printf \"%.0f\" }}%"
+ description: "Available memory has been below 10% for more than 5 minutes."
+
+ - alert: HighDiskUsage
+ expr: >
+ (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|squashfs|ramfs|overlay"}
+ / node_filesystem_size_bytes) * 100 > 85
+ for: 5m
+ labels:
+ severity: warning
+ annotations:
+ summary: "High disk usage on {{ $labels.mountpoint }}: {{ $value | printf \"%.0f\" }}%"
+ description: >
+ Disk usage on {{ $labels.mountpoint }} ({{ $labels.device }}) has been
+ above 85% for more than 5 minutes.
+
+ - alert: DiskWillFillSoon
+ expr: >
+ predict_linear(
+ node_filesystem_avail_bytes{fstype!~"tmpfs|fuse.lxcfs|squashfs|ramfs|overlay"}[6h],
+ 4 * 3600
+ ) < 0
+ for: 30m
+ labels:
+ severity: warning
+ annotations:
+ summary: "Disk {{ $labels.mountpoint }} predicted to fill within 4 hours"
+ description: >
+ Based on the last 6 hours of growth, {{ $labels.mountpoint }} will run
+ out of space within 4 hours.
+
+ - alert: HighSystemLoad
+ expr: node_load15 / count without(cpu, mode) (node_cpu_seconds_total{mode="idle"}) > 2
+ for: 15m
+ labels:
+ severity: warning
+ annotations:
+ summary: "High 15-minute load average: {{ $value | printf \"%.2f\" }}"
+ description: "The 15-minute load average per CPU core has been above 2 for 15 minutes."
+
+ # ── Systemd unit health (systemd_exporter) ─────────────────────────────────
+ - name: systemd
+ rules:
+ - alert: SystemdUnitFailed
+ expr: node_systemd_unit_state{state="failed"} == 1
+ for: 5m
+ labels:
+ severity: critical
+ annotations:
+ summary: "Systemd unit {{ $labels.name }} failed"
+ description: >
+ The systemd unit {{ $labels.name }} has been in a failed state
+ for more than 5 minutes.
+
+ # ── Container resources (cAdvisor) ─────────────────────────────────────────
+ - name: containers
+ rules:
+ - alert: ContainerHighCPU
+ expr: sum(rate(container_cpu_usage_seconds_total{name!=""}[5m])) by (name) * 100 > 80
+ for: 10m
+ labels:
+ severity: warning
+ annotations:
+ summary: "Container {{ $labels.name }} CPU usage: {{ $value | printf \"%.0f\" }}%"
+ description: "Container {{ $labels.name }} CPU usage has been above 80% for 10 minutes."
+
+ - alert: ContainerHighMemory
+ expr: >
+ (
+ container_memory_usage_bytes{name!=""}
+ / container_spec_memory_limit_bytes{name!=""}
+ ) * 100 > 80
+ unless container_spec_memory_limit_bytes{name!=""} == 0
+ for: 5m
+ labels:
+ severity: warning
+ annotations:
+ summary: "Container {{ $labels.name }} memory: {{ $value | printf \"%.0f\" }}% of limit"
+ description: "Container {{ $labels.name }} memory usage has been above 80% of its limit for 5 minutes."
+
+ - alert: ContainerOOMKilled
+ expr: increase(container_oom_events_total{name!=""}[5m]) > 0
+ for: 1m
+ labels:
+ severity: critical
+ annotations:
+ summary: "Container {{ $labels.name }} was OOM-killed"
+ description: "Container {{ $labels.name }} was killed due to out-of-memory in the last 5 minutes."
+
+ # ── SeaweedFS cluster (metrics on ports 9324–9327) ─────────────────────────
+ - name: seaweedfs
+ rules:
+ - alert: SeaweedFSDown
+ expr: up{job="seaweedfs"} == 0
+ for: 2m
+ labels:
+ severity: critical
+ annotations:
+ summary: "SeaweedFS component {{ $labels.instance }} is down"
+ description: "SeaweedFS at {{ $labels.instance }} has been unreachable for more than 2 minutes."
+
+ # ── VictoriaMetrics self-health ─────────────────────────────────────────────
+ - name: victoriametrics
+ rules:
+ - alert: HighScrapeErrorRate
+ expr: rate(vm_promscrape_scrapes_failed_total[5m]) > 0.1
+ for: 10m
+ labels:
+ severity: warning
+ annotations:
+ summary: "VictoriaMetrics scrape errors: {{ $value | printf \"%.2f\" }}/s"
+ description: "More than 10% of scrapes have been failing for the last 10 minutes."
+
+ # ── vmalert self-health ─────────────────────────────────────────────────────
+ - name: vmalert
+ rules:
+ - alert: VMAlertRuleErrors
+ expr: increase(vmalert_execution_errors_total[5m]) > 0
+ for: 10m
+ labels:
+ severity: warning
+ annotations:
+ summary: "vmalert rule evaluation errors"
+ description: "vmalert has been encountering rule evaluation errors. Check vmalert logs."
diff --git a/modules/services/monitoring.nix b/modules/services/monitoring.nix
@@ -6,6 +6,12 @@
}: let
cfg = config.services.haravara.monitoring;
+ authSnippet = lib.optionalString (cfg.authLabel != null) "import ${
+ if cfg.authLabel == "default"
+ then "auth"
+ else "auth_${cfg.authLabel}"
+ }";
+
fetchDashboard = {
name,
sha256,
@@ -29,6 +35,11 @@
sha256 = "665bc3e86b79749b2fdae7787f4b7502e00830898bf48f73c68134471f72bf33";
}
{
+ name = "alertmanager.json";
+ url = "https://grafana.com/api/dashboards/9578/revisions/4/download";
+ sha256 = "16jvazvjswiyn281z4r7wmwb18841d3ar24951538kma28l05izy";
+ }
+ {
name = "seaweedfs.json";
url = "https://raw.githubusercontent.com/chrislusf/seaweedfs/c9100a7213e27963f58a0ba783c9598a177384b6/other/metrics/grafana_seaweedfs.json";
sha256 = "e364d113fed54127cbda10c4c2809d99a90dcc38bcd949b91846d0831e1a68e4";
@@ -52,10 +63,11 @@
'';
}
]
- ++ lib.optional config.services.haravara.corpus.enable {
+ ++ lib.optional config.services.haravara.corpus.enable (fetchDashboard {
name = "corpus.json";
- path = ../mixins/corpus/grafana-dashboard.json;
- });
+ url = "https://raw.githubusercontent.com/mtmn/corpus/master/monitoring/grafana-dashboard.json";
+ sha256 = "0b6d4kvcwarv1isy3i4x0l4an7b3zyxyy9af3acr9gbfkkrbyjv5";
+ }));
in {
options.services.haravara.monitoring = {
enable = lib.mkEnableOption "Monitoring stack (VictoriaMetrics + Grafana)";
@@ -70,6 +82,50 @@ in {
default = null;
description = "The auth snippet to import";
};
+
+ alertmanager = {
+ enable = lib.mkEnableOption "Alertmanager for alert routing and notifications";
+ domain = lib.mkOption {
+ type = lib.types.nullOr lib.types.str;
+ default = null;
+ description = "Domain to expose the Alertmanager UI via Caddy (optional)";
+ };
+ port = lib.mkOption {
+ type = lib.types.port;
+ default = 9093;
+ };
+ configuration = lib.mkOption {
+ type = lib.types.attrs;
+ description = "Alertmanager configuration (route, receivers, etc.)";
+ example = lib.literalExpression ''
+ {
+ route = {
+ receiver = "default";
+ group_wait = "30s";
+ group_interval = "5m";
+ repeat_interval = "4h";
+ };
+ receivers = [{
+ name = "default";
+ webhook_configs = [{ url = "https://example.com/hook"; }];
+ }];
+ }
+ '';
+ };
+ };
+
+ vmalert = {
+ enable = lib.mkEnableOption "vmalert for Prometheus-compatible alert rule evaluation";
+ port = lib.mkOption {
+ type = lib.types.port;
+ default = 8880;
+ };
+ rules = lib.mkOption {
+ type = lib.types.listOf lib.types.path;
+ default = [];
+ description = "Paths to YAML files containing alert/recording rules";
+ };
+ };
};
config = lib.mkIf cfg.enable {
@@ -87,6 +143,29 @@ in {
systemd = {
services = {
+ vmalert = lib.mkIf cfg.vmalert.enable {
+ description = "VictoriaMetrics alert rule evaluator";
+ wantedBy = ["multi-user.target"];
+ after = ["network.target" "victoriametrics.service"];
+ serviceConfig = {
+ ExecStart = lib.concatStringsSep " " (
+ [
+ "${pkgs.victoriametrics}/bin/vmalert"
+ "-datasource.url=http://127.0.0.1:8428"
+ "-remoteWrite.url=http://127.0.0.1:8428"
+ "-httpListenAddr=127.0.0.1:${toString cfg.vmalert.port}"
+ "-evaluationInterval=30s"
+ ]
+ ++ lib.optional cfg.alertmanager.enable
+ "-notifier.url=http://127.0.0.1:${toString cfg.alertmanager.port}"
+ ++ map (r: "-rule=${r}") cfg.vmalert.rules
+ );
+ DynamicUser = true;
+ Restart = "on-failure";
+ RestartSec = "5s";
+ };
+ };
+
victoriametrics = {
serviceConfig = {
User = "victoriametrics";
@@ -122,6 +201,9 @@ in {
victoriametrics = {
enable = true;
listenAddress = "127.0.0.1:8428";
+ extraOptions =
+ lib.optional cfg.vmalert.enable
+ "-vmalert.proxyURL=http://127.0.0.1:${toString cfg.vmalert.port}";
prometheusConfig = {
global.scrape_interval = "30s";
scrape_configs =
@@ -162,29 +244,49 @@ in {
job_name = "caddy";
static_configs = [{targets = ["127.0.0.1:2019"];}];
}
+ {
+ job_name = "loki";
+ static_configs = [{targets = ["127.0.0.1:3100"];}];
+ }
]
++ lib.optional config.services.haravara.corpus.enable {
job_name = "corpus";
static_configs = [{targets = ["127.0.0.1:${toString config.services.haravara.corpus.port}"];}];
metrics_path = "/metrics";
+ }
+ ++ lib.optional cfg.vmalert.enable {
+ job_name = "vmalert";
+ static_configs = [{targets = ["127.0.0.1:${toString cfg.vmalert.port}"];}];
+ }
+ ++ lib.optional cfg.alertmanager.enable {
+ job_name = "alertmanager";
+ static_configs = [{targets = ["127.0.0.1:${toString cfg.alertmanager.port}"];}];
};
};
};
- prometheus.exporters.node = {
- enable = true;
- listenAddress = "127.0.0.1";
- enabledCollectors = ["systemd"];
- port = 9100;
- };
+ prometheus = {
+ exporters.node = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ enabledCollectors = ["systemd"];
+ port = 9100;
+ };
- prometheus.exporters.systemd = {
- enable = true;
- listenAddress = "127.0.0.1";
- port = 9558;
- extraFlags = [
- "--systemd.collector.unit-exclude=.*\\\\.(mount|slice|scope|target|device|path|timer|automount|socket|swap)"
- ];
+ exporters.systemd = {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ port = 9558;
+ extraFlags = [
+ "--systemd.collector.unit-exclude=.*\\\\.(mount|slice|scope|target|device|path|timer|automount|socket|swap)"
+ ];
+ };
+
+ alertmanager = lib.mkIf cfg.alertmanager.enable {
+ enable = true;
+ listenAddress = "127.0.0.1";
+ inherit (cfg.alertmanager) port configuration;
+ };
};
cadvisor = {
@@ -320,24 +422,23 @@ in {
};
};
- caddy.virtualHosts = {
- "${cfg.grafanaDomain}".extraConfig = ''
- ${lib.optionalString (cfg.authLabel != null) "import ${
- if cfg.authLabel == "default"
- then "auth"
- else "auth_${cfg.authLabel}"
- }"}
- reverse_proxy 127.0.0.1:3001
- '';
- "${cfg.victoriaDomain}".extraConfig = ''
- ${lib.optionalString (cfg.authLabel != null) "import ${
- if cfg.authLabel == "default"
- then "auth"
- else "auth_${cfg.authLabel}"
- }"}
- reverse_proxy 127.0.0.1:8428
- '';
- };
+ caddy.virtualHosts =
+ {
+ "${cfg.grafanaDomain}".extraConfig = ''
+ ${authSnippet}
+ reverse_proxy 127.0.0.1:3001
+ '';
+ "${cfg.victoriaDomain}".extraConfig = ''
+ ${authSnippet}
+ reverse_proxy 127.0.0.1:8428
+ '';
+ }
+ // lib.optionalAttrs (cfg.alertmanager.enable && cfg.alertmanager.domain != null) {
+ "${cfg.alertmanager.domain}".extraConfig = ''
+ ${authSnippet}
+ reverse_proxy 127.0.0.1:${toString cfg.alertmanager.port}
+ '';
+ };
};
};
}
diff --git a/secrets.nix b/secrets.nix
@@ -16,4 +16,5 @@ in {
"secrets/listenbrainz-token.age".publicKeys = allKeys;
"secrets/mpd-host.age".publicKeys = allKeys;
"secrets/corpus.age".publicKeys = allKeys;
+ "secrets/smtp-alerts-password.age".publicKeys = allKeys;
}
diff --git a/secrets/smtp-alerts-password.age b/secrets/smtp-alerts-password.age
Binary files differ.
