nix

configuration files that power my machines
Log | Files | Refs | README | LICENSE

commit 5d5db0ec88ba7bf3a19a75d525dc25e66ce0891e
parent 1d60e30e8f127084490b89c300570e593bb9ae89
Author: mtmn <miro@haravara.org>
Date:   Sat, 13 Jun 2026 12:35:48 +0200

monitoring: add zfs_exporter, h2o, grafonnet, etc.

Diffstat:
Mhosts/nixaran/configuration.nix | 0
Mhosts/void/overlays/config/shell/rc/void | 0
Amodules/mixins/monitoring/caddy.jsonnet | 762+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Amodules/mixins/monitoring/h2o.jsonnet | 494+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Amodules/mixins/monitoring/zfs_exporter.jsonnet | 539+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mmodules/services/monitoring/grafana.nix | 34+++++++++++++++++++++++++---------
6 files changed, 1820 insertions(+), 9 deletions(-)

diff --git a/hosts/nixaran/configuration.nix b/hosts/nixaran/configuration.nix Binary files differ. diff --git a/hosts/void/overlays/config/shell/rc/void b/hosts/void/overlays/config/shell/rc/void Binary files differ. diff --git a/modules/mixins/monitoring/caddy.jsonnet b/modules/mixins/monitoring/caddy.jsonnet @@ -0,0 +1,762 @@ +{ + local matcher = 'job=~"$job", instance=~"$instance"', + annotations: { + list: [ + { + builtIn: 1, + datasource: { + type: 'datasource', + uid: 'grafana', + }, + enable: true, + hide: true, + iconColor: 'rgba(0, 211, 255, 1)', + limit: 100, + name: 'Annotations & Alerts', + showIn: 0, + target: { + limit: 100, + matchAny: false, + tags: [], + type: 'dashboard', + }, + type: 'dashboard', + }, + ], + }, + editable: false, + gnetId: 13460, + graphTooltip: 0, + id: 10, + iteration: 1633116262227, + links: [], + panels: [ + { + datasource: { + uid: '$datasource', + }, + fieldConfig: { + defaults: { + color: { + mode: 'palette-classic', + }, + custom: { + axisBorderShow: false, + axisCenteredZero: false, + axisColorMode: 'text', + axisLabel: '', + axisPlacement: 'auto', + barAlignment: 0, + drawStyle: 'line', + fillOpacity: 10, + gradientMode: 'none', + hideFrom: { + legend: false, + tooltip: false, + viz: false, + }, + insertNulls: false, + lineInterpolation: 'linear', + lineWidth: 2, + pointSize: 5, + scaleDistribution: { + log: 10, + type: 'log', + }, + showPoints: 'never', + spanNulls: false, + stacking: { + group: 'A', + mode: 'none', + }, + thresholdsStyle: { + mode: 'off', + }, + }, + mappings: [], + thresholds: { + mode: 'absolute', + steps: [ + { + color: 'green', + value: null, + }, + { + color: 'red', + value: 80, + }, + ], + }, + unit: 'short', + }, + overrides: [], + }, + gridPos: { + h: 11, + w: 12, + x: 0, + y: 0, + }, + id: 2, + options: { + legend: { + calcs: [], + displayMode: 'list', + placement: 'bottom', + showLegend: false, + }, + tooltip: { + mode: 'multi', + sort: 'none', + }, + }, + pluginVersion: '10.2.0', + targets: [ + { + datasource: { + uid: '$datasource', + }, + expr: 'sum(rate(caddy_http_requests_total{' + matcher + '}[$__rate_interval])) by (handler)', + interval: '', + legendFormat: '{{handler}}', + refId: 'A', + }, + ], + title: 'Requests', + type: 'timeseries', + }, + { + datasource: { + uid: '$datasource', + }, + description: '', + fieldConfig: { + defaults: { + color: { + mode: 'palette-classic', + }, + custom: { + axisBorderShow: false, + axisCenteredZero: false, + axisColorMode: 'text', + axisLabel: '', + axisPlacement: 'auto', + barAlignment: 0, + drawStyle: 'line', + fillOpacity: 10, + gradientMode: 'none', + hideFrom: { + legend: false, + tooltip: false, + viz: false, + }, + insertNulls: false, + lineInterpolation: 'linear', + lineWidth: 2, + pointSize: 5, + scaleDistribution: { + type: 'linear', + }, + showPoints: 'never', + spanNulls: false, + stacking: { + group: 'A', + mode: 'normal', + }, + thresholdsStyle: { + mode: 'off', + }, + }, + mappings: [], + thresholds: { + mode: 'absolute', + steps: [ + { + color: 'green', + value: null, + }, + { + color: 'red', + value: 80, + }, + ], + }, + unit: 'short', + }, + overrides: [], + }, + gridPos: { + h: 11, + w: 12, + x: 12, + y: 0, + }, + id: 7, + options: { + legend: { + calcs: [], + displayMode: 'list', + placement: 'bottom', + showLegend: true, + }, + tooltip: { + mode: 'multi', + sort: 'none', + }, + }, + pluginVersion: '10.2.0', + targets: [ + { + datasource: { + uid: '$datasource', + }, + expr: 'sum(irate(caddy_http_request_duration_seconds_count{' + matcher + '}[$__rate_interval])) by (code)', + interval: '', + legendFormat: '{{code}}', + refId: 'A', + }, + ], + title: 'Requests by Response Code', + type: 'timeseries', + }, + { + datasource: { + uid: '$datasource', + }, + fieldConfig: { + defaults: { + color: { + mode: 'palette-classic', + }, + custom: { + axisBorderShow: false, + axisCenteredZero: false, + axisColorMode: 'text', + axisLabel: '', + axisPlacement: 'auto', + barAlignment: 0, + drawStyle: 'line', + fillOpacity: 10, + gradientMode: 'none', + hideFrom: { + legend: false, + tooltip: false, + viz: false, + }, + insertNulls: false, + lineInterpolation: 'linear', + lineWidth: 2, + pointSize: 5, + scaleDistribution: { + type: 'linear', + }, + showPoints: 'never', + spanNulls: false, + stacking: { + group: 'A', + mode: 'none', + }, + thresholdsStyle: { + mode: 'off', + }, + }, + mappings: [], + thresholds: { + mode: 'absolute', + steps: [ + { + color: 'green', + value: null, + }, + { + color: 'red', + value: 80, + }, + ], + }, + unit: 'short', + }, + overrides: [], + }, + gridPos: { + h: 9, + w: 12, + x: 0, + y: 11, + }, + id: 8, + options: { + legend: { + calcs: [], + displayMode: 'list', + placement: 'bottom', + showLegend: false, + }, + tooltip: { + mode: 'multi', + sort: 'none', + }, + }, + pluginVersion: '10.2.0', + targets: [ + { + datasource: { + uid: '$datasource', + }, + expr: 'avg(avg_over_time(caddy_http_requests_in_flight{' + matcher + '}[$__rate_interval])) by (handler)', + hide: false, + interval: '', + legendFormat: '{{handler}}', + refId: 'E', + }, + ], + title: 'Requests In Flight', + type: 'timeseries', + }, + { + datasource: { + uid: '$datasource', + }, + description: '', + fieldConfig: { + defaults: { + color: { + mode: 'palette-classic', + }, + custom: { + axisBorderShow: false, + axisCenteredZero: false, + axisColorMode: 'text', + axisLabel: '', + axisPlacement: 'auto', + barAlignment: 0, + drawStyle: 'bars', + fillOpacity: 100, + gradientMode: 'none', + hideFrom: { + legend: false, + tooltip: false, + viz: false, + }, + insertNulls: false, + lineInterpolation: 'linear', + lineWidth: 2, + pointSize: 5, + scaleDistribution: { + type: 'linear', + }, + showPoints: 'never', + spanNulls: false, + stacking: { + group: 'A', + mode: 'normal', + }, + thresholdsStyle: { + mode: 'off', + }, + }, + mappings: [], + min: 0, + thresholds: { + mode: 'absolute', + steps: [ + { + color: 'green', + value: null, + }, + { + color: 'red', + value: 80, + }, + ], + }, + unit: 'none', + }, + overrides: [], + }, + gridPos: { + h: 9, + w: 12, + x: 12, + y: 11, + }, + id: 5, + options: { + legend: { + calcs: [], + displayMode: 'list', + placement: 'bottom', + showLegend: true, + }, + tooltip: { + mode: 'multi', + sort: 'none', + }, + }, + pluginVersion: '10.2.0', + targets: [ + { + datasource: { + uid: '$datasource', + }, + expr: 'sum(irate(caddy_http_request_duration_seconds_count{' + matcher + '}[$__rate_interval])) by (code)', + interval: '', + legendFormat: '{{code}}', + refId: 'A', + }, + ], + title: 'Requests by Response Code (%)', + type: 'timeseries', + }, + { + datasource: { + uid: '$datasource', + }, + fieldConfig: { + defaults: { + color: { + mode: 'palette-classic', + }, + custom: { + axisBorderShow: false, + axisCenteredZero: false, + axisColorMode: 'text', + axisLabel: '', + axisPlacement: 'auto', + barAlignment: 0, + drawStyle: 'line', + fillOpacity: 10, + gradientMode: 'none', + hideFrom: { + legend: false, + tooltip: false, + viz: false, + }, + insertNulls: false, + lineInterpolation: 'linear', + lineWidth: 2, + pointSize: 5, + scaleDistribution: { + log: 2, + type: 'log', + }, + showPoints: 'never', + spanNulls: false, + stacking: { + group: 'A', + mode: 'none', + }, + thresholdsStyle: { + mode: 'off', + }, + }, + mappings: [], + thresholds: { + mode: 'absolute', + steps: [ + { + color: 'green', + value: null, + }, + { + color: 'red', + value: 80, + }, + ], + }, + unit: 's', + }, + overrides: [], + }, + gridPos: { + h: 9, + w: 12, + x: 0, + y: 20, + }, + id: 4, + options: { + legend: { + calcs: [], + displayMode: 'list', + placement: 'bottom', + showLegend: true, + }, + tooltip: { + mode: 'multi', + sort: 'none', + }, + }, + pluginVersion: '10.2.0', + targets: [ + { + datasource: { + uid: '$datasource', + }, + expr: 'histogram_quantile(0.99, sum(rate(caddy_http_request_duration_seconds_bucket{' + matcher + '}[$__rate_interval])) by (le))', + interval: '', + legendFormat: 'p99', + refId: 'A', + }, + { + datasource: { + uid: '$datasource', + }, + expr: 'histogram_quantile(0.95, sum(rate(caddy_http_request_duration_seconds_bucket{' + matcher + '}[$__rate_interval])) by (le))', + interval: '', + legendFormat: 'p95', + refId: 'B', + }, + { + datasource: { + uid: '$datasource', + }, + expr: 'histogram_quantile(0.90, sum(rate(caddy_http_request_duration_seconds_bucket{' + matcher + '}[$__rate_interval])) by (le))', + interval: '', + legendFormat: 'p90', + refId: 'C', + }, + { + datasource: { + uid: '$datasource', + }, + expr: 'histogram_quantile(0.75, sum(rate(caddy_http_request_duration_seconds_bucket{' + matcher + '}[$__rate_interval])) by (le))', + interval: '', + legendFormat: 'p75', + refId: 'D', + }, + { + datasource: { + uid: '$datasource', + }, + expr: 'histogram_quantile(0.5, sum(rate(caddy_http_request_duration_seconds_bucket{' + matcher + '}[$__rate_interval])) by (le))', + interval: '', + legendFormat: 'p50', + refId: 'E', + }, + ], + title: 'Request Duration (percentile)', + type: 'timeseries', + }, + { + cards: {}, + color: { + cardColor: '#b4ff00', + colorScale: 'linear', + colorScheme: 'interpolateInferno', + exponent: 0.5, + mode: 'spectrum', + }, + dataFormat: 'tsbuckets', + datasource: { + uid: '$datasource', + }, + fieldConfig: { + defaults: { + custom: { + hideFrom: { + legend: false, + tooltip: false, + viz: false, + }, + scaleDistribution: { + type: 'linear', + }, + }, + }, + overrides: [], + }, + gridPos: { + h: 9, + w: 12, + x: 12, + y: 20, + }, + heatmap: {}, + hideZeroBuckets: true, + highlightCards: true, + id: 6, + interval: '', + legend: { + show: true, + }, + maxDataPoints: 25, + options: { + calculate: false, + calculation: {}, + cellGap: 2, + cellValues: {}, + color: { + exponent: 0.5, + fill: '#b4ff00', + mode: 'scheme', + reverse: false, + scale: 'exponential', + scheme: 'Inferno', + steps: 128, + }, + exemplars: { + color: 'rgba(255,0,255,0.7)', + }, + filterValues: { + le: 1e-9, + }, + legend: { + show: true, + }, + rowsFrame: { + layout: 'auto', + }, + showValue: 'never', + tooltip: { + show: true, + yHistogram: false, + }, + yAxis: { + axisPlacement: 'left', + reverse: false, + unit: 's', + }, + }, + pluginVersion: '10.2.0', + reverseYBuckets: false, + targets: [ + { + datasource: { + uid: '$datasource', + }, + expr: 'sum(increase(caddy_http_request_duration_seconds_bucket{' + matcher + '}[$__rate_interval])) by (le)', + format: 'heatmap', + interval: '', + legendFormat: '{{le}}', + refId: 'A', + }, + ], + title: 'Request Duration (heatmap)', + tooltip: { + show: true, + showHistogram: false, + }, + type: 'heatmap', + xAxis: { + show: true, + }, + yAxis: { + format: 's', + logBase: 1, + show: true, + }, + yBucketBound: 'auto', + }, + ], + refresh: '30s', + schemaVersion: 38, + style: 'dark', + tags: [ + 'caddy-integration', + ], + templating: { + list: [ + { + current: { + selected: false, + text: 'Prometheus', + value: 'Prometheus', + }, + hide: 0, + includeAll: false, + label: 'Data source', + multi: false, + name: 'datasource', + options: [], + query: 'prometheus', + queryValue: '', + refresh: 1, + regex: '(?!grafanacloud-usage|grafanacloud-ml-metrics).+', + skipUrlSync: false, + type: 'datasource', + }, + { + allValue: '.+', + current: { + selected: false, + text: 'All', + value: '$__all', + }, + datasource: { + uid: '$datasource', + }, + definition: '', + hide: 0, + includeAll: true, + label: 'Job', + multi: true, + name: 'job', + options: [], + query: 'label_values(caddy_http_requests_total, job)', + refresh: 2, + regex: '', + skipUrlSync: false, + sort: 1, + tagValuesQuery: '', + tags: [], + tagsQuery: '', + type: 'query', + useTags: false, + }, + { + allValue: '.+', + current: { + selected: false, + text: 'All', + value: '$__all', + }, + datasource: { + uid: '$datasource', + }, + definition: 'label_values(caddy_http_requests_total{job=~"$job"}, instance)', + hide: 0, + includeAll: true, + label: 'Instance', + multi: true, + name: 'instance', + options: [], + query: { + qryType: 1, + query: 'label_values(caddy_http_requests_total{job=~"$job"}, instance)', + refId: 'PrometheusVariableQueryEditor-VariableQuery', + }, + refresh: 2, + regex: '', + skipUrlSync: false, + sort: 1, + tagValuesQuery: '', + tags: [], + tagsQuery: '', + type: 'query', + useTags: false, + }, + ], + }, + time: { + from: 'now-30m', + to: 'now', + }, + timepicker: { + refresh_intervals: [ + '10s', + '30s', + '1m', + '5m', + '15m', + '30m', + '1h', + '2h', + '1d', + ], + }, + timezone: '', + title: 'Caddy Overview', + uid: '9B0qPnfMz', + version: 9, +} diff --git a/modules/mixins/monitoring/h2o.jsonnet b/modules/mixins/monitoring/h2o.jsonnet @@ -0,0 +1,494 @@ +// H2O exposes every metric as its own metric name with only a `version` label, +// so metric families (status-errors.*, http2-errors.*, quic.num-frames-*, ...) +// are grouped here with {__name__=~"..."} selectors + label_replace. + +local g = import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet'; + +local dashboard = g.dashboard; +local var = dashboard.variable; +local ts = g.panel.timeSeries; +local stat = g.panel.stat; +local gauge = g.panel.gauge; +local barGauge = g.panel.barGauge; +local row = g.panel.row; +local prometheus = g.query.prometheus; + +// --------------------------------------------------------------------------- +// Shared selectors +// --------------------------------------------------------------------------- +local dsRef = { type: 'prometheus', uid: '${datasource}' }; +local selInner = 'job=~"$job",instance=~"$instance"'; +local sel = '{%s}' % selInner; + +// rate() of a metric family by __name__ prefix, relabelled to a short legend. +// `keep_metric_names` is required because rate() otherwise drops __name__, +// collapsing every family member to the same labelset (a "duplicate output +// timeseries" error) and leaving label_replace nothing to match on. +local groupRate(prefix, label) = + 'label_replace(rate({__name__=~"%s.+",%s}[$__rate_interval]) keep_metric_names, "%s", "$1", "__name__", "%s(.+)")' + % [prefix, selInner, label, prefix]; + +// rate() of a single counter. +local rate(metric) = 'rate(%s%s[$__rate_interval])' % [metric, sel]; + +// --------------------------------------------------------------------------- +// Threshold step sets +// --------------------------------------------------------------------------- +local greenOnly = [{ color: 'green', value: null }]; +local usedSteps = [ + { color: 'green', value: null }, + { color: 'yellow', value: 75 }, + { color: 'orange', value: 85 }, + { color: 'red', value: 90 }, +]; +local errSteps = [ + { color: 'green', value: null }, + { color: 'yellow', value: 1 }, + { color: 'red', value: 10 }, +]; +local upDownMap = [{ + type: 'value', + options: { + '0': { text: 'DOWN', color: 'red', index: 0 }, + '1': { text: 'UP', color: 'green', index: 1 }, + }, +}]; + +// --------------------------------------------------------------------------- +// Query helper +// --------------------------------------------------------------------------- +local prom(expr, legend=null, refId='A', instant=false) = + prometheus.new('${datasource}', expr) + + prometheus.withRefId(refId) + + prometheus.withEditorMode('code') + + (if legend != null then prometheus.withLegendFormat(legend) else {}) + + ( + if instant then + prometheus.withInstant(true) + + prometheus.withFormat('table') + + prometheus.withRange(false) + else prometheus.withRange(true) + ); + +// --------------------------------------------------------------------------- +// Panel helpers +// --------------------------------------------------------------------------- +local tsPanel(title, gp, targets, unit='short', steps=greenOnly, min=null, max=null, decimals=null, desc=null, fill=10, stack='none', interp='smooth', threshStyle='off', calcs=['lastNotNull', 'max', 'mean']) = + ts.new(title) + + ts.panelOptions.withGridPos(gp.h, gp.w, gp.x, gp.y) + + { datasource: dsRef } + + ts.queryOptions.withTargets(targets) + + ts.standardOptions.withUnit(unit) + + ts.standardOptions.color.withMode('palette-classic') + + ts.standardOptions.thresholds.withMode('absolute') + + ts.standardOptions.thresholds.withSteps(steps) + + ts.fieldConfig.defaults.custom.withDrawStyle('line') + + ts.fieldConfig.defaults.custom.withFillOpacity(fill) + + ts.fieldConfig.defaults.custom.withGradientMode('opacity') + + ts.fieldConfig.defaults.custom.withLineWidth(2) + + ts.fieldConfig.defaults.custom.withLineInterpolation(interp) + + ts.fieldConfig.defaults.custom.withShowPoints('never') + + ts.fieldConfig.defaults.custom.withSpanNulls(false) + + ts.fieldConfig.defaults.custom.withStacking({ mode: stack, group: 'A' }) + + ts.fieldConfig.defaults.custom.thresholdsStyle.withMode(threshStyle) + + ts.options.legend.withDisplayMode('table') + + ts.options.legend.withPlacement('bottom') + + ts.options.legend.withShowLegend(true) + + ts.options.legend.withCalcs(calcs) + + ts.options.tooltip.withMode('multi') + + ts.options.tooltip.withSort('desc') + + (if min != null then ts.standardOptions.withMin(min) else {}) + + (if max != null then ts.standardOptions.withMax(max) else {}) + + (if decimals != null then ts.standardOptions.withDecimals(decimals) else {}) + + (if desc != null then ts.panelOptions.withDescription(desc) else {}); + +local statPanel(title, gp, targets, unit='short', mappings=[], steps=greenOnly, colorMode='value', textMode='auto', graphMode='area', decimals=null, calc='lastNotNull', desc=null) = + stat.new(title) + + stat.panelOptions.withGridPos(gp.h, gp.w, gp.x, gp.y) + + { datasource: dsRef } + + stat.queryOptions.withTargets(targets) + + stat.standardOptions.withUnit(unit) + + stat.standardOptions.withMappings(mappings) + + stat.standardOptions.color.withMode('thresholds') + + stat.standardOptions.thresholds.withMode('absolute') + + stat.standardOptions.thresholds.withSteps(steps) + + stat.options.withColorMode(colorMode) + + stat.options.withGraphMode(graphMode) + + stat.options.withTextMode(textMode) + + stat.options.withJustifyMode('auto') + + stat.options.withOrientation('auto') + + stat.options.reduceOptions.withCalcs([calc]) + + stat.options.reduceOptions.withFields('') + + stat.options.reduceOptions.withValues(false) + + (if decimals != null then stat.standardOptions.withDecimals(decimals) else {}) + + (if desc != null then stat.panelOptions.withDescription(desc) else {}); + +local gaugePanel(title, gp, targets, unit='percent', steps=usedSteps, min=0, max=100, desc=null) = + gauge.new(title) + + gauge.panelOptions.withGridPos(gp.h, gp.w, gp.x, gp.y) + + { datasource: dsRef } + + gauge.queryOptions.withTargets(targets) + + gauge.standardOptions.withUnit(unit) + + gauge.standardOptions.withMin(min) + + gauge.standardOptions.withMax(max) + + gauge.standardOptions.color.withMode('thresholds') + + gauge.standardOptions.thresholds.withMode('absolute') + + gauge.standardOptions.thresholds.withSteps(steps) + + gauge.options.withShowThresholdLabels(false) + + gauge.options.withShowThresholdMarkers(true) + + gauge.options.withOrientation('auto') + + gauge.options.reduceOptions.withCalcs(['lastNotNull']) + + gauge.options.reduceOptions.withFields('') + + gauge.options.reduceOptions.withValues(false) + + (if desc != null then gauge.panelOptions.withDescription(desc) else {}); + +local barGaugePanel(title, gp, targets, unit='short', colorMode='continuous-GrYlRd', steps=greenOnly, display='gradient', orientation='horizontal', decimals=null, desc=null) = + barGauge.new(title) + + barGauge.panelOptions.withGridPos(gp.h, gp.w, gp.x, gp.y) + + { datasource: dsRef } + + barGauge.queryOptions.withTargets(targets) + + barGauge.standardOptions.withUnit(unit) + + barGauge.standardOptions.color.withMode(colorMode) + + barGauge.standardOptions.thresholds.withMode('absolute') + + barGauge.standardOptions.thresholds.withSteps(steps) + + barGauge.options.withDisplayMode(display) + + barGauge.options.withOrientation(orientation) + + barGauge.options.withShowUnfilled(true) + + barGauge.options.withValueMode('color') + + barGauge.options.reduceOptions.withCalcs(['lastNotNull']) + + barGauge.options.reduceOptions.withFields('') + + barGauge.options.reduceOptions.withValues(false) + + (if decimals != null then barGauge.standardOptions.withDecimals(decimals) else {}) + + (if desc != null then barGauge.panelOptions.withDescription(desc) else {}); + +local rowPanel(title, y) = + row.new(title) + + row.withGridPos({ h: 1, w: 24, x: 0, y: y }) + + row.withCollapsed(false); + +// percentile series for a latency phase (h2o_<phase>_time_{0,25,50,75,99}). +local pctTargets(metric) = [ + prom('%s_0%s' % [metric, sel], legend='p0 (min)', refId='A'), + prom('%s_25%s' % [metric, sel], legend='p25', refId='B'), + prom('%s_50%s' % [metric, sel], legend='p50', refId='C'), + prom('%s_75%s' % [metric, sel], legend='p75', refId='D'), + prom('%s_99%s' % [metric, sel], legend='p99', refId='E'), +]; + +// --------------------------------------------------------------------------- +// Variables +// --------------------------------------------------------------------------- +local datasourceVar = + var.datasource.new('datasource', 'prometheus') + + var.datasource.generalOptions.withLabel('Data source') + + var.datasource.withRegex('') + + { current: { selected: false, text: 'VictoriaMetrics', value: 'DS_PROMETHEUS' } }; + +local queryVar(name, label, expr, valueLabel) = + var.query.new(name) + + var.query.withDatasourceFromVariable(datasourceVar) + + var.query.queryTypes.withLabelValues(valueLabel, expr) + + var.query.generalOptions.withLabel(label) + + var.query.selectionOptions.withMulti(true) + + var.query.selectionOptions.withIncludeAll(true) + + var.query.withRefresh('time') + + var.query.withSort(1); + +local jobVar = queryVar('job', 'Job', 'h2o_uptime', 'job'); +local instanceVar = queryVar('instance', 'Instance', 'h2o_uptime{job=~"$job"}', 'instance'); + +// --------------------------------------------------------------------------- +// Panels +// --------------------------------------------------------------------------- +local panels = [ + //=== Overview ============================================================ + rowPanel('Overview', 0), + + statPanel('Version', { h: 4, w: 3, x: 0, y: 1 }, + [prom('h2o_uptime%s' % sel, legend='{{version}}')], + unit='short', graphMode='none', textMode='name', colorMode='none', + desc='Running H2O version.'), + + statPanel('Uptime', { h: 4, w: 3, x: 3, y: 1 }, + [prom('h2o_uptime%s' % sel)], + unit='s', graphMode='none', + desc='Seconds since the current worker process started.'), + + statPanel('Connections', { h: 4, w: 3, x: 6, y: 1 }, + [prom('h2o_connections%s' % sel)], + unit='short'), + + statPanel('Sessions', { h: 4, w: 3, x: 9, y: 1 }, + [prom('h2o_num_sessions%s' % sel)], + unit='short'), + + statPanel('Generation', { h: 4, w: 3, x: 12, y: 1 }, + [prom('h2o_generation%s' % sel)], + unit='short', graphMode='none', + desc='Configuration generation — increments on each graceful reload.'), + + statPanel('Worker Threads', { h: 4, w: 3, x: 15, y: 1 }, + [prom('h2o_worker_threads%s' % sel)], + unit='short', graphMode='none'), + + statPanel('Listeners', { h: 4, w: 3, x: 18, y: 1 }, + [prom('h2o_listeners%s' % sel)], + unit='short', graphMode='none'), + + gaugePanel('Connection Utilization', { h: 4, w: 3, x: 21, y: 1 }, + [prom('h2o_connections%s / h2o_max_connections%s * 100' % [sel, sel])], + desc='Current connections as a percentage of max-connections.'), + + //=== Connections ========================================================= + rowPanel('Connections', 5), + + tsPanel('Connections & Sessions', { h: 8, w: 12, x: 0, y: 6 }, + [ + prom('h2o_connections%s' % sel, legend='connections', refId='A'), + prom('h2o_num_sessions%s' % sel, legend='sessions', refId='B'), + prom('h2o_connections_active%s' % sel, legend='active', refId='C'), + prom('h2o_connections_idle%s' % sel, legend='idle', refId='D'), + ], + unit='short', + desc='Live connection and session counts.'), + + tsPanel('Connection Churn', { h: 8, w: 12, x: 12, y: 6 }, + [ + prom(rate('h2o_connections_idle_closed'), legend='idle-closed', refId='A'), + prom(rate('h2o_connections_shutdown'), legend='shutdown', refId='B'), + ], + unit='cps', + desc='Rate of connections closed for being idle, and connections shut down.'), + + //=== Request Latency ===================================================== + rowPanel('Request Latency (per H2O status durations, ms)', 14), + + tsPanel('Request Total Time — percentiles', { h: 8, w: 12, x: 0, y: 15 }, + pctTargets('h2o_request_total_time'), + unit='ms', threshStyle='off', + desc='End-to-end request time distribution (p0/p25/p50/p75/p99).'), + + tsPanel('Duration — percentiles', { h: 8, w: 12, x: 12, y: 15 }, + pctTargets('h2o_duration'), + unit='ms', + desc='Connection duration distribution.'), + + tsPanel('Request Phases @ p50', { h: 8, w: 12, x: 0, y: 23 }, + [ + prom('h2o_connect_time_50%s' % sel, legend='connect', refId='A'), + prom('h2o_header_time_50%s' % sel, legend='header', refId='B'), + prom('h2o_body_time_50%s' % sel, legend='body', refId='C'), + prom('h2o_process_time_50%s' % sel, legend='process', refId='D'), + prom('h2o_response_time_50%s' % sel, legend='response', refId='E'), + prom('h2o_request_total_time_50%s' % sel, legend='total', refId='F'), + ], + unit='ms', + desc='Median latency contribution of each request phase.'), + + tsPanel('Request Phases @ p99', { h: 8, w: 12, x: 12, y: 23 }, + [ + prom('h2o_connect_time_99%s' % sel, legend='connect', refId='A'), + prom('h2o_header_time_99%s' % sel, legend='header', refId='B'), + prom('h2o_body_time_99%s' % sel, legend='body', refId='C'), + prom('h2o_process_time_99%s' % sel, legend='process', refId='D'), + prom('h2o_response_time_99%s' % sel, legend='response', refId='E'), + prom('h2o_request_total_time_99%s' % sel, legend='total', refId='F'), + ], + unit='ms', + desc='Tail latency contribution of each request phase.'), + + //=== HTTP Status & Errors ================================================ + rowPanel('HTTP Status & Errors', 31), + + tsPanel('HTTP Status Errors', { h: 8, w: 12, x: 0, y: 32 }, + [prom(groupRate('h2o_status_errors_', 'code'), legend='{{code}}')], + unit='cps', stack='normal', fill=25, + desc='Per-status-code error response rate (4xx / 5xx).'), + + tsPanel('Protocol Errors', { h: 8, w: 12, x: 12, y: 32 }, + [ + prom(groupRate('h2o_http2_errors_', 'kind'), legend='h2: {{kind}}', refId='A'), + prom(groupRate('h2o_http1_errors_', 'kind'), legend='h1: {{kind}}', refId='B'), + ], + unit='cps', + desc='HTTP/1 and HTTP/2 protocol error rates by kind.'), + + barGaugePanel('Status Errors — total', { h: 7, w: 12, x: 0, y: 40 }, + [prom('label_replace({__name__=~"h2o_status_errors_.+",%s}, "code", "$1", "__name__", "h2o_status_errors_(.+)")' % selInner, legend='{{code}}')], + unit='short', display='gradient'), + + tsPanel('HTTP/2 Activity', { h: 7, w: 12, x: 12, y: 40 }, + [ + prom(rate('h2o_http2_read_closed'), legend='read-closed', refId='A'), + prom(rate('h2o_http2_write_closed'), legend='write-closed', refId='B'), + prom(rate('h2o_http2_idle_timeout'), legend='idle-timeout', refId='C'), + prom(rate('h2o_http2_streaming_requests'), legend='streaming-requests', refId='D'), + ], + unit='cps', + desc='HTTP/2 stream lifecycle events.'), + + //=== TLS / SSL =========================================================== + rowPanel('TLS / SSL', 47), + + tsPanel('TLS Handshakes', { h: 7, w: 12, x: 0, y: 48 }, + [ + prom(rate('h2o_ssl_handshake_full'), legend='full', refId='A'), + prom(rate('h2o_ssl_handshake_resume'), legend='resume', refId='B'), + ], + unit='ops', + desc='Full vs. resumed TLS handshakes per second.'), + + tsPanel('ALPN Negotiation', { h: 7, w: 12, x: 12, y: 48 }, + [ + prom(rate('h2o_ssl_alpn_h1'), legend='http/1.1', refId='A'), + prom(rate('h2o_ssl_alpn_h2'), legend='h2', refId='B'), + ], + unit='ops', + desc='Negotiated ALPN protocol per second.'), + + gaugePanel('Handshake Resume Ratio', { h: 6, w: 6, x: 0, y: 55 }, + [prom('sum(%s) / clamp_min(sum(%s) + sum(%s), 1) * 100' + % [rate('h2o_ssl_handshake_resume'), rate('h2o_ssl_handshake_full'), rate('h2o_ssl_handshake_resume')])], + steps=[{ color: 'red', value: null }, { color: 'yellow', value: 40 }, { color: 'green', value: 70 }], + desc='Share of handshakes served from session resumption (higher is cheaper).'), + + statPanel('SSL Errors', { h: 6, w: 6, x: 6, y: 55 }, + [prom('h2o_ssl_errors%s' % sel)], + unit='short', graphMode='area', steps=errSteps), + + statPanel('Avg Full Handshake', { h: 6, w: 6, x: 12, y: 55 }, + [prom('%s / clamp_min(%s, 1)' % [rate('h2o_ssl_handshake_accumulated_time_full'), rate('h2o_ssl_handshake_full')])], + unit='µs', graphMode='area', + desc='Mean wall time per full handshake (H2O reports accumulated time in microseconds).'), + + statPanel('Avg Resume Handshake', { h: 6, w: 6, x: 18, y: 55 }, + [prom('%s / clamp_min(%s, 1)' % [rate('h2o_ssl_handshake_accumulated_time_resume'), rate('h2o_ssl_handshake_resume')])], + unit='µs', graphMode='area', + desc='Mean wall time per resumed handshake.'), + + //=== QUIC / HTTP3 ======================================================== + rowPanel('QUIC / HTTP3', 61), + + tsPanel('QUIC Packets', { h: 8, w: 12, x: 0, y: 62 }, + [ + prom(rate('h2o_quic_num_packets_received'), legend='received', refId='A'), + prom(rate('h2o_quic_num_packets_sent'), legend='sent', refId='B'), + prom(rate('h2o_quic_num_packets_lost'), legend='lost', refId='C'), + prom(rate('h2o_quic_num_packets_ack_received'), legend='ack-received', refId='D'), + prom(rate('h2o_quic_packet_processed'), legend='processed', refId='E'), + ], + unit='pps', + desc='QUIC packet throughput and loss.'), + + tsPanel('QUIC Bytes', { h: 8, w: 12, x: 12, y: 62 }, + [ + prom(rate('h2o_quic_num_bytes_received'), legend='received', refId='A'), + prom(rate('h2o_quic_num_bytes_sent'), legend='sent', refId='B'), + prom(rate('h2o_quic_num_bytes_lost'), legend='lost', refId='C'), + prom(rate('h2o_quic_num_bytes_stream_data_sent'), legend='stream-data-sent', refId='D'), + ], + unit='Bps', + desc='QUIC byte throughput.'), + + statPanel('QUIC Packet Loss', { h: 6, w: 6, x: 0, y: 70 }, + [prom('%s / clamp_min(%s, 1) * 100' % [rate('h2o_quic_num_packets_lost'), rate('h2o_quic_num_packets_sent')])], + unit='percent', graphMode='area', steps=errSteps, + desc='Lost packets as a fraction of sent.'), + + tsPanel('QUIC Paths', { h: 6, w: 9, x: 6, y: 70 }, + [prom(groupRate('h2o_quic_num_paths_', 'event'), legend='{{event}}')], + unit='cps', + desc='QUIC path validation / migration events.'), + + tsPanel('HTTP/3 Packet Forwarding', { h: 6, w: 9, x: 15, y: 70 }, + [ + prom(rate('h2o_http3_packet_forwarded'), legend='forwarded', refId='A'), + prom(rate('h2o_http3_forwarded_packet_received'), legend='forwarded-received', refId='B'), + ], + unit='pps'), + + tsPanel('QUIC Frames Received', { h: 8, w: 12, x: 0, y: 76 }, + [prom(groupRate('h2o_quic_num_frames_received_', 'frame'), legend='{{frame}}')], + unit='cps', + desc='Inbound QUIC frames by type.'), + + tsPanel('QUIC Frames Sent', { h: 8, w: 12, x: 12, y: 76 }, + [prom(groupRate('h2o_quic_num_frames_sent_', 'frame'), legend='{{frame}}')], + unit='cps', + desc='Outbound QUIC frames by type.'), + + tsPanel('QUIC Recovery & Congestion', { h: 7, w: 12, x: 0, y: 84 }, + [ + prom(rate('h2o_quic_num_ptos'), legend='ptos', refId='A'), + prom(rate('h2o_quic_num_handshake_timeouts'), legend='handshake-timeouts', refId='B'), + prom(rate('h2o_quic_num_initial_handshake_exceeded'), legend='initial-handshake-exceeded', refId='C'), + prom(rate('h2o_quic_num_paced'), legend='paced', refId='D'), + prom(rate('h2o_quic_num_rapid_start'), legend='rapid-start', refId='E'), + prom(rate('h2o_quic_num_respected_app_limited'), legend='respected-app-limited', refId='F'), + ], + unit='cps', + desc='QUIC loss recovery, pacing and congestion-control signals.'), + + tsPanel('QUIC ECN', { h: 7, w: 12, x: 12, y: 84 }, + [ + prom(rate('h2o_quic_num_packets_received_ecn_ect0'), legend='rx ect0', refId='A'), + prom(rate('h2o_quic_num_packets_received_ecn_ect1'), legend='rx ect1', refId='B'), + prom(rate('h2o_quic_num_packets_received_ecn_ce'), legend='rx ce', refId='C'), + prom(rate('h2o_quic_num_packets_acked_ecn_ect0'), legend='acked ect0', refId='D'), + prom(rate('h2o_quic_num_packets_acked_ecn_ect1'), legend='acked ect1', refId='E'), + prom(rate('h2o_quic_num_packets_acked_ecn_ce'), legend='acked ce', refId='F'), + ], + unit='pps', + desc='Explicit Congestion Notification marks on received and acked packets.'), + + //=== Memory & Internals ================================================== + rowPanel('Memory & Internals', 91), + + tsPanel('Memory Pool Chunks', { h: 7, w: 12, x: 0, y: 92 }, + [ + prom('h2o_memory_mem_pool_chunks%s' % sel, legend='mem-pool', refId='A'), + prom('h2o_memory_socket_ssl_chunks%s' % sel, legend='socket-ssl', refId='B'), + prom('h2o_memory_socket_zerocopy_chunks%s' % sel, legend='socket-zerocopy', refId='C'), + prom('h2o_memory_socket_zerocopy_inflight%s' % sel, legend='zerocopy-inflight', refId='D'), + ], + unit='short', + desc='Recycled buffer chunk counts per allocator.'), + + tsPanel('Event Loop Latency', { h: 7, w: 12, x: 12, y: 92 }, + [prom('h2o_evloop_latency_nanosec%s' % sel, legend='evloop')], + unit='ns', + desc='Event-loop latency (only populated when latency-optimization probing is enabled).'), + + statPanel('mmap Errors', { h: 5, w: 6, x: 0, y: 99 }, + [prom('h2o_memory_mmap_errors%s' % sel)], + unit='short', graphMode='area', steps=errSteps), + + statPanel('h2olog Lost', { h: 5, w: 6, x: 6, y: 99 }, + [prom('h2o_h2olog_lost%s' % sel)], + unit='short', graphMode='area', steps=errSteps, + desc='h2olog events dropped because the ring buffer was full.'), + + statPanel('Max Connections', { h: 5, w: 6, x: 12, y: 99 }, + [prom('h2o_max_connections%s' % sel)], + unit='short', graphMode='none', colorMode='none'), + + statPanel('Soft Limit Min Age', { h: 5, w: 6, x: 18, y: 99 }, + [prom('h2o_soft_connection_limit_min_age%s' % sel)], + unit='s', graphMode='none', colorMode='none', + desc='Minimum connection age before the soft connection limit may close it.'), +]; + +// Assign stable, unique panel IDs. +local panelsWithIds = std.mapWithIndex(function(i, p) p + { id: i + 1 }, panels); + +// --------------------------------------------------------------------------- +// Dashboard +// --------------------------------------------------------------------------- +dashboard.new('H2O') ++ dashboard.withUid('h2o') ++ dashboard.withTags(['h2o', 'http', 'webserver']) ++ dashboard.withRefresh('30s') ++ dashboard.withTimezone('browser') ++ dashboard.withVariables([datasourceVar, jobVar, instanceVar]) ++ dashboard.withPanels(panelsWithIds) ++ { graphTooltip: 1, time: { from: 'now-6h', to: 'now' } } diff --git a/modules/mixins/monitoring/zfs_exporter.jsonnet b/modules/mixins/monitoring/zfs_exporter.jsonnet @@ -0,0 +1,539 @@ +// ZFS Exporter dashboard (pdf/zfs_exporter) + +local g = import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet'; + +local dashboard = g.dashboard; +local var = dashboard.variable; +local ts = g.panel.timeSeries; +local stat = g.panel.stat; +local gauge = g.panel.gauge; +local barGauge = g.panel.barGauge; +local table = g.panel.table; +local row = g.panel.row; +local prometheus = g.query.prometheus; + +// --------------------------------------------------------------------------- +// Shared selectors and datasource reference +// --------------------------------------------------------------------------- +local dsRef = { type: 'prometheus', uid: '${datasource}' }; +local poolSel = '{instance=~"$instance",pool=~"$pool"}'; +local dsSel = '{type="filesystem",instance=~"$instance",pool=~"$pool",name=~"$dataset"}'; + +// --------------------------------------------------------------------------- +// Threshold step sets and value mappings +// --------------------------------------------------------------------------- +local greenOnly = [{ color: 'green', value: null }]; +local usedSteps = [ + { color: 'green', value: null }, + { color: 'yellow', value: 75 }, + { color: 'orange', value: 85 }, + { color: 'red', value: 90 }, +]; +local healthyStep = [{ color: 'red', value: null }, { color: 'green', value: 1 }]; +local fragSteps = [ + { color: 'green', value: null }, + { color: 'yellow', value: 0.4 }, + { color: 'red', value: 0.7 }, +]; +local compSteps = [ + { color: 'blue', value: null }, + { color: 'green', value: 1.1 }, + { color: 'super-light-green', value: 2 }, +]; + +local healthMap = [{ + type: 'value', + options: { + '0': { text: 'ONLINE', color: 'green', index: 0 }, + '1': { text: 'DEGRADED', color: 'orange', index: 1 }, + '2': { text: 'FAULTED', color: 'red', index: 2 }, + '3': { text: 'OFFLINE', color: 'red', index: 3 }, + '4': { text: 'UNAVAIL', color: 'red', index: 4 }, + '5': { text: 'REMOVED', color: 'red', index: 5 }, + '6': { text: 'SUSPENDED', color: 'red', index: 6 }, + }, +}]; +local okFailMap = [{ + type: 'value', + options: { + '0': { text: 'FAILED', color: 'red', index: 0 }, + '1': { text: 'OK', color: 'green', index: 1 }, + }, +}]; +local rwroMap = [{ + type: 'value', + options: { + '0': { text: 'read-write', color: 'green', index: 0 }, + '1': { text: 'read-only', color: 'orange', index: 1 }, + }, +}]; +local upDownMap = [{ + type: 'value', + options: { + '0': { text: 'DOWN', color: 'red', index: 0 }, + '1': { text: 'UP', color: 'green', index: 1 }, + }, +}]; + +// --------------------------------------------------------------------------- +// Query helper +// --------------------------------------------------------------------------- +local prom(expr, legend=null, refId='A', instant=false) = + prometheus.new('${datasource}', expr) + + prometheus.withRefId(refId) + + prometheus.withEditorMode('code') + + (if legend != null then prometheus.withLegendFormat(legend) else {}) + + ( + if instant then + prometheus.withInstant(true) + + prometheus.withFormat('table') + + prometheus.withRange(false) + else prometheus.withRange(true) + ); + +// --------------------------------------------------------------------------- +// Panel helpers +// --------------------------------------------------------------------------- +local tsPanel(title, gp, targets, unit='bytes', steps=greenOnly, mappings=[], min=null, max=null, decimals=null, desc=null, fill=10, stack='none', interp='smooth', threshStyle='off', calcs=['lastNotNull', 'max', 'mean']) = + ts.new(title) + + ts.panelOptions.withGridPos(gp.h, gp.w, gp.x, gp.y) + + { datasource: dsRef } + + ts.queryOptions.withTargets(targets) + + ts.standardOptions.withUnit(unit) + + ts.standardOptions.color.withMode('palette-classic') + + ts.standardOptions.thresholds.withMode('absolute') + + ts.standardOptions.thresholds.withSteps(steps) + + ts.standardOptions.withMappings(mappings) + + ts.fieldConfig.defaults.custom.withDrawStyle('line') + + ts.fieldConfig.defaults.custom.withFillOpacity(fill) + + ts.fieldConfig.defaults.custom.withGradientMode('opacity') + + ts.fieldConfig.defaults.custom.withLineWidth(2) + + ts.fieldConfig.defaults.custom.withLineInterpolation(interp) + + ts.fieldConfig.defaults.custom.withShowPoints('never') + + ts.fieldConfig.defaults.custom.withSpanNulls(false) + + ts.fieldConfig.defaults.custom.withStacking({ mode: stack, group: 'A' }) + + ts.fieldConfig.defaults.custom.thresholdsStyle.withMode(threshStyle) + + ts.options.legend.withDisplayMode('table') + + ts.options.legend.withPlacement('bottom') + + ts.options.legend.withShowLegend(true) + + ts.options.legend.withCalcs(calcs) + + ts.options.tooltip.withMode('multi') + + ts.options.tooltip.withSort('desc') + + (if min != null then ts.standardOptions.withMin(min) else {}) + + (if max != null then ts.standardOptions.withMax(max) else {}) + + (if decimals != null then ts.standardOptions.withDecimals(decimals) else {}) + + (if desc != null then ts.panelOptions.withDescription(desc) else {}); + +local statPanel(title, gp, targets, unit='none', mappings=[], steps=greenOnly, colorMode='value', textMode='auto', graphMode='area', decimals=null, calc='lastNotNull', desc=null) = + stat.new(title) + + stat.panelOptions.withGridPos(gp.h, gp.w, gp.x, gp.y) + + { datasource: dsRef } + + stat.queryOptions.withTargets(targets) + + stat.standardOptions.withUnit(unit) + + stat.standardOptions.withMappings(mappings) + + stat.standardOptions.color.withMode('thresholds') + + stat.standardOptions.thresholds.withMode('absolute') + + stat.standardOptions.thresholds.withSteps(steps) + + stat.options.withColorMode(colorMode) + + stat.options.withGraphMode(graphMode) + + stat.options.withTextMode(textMode) + + stat.options.withJustifyMode('auto') + + stat.options.withOrientation('auto') + + stat.options.reduceOptions.withCalcs([calc]) + + stat.options.reduceOptions.withFields('') + + stat.options.reduceOptions.withValues(false) + + (if decimals != null then stat.standardOptions.withDecimals(decimals) else {}) + + (if desc != null then stat.panelOptions.withDescription(desc) else {}); + +local gaugePanel(title, gp, targets, unit='percent', steps=usedSteps, min=0, max=100, desc=null) = + gauge.new(title) + + gauge.panelOptions.withGridPos(gp.h, gp.w, gp.x, gp.y) + + { datasource: dsRef } + + gauge.queryOptions.withTargets(targets) + + gauge.standardOptions.withUnit(unit) + + gauge.standardOptions.withMin(min) + + gauge.standardOptions.withMax(max) + + gauge.standardOptions.color.withMode('thresholds') + + gauge.standardOptions.thresholds.withMode('absolute') + + gauge.standardOptions.thresholds.withSteps(steps) + + gauge.options.withShowThresholdLabels(false) + + gauge.options.withShowThresholdMarkers(true) + + gauge.options.withOrientation('auto') + + gauge.options.reduceOptions.withCalcs(['lastNotNull']) + + gauge.options.reduceOptions.withFields('') + + gauge.options.reduceOptions.withValues(false) + + (if desc != null then gauge.panelOptions.withDescription(desc) else {}); + +local barGaugePanel(title, gp, targets, unit='bytes', colorMode='continuous-GrYlRd', steps=greenOnly, display='gradient', orientation='horizontal', min=null, max=null, decimals=null, desc=null) = + barGauge.new(title) + + barGauge.panelOptions.withGridPos(gp.h, gp.w, gp.x, gp.y) + + { datasource: dsRef } + + barGauge.queryOptions.withTargets(targets) + + barGauge.standardOptions.withUnit(unit) + + barGauge.standardOptions.color.withMode(colorMode) + + barGauge.standardOptions.thresholds.withMode('absolute') + + barGauge.standardOptions.thresholds.withSteps(steps) + + barGauge.options.withDisplayMode(display) + + barGauge.options.withOrientation(orientation) + + barGauge.options.withShowUnfilled(true) + + barGauge.options.withValueMode('color') + + barGauge.options.reduceOptions.withCalcs(['lastNotNull']) + + barGauge.options.reduceOptions.withFields('') + + barGauge.options.reduceOptions.withValues(false) + + (if min != null then barGauge.standardOptions.withMin(min) else {}) + + (if max != null then barGauge.standardOptions.withMax(max) else {}) + + (if decimals != null then barGauge.standardOptions.withDecimals(decimals) else {}) + + (if desc != null then barGauge.panelOptions.withDescription(desc) else {}); + +local tablePanel(title, gp, targets, transformations, overrides, sortBy, footer, desc) = + table.new(title) + + table.panelOptions.withGridPos(gp.h, gp.w, gp.x, gp.y) + + { datasource: dsRef } + + table.panelOptions.withDescription(desc) + + table.queryOptions.withTargets(targets) + + table.queryOptions.withTransformations(transformations) + + table.standardOptions.withUnit('bytes') + + table.standardOptions.color.withMode('thresholds') + + table.standardOptions.thresholds.withMode('absolute') + + table.standardOptions.thresholds.withSteps(greenOnly) + + table.standardOptions.withOverrides(overrides) + + { fieldConfig+: { defaults+: { custom+: { + align: 'auto', + cellOptions: { type: 'auto' }, + filterable: true, + inspect: false, + } } } } + + table.options.withShowHeader(true) + + table.options.withCellHeight('sm') + + table.options.withFooter(footer) + + table.options.withSortBy(sortBy); + +local rowPanel(title, y) = + row.new(title) + + row.withGridPos({ h: 1, w: 24, x: 0, y: y }) + + row.withCollapsed(false); + +local override(name, properties) = { + matcher: { id: 'byName', options: name }, + properties: properties, +}; + +// --------------------------------------------------------------------------- +// Variables +// --------------------------------------------------------------------------- +local datasourceVar = + var.datasource.new('datasource', 'prometheus') + + var.datasource.generalOptions.withLabel('Data source') + + var.datasource.withRegex('') + + { current: { selected: false, text: 'VictoriaMetrics', value: 'DS_PROMETHEUS' } }; + +local queryVar(name, label, expr, valueLabel) = + var.query.new(name) + + var.query.withDatasourceFromVariable(datasourceVar) + + var.query.queryTypes.withLabelValues(valueLabel, expr) + + var.query.generalOptions.withLabel(label) + + var.query.selectionOptions.withMulti(true) + + var.query.selectionOptions.withIncludeAll(true) + + var.query.withRefresh('time') + + var.query.withSort(1); + +local instanceVar = queryVar('instance', 'Instance', 'zfs_pool_health', 'instance'); +local poolVar = queryVar('pool', 'Pool', 'zfs_pool_health{instance=~"$instance"}', 'pool'); +local datasetVar = queryVar('dataset', 'Dataset', 'zfs_dataset_used_bytes{type="filesystem",instance=~"$instance",pool=~"$pool"}', 'name'); + +// --------------------------------------------------------------------------- +// Panels +// --------------------------------------------------------------------------- +local panels = [ + //=== Pool Overview ======================================================= + rowPanel('Pool Overview', 0), + + statPanel('Pools', { h: 4, w: 3, x: 0, y: 1 }, + [prom('count(zfs_pool_health%s)' % poolSel)], + unit='none', graphMode='none', + desc='Number of ZFS pools matched by the current filters.'), + + statPanel('Pools Healthy', { h: 4, w: 3, x: 3, y: 1 }, + [prom('count(zfs_pool_health%s == 0) or vector(0)' % poolSel)], + unit='none', graphMode='none', steps=healthyStep, + desc='Pools currently in the ONLINE (healthy) state.'), + + statPanel('Total Capacity', { h: 4, w: 3, x: 6, y: 1 }, + [prom('sum(zfs_pool_size_bytes%s)' % poolSel)], + unit='bytes', graphMode='none'), + + statPanel('Allocated', { h: 4, w: 3, x: 9, y: 1 }, + [prom('sum(zfs_pool_allocated_bytes%s)' % poolSel)], + unit='bytes'), + + statPanel('Free', { h: 4, w: 3, x: 12, y: 1 }, + [prom('sum(zfs_pool_free_bytes%s)' % poolSel)], + unit='bytes'), + + gaugePanel('Used %', { h: 4, w: 3, x: 15, y: 1 }, + [prom('sum(zfs_pool_allocated_bytes%s) / sum(zfs_pool_size_bytes%s) * 100' % [poolSel, poolSel])], + desc='Aggregate space used across all matched pools.'), + + statPanel('Datasets', { h: 4, w: 3, x: 18, y: 1 }, + [prom('count(zfs_dataset_used_bytes{type="filesystem",instance=~"$instance",pool=~"$pool"})')], + unit='none', graphMode='none'), + + statPanel('Exporter', { h: 4, w: 3, x: 21, y: 1 }, + [prom('min(up{job="zfs_exporter",instance=~"$instance"})')], + unit='none', graphMode='none', mappings=upDownMap, + colorMode='background', textMode='value', + desc='Scrape state of the zfs_exporter target(s).'), + + //=== Pool Health & Capacity ============================================== + rowPanel('Pool Health & Capacity', 5), + + tablePanel( + 'Pool Status', { h: 9, w: 15, x: 0, y: 6 }, + [ + prom('max by (pool) (zfs_pool_health%s)' % poolSel, refId='A', instant=true), + prom('max by (pool) (zfs_pool_size_bytes%s)' % poolSel, refId='B', instant=true), + prom('max by (pool) (zfs_pool_allocated_bytes%s)' % poolSel, refId='C', instant=true), + prom('max by (pool) (zfs_pool_free_bytes%s)' % poolSel, refId='D', instant=true), + prom('max by (pool) (zfs_pool_allocated_bytes%s / zfs_pool_size_bytes%s * 100)' % [poolSel, poolSel], refId='E', instant=true), + prom('max by (pool) (zfs_pool_fragmentation_ratio%s)' % poolSel, refId='F', instant=true), + prom('max by (pool) (zfs_pool_deduplication_ratio%s)' % poolSel, refId='G', instant=true), + prom('max by (pool) (zfs_pool_readonly%s)' % poolSel, refId='H', instant=true), + ], + [ + { id: 'joinByField', options: { byField: 'pool', mode: 'outer' } }, + { id: 'organize', options: { + excludeByName: { Time: true, 'Time 1': true, 'Time 2': true, 'Time 3': true, 'Time 4': true, 'Time 5': true, 'Time 6': true, 'Time 7': true, 'Time 8': true }, + renameByName: { + pool: 'Pool', + 'Value #A': 'Health', + 'Value #B': 'Size', + 'Value #C': 'Allocated', + 'Value #D': 'Free', + 'Value #E': 'Used %', + 'Value #F': 'Fragmentation', + 'Value #G': 'Dedup', + 'Value #H': 'Read-only', + }, + indexByName: { pool: 0, 'Value #A': 1, 'Value #B': 2, 'Value #C': 3, 'Value #D': 4, 'Value #E': 5, 'Value #F': 6, 'Value #G': 7, 'Value #H': 8 }, + } }, + ], + [ + override('Health', [ + { id: 'mappings', value: healthMap }, + { id: 'custom.cellOptions', value: { type: 'color-text' } }, + { id: 'custom.width', value: 110 }, + ]), + override('Used %', [ + { id: 'unit', value: 'percent' }, + { id: 'decimals', value: 1 }, + { id: 'custom.cellOptions', value: { type: 'gauge', mode: 'gradient' } }, + { id: 'min', value: 0 }, + { id: 'max', value: 100 }, + { id: 'thresholds', value: { mode: 'absolute', steps: usedSteps } }, + ]), + override('Fragmentation', [ + { id: 'unit', value: 'percentunit' }, + { id: 'decimals', value: 1 }, + { id: 'custom.cellOptions', value: { type: 'color-text' } }, + { id: 'thresholds', value: { mode: 'absolute', steps: fragSteps } }, + ]), + override('Dedup', [ + { id: 'unit', value: 'none' }, + { id: 'decimals', value: 2 }, + ]), + override('Read-only', [ + { id: 'mappings', value: rwroMap }, + { id: 'custom.cellOptions', value: { type: 'color-text' } }, + ]), + ], + [{ displayName: 'Used %', desc: true }], + { show: false, reducer: ['sum'], countRows: false, fields: '' }, + 'Per-pool health, capacity, fragmentation and dedup at a glance.' + ), + + barGaugePanel( + 'Capacity Used per Pool', { h: 9, w: 9, x: 15, y: 6 }, + [prom('zfs_pool_allocated_bytes%s / zfs_pool_size_bytes%s * 100' % [poolSel, poolSel], legend='{{pool}}')], + unit='percent', colorMode='thresholds', steps=usedSteps, display='lcd', + min=0, max=100, decimals=1, + desc='Fill percentage per pool. ZFS performance degrades sharply above ~80%.' + ), + + //=== Capacity Trends ===================================================== + rowPanel('Capacity Trends', 15), + + tsPanel('Pool Used % Over Time', { h: 8, w: 12, x: 0, y: 16 }, + [prom('zfs_pool_allocated_bytes%s / zfs_pool_size_bytes%s * 100' % [poolSel, poolSel], legend='{{pool}}')], + unit='percent', min=0, max=100, steps=usedSteps, threshStyle='dashed', + desc='Capacity fill trend with the 90% danger threshold marked.'), + + tsPanel('Pool Free Space Trend', { h: 8, w: 12, x: 12, y: 16 }, + [ + prom('zfs_pool_free_bytes%s' % poolSel, legend='{{pool}} free', refId='A'), + prom('zfs_pool_allocated_bytes%s' % poolSel, legend='{{pool}} allocated', refId='B'), + ], + unit='bytes', + desc='Absolute free vs. allocated space — useful for eyeballing time-to-full.'), + + tsPanel('Fragmentation', { h: 7, w: 8, x: 0, y: 24 }, + [prom('zfs_pool_fragmentation_ratio%s' % poolSel, legend='{{pool}}')], + unit='percentunit', min=0, max=1, + desc='Free-space fragmentation ratio per pool.'), + + tsPanel('Deduplication Ratio', { h: 7, w: 8, x: 8, y: 24 }, + [prom('zfs_pool_deduplication_ratio%s' % poolSel, legend='{{pool}}')], + unit='none', decimals=2, + desc='Dedup ratio (1.00 = no deduplication).'), + + tsPanel('Freeing & Leaked Bytes', { h: 7, w: 8, x: 16, y: 24 }, + [ + prom('zfs_pool_freeing_bytes%s' % poolSel, legend='{{pool}} freeing', refId='A'), + prom('zfs_pool_leaked_bytes%s' % poolSel, legend='{{pool}} leaked', refId='B'), + ], + unit='bytes', + desc='Space pending free after destroys, and leaked (unaccounted) bytes — both should trend to 0.'), + + //=== Datasets ============================================================ + rowPanel('Datasets', 31), + + tablePanel( + 'Dataset Usage', { h: 11, w: 15, x: 0, y: 32 }, + [ + prom('max by (name, pool) (zfs_dataset_used_bytes%s)' % dsSel, refId='A', instant=true), + prom('max by (name) (zfs_dataset_available_bytes%s)' % dsSel, refId='B', instant=true), + prom('max by (name) (zfs_dataset_referenced_bytes%s)' % dsSel, refId='C', instant=true), + prom('max by (name) (zfs_dataset_logical_used_bytes%s)' % dsSel, refId='D', instant=true), + prom('max by (name) (zfs_dataset_quota_bytes%s)' % dsSel, refId='E', instant=true), + prom('max by (name) (zfs_dataset_used_bytes%s / (zfs_dataset_used_bytes%s + zfs_dataset_available_bytes%s) * 100)' % [dsSel, dsSel, dsSel], refId='F', instant=true), + prom('max by (name) (zfs_dataset_logical_used_bytes%s / zfs_dataset_used_by_dataset_bytes%s)' % [dsSel, dsSel], refId='G', instant=true), + ], + [ + { id: 'joinByField', options: { byField: 'name', mode: 'outer' } }, + { id: 'organize', options: { + excludeByName: { Time: true, 'Time 1': true, 'Time 2': true, 'Time 3': true, 'Time 4': true, 'Time 5': true, 'Time 6': true, 'Time 7': true }, + renameByName: { + name: 'Dataset', + pool: 'Pool', + 'Value #A': 'Used', + 'Value #B': 'Available', + 'Value #C': 'Referenced', + 'Value #D': 'Logical Used', + 'Value #E': 'Quota', + 'Value #F': 'Fill %', + 'Value #G': 'Compression', + }, + indexByName: { name: 0, pool: 1, 'Value #A': 2, 'Value #F': 3, 'Value #B': 4, 'Value #C': 5, 'Value #D': 6, 'Value #E': 7, 'Value #G': 8 }, + } }, + { id: 'sortBy', options: { fields: '', sort: [{ field: 'Used', desc: true }] } }, + ], + [ + override('Dataset', [{ id: 'custom.width', value: 240 }]), + override('Pool', [{ id: 'custom.width', value: 80 }]), + override('Used', [ + { id: 'unit', value: 'bytes' }, + { id: 'custom.cellOptions', value: { type: 'gauge', mode: 'gradient', valueDisplayMode: 'text' } }, + ]), + override('Fill %', [ + { id: 'unit', value: 'percent' }, + { id: 'decimals', value: 1 }, + { id: 'custom.cellOptions', value: { type: 'color-background', mode: 'gradient' } }, + { id: 'thresholds', value: { mode: 'absolute', steps: usedSteps } }, + ]), + override('Compression', [ + { id: 'unit', value: 'none' }, + { id: 'decimals', value: 2 }, + { id: 'custom.cellOptions', value: { type: 'color-text' } }, + { id: 'thresholds', value: { mode: 'absolute', steps: [{ color: 'text', value: null }, { color: 'green', value: 1.1 }] } }, + ]), + ], + [{ displayName: 'Used', desc: true }], + { show: true, reducer: ['sum'], countRows: false, fields: ['Used', 'Logical Used'] }, + 'Per-dataset space accounting, quota fill and compression ratio.' + ), + + barGaugePanel( + 'Top Datasets by Used Space', { h: 11, w: 9, x: 15, y: 32 }, + [prom('topk(15, max by (name) (zfs_dataset_used_bytes%s))' % dsSel, legend='{{name}}')], + unit='bytes', display='gradient', + desc='Largest datasets by space consumed (incl. descendants).' + ), + + //=== Dataset Trends ====================================================== + rowPanel('Dataset Trends', 43), + + tsPanel('Dataset Used Over Time', { h: 8, w: 12, x: 0, y: 44 }, + [prom('topk(20, max by (name) (zfs_dataset_used_bytes%s))' % dsSel, legend='{{name}}')], + unit='bytes', + desc='Growth of the top datasets over the selected window.'), + + tsPanel('Quota Fill', { h: 8, w: 12, x: 12, y: 44 }, + [prom('zfs_dataset_used_bytes%s / (zfs_dataset_quota_bytes%s > 0) * 100' % [dsSel, dsSel], legend='{{name}}')], + unit='percent', min=0, max=100, steps=usedSteps, threshStyle='dashed', + desc='Used space as a percentage of quota, for datasets that have a quota set.'), + + barGaugePanel( + 'Compression Ratio (logical / physical)', { h: 8, w: 12, x: 0, y: 52 }, + [prom('topk(15, max by (name) (zfs_dataset_logical_used_bytes%s / zfs_dataset_used_by_dataset_bytes%s))' % [dsSel, dsSel], legend='{{name}}')], + unit='none', colorMode='thresholds', steps=compSteps, display='gradient', decimals=2, + desc="How much each dataset's data shrinks on disk. 1.00 = incompressible." + ), + + tsPanel('Written Since Last Snapshot', { h: 8, w: 12, x: 12, y: 52 }, + [prom('topk(15, max by (name) (zfs_dataset_written_bytes%s))' % dsSel, legend='{{name}}')], + unit='bytes', fill=25, interp='linear', + desc='Referenced bytes written since the previous snapshot — a proxy for recent write activity (resets on snapshot).'), + + //=== Exporter Health ===================================================== + rowPanel('Exporter Health', 60), + + statPanel('Collectors Healthy', { h: 5, w: 4, x: 0, y: 61 }, + [prom('count(zfs_scrape_collector_success{instance=~"$instance"} == 1) or vector(0)')], + unit='none', graphMode='none', steps=healthyStep, + desc='ZFS exporter sub-collectors reporting success.'), + + statPanel('Collector Status', { h: 5, w: 8, x: 4, y: 61 }, + [prom('zfs_scrape_collector_success{instance=~"$instance"}', legend='{{collector}}')], + unit='none', graphMode='none', mappings=okFailMap, + colorMode='background', textMode='value_and_name', + desc='Success/failure per exporter collector.'), + + tsPanel('Collector Scrape Duration', { h: 5, w: 12, x: 12, y: 61 }, + [prom('zfs_scrape_collector_duration_seconds{instance=~"$instance"}', legend='{{collector}}')], + unit='s', + desc='Time each collector takes to gather metrics.'), + + statPanel('Goroutines', { h: 4, w: 4, x: 0, y: 66 }, + [prom('go_goroutines{job="zfs_exporter",instance=~"$instance"}')], + unit='none'), + + statPanel('Heap In Use', { h: 4, w: 4, x: 4, y: 66 }, + [prom('go_memstats_heap_inuse_bytes{job="zfs_exporter",instance=~"$instance"}')], + unit='bytes'), + + statPanel('Go Version', { h: 4, w: 4, x: 8, y: 66 }, + [prom('go_info{job="zfs_exporter",instance=~"$instance"}', legend='{{version}}')], + unit='none', graphMode='none', textMode='name', colorMode='none'), + + tsPanel('Exporter Memory', { h: 4, w: 12, x: 12, y: 66 }, + [ + prom('go_memstats_heap_inuse_bytes{job="zfs_exporter",instance=~"$instance"}', legend='heap in-use', refId='A'), + prom('go_memstats_stack_inuse_bytes{job="zfs_exporter",instance=~"$instance"}', legend='stack in-use', refId='B'), + prom('go_memstats_sys_bytes{job="zfs_exporter",instance=~"$instance"}', legend='sys', refId='C'), + ], + unit='bytes', calcs=['lastNotNull', 'max']), +]; + +// Assign stable, unique panel IDs. +local panelsWithIds = std.mapWithIndex(function(i, p) p + { id: i + 1 }, panels); + +// --------------------------------------------------------------------------- +// Dashboard +// --------------------------------------------------------------------------- +dashboard.new('ZFS Exporter') ++ dashboard.withUid('zfs-exporter') ++ dashboard.withTags(['zfs', 'storage', 'freebsd']) ++ dashboard.withRefresh('30s') ++ dashboard.withTimezone('browser') ++ dashboard.withVariables([datasourceVar, instanceVar, poolVar, datasetVar]) ++ dashboard.withPanels(panelsWithIds) ++ { graphTooltip: 1, time: { from: 'now-24h', to: 'now' } } diff --git a/modules/services/monitoring/grafana.nix b/modules/services/monitoring/grafana.nix @@ -19,6 +19,21 @@ ''; }; + grafonnet = pkgs.fetchFromGitHub { + owner = "grafana"; + repo = "grafonnet"; + rev = "7380c9c64fb973f34c3ec46265621a2b0dee0058"; + sha256 = "0pablj8mw48hcqqzkgs5cklbcjgma0qksmd45dg2j3az9zzdjbar"; + }; + grafonnetPath = pkgs.runCommand "grafonnet-jpath" {} '' + mkdir -p $out/github.com/grafana + ln -s ${grafonnet} $out/github.com/grafana/grafonnet + ''; + mkGrafonnet = name: src: + pkgs.runCommand name {nativeBuildInputs = [pkgs.go-jsonnet];} '' + jsonnet -J ${grafonnetPath} ${src} > $out + ''; + dashboards = pkgs.linkFarm "grafana-dashboards" (map fetchDashboard [ { name = "node_exporter.json"; @@ -49,20 +64,21 @@ ++ [ { name = "caddy.json"; - path = let - src = pkgs.fetchFromGitHub { - owner = "grafana"; - repo = "jsonnet-libs"; - rev = "2e9c7f4a276f6f6a9bc3f994f2b8804555f31bb9"; - sha256 = "12ikl9p9s6sv1qy5377dk95naz7ckwd767wgq4503xbaq0nq3k11"; - }; - in + path = pkgs.runCommand "caddy.json" { nativeBuildInputs = [pkgs.go-jsonnet]; } '' - jsonnet -J ${src} ${src}/caddy-mixin/dashboards/caddy-overview.jsonnet > $out + jsonnet ${../../mixins/monitoring/caddy.jsonnet} > $out ''; } + { + name = "zfs_exporter.json"; + path = mkGrafonnet "zfs_exporter.json" ../../mixins/monitoring/zfs_exporter.jsonnet; + } + { + name = "h2o.json"; + path = mkGrafonnet "h2o.json" ../../mixins/monitoring/h2o.jsonnet; + } ] ++ lib.optional config.services.haravara.corpus.enable (fetchDashboard { name = "corpus.json";