Skip to content
Snippets Groups Projects
Commit ad64c1b3 authored by Ben Kochie's avatar Ben Kochie
Browse files

Tune node alerts

* Increase time to alert for 24h NodeFilesystemFullSoon to avoid flaps.
* Re-add NodeFilesystemFull 20% warning alert.
* Tune NodeFilesystemFull to 15min to avoid flaps.
* Remove `humanize` from percentages.
* Ignore backup snapshot mountpoints in `predict_linear()` alerts.
parent aed9cdd1
No related branches found
No related tags found
No related merge requests found
# Use 6 hours of history to predict 24 hours into the future.
ALERT NodeFilesystemFullSoon
IF predict_linear(node_filesystem_avail{fstype=~"ext.|xfs"}[6h], 24*60*60) < 0
FOR 1h
IF predict_linear(node_filesystem_avail{fstype=~"ext.|xfs",mountpoint!~"/mnt/(backup-block-for|remote).*"}[6h], 24*60*60) < 0
FOR 2h
LABELS {
severity = "crit",
}
Loading
Loading
@@ -13,7 +13,7 @@ ALERT NodeFilesystemFullSoon
 
# Use 1 hour of history to predict 4 hours into the future.
ALERT NodeFilesystemFullSoon
IF predict_linear(node_filesystem_avail{fstype=~"ext.|xfs"}[1h], 4*60*60) < 0
IF predict_linear(node_filesystem_avail{fstype=~"ext.|xfs",mountpoint!~"/mnt/(backup-block-for|remote).*"}[1h], 4*60*60) < 0
FOR 10m
LABELS {
severity = "crit",
Loading
Loading
@@ -24,15 +24,27 @@ ALERT NodeFilesystemFullSoon
runbook = "troubleshooting/filesystem_alerts.md",
}
 
ALERT NodeFilesystemFull
IF ((node_filesystem_avail{fstype=~"ext4|xfs"} / node_filesystem_size{fstype=~"ext4|xfs"}) * 100) <= 20
FOR 15m
LABELS {
severity = "warn",
}
ANNOTATIONS {
summary = "Filesystem is full",
description = "Filesystem '{{ $labels.mountpoint }}' on {{ $labels.instance }} is full: {{ $value }}%",
runbook = "troubleshooting/filesystem_alerts.md",
}
ALERT NodeFilesystemFull
IF ((node_filesystem_avail{fstype=~"ext4|xfs"} / node_filesystem_size{fstype=~"ext4|xfs"}) * 100) <= 10
FOR 1m
FOR 15m
LABELS {
severity = "crit",
}
ANNOTATIONS {
summary = "Filesystem is full",
description = "Filesystem '{{ $labels.mountpoint }}' on {{ $labels.instance }} is full: {{ $value | humanize }}%",
description = "Filesystem '{{ $labels.mountpoint }}' on {{ $labels.instance }} is full: {{ $value }}%",
runbook = "troubleshooting/filesystem_alerts.md",
}
 
Loading
Loading
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment