Merge branch 'bjk/alerts_tune' into 'master'

Tune node alerts

See merge request !2

alerts/githost.rules

 # Use 6 hours of history to predict 24 hours into the future.
 ALERT NodeFilesystemFullSoon
-  IF predict_linear(node_filesystem_avail{fstype=~"ext.|xfs"}[6h], 24*60*60) < 0
-  FOR 1h
+  IF predict_linear(node_filesystem_avail{fstype=~"ext.|xfs",mountpoint!~"/mnt/(backup-block-for|remote).*"}[6h], 24*60*60) < 0
+  FOR 2h
   LABELS {
     severity = "crit",
   }
@@ -13,7 +13,7 @@ ALERT NodeFilesystemFullSoon
 # Use 1 hour of history to predict 4 hours into the future.
 ALERT NodeFilesystemFullSoon
-  IF predict_linear(node_filesystem_avail{fstype=~"ext.|xfs"}[1h], 4*60*60) < 0
+  IF predict_linear(node_filesystem_avail{fstype=~"ext.|xfs",mountpoint!~"/mnt/(backup-block-for|remote).*"}[1h], 4*60*60) < 0
   FOR 10m
   LABELS {
     severity = "crit",
@@ -24,15 +24,27 @@ ALERT NodeFilesystemFullSoon
     runbook = "troubleshooting/filesystem_alerts.md",
   }
+ALERT NodeFilesystemFull
+  IF ((node_filesystem_avail{fstype=~"ext4|xfs"} / node_filesystem_size{fstype=~"ext4|xfs"}) * 100) <= 20
+  FOR 15m
+  LABELS {
+    severity = "warn",
+  }
+  ANNOTATIONS {
+    summary = "Filesystem is full",
+    description = "Filesystem '{{ $labels.mountpoint }}' on {{ $labels.instance }} is full: {{ $value }}%",
+    runbook = "troubleshooting/filesystem_alerts.md",
+  }
+
 ALERT NodeFilesystemFull
   IF ((node_filesystem_avail{fstype=~"ext4|xfs"} / node_filesystem_size{fstype=~"ext4|xfs"}) * 100) <= 10
-  FOR 1m
+  FOR 15m
   LABELS {
     severity = "crit",
   }
   ANNOTATIONS {
     summary = "Filesystem is full",
-    description = "Filesystem '{{ $labels.mountpoint }}' on {{ $labels.instance }} is full: {{ $value | humanize }}%",
+    description = "Filesystem '{{ $labels.mountpoint }}' on {{ $labels.instance }} is full: {{ $value }}%",
     runbook = "troubleshooting/filesystem_alerts.md",
   }