Merge branch 'bjk/alerts' into 'master'

Update githost alerting rules

See merge request !1

alerts/githost.rules

-ALERT instance_20_disk
-  IF ((node_filesystem_avail{fstype=~"ext4|xfs"} / node_filesystem_size{fstype=~"ext4|xfs"}) * 100) < 20 AND ((node_filesystem_avail{fstype=~"ext4|xfs"} / node_filesystem_size{fstype=~"ext4|xfs"}) * 100) > 10
-  FOR 1m
-  LABELS { severity = "warn" }
+# Use 6 hours of history to predict 24 hours into the future.
+ALERT NodeFilesystemFullSoon
+  IF predict_linear(node_filesystem_avail{fstype=~"ext.|xfs"}[6h], 24*60*60) < 0
+  FOR 1h
+  LABELS {
+    severity = "crit",
+  }
+  ANNOTATIONS {
+    summary = "Filesystem is filling up",
+    description = "Filesystem '{{ $labels.mountpoint }}' on {{ $labels.instance }} is filling up and will be full in the next 24 hours",
+    runbook = "troubleshooting/filesystem_alerts.md",
+  }
+
+# Use 1 hour of history to predict 4 hours into the future.
+ALERT NodeFilesystemFullSoon
+  IF predict_linear(node_filesystem_avail{fstype=~"ext.|xfs"}[1h], 4*60*60) < 0
+  FOR 10m
+  LABELS {
+    severity = "crit",
+  }
   ANNOTATIONS {
-    title = "Low disk space left on {{ $labels.mountpoint }} on {{ $labels.instance }}: {{ $value | humanize }}%",
-    description = "",
-    runbook="troubleshooting/filesystem_alerts.md",
+    summary = "Filesystem is filling up",
+    description = "Filesystem '{{ $labels.mountpoint }}' on {{ $labels.instance }} is filling up and will be full in the next 4 hours",
+    runbook = "troubleshooting/filesystem_alerts.md",
   }
-ALERT instance_10_disk
+ALERT NodeFilesystemFull
   IF ((node_filesystem_avail{fstype=~"ext4|xfs"} / node_filesystem_size{fstype=~"ext4|xfs"}) * 100) <= 10
   FOR 1m
-  LABELS { severity = "crit" }
+  LABELS {
+    severity = "crit",
+  }
   ANNOTATIONS {
-    title = "Really low disk space left on {{ $labels.mountpoint }} on {{ $labels.instance }}: {{ $value | humanize }}%",
-    description = "",
-    runbook="troubleshooting/filesystem_alerts.md"
-    }
-ALERT down
+    summary = "Filesystem is full",
+    description = "Filesystem '{{ $labels.mountpoint }}' on {{ $labels.instance }} is full: {{ $value | humanize }}%",
+    runbook = "troubleshooting/filesystem_alerts.md",
+  }
+ALERT NodeFilesystemReadOnly
+  IF node_filesystem_readonly{fstype=~"ext4|xfs"} == 1
+  FOR 5m
+  LABELS {
+    severity = "crit",
+  }
+  ANNOTATIONS {
+    summary = "Filesystem is read-only",
+    description = "Filesystem '{{ $labels.mountpoint }}' on {{ $labels.instance }} is read-only",
+    runbook = "troubleshooting/filesystem_alerts.md",
+  }
+
+ALERT InstanceDown
   IF up == 0
   FOR 5m
-  LABELS { severity = "crit" }
+  LABELS {
+    severity = "crit",
+  }
   ANNOTATIONS {
     summary = "Instance {{ $labels.instance }} is down.",
     description = "{{ $labels.instance }} is reporting as down, prometheus cannot reach the exporter.",
-    runbook="troubleshooting/instance_down.md"
-  }
\ No newline at end of file
+    runbook = "troubleshooting/instance_down.md",
+  }