commit 417da0ad0a8e8c4444740472d0cd9af98a073cae
author Kevin Lubick <kjlubick@google.com> Wed Aug 21 14:13:39 2019 -0400
committer Kevin Lubick <kjlubick@google.com> Wed Aug 21 18:24:29 2019 +0000
tree 719296a19114472f6d19acd38ddadd22eb29bf5e
parent c47114a11323680463e5d6fda07ba16fe2fcd45c

[gold] make polling ingestion alerts less touchy

Most instances backed off to polling every 60 minutes, so we don't need
to alert until it's been much longer than that.

NOTRY=true

Change-Id: I3c9ae91e039df5994044848aaf23df6bd8e8d940
Reviewed-on: https://skia-review.googlesource.com/c/buildbot/+/236119
Reviewed-by: Joe Gregorio <jcgregorio@google.com>
Auto-Submit: Kevin Lubick <kjlubick@google.com>

promk/prometheus/alerts_general.yml

diff --git a/promk/prometheus/alerts_general.yml b/promk/prometheus/alerts_general.yml
index a0eb584..b011aa5 100644
--- a/promk/prometheus/alerts_general.yml
+++ b/promk/prometheus/alerts_general.yml
@@ -542,7 +542,7 @@
       description: '{{ $labels.app }} has not checked for expired ignore rules in a while. https://skia.googlesource.com/buildbot/+/refs/heads/master/golden/docs/PROD.md#goldignoremonitoring'
 
   - alert: GoldPollingIngestionStalled
-    expr: liveness_gold_bt_s{metric="since-last-run"} > 60*60
+    expr: liveness_gold_bt_s{metric="since-last-run"} > 150*60
     labels:
       category: infra
       severity: warning