alertserver/alerts.cfg

# This file defines alerts to be triggered by the server.

#
# AlertServer should tolerate no errors.


[[rule]]
name = "AlertServer Errors"
message = "The Error rate for the alertserver is too high, please check the logs."
database = "skmetrics"
query = "SELECT derivative(mean(value)) FROM \"logs\" WHERE \"name\" = 'alertserver' AND level = 'ERROR' AND host = 'skia-alerts' AND  time > now() - 4m group by time(2m) order by time desc limit 1"
category = "infra"
conditions = ["x > 0"]
actions = ["Email(alerts@skia.org)"]
auto-dismiss = true
nag = "24h"


#
# General
#

[[rule]]
name = "Too many Go routines in %(app)s"
message = "Too many Go routines in %(app)s running on $(host)s"
database = "skmetrics"
query = "SELECT mean(value) FROM \"runtime-metrics\" WHERE metric='num-goroutine' AND time > now() - 10m GROUP BY app,host"
category = "infra"
conditions = ["x > 3000"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Root Disk Space on %(host)s"
message = "Free space has fallen below 1GB on %(host)s root drive."
database = "graphite"
query = "SELECT mean(value) FROM \"df_complex-free\" WHERE resource = 'df-root' AND time > now() - 5m GROUP BY host"
category = "infra"
conditions = ["x <= 1e9"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Low Disk Space on %(host)s"
message = "Free space has fallen below 10GB on %(host)s drive %(resource)s."
database = "graphite"
query = "SELECT mean(value) FROM \"df_complex-free\" WHERE resource = 'df-mnt-pd0' OR resource = 'df-mnt-ssd0' AND time > now() - 5m GROUP BY host,resource"
category = "infra"
conditions = ["x <= 1e10"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Probe failed %(probename)s"
message = "Endpoint %(probename)s has failed to respond in at least 10 minutes. See https://github.com/google/skia-buildbot/search?q=%(probename)s+filename%3Aprobers.json for the endpoing URL."
database = "skmetrics"
query = "select mean(value) from \"prober\" where type='failure' AND time > now() - 10m group by probename;"
category = "infra"
conditions = ["x >= 1"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

#
# SkiaPerf and SkiaGold
#

[[rule]]
name = "Perf Alerts"
message = "At least one perf alert has been found. Please visit https://perf.skia.org/alerts/ to triage."
database = "skmetrics"
query = "select value from \"perf.clustering.untriaged\" WHERE app = 'skiaperf' AND host = 'skia-perf' order by time desc limit 1"
category = "Perf"
conditions = ["x > 0"]
actions = ["Email(alerts@skia.org)"]
auto-dismiss = true
nag = "24h"

[[rule]]
name = "Gold Alert (GM)"
message = "At least one untriaged GM has been found. Please visit https://gold.skia.org/ to triage."
database = "skmetrics"
query = "select value from \"gold.status.by-corpus\" WHERE type='untriaged' AND app = 'skiacorrectness' AND host = 'skia-gold-prod' order by time desc limit 1"
category = "Gold"
conditions = ["x > 0"]
actions = ["Email(alerts@skia.org)"]
auto-dismiss = true
nag = "24h"

[[rule]]
name = "Expired Ignores (Gold)"
message = "At least one expired ignore rule has been found. Please visit https://gold.skia.org/ignores to delete or extend."
database = "skmetrics"
query = "select value from \"gold.num-expired-ignore-rules\" WHERE app = 'skiacorrectness' AND host = 'skia-gold-prod' order by time desc limit 1"
category = "Gold"
conditions = ["x > 0"]
actions = ["Email(alerts@skia.org)"]
auto-dismiss = true
nag = "24h"

[[rule]]
name = "Ingestion Failure (Gold)"
message = """At least two rounds of gold ingestion have failed back to back.

See: https://mon.skia.org/dashboard/db/ingestion
"""
database = "skmetrics"
query = "SELECT mean(value) FROM liveness WHERE app='gold_ingestion' AND \"name\"='gold' AND source='poll' AND metric='since-last-run' AND time>now() - 10m"
category = "infra"
conditions = ["x >= 750"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Ignore Monitoring Failure (Gold)"
message = "At least two rounds of monitoring for expired ignore rules have failed back to back."
database = "skmetrics"
query = "select mean(value) from \"liveness\" where \"name\"='gold.expired-ignore-rules-monitoring' AND app='skiacorrectness' AND host='skia-gold-prod' AND  time > now() - 10m"
category = "infra"
conditions = ["x >= 200"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Gold Error Rate"
message = "The error rate for Gold is too high."
database = "skmetrics"
query = "select derivative(value, 10m) from \"logs\" WHERE \"name\"='skiacorrectness' AND level='ERROR' AND host='skia-gold-prod' order by time desc limit 1"
category = "infra"
conditions = ["x >= 1"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = false
nag = "1h"

[[rule]]
name = "Gold Ingestion Error Rate"
message = """The error rate for Gold Ingestion is too high.

See: https://mon.skia.org/dashboard/db/ingestion
"""
database = "skmetrics"
query = "SELECT  MAX(value) FROM ingestion WHERE ingester='gold' AND \"metric\"='errors' AND source='poll' AND time>now() - 10m"
category = "infra"
conditions = ["x >= 2"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Gold Trybot Ingestion Error Rate"
message = """The error rate for Gold Trybot Ingestion is too high.

See: https://mon.skia.org/dashboard/db/ingestion
"""
database = "skmetrics"
query = "SELECT  MAX(value) FROM ingestion WHERE ingester='gold-trybot' AND \"metric\"='errors' AND source='poll' AND time>now() - 10m"
category = "infra"
conditions = ["x >= 2"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "PDF Ingestion Error Rate"
message = "The error rate for PDF Ingestion is too high."
database = "skmetrics"
query = "SELECT  MAX(value) FROM ingestion WHERE ingester='pdf' AND \"metric\"='errors' AND source='poll' AND time>now() - 30m"
category = "infra"
conditions = ["x >= 1"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = false
nag = "1h"

#
# Skia Status
#

[[rule]]
name = "Skia Status Latency (JSON endpoint)"
message = "The JSON endpoint at https://status.skia.org/json/skia/commits/ took more than 300ms to respond."
database = "skmetrics"
query = "select mean(value) from \"prober\" where type='latency' AND probename='skiastatus_json' AND time > now() - 10m;"
category = "infra"
conditions = ["x / 1000000 > 300"]
actions = ["Email(infra-alerts@skia.org)", "Email(borenet@google.com)"]
auto-dismiss = true
nag = "1h"

#
# Skia Alerts
#

[[rule]]
name = "Skia Alerts Latency (JSON endpoint)"
message = "The JSON endpoint at https://alerts.skia.org/json/alerts/ took more than 200ms to respond."
database = "skmetrics"
query = "select mean(value) from \"prober\" where type='latency' AND probename='skiaalerts_json' AND time > now() - 10m;"
category = "infra"
conditions = ["x / 1000000 > 200"]
actions = ["Email(infra-alerts@skia.org)", "Email(borenet@google.com)"]
auto-dismiss = true
nag = "1h"

#
# Skia AutoRoll Bot
#

[[rule]]
name = "AutoRoll Failed"
message = "The last DEPS roll attempt for Skia failed."
database = "skmetrics"
query = "select value from \"autoroll.last-roll-result\" where \"child-path\"='src/third_party/skia' order by time desc limit 1"
category = "Skia"
conditions = ["x == 0"]
actions = ["Email(alerts@skia.org)"]
auto-dismiss = true
nag = "2h"

[[rule]]
name = "No AutoRolls Landed for 24h (%(child-path)s)"
message = "The last-landed AutoRoll for %(child-path)s was over 24h ago."
database = "skmetrics"
query = "select value from \"liveness\" where \"name\"='last-autoroll-landed' AND \"child-path\"='src/third_party/skia' order by time desc limit 1"
category = "Skia"
conditions = ["x > 24 * 60 * 60 * 1000000000"]
actions = ["Email(alerts@skia.org)"]
auto-dismiss = true
nag = "2h"

[[rule]]
name = "Skia AutoRoll Latency (JSON endpoint)"
message = "The JSON endpoint at https://autoroll.skia.org/json/status took more than 200ms to respond."
database = "skmetrics"
query = "select mean(value) from \"prober\" where type='latency' AND probename='skiaautoroll_json' AND time > now() - 10m;"
category = "infra"
conditions = ["x / 1000000 > 200"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "AutoRoll Error Rate"
message = "The error rate for autoroll is too high. http://104.154.112.12:10115/file_server/autoroll.ERROR"
database = "skmetrics"
query = "select derivative(value, 10m) from \"logs\" where \"name\"='autoroll' AND level='ERROR' AND host='skia-autoroll' order by time desc limit 1"
category = "infra"
conditions = ["x >= 5"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = false
nag = "1h"

#
# Skia Prober
#

[[rule]]
name = "Probe Failure (probes)"
message = "Probing has failed to run in at least 10 minutes."
database = "skmetrics"
query = "select value from \"liveness\" where \"name\" = 'probes' AND app='probeserver' AND host='skia-monitoring' order by time desc limit 1"
category = "infra"
conditions = ["x >= 600"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Probe Failure (issue-tracker)"
message = "Ingesting issue tracker issued has failed to run in at least 30 minutes."
database = "skmetrics"
query = "select mean(value) from \"liveness\" where \"name\" = 'issue-tracker' AND app='probeserver' AND host='skia-monitoring' AND time > now() - 10m"
category = "infra"
conditions = ["x >= 1800"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"


#
# Datahopper
#

[[rule]]
name = "Datahopper Buildbot Ingestion Stalled (client.skia)"
message = "Buildbot Ingestion in Datahopper has failed to run in at least 5 minutes for client.skia."
database = "skmetrics"
query = "select mean(value) from liveness where app='datahopper' AND host='skia-datahopper2' AND \"name\"='buildbot-ingest' AND master='client.skia' AND time > now() - 5m"
category = "infra"
conditions = ["x >= 150"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Datahopper Buildbot Ingestion Stalled (client.skia.android)"
message = "Buildbot Ingestion in Datahopper has failed to run in at least 5 minutes for client.skia.android."
database = "skmetrics"
query = "select mean(value) from liveness where app='datahopper' AND host='skia-datahopper2' AND \"name\"='buildbot-ingest' AND master='client.skia.android' AND time > now() - 5m"
category = "infra"
conditions = ["x >= 150"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Datahopper Buildbot Ingestion Stalled (client.skia.compile)"
message = "Buildbot Ingestion in Datahopper has failed to run in at least 5 minutes for client.skia.compile."
database = "skmetrics"
query = "select mean(value) from liveness where app='datahopper' AND host='skia-datahopper2' AND \"name\"='buildbot-ingest' AND master='client.skia.compile' AND time > now() - 5m"
category = "infra"
conditions = ["x >= 150"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Datahopper Buildbot Ingestion Stalled (client.skia.fyi)"
message = "Buildbot Ingestion in Datahopper has failed to run in at least 5 minutes for client.skia.fyi."
database = "skmetrics"
query = "select mean(value) from liveness where app='datahopper' AND host='skia-datahopper2' AND \"name\"='buildbot-ingest' AND master='client.skia.fyi' AND time > now() - 5m"
category = "infra"
conditions = ["x >= 150"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Datahopper Buildbot Ingestion Too Slow"
message = "Buildbot ingestion time is more than 2 minutes."
database = "skmetrics"
query = "select mean(value) from timer where \"name\"='func-timer' and package='go.skia.org/infra/go/buildbot' and func='ingestNewBuilds' and app='datahopper' AND host='skia-datahopper2' AND time > now() - 5m"
category = "infra"
conditions = ["x >= 120 * 1000 * 1000 * 1000"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Datahopper Buildbot Start to Ingestion Latency Too High"
message = "Average time between build start and first ingestion is more than 1 minute."
database = "skmetrics"
query = "select mean(value) from \"buildbot.ingest.latency\" where app='datahopper' AND host='skia-datahopper2' AND master!='client.skia.fake_internal' AND time > now() - 24h"
category = "infra"
conditions = ["x >= 60 * 1000 * 1000 * 1000"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "3h"

[[rule]]
name = "Datahopper Error Rate"
message = "The error rate for datahopper is too high. http://104.154.112.116:10115/file_server/datahopper.ERROR"
database = "skmetrics"
query = "select derivative(value, 10m) from \"logs\" where \"name\"='datahopper' AND level='ERROR' AND host='skia-datahopper2' order by time desc limit 1"
category = "infra"
conditions = ["x >= 5"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = false
nag = "1h"

[[rule]]
name = "Buildbot DB backup"
message = "The last buildbot DB backup was more than 25 hours ago."
database = "skmetrics"
query = "select mean(value) from \"liveness\" where app='datahopper' AND host='skia-datahopper2' AND \"name\" = 'last-buildbot-db-backup' AND time > now() - 1h"
category = "infra"
conditions = ["x > 25 * 60 * 60 * 1000 * 1000 * 1000"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "6h"

#
# Buildbots
#

[[rule]]
name = "Buildslave %(buildslave)s Disconnected"
message = """Buildslave %(buildslave)s is not connected to %(master)s

Link: https://uberchromegw.corp.google.com/i/%(master)s/buildslaves/%(buildslave)s
Host info: https://status.skia.org/hosts?filter=%(buildslave)s
"""
database = "skmetrics"
query = "select mean(value) from \"buildbot.buildslaves.connected\" where host='skia-datahopper2' AND app='datahopper' AND time > now() - 10m group by buildslave,master"
category = "infra"
conditions = ["x < 1"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Possibly hung buildslave (%(buildslave)s)"
message = """Possibly hung buildslave (%(buildslave)s)

Step "%(step)s" has been running for over 3h:
https://uberchromegw.corp.google.com/i/%(master)s/builders/%(builder)s/builds/%(number)s
Dashboard: https://status.skia.org/buildbots?botGrouping=buildslave&filterBy=buildslave&include=%%5E%(buildslave)s%%24&tab=builds
Host info: https://status.skia.org/hosts?filter=%(buildslave)s"""
database = "skmetrics"
query = "select mean(value) from \"buildbot.buildsteps.running-time\" where builder != 'Housekeeper-Nightly-RecreateSKPs_Canary' AND builder != 'Housekeeper-Weekly-RecreateSKPs' AND builder != 'Linux Builder' AND builder != 'Mac Builder' AND builder != 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Valgrind' AND builder != 'Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind' AND builder != 'Win Builder' AND step != 'steps' AND step !~ /wait for device/ AND time > now() - 2m group by builder,master,number,buildslave,step"
category = "infra"
conditions = ["x > 3 * 60 * 60 * 1000 * 1000 * 1000"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "The Android device for %(buildslave)s appears to be disconnected."
message = """The Android device for %(buildslave)s appears to be disconnected.

Build: https://uberchromegw.corp.google.com/i/%(master)s/builders/%(builder)s/builds/%(number)s
Dashboard: https://status.skia.org/buildbots?botGrouping=buildslave&filterBy=buildslave&include=%%5E%(buildslave)s%%24&tab=builds
Host info: https://status.skia.org/hosts?filter=%(buildslave)s"""
database = "skmetrics"
query = "select mean(value) from \"buildbot.buildsteps.running-time\" where step != 'steps' AND step =~ /wait for device/ AND time > now() - 2m group by builder,master,number,buildslave,step"
category = "infra"
conditions = ["x > 30 * 60 * 1000 * 1000 * 1000"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "The Android device for %(buildslave)s has been charging a long time."
message = """The Android device for %(buildslave)s has been charging for longer than 3 hours.  Perhaps it has been disconnected or needs a new power hub?

Build: https://uberchromegw.corp.google.com/i/%(master)s/builders/%(builder)s/builds/%(number)s
Dashboard: https://status.skia.org/buildbots?botGrouping=buildslave&filterBy=buildslave&include=%%5E%(buildslave)s%%24&tab=builds
Host info: https://status.skia.org/hosts?filter=%(buildslave)s"""
database = "skmetrics"
query = "select mean(value) from \"buildbot.buildsteps.running-time\" where step != 'steps' AND step =~ /wait for charge/ AND time > now() - 2m group by builder,master,number,buildslave,step"
category = "infra"
conditions = ["x > 180 * 60 * 1000 * 1000 * 1000"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "Buildslave %(buildslave)s failure rate too high"
message = """%(buildslave)s has been failing consistently for the past 3 hours.

Link: https://uberchromegw.corp.google.com/i/%(master)s/buildslaves/%(buildslave)s/
Host info: https://status.skia.org/hosts?filter=%(buildslave)s"""
database = "skmetrics"
query = "select mean(value), sum(value) from \"buildbot.builds.failure-status\" where app='datahopper' and host='skia-datahopper2' and time > now() - 3h group by buildslave, master"
category = "infra"
conditions = ["x == 1", "y > 1"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

#
# Buildbot Scheduler
#

[[rule]]
name = "Buildbot Scheduler Failing"
message = "The buildbot scheduler has failed to schedule builds for the last 10 minutes."
database = "skmetrics"
query = "select mean(value) from liveness where app='buildbot_scheduler' and host='skia-build-scheduler' and \"name\"='time-since-last-successful-scheduling' and time > now() - 5m group by app, host"
category = "infra"
conditions = ["x > 10 * 60 * 1000 * 1000 * 1000"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

#
# CTFE
#

[[rule]]
name = "CTFE Pending Task Count"
message = "There are a lot of pending tasks. https://ct.skia.org/queue/"
database = "graphite"
query = "select mean(value) from /^num-pending-tasks.value$/ where app='ctfe' AND host='skia-ctfe' AND time > now() - 10m;"
category = "infra"
conditions = ["x >= 100"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "CTFE Pending Task Status"
message = "A task has been waiting to be executed for a while and it's still not started. https://ct.skia.org/queue/"
database = "graphite"
query = "select mean(value) from /^oldest-pending-task-status.value$/ where app='ctfe' AND host='skia-ctfe' AND time > now() - 10m;"
category = "infra"
conditions = ["x >= 2"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "CTFE Last Metrics Update"
message = "No recent update from the CTFE metrics goroutine."
database = "graphite"
query = "select count(value) from /^oldest-pending-task-status.value$/ where app='ctfe' AND host='skia-ctfe' AND time > now() - 10m;"
category = "infra"
conditions = ["x < 1"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "CTFE Number of Goroutines"
message = "There are more goroutines running than expected for CTFE."
database = "graphite"
query = "select mean(value) from /^NumGoroutine.value$/ where app='ctfe' AND host='skia-ctfe' AND time > now() - 10m;"
category = "infra"
conditions = ["x >= 100"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "CTFE Error Rate"
message = "The error rate for CTFE is too high. http://104.154.112.110:10115/ctfe.ERROR"
database = "skmetrics"
query = "select derivative(value, 10m) from \"logs\" where \"name\"='ctfe' AND level='ERROR' AND host='skia-ctfe' order by time desc limit 1"
category = "infra"
conditions = ["x >= 5"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = false
nag = "1h"

[[rule]]
name = "CT Poller Health Check"
message = "CT poller health check is failing. https://uberchromegw.corp.google.com/i/skia-ct-master/poller.ERROR?page_y=end"
database = "graphite"
query = "select mean(value) from /^healthy.value$/ where app='ct-poller' AND host='build101-m5' AND time > now() - 10m;"
category = "infra"
conditions = ["x < 0.01"]
actions = ["Email(infra-alerts@skia.org)", "Email(rmistry@google.com)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "CT Poller Last Metrics Update"
message = "No recent update from the CT Poller metrics goroutine."
database = "graphite"
query = "select count(value) from /^healthy.value$/ where app='ct-poller' AND host='build101-m5' AND time > now() - 10m;"
category = "infra"
conditions = ["x < 1"]
actions = ["Email(infra-alerts@skia.org)", "Email(rmistry@google.com)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "CT Poller Number of Goroutines"
message = "There are more goroutines running than expected for CT poller."
database = "graphite"
query = "select mean(value) from /^NumGoroutine.value$/ where app='ct-poller' AND host='build101-m5' AND time > now() - 10m;"
category = "infra"
conditions = ["x >= 100"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

#
# datahopper_internal (internal.skia.org)
#

[[rule]]
name = "datahopper_internal liveness"
message = "Too long between successful gold Android ingestions."
database = "skmetrics"
query = "select mean(value) from \"liveness\" where \"name\" = 'android-internal-ingest' AND app='datahopper_internal' AND host='skia-internal' AND time > now() - 10m"
category = "infra"
conditions = ["x >= 600"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "datahopper_internal Go routines"
message = "Too many Go routines."
database = "skmetrics"
query = "select mean(value) from \"runtime-metrics\" where \"metric\" = 'num-goroutine' AND app='datahopper_internal' AND  host='skia-internal' AND time > now() - 10m"
category = "infra"
conditions = ["x >= 100"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "datahopper_internal ingestBuild metrics"
message = "datahopper_internal goroutine for updateWebhookMetrics is dead or failing."
database = "skmetrics"
query = "select mean(value) from \"liveness\" where \"name\" = 'ingest-build-webhook-oldest-untested-commit-age-metric' AND app='datahopper_internal' AND host='skia-internal' AND time > now() - 10m"
category = "infra"
conditions = ["x >= 600"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "datahopper_internal Google3-Autoroller up-to-date"
message = "Commit has not been picked up by Google3-Autoroller for over two hours."
database = "skmetrics"
query = "select mean(value) from \"datahopper_internal.ingest-build-webhook.oldest-untested-commit-age\" where app='datahopper_internal' AND host='skia-internal' AND codename='Google3-Autoroller' AND time > now() - 10m"
category = "infra"
conditions = ["x >= 7200"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"


#
# Fuzzer
#
[[rule]]
name = "fuzzer upload queue"
message = "Fuzzer upload queue has been very full."
database = "skmetrics"
query = "SELECT max(value) from \"fuzzer.queue-size.upload\" WHERE time > now() - 10m AND app='fuzzer-be' AND host='skia-fuzzer'"
category = "infra"
conditions = ["x >= 90"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "fuzzer analysis queue"
message = "Fuzzer analysis queue has been very full."
database = "skmetrics"
query = "SELECT max(value) from \"fuzzer.queue-size.analysis\" WHERE time > now() - 10m AND app='fuzzer-be' AND host='skia-fuzzer'"
category = "infra"
conditions = ["x >= 9000"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "non-useful fuzzer"
message = "The %(category)s generator hasn't made much progress fuzzing. Perhaps it is misconfigured?"
database = "skmetrics"
query = "SELECT max(value) from \"fuzzer.stats.paths-total\" WHERE time > now() - 10m AND app='fuzzer-be' AND host='skia-fuzzer' GROUP BY category"
category = "infra"
conditions = ["x < 20"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

#
# Fiddle
#
[[rule]]
name = "fiddle is failing to build"
message = "Fiddle has failed to build at the last 2 chrome DEPS rolls. https://skia.googlesource.com/buildbot/+/master/fiddle/PROD.md#build_fail"
database = "skmetrics"
query = "SELECT mean(value) FROM \"counter\" WHERE app='fiddle' AND \"name\"='builds-failed' AND time > now() - 10m"
category = "infra"
conditions = ["x >= 2"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"

[[rule]]
name = "fiddle is failing to sync"
message = "Fiddle has failed to sync to HEAD at least 2 times in a row. https://skia.googlesource.com/buildbot/+/master/fiddle/PROD.md#sync_fail"
database = "skmetrics"
query = "SELECT mean(value) FROM \"counter\" WHERE app='fiddle' AND \"name\"='repo-sync-failed' AND time > now() - 10m"
category = "infra"
conditions = ["x >= 2"]
actions = ["Email(infra-alerts@skia.org)"]
auto-dismiss = true
nag = "1h"