| # General |
| ALERT InstanceDown |
| IF up == 0 |
| FOR 5m |
| LABELS { category = "infra", severity = "critical" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes. Logs: https://console.cloud.google.com/logs/viewer?project=google.com:skia-buildbots&minLogLevel=500&expandAll=false&resource=logging_log%2Fname%2F{{ reReplaceAll `:[0-9]+` `` $labels.instance }}&logName=projects%2Fgoogle.com:skia-buildbots%2Flogs%2F{{ $labels.job }}", |
| } |
| |
| ALERT CrashLoop |
| IF max_over_time(liveness_uptime_s[6m]) < 60*3 |
| FOR 5m |
| LABELS { category = "infra", severity = "critical" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "{{ $labels.instance }} of job {{ $labels.job }} is crashing on startup. Logs: https://console.cloud.google.com/logs/viewer?project=google.com:skia-buildbots&minLogLevel=500&expandAll=false&resource=logging_log%2Fname%2F{{ reReplaceAll `:[0-9]+` `` $labels.instance }}&logName=projects%2Fgoogle.com:skia-buildbots%2Flogs%2F{{ $labels.job }}", |
| } |
| |
| ALERT ExcessiveLoad |
| IF max_over_time(load[6m]) > 200 |
| FOR 5m |
| LABELS { category = "infra", severity = "critical" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.host }}", |
| description = "{{ $labels.host }} is experiencing excessive {{ $labels.sub }} load. Logs: https://console.cloud.google.com/logs/viewer?project=google.com:skia-buildbots&minLogLevel=200&expandAll=false&resource=logging_log%2Fname%2F{{ $labels.host }}" |
| } |
| |
| ALERT TooManyGoRoutines |
| IF go_goroutines > 3000 |
| FOR 2m |
| LABELS { category = "infra", severity = "warning"} |
| ANNOTATIONS { |
| abbr = "{{ $labels.job }}", |
| description = "Too many Go routines in {{ $labels.job }} running on {{ $labels.instance }}. Logs: https://console.cloud.google.com/logs/viewer?project=google.com:skia-buildbots&minLogLevel=400&expandAll=false&resource=logging_log%2Fname%2F{{ reReplaceAll `:[0-9]+` `` $labels.instance }}&logName=projects%2Fgoogle.com:skia-buildbots%2Flogs%2F{{ $labels.job }}" |
| } |
| |
| ALERT ProbeFailure |
| IF prober{type="failure"} > 0 |
| FOR 5m |
| LABELS { category = "infra", severity = "critical" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.probename }} {{ $labels.url }}", |
| description = "Endpoint {{ $labels.probename }} {{ $labels.url }} has failed to respond in at least 5 minutes. See https://github.com/google/skia-buildbot/search?q={{ $labels.probename }}+filename%3Aprobers.json5 for the endpoint URL." |
| } |
| |
| ALERT RebootRequired |
| IF reboot_required_i > 0 |
| LABELS { category = "infra", severity = "warning" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.host }}", |
| description = "Instance {{ $labels.host }} needs rebooting. Owner(s): {{ $labels.owners }}. See https://mon.skia.org/dashboard/db/reboots-required for the full list of instances that need rebooting.", |
| } |
| |
| ALERT DiskSpaceLow |
| IF df_complex_free{resource="df-root",host!~".*rpi-.+"} < 1e9 |
| FOR 5m |
| LABELS { category = "infra", severity = "warning" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.host }}", |
| description = "Low Root Disk Space on {{ $labels.host }}.", |
| } |
| |
| ALERT DiskSpaceLow |
| IF df_complex_free{resource=~"df-mnt-.*",resource!~".+docker.+"} < 1e10 |
| FOR 5m |
| LABELS { category = "infra", severity = "warning" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.host }}", |
| description = "Low Disk Space on {{ $labels.host }} on disk {{ $labels.resource }}.", |
| } |
| |
| ALERT DiskSpaceLow |
| IF df_complex_free{resource=~"df-var|df-tmp",host=~"skia-rpi-.+"} < 1e8 |
| FOR 5m |
| LABELS { category = "infra", severity = "warning" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.host }}", |
| description = "Free space has fallen below 100MB on {{ $labels.host }} drive {{ $labels.resource}}.", |
| } |
| |
| ALERT DirtyPackages |
| IF min_over_time(dirty_packages[25h]) >= 1 |
| LABELS { category = "infra", severity = "warning" } |
| ANNOTATIONS { |
| description = "One or more dirty packages have been running for more than 24 hours. https://push.skia.org", |
| } |
| |
| ALERT PackageInstall |
| IF rate(pulld_failed_install[10m]) > 1 |
| LABELS { category = "infra", severity = "warning" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.host }}", |
| description = "Package failing to install via pulld on {{ $labels.host }}.", |
| } |
| |
| ALERT TooManyOpenFDs |
| IF process_open_fds > 2000 |
| LABELS { category = "infra", severity = "warning" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.host }}", |
| description = "Too many open file handles on {{ $labels.host }}.", |
| } |
| |
| |
| # Datahopper |
| |
| ALERT DatahopperErrorRate |
| IF rate(num_log_lines{level="ERROR",log_source="datahopper"}[10m]) > 5 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| description = "The error rate for datahopper is too high. https://console.cloud.google.com/logs/viewer?project=google.com:skia-buildbots&minLogLevel=500&expandAll=false&resource=logging_log%2Fname%2Fskia-datahopper2&logName=projects%2Fgoogle.com:skia-buildbots%2Flogs%2Fdatahopper" |
| } |
| |
| ALERT JobMetricsLiveness |
| IF liveness_last_successful_job_metrics_update_s/60 > 30 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "{{ $labels.instance }} has failed to update job metrics for the last 30 minutes. https://skia.googlesource.com/buildbot/%2B/master/datahopper/PROD.md#job_metrics" |
| } |
| |
| ALERT BotCoverageMetricsLiveness |
| IF liveness_last_successful_bot_coverage_metrics_s/60 > 60 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "{{ $labels.instance }} has failed to update bot coverage metrics for the last 1 hour. https://skia.googlesource.com/buildbot/%2B/master/datahopper/PROD.md#bot_coverage_metrics" |
| } |
| |
| ALERT SwarmingTaskMetricsLiveness |
| IF liveness_last_successful_swarming_task_metrics_s/60 > 60 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "{{ $labels.instance }} has failed to update swarming task metrics for the last 1 hour. https://skia.googlesource.com/buildbot/%2B/master/datahopper/PROD.md#swarming_task_metrics" |
| } |
| |
| ALERT EventMetricsLiveness |
| IF liveness_last_successful_event_metrics_update_s/60 > 30 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "{{ $labels.instance }} has failed to update event metrics for {{ $labels.measurement }} for the last 30 minutes. https://skia.googlesource.com/buildbot/%2B/master/datahopper/PROD.md#event_metrics" |
| } |
| |
| ALERT SwarmingBotMetricsLiveness |
| IF liveness_last_successful_report_bot_metrics_s/60 > 10 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.pool }}", |
| description = "{{ $labels.instance }} has failed to update swarming task metrics for pool {{ $labels.pool }} on {{ $labels.server }} for the last 10 minutes. https://skia.googlesource.com/buildbot/%2B/master/datahopper/PROD.md#swarming_bot_metrics" |
| } |
| |
| # Swarming |
| |
| ALERT BotMissing |
| IF swarming_bots_last_seen{bot!~"(ct-gce-.*)|(build4.+device.+)"}/1000/1000/1000/60 > 15 |
| LABELS { category = "infra", severity = "critical"} |
| ANNOTATIONS { |
| abbr = "{{ $labels.bot }}", |
| description = "Swarming bot {{ $labels.bot }} is missing. https://{{ $labels.swarming }}/bot?id={{ $labels.bot }} https://goto.google.com/skolo-maintenance" |
| } |
| |
| ALERT CtGceBotMissing |
| IF max(swarming_bots_last_seen{bot=~"ct-gce-.*"})/1024/1024/1024/60 * max(ct_gce_bots_up) > 15 |
| LABELS { category = "infra", severity = "critical", owner = "rmistry@google.com"} |
| ANNOTATIONS { |
| description = "1 or more CT GCE bots are down: https://chrome-swarming.appspot.com/botlist?f=status%3Adead&f=gpu%3Anone&f=pool%3ACT&l=100" |
| } |
| |
| ALERT BotUnemployed |
| IF swarming_bots_last_task{pool=~"Skia.*"}/1000/1000/1000/60/60 >= 72 |
| LABELS { category = "infra", severity = "critical"} |
| ANNOTATIONS { |
| abbr = "{{ $labels.bot }}", |
| description = "Swarming bot {{ $labels.bot }} hasn't run a job in 72 hours. Maybe its dimensions need changing? https://{{ $labels.swarming }}/bot?id={{ $labels.bot }} https://goto.google.com/skolo-maintenance" |
| } |
| |
| ALERT BotQuarantined |
| IF avg_over_time(swarming_bots_quarantined{device_state!~"(too_hot)|(low_battery)"}[10m]) >= 1 |
| LABELS { category = "infra", severity = "critical"} |
| ANNOTATIONS { |
| abbr = "{{ $labels.bot }}", |
| description = "Swarming bot {{ $labels.bot }} is quarantined. https://{{ $labels.swarming }}/bot?id={{ $labels.bot }} https://goto.google.com/skolo-maintenance" |
| } |
| |
| ALERT DeviceUnhealthy |
| IF avg_over_time(swarming_bots_quarantined{device_state=~"(too_hot)|(low_battery)"}[1h]) >= 1 |
| LABELS { category = "infra", severity = "critical"} |
| ANNOTATIONS { |
| abbr = "{{ $labels.bot }}", |
| description = "Swarming bot {{ $labels.bot }} is quarantined because the device is {{ $labels.device_state }} and hasn't resolved itself in 1+ hours. https://{{ $labels.swarming }}/bot?id={{ $labels.bot }} https://goto.google.com/skolo-maintenance" |
| } |
| |
| ALERT SwarmingBotRebootRequired |
| IF avg_over_time(swarming_bots_reboot_required[10m]) >= 1 |
| LABELS { category = "infra", severity = "warning"} |
| ANNOTATIONS { |
| abbr = "{{ $labels.bot }}", |
| description = "Swarming bot {{ $labels.bot }} requires reboot. After reboot, check if os dimension has changed. https://{{ $labels.swarming }}/bot?id={{ $labels.bot }} https://goto.google.com/skolo-maintenance" |
| } |
| |
| ALERT WindowsSkoloOSVersion |
| IF round(avg_over_time(windows_skolo_os_version_count{pool="Skia"}[1h])) != 1 |
| LABELS { category = "infra", severity = "warning"} |
| ANNOTATIONS { |
| description = "Windows Skolo bots OS version has diverged. https://goto.google.com/skolo-maintenance" |
| } |
| |
| |
| # Status |
| |
| ALERT StatusLatency |
| IF avg_over_time(prober{probename="skiastatus_json",type="latency"}[10m])/1024 > 10 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| description = "The JSON endpoint at https://status.skia.org/json/skia/commits/ took more than 10s to respond." |
| } |
| |
| ALERT StatusIncrementalCacheUpdate |
| IF liveness_last_successful_incremental_cache_update_s > 5*60 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| description = "IncrementalCache UpdateLoop on {{ $labels.instance }} has failed to update data for more than 5 minutes. Playbook: Logs: https://console.cloud.google.com/logs/viewer?project=google.com:skia-buildbots&minLogLevel=500&expandAll=false&resource=logging_log%2Fname%2F{{ reReplaceAll `:[0-9]+` `` $labels.instance }}&logName=projects%2Fgoogle.com:skia-buildbots%2Flogs%2Fstatus" |
| } |
| |
| # Prober |
| |
| ALERT ProberLiveness |
| IF liveness_probes_s/60 > 10 |
| LABELS { category = "infra", severity = "critical"} |
| ANNOTATIONS { |
| description = "Probing has failed to run in at least 10 minutes." |
| } |
| |
| ALERT IssueTrackerLiveness |
| IF liveness_issue_tracker_s/60 > 30 |
| LABELS { category = "infra", severity = "critical"} |
| ANNOTATIONS { |
| description = "Issue tracker ingestion has failed to run in at least 30 minutes." |
| } |
| |
| # Gold |
| |
| ALERT GoldIgnoreMonitoring |
| IF liveness_gold_expired_ignore_rules_monitoring_s{instance="skia-gold-prod:20001"} > 200 |
| LABELS { category = "infra", severity = "critical", owner = "stephana@google.com"} |
| ANNOTATIONS { |
| description = "At least two rounds of monitoring for expired ignore rules have failed back to back.", |
| } |
| |
| ALERT GoldErrorRate |
| IF rate(num_log_lines{level="ERROR",job=~"skiacorrectness-.*"}[2m]) > 1 |
| LABELS { category = "infra", severity = "critical", owner = "stephana@google.com"} |
| ANNOTATIONS { |
| description = "The error rate for Gold {{ $labels.instance }} is too high. https://console.cloud.google.com/logs/viewer?project=google.com:skia-buildbots&minLogLevel=500&expandAll=false&resource=logging_log%2Fname%2F{{ $labels.log_group }}&logName=projects%2Fgoogle.com:skia-buildbots%2Flogs%2F{{ $labels.log_source }}", |
| } |
| |
| ALERT GoldDiffServerErrorRate |
| IF rate(num_log_lines{level="ERROR", instance="skia-diffserver-prod:20000"}[2m]) > 1 |
| LABELS { category = "infra", severity = "critical", owner = "stephana@google.com"} |
| ANNOTATIONS { |
| description = "The error rate for Gold Diffserver {{ $labels.instance }} is too high. https://console.cloud.google.com/logs/viewer?project=google.com:skia-buildbots&minLogLevel=500&expandAll=false&resource=logging_log%2Fname%2F{{ $labels.log_group }}&logName=projects%2Fgoogle.com:skia-buildbots%2Flogs%2F{{ $labels.log_source }}", |
| } |
| |
| ALERT GoldIngestionStalled |
| IF liveness_gold_s{metric="since-last-run",source="poll"} > 750 |
| LABELS { category = "infra", severity = "critical", owner = "stephana@google.com"} |
| ANNOTATIONS { |
| description = "At least two rounds of gold ingestion have failed back to back for {{ $labels.corpus }}. See https://mon.skia.org/dashboard/db/gold-panel", |
| } |
| |
| ALERT GoldIngestionErrorRate |
| IF rate(num_log_lines{level="ERROR",job=~".*_ingestion"}[2m]) > 1 |
| LABELS { category = "infra", severity = "critical", owner = "stephana@google.com"} |
| ANNOTATIONS { |
| description = "The error rate for Gold Ingestion {{ $labels.corpus }} is too high. See https://mon.skia.org/dashboard/db/gold-panel https://console.cloud.google.com/logs/viewer?project=google.com:skia-buildbots&minLogLevel=500&expandAll=false&resource=logging_log%2Fname%2F{{ $labels.log_group }}&logName=projects%2Fgoogle.com:skia-buildbots%2Flogs%2F{{ $labels.log_source }}", |
| } |
| |
| # Task Scheduler |
| |
| ALERT TaskSchedulerLiveness |
| IF liveness_last_successful_task_scheduling_s/60 > 10 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "{{ $labels.instance }} has failed to schedule for the last 10 minutes. https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#scheduling_failed" |
| } |
| |
| ALERT TaskSchedulerLatency |
| IF prober{type="latency",probename="task_scheduler"} > 300 |
| FOR 10m |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.probename }}", |
| description = "The endpoint for {{ $labels.probename }} took more than 300ms to respond. https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#http_latency" |
| } |
| |
| ALERT TaskSchedulerErrorRate |
| IF rate(num_log_lines{level="ERROR",log_source="task_scheduler"}[2m]) > 0.05 |
| FOR 2m |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "The error rate for task_scheduler on {{ $labels.instance }} is too high. https://console.cloud.google.com/logs/viewer?project=google.com:skia-buildbots&minLogLevel=500&expandAll=false&resource=logging_log%2Fname%2F{{ $labels.log_group }}&logName=projects%2Fgoogle.com:skia-buildbots%2Flogs%2F{{ $labels.log_source }} https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#error_rate" |
| } |
| |
| ALERT TaskSchedulerDBBackup |
| IF liveness_last_db_backup_s/60/60 > 25 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "The last Task Scheduler DB backup on {{ $labels.instance }} was more than 25 hours ago. https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#old_db_backup" |
| } |
| |
| ALERT TaskSchedulerExtraDBBackups |
| IF recent_db_backup_count > 9 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "There are too many recent Task Scheduler DB backups for {{ $labels.instance }}. https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#too_many_recent_db_backups" |
| } |
| |
| ALERT TaskSchedulerDBBackupTrigger |
| IF liveness_db_backup_maybe_backup_db_s/60 > 10 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "The last time we checked for a Task Scheduler DB backup trigger file on {{ $labels.instance }} was more than 10 minutes ago. https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#db_backup_trigger_liveness" |
| } |
| |
| ALERT TaskSchedulerIncrementalBackup |
| IF liveness_incremental_backup_s/60 > 10 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "The last time a Task Scheduler incremental backup succeeded on {{ $labels.instance }} was more than 10 minutes ago. https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#incremental_backup_liveness" |
| } |
| |
| ALERT TaskSchedulerIncrementalBackupReset |
| IF incremental_backup_reset > 0 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "Task Scheduler modified job tracking for incremental backups has been reset since last full backup on {{ $labels.instance }}. https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#incremental_backup_reset" |
| } |
| |
| ALERT TaskSchedulerDBFreePages |
| IF avg_over_time(bolt_db{metric="FreePageCount",database="task_scheduler_db",instance!="skia-task-scheduler-internal:20000"}[30m]) > 1000 |
| FOR 1h |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "There are a large number of free pages in the Task Scheduler DB on {{ $labels.instance }}. https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#db_too_many_free_pages" |
| } |
| |
| ALERT DbMetricsLiveness |
| IF liveness_DbMetric_s/60 > 30 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "{{ $labels.instance }} has failed to update boltutil.DbMetrics for the last 30 minutes. Logs: https://console.cloud.google.com/logs/viewer?project=google.com:skia-buildbots&minLogLevel=500&expandAll=false&resource=logging_log%2Fname%2F{{ reReplaceAll `:[0-9]+` `` $labels.instance }}&logName=projects%2Fgoogle.com:skia-buildbots%2Flogs%2F{{ $labels.job }}" |
| } |
| |
| |
| ALERT TaskSchedulerTooManyCandidates |
| IF task_candidate_count > 1500 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "There are too many task candidates for dimensions: {{ $labels.dimensions }} https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#too_many_candidates" |
| } |
| |
| |
| ALERT OverdueMetricsLiveness |
| IF liveness_last_successful_overdue_metrics_update_s/60 > 10 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.instance }}", |
| description = "{{ $labels.instance }} has failed to update overdue_job_specs_s for the last 10 minutes. Logs: https://console.cloud.google.com/logs/viewer?project=google.com:skia-buildbots&minLogLevel=500&expandAll=false&resource=logging_log%2Fname%2F{{ reReplaceAll `:[0-9]+` `` $labels.instance }}&logName=projects%2Fgoogle.com:skia-buildbots%2Flogs%2F{{ $labels.job }} Production Manual: https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#overdue_metrics_liveness" |
| } |
| |
| # These jobs have tasks with an expiration of 4 hours, and we allow 2 attempts, so they should |
| # normally finish within 8 hours. |
| ALERT OverdueJobSpec |
| IF overdue_job_specs_s{job_trigger=~"|master",job_name!~".*(Valgrind|MSAN|-x86-).*"}/60/60 > 8 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.job_name }}", |
| description = "{{ $labels.job_name }} has not finished for any commit in the last 8 hours. Maybe the dimensions need changing? (Job defined here: {{ $labels.repo }}/+/master/infra/bots/tasks.json) Production Manual: https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#overdue_job_spec" |
| } |
| |
| # These jobs have tasks with an expiration of 9 hours, and we allow 2 attempts, so they should |
| # normally finish within 18 hours. |
| ALERT OverdueJobSpecLong |
| IF overdue_job_specs_s{job_trigger=~"|master",job_name=~".*(Valgrind|MSAN|-x86-).*"}/60/60 > 18 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.job_name }}", |
| description = "{{ $labels.job_name }} has not finished for any commit in the last 9 hours. Maybe the dimensions need changing? (Job defined here: {{ $labels.repo }}/+/master/infra/bots/tasks.json) Production Manual: https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#overdue_job_spec" |
| } |
| |
| ALERT OverdueJobSpecNightly |
| IF overdue_job_specs_s{job_trigger="nightly"}/60/60 > 28 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.job_name }}", |
| description = "{{ $labels.job_name }} has not completed in the last 28 hours (nightly job). Maybe the dimensions need changing? (Job defined here: {{ $labels.repo }}/+/master/infra/bots/tasks.json) Production Manual: https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#overdue_job_spec" |
| } |
| |
| ALERT OverdueJobSpecWeekly |
| IF overdue_job_specs_s{job_trigger="weekly"}/60/60 > 7*24+4 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.job_name }}", |
| description = "{{ $labels.job_name }} has not completed in the last week + 4 hours (weekly job). Maybe the dimensions need changing? (Job defined here: {{ $labels.repo }}/+/master/infra/bots/tasks.json) Production Manual: https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#overdue_job_spec" |
| } |
| |
| ALERT LatestJobAgeNightly |
| IF latest_job_age_s{job_trigger="nightly"}/60/60 > 25 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.job_name }}", |
| description = "{{ $labels.job_name }} has not been triggered in the last 25 hours (nightly job). Double check whether the periodic triggers are running correctly (Job defined here: {{ $labels.repo }}/+/master/infra/bots/tasks.json) Production Manual: https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#latest_job_age" |
| } |
| |
| ALERT LatestJobAgeWeekly |
| IF latest_job_age_s{job_trigger="weekly"}/60/60 > 7*24+1 |
| LABELS { category = "infra", severity = "critical", owner = "borenet@google.com" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.job_name }}", |
| description = "{{ $labels.job_name }} has not been triggered in the last week + 1 hour (weekly job). Double check whether the periodic triggers are running correctly (Job defined here: {{ $labels.repo }}/+/master/infra/bots/tasks.json) Production Manual: https://skia.googlesource.com/buildbot/%2B/master/task_scheduler/PROD.md#latest_job_age" |
| } |
| |
| |
| # Skolo |
| |
| ALERT BackupNotDone |
| IF liveness_skolo_last_backup_s{}/60/60 > 25 |
| LABELS { category = "infra", severity = "critical" } |
| ANNOTATIONS { |
| abbr = "{{ $labels.federated }} {{ $labels.backup }}", |
| description = "The automated backup of {{ $labels.backup }} was not completed by {{ $labels.federated }} in the last 25 hours. http://go/skolo-maintenance", |
| } |
| |
| ALERT RpiMaster |
| IF skolo_hotspare_spare_active == 1 |
| LABELS { category = "infra", severity = "critical"} |
| ANNOTATIONS { |
| description = "skia-rpi-master-spare is active. Something is possibly wrong with skia-rpi-master. http://go/skolo-maintenance", |
| } |
| |
| # |
| # QPS to external services. |
| # |
| |
| # General. |
| # TODO(borenet): Specific alerts for Swarming. |
| ALERT HighExternalQPS |
| IF sum(rate(http_request_metrics{job!="skiaperf",host!="www.googleapis.com"}[30m])) by (host) > 25 |
| LABELS { category = "infra", severity = "warning" } |
| ANNOTATIONS { |
| description = "QPS to {{ $labels.host }} is high. Verify that this is expected. See https://mon.skia.org/dashboard/db/outgoing-qps" |
| } |
| |
| # www.googleapis.com has a higher threshold. |
| ALERT HighExternalQPSGoogleAPIs |
| IF sum(rate(http_request_metrics{host="www.googleapis.com"}[30m])) > 60 |
| LABELS { category = "infra", severity = "warning" } |
| ANNOTATIONS { |
| description = "QPS to www.googleapis.com is high. Verify that this is expected. See https://mon.skia.org/dashboard/db/outgoing-qps" |
| } |
| |
| # Datastore Backup |
| |
| ALERT Bootloop |
| IF avg_over_time(liveness_backup_step_s[5m]) < 60 |
| LABELS { category = "infra", severity = "critical" } |
| ANNOTATIONS { |
| description = "datastore_backup appears to be in a boot loop. https://skia.googlesource.com/buildbot/%2B/master/ds/PROD.md#boot_loop" |
| } |
| |
| ALERT BackupNotDone |
| IF liveness_backup_success_s/60/60 > 25 |
| LABELS { category = "infra", severity = "critical" } |
| ANNOTATIONS { |
| description = "A backup of Cloud Datastore hasn't succeeded in the last 25 hours. https://skia.googlesource.com/buildbot/%2B/master/ds/PROD.md#backup_not_done" |
| } |
| |
| |
| # alert-to-pubsub |
| |
| ALERT AlertToPubSub |
| IF rate(pubsub_send_failure[5m]) > 0.003 |
| LABELS { category = "infra", severity = "critical", owner = "jcgregorio@google.com" } |
| ANNOTATIONS { |
| abbr = "google.com:skia-buildbots", |
| description = "Failed to send alert via PubSub. https://console.cloud.google.com/logs/viewer?project=google.com:skia-buildbots&minLogLevel=500&expandAll=false&resource=logging_log%2Fname%2F{{ reReplaceAll `:[0-9]+` `` $labels.instance }}&logName=projects%2Fgoogle.com:skia-buildbots%2Flogs%2F{{ $labels.job }}" |
| } |
