# Alerts for things in the skia-public cluster only.
#
# If anything in this file starts to run in another cluster, such as
# skia-corp, then break it out into its own alerts_NNNN.yml file
# and include it in each prometheus-CLUSTER.yml file that is it running in.
groups:
- name: general
  rules:

  # This alert belongs in alerts_general.yml, except that skia-corp doesn't have any scrape_configs
  # for processes with this metric.
  - alert: CrashLoop
    expr: max_over_time(liveness_uptime_s{kubernetes_pod_name=""}[6m]) < 60 * 3
    for: 5m
    labels:
      category: infra
      severity: critical
    annotations:
      abbr: '{{ $labels.instance }}'
      description: '{{ $labels.instance }} of job {{ $labels.job }} is crashing on
        startup. Logs:

          kubectl logs -l app={{ reReplaceAll `:[0-9]+` `` $labels.instance }} -c {{ $labels.job }}

          https://console.cloud.google.com/logs/viewer?project={{ $labels.project }}&advancedFilter=resource.type%3D%22k8s_container%22%0Aresource.labels.pod_name%3D%22{{ $labels.kubernetes_pod_name }}%22"
        '

  # Docker Pushes Watcher
  - alert: DockerPushesWatcherLiveness
    expr: liveness_docker_watcher_pubsub_receive_s > 60 * 60 * 24 * 2
    for: 5m
    labels:
      category: infra
      severity: critical
      owner: rmistry@google.com
    annotations:
      description: 'Docker pushes watcher has failed to recieve a pubsub event in the last 48 hours.

          https://console.cloud.google.com/logs/viewer?project=skia-public&resource=k8s_container%2Fcluster_name%2Fskia-public%2Fnamespace_name%2Fdefault%2Fcontainer_name%2Fdocker-pushes-watcher
        '

  - alert: DockerPushesWatcherTagFailures
    expr: docker_watcher_tag_failure > 0
    for: 5m
    labels:
      category: infra
      severity: critical
      owner: rmistry@google.com
    annotations:
      abbr: '{{ $labels.image }}'
      description: 'Docker pushes watcher has failed to successfully add "prod" tag to the docker image of {{ $labels.image }} in the repo {{ $labels.repo }}

          https://console.cloud.google.com/logs/viewer?project=skia-public&resource=k8s_container%2Fcluster_name%2Fskia-public%2Fnamespace_name%2Fdefault%2Fcontainer_name%2Fdocker-pushes-watcher
        '

  - alert: DockerPushesWatcherPushFailures
    expr: docker_watcher_push_failure > 0
    for: 5m
    labels:
      category: infra
      severity: critical
      owner: rmistry@google.com
    annotations:
      abbr: '{{ $labels.image }}'
      description: 'Docker pushes watcher has failed to successfully pushk the docker image of {{ $labels.image }} in the repo {{ $labels.repo }}

          https://console.cloud.google.com/logs/viewer?project=skia-public&resource=k8s_container%2Fcluster_name%2Fskia-public%2Fnamespace_name%2Fdefault%2Fcontainer_name%2Fdocker-pushes-watcher
        '

  # Fiddle
  - alert: InsufficientFiddlerPods
    expr: avg_over_time(pods_idle[2m]) < 5
    for: 15m
    labels:
      category: infra
      severity: warning
      owner: jcgregorio@google.com
    annotations:
      description: 'Fiddle is experiencing heavy load and has insufficient idle fiddler pods. https://skia.googlesource.com/buildbot/+doc/refs/heads/main/fiddlek/PROD.md#fiddler_pods'

  - alert: NamedFiddlesFailing
    expr: named_fiddles_errors_in_examples_run > 0
    for: 15m
    labels:
      category: infra
      severity: warning
      owner: jcgregorio@google.com
    annotations:
      description: 'Some named fiddles are failing to compile or run. https://skia.googlesource.com/buildbot/+doc/refs/heads/main/fiddlek/PROD.md#NamedFiddlesFailing'

  - alert: InvalidNamedFiddles
    expr: named_fiddles_examples_total_invalid > 0
    for: 15m
    labels:
      category: infra
      severity: warning
      owner: jcgregorio@google.com
    annotations:
      description: 'Some named fiddles are failing to be parsed. https://skia.googlesource.com/buildbot/+doc/refs/heads/main/fiddlek/PROD.md#InvalidNamedFiddles'

  - alert: FiddlerPodCommunicationErrors
    expr: rate(run_exhaustion[20m]) * 20 * 60 > 5
    for: 5m
    labels:
      category: infra
      severity: warning
      owner: jcgregorio@google.com
    annotations:
      description: 'Fiddle is having trouble communicating with fiddler pods.'

  # CQ Watcher
  - alert: TooManyCLsInCQ
    expr: cq_watcher_in_flight_waiting_in_cq{app="cq-watcher"} >= 10
    for: 5m
    labels:
      category: infra
      severity: warning
    annotations:
      description: 'There are 10 CLs or more waiting in the CQ. Dry run queue: https://skia-review.googlesource.com/q/label:Commit-Queue%253D1+status:open and Commit queue: https://skia-review.googlesource.com/q/label:Commit-Queue%253D2+status:open Playbook: https://skia.googlesource.com/buildbot/%2B/main/cq_watcher/PROD.md#too_many_cls'

  - alert: CQTrybotRunningTooLong
    expr: max_over_time(cq_watcher_in_flight_trybot_duration{app="cq-watcher"}[20m]) > 2700
    labels:
      category: infra
      severity: warning
    annotations:
      abbr: '{{ $labels.trybot }}'
      description: '{{ $labels.trybot }} ran longer than 45 mins on {{ $labels.gerritURL }} Playbook: https://skia.googlesource.com/buildbot/%2B/main/cq_watcher/PROD.md#trybot_duration_beyond_threshold'

  - alert: TooManyCQTrybotsForCL
    expr: max_over_time(cq_watcher_in_flight_trybot_num{app="cq-watcher"}[20m]) > 100
    labels:
      category: infra
      severity: warning
    annotations:
      description: 'There are more than 100 CQ trybots triggered by {{ $labels.gerritURL }} Playbook: https://skia.googlesource.com/buildbot/%2B/main/cq_watcher/PROD.md#too_many_trybots_triggered'

 # datatore backups
  - alert: BackupNotDone
    expr: liveness_backup_success_s/60/60/24 > 7
    labels:
      category: infra
      severity: critical
    annotations:
      abbr: skia-public
      description: 'A backup of Cloud Datastore has not succeeded in the last week. https://skia.googlesource.com/buildbot/%2B/main/ds/PROD.md#backup_not_done'

# alert-to-pubsub liveness
  - alert: AlertToPubSubLiveness
    expr: (min(liveness_alive_s{location=~".+"}) by (location)) > 90
    labels:
      category: infra
      severity: critical
      owner: jcgregorio@google.com
    annotations:
      abbr: '{{ $labels.location }}'
      description: 'alert-to-pubsub for {{ $labels.location }} has failed to send a healthz PubSub event in 90s. https://skia.googlesource.com/buildbot/%2B/main/am/PROD.md#alert_to_pubsub'

# CT
# TODO(rmistry): Add error rate alert once logmetrics is ported to skia-public.
  - alert: CTFEPendingTaskCount
    expr: num_pending_tasks{app="ctfe"} >= 10
    for: 5m
    labels:
      category: infra
      severity: critical
      owner: rmistry@google.com
    annotations:
      description: 'There are a lot of CTFE pending tasks. https://skia.googlesource.com/buildbot/%2B/main/ct/PROD.md#ctfe_pending_tasks'

  - alert: CTFEPendingTaskNotRunning
    expr: oldest_pending_task_status{app="ctfe"} >= 2
    for: 5m
    labels:
      category: infra
      severity: critical
      owner: rmistry@google.com
    annotations:
      description: 'A task has been waiting to be executed for a while and it has still not started. https://skia.googlesource.com/buildbot/%2B/main/ct/PROD.md#ctfe_pending_tasks'

  - alert: AutoRollLatency
    expr: prober{type="latency",probename="autoroll"} > 200
    for: 10m
    labels:
      category: infra
      severity: critical
      owner: borenet@google.com
    annotations:
      description: 'The endpoint for {{ $labels.probename }} {{ $labels.url }} took more than 200ms to respond. https://skia.googlesource.com/buildbot/%2B/main/autoroll/PROD.md#http_latency'

  - alert: FlutterLicenseScriptFailure
    expr: flutter_license_script_failure{app="autoroll-be-skia-flutter-autoroll"} > 0
    for: 5m
    labels:
      category: infra
      severity: critical
      owner: rmistry@google.com
    annotations:
      description: 'The License scripts in the Skia->Flutter roller have failed.
      https://console.cloud.google.com/logs/viewer?project={{ $labels.project }}&resource=k8s_container%2Fcluster_name%2F{{ $labels.cluster }}%2Fnamespace_name%2Fdefault%2Fcontainer_name%2F{{ $labels.app }}
      https://skia.googlesource.com/buildbot/%2B/main/autoroll/PROD.md#flutter_license_script_failure'

# skia-flutter-autoroll takes a long time to transition because its pre-upload
# scripts run flutter's license script which can take around 40 minutes.
  - alert: AutoRollLastTransition
    expr: liveness_last_successful_autoroll_tick_s{roller="skia-flutter-autoroll"} > 50*60
    labels:
      category: infra
      severity: critical
      owner: rmistry@google.com
    annotations:
      abbr: 'skia-flutter-autoroll'
      description: 'Autoroll on {{ $labels.app }} has failed to transition for more than 50 minutes.
      https://console.cloud.google.com/logs/viewer?project={{ $labels.project }}&resource=k8s_container%2Fcluster_name%2F{{ $labels.cluster }}%2Fnamespace_name%2Fdefault%2Fcontainer_name%2F{{ $labels.app }}
      '

# Perf
  - alert: AndroidIngestStalled
    expr: rate(ack[30m]) < 0.01
    labels:
      category: infra
      severity: critical
      owner: jcgregorio@google.com
    annotations:
      description: 'The Android Perf instances that run using --event_driven_regression_detection have stopped receiving data. Confirm via https://android-metric-ingest.skia.org/ that data is arriving.'

  - alert: AndroidRegressionDetectionTooSlow
    expr: sum(rate(perf_regression_store_found{app=~"perf-clustering-android|skiaperf|skiaperf-android-x"}[1h])) by (app) < 0.01
    labels:
      category: infra
      severity: critical
      owner: jcgregorio@google.com
    annotations:
      description: 'Perf failed to find any regressions in an hour. See https://github.com/google/skia-buildbot/blob/main/android_ingest/PROD.md#regression_detection_slow'

  - alert: AndroidIngestFailures
    expr: rate(process_failures[1h]) > 0.01
    labels:
      category: infra
      severity: critical
      owner: jcgregorio@google.com
    annotations:
      description: 'Error rate for processing buildids is too high. See https://github.com/google/skia-buildbot/blob/main/android_ingest/PROD.md#process_failures'

  - alert: AndroidIngestLiveness
    expr: liveness_last_successful_add_s > 300
    labels:
      category: infra
      severity: critical
      owner: jcgregorio@google.com
    annotations:
      description: 'Liveness for processing buildids is too high. See https://github.com/google/skia-buildbot/blob/main/android_ingest/PROD.md#liveness'


  - alert: AndroidIngestBadFilesTooHigh
    expr: bad_files/uploads > 0.5
    labels:
      category: infra
      severity: critical
      owner: jcgregorio@google.com
    annotations:
      description: 'Error rate for bad_files is too high. See https://github.com/google/skia-buildbot/blob/main/android_ingest/PROD.md#bad_files'

  - alert: AndroidTxLogFailures
    expr: tx_log_write_failure > 0
    labels:
      category: infra
      severity: critical
      owner: jcgregorio@google.com
    annotations:
      description: 'android_ingest failing to record incoming data to transaction log. See https://github.com/google/skia-buildbot/blob/main/android_ingest/PROD.md#tx_log'

  - alert: PerfIngestionSuccessRateTooLow
    expr: rate(perfserver_ingest_successful_write[1h]) / rate(perfserver_ingest_files_received[1h])  < 0.95
    for: 10m
    labels:
      category: infra
      severity: critical
      owner: jcgregorio@google.com
    annotations:
      description: 'Too few ingestion files are being successfully ingested. See https://github.com/google/skia-buildbot/blob/main/perf/PROD.md#success_rate_too_low'

  - alert: PerfAndroidClusteringSlow
    expr: sum(rate(perf_clustering_runs{app="perf-clustering-android"}[1h])*60*60) < 1
    for: 15m
    labels:
      category: infra
      severity: critical
      owner: jcgregorio@google.com
    annotations:
      description: 'Android Perf Clustering (https://android-main-perf.skia.org)Rate is too low. See https://github.com/google/skia-buildbot/blob/main/perf/PROD.md#android_clustering_rate'

  - alert: PerfClusteringSlow
    expr: rate(perf_clustering_runs{app!="skiaperf-android"}[1h])*60*60 < 1
    for: 60m
    labels:
      category: infra
      severity: critical
      owner: jcgregorio@google.com
    annotations:
      description: 'Perf Clustering Rate is too low. See https://github.com/google/skia-buildbot/blob/main/perf/PROD.md#clustering_rate'

# Prober
  - alert: ProbeFailure
    expr: prober{type="failure"} > 0
    for: 5m
    labels:
      category: infra
      severity: critical
    annotations:
      abbr: '{{ $labels.probename }} {{ $labels.url }}'
      description: 'Endpoint {{ $labels.probename }} {{ $labels.url }} has failed to
        respond in at least 5 minutes. See https://github.com/google/skia-buildbot/search?q={{
        $labels.probename }}+filename%3Aprobersk.json5 for the endpoint URL.'

  - alert: ProberLiveness
    expr: liveness_probes_s > 300
    for: 5m
    labels:
      category: infra
      severity: critical
    annotations:
      description: 'The prober has failed to probe in the last 5 minutes.'


# Grafana Backup
  - alert: GrafanaBackupLiveness
    expr: liveness_backup_s > 60*60*25
    for: 5m
    labels:
      category: infra
      severity: critical
    annotations:
      description: 'backup-to-gcs has failed to back up the Grafana db in the last 24 hours. Check the logs.'

# Skia Status
  - alert: StatusLatency
    expr: avg_over_time(prober{probename="skiastatus_json",type="latency"}[10m])/1024  > 10
    labels:
      category: infra
      severity: critical
      owner: borenet@google.com
    annotations:
      description: 'The JSON endpoint at https://status.skia.org/json/skia/commits/ took more than 10s to respond.'

# Datahopper

  - alert: FirestoreBackupTooOld
    expr: liveness_last_successful_firestore_backup_s{app="datahopper"}/60/60/24 > 7
    labels:
      category: infra
      severity: critical
      owner: borenet@google.com
    annotations:
      description: 'The most recent successful Firestore weekly backup was more than 7 days ago. https://skia.googlesource.com/buildbot/%2B/main/datahopper/PROD.md#firestore_weekly_backup'

  - alert: GoDEPSTooOld
    expr: liveness_last_file_modification_s{file="go.mod"}/60/60/24 > 3
    labels:
      category: infra
      severity: critical
      owner: borenet@google.com
    annotations:
      abbr: '{{ $labels.file }} in {{ $labels.repo }} too old'
      description: 'The {{ $labels.file }} file in {{ $labels.repo }} was last updated more than 3 days ago. https://skia.googlesource.com/buildbot/%2B/main/infra/bots/task_drivers/update_go_deps/update_go_deps.go'

# External RPis.
  - alert: DiskSpaceLow
    expr: collectd_df_df_complex{df=~"var|tmp",exported_instance=~"skia-(e-)?rpi-.+",type="free"} < 1e8
    for: 5m
    labels:
      category: infra
      severity: warning
    annotations:
      abbr: '{{ $labels.exported_instance }}'
      description: 'Free space has fallen below 100MB on {{ $labels.exported_instance }} drive {{ $labels.df }}. https://chromium-swarm.appspot.com/bot?id={{ $labels.exported_instance }}  https://skia.googlesource.com/buildbot/+doc/main/docs/PROD.md#diskspacelow'

# External bots except RPis.
  - alert: DiskSpaceLow
    expr: collectd_df_df_complex{df="root",exported_instance=~"skia-e-[^r].+",type="free"} < 1e9
    for: 5m
    labels:
      category: infra
      severity: warning
    annotations:
      abbr: '{{ $labels.exported_instance }}'
      description: 'Low Root Disk Space on {{ $labels.exported_instance }}. https://chromium-swarm.appspot.com/bot?id={{ $labels.exported_instance }}  https://skia.googlesource.com/buildbot/+doc/main/docs/PROD.md#diskspacelow'

# Dev RPis.
  - alert: DiskSpaceLow
    expr: collectd_df_df_complex{df=~"var|tmp",exported_instance=~"skia-d-rpi-.+",type="free"} < 1e8
    for: 5m
    labels:
      category: infra
      severity: warning
    annotations:
      abbr: '{{ $labels.exported_instance }}'
      description: 'Free space has fallen below 100MB on {{ $labels.exported_instance }} drive {{ $labels.df }}. https://chromium-swarm-dev.appspot.com/bot?id={{ $labels.exported_instance }}  https://skia.googlesource.com/buildbot/+doc/main/docs/PROD.md#diskspacelow'

# Dev bots except RPis.
  - alert: DiskSpaceLow
    expr: collectd_df_df_complex{df="root",exported_instance=~"skia-d-[^r].+",type="free"} < 1e9
    for: 5m
    labels:
      category: infra
      severity: warning
    annotations:
      abbr: '{{ $labels.exported_instance }}'
      description: 'Low Root Disk Space on {{ $labels.exported_instance }}. https://chromium-swarm-dev.appspot.com/bot?id={{ $labels.exported_instance }}  https://skia.googlesource.com/buildbot/+doc/main/docs/PROD.md#diskspacelow'

# Internal RPis.
  - alert: DiskSpaceLow
    expr: collectd_df_df_complex{df=~"var|tmp",exported_instance=~"skia-i-rpi-.+",type="free"} < 1e8
    for: 5m
    labels:
      category: infra
      severity: warning
    annotations:
      abbr: '{{ $labels.exported_instance }}'
      description: 'Free space has fallen below 100MB on {{ $labels.exported_instance }} drive {{ $labels.df }}. https://chrome-swarming.appspot.com/bot?id={{ $labels.exported_instance }}  https://skia.googlesource.com/buildbot/+doc/main/docs/PROD.md#diskspacelow'

# Internal bots except RPis.
  - alert: DiskSpaceLow
    expr: collectd_df_df_complex{df="root",exported_instance=~"skia-i-[^r].+",type="free"} < 1e9
    for: 5m
    labels:
      category: infra
      severity: warning
    annotations:
      abbr: '{{ $labels.exported_instance }}'
      description: 'Low Root Disk Space on {{ $labels.exported_instance }}. https://chrome-swarming.appspot.com/bot?id={{ $labels.exported_instance }}  https://skia.googlesource.com/buildbot/+doc/main/docs/PROD.md#diskspacelow'

# GCE machines (other than bots), root disk.
  - alert: DiskSpaceLow
    expr: collectd_df_df_complex{df="root",exported_instance!~"skia-(e|i|d|rpi)-.+",type="free"} < 1e9
    for: 5m
    labels:
      category: infra
      severity: warning
    annotations:
      abbr: '{{ $labels.exported_instance }}'
      description: 'Low Root Disk Space on {{ $labels.exported_instance }}.  https://skia.googlesource.com/buildbot/+doc/main/docs/PROD.md#diskspacelow'

# GCE bots, /b (aka /mnt/pd0)
  - alert: DiskSpaceLow
    expr: collectd_df_df_complex{df="mnt-pd0", exported_instance=~"skia-e-gce.+",type="free"} < 1e10
    for: 5m
    labels:
      category: infra
      severity: critical
    annotations:
      abbr: '{{ $labels.exported_instance }}'
      description: 'Low Disk Space on /b for {{ $labels.exported_instance }}. https://skia.googlesource.com/buildbot/+doc/main/docs/PROD.md#diskspacelow'


# Envoy
  - alert: EnvoyClusterBindError
    expr: envoy_cluster_bind_errors > 0
    for: 5m
    labels:
      category: infra
      severity: error
    annotations:
      abbr: '{{ $labels.envoy_cluster_name }}'
      description: 'Envoy Cluster Bind Error for {{ $labels.envoy_cluster_name
      }}. https://skia.googlesource.com/buildbot/+doc/main/skfe/PROD.md#cluster_bind_error'

  - alert: EnvoyRuntimeLoadError
    expr: envoy_runtime_load_error > 0
    for: 5m
    labels:
      category: infra
      severity: error
    annotations:
      abbr: '{{ $labels.app }}'
      description: 'Envoy Runtime Load Error for {{ $labels.app }}. https://skia.googlesource.com/buildbot/%2B/main/skfe/PROD.md#runtime_load_error'

  - alert: EnvoyClusterNotOK
    expr: envoy_cluster_lb_local_cluster_not_ok > 0
    for: 5m
    labels:
      category: infra
      severity: error
    annotations:
      abbr: '{{ $labels.envoy_cluster_name }}'
      description: 'Envoy Cluster Not OK for {{ $labels.envoy_cluster_name }}. https://skia.googlesource.com/buildbot/%2B/main/skfe/PROD.md#envoy_cluster_lb_local_cluster_not_ok'

# Push
  - alert: DirtyPackages
    expr: min_over_time(dirty_packages[25h]) >= 1
    labels:
      category: infra
      severity: warning
    annotations:
      description: 'One or more dirty packages have been running for more than 24 hours. https://push.skia.org'

# Docsyserver
  - alert: SkiaDotOrgRefreshFail
    expr: liveness_docsy_docset_refresh_s > 600
    labels:
      category: infra
      severity: critical
      owner: jcgregorio@google.com
    annotations:
      abbr: '{{ $labels.app }}'
      description: 'docsyserver has failed to successfully refresh in the last 10 minutes. Check the logs for docsyserver and look for errors.'