{"meta":{"github_repo":"https://github.com/PixiBixi/prometheus-rules","branch":"main"},"rules":[{"file":"aerospike.records.yml","name":"aerospike.records","category":"Databases","rules":[{"kind":"record","record":"aerospike:namespace_client_timeout:ratio5m","expr":"100 * (\n    rate(aerospike_namespace_client_read_timeout[5m])\n  + rate(aerospike_namespace_client_write_timeout[5m])\n)\n/ (\n    rate(aerospike_namespace_client_read_success[5m])\n  + rate(aerospike_namespace_client_write_success[5m])\n  + rate(aerospike_namespace_client_read_timeout[5m])\n  + rate(aerospike_namespace_client_write_timeout[5m])\n  > 10\n)\n","metric_refs":["aerospike_namespace_client_read_success","aerospike_namespace_client_read_timeout","aerospike_namespace_client_write_success","aerospike_namespace_client_write_timeout"],"line":7}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/aerospike.records.yml"},{"file":"aerospike.rules.yml","name":"aerospike.rules","category":"Databases","rules":[{"kind":"alert","alert":"AerospikeExporterDown","expr":"# Two distinct failure modes: a failing scrape (up == 0, exporter crashed\n# or host unreachable) and the target vanishing from service discovery,\n# where no up series exists to be 0. Both branches are grouped\n# identically so the alert carries the same labels either way.\ngroup by (job, instance) (up{job=~\".*aerospike.*\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=~\".*aerospike.*\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=~\".*aerospike.*\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"Job {{ $labels.job }} having an issue on {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"AerospikeDown","expr":"sum(aerospike_node_up) by (job,instance,cluster_name) == 0","for":"30m","severity":"critical","summary":"Node {{ $labels.instance }} having an issue on {{ $labels.cluster_name }}","description":"{{ $labels.instance }} of cluster {{ $labels.cluster_name }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":["aerospike_node_up","cluster_name"],"line":22},{"kind":"alert","alert":"AerospikeMultipleNodesDown","expr":"count by (job, cluster_name) (aerospike_node_up == 0) >= 2","for":"5m","severity":"critical","summary":"AS - {{ $value }} nodes down on {{ $labels.cluster_name }}","description":"{{ $value }} nodes of cluster {{ $labels.cluster_name }} are down at the same time. With replication factor 2, some partitions have no copy left.","runbook_url":"","metric_refs":["aerospike_node_up","cluster_name"],"line":33},{"kind":"alert","alert":"NamespaceIsDownWrite","expr":"sum(max_over_time(aerospike_namespace_stop_writes[15m])) by (cluster_name,ns,job) > 0","for":"3m","severity":"critical","summary":"AS - Down write for {{ $labels.ns }} @ {{ $labels.cluster_name }}","description":"AS can't write any bin on {{ $labels.ns }} @ {{ $labels.cluster_name }}, please dig ASAP","runbook_url":"","metric_refs":["aerospike_namespace_stop_writes","cluster_name"],"line":43},{"kind":"alert","alert":"UnavailablePartitions","expr":"sum(aerospike_namespace_unavailable_partitions) by (ns,cluster_name,job) > 0","for":"1m","severity":"critical","summary":"AS - Partitions unavailable for {{ $labels.ns }} @ {{ $labels.cluster_name }}","description":"AS can't access to all partitions for {{ $labels.ns }} @ {{ $labels.cluster_name }}","runbook_url":"","metric_refs":["aerospike_namespace_unavailable_partitions","cluster_name"],"line":53},{"kind":"alert","alert":"DeviceLowOnFreespace","expr":"sum(aerospike_namespace_device_available_pct) by (instance,ns,job) < 15","for":"1m","severity":"critical","summary":"AS - Low Freespace for {{ $labels.ns }} @ {{ $labels.instance }}","description":"AS Low Freespace for {{ $labels.ns }} @ {{ $labels.instance }}","runbook_url":"","metric_refs":["aerospike_namespace_device_available_pct"],"line":61},{"kind":"alert","alert":"RAMExceedHighWatermarkPCT","expr":"100-(sum(aerospike_namespace_memory_free_pct) by (instance,ns)) > (sum(aerospike_namespace_high_water_memory_pct) by (instance,ns))","for":"1m","severity":"warning","summary":"AS - Low Memory for {{ $labels.ns }} @ {{ $labels.instance }} (< 20%)","description":"AS Low Memory for {{ $labels.ns }} @ {{ $labels.instance }}, if it continues, cluster will be down for OOM, please dig","runbook_url":"","metric_refs":["aerospike_namespace_high_water_memory_pct","aerospike_namespace_memory_free_pct"],"line":69},{"kind":"alert","alert":"DeviceCloseToStopWrite","expr":"100 - (sum(aerospike_namespace_memory_free_pct) by (instance,ns))\n> (sum(aerospike_namespace_stop_writes_pct) by (instance,ns)) - 15\n","for":"1m","severity":"critical","summary":"AS - Stop Write Limit close for {{ $labels.ns }} @ {{ $labels.instance }} (< 15%)","description":"AS stop write threshold (aerospike_namespace_stop_writes_pct) for ns={{ $labels.ns }} @ {{ $labels.instance }} is nearly reached - only 15% remaining before writes are blocked.","runbook_url":"","metric_refs":["aerospike_namespace_memory_free_pct","aerospike_namespace_stop_writes_pct"],"line":77},{"kind":"alert","alert":"AerospikeAvailCloseToStopWrites","expr":"aerospike_namespace_data_avail_pct - (aerospike_namespace_storage_engine_stop_writes_avail_pct > 0) <= 15","for":"15m","severity":"warning","summary":"AS - avail stop-writes margin low for {{ $labels.ns }} @ {{ $labels.instance }}","description":"{{ $labels.ns }} @ {{ $labels.instance }} ({{ $labels.storage_engine }}) is {{ $value | printf \"%.1f\" }} points from its avail stop-writes threshold (<= 15)","runbook_url":"","metric_refs":["aerospike_namespace_data_avail_pct","aerospike_namespace_storage_engine_stop_writes_avail_pct"],"line":95},{"kind":"alert","alert":"AerospikeAvailCloseToStopWritesCritical","expr":"aerospike_namespace_data_avail_pct - (aerospike_namespace_storage_engine_stop_writes_avail_pct > 0) <= 8","for":"5m","severity":"critical","summary":"AS - avail stop-writes margin low for {{ $labels.ns }} @ {{ $labels.instance }}","description":"{{ $labels.ns }} @ {{ $labels.instance }} ({{ $labels.storage_engine }}) is {{ $value | printf \"%.1f\" }} points from its avail stop-writes threshold (<= 8)","runbook_url":"","metric_refs":["aerospike_namespace_data_avail_pct","aerospike_namespace_storage_engine_stop_writes_avail_pct"],"line":103},{"kind":"alert","alert":"AerospikeUsedCloseToStopWrites","expr":"(aerospike_namespace_storage_engine_stop_writes_used_pct > 0) - aerospike_namespace_data_used_pct <= 15","for":"15m","severity":"warning","summary":"AS - used stop-writes margin low for {{ $labels.ns }} @ {{ $labels.instance }}","description":"{{ $labels.ns }} @ {{ $labels.instance }} ({{ $labels.storage_engine }}) is {{ $value | printf \"%.1f\" }} points from its used stop-writes threshold (<= 15)","runbook_url":"","metric_refs":["aerospike_namespace_data_used_pct","aerospike_namespace_storage_engine_stop_writes_used_pct"],"line":111},{"kind":"alert","alert":"AerospikeUsedCloseToStopWritesCritical","expr":"(aerospike_namespace_storage_engine_stop_writes_used_pct > 0) - aerospike_namespace_data_used_pct <= 8","for":"5m","severity":"critical","summary":"AS - used stop-writes margin low for {{ $labels.ns }} @ {{ $labels.instance }}","description":"{{ $labels.ns }} @ {{ $labels.instance }} ({{ $labels.storage_engine }}) is {{ $value | printf \"%.1f\" }} points from its used stop-writes threshold (<= 8)","runbook_url":"","metric_refs":["aerospike_namespace_data_used_pct","aerospike_namespace_storage_engine_stop_writes_used_pct"],"line":119},{"kind":"alert","alert":"ClockAreUnsync","expr":"sum(aerospike_node_stats_cluster_clock_skew_stop_writes_sec) by (cluster_name,instance,job) > 20","for":"1m","severity":"warning","summary":"AS - Clock skew high on {{ $labels.instance }} (> 20s)","description":"Ensure clocks are synchronized across the cluster. {{ $labels.instance }} clock skew > 20s","runbook_url":"","metric_refs":["aerospike_node_stats_cluster_clock_skew_stop_writes_sec","cluster_name"],"line":127},{"kind":"alert","alert":"ClockAreUnsyncCritical","expr":"sum(aerospike_node_stats_cluster_clock_skew_stop_writes_sec) by (cluster_name,instance,job) > 40","for":"1m","severity":"critical","summary":"AS - Clock skew critical on {{ $labels.instance }} (> 40s)","description":"Ensure clocks are synchronized across the cluster. {{ $labels.instance }} clock skew > 40s - writes may be stopped","runbook_url":"","metric_refs":["aerospike_node_stats_cluster_clock_skew_stop_writes_sec","cluster_name"],"line":137},{"kind":"alert","alert":"AerospikeNamespaceClientTimeoutsHigh","expr":"aerospike:namespace_client_timeout:ratio5m > 1","for":"10m","severity":"warning","summary":"AS - Client timeouts high for {{ $labels.ns }} @ {{ $labels.cluster_name }}","description":"{{ $value | printf \"%.2f\" }}% of client transactions on {{ $labels.ns }} @ {{ $labels.cluster_name }} time out (> 1%)","runbook_url":"","metric_refs":["aerospike:namespace_client_timeout:ratio5m"],"line":145},{"kind":"alert","alert":"AerospikeNamespaceClientTimeoutsCritical","expr":"aerospike:namespace_client_timeout:ratio5m > 5","for":"5m","severity":"critical","summary":"AS - Client timeouts critical for {{ $labels.ns }} @ {{ $labels.cluster_name }}","description":"{{ $value | printf \"%.2f\" }}% of client transactions on {{ $labels.ns }} @ {{ $labels.cluster_name }} time out (> 5%)","runbook_url":"","metric_refs":["aerospike:namespace_client_timeout:ratio5m"],"line":153},{"kind":"alert","alert":"LowHeapEfficiency","expr":"sum(aerospike_node_stats_heap_efficiency_pct) by (instance) < 50","for":"1m","severity":"critical","summary":"AS - Heap Efficiency {{ $labels.instance }} (< 50%)","description":"Heap efficiency is very low for {{ $labels.instance }} (< 50%). You should check your configuration","runbook_url":"","metric_refs":["aerospike_node_stats_heap_efficiency_pct"],"line":161},{"kind":"alert","alert":"AerospikeBackupFail","expr":"backup_aerospike_state < 1","for":"2m","severity":"warning","summary":"Backup for {{ $labels.instance }}/{{ $labels.namespace }} has failed","description":"The last backup run of {{ $labels.namespace }} reported a failure. A restore would use the previous successful backup: check the backup job logs on {{ $labels.instance }}.","runbook_url":"","metric_refs":["backup_aerospike_state"],"line":169},{"kind":"alert","alert":"AerospikePopBackupFail","expr":"sum(backup_aerospike_state) by (pop) == 0","for":"2m","severity":"critical","summary":"All Aerospike backups for pop {{ $labels.pop }} have failed","description":"No Aerospike backup succeeded in pop {{ $labels.pop }}. A shared cause (storage target, credentials, network) is more likely than a single node.","runbook_url":"","metric_refs":["backup_aerospike_state"],"line":177},{"kind":"alert","alert":"AerospikeDeadPartitions","expr":"sum(aerospike_namespace_dead_partitions) by (ns,cluster_name,job) > 0","for":"3m","severity":"critical","summary":"AS - Dead partitions for {{ $labels.ns }} @ {{ $labels.cluster_name }}","description":"{{ $labels.ns }} @ {{ $labels.cluster_name }} has {{ $value | printf \"%.0f\" }} dead partition(s): potential data loss, writes to them are refused until they are revived.","runbook_url":"","metric_refs":["aerospike_namespace_dead_partitions","cluster_name"],"line":185},{"kind":"alert","alert":"AerospikeNamespaceClockSkewStopWrites","expr":"aerospike_namespace_clock_skew_stop_writes == 1","for":"3m","severity":"critical","summary":"AS - Stop-writes due to clock skew for {{ $labels.ns }} @ {{ $labels.cluster_name }}","description":"{{ $labels.ns }} @ {{ $labels.cluster_name }} rejects writes because of cluster clock skew, check NTP on every node.","runbook_url":"","metric_refs":["aerospike_namespace_clock_skew_stop_writes"],"line":193},{"kind":"alert","alert":"AerospikeNodeClockSkew","expr":"aerospike_node_stats_cluster_clock_skew_ms > 2000","for":"3m","severity":"warning","summary":"AS - Clock skew above 2s on cluster {{ $labels.cluster_name }}","description":"Cluster {{ $labels.cluster_name }} clock skew is {{ $value | printf \"%.0f\" }}ms (> 2s) as seen from {{ $labels.instance }}. Ensure clocks are synchronized.","runbook_url":"","metric_refs":["aerospike_node_stats_cluster_clock_skew_ms"],"line":203},{"kind":"alert","alert":"AerospikeNodeClockSkewCritical","expr":"aerospike_node_stats_cluster_clock_skew_ms > 20000","for":"3m","severity":"critical","summary":"AS - Clock skew above 20s on cluster {{ $labels.cluster_name }}","description":"Cluster {{ $labels.cluster_name }} clock skew is {{ $value | printf \"%.0f\" }}ms (> 20s) as seen from {{ $labels.instance }}, writes may be stopped.","runbook_url":"","metric_refs":["aerospike_node_stats_cluster_clock_skew_ms"],"line":211},{"kind":"alert","alert":"AerospikeLowSystemMemory","expr":"aerospike_node_stats_system_free_mem_pct < 15\nor\n(aerospike_node_stats_system_free_mem_pct < 17 and on (cluster_name, service) ALERTS{alertname=\"AerospikeLowSystemMemory\", alertstate=\"firing\"})\n","for":"3m","severity":"warning","summary":"AS - Low system memory on {{ $labels.service }} @ {{ $labels.cluster_name }} (< 15%)","description":"System free memory of {{ $labels.service }} @ {{ $labels.cluster_name }} is {{ $value | printf \"%.1f\" }}%. Stop-writes triggers at 10% free.","runbook_url":"","metric_refs":["aerospike_node_stats_system_free_mem_pct","cluster_name"],"line":219},{"kind":"alert","alert":"AerospikeLowSystemMemoryCritical","expr":"aerospike_node_stats_system_free_mem_pct < 12","for":"3m","severity":"critical","summary":"AS - Low system memory on {{ $labels.service }} @ {{ $labels.cluster_name }} (< 12%)","description":"System free memory of {{ $labels.service }} @ {{ $labels.cluster_name }} is {{ $value | printf \"%.1f\" }}%, close to the 10% stop-writes threshold.","runbook_url":"","metric_refs":["aerospike_node_stats_system_free_mem_pct"],"line":233},{"kind":"alert","alert":"AerospikeMigrationsStuck","expr":"sum by (cluster_name, ns) (aerospike_namespace_migrate_tx_partitions_remaining + aerospike_namespace_migrate_rx_partitions_remaining) > 0\nand\ndelta(sum by (cluster_name, ns) (aerospike_namespace_migrate_tx_partitions_remaining + aerospike_namespace_migrate_rx_partitions_remaining)[1h:1m]) >= 0\n","for":"1h","severity":"warning","summary":"AS - Migrations stuck for {{ $labels.ns }} @ {{ $labels.cluster_name }}","description":"{{ $value | printf \"%.0f\" }} partitions of {{ $labels.ns }} @ {{ $labels.cluster_name }} are migrating with no progress for 1h. Migrations delay rolling restarts: check node size and the migrate settings.","runbook_url":"","metric_refs":["aerospike_namespace_migrate_rx_partitions_remaining","aerospike_namespace_migrate_tx_partitions_remaining","cluster_name"],"line":243},{"kind":"alert","alert":"AerospikeXdrLag","expr":"max by (cluster_name, dc) (aerospike_xdr_lag) > 600","for":"15m","severity":"warning","summary":"AS - XDR lag to {{ $labels.dc }} from {{ $labels.cluster_name }}","description":"XDR lag from {{ $labels.cluster_name }} to {{ $labels.dc }} is {{ $value | humanizeDuration }} (> 10m). The remote datacenter is the fallback for losing a cluster.","runbook_url":"","metric_refs":["aerospike_xdr_lag","cluster_name"],"line":256}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/aerospike.rules.yml"},{"file":"apache.rules.yml","name":"apache.rules","category":"Web & load balancing","rules":[{"kind":"alert","alert":"ApacheExporterDown","expr":"group by (job, instance) (up{job=~\".*apache.*\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=~\".*apache.*\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=~\".*apache.*\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"Apache exporter down - {{ $labels.job }} @ {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"ApacheDown","expr":"apache_up == 0","for":"2m","severity":"critical","summary":"Apache down on {{ $labels.instance }}","description":"The exporter on {{ $labels.instance }} cannot reach Apache's mod_status endpoint. Apache is likely stopped or refusing connections.","runbook_url":"","metric_refs":["apache_up"],"line":23},{"kind":"alert","alert":"ApacheWorkersSaturation","expr":"sum by (instance, job) (apache_workers{state=\"busy\"})\n/ (sum by (instance, job) (apache_workers) > 0)\n> 0.80\n","for":"5m","severity":"warning","summary":"Apache worker saturation on {{ $labels.instance }} ({{ $value | humanizePercentage }})","description":"More than 80% of Apache workers are busy on {{ $labels.instance }}. Consider raising MaxRequestWorkers before the pool is exhausted.","runbook_url":"","metric_refs":["apache_workers"],"line":41},{"kind":"alert","alert":"ApacheWorkersSaturationCritical","expr":"sum by (instance, job) (apache_workers{state=\"busy\"})\n/ (sum by (instance, job) (apache_workers) > 0)\n> 0.90\n","for":"5m","severity":"critical","summary":"Apache worker pool nearly exhausted on {{ $labels.instance }} ({{ $value | humanizePercentage }})","description":"More than 90% of Apache workers are busy on {{ $labels.instance }}. New connections will start queueing.","runbook_url":"","metric_refs":["apache_workers"],"line":53},{"kind":"alert","alert":"ApacheNoFreeSlot","expr":"apache_scoreboard{state=\"open_slot\"} == 0","for":"2m","severity":"critical","summary":"Apache has no free scoreboard slot on {{ $labels.instance }}","description":"Apache on {{ $labels.instance }} has exhausted its scoreboard: ServerLimit is reached and no further worker can be spawned.","runbook_url":"","metric_refs":["apache_scoreboard"],"line":65},{"kind":"alert","alert":"ApacheRestarted","expr":"apache_uptime_seconds_total < 300","for":"0m","severity":"warning","summary":"Apache restarted on {{ $labels.instance }}","description":"Apache on {{ $labels.instance }} has been up for {{ $value | humanizeDuration }}. A graceful reload does not reset this counter, so the process actually restarted.","runbook_url":"","metric_refs":["apache_uptime_seconds_total"],"line":79},{"kind":"alert","alert":"ApacheWorkersStopping","expr":"apache_processes{state=\"stopping\"} > 0","for":"15m","severity":"warning","summary":"Apache has workers stuck stopping on {{ $labels.instance }}","description":"{{ $value }} Apache worker(s) on {{ $labels.instance }} have been in the stopping state for 15 minutes. A graceful restart is not completing.","runbook_url":"","metric_refs":["apache_processes"],"line":90},{"kind":"alert","alert":"ApacheSlowRequests","expr":"rate(apache_duration_ms_total[5m])\n/ (rate(apache_accesses_total[5m]) > 1)\n> 1000\n","for":"10m","severity":"warning","summary":"Apache average request duration high on {{ $labels.instance }} ({{ $value | humanize }}ms)","description":"Mean request duration on {{ $labels.instance }} is above 1s over the last 5 minutes.","runbook_url":"","metric_refs":["apache_accesses_total","apache_duration_ms_total"],"line":103}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/apache.rules.yml"},{"file":"argo-rollouts.rules.yml","name":"argo-rollouts.rules","category":"Other","rules":[{"kind":"alert","alert":"ArgoRolloutsControllerDown","expr":"absent(up{job=~\".*argo-rollouts.*\"} == 1)","for":"10m","severity":"critical","summary":"No Argo Rollouts controller answers","description":"Rollouts are no longer reconciled: canary and blue-green deployments are blocked.","runbook_url":"","metric_refs":[],"line":6},{"kind":"alert","alert":"ArgoRolloutsRolloutFailed","expr":"max by (exported_namespace, namespace, name, phase) (rollout_info{phase=~\"Abort|Error|Timeout|InvalidSpec\"}) == 1","for":"10m","severity":"warning","summary":"Rollout {{ if $labels.exported_namespace }}{{ $labels.exported_namespace }}{{ else }}{{ $labels.namespace }}{{ end }}/{{ $labels.name }} is in phase {{ $labels.phase }}","description":"The new version of the rollout is not being promoted.","runbook_url":"","metric_refs":["exported_namespace","rollout_info"],"line":16},{"kind":"alert","alert":"ArgoRolloutsRolloutReconcileErrors","expr":"sum by (exported_namespace, namespace, name) (increase(rollout_reconcile_error[15m])) > 0","for":"30m","severity":"warning","summary":"Argo Rollouts cannot reconcile {{ if $labels.exported_namespace }}{{ $labels.exported_namespace }}{{ else }}{{ $labels.namespace }}{{ end }}/{{ $labels.name }} ({{ $value | printf \"%.0f\" }} errors in 15m)","description":"The controller keeps failing to reconcile this rollout. Check the controller logs.","runbook_url":"","metric_refs":["exported_namespace","rollout_reconcile_error"],"line":25}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/argo-rollouts.rules.yml"},{"file":"argocd.rules.yml","name":"argocd.rules","category":"Other","rules":[{"kind":"alert","alert":"ArgoCDAppOutOfSync","expr":"max by (job, project, name, sync_status) (\n  argocd_app_info{autosync_enabled=\"true\", sync_status!=\"Synced\"}\n) == 1\n","for":"1h","severity":"warning","summary":"ArgoCD application {{ $labels.name }} cannot sync ({{ $labels.sync_status }})","description":"Auto-sync has not converged for application {{ $labels.name }} (project {{ $labels.project }}) for over 1 hour, so merged changes do not reach the cluster. Read .status.operationState.message on the Application.","runbook_url":"","metric_refs":["argocd_app_info","sync_status"],"line":4},{"kind":"alert","alert":"ArgoCDAppUnhealthy","expr":"max by (job, project, name, health_status) (\n  argocd_app_info{health_status=~\"Degraded|Missing\"}\n) == 1\n","for":"15m","severity":"warning","summary":"ArgoCD application {{ $labels.name }} is {{ $labels.health_status }}","description":"Application {{ $labels.name }} (project {{ $labels.project }}) has been {{ $labels.health_status }} for 15 minutes.","runbook_url":"","metric_refs":["argocd_app_info","health_status"],"line":16},{"kind":"alert","alert":"ArgoCDAppSyncFailed","expr":"sum by (job, project, name) (\n  increase(argocd_app_sync_total{phase=~\"Error|Failed\", dry_run=\"false\"}[30m])\n) > 0\n","for":"5m","severity":"warning","summary":"ArgoCD sync of {{ $labels.name }} failed","description":"Application {{ $labels.name }} (project {{ $labels.project }}) had {{ $value | humanize }} failed sync operations in the last 30 minutes.","runbook_url":"","metric_refs":["argocd_app_sync_total"],"line":28},{"kind":"alert","alert":"ArgoCDClusterConnectionFailed","expr":"max by (job, server) (\n  argocd_cluster_connection_status\n) == 0\n","for":"10m","severity":"critical","summary":"ArgoCD cannot reach cluster {{ $labels.server }}","description":"The application controller has had no working connection to cluster {{ $labels.server }} for 10 minutes. Applications targeting it are not reconciled.","runbook_url":"","metric_refs":["argocd_cluster_connection_status"],"line":40},{"kind":"alert","alert":"ArgoCDRepoUnreachable","expr":"sum by (job, repo) (\n  increase(argocd_git_lsremote_fail_total[15m])\n) > 0\n","for":"15m","severity":"warning","summary":"ArgoCD repo-server cannot reach {{ $labels.repo }}","description":"git ls-remote against {{ $labels.repo }} keeps failing. Applications sourced from it stop seeing new commits.","runbook_url":"","metric_refs":["argocd_git_lsremote_fail_total"],"line":52}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/argocd.rules.yml"},{"file":"basis.records.yml","name":"basis.records","category":"Hosts & hardware","rules":[{"kind":"record","record":"instance:node_memory_available:ratio","expr":"(\n  node_memory_MemAvailable_bytes or\n  (\n    node_memory_Buffers_bytes +\n    node_memory_Cached_bytes +\n    node_memory_MemFree_bytes +\n    node_memory_Slab_bytes\n  )\n) / node_memory_MemTotal_bytes\n","metric_refs":["node_memory_Buffers_bytes","node_memory_Cached_bytes","node_memory_MemAvailable_bytes","node_memory_MemFree_bytes","node_memory_MemTotal_bytes","node_memory_Slab_bytes"],"line":10},{"kind":"record","record":"instance:node_memory_utilization:ratio","expr":"1 - instance:node_memory_available:ratio","metric_refs":["instance:node_memory_available:ratio"],"line":22},{"kind":"record","record":"instance:node_cpus:count","expr":"count without (cpu, mode) (\n  node_cpu_seconds_total{mode=\"idle\"}\n)\n","metric_refs":["node_cpu_seconds_total"],"line":26},{"kind":"record","record":"instance_cpu:node_cpu_seconds_not_idle:rate1m","expr":"sum without (mode) (\n  1 - rate(node_cpu_seconds_total{mode=\"idle\"}[1m])\n)\n","metric_refs":["node_cpu_seconds_total"],"line":32},{"kind":"record","record":"instance_mode:node_cpu_seconds:rate1m","expr":"sum without (cpu) (\n  rate(node_cpu_seconds_total[1m])\n)\n","metric_refs":["node_cpu_seconds_total"],"line":46},{"kind":"record","record":"instance:node_cpu_utilization:ratio","expr":"avg without (cpu) (\n  instance_cpu:node_cpu_seconds_not_idle:rate1m\n)\n","metric_refs":["instance_cpu:node_cpu_seconds_not_idle:rate1m"],"line":52},{"kind":"record","record":"job:node_cpu_utilization:min_ratio","expr":"min without (fqdn,instance,node,pod) (\n  instance:node_cpu_utilization:ratio\n)\n","metric_refs":["instance:node_cpu_utilization:ratio"],"line":58},{"kind":"record","record":"job:node_cpu_utilization:avg_ratio","expr":"avg without (fqdn,instance,node,pod) (\n  instance:node_cpu_utilization:ratio\n)\n","metric_refs":["instance:node_cpu_utilization:ratio"],"line":63},{"kind":"record","record":"job:node_cpu_utilization:max_ratio","expr":"max without (fqdn,instance,node,pod) (\n  instance:node_cpu_utilization:ratio\n)\n","metric_refs":["instance:node_cpu_utilization:ratio"],"line":68},{"kind":"record","record":"instance_device:node_nic_receive_saturation:ratio","expr":"rate(node_network_receive_bytes_total{device!~\"lo|veth.*|cali.*|cilium.*|docker.*|virbr.*\"}[5m]) / (node_network_speed_bytes{device!~\"lo|veth.*|cali.*|cilium.*|docker.*|virbr.*\"} > 0)\n","metric_refs":["node_network_receive_bytes_total","node_network_speed_bytes"],"line":74},{"kind":"record","record":"instance_device:node_nic_transmit_saturation:ratio","expr":"rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|cali.*|cilium.*|docker.*|virbr.*\"}[5m]) / (node_network_speed_bytes{device!~\"lo|veth.*|cali.*|cilium.*|docker.*|virbr.*\"} > 0)\n","metric_refs":["node_network_speed_bytes","node_network_transmit_bytes_total"],"line":78}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/basis.records.yml"},{"file":"basis.rules.yml","name":"basis.rules","category":"Hosts & hardware","rules":[{"kind":"alert","alert":"HostDown","expr":"# Two distinct failure modes: a failing scrape (up == 0, exporter crashed\n# or host unreachable) and the target vanishing from service discovery,\n# where no up series exists to be 0. Both branches are grouped\n# identically so the alert carries the same labels either way.\ngroup by (job, instance) (up{job=~\".*node-exporter\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=~\".*node-exporter\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=~\".*node-exporter\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"Host down {{$labels.instance}} (node_exporter)","description":"node_exporter seems down on {{$labels.instance}}, then the host is probably down","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"CPUThrottled","expr":"sum(rate(node_cpu_core_throttles_total[5m]) > 0) by (instance)","for":"5m","severity":"warning","summary":"CPU Throttled on {{ $labels.instance }}","description":"CPU has been throttled on {{ $labels.instance }} for at least 5 minutes, it directly impact performances because it reduces frequency. Please check T\u00b0 of your processor","runbook_url":"","metric_refs":["node_cpu_core_throttles_total"],"line":23},{"kind":"alert","alert":"SlowCPU","expr":"avg(node_cpu_scaling_frequency_hertz) by (instance) < avg(node_cpu_frequency_max_hertz) by (instance)*0.30","for":"5m","severity":"warning","summary":"Slow CPU on {{ $labels.instance }}","description":"Slow CPU on {{ $labels.instance }} ({{ $value }}) for at least 5 minutes (< 30% of maximum frequency). It directly impact performances because it reduces frequency.","runbook_url":"","metric_refs":["node_cpu_frequency_max_hertz","node_cpu_scaling_frequency_hertz"],"line":31},{"kind":"alert","alert":"CantScrapeTextfile","expr":"node_textfile_scrape_error > 0","for":"5m","severity":"critical","summary":"Cant scrape {{ $labels.instance }} file","description":"Cant scrape {{ $labels.instance }} for more than 5 minutes, please check the instance","runbook_url":"","metric_refs":["node_textfile_scrape_error"],"line":40},{"kind":"alert","alert":"OldMetricsTextfile","expr":"node_textfile_mtime_seconds{file!~\".*(server_metrics|aerospike_backup).*\"} < time() - 60\nor\nnode_textfile_mtime_seconds{file=~\".*aerospike_backup.*\"} < time() - 3700\n","for":"2m","severity":"critical","summary":"Old metrics of {{ $labels.file }} on {{ $labels.instance }}","description":"`{{ $labels.file }}` on {{ $labels.instance }} was last written {{ $value | humanizeTimestamp }}: check the job that writes it.","runbook_url":"","metric_refs":["node_textfile_mtime_seconds"],"line":48},{"kind":"alert","alert":"HighDroppedPackages","expr":"(\n  irate(node_network_receive_drop_total{ignoredDroppedPackage=\"\"}[5m])\n  / (irate(node_network_receive_packets_total[5m]) > 0)\n) > 0.1\n","for":"5m","severity":"critical","summary":"High Drop on {{ $labels.instance }}","description":"High drop detected (> 10%) on {{ $labels.device }} @ {{ $labels.instance }}. Misconfiguration or NIC has an issue","runbook_url":"","metric_refs":["node_network_receive_drop_total","node_network_receive_packets_total"],"line":61},{"kind":"alert","alert":"HighListenDrop","expr":"(\n  irate(node_netstat_TcpExt_ListenDrops[1m])\n  /\n  irate(node_netstat_Tcp_PassiveOpens[1m])\n) > 0.10\nand\nirate(node_netstat_Tcp_PassiveOpens[1m]) > 1\n","for":"5m","severity":"critical","summary":"High TCP listen drops on {{ $labels.instance }}","description":"More than 10% of incoming TCP connections are being dropped on {{ $labels.instance }} ({{ $value | humanizePercentage }}) - accept queue full, check backlog/somaxconn","runbook_url":"","metric_refs":["node_netstat_TcpExt_ListenDrops","node_netstat_Tcp_PassiveOpens"],"line":75},{"kind":"alert","alert":"ConntrackAlmostFull","expr":"(node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.80","for":"5m","severity":"warning","summary":"{{ $labels.instance }} of job {{ $labels.job }} has its conntrack stack almost full, please check if HAproxy/whatever is running fine, also, check sysctl conntrack value","description":"{{ $labels.instance }} of job {{ $labels.job }} has its conntrack stack almost full","runbook_url":"","metric_refs":["node_nf_conntrack_entries","node_nf_conntrack_entries_limit"],"line":90},{"kind":"alert","alert":"NodeRAIDDiskFailure","expr":"node_md_disks{state=\"failed\",device!=\"\"} > 0","for":"1m","severity":"critical","summary":"Failed device in RAID array","description":"At least one device in RAID array on {{ $labels.instance }} failed. Array \"{{ $labels.device }}\" needs attention and possibly a disk swap.","runbook_url":"","metric_refs":["node_md_disks"],"line":101},{"kind":"alert","alert":"RaidDegraded","expr":"sum(node_md_disks{state=\"active\"}) by (instance,device,job) < sum(node_md_disks_required) by (instance,device,job)","for":"1m","severity":"critical","summary":"MDRaid is degraded {{ $labels.device }} for {{ $labels.instance }}","description":"A disk seems disabled by kernel {{ $labels.device }} for {{ $labels.instance }}","runbook_url":"","metric_refs":["node_md_disks","node_md_disks_required"],"line":110},{"kind":"alert","alert":"NTPUnsync","expr":"sum(node_timex_sync_status < 1) by (instance,job)","for":"1m","severity":"critical","summary":"NTP Unsync for `{{ $labels.instance }}`, take a look to `timedatectl status` and 123 UDP OUT FW","description":"NTP Unsync {{ $labels.instance }}","runbook_url":"","metric_refs":["node_timex_sync_status"],"line":122},{"kind":"alert","alert":"HostLowOfMemory","expr":"sum ((node_memory_MemAvailable_bytes{ignoreMonitoringMemory=\"\"} / node_memory_MemTotal_bytes * 100) < 10) by (instance,job)","for":"5m","severity":"critical","summary":"Host out of memory (instance {{ $labels.instance }})","description":"Node memory is filling up (< 10% left)\n VALUE = {{ $value }}","runbook_url":"","metric_refs":["node_memory_MemAvailable_bytes","node_memory_MemTotal_bytes"],"line":133},{"kind":"alert","alert":"HostOomKillDetected","expr":"increase(node_vmstat_oom_kill{ignoreOOM=\"\"}[5m]) > 0","for":"0m","severity":"critical","summary":"Host OOM kill detected instance on {{ $labels.instance }}","description":"OOM kill detected\n","runbook_url":"","metric_refs":["node_vmstat_oom_kill"],"line":143},{"kind":"alert","alert":"Segfault","expr":"server_metrics{dimension=\"segfault\"} > 0","for":"0m","severity":"critical","summary":"Segfault detected on {{ $labels.instance }}","description":"Segfault has been detected in the last minute, please check `/var/log/kern.log`","runbook_url":"","metric_refs":["server_metrics"],"line":153},{"kind":"alert","alert":"HostOutOfDiskSpace","expr":"(node_filesystem_avail_bytes{fstype!=\"tmpfs\",mountpoint!~\"(/boot|/boot/efi)\",job!=\"rds-enhanced\",ignoreDiskUsage=\"\"} / node_filesystem_size_bytes * 100) < 20","for":"5m","severity":"critical","summary":"Host out of disk space (instance {{ $labels.instance }})","description":"Disk is almost full (< 20% left)\n  Instance: {{ $labels.instance }}\n  Mount: {{ $labels.mountpoint }}\n  Free: {{ $value | humanize }}%","runbook_url":"","metric_refs":["node_filesystem_avail_bytes","node_filesystem_size_bytes"],"line":164},{"kind":"alert","alert":"NodeFilesystemAlmostOutOfFiles","expr":"(\n  node_filesystem_files_free{fstype!=\"tmpfs\",mountpoint!~\"(/boot|/boot/efi)\"} / node_filesystem_files{fstype!=\"\",mountpoint!=\"\"} * 100 < 10\nand\n  node_filesystem_readonly{fstype!=\"\",mountpoint!=\"\"} == 0\n)\n","for":"1h","severity":"critical","summary":"Filesystem has less than 10% inodes left.","description":"Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left.","runbook_url":"","metric_refs":["node_filesystem_files","node_filesystem_files_free","node_filesystem_readonly"],"line":173},{"kind":"alert","alert":"DownSystemdNginx","expr":"node_systemd_unit_state{state=\"failed\",name=\"nginx.service\",ignoreNginxDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for nginx on {{ $labels.instance }}","description":"Systemd failed for nginx, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":191},{"kind":"alert","alert":"DownSystemdDocker","expr":"node_systemd_unit_state{state=\"failed\",name=\"docker.service\",ignoreDockerDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for docker on {{ $labels.instance }}","description":"Systemd failed for docker, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":199},{"kind":"alert","alert":"DownSystemdMySQL","expr":"node_systemd_unit_state{state=\"failed\",name=\"mysql.service\",ignoreMysqlDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for MySQL on {{ $labels.instance }}","description":"Systemd failed for MySQL, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":207},{"kind":"alert","alert":"DownSystemdAerospike","expr":"node_systemd_unit_state{state=\"failed\",name=\"aerospike.service\",ignoreAerospikeDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for aerospike {{ $labels.instance }}","description":"Systemd failed for aerospike, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":215},{"kind":"alert","alert":"DownSystemdASGateway","expr":"node_systemd_unit_state{state=\"failed\",name=\"as-rest-gateway.service\",ignoreASRestDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for as-rest-gateway {{ $labels.instance }}","description":"Systemd failed for as-rest-gateway, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":223},{"kind":"alert","alert":"DownSystemdPromtail","expr":"node_systemd_unit_state{state=\"failed\",name=\"promtail.service\",ignorePromtailDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for promtail {{ $labels.instance }}","description":"Systemd failed for promtail, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":231},{"kind":"alert","alert":"DownSystemdCron","expr":"node_systemd_unit_state{state=\"failed\",name=~\"crond?\\\\.service\",ignoreCronDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for cron {{ $labels.instance }}","description":"Systemd failed for cron, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":239},{"kind":"alert","alert":"DownSystemdSSH","expr":"node_systemd_unit_state{state=\"failed\",name=~\"sshd?\\\\.service\",ignoreSSHDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for SSH {{ $labels.instance }}","description":"Systemd failed for SSH, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":247},{"kind":"alert","alert":"DownSystemdHAproxy","expr":"node_systemd_unit_state{state=\"failed\",name=\"haproxy.service\",ignoreHAproxyDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for HAproxy {{ $labels.instance }}","description":"Systemd failed for HAproxy, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":255},{"kind":"alert","alert":"DownSystemdKeyDB","expr":"node_systemd_unit_state{state=\"failed\",name=\"keydb.service\",ignoreKeydbDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for KeyDB {{ $labels.instance }}","description":"Systemd failed for KeyDB, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":263},{"kind":"alert","alert":"DownSystemdRedis","expr":"node_systemd_unit_state{state=\"failed\",name=~\"redis_.*.service\",ignoreRedisDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for Redis {{ $labels.instance }}","description":"Systemd failed for Redis, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":271},{"kind":"alert","alert":"DownSystemdBidRequest","expr":"node_systemd_unit_state{state=\"failed\",name=~\"bid_request_.*.service\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for Bid Request Logger {{ $labels.instance }}","description":"Systemd failed for Bid Request Logger, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":279},{"kind":"alert","alert":"DownSystemdPHPFPM","expr":"node_systemd_unit_state{state=\"failed\",name=~\"php.*-fpm.service\",ignorePHPDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for PHPFPM {{ $labels.instance }}","description":"Systemd failed for PHPFPM, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":287},{"kind":"alert","alert":"DownSystemdNTP","expr":"node_systemd_unit_state{state=\"failed\",name=~\"(chrony|systemd-timesyncd).*\",ignoreNTPDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for NTP {{ $labels.instance }}","description":"Systemd failed for NTP, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":295},{"kind":"alert","alert":"DownSystemdTeleport","expr":"node_systemd_unit_state{state=\"failed\",name=~\"teleport.service\",ignoreTeleportDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for Teleport {{ $labels.instance }}","description":"Systemd failed for Teleport, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":303},{"kind":"alert","alert":"DownSystemdKafkaConnect","expr":"node_systemd_unit_state{state=\"failed\",name=\"confluent-kafka-connect.service\",ignoreKafkaConnectDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for {{ $labels.name }} @ {{ $labels.instance }}","description":"Systemd failed for {{ $labels.name }}, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":311},{"kind":"alert","alert":"DownSystemdSchemaRegistry","expr":"node_systemd_unit_state{state=\"failed\",name=\"confluent-schema-registry.service\",ignoreSchemaRegistryDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for {{ $labels.name }} @ {{ $labels.instance }}","description":"Systemd failed for {{ $labels.name }}, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":319},{"kind":"alert","alert":"DownSystemdKsqlDB","expr":"node_systemd_unit_state{state=\"failed\",name=\"confluent-ksqldb.service\",ignoreKsqlDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for {{ $labels.name }} @ {{ $labels.instance }}","description":"Systemd failed for {{ $labels.name }}, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":327},{"kind":"alert","alert":"DownSystemdZookeeper","expr":"node_systemd_unit_state{state=\"failed\",name=\"confluent-zookeeper.service\",ignoreZookeeperDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for {{ $labels.name }} @ {{ $labels.instance }}","description":"Systemd failed for {{ $labels.name }}, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":335},{"kind":"alert","alert":"DownSystemdKafka","expr":"node_systemd_unit_state{state=\"failed\",name=\"confluent-kafka.service\",ignoreKafkaDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for {{ $labels.name }} @ {{ $labels.instance }}","description":"Systemd failed for {{ $labels.name }}, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":343},{"kind":"alert","alert":"DownSystemdKafkaRest","expr":"node_systemd_unit_state{state=\"failed\",name=\"confluent-kafka-rest.service\",ignoreKafkaRestDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for {{ $labels.name }} @ {{ $labels.instance }}","description":"Systemd failed for {{ $labels.name }}, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":351},{"kind":"alert","alert":"DownSystemdSupervisor","expr":"node_systemd_unit_state{state=\"failed\",name=\"supervisor.service\",ignoreSupervisorDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for supervisor on {{ $labels.instance }}","description":"Systemd failed for supervisor, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":359},{"kind":"alert","alert":"DownSystemdSquid","expr":"node_systemd_unit_state{state=\"failed\",name=\"squid.service\",ignoreSquidDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for {{ $labels.name }} @ {{ $labels.instance }}","description":"Systemd failed for {{ $labels.name }}, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":367},{"kind":"alert","alert":"DownSystemdOpenVPN","expr":"node_systemd_unit_state{state=\"failed\",name=~\"openvpn-client@.*\",ignoreOpenVPNDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for {{ $labels.name }} @ {{ $labels.instance }}","description":"Systemd failed for {{ $labels.name }}, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":375},{"kind":"alert","alert":"DownSystemdKeepalived","expr":"node_systemd_unit_state{state=\"failed\",name=\"keepalived.service\",ignoreKeepalivedDown=\"\"} > 0","for":"1m","severity":"critical","summary":"Systemd unit failed for {{ $labels.name }} @ {{ $labels.instance }}","description":"Systemd failed for {{ $labels.name }}, please take a look with `systemctl status {{ $labels.name }}`","runbook_url":"","metric_refs":["node_systemd_unit_state"],"line":387},{"kind":"alert","alert":"CPULoad","expr":"instance:node_cpu_utilization:ratio{ignoreCPULoad=\"\",ignoreSingleCPU=\"\"} > 0.9","for":"15m","severity":"critical","summary":"Host high CPU load (instance {{ $labels.instance }})","description":"The CPU load reported from {{ $labels.instance }} has exceeded 90% for more than 20 minutes.","runbook_url":"","metric_refs":["instance:node_cpu_utilization:ratio"],"line":397},{"kind":"alert","alert":"CPULoadRDS","expr":"node_cpu_average{mode=\"idle\"} < 10","for":"1m","severity":"critical","summary":"Host high CPU load (instance RDS {{ $labels.instance }})","description":"The CPU load reported from RDS {{ $labels.instance }} has exceeded 90% for more than 1 minute.","runbook_url":"","metric_refs":["node_cpu_average"],"line":405},{"kind":"alert","alert":"CPUwaitIO","expr":"instance_mode:node_cpu_seconds:rate1m{ignoreCPULoad=\"\",mode=\"iowait\",ignoreSingleCPU=\"\"}\n/ ignoring(mode) instance:node_cpus:count\n> 0.1\n","for":"5m","severity":"critical","summary":"High IOWait ({{ $value | humanizePercentage }}) on {{ $labels.instance }}","description":"CPU time spent waiting on I/O is above 10% on {{ $labels.instance }} - check disk latency and saturation","runbook_url":"","metric_refs":["instance:node_cpus:count","instance_mode:node_cpu_seconds:rate1m"],"line":414},{"kind":"alert","alert":"CPULoadServerType","expr":"job:node_cpu_utilization:avg_ratio > 0.80","for":"15m","severity":"critical","summary":"Host high CPU load for {{ $labels.servertype }} @ {{ $labels.pop }} : {{ $value | humanizePercentage }}","description":"The CPU load reported from {{ $labels.servertype }} on {{ $labels.pop }} has exceeded 80% for more than 15 minutes.","runbook_url":"","metric_refs":["job:node_cpu_utilization:avg_ratio"],"line":428},{"kind":"alert","alert":"NodeFilesystemFilesFillingUp","expr":"(\n  node_filesystem_files_free{fstype!=\"\",mountpoint!=\"\"} / node_filesystem_files{fstype!=\"\",mountpoint!=\"\"} * 100 < 40\n  and predict_linear(node_filesystem_files_free{fstype!=\"\",mountpoint!=\"\"}[6h], 24*60*60) < 0\n  and node_filesystem_readonly{fstype!=\"\",mountpoint!=\"\"} == 0\n)\n","for":"1h","severity":"warning","summary":"Filesystem inodes filling up on {{ $labels.instance }} ({{ $labels.mountpoint }})","description":"Filesystem {{ $labels.device }} mounted on {{ $labels.mountpoint }} at {{ $labels.instance }} is predicted to run out of inodes within 24 hours ({{ printf \"%.2f\" $value }}% free).","runbook_url":"","metric_refs":["node_filesystem_files","node_filesystem_files_free","node_filesystem_readonly"],"line":439},{"kind":"alert","alert":"NodeFilesystemFilesFillingUp","expr":"(\n  node_filesystem_files_free{fstype!=\"\",mountpoint!=\"\"} / node_filesystem_files{fstype!=\"\",mountpoint!=\"\"} * 100 < 20\n  and predict_linear(node_filesystem_files_free{fstype!=\"\",mountpoint!=\"\"}[6h], 4*60*60) < 0\n  and node_filesystem_readonly{fstype!=\"\",mountpoint!=\"\"} == 0\n)\n","for":"1h","severity":"critical","summary":"Filesystem inodes filling up fast on {{ $labels.instance }} ({{ $labels.mountpoint }})","description":"Filesystem {{ $labels.device }} mounted on {{ $labels.mountpoint }} at {{ $labels.instance }} is predicted to run out of inodes within 4 hours ({{ printf \"%.2f\" $value }}% free).","runbook_url":"","metric_refs":["node_filesystem_files","node_filesystem_files_free","node_filesystem_readonly"],"line":452},{"kind":"alert","alert":"NodeNetworkReceiveErrs","expr":"rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01\n","for":"1h","severity":"warning","summary":"High receive errors on {{ $labels.instance }} ({{ $labels.device }})","description":"Interface {{ $labels.device }} at {{ $labels.instance }} has more than 1% receive errors over the last 2 minutes.","runbook_url":"","metric_refs":["node_network_receive_errs_total","node_network_receive_packets_total"],"line":468},{"kind":"alert","alert":"NodeNetworkTransmitErrs","expr":"rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01\n","for":"1h","severity":"warning","summary":"High transmit errors on {{ $labels.instance }} ({{ $labels.device }})","description":"Interface {{ $labels.device }} at {{ $labels.instance }} has more than 1% transmit errors over the last 2 minutes.","runbook_url":"","metric_refs":["node_network_transmit_errs_total","node_network_transmit_packets_total"],"line":477},{"kind":"alert","alert":"NodeNICReceiveBandwidthSaturation","expr":"rate(node_network_receive_bytes_total{device!~\"lo|veth.*|cali.*|cilium.*|docker.*|virbr.*\"}[5m])\n/ (node_network_speed_bytes{device!~\"lo|veth.*|cali.*|cilium.*|docker.*|virbr.*\"} > 0)\n> 0.8\n","for":"5m","severity":"warning","summary":"NIC receive saturation on {{ $labels.instance }} ({{ $labels.device }})","description":"Interface {{ $labels.device }} at {{ $labels.instance }} receive utilization is {{ $value | humanizePercentage }} of link capacity (warning threshold: 80%).","runbook_url":"","metric_refs":["node_network_receive_bytes_total","node_network_speed_bytes"],"line":489},{"kind":"alert","alert":"NodeNICReceiveBandwidthSaturation","expr":"rate(node_network_receive_bytes_total{device!~\"lo|veth.*|cali.*|cilium.*|docker.*|virbr.*\"}[5m])\n/ (node_network_speed_bytes{device!~\"lo|veth.*|cali.*|cilium.*|docker.*|virbr.*\"} > 0)\n> 0.9\n","for":"5m","severity":"critical","summary":"NIC receive saturation critical on {{ $labels.instance }} ({{ $labels.device }})","description":"Interface {{ $labels.device }} at {{ $labels.instance }} receive utilization is {{ $value | humanizePercentage }} of link capacity (critical threshold: 90%).","runbook_url":"","metric_refs":["node_network_receive_bytes_total","node_network_speed_bytes"],"line":500},{"kind":"alert","alert":"NodeNICTransmitBandwidthSaturation","expr":"rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|cali.*|cilium.*|docker.*|virbr.*\"}[5m])\n/ (node_network_speed_bytes{device!~\"lo|veth.*|cali.*|cilium.*|docker.*|virbr.*\"} > 0)\n> 0.8\n","for":"5m","severity":"warning","summary":"NIC transmit saturation on {{ $labels.instance }} ({{ $labels.device }})","description":"Interface {{ $labels.device }} at {{ $labels.instance }} transmit utilization is {{ $value | humanizePercentage }} of link capacity (warning threshold: 80%).","runbook_url":"","metric_refs":["node_network_speed_bytes","node_network_transmit_bytes_total"],"line":511},{"kind":"alert","alert":"NodeNICTransmitBandwidthSaturation","expr":"rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|cali.*|cilium.*|docker.*|virbr.*\"}[5m])\n/ (node_network_speed_bytes{device!~\"lo|veth.*|cali.*|cilium.*|docker.*|virbr.*\"} > 0)\n> 0.9\n","for":"5m","severity":"critical","summary":"NIC transmit saturation critical on {{ $labels.instance }} ({{ $labels.device }})","description":"Interface {{ $labels.device }} at {{ $labels.instance }} transmit utilization is {{ $value | humanizePercentage }} of link capacity (critical threshold: 90%).","runbook_url":"","metric_refs":["node_network_speed_bytes","node_network_transmit_bytes_total"],"line":522},{"kind":"alert","alert":"NodeClockSkewDetected","expr":"(\n  node_timex_offset_seconds > 0.05\n  and deriv(node_timex_offset_seconds[5m]) >= 0\n)\nor\n(\n  node_timex_offset_seconds < -0.05\n  and deriv(node_timex_offset_seconds[5m]) <= 0\n)\n","for":"10m","severity":"warning","summary":"Clock skew detected on {{ $labels.instance }}","description":"Clock at {{ $labels.instance }} is out of sync by more than 0.05s and diverging. Check NTP configuration.","runbook_url":"","metric_refs":["node_timex_offset_seconds"],"line":536},{"kind":"alert","alert":"NodeFileDescriptorLimit","expr":"node_filefd_allocated * 100 / (node_filefd_maximum < 1e9) > 70","for":"15m","severity":"warning","summary":"File descriptor limit approaching on {{ $labels.instance }}","description":"{{ $labels.instance }} is using {{ printf \"%.2f\" $value }}% of available file descriptors (warning threshold: 70%).","runbook_url":"","metric_refs":["node_filefd_allocated","node_filefd_maximum"],"line":556},{"kind":"alert","alert":"NodeFileDescriptorLimit","expr":"node_filefd_allocated * 100 / (node_filefd_maximum < 1e9) > 90","for":"15m","severity":"critical","summary":"File descriptor limit critical on {{ $labels.instance }}","description":"{{ $labels.instance }} is using {{ printf \"%.2f\" $value }}% of available file descriptors (critical threshold: 90%).","runbook_url":"","metric_refs":["node_filefd_allocated","node_filefd_maximum"],"line":566},{"kind":"alert","alert":"NodeMemoryMajorPagesFaults","expr":"(\n  rate(node_vmstat_pgmajfault[5m])\n  /\n  rate(node_vmstat_pgfault[5m])\n) > 0.05\nand\nrate(node_vmstat_pgfault[5m]) > 100\n","for":"15m","severity":"warning","summary":"Memory major page faults high on {{ $labels.instance }}","description":"More than 5% of page faults require disk I/O on {{ $labels.instance }} ({{ $value | humanizePercentage }}) - possible memory pressure or thrashing","runbook_url":"","metric_refs":["node_vmstat_pgfault","node_vmstat_pgmajfault"],"line":579},{"kind":"alert","alert":"NodeDiskIOSaturation","expr":"rate(node_disk_io_time_seconds_total{device!=\"\"}[5m]) > 0.9","for":"30m","severity":"warning","summary":"Disk IO saturation on {{ $labels.instance }} ({{ $labels.device }})","description":"Disk {{ $labels.device }} at {{ $labels.instance }} is busy more than 90% of the time for 30 minutes ({{ $value | humanizePercentage }}) - possible disk saturation.","runbook_url":"","metric_refs":["node_disk_io_time_seconds_total"],"line":597},{"kind":"alert","alert":"NodeSystemdServiceCrashlooping","expr":"increase(node_systemd_service_restart_total[5m]) > 2","for":"15m","severity":"warning","summary":"Systemd service crash looping on {{ $labels.instance }}","description":"Service {{ $labels.name }} at {{ $labels.instance }} has restarted more than twice in 5 minutes.","runbook_url":"","metric_refs":["node_systemd_service_restart_total"],"line":608},{"kind":"alert","alert":"NodeHardwareTemperatureTooHigh","expr":"node_hwmon_temp_celsius >= node_hwmon_temp_crit_celsius","for":"2m","severity":"critical","summary":"Critical hardware temperature on {{ $labels.instance }} ({{ $labels.chip }}/{{ $labels.sensor }})","description":"Hardware temperature on {{ $labels.instance }} sensor {{ $labels.chip }}/{{ $labels.sensor }} has reached critical threshold ({{ $value }}C).","runbook_url":"","metric_refs":["node_hwmon_temp_celsius","node_hwmon_temp_crit_celsius"],"line":619},{"kind":"alert","alert":"NodeHardwareTemperatureWarning","expr":"node_hwmon_temp_celsius >= node_hwmon_temp_max_celsius","for":"5m","severity":"warning","summary":"High hardware temperature on {{ $labels.instance }} ({{ $labels.chip }}/{{ $labels.sensor }})","description":"Hardware temperature on {{ $labels.instance }} sensor {{ $labels.chip }}/{{ $labels.sensor }} above max threshold ({{ $value }}C).","runbook_url":"","metric_refs":["node_hwmon_temp_celsius","node_hwmon_temp_max_celsius"],"line":628},{"kind":"alert","alert":"NodeRaidRecovering","expr":"node_md_state{state=\"recovering\"} == 1","for":"0m","severity":"warning","summary":"RAID array recovering on {{ $labels.instance }} ({{ $labels.device }})","description":"RAID array {{ $labels.device }} on {{ $labels.instance }} is recovering. Monitor progress.","runbook_url":"","metric_refs":["node_md_state"],"line":639},{"kind":"alert","alert":"NodeNetworkLinkDown","expr":"node_network_carrier{device!~\"lo|veth.*|docker.*|br-.*|virbr.*|tun.*\",ignoreLinkDown=\"\"} == 0\nand on(instance, device) node_network_up == 1\n","for":"2m","severity":"critical","summary":"Network link down on {{ $labels.instance }} ({{ $labels.device }})","description":"Network interface {{ $labels.device }} on {{ $labels.instance }} has lost carrier.","runbook_url":"","metric_refs":["node_network_carrier","node_network_up"],"line":650},{"kind":"alert","alert":"NodeNetworkInterfaceFlapping","expr":"increase(node_network_carrier_changes_total{device!~\"lo|veth.*|docker.*|br-.*|virbr.*|tun.*\"}[5m]) > 2","for":"0m","severity":"warning","summary":"Network interface flapping on {{ $labels.instance }} ({{ $labels.device }})","description":"Network interface {{ $labels.device }} on {{ $labels.instance }} is flapping ({{ $value }} carrier changes in 5 minutes).","runbook_url":"","metric_refs":["node_network_carrier_changes_total"],"line":664},{"kind":"alert","alert":"NodeMemoryHardwareCorrupted","expr":"node_memory_HardwareCorrupted_bytes > 0","for":"0m","severity":"critical","summary":"Hardware memory corruption detected on {{ $labels.instance }}","description":"Hardware memory corruption detected on {{ $labels.instance }} ({{ $value | humanize1024 }}). Schedule hardware replacement.","runbook_url":"","metric_refs":["node_memory_HardwareCorrupted_bytes"],"line":678},{"kind":"alert","alert":"NodeTCPOrphanSockets","expr":"(node_sockstat_TCP_orphan / node_sockstat_TCP_alloc) > 0.10\nand node_sockstat_TCP_alloc > 100\n","for":"5m","severity":"warning","summary":"High TCP orphan sockets on {{ $labels.instance }}","description":"More than 10% of allocated TCP sockets are orphaned on {{ $labels.instance }} ({{ $value | humanizePercentage }}) - possible TCP connection leak","runbook_url":"","metric_refs":["node_sockstat_TCP_alloc","node_sockstat_TCP_orphan"],"line":689},{"kind":"alert","alert":"NodeSystemdNotRunning","expr":"node_systemd_system_running{alertSystemdDegraded=\"yes\"} == 0","for":"15m","severity":"warning","summary":"systemd not fully operational on {{ $labels.instance }}","description":"systemd reports system is not fully operational on {{ $labels.instance }}.","runbook_url":"","metric_refs":["node_systemd_system_running"],"line":702},{"kind":"alert","alert":"NodeBondingDegraded","expr":"(node_bonding_slaves - node_bonding_active) != 0","for":"5m","severity":"warning","summary":"Bonding interface degraded on {{ $labels.instance }}","description":"Bonding interface {{ $labels.master }} at {{ $labels.instance }} has failed member links.","runbook_url":"","metric_refs":["node_bonding_active","node_bonding_slaves"],"line":736},{"kind":"alert","alert":"NodeSwapSaturation","expr":"node_memory_SwapTotal_bytes > 0\n  and\n(node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes > 0.8\n","for":"5m","severity":"warning","summary":"Swap usage high on {{ $labels.instance }}","description":"Swap is more than 80% used on {{ $labels.instance }} - memory pressure likely.","runbook_url":"","metric_refs":["node_memory_SwapFree_bytes","node_memory_SwapTotal_bytes"],"line":747},{"kind":"alert","alert":"NodeSwapSaturationCritical","expr":"node_memory_SwapTotal_bytes > 0\n  and\n(node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes > 0.9\n","for":"5m","severity":"critical","summary":"Swap almost exhausted on {{ $labels.instance }}","description":"Swap is more than 90% used on {{ $labels.instance }} - OOM risk imminent.","runbook_url":"","metric_refs":["node_memory_SwapFree_bytes","node_memory_SwapTotal_bytes"],"line":759},{"kind":"alert","alert":"NodeActiveSwapping","expr":"rate(node_vmstat_pswpin[5m]) + rate(node_vmstat_pswpout[5m]) > 256","for":"5m","severity":"critical","summary":"Active swapping on {{ $labels.instance }}","description":"{{ $labels.instance }} is actively swapping ({{ $value | humanize }} pages/sec) - RAM exhausted, severe performance degradation.","runbook_url":"","metric_refs":["node_vmstat_pswpin","node_vmstat_pswpout"],"line":771},{"kind":"alert","alert":"NodeProcsBlocked","expr":"node_procs_blocked > 10","for":"5m","severity":"warning","summary":"Processes blocked on I/O on {{ $labels.instance }}","description":"{{ $value }} processes blocked waiting for I/O on {{ $labels.instance }} - possible disk saturation or hung device.","runbook_url":"","metric_refs":["node_procs_blocked"],"line":782},{"kind":"alert","alert":"NodeTCPListenOverflows","expr":"increase(node_netstat_TcpExt_ListenOverflows[5m]) > 10","for":"5m","severity":"critical","summary":"TCP listen queue overflows on {{ $labels.instance }}","description":"{{ $labels.instance }} is dropping connections due to full TCP listen queue - application cannot accept connections fast enough.","runbook_url":"","metric_refs":["node_netstat_TcpExt_ListenOverflows"],"line":793},{"kind":"alert","alert":"NodeSoftnetDrops","expr":"increase(node_softnet_dropped_total[5m]) > 10","for":"5m","severity":"warning","summary":"Kernel softnet drops on {{ $labels.instance }}","description":"{{ $labels.instance }} is dropping packets at the kernel NIC ring buffer level - possible NIC saturation or insufficient RX buffer.","runbook_url":"","metric_refs":["node_softnet_dropped_total"],"line":806},{"kind":"alert","alert":"NodeFilesystemReadonly","expr":"node_filesystem_readonly{fstype!~\"(tmpfs|iso9660|squashfs)\",mountpoint!~\"(/boot|/boot/efi)\"} == 1","for":"0m","severity":"critical","summary":"Filesystem remounted read-only on {{ $labels.instance }}","description":"Filesystem {{ $labels.device }} mounted on {{ $labels.mountpoint }} at {{ $labels.instance }} is read-only - possible disk error or kernel panic.","runbook_url":"","metric_refs":["node_filesystem_readonly"],"line":819},{"kind":"alert","alert":"NodeFilesystemDeviceError","expr":"node_filesystem_device_error{fstype!~\"(tmpfs|iso9660|squashfs)\"} > 0","for":"0m","severity":"critical","summary":"Filesystem device error on {{ $labels.instance }}","description":"Filesystem {{ $labels.device }} mounted on {{ $labels.mountpoint }} at {{ $labels.instance }} reports a device error.","runbook_url":"","metric_refs":["node_filesystem_device_error"],"line":828},{"kind":"alert","alert":"NodeUDPReceiveBufferErrors","expr":"increase(node_netstat_Udp_RcvbufErrors[5m]) > 10","for":"5m","severity":"warning","summary":"UDP receive buffer errors on {{ $labels.instance }}","description":"{{ $labels.instance }} is dropping UDP packets due to full receive buffers - consider increasing net.core.rmem_max.","runbook_url":"","metric_refs":["node_netstat_Udp_RcvbufErrors"],"line":839},{"kind":"alert","alert":"NodeHighTcpRetransmitRatio","expr":"rate(node_netstat_Tcp_RetransSegs[30m])\n/\nrate(node_netstat_Tcp_OutSegs[30m])\n> 0.02\n","for":"15m","severity":"warning","summary":"High TCP retransmit ratio on {{ $labels.instance }}","description":"{{ $value | humanizePercentage }} of TCP segments sent by {{ $labels.instance }} are retransmissions, against a fleet median of 0.002%. Points at the network path rather than the host.","runbook_url":"","metric_refs":["node_netstat_Tcp_OutSegs","node_netstat_Tcp_RetransSegs"],"line":850},{"kind":"alert","alert":"RDSTasksBlocked","expr":"rdsosmetrics_tasks_blocked > 0","for":"10m","severity":"warning","summary":"RDS tasks blocked on {{ $labels.instance }}","description":"{{ $value | printf \"%.0f\" }} tasks blocked in uninterruptible sleep on {{ $labels.instance }} ({{ $labels.region }}) for 10 minutes. Usually storage that stopped answering. The instance looks idle because blocked tasks burn no CPU.","runbook_url":"","metric_refs":["rdsosmetrics_tasks_blocked"],"line":870}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/basis.rules.yml"},{"file":"blackbox.rules.yml","name":"blackbox.rules","category":"Network, DNS & TLS","rules":[{"kind":"alert","alert":"BlackboxProbeFailed","expr":"probe_success == 0","for":"1m","severity":"critical","summary":"Blackbox probe failed - {{ $labels.job }} @ {{ $labels.instance }}","description":"Probe {{ $labels.job }} failed on {{ $labels.instance }}","runbook_url":"","metric_refs":["probe_success"],"line":3},{"kind":"alert","alert":"BlackboxProbeHttpFailure","expr":"probe_http_status_code <= 199 OR probe_http_status_code >= 400","for":"1m","severity":"critical","summary":"Blackbox probe HTTP failure (instance {{ $labels.instance }})","description":"HTTP status code is not 200-399 on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["probe_http_status_code"],"line":12},{"kind":"alert","alert":"BlackboxSslCertificateWillExpireSoon","expr":"3 <= round((last_over_time(probe_ssl_earliest_cert_expiry[10m]) - time()) / 86400, 0.1) < 20","for":"0m","severity":"warning","summary":"Blackbox SSL certificate will expire soon (instance {{ $labels.instance }})","description":"SSL certificate expires in less than 20 days on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["probe_ssl_earliest_cert_expiry"],"line":21},{"kind":"alert","alert":"BlackboxSslCertificateWillExpireVerySoon","expr":"0 <= round((last_over_time(probe_ssl_earliest_cert_expiry[10m]) - time()) / 86400, 0.1) < 3","for":"0m","severity":"critical","summary":"Blackbox SSL certificate will expire very soon (instance {{ $labels.instance }})","description":"SSL certificate expires in less than 3 days on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["probe_ssl_earliest_cert_expiry"],"line":30},{"kind":"alert","alert":"BlackboxSslCertificateExpired","expr":"round((last_over_time(probe_ssl_earliest_cert_expiry[10m]) - time()) / 86400, 0.1) < 0","for":"0m","severity":"critical","summary":"Blackbox SSL certificate expired (instance {{ $labels.instance }})","description":"SSL certificate has already expired on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["probe_ssl_earliest_cert_expiry"],"line":39},{"kind":"alert","alert":"BlackboxProbeSlowHttp","expr":"probe_http_duration_seconds > 1","for":"1m","severity":"warning","summary":"Blackbox probe slow HTTP (instance {{ $labels.instance }})","description":"HTTP request took more than 1s on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["probe_http_duration_seconds"],"line":48},{"kind":"alert","alert":"BlackboxConfigurationReloadFailure","expr":"blackbox_exporter_config_last_reload_successful != 1","for":"0m","severity":"critical","summary":"Blackbox configuration reload failure (instance {{ $labels.instance }})","description":"Blackbox configuration reload failed on {{ $labels.instance }}","runbook_url":"","metric_refs":["blackbox_exporter_config_last_reload_successful"],"line":57},{"kind":"alert","alert":"BlackboxConfigReloadStale","expr":"time() - blackbox_exporter_config_last_reload_success_timestamp_seconds > 300","for":"0m","severity":"warning","summary":"Blackbox configuration reload stale (instance {{ $labels.instance }})","description":"Configuration hasn't been successfully reloaded in 5 minutes on {{ $labels.instance }}","runbook_url":"","metric_refs":["blackbox_exporter_config_last_reload_success_timestamp_seconds"],"line":66},{"kind":"alert","alert":"BlackboxUnknownModule","expr":"increase(blackbox_module_unknown_total[5m]) > 0","for":"0m","severity":"warning","summary":"Blackbox unknown module (instance {{ $labels.instance }})","description":"Scrape job references a module not defined in blackbox config on {{ $labels.instance }}","runbook_url":"","metric_refs":["blackbox_module_unknown_total"],"line":75}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/blackbox.rules.yml"},{"file":"cert-manager.rules.yml","name":"cert-manager.rules","category":"Kubernetes","rules":[{"kind":"alert","alert":"CertManagerAbsent","expr":"absent(up{job=\"cert-manager\"})","for":"10m","severity":"critical","summary":"Cert Manager absent from service discovery","description":"Cert Manager has disappeared. No certificates can be minted or renewed until it is back.","runbook_url":"https://github.com/imusmanmalik/cert-manager-mixin/blob/main/RUNBOOK.md#certmanagerabsent","metric_refs":[],"line":15},{"kind":"alert","alert":"CertManagerCertExpirySoon","expr":"avg by (exported_namespace, namespace, name) (\n  (certmanager_certificate_expiration_timestamp_seconds - time())\n  and on (exported_namespace, namespace, name)\n  (certmanager_certificate_ready_status{condition=\"True\"} == 1)\n)\n<\navg by (exported_namespace, namespace, name) (\n  certmanager_certificate_expiration_timestamp_seconds - certmanager_certificate_renewal_timestamp_seconds\n) * 0.7\n","for":"1h","severity":"warning","summary":"Certificate {{ $labels.name }} expiring soon ({{ $value | humanizeDuration }})","description":"Certificate {{ $labels.name }} in namespace {{ if $labels.exported_namespace }}{{ $labels.exported_namespace }}{{ else }}{{ $labels.namespace }}{{ end }} has not been renewed and is {{ $value | humanizeDuration }} from expiry.","runbook_url":"https://github.com/imusmanmalik/cert-manager-mixin/blob/main/RUNBOOK.md#certmanagercertexpirysoon","metric_refs":["certmanager_certificate_expiration_timestamp_seconds","certmanager_certificate_ready_status","certmanager_certificate_renewal_timestamp_seconds","exported_namespace"],"line":25},{"kind":"alert","alert":"CertManagerCertExpiryCritical","expr":"min by (exported_namespace, namespace, name) (\n  certmanager_certificate_expiration_timestamp_seconds > 0\n) - time() < 4 * 86400\n","for":"1h","severity":"critical","summary":"Certificate {{ $labels.name }} expires in {{ $value | humanizeDuration }}","description":"Certificate {{ $labels.name }} in namespace {{ if $labels.exported_namespace }}{{ $labels.exported_namespace }}{{ else }}{{ $labels.namespace }}{{ end }} expires in {{ $value | humanizeDuration }}. TLS will break on every endpoint it serves. Issue it manually if the renewal cannot be unblocked in time.","runbook_url":"https://github.com/imusmanmalik/cert-manager-mixin/blob/main/RUNBOOK.md#certmanagercertexpirysoon","metric_refs":["certmanager_certificate_expiration_timestamp_seconds","exported_namespace"],"line":44},{"kind":"alert","alert":"CertManagerCertNotReady","expr":"max by (name, exported_namespace, namespace, condition) (\n  certmanager_certificate_ready_status{condition!=\"True\"} == 1\n)\n","for":"10m","severity":"critical","summary":"Certificate {{ $labels.name }} not ready","description":"Certificate {{ $labels.name }} in namespace {{ if $labels.exported_namespace }}{{ $labels.exported_namespace }}{{ else }}{{ $labels.namespace }}{{ end }} is not ready (condition={{ $labels.condition }}) for more than 10 minutes.","runbook_url":"https://github.com/imusmanmalik/cert-manager-mixin/blob/main/RUNBOOK.md#certmanagercertnotready","metric_refs":["certmanager_certificate_ready_status","exported_namespace"],"line":59},{"kind":"alert","alert":"CertManagerIssuerNotReady","expr":"max by (name, exported_namespace, namespace, condition) (\n  certmanager_issuer_ready_status{condition!=\"True\"} == 1\n)\n","for":"10m","severity":"critical","summary":"cert-manager Issuer {{ $labels.name }} not ready","description":"Issuer {{ $labels.name }} in namespace {{ if $labels.exported_namespace }}{{ $labels.exported_namespace }}{{ else }}{{ $labels.namespace }}{{ end }} is not ready (condition={{ $labels.condition }}). Every certificate it backs has stopped renewing.","runbook_url":"","metric_refs":["certmanager_issuer_ready_status","exported_namespace"],"line":72},{"kind":"alert","alert":"CertManagerClusterIssuerNotReady","expr":"max by (name, condition) (\n  certmanager_clusterissuer_ready_status{condition!=\"True\"} == 1\n)\n","for":"10m","severity":"critical","summary":"cert-manager ClusterIssuer {{ $labels.name }} not ready","description":"ClusterIssuer {{ $labels.name }} is not ready (condition={{ $labels.condition }}). Certificates across every namespace it backs have stopped renewing.","runbook_url":"","metric_refs":["certmanager_clusterissuer_ready_status"],"line":89},{"kind":"alert","alert":"CertManagerHittingRateLimits","expr":"sum by (host) (\n  rate(certmanager_http_acme_client_request_count{status=\"429\"}[5m])\n) > 0\n","for":"5m","severity":"critical","summary":"Cert Manager hitting Let's Encrypt rate limits on {{ $labels.host }}","description":"Cert Manager is receiving HTTP 429 from Let's Encrypt for {{ $labels.host }}. Rate limit can block certificate issuance for up to 7 days.","runbook_url":"https://github.com/imusmanmalik/cert-manager-mixin/blob/main/RUNBOOK.md#certmanagerhittingratelimits","metric_refs":["certmanager_http_acme_client_request_count"],"line":104},{"kind":"alert","alert":"CertManagerControllerSyncErrors","expr":"rate(certmanager_controller_sync_error_count[5m]) / rate(certmanager_controller_sync_call_count[5m]) > 0.1","for":"5m","severity":"warning","summary":"Cert Manager controller {{ $labels.controller }} sync error rate is high","description":"cert-manager controller {{ $labels.controller }} is failing more than 10% of sync operations. Certificate issuance may be delayed.","runbook_url":"","metric_refs":["certmanager_controller_sync_call_count","certmanager_controller_sync_error_count"],"line":117},{"kind":"alert","alert":"CertManagerACMEHighErrorRate","expr":"sum by (instance, job, host) (rate(certmanager_http_acme_client_request_duration_seconds_count{status=~\"4..|5..\"}[5m]))\n/\nsum by (instance, job, host) (rate(certmanager_http_acme_client_request_duration_seconds_count[5m]))\n> 0.05\n","for":"5m","severity":"warning","summary":"Cert Manager ACME client high error rate on {{ $labels.host }}","description":"cert-manager ACME client has more than 5% error rate ({{ $value | humanizePercentage }}) talking to {{ $labels.host }}.","runbook_url":"","metric_refs":["certmanager_http_acme_client_request_duration_seconds_count"],"line":126}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/cert-manager.rules.yml"},{"file":"certs.rules.yml","name":"certs.rules","category":"Network, DNS & TLS","rules":[{"kind":"alert","alert":"CertExpireLessThanFourDays","expr":"min(probe_ssl_last_chain_expiry_timestamp_seconds{job=~\"blackbox.*\"} - time()) by (job, instance, cust) / 86400 < 4","for":"1m","severity":"critical","summary":"Certificate for {{ $labels.job }} @ {{ $labels.instance }} will expire soon (< 4 days)","description":"It seems HAproxy/whatever didnt reloaded well. You should probably dig in this way (or check the certificate itself)","runbook_url":"","metric_refs":["probe_ssl_last_chain_expiry_timestamp_seconds"],"line":4},{"kind":"alert","alert":"OCSPInvalid","expr":"sum(ssl_ocsp_response_status{job=\"ssl\"}) by (instance,job) > 0","for":"1m","severity":"critical","summary":"OCSP for {{ $labels.instance }} is invalid","description":"OCSP response status for {{ $labels.instance }} is invalid (status > 0)","runbook_url":"","metric_refs":["ssl_ocsp_response_status"],"line":15}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/certs.rules.yml"},{"file":"controller-runtime.rules.yml","name":"controller-runtime.rules","category":"Other","rules":[{"kind":"alert","alert":"ControllerRuntimeReconcileErrors","expr":"sum by (job, controller) (\n  rate(controller_runtime_reconcile_errors_total[10m])\n) > 0.05\n","for":"15m","severity":"warning","summary":"Controller {{ $labels.controller }} ({{ $labels.job }}) is failing to reconcile","description":"Controller {{ $labels.controller }} of {{ $labels.job }} has been erroring at {{ $value | humanize }}/s for 15 minutes. Objects it owns are not converging. Check the operator logs.","runbook_url":"","metric_refs":["controller_runtime_reconcile_errors_total"],"line":3},{"kind":"alert","alert":"ControllerRuntimeReconcilePanics","expr":"sum by (job, controller) (\n  increase(controller_runtime_reconcile_panics_total[15m])\n) > 0\n","for":"5m","severity":"warning","summary":"Controller {{ $labels.controller }} ({{ $labels.job }}) panicked during reconcile","description":"Controller {{ $labels.controller }} of {{ $labels.job }} recovered from {{ $value | humanize }} reconcile panics in the last 15 minutes. The offending object is retried and will keep panicking.","runbook_url":"","metric_refs":["controller_runtime_reconcile_panics_total"],"line":15},{"kind":"alert","alert":"ControllerRuntimeWebhookErrors","expr":"sum by (job, webhook) (\n  rate(controller_runtime_webhook_requests_total{code!=\"200\"}[10m])\n) > 0.05\n","for":"10m","severity":"critical","summary":"Admission webhook {{ $labels.webhook }} ({{ $labels.job }}) is returning errors","description":"Webhook {{ $labels.webhook }} of {{ $labels.job }} has been returning non-200 at {{ $value | humanize }}/s for 10 minutes. With a failurePolicy of Fail the API server rejects the matching objects.","runbook_url":"","metric_refs":["controller_runtime_webhook_requests_total"],"line":27},{"kind":"alert","alert":"ControllerRuntimeWorkqueueStuck","expr":"max by (job, name) (\n  workqueue_unfinished_work_seconds\n) > 300\n","for":"10m","severity":"warning","summary":"Workqueue {{ $labels.name }} ({{ $labels.job }}) is not draining","description":"Workqueue {{ $labels.name }} of {{ $labels.job }} has had unfinished work for {{ $value | humanizeDuration }}. A reconcile is wedged rather than failing, so nothing errors but nothing progresses.","runbook_url":"","metric_refs":["workqueue_unfinished_work_seconds"],"line":39}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/controller-runtime.rules.yml"},{"file":"coredns.rules.yml","name":"coredns.rules","category":"Network, DNS & TLS","rules":[{"kind":"alert","alert":"CoreDNSDown","expr":"absent(up{job=\"kube-dns\"} == 1)","for":"15m","severity":"critical","summary":"CoreDNS has disappeared from Prometheus target discovery","description":"CoreDNS is down. No DNS resolution is possible for in-cluster traffic.","runbook_url":"https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednsdown","metric_refs":[],"line":3},{"kind":"alert","alert":"CoreDNSLatencyHigh","expr":"histogram_quantile(0.99,\n  sum(rate(coredns_dns_request_duration_seconds_bucket{job=\"kube-dns\"}[5m])) without (instance, pod)\n) > 4\n","for":"10m","severity":"critical","summary":"CoreDNS p99 latency high ({{ $value | humanizeDuration }})","description":"CoreDNS 99th percentile latency is {{ $value | humanizeDuration }} for server {{ $labels.server }} zone {{ $labels.zone }}.","runbook_url":"https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednslatencyhigh","metric_refs":["coredns_dns_request_duration_seconds_bucket"],"line":13},{"kind":"alert","alert":"CoreDNSErrorsHigh","expr":"sum without (pod, instance, server, zone, view, rcode, plugin) (rate(coredns_dns_responses_total{job=\"kube-dns\", rcode=\"SERVFAIL\"}[5m]))\n/\nsum without (pod, instance, server, zone, view, rcode, plugin) (rate(coredns_dns_responses_total{job=\"kube-dns\"}[5m])) > 0.03\n","for":"10m","severity":"critical","summary":"CoreDNS high SERVFAIL rate ({{ $value | humanizePercentage }})","description":"CoreDNS is returning SERVFAIL for {{ $value | humanizePercentage }} of requests.","runbook_url":"https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednserrorshigh","metric_refs":["coredns_dns_responses_total"],"line":26},{"kind":"alert","alert":"CoreDNSErrorsElevated","expr":"sum without (pod, instance, server, zone, view, rcode, plugin) (rate(coredns_dns_responses_total{job=\"kube-dns\", rcode=\"SERVFAIL\"}[5m]))\n/\nsum without (pod, instance, server, zone, view, rcode, plugin) (rate(coredns_dns_responses_total{job=\"kube-dns\"}[5m])) > 0.01\n","for":"10m","severity":"warning","summary":"CoreDNS elevated SERVFAIL rate ({{ $value | humanizePercentage }})","description":"CoreDNS is returning SERVFAIL for {{ $value | humanizePercentage }} of requests.","runbook_url":"https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednserrorselevated","metric_refs":["coredns_dns_responses_total"],"line":38},{"kind":"alert","alert":"CoreDNSForwardLatencyHigh","expr":"histogram_quantile(0.99,\n  sum(rate(coredns_proxy_request_duration_seconds_bucket{job=\"kube-dns\"}[5m])) without (pod, instance, rcode)\n  or\n  sum(rate(coredns_forward_request_duration_seconds_bucket{job=\"kube-dns\"}[5m])) without (pod, instance, rcode)\n) > 4\n","for":"10m","severity":"critical","summary":"CoreDNS forward p99 latency high ({{ $value | humanizeDuration }}) to {{ $labels.to }}","description":"CoreDNS 99th percentile latency forwarding to {{ $labels.to }} is {{ $value | humanizeDuration }}.","runbook_url":"https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednsforwardlatencyhigh","metric_refs":["coredns_forward_request_duration_seconds_bucket","coredns_proxy_request_duration_seconds_bucket"],"line":51},{"kind":"alert","alert":"CoreDNSForwardErrorsHigh","expr":"(\n  sum without (pod, instance, rcode) (rate(coredns_forward_responses_total{job=\"kube-dns\", rcode=\"SERVFAIL\"}[5m]))\n  or\n  sum without (pod, instance, rcode) (rate(coredns_proxy_responses_total{job=\"kube-dns\", rcode=\"SERVFAIL\"}[5m]))\n)\n/\n(\n  sum without (pod, instance, rcode) (rate(coredns_forward_responses_total{job=\"kube-dns\"}[5m]))\n  or\n  sum without (pod, instance, rcode) (rate(coredns_proxy_responses_total{job=\"kube-dns\"}[5m]))\n) > 0.03\n","for":"10m","severity":"critical","summary":"CoreDNS high forward SERVFAIL rate to {{ $labels.to }} ({{ $value | humanizePercentage }})","description":"CoreDNS is returning SERVFAIL for {{ $value | humanizePercentage }} of forward requests to {{ $labels.to }}.","runbook_url":"https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednsforwarderrorshigh","metric_refs":["coredns_forward_responses_total","coredns_proxy_responses_total"],"line":70},{"kind":"alert","alert":"CoreDNSForwardErrorsElevated","expr":"(\n  sum without (pod, instance, rcode) (rate(coredns_forward_responses_total{job=\"kube-dns\", rcode=\"SERVFAIL\"}[5m]))\n  or\n  sum without (pod, instance, rcode) (rate(coredns_proxy_responses_total{job=\"kube-dns\", rcode=\"SERVFAIL\"}[5m]))\n)\n/\n(\n  sum without (pod, instance, rcode) (rate(coredns_forward_responses_total{job=\"kube-dns\"}[5m]))\n  or\n  sum without (pod, instance, rcode) (rate(coredns_proxy_responses_total{job=\"kube-dns\"}[5m]))\n) > 0.01\n","for":"10m","severity":"warning","summary":"CoreDNS elevated forward SERVFAIL rate to {{ $labels.to }} ({{ $value | humanizePercentage }})","description":"CoreDNS is returning SERVFAIL for {{ $value | humanizePercentage }} of forward requests to {{ $labels.to }}.","runbook_url":"https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednsforwarderrorselevated","metric_refs":["coredns_forward_responses_total","coredns_proxy_responses_total"],"line":94},{"kind":"alert","alert":"CoreDNSForwardHealthcheckFailureCount","expr":"(\n  sum without (pod, instance) (rate(coredns_proxy_healthcheck_failures_total{job=\"kube-dns\"}[5m]))\n  or\n  sum without (pod, instance) (rate(coredns_forward_healthcheck_failures_total{job=\"kube-dns\"}[5m]))\n) > 0\n","for":"10m","severity":"warning","summary":"CoreDNS healthcheck failures to upstream {{ $labels.to }}","description":"CoreDNS health checks are failing to upstream server {{ $labels.to }}.","runbook_url":"https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednsforwardhealthcheckfailurecount","metric_refs":["coredns_forward_healthcheck_failures_total","coredns_proxy_healthcheck_failures_total"],"line":115},{"kind":"alert","alert":"CoreDNSForwardHealthcheckBrokenCount","expr":"sum without (pod, instance) (rate(coredns_forward_healthcheck_broken_total{job=\"kube-dns\"}[5m])) > 0\n","for":"10m","severity":"critical","summary":"CoreDNS all upstream healthchecks failing","description":"CoreDNS health checks have failed for all upstream DNS servers. Forward resolution is broken.","runbook_url":"https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednsforwardhealthcheckbrokencount","metric_refs":["coredns_forward_healthcheck_broken_total"],"line":130}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/coredns.rules.yml"},{"file":"external-dns.rules.yml","name":"external-dns.rules","category":"Network, DNS & TLS","rules":[{"kind":"alert","alert":"ExternalDnsReconciliationFailing","expr":"max by (job, namespace) (external_dns_controller_consecutive_soft_errors) > 0","for":"10m","severity":"critical","summary":"external-dns failing to reconcile ({{ $labels.namespace }})","description":"external-dns in {{ $labels.namespace }} reports {{ $value }} consecutive failed reconciliations. DNS records are no longer created, updated or deleted. Check the DNS provider credentials and API quota.","runbook_url":"","metric_refs":["external_dns_controller_consecutive_soft_errors"],"line":6}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/external-dns.rules.yml"},{"file":"external-secrets.rules.yml","name":"external-secrets.rules","category":"Kubernetes","rules":[{"kind":"alert","alert":"ExternalSecretsSyncDegraded","expr":"count(externalsecret_status_condition{condition=\"Ready\", status=\"False\"} == 1)\n/\nmax_over_time(count(externalsecret_status_condition{condition=\"Ready\"} == 1)[1h:1m])\n> 0.05\n","for":"15m","severity":"warning","summary":"More than 5% of ExternalSecrets are failing to sync","description":"{{ $value | humanizePercentage }} of ExternalSecrets have been Ready=False for 15 minutes. A shared cause (secret store credentials, IAM binding) is far more likely than that many independent failures.","runbook_url":"","metric_refs":["externalsecret_status_condition"],"line":13},{"kind":"alert","alert":"ExternalSecretsSyncMassFailure","expr":"(\n  count(externalsecret_status_condition{condition=\"Ready\", status=\"False\"} == 1)\n  /\n  max_over_time(count(externalsecret_status_condition{condition=\"Ready\"} == 1)[1h:1m])\n  > 0.25\n)\nand\ncount(externalsecret_status_condition{condition=\"Ready\", status=\"False\"} == 1) > 10\n","for":"5m","severity":"critical","summary":"Over a quarter of ExternalSecrets are failing to sync","description":"{{ $value | humanizePercentage }} of ExternalSecrets, and more than 10 of them, have been Ready=False for 5 minutes. Check the secret store credentials used by the operator.","runbook_url":"","metric_refs":["externalsecret_status_condition"],"line":26}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/external-secrets.rules.yml"},{"file":"haproxy.records.yml","name":"haproxy.records","category":"Web & load balancing","rules":[{"kind":"record","record":"haproxy:backend_http_responses:rate5m","expr":"sum by (namespace, proxy, instance, job, code) (rate(haproxy_backend_http_responses_total[5m]))","metric_refs":["haproxy_backend_http_responses_total"],"line":24},{"kind":"record","record":"haproxy:backend_http_responses_5xx:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_backend_http_responses_total{code=\"5xx\"}[5m]))","metric_refs":["haproxy_backend_http_responses_total"],"line":27},{"kind":"record","record":"haproxy:backend_http_responses_4xx:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_backend_http_responses_total{code=\"4xx\"}[5m]))","metric_refs":["haproxy_backend_http_responses_total"],"line":30},{"kind":"record","record":"haproxy:backend_bytes_in:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_server_bytes_in_total[5m]))","metric_refs":["haproxy_server_bytes_in_total"],"line":33},{"kind":"record","record":"haproxy:backend_bytes_out:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_server_bytes_out_total[5m]))","metric_refs":["haproxy_server_bytes_out_total"],"line":36},{"kind":"record","record":"haproxy:backend_sessions:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_server_sessions_total[5m]))","metric_refs":["haproxy_server_sessions_total"],"line":39},{"kind":"record","record":"haproxy:backend_connection_errors:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_server_connection_errors_total[5m]))","metric_refs":["haproxy_server_connection_errors_total"],"line":42},{"kind":"record","record":"haproxy:backend_response_errors:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_server_response_errors_total[5m]))","metric_refs":["haproxy_server_response_errors_total"],"line":45},{"kind":"record","record":"haproxy:backend_connection_attempts:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_backend_connection_attempts_total[5m]))","metric_refs":["haproxy_backend_connection_attempts_total"],"line":48},{"kind":"record","record":"haproxy:backend_connection_reuses:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_backend_connection_reuses_total[5m]))","metric_refs":["haproxy_backend_connection_reuses_total"],"line":51},{"kind":"record","record":"haproxy:backend_5xx_error_rate:ratio5m","expr":"haproxy:backend_http_responses_5xx:rate5m\n/ clamp_min(\n    sum by (namespace, proxy, instance, job) (haproxy:backend_http_responses:rate5m),\n    0.01\n  ) * 100\n","metric_refs":["haproxy:backend_http_responses:rate5m","haproxy:backend_http_responses_5xx:rate5m"],"line":57},{"kind":"record","record":"haproxy:backend_4xx_error_rate:ratio5m","expr":"haproxy:backend_http_responses_4xx:rate5m\n/ clamp_min(\n    sum by (namespace, proxy, instance, job) (haproxy:backend_http_responses:rate5m),\n    0.01\n  ) * 100\n","metric_refs":["haproxy:backend_http_responses:rate5m","haproxy:backend_http_responses_4xx:rate5m"],"line":69},{"kind":"record","record":"haproxy:backend_error_rate:ratio5m","expr":"sum by (namespace, proxy, instance, job) (haproxy:backend_http_responses:rate5m{code=~\"4xx|5xx\"})\n/ clamp_min(\n    sum by (namespace, proxy, instance, job) (haproxy:backend_http_responses:rate5m),\n    0.01\n  ) * 100\n","metric_refs":["haproxy:backend_http_responses:rate5m"],"line":78},{"kind":"record","record":"haproxy:backend_session_saturation:ratio","expr":"sum by (namespace, proxy, instance, job) (haproxy_backend_current_sessions)\n/ (sum by (namespace, proxy, instance, job) (haproxy_backend_limit_sessions) > 0)\n* 100\n","metric_refs":["haproxy_backend_current_sessions","haproxy_backend_limit_sessions"],"line":89},{"kind":"record","record":"haproxy:backend_connection_reuse:ratio5m","expr":"haproxy:backend_connection_reuses:rate5m\n/ clamp_min(\n    haproxy:backend_connection_reuses:rate5m + haproxy:backend_connection_attempts:rate5m,\n    0.01\n  ) * 100\n","metric_refs":["haproxy:backend_connection_attempts:rate5m","haproxy:backend_connection_reuses:rate5m"],"line":99},{"kind":"record","record":"haproxy:backend_aborts:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_backend_client_aborts_total[5m]))\n+ sum by (namespace, proxy, instance, job) (rate(haproxy_backend_server_aborts_total[5m]))\n","metric_refs":["haproxy_backend_client_aborts_total","haproxy_backend_server_aborts_total"],"line":110},{"kind":"record","record":"haproxy:backend_retries:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_backend_retry_warnings_total[5m]))","metric_refs":["haproxy_backend_retry_warnings_total"],"line":116},{"kind":"record","record":"haproxy:backend_downtime:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_backend_downtime_seconds_total[5m]))","metric_refs":["haproxy_backend_downtime_seconds_total"],"line":120},{"kind":"record","record":"haproxy:server_http_responses:rate5m","expr":"sum by (namespace, proxy, server, instance, job, code) (rate(haproxy_server_http_responses_total[5m]))","metric_refs":["haproxy_server_http_responses_total"],"line":129},{"kind":"record","record":"haproxy:frontend_http_responses_5xx:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_frontend_http_responses_total{code=\"5xx\"}[5m]))","metric_refs":["haproxy_frontend_http_responses_total"],"line":134},{"kind":"record","record":"haproxy:frontend_http_responses:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_frontend_http_responses_total[5m]))","metric_refs":["haproxy_frontend_http_responses_total"],"line":137},{"kind":"record","record":"haproxy:frontend_5xx_error_rate:ratio5m","expr":"haproxy:frontend_http_responses_5xx:rate5m\n/ clamp_min(haproxy:frontend_http_responses:rate5m, 0.01)\n* 100\n","metric_refs":["haproxy:frontend_http_responses:rate5m","haproxy:frontend_http_responses_5xx:rate5m"],"line":141},{"kind":"record","record":"haproxy:frontend_denied_connections:rate5m","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_frontend_denied_connections_total[5m]))","metric_refs":["haproxy_frontend_denied_connections_total"],"line":148}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/haproxy.records.yml"},{"file":"haproxy.rules.yml","name":"haproxy.rules","category":"Web & load balancing","rules":[{"kind":"alert","alert":"HAproxyDown","expr":"# Two distinct failure modes: a failing scrape (up == 0, exporter crashed\n# or host unreachable) and the target vanishing from service discovery,\n# where no up series exists to be 0. Both branches are grouped\n# identically so the alert carries the same labels either way.\ngroup by (job, instance) (up{job=~\".*haproxy.*\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=~\".*haproxy.*\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=~\".*haproxy.*\"}[5m]))\n)\n","for":"5m","severity":"critical","summary":"HAProxy scrape target down - {{ $labels.job }} @ {{ $labels.instance }}","description":"{{ $labels.job }} @ {{ $labels.instance }} has been down for more than 5 minutes","runbook_url":"","metric_refs":[],"line":5},{"kind":"alert","alert":"HAproxyProcessStopping","expr":"haproxy_process_stopping == 1","for":"5m","severity":"critical","summary":"HAProxy process stopping - {{ $labels.instance }}","description":"HAProxy worker process is in stopping/draining state on {{ $labels.instance }} - possible crash or stuck reload","runbook_url":"","metric_refs":["haproxy_process_stopping"],"line":25},{"kind":"alert","alert":"HAproxyProcessPoolFailures","expr":"rate(haproxy_process_pool_failures_total[5m]) > 0","for":"0m","severity":"critical","summary":"HAProxy process memory pool failures - {{ $labels.instance }}","description":"HAProxy memory pool allocation failing on {{ $labels.instance }} - OOM risk, connections may be rejected","runbook_url":"","metric_refs":["haproxy_process_pool_failures_total"],"line":37},{"kind":"alert","alert":"HAproxyProcessIdleLow","expr":"haproxy_process_idle_time_percent < 10","for":"5m","severity":"warning","summary":"HAProxy process idle low - {{ $labels.instance }}","description":"HAProxy process idle time below 10% on {{ $labels.instance }} - process is saturated","runbook_url":"","metric_refs":["haproxy_process_idle_time_percent"],"line":46},{"kind":"alert","alert":"HAproxyProcessDroppedLogs","expr":"rate(haproxy_process_dropped_logs_total[5m]) > 0","for":"5m","severity":"warning","summary":"HAProxy process dropped logs - {{ $labels.instance }}","description":"HAProxy is dropping log messages on {{ $labels.instance }} - syslog or log socket overloaded","runbook_url":"","metric_refs":["haproxy_process_dropped_logs_total"],"line":55},{"kind":"alert","alert":"HAproxyProcessFailedResolutions","expr":"increase(haproxy_process_failed_resolutions[5m]) > 0","for":"5m","severity":"warning","summary":"HAProxy DNS resolution failures - {{ $labels.instance }}","description":"HAProxy is failing DNS resolutions on {{ $labels.instance }} - backends may become unreachable","runbook_url":"","metric_refs":["haproxy_process_failed_resolutions"],"line":64},{"kind":"alert","alert":"HAproxyMaxconnConnectionReuseRisk","expr":"haproxy_process_max_connections > 0\n  and\nhaproxy_process_current_connections / haproxy_process_max_connections > 0.5\n","for":"10m","severity":"warning","summary":"HAProxy maxconn saturation disabling connection reuse - {{ $labels.instance }}","description":"HAProxy connections at {{ $value | humanizePercentage }} of maxconn on {{ $labels.instance }}. Above ~50%, HAProxy stops pooling idle backend connections to keep FD usage under the global limit (tune.pool-low-fd-ratio / tune.pool-high-fd-ratio). Increase maxconn and FD limits (ulimit -n), tune pool FD ratios, or reduce frontend session count.","runbook_url":"","metric_refs":["haproxy_process_current_connections","haproxy_process_max_connections"],"line":73},{"kind":"alert","alert":"HAproxyProcessConnectionsCritical","expr":"haproxy_process_max_connections > 0\n  and\nhaproxy_process_current_connections / haproxy_process_max_connections > 0.9\n","for":"5m","severity":"critical","summary":"HAProxy process connections critical - {{ $labels.instance }}","description":"HAProxy current connections exceed 90% of max on {{ $labels.instance }} - imminent connection rejection","runbook_url":"","metric_refs":["haproxy_process_current_connections","haproxy_process_max_connections"],"line":85},{"kind":"alert","alert":"HAproxyProcessConnectionsSaturation","expr":"haproxy_process_max_connections > 0\n  and\nhaproxy_process_current_connections / haproxy_process_max_connections > 0.8\n","for":"5m","severity":"warning","summary":"HAProxy process connections near limit - {{ $labels.instance }}","description":"HAProxy current connections exceed 80% of max on {{ $labels.instance }} - risk of connection rejection","runbook_url":"","metric_refs":["haproxy_process_current_connections","haproxy_process_max_connections"],"line":97},{"kind":"alert","alert":"HAproxyProcessConnectionsNearHardMax","expr":"haproxy_process_hard_max_connections > 0\n  and\nhaproxy_process_current_connections / haproxy_process_hard_max_connections > 0.7\n","for":"15m","severity":"warning","summary":"HAProxy connections near hard max - {{ $labels.instance }}","description":"HAProxy current connections exceed 70% of the hard connection ceiling on {{ $labels.instance }}, already beyond the configured maxconn","runbook_url":"","metric_refs":["haproxy_process_current_connections","haproxy_process_hard_max_connections"],"line":109},{"kind":"alert","alert":"HAproxyProcessConnectionsNearHardMaxCritical","expr":"haproxy_process_hard_max_connections > 0\n  and\nhaproxy_process_current_connections / haproxy_process_hard_max_connections > 0.9\n","for":"15m","severity":"critical","summary":"HAProxy connections critically near hard max - {{ $labels.instance }}","description":"HAProxy current connections exceed 90% of the hard connection ceiling on {{ $labels.instance }} - the process is about to be unable to accept new sockets","runbook_url":"","metric_refs":["haproxy_process_current_connections","haproxy_process_hard_max_connections"],"line":123},{"kind":"alert","alert":"HAproxyProcessSslConnectionsCritical","expr":"haproxy_process_max_ssl_connections > 0\n  and\nhaproxy_process_current_ssl_connections / haproxy_process_max_ssl_connections > 0.9\n","for":"5m","severity":"critical","summary":"HAProxy SSL connections critical - {{ $labels.instance }}","description":"HAProxy current SSL connections exceed 90% of max on {{ $labels.instance }} - imminent SSL endpoint rejection","runbook_url":"","metric_refs":["haproxy_process_current_ssl_connections","haproxy_process_max_ssl_connections"],"line":135},{"kind":"alert","alert":"HAproxyProcessSslConnectionsSaturation","expr":"haproxy_process_max_ssl_connections > 0\n  and\nhaproxy_process_current_ssl_connections / haproxy_process_max_ssl_connections > 0.8\n","for":"5m","severity":"warning","summary":"HAProxy SSL connections near limit - {{ $labels.instance }}","description":"HAProxy current SSL connections exceed 80% of max on {{ $labels.instance }}","runbook_url":"","metric_refs":["haproxy_process_current_ssl_connections","haproxy_process_max_ssl_connections"],"line":147},{"kind":"alert","alert":"HAproxyConnectionRateNearLimit","expr":"haproxy_process_current_connection_rate / (haproxy_process_limit_connection_rate > 0) > 0.8","for":"5m","severity":"warning","summary":"HAProxy connection rate near limit - {{ $labels.instance }}","description":"HAProxy connection rate at 80% of limit on {{ $labels.instance }}","runbook_url":"","metric_refs":["haproxy_process_current_connection_rate","haproxy_process_limit_connection_rate"],"line":159},{"kind":"alert","alert":"HAproxyProcessSessionRateNearLimit","expr":"haproxy_process_limit_session_rate > 0\n  and\nhaproxy_process_current_session_rate / haproxy_process_limit_session_rate > 0.8\n","for":"5m","severity":"warning","summary":"HAProxy session rate near limit - {{ $labels.instance }}","description":"HAProxy session rate at 80% of limit on {{ $labels.instance }}","runbook_url":"","metric_refs":["haproxy_process_current_session_rate","haproxy_process_limit_session_rate"],"line":170},{"kind":"alert","alert":"HAproxyFrontendDown","expr":"haproxy_frontend_status{state=\"DOWN\"} == 1","for":"1m","severity":"critical","summary":"HAProxy frontend down - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"Frontend {{ $labels.proxy }} @ {{ $labels.instance }} is DOWN and not accepting connections","runbook_url":"","metric_refs":["haproxy_frontend_status"],"line":184},{"kind":"alert","alert":"HAproxyFrontendSessionCritical","expr":"haproxy_frontend_current_sessions / haproxy_frontend_limit_sessions > 0.85","for":"2m","severity":"critical","summary":"HAProxy frontend session critical saturation - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"Frontend {{ $labels.proxy }} @ {{ $labels.instance }} session usage above 85% - connection refusal imminent","runbook_url":"","metric_refs":["haproxy_frontend_current_sessions","haproxy_frontend_limit_sessions"],"line":197},{"kind":"alert","alert":"HAproxyFrontendSessionSaturation","expr":"haproxy_frontend_current_sessions / haproxy_frontend_limit_sessions > 0.7","for":"5m","severity":"warning","summary":"HAProxy frontend session saturation - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"Frontend {{ $labels.proxy }} @ {{ $labels.instance }} session usage above 70%","runbook_url":"","metric_refs":["haproxy_frontend_current_sessions","haproxy_frontend_limit_sessions"],"line":206},{"kind":"alert","alert":"HAproxyFrontendRequestErrors","expr":"(\n  sum by (proxy, instance, job) (rate(haproxy_frontend_request_errors_total[5m]))\n  /\n  sum by (proxy, instance, job) (rate(haproxy_frontend_http_requests_total[5m]))\n) > 0.01\nand\nsum by (proxy, instance, job) (rate(haproxy_frontend_http_requests_total[5m])) > 10\n","for":"5m","severity":"warning","summary":"HAProxy frontend request errors - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"More than 1% invalid requests on frontend {{ $labels.proxy }} @ {{ $labels.instance }} ({{ $value | humanizePercentage }})","runbook_url":"","metric_refs":["haproxy_frontend_http_requests_total","haproxy_frontend_request_errors_total"],"line":215},{"kind":"alert","alert":"HAproxyFrontendSecurityBlockedRequests","expr":"(\n  sum by (proxy, instance, job) (rate(haproxy_frontend_denied_connections_total[2m]))\n  /\n  sum by (proxy, instance, job) (rate(haproxy_frontend_connections_total[2m]))\n) > 0.05\nand\nsum by (proxy, instance, job) (rate(haproxy_frontend_connections_total[2m])) > 10\n","for":"2m","severity":"warning","summary":"HAProxy frontend security blocked requests - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"More than 5% of connections blocked by ACL on frontend {{ $labels.proxy }} @ {{ $labels.instance }} ({{ $value | humanizePercentage }})","runbook_url":"","metric_refs":["haproxy_frontend_connections_total","haproxy_frontend_denied_connections_total"],"line":231},{"kind":"alert","alert":"HAproxyBackendDown","expr":"(sum by (namespace, proxy) (haproxy_backend_agg_server_check_status{state=\"UP\"}) == 0)","for":"5m","severity":"critical","summary":"All HAProxy backends down - {{ $labels.proxy }} @ {{ $labels.namespace }}","description":"All HAProxy backends are down for proxy {{ $labels.proxy }} in namespace {{ $labels.namespace }}","runbook_url":"","metric_refs":["haproxy_backend_agg_server_check_status"],"line":249},{"kind":"alert","alert":"HAproxyBackendStatusDown","expr":"haproxy_backend_status{state!=\"UP\"} == 1","for":"0m","severity":"critical","summary":"HAProxy backend status not UP - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"HAProxy backend {{ $labels.proxy }} @ {{ $labels.instance }} is not UP","runbook_url":"","metric_refs":["haproxy_backend_status"],"line":258},{"kind":"alert","alert":"HAproxyHAsNoAliveBackends","expr":"(haproxy_backend_active_servers + haproxy_backend_backup_servers) == 0","for":"5m","severity":"critical","summary":"HAProxy no alive backends - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"HAProxy has no alive active or backup backends for {{ $labels.proxy }} @ {{ $labels.instance }}","runbook_url":"","metric_refs":["haproxy_backend_active_servers","haproxy_backend_backup_servers"],"line":267},{"kind":"alert","alert":"HAproxyBackendConnectionErrors","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_backend_connection_errors_total[5m]))\n/ haproxy:backend_connection_attempts:rate5m\n> 0.05\nand\nhaproxy:backend_connection_attempts:rate5m > 10\n","for":"5m","severity":"critical","summary":"HAProxy backend connection errors - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"More than 5% of connection attempts failing on {{ $labels.proxy }} @ {{ $labels.instance }} ({{ $value | humanizePercentage }}) - check backend health or network connectivity","runbook_url":"","metric_refs":["haproxy:backend_connection_attempts:rate5m","haproxy_backend_connection_errors_total"],"line":276},{"kind":"alert","alert":"HAproxyHighHttp5xxErrorRateBackend","expr":"haproxy:backend_5xx_error_rate:ratio5m > 5\nand\nsum by (namespace, proxy, instance, job) (haproxy:backend_http_responses:rate5m) > 1\n","for":"5m","severity":"critical","summary":"HAProxy high HTTP 5xx error rate - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"Too many HTTP requests with status 5xx (> 5%) on {{ $labels.proxy }} @ {{ $labels.instance }}","runbook_url":"","metric_refs":["haproxy:backend_5xx_error_rate:ratio5m","haproxy:backend_http_responses:rate5m"],"line":290},{"kind":"alert","alert":"HAproxyHighHttp4xxErrorRateBackend","expr":"haproxy:backend_4xx_error_rate:ratio5m{proxy!~\".*default-local-service.*\"} > 50\nand\nsum by (namespace, proxy, instance, job) (haproxy:backend_http_responses:rate5m{proxy!~\".*default-local-service.*\"}) > 1\n","for":"5m","severity":"critical","summary":"HAProxy high HTTP 4xx error rate - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"Too many HTTP requests with status 4xx (> 50%) on {{ $labels.proxy }} @ {{ $labels.instance }}","runbook_url":"","metric_refs":["haproxy:backend_4xx_error_rate:ratio5m","haproxy:backend_http_responses:rate5m"],"line":304},{"kind":"alert","alert":"HAproxyHighHttp4xxErrorRateBackendWarning","expr":"haproxy:backend_4xx_error_rate:ratio5m{proxy!~\".*default-local-service.*\"} > 15\nand\nsum by (namespace, proxy, instance, job) (haproxy:backend_http_responses:rate5m{proxy!~\".*default-local-service.*\"}) > 1\n","for":"5m","severity":"warning","summary":"HAProxy high HTTP 4xx error rate (warning) - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"Too many HTTP requests with status 4xx (> 15%) on {{ $labels.proxy }} @ {{ $labels.instance }}","runbook_url":"","metric_refs":["haproxy:backend_4xx_error_rate:ratio5m","haproxy:backend_http_responses:rate5m"],"line":316},{"kind":"alert","alert":"HAproxyBackendConnectionReuseCollapse","expr":"haproxy:backend_connection_reuse:ratio5m < 20\nand\nhaproxy:backend_connection_attempts:rate5m > 10\n","for":"10m","severity":"critical","summary":"HAProxy backend connection reuse collapsed - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"Backend {{ $labels.proxy }} @ {{ $labels.instance }} connection reuse rate is {{ $value | printf \"%.0f\" }}% (< 20%) for the last 10 minutes. Almost no connections are being reused - check keep-alive settings, backend health, or HTTP version negotiation.","runbook_url":"","metric_refs":["haproxy:backend_connection_attempts:rate5m","haproxy:backend_connection_reuse:ratio5m"],"line":328},{"kind":"alert","alert":"HAproxyBackendConnectionReuseDrop","expr":"haproxy:backend_connection_reuse:ratio5m < 50\nand\nhaproxy:backend_connection_attempts:rate5m > 10\n","for":"10m","severity":"warning","summary":"HAProxy backend connection reuse dropped - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"Backend {{ $labels.proxy }} @ {{ $labels.instance }} connection reuse rate is {{ $value | printf \"%.0f\" }}% (< 50%) for the last 10 minutes. This may indicate misconfigured keep-alive, backend restarts, or HTTP/1.0 fallback.","runbook_url":"","metric_refs":["haproxy:backend_connection_attempts:rate5m","haproxy:backend_connection_reuse:ratio5m"],"line":340},{"kind":"alert","alert":"HAproxyBackendMaxActiveSessionHigh","expr":"(haproxy_server_current_sessions * 100) / (haproxy_server_limit_sessions > 0) > 80","for":"5m","severity":"warning","summary":"HAProxy backend max active session > 80% - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"Session limit from {{ $labels.proxy }}/{{ $labels.server }} @ {{ $labels.instance }} reached 80% of limit - {{ $value | printf \"%.2f\"}}%","runbook_url":"","metric_refs":["haproxy_server_current_sessions","haproxy_server_limit_sessions"],"line":352},{"kind":"alert","alert":"HAproxyBackendQueueTimeHigh","expr":"haproxy_backend_queue_time_average_seconds > 0.5","for":"5m","severity":"warning","summary":"HAProxy backend queue time high - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"Backend {{ $labels.proxy }} @ {{ $labels.instance }} queue wait time above 500ms ({{ $value | humanizeDuration }})","runbook_url":"","metric_refs":["haproxy_backend_queue_time_average_seconds"],"line":365},{"kind":"alert","alert":"HAproxyBackendResponseTimeHigh","expr":"haproxy_backend_response_time_average_seconds > 1","for":"5m","severity":"warning","summary":"HAProxy backend response time high - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"Backend {{ $labels.proxy }} @ {{ $labels.instance }} average response time above 1s ({{ $value | humanizeDuration }})","runbook_url":"","metric_refs":["haproxy_backend_response_time_average_seconds"],"line":374},{"kind":"alert","alert":"HAproxyBackendRedispatch","expr":"sum by (namespace, proxy, instance, job) (rate(haproxy_backend_redispatch_warnings_total[5m]))\n/ haproxy:backend_connection_attempts:rate5m\n> 0.05\nand\nhaproxy:backend_connection_attempts:rate5m > 10\n","for":"5m","severity":"warning","summary":"HAProxy backend redispatch - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"More than 5% of connections redispatched on {{ $labels.proxy }} @ {{ $labels.instance }} ({{ $value | humanizePercentage }}) - a server is likely flapping","runbook_url":"","metric_refs":["haproxy:backend_connection_attempts:rate5m","haproxy_backend_redispatch_warnings_total"],"line":383},{"kind":"alert","alert":"HAproxyPendingRequests","expr":"sum by (proxy,job,instance) (min_over_time(haproxy_backend_current_queue[5m])) > 0","for":"5m","severity":"warning","summary":"HAProxy pending requests - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"Some HAProxy requests are pending on {{ $labels.proxy }} @ {{ $labels.instance }} - {{ $value | printf \"%.2f\"}}","runbook_url":"","metric_refs":["haproxy_backend_current_queue"],"line":397},{"kind":"alert","alert":"HAproxyRetryHigh","expr":"haproxy:backend_retries:rate5m\n/ (haproxy:backend_connection_attempts:rate5m > 0)\n> 0.05\nand\nhaproxy:backend_connection_attempts:rate5m > 1\n","for":"5m","severity":"warning","summary":"HAProxy retry high - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"High retry rate on {{ $labels.proxy }} @ {{ $labels.instance }}: {{ $value | humanizePercentage }} of connection attempts","runbook_url":"","metric_refs":["haproxy:backend_connection_attempts:rate5m","haproxy:backend_retries:rate5m"],"line":410},{"kind":"alert","alert":"HAproxyBlackholedTrafficNamespaceWarning","expr":"(\n  sum by (namespace) (haproxy:backend_http_responses:rate5m{proxy=~\".*default-local-service.*\"})\n  /\n  sum by (namespace) (haproxy:backend_http_responses:rate5m)\n) * 100 > 0.5\nand\nsum by (namespace) (haproxy:backend_http_responses:rate5m) > 1\n","for":"5m","severity":"warning","summary":"HAProxy blackholed traffic in {{ $labels.namespace }} (> 0.5%)","description":"More than 0.5% of requests in {{ $labels.namespace }} are hitting the default backend - possible ingress misconfiguration or high 404 rate","runbook_url":"","metric_refs":["haproxy:backend_http_responses:rate5m"],"line":424},{"kind":"alert","alert":"HAproxyBlackholedTrafficNamespaceCritical","expr":"(\n  sum by (namespace) (haproxy:backend_http_responses:rate5m{proxy=~\".*default-local-service.*\"})\n  /\n  sum by (namespace) (haproxy:backend_http_responses:rate5m)\n) * 100 > 2\nand\nsum by (namespace) (haproxy:backend_http_responses:rate5m) > 1\n","for":"5m","severity":"critical","summary":"HAProxy blackholed traffic in {{ $labels.namespace }} (> 2%)","description":"More than 2% of requests in {{ $labels.namespace }} are hitting the default backend - possible ingress misconfiguration or high 404 rate","runbook_url":"","metric_refs":["haproxy:backend_http_responses:rate5m"],"line":440},{"kind":"alert","alert":"HAproxyServerDown","expr":"haproxy_server_status{state=\"DOWN\"} == 1","for":"2m","severity":"critical","summary":"HAProxy server down - {{ $labels.proxy }}/{{ $labels.server }} @ {{ $labels.instance }}","description":"Server {{ $labels.proxy }}/{{ $labels.server }} @ {{ $labels.instance }} is DOWN","runbook_url":"","metric_refs":["haproxy_server_status"],"line":458},{"kind":"alert","alert":"HAproxyServerDownOnSingleInstance","expr":"count by (job, instance, proxy) (\n  haproxy_server_status{state=\"DOWN\"} == 1\n  and on (job, proxy, server) haproxy_server_status{state=\"UP\"} == 1\n)\n","for":"1h","severity":"warning","summary":"HAProxy servers down on one instance only - {{ $labels.proxy }} @ {{ $labels.instance }}","description":"{{ $value }} servers of {{ $labels.proxy }} are DOWN on {{ $labels.instance }} while other instances of {{ $labels.job }} see them UP. Check the network path from {{ $labels.instance }} to those servers (shared rack, switch, VLAN).","runbook_url":"","metric_refs":["haproxy_server_status"],"line":471},{"kind":"alert","alert":"HAproxyServerConnectionErrors","expr":"(\n  sum by (proxy, job, instance, server) (rate(haproxy_server_connection_errors_total[5m]))\n  /\n  sum by (proxy, job, instance, server) (rate(haproxy_server_connection_attempts_total[5m]))\n) > 0.05\nand\nsum by (proxy, job, instance, server) (rate(haproxy_server_connection_attempts_total[5m])) > 10\n","for":"5m","severity":"critical","summary":"HAProxy server connection errors - {{ $labels.proxy }}/{{ $labels.server }} @ {{ $labels.instance }}","description":"More than 5% of connection attempts failing on {{ $labels.proxy }}/{{ $labels.server }} @ {{ $labels.instance }} ({{ $value | humanizePercentage }})","runbook_url":"","metric_refs":["haproxy_server_connection_attempts_total","haproxy_server_connection_errors_total"],"line":487},{"kind":"alert","alert":"HAproxyHighHttp5xxErrorRateServer","expr":"sum by (namespace, proxy, server, instance, job) (haproxy:server_http_responses:rate5m{code=\"5xx\"})\n/ clamp_min(\n    sum by (namespace, proxy, server, instance, job) (haproxy:server_http_responses:rate5m),\n    0.01\n  ) * 100 > 5\nand\nsum by (namespace, proxy, server, instance, job) (haproxy:server_http_responses:rate5m) > 1\n","for":"5m","severity":"critical","summary":"HAProxy high HTTP 5xx error rate - {{ $labels.proxy }}/{{ $labels.server }} @ {{ $labels.instance }}","description":"Too many HTTP requests with status 5xx (> 5%) on {{ $labels.proxy }}/{{ $labels.server }} @ {{ $labels.instance }}","runbook_url":"","metric_refs":["haproxy:server_http_responses:rate5m"],"line":503},{"kind":"alert","alert":"HAproxyHighHttp4xxErrorRateServer","expr":"sum by (namespace, proxy, server, instance, job) (haproxy:server_http_responses:rate5m{code=\"4xx\"})\n/ clamp_min(\n    sum by (namespace, proxy, server, instance, job) (haproxy:server_http_responses:rate5m),\n    0.01\n  ) * 100 > 50\nand\nsum by (namespace, proxy, server, instance, job) (haproxy:server_http_responses:rate5m) > 1\n","for":"5m","severity":"critical","summary":"HAProxy high HTTP 4xx error rate - {{ $labels.proxy }}/{{ $labels.server }} @ {{ $labels.instance }}","description":"Too many HTTP requests with status 4xx (> 50%) on {{ $labels.proxy }}/{{ $labels.server }} @ {{ $labels.instance }}","runbook_url":"","metric_refs":["haproxy:server_http_responses:rate5m"],"line":519},{"kind":"alert","alert":"HAproxyHighHttp4xxErrorRateServerWarning","expr":"sum by (namespace, proxy, server, instance, job) (haproxy:server_http_responses:rate5m{code=\"4xx\"})\n/ clamp_min(\n    sum by (namespace, proxy, server, instance, job) (haproxy:server_http_responses:rate5m),\n    0.01\n  ) * 100 > 15\nand\nsum by (namespace, proxy, server, instance, job) (haproxy:server_http_responses:rate5m) > 1\n","for":"5m","severity":"warning","summary":"HAProxy high HTTP 4xx error rate (warning) - {{ $labels.proxy }}/{{ $labels.server }} @ {{ $labels.instance }}","description":"Too many HTTP requests with status 4xx (> 15%) on {{ $labels.proxy }}/{{ $labels.server }} @ {{ $labels.instance }}","runbook_url":"","metric_refs":["haproxy:server_http_responses:rate5m"],"line":535},{"kind":"alert","alert":"HAproxyServerHealthcheckFailure","expr":"increase(haproxy_server_check_failures_total[5m]) > 3","for":"0m","severity":"warning","summary":"HAProxy server healthcheck failure - {{ $labels.server }} @ {{ $labels.instance }}","description":"{{ $value }} healthcheck failures in the last minute on {{ $labels.server }} @ {{ $labels.instance }}","runbook_url":"","metric_refs":["haproxy_server_check_failures_total"],"line":551},{"kind":"alert","alert":"HAproxyServerQueueFull","expr":"haproxy_server_current_queue > 0","for":"5m","severity":"warning","summary":"HAProxy server queue full - {{ $labels.proxy }}/{{ $labels.server }} @ {{ $labels.instance }}","description":"Server {{ $labels.proxy }}/{{ $labels.server }} @ {{ $labels.instance }} has pending queued connections","runbook_url":"","metric_refs":["haproxy_server_current_queue"],"line":561},{"kind":"alert","alert":"HAproxyServerDowntimeAccumulating","expr":"increase(haproxy_server_downtime_seconds_total[1h]) > 300","for":"0m","severity":"warning","summary":"HAProxy server downtime accumulating - {{ $labels.server }} @ {{ $labels.instance }}","description":"{{ $labels.server }} @ {{ $labels.instance }} accumulated more than 5 minutes of downtime in the last hour","runbook_url":"","metric_refs":["haproxy_server_downtime_seconds_total"],"line":570},{"kind":"alert","alert":"HAproxySticktableCritical","expr":"haproxy_sticktable_size > 0\n  and\nhaproxy_sticktable_used / haproxy_sticktable_size > 0.9\n","for":"5m","severity":"critical","summary":"HAProxy stick table critical - {{ $labels.name }} @ {{ $labels.instance }}","description":"HAProxy stick table {{ $labels.name }} @ {{ $labels.instance }} at 90% capacity - rate limiting and ACLs at risk of failure","runbook_url":"","metric_refs":["haproxy_sticktable_size","haproxy_sticktable_used"],"line":581},{"kind":"alert","alert":"HAproxySticktableSaturation","expr":"haproxy_sticktable_size > 0\n  and\nhaproxy_sticktable_used / haproxy_sticktable_size > 0.8\n","for":"5m","severity":"warning","summary":"HAProxy stick table near full - {{ $labels.name }} @ {{ $labels.instance }}","description":"HAProxy stick table {{ $labels.name }} @ {{ $labels.instance }} at 80% capacity - rate limiting and ACLs may malfunction","runbook_url":"","metric_refs":["haproxy_sticktable_size","haproxy_sticktable_used"],"line":593},{"kind":"alert","alert":"HAProxyBackendFlapping","expr":"sum by (proxy, instance) (increase(haproxy_backend_check_up_down_total[6h])) > 5","for":"15m","severity":"warning","summary":"HAProxy backend {{ $labels.proxy }} flapping @ {{ $labels.instance }}","description":"Backend {{ $labels.proxy }} on {{ $labels.instance }} recorded {{ $value | printf \"%.0f\" }} UP\u2192DOWN check transitions in 6h. Each one drops in-flight connections. The backend looks healthy between transitions, so the down alerts stay silent.","runbook_url":"","metric_refs":["haproxy_backend_check_up_down_total"],"line":605},{"kind":"alert","alert":"HAProxyBackendInternalErrors","expr":"sum by (proxy, instance) (increase(haproxy_backend_internal_errors_total[1h])) > 0","for":"5m","severity":"warning","summary":"HAProxy internal errors on {{ $labels.proxy }} @ {{ $labels.instance }}","description":"{{ $value | printf \"%.0f\" }} internal errors on backend {{ $labels.proxy }} @ {{ $labels.instance }} in the last hour. These are HAProxy's own failures (allocation, buffers, internal state), not backend problems, and they do not show up in the health or saturation alerts.","runbook_url":"","metric_refs":["haproxy_backend_internal_errors_total"],"line":623}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/haproxy.rules.yml"},{"file":"ipsec.rules.yml","name":"ipsec.rules","category":"Network, DNS & TLS","rules":[{"kind":"alert","alert":"ExporterIPSECDown","expr":"# Two distinct failure modes: a failing scrape (up == 0, exporter crashed\n# or host unreachable) and the target vanishing from service discovery,\n# where no up series exists to be 0. Both branches are grouped\n# identically so the alert carries the same labels either way.\ngroup by (job, instance) (up{job=\"vpn-ipsec\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=\"vpn-ipsec\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=\"vpn-ipsec\"}[5m]))\n)\n","for":"1m","severity":"critical","summary":"IPSEC Exporter seems down for {{ $labels.instance }}","description":"The IPsec exporter on {{ $labels.instance }} is failing its scrape or has vanished from service discovery. Tunnel alerts for this host are blind until it is back.","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"IPSECDown","expr":"ipsec_status != 0","for":"1m","severity":"critical","summary":"Tunnel {{ $labels.tunnel }} @ {{ $labels.instance }} down","description":"Tunnel {{ $labels.tunnel }} @ {{ $labels.instance }} down, check for systemd status and syslog","runbook_url":"","metric_refs":["ipsec_status"],"line":22}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/ipsec.rules.yml"},{"file":"jmx.rules.yml","name":"jmx.rules","category":"Messaging & streaming","rules":[{"kind":"alert","alert":"JvmMemoryFillingUp","expr":"(sum by (instance)(jvm_memory_bytes_used{area=\"heap\"}) / sum by (instance)(jvm_memory_bytes_max{area=\"heap\"})) * 100 > 80 and sum by (instance)(jvm_memory_bytes_max{area=\"heap\"}) > 0","for":"2m","severity":"warning","summary":"JVM heap memory filling up (instance {{ $labels.instance }})","description":"JVM heap memory is filling up (> 80%) on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["jvm_memory_bytes_max","jvm_memory_bytes_used"],"line":3},{"kind":"alert","alert":"JvmHeapMemoryCritical","expr":"(sum by (instance)(jvm_memory_bytes_used{area=\"heap\"}) / sum by (instance)(jvm_memory_bytes_max{area=\"heap\"})) * 100 > 90 and sum by (instance)(jvm_memory_bytes_max{area=\"heap\"}) > 0","for":"5m","severity":"critical","summary":"JVM heap memory critical (instance {{ $labels.instance }})","description":"JVM heap memory above 90% on {{ $labels.instance }} - raise -Xmx or investigate a leak\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["jvm_memory_bytes_max","jvm_memory_bytes_used"],"line":12},{"kind":"alert","alert":"JvmGcTimeTooHigh","expr":"sum by (instance)(rate(jvm_gc_collection_seconds_sum[5m])) > 0.05","for":"5m","severity":"warning","summary":"JVM GC time too high (instance {{ $labels.instance }})","description":"JVM is spending more than 5% of wall clock time in GC on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["jvm_gc_collection_seconds_sum"],"line":21},{"kind":"alert","alert":"JvmOldGenGcFrequency","expr":"rate(jvm_gc_collection_seconds_count{gc=~\"(?i).*old.*|.*major.*\"}[5m]) > 0.3","for":"5m","severity":"warning","summary":"JVM old gen GC frequency high (instance {{ $labels.instance }})","description":"Frequent old/major GC cycles indicate memory pressure on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["jvm_gc_collection_seconds_count"],"line":30},{"kind":"alert","alert":"JvmThreadsDeadlocked","expr":"jvm_threads_deadlocked > 0","for":"1m","severity":"critical","summary":"JVM threads deadlocked (instance {{ $labels.instance }})","description":"JVM has {{ $value }} deadlocked threads on {{ $labels.instance }}","runbook_url":"","metric_refs":["jvm_threads_deadlocked"],"line":41},{"kind":"alert","alert":"JvmThreadCountHigh","expr":"jvm_threads_current > 300","for":"5m","severity":"warning","summary":"JVM thread count high (instance {{ $labels.instance }})","description":"JVM thread count is {{ $value }} (> 300) on {{ $labels.instance }} - potential thread leak","runbook_url":"","metric_refs":["jvm_threads_current"],"line":50},{"kind":"alert","alert":"JvmThreadsBlocked","expr":"(\n  jvm_threads_state{state=\"BLOCKED\"}\n  /\n  ignoring(state) jvm_threads_current\n) > 0.10\nand\njvm_threads_current > 10\n","for":"5m","severity":"warning","summary":"JVM threads BLOCKED (instance {{ $labels.instance }})","description":"More than 10% of JVM threads are BLOCKED on {{ $labels.instance }} ({{ $value | humanizePercentage }}) - lock contention suspected","runbook_url":"","metric_refs":["jvm_threads_current","jvm_threads_state"],"line":59},{"kind":"alert","alert":"JvmDirectBufferPoolFillingUp","expr":"(jvm_buffer_pool_used_bytes / jvm_buffer_pool_capacity_bytes) * 100 > 90 and jvm_buffer_pool_capacity_bytes > 0","for":"5m","severity":"warning","summary":"JVM direct buffer pool filling up (instance {{ $labels.instance }})","description":"JVM direct buffer pool > 90% on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["jvm_buffer_pool_capacity_bytes","jvm_buffer_pool_used_bytes"],"line":75},{"kind":"alert","alert":"JvmFileDescriptorsExhaustion","expr":"(process_open_fds / process_max_fds) * 100 > 90 and process_max_fds > 0","for":"5m","severity":"warning","summary":"JVM file descriptors near exhaustion (instance {{ $labels.instance }})","description":"JVM process is using > 90% of available file descriptors on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["process_max_fds","process_open_fds"],"line":84}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/jmx.rules.yml"},{"file":"k8s.records.yml","name":"k8s.records","category":"Kubernetes","rules":[{"kind":"record","record":"kube_container_cpu_usage_5m","expr":"sum by (dc, k8s_cluster_name, namespace, pod) (rate(container_cpu_usage_seconds_total{container!=\"\"}[5m]))","metric_refs":["container_cpu_usage_seconds_total","k8s_cluster_name"],"line":17},{"kind":"record","record":"kube_container_memory_usage","expr":"sum(container_memory_working_set_bytes{image!=\"\",container!=\"\"}) by (dc, k8s_cluster_name, namespace, pod)","metric_refs":["container_memory_working_set_bytes","k8s_cluster_name"],"line":20},{"kind":"record","record":"kube_namespace_cpu_limit","expr":"sum by(dc, k8s_cluster_name, namespace, pod) (kube_pod_container_resource_limits{resource=\"cpu\"} * on(namespace,pod,cluster_id,container,instance) group_right() (kube_pod_container_status_running == 1))","metric_refs":["cluster_id","k8s_cluster_name","kube_pod_container_resource_limits","kube_pod_container_status_running"],"line":25},{"kind":"record","record":"kube_namespace_cpu_request","expr":"sum by(dc, k8s_cluster_name, namespace, pod) (kube_pod_container_resource_requests{resource=\"cpu\"} * on(namespace,pod,cluster_id,container,instance) group_right() (kube_pod_container_status_running == 1))","metric_refs":["cluster_id","k8s_cluster_name","kube_pod_container_resource_requests","kube_pod_container_status_running"],"line":28},{"kind":"record","record":"kube_pod_cpu5m_request_ratio","expr":"kube_container_cpu_usage_5m/kube_namespace_cpu_request","metric_refs":["kube_container_cpu_usage_5m","kube_namespace_cpu_request"],"line":31},{"kind":"record","record":"kube_pod_cpu5m_limit_ratio","expr":"kube_container_cpu_usage_5m/kube_namespace_cpu_limit","metric_refs":["kube_container_cpu_usage_5m","kube_namespace_cpu_limit"],"line":34},{"kind":"record","record":"kube_namespace_cpu5m_limit_ratio","expr":"avg(kube_pod_cpu5m_limit_ratio) without(pod)","metric_refs":["kube_pod_cpu5m_limit_ratio"],"line":37},{"kind":"record","record":"kube_namespace_cpu5m_request_ratio","expr":"avg(kube_pod_cpu5m_request_ratio) without(pod)","metric_refs":["kube_pod_cpu5m_request_ratio"],"line":40},{"kind":"record","record":"kube_namespace_memory_limit","expr":"sum by(dc, k8s_cluster_name, namespace, pod) (kube_pod_container_resource_limits{resource=\"memory\"} * on(namespace,pod,cluster_id,container,instance) group_right() (kube_pod_container_status_running == 1))","metric_refs":["cluster_id","k8s_cluster_name","kube_pod_container_resource_limits","kube_pod_container_status_running"],"line":43},{"kind":"record","record":"kube_namespace_memory_request","expr":"sum by(dc, k8s_cluster_name, namespace, pod) (kube_pod_container_resource_requests{resource=\"memory\"} * on(namespace,pod,cluster_id,container,instance) group_right() (kube_pod_container_status_running == 1))","metric_refs":["cluster_id","k8s_cluster_name","kube_pod_container_resource_requests","kube_pod_container_status_running"],"line":46},{"kind":"record","record":"kube_pod_memory_request_ratio","expr":"kube_container_memory_usage/kube_namespace_memory_request","metric_refs":["kube_container_memory_usage","kube_namespace_memory_request"],"line":49},{"kind":"record","record":"kube_pod_memory_limit_ratio","expr":"kube_container_memory_usage/kube_namespace_memory_limit","metric_refs":["kube_container_memory_usage","kube_namespace_memory_limit"],"line":52},{"kind":"record","record":"kube_namespace_memory_request_ratio","expr":"avg(kube_pod_memory_request_ratio) without(pod)","metric_refs":["kube_pod_memory_request_ratio"],"line":55},{"kind":"record","record":"kube_namespace_memory_limit_ratio","expr":"avg(kube_pod_memory_limit_ratio) without(pod)","metric_refs":["kube_pod_memory_limit_ratio"],"line":58}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/k8s.records.yml"},{"file":"kafka-consumer.rules.yml","name":"kafka-consumer.rules","category":"Other","rules":[{"kind":"alert","alert":"KafkaConsumerGroupLagHigh","expr":"max by (consumergroup, topic) (kafka_consumergroup_lag_sum) > 100000","for":"15m","severity":"warning","summary":"Consumer group {{ $labels.consumergroup }} is {{ $value | humanize }} messages behind on {{ $labels.topic }}","description":"Total lag above 100000 messages for 15 minutes. Tune the threshold to the topic throughput.","runbook_url":"","metric_refs":["kafka_consumergroup_lag_sum"],"line":6},{"kind":"alert","alert":"KafkaConsumerGroupPartitionStalled","expr":"max by (consumergroup, topic) (\n  (max by (consumergroup, topic, partition) (kafka_consumergroup_lag) > 1000)\n  and on (consumergroup, topic, partition)\n  (sum by (consumergroup, topic, partition) (rate(kafka_consumergroup_current_offset[15m])) == 0)\n)\n","for":"15m","severity":"critical","summary":"Consumer group {{ $labels.consumergroup }} is stuck on {{ $labels.topic }}","description":"At least one partition committed no offset for 15m, worst backlog {{ $value }} messages.","runbook_url":"","metric_refs":["kafka_consumergroup_current_offset","kafka_consumergroup_lag"],"line":16},{"kind":"alert","alert":"KafkaConsumerGroupPartitionSkew","expr":"(\n  max by (consumergroup, topic) (kafka_consumergroup_lag)\n  / on (consumergroup, topic)\n  (quantile by (consumergroup, topic) (0.5, max by (consumergroup, topic, partition) (kafka_consumergroup_lag)) > 0)\n) > 10\nand on (consumergroup, topic)\n(max by (consumergroup, topic) (kafka_consumergroup_lag) > 1000)\n","for":"15m","severity":"warning","summary":"Consumer group {{ $labels.consumergroup }} is uneven on {{ $labels.topic }}","description":"The worst partition is {{ $value | humanize }}x further behind than the median partition.","runbook_url":"","metric_refs":["kafka_consumergroup_lag"],"line":31}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/kafka-consumer.rules.yml"},{"file":"kafka.rules.yml","name":"kafka.rules","category":"Messaging & streaming","rules":[{"kind":"alert","alert":"KafkaExporter","expr":"# Two distinct failure modes: a failing scrape (up == 0, exporter crashed\n# or host unreachable) and the target vanishing from service discovery,\n# where no up series exists to be 0. Both branches are grouped\n# identically so the alert carries the same labels either way.\ngroup by (job, instance) (up{job=\"kafka\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=\"kafka\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=\"kafka\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"Job {{ $labels.job }} having an issue on {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"KafkaUnderReplicatedPartitions","expr":"kafka_server_replicamanager_underreplicatedpartitions > 0","for":"5m","severity":"critical","summary":"{{ $labels.instance }} of job {{ $labels.job }} has some underreplicated partitions","description":"{{ $value }} partitions have fewer in-sync replicas than their replication factor. A broker is down, slow or partitioned: check broker health and ISR shrinks.","runbook_url":"","metric_refs":["kafka_server_replicamanager_underreplicatedpartitions"],"line":22},{"kind":"alert","alert":"KafkaOfflinePartitions","expr":"kafka_controller_kafkacontroller_offlinepartitionscount > 0","for":"1m","severity":"critical","summary":"{{ $labels.instance }} of job {{ $labels.job }} has offline partitions, some services may be unavailable","description":"{{ $labels.instance }} of job {{ $labels.job }} has offline partitions","runbook_url":"","metric_refs":["kafka_controller_kafkacontroller_offlinepartitionscount"],"line":32},{"kind":"alert","alert":"KafkaSlowReassigningPartition","expr":"kafka_server_replicamanager_reassigningpartitions > 0","for":"5m","severity":"warning","summary":"{{ $labels.instance }} of job {{ $labels.job }} has some reassigning partitions","description":"{{ $value }} partitions have been reassigning for over 5 minutes. A large reassignment or a replication throttle set too low: check the reassignment progress.","runbook_url":"","metric_refs":["kafka_server_replicamanager_reassigningpartitions"],"line":40},{"kind":"alert","alert":"KafkaLagExporter","expr":"# Two distinct failure modes: a failing scrape (up == 0, exporter crashed\n# or host unreachable) and the target vanishing from service discovery,\n# where no up series exists to be 0. Both branches are grouped\n# identically so the alert carries the same labels either way.\ngroup by (job, instance) (up{job=\"consumer\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=\"consumer\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=\"consumer\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"Job {{ $labels.job }} having an issue on {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":48},{"kind":"alert","alert":"KafkaNoActiveController","expr":"sum by (job) (kafka_controller_kafkacontroller_activecontrollercount) != 1","for":"1m","severity":"critical","summary":"Wrong active Kafka controller count on {{ $labels.job }}","description":"Kafka cluster {{ $labels.job }} has {{ $value }} active controllers instead of exactly 1. Cluster operations are impaired.","runbook_url":"","metric_refs":["kafka_controller_kafkacontroller_activecontrollercount"],"line":67},{"kind":"alert","alert":"KafkaBrokerFenced","expr":"kafka_controller_kafkacontroller_fencedbrokercount > 0","for":"2m","severity":"critical","summary":"{{ $value }} broker(s) fenced on {{ $labels.instance }}","description":"{{ $value }} Kafka broker(s) are fenced (unresponsive to controller heartbeat) on {{ $labels.instance }}.","runbook_url":"","metric_refs":["kafka_controller_kafkacontroller_fencedbrokercount"],"line":78},{"kind":"alert","alert":"KafkaUncleanLeaderElection","expr":"increase(kafka_controller_controllerstats_uncleanleaderelections_total[5m]) > 0","for":"0m","severity":"critical","summary":"Unclean leader election on {{ $labels.instance }}","description":"Unclean leader election detected on {{ $labels.instance }} - potential data loss. Inspect consumer lag immediately.","runbook_url":"","metric_refs":["kafka_controller_controllerstats_uncleanleaderelections_total"],"line":86},{"kind":"alert","alert":"KafkaOfflineReplicas","expr":"kafka_server_replicamanager_offlinereplicacount > 0","for":"2m","severity":"critical","summary":"{{ $value }} replica(s) offline on {{ $labels.instance }}","description":"{{ $value }} Kafka replica(s) are offline on {{ $labels.instance }}.","runbook_url":"","metric_refs":["kafka_server_replicamanager_offlinereplicacount"],"line":94},{"kind":"alert","alert":"KafkaPartitionsUnderMinISR","expr":"kafka_server_replicamanager_underminisrpartitioncount > 0","for":"2m","severity":"critical","summary":"{{ $value }} partition(s) under min ISR on {{ $labels.instance }}","description":"{{ $value }} partition(s) are below the minimum ISR on {{ $labels.instance }}. Producer acks=all will fail.","runbook_url":"","metric_refs":["kafka_server_replicamanager_underminisrpartitioncount"],"line":102},{"kind":"alert","alert":"KafkaPartitionsAtMinISR","expr":"kafka_server_replicamanager_atminisrpartitioncount > 0","for":"5m","severity":"warning","summary":"{{ $value }} partition(s) at min ISR on {{ $labels.instance }}","description":"{{ $value }} partition(s) are exactly at min ISR on {{ $labels.instance }} - one more failure risks under-min-ISR.","runbook_url":"","metric_refs":["kafka_server_replicamanager_atminisrpartitioncount"],"line":110},{"kind":"alert","alert":"KafkaOfflineLogDirectory","expr":"kafka_log_logmanager_offlinelogdirectorycount > 0","for":"0m","severity":"critical","summary":"{{ $value }} log directory(ies) offline on {{ $labels.instance }}","description":"{{ $value }} log directory(ies) are offline on {{ $labels.instance }}. Partitions on those dirs are unavailable.","runbook_url":"","metric_refs":["kafka_log_logmanager_offlinelogdirectorycount"],"line":118},{"kind":"alert","alert":"KafkaRequestHandlerIdleLow","expr":"kafka_server_kafkarequesthandlerpool_requesthandleravgidle_percent < 0.30","for":"5m","severity":"warning","summary":"Request handler idle low on {{ $labels.instance }}","description":"Kafka request handler thread pool idle below 30% on {{ $labels.instance }}. Broker may be overloaded.","runbook_url":"","metric_refs":["kafka_server_kafkarequesthandlerpool_requesthandleravgidle_percent"],"line":126},{"kind":"alert","alert":"KafkaNetworkProcessorIdleLow","expr":"kafka_network_socketserver_networkprocessoravgidlepercent < 0.30","for":"5m","severity":"warning","summary":"Network processor idle low on {{ $labels.instance }}","description":"Kafka network processor idle below 30% on {{ $labels.instance }}. Consider increasing num.network.threads.","runbook_url":"","metric_refs":["kafka_network_socketserver_networkprocessoravgidlepercent"],"line":134},{"kind":"alert","alert":"KafkaDeadLogCleanerThread","expr":"kafka_log_logcleaner_deadthreadcount > 0","for":"0m","severity":"critical","summary":"Dead log cleaner thread on {{ $labels.instance }}","description":"Dead log cleaner thread on {{ $labels.instance }}. Compacted topics will stop being cleaned.","runbook_url":"","metric_refs":["kafka_log_logcleaner_deadthreadcount"],"line":142},{"kind":"alert","alert":"KafkaJVMDeadlocked","expr":"jvm_threads_deadlocked_monitor > 0","for":"0m","severity":"critical","summary":"JVM deadlock on {{ $labels.instance }}","description":"JVM thread deadlock detected on {{ $labels.instance }}. Restart required.","runbook_url":"","metric_refs":["jvm_threads_deadlocked_monitor"],"line":150},{"kind":"alert","alert":"KafkaJMXScrapeFailed","expr":"jmx_scrape_error != 0","for":"2m","severity":"critical","summary":"JMX scrape failed on {{ $labels.instance }}","description":"JMX scrape failed on {{ $labels.instance }}. Kafka metrics are incomplete.","runbook_url":"","metric_refs":["jmx_scrape_error"],"line":158},{"kind":"alert","alert":"KafkaProduceErrorRate","expr":"sum by (job) (increase(kafka_server_brokertopicmetrics_failedproducerequests_total{topic=\"\"}[5m]))\n/\n(sum by (job) (increase(kafka_server_brokertopicmetrics_totalproducerequests_total{topic=\"\"}[5m])) > 0)\n> 0.01\n","for":"10m","severity":"critical","summary":"Kafka produce error rate at {{ $value | humanizePercentage }} on job {{ $labels.job }}","description":"Over 1% of produce requests failed on job {{ $labels.job }} for 10 minutes ({{ $value | humanizePercentage }}).","runbook_url":"","metric_refs":["kafka_server_brokertopicmetrics_failedproducerequests_total","kafka_server_brokertopicmetrics_totalproducerequests_total"],"line":166},{"kind":"alert","alert":"KafkaProduceLatencyHigh","expr":"max by (job) (kafka_network_requestmetrics_totaltimems{request=\"Produce\", quantile=\"0.99\"}) > 500","for":"15m","severity":"warning","summary":"Kafka produce p99 latency at {{ $value | humanize }} ms on job {{ $labels.job }}","description":"The worst broker p99 of Produce total time on job {{ $labels.job }} stayed above 500 ms for 15 minutes.","runbook_url":"","metric_refs":["kafka_network_requestmetrics_totaltimems"],"line":179}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/kafka.rules.yml"},{"file":"keepalived.rules.yml","name":"keepalived.rules","category":"Web & load balancing","rules":[{"kind":"alert","alert":"KeepalivedDown","expr":"keepalived_up == 0","for":"2m","severity":"critical","summary":"Keepalived down on {{ $labels.instance }}","description":"Keepalived is unreachable on {{ $labels.instance }}. VIP failover may be broken.","runbook_url":"","metric_refs":["keepalived_up"],"line":3},{"kind":"alert","alert":"KeepalivedSplitBrain","expr":"count by (vrid) (keepalived_vrrp_state == 2) > 1","for":"1m","severity":"critical","summary":"Keepalived split-brain detected on VRID {{ $labels.vrid }}","description":"Multiple instances are in MASTER state for VRID {{ $labels.vrid }}. This causes IP conflicts on the network.","runbook_url":"","metric_refs":["keepalived_vrrp_state"],"line":12},{"kind":"alert","alert":"KeepalivedNoMaster","expr":"count by (vrid) (keepalived_vrrp_state)\nunless\ncount by (vrid) (keepalived_vrrp_state == 2)\n","for":"2m","severity":"critical","summary":"No Keepalived master for VRID {{ $labels.vrid }}","description":"No instance is in MASTER state for VRID {{ $labels.vrid }}. The VIP is unreachable.","runbook_url":"","metric_refs":["keepalived_vrrp_state"],"line":25},{"kind":"alert","alert":"KeepalivedFlapping","expr":"increase(keepalived_become_master_total[5m]) > 3","for":"0m","severity":"warning","summary":"Keepalived VRRP flapping on {{ $labels.instance }}","description":"{{ $labels.instance }} has transitioned to MASTER {{ $value }} times in 5 minutes for VRID {{ $labels.vrid }} - network or configuration instability.","runbook_url":"","metric_refs":["keepalived_become_master_total"],"line":42},{"kind":"alert","alert":"KeepalivedAuthFailure","expr":"increase(keepalived_authentication_failure_total[5m]) > 0","for":"0m","severity":"warning","summary":"Keepalived authentication failures on {{ $labels.instance }}","description":"{{ $labels.instance }} is receiving VRRP packets with authentication failures - possible misconfiguration or unexpected peer on the network.","runbook_url":"","metric_refs":["keepalived_authentication_failure_total"],"line":51},{"kind":"alert","alert":"KeepalivedAdIntervalErrors","expr":"(\n  increase(keepalived_advertisements_interval_errors_total[5m])\n  /\n  increase(keepalived_advertisements_received_total[5m])\n) > 0.05\nand\nincrease(keepalived_advertisements_received_total[5m]) > 5\n","for":"5m","severity":"warning","summary":"Keepalived advertisement interval errors on {{ $labels.instance }}","description":"More than 5% of received VRRP advertisements have interval errors on {{ $labels.instance }} vrid={{ $labels.vrid }} ({{ $value | humanizePercentage }}) - risk of spurious master election.","runbook_url":"","metric_refs":["keepalived_advertisements_interval_errors_total","keepalived_advertisements_received_total"],"line":60},{"kind":"alert","alert":"KeepalivedAuthenticationInvalid","expr":"increase(keepalived_authentication_invalid_total[5m]) > 0","for":"0m","severity":"warning","summary":"Keepalived authentication invalid on {{ $labels.instance }}","description":"VRRP authentication failures on {{ $labels.instance }} vrid={{ $labels.vrid }} - likely misconfiguration or rogue peer.","runbook_url":"","metric_refs":["keepalived_authentication_invalid_total"],"line":76},{"kind":"alert","alert":"KeepalivedTTLErrors","expr":"(\n  increase(keepalived_ip_ttl_errors_total[5m])\n  /\n  increase(keepalived_advertisements_received_total[5m])\n) > 0.02\nand\nincrease(keepalived_advertisements_received_total[5m]) > 5\n","for":"0m","severity":"warning","summary":"Keepalived TTL errors on {{ $labels.instance }}","description":"More than 2% of received VRRP advertisements have TTL errors on {{ $labels.instance }} vrid={{ $labels.vrid }} ({{ $value | humanizePercentage }}) - possible routing misconfiguration or rogue peer.","runbook_url":"","metric_refs":["keepalived_advertisements_received_total","keepalived_ip_ttl_errors_total"],"line":85},{"kind":"alert","alert":"KeepalivedPacketErrors","expr":"increase(keepalived_packet_length_errors_total[5m]) > 0","for":"0m","severity":"warning","summary":"Keepalived packet length errors on {{ $labels.instance }}","description":"VRRP packet length errors on {{ $labels.instance }} - possible network corruption or incompatible version.","runbook_url":"","metric_refs":["keepalived_packet_length_errors_total"],"line":101}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/keepalived.rules.yml"},{"file":"keydb.rules.yml","name":"redis.rules","category":"Databases","rules":[{"kind":"alert","alert":"RedisDown","expr":"sum(redis_up) by (instance,job) != 1","for":"5m","severity":"critical","summary":"Redis is down for {{ $labels.instance }}","description":"Redis is down for {{ $labels.instance }} for more than 5 minutes.","runbook_url":"","metric_refs":["redis_up"],"line":3},{"kind":"alert","alert":"RedisClusterSlotFail","expr":"redis_cluster_slots_fail > 0","for":"2m","severity":"critical","summary":"Number of hash slots mapping to a node in FAIL state for instance {{ $labels.instance }}","description":"Redis cluster has slots fail\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["redis_cluster_slots_fail"],"line":11},{"kind":"alert","alert":"RedisClusterStateNotOk","expr":"redis_cluster_state == 0","for":"5m","severity":"critical","summary":"Redis cluster state is not ok for instance {{ $labels.instance }}","description":"Redis cluster is not ok","runbook_url":"","metric_refs":["redis_cluster_state"],"line":19},{"kind":"alert","alert":"RedisMissingMaster","expr":"(count(redis_instance_info{role=\"master\"}) or vector(0)) < 1","for":"5m","severity":"critical","summary":"Redis missing master (instance {{ $labels.instance }})","description":"Redis cluster has no node marked as master","runbook_url":"","metric_refs":["redis_instance_info"],"line":28},{"kind":"alert","alert":"RedisTooManyMasters","expr":"count(redis_instance_info{role=\"master\"}) > 1","for":"5m","severity":"critical","summary":"Redis too many masters (instance {{ $labels.instance }})","description":"Redis cluster has more than one master node","runbook_url":"","metric_refs":["redis_instance_info"],"line":39},{"kind":"alert","alert":"RedisDisconnectedSlaves","expr":"count without (instance, job) (redis_connected_slaves) - sum without (instance, job) (redis_connected_slaves) - 1 > 0","for":"0m","severity":"critical","summary":"Redis disconnected slaves (instance {{ $labels.instance }})","description":"Redis not replicating for all slaves on {{ $labels.instance }}","runbook_url":"","metric_refs":["redis_connected_slaves"],"line":49},{"kind":"alert","alert":"RedisReplicationBroken","expr":"delta(redis_connected_slaves[1m]) < 0","for":"0m","severity":"critical","summary":"Redis replication broken (instance {{ $labels.instance }})","description":"Redis instance lost a slave on {{ $labels.instance }}","runbook_url":"","metric_refs":["redis_connected_slaves"],"line":58},{"kind":"alert","alert":"RedisClusterFlapping","expr":"changes(redis_connected_slaves[1m]) > 1","for":"2m","severity":"critical","summary":"Redis cluster flapping (instance {{ $labels.instance }})","description":"Redis replica connections are flapping on {{ $labels.instance }}","runbook_url":"","metric_refs":["redis_connected_slaves"],"line":67},{"kind":"alert","alert":"RedisMissingBackup","expr":"time() - redis_rdb_last_save_timestamp_seconds > 60 * 60 * 48","for":"0m","severity":"critical","summary":"Redis missing backup (instance {{ $labels.instance }})","description":"Redis has not been backed up for 48 hours on {{ $labels.instance }}","runbook_url":"","metric_refs":["redis_rdb_last_save_timestamp_seconds"],"line":76},{"kind":"alert","alert":"RedisOutOfConfiguredMaxmemory","expr":"redis_memory_used_bytes / redis_memory_max_bytes * 100 > 90 and on(instance) redis_memory_max_bytes > 0","for":"2m","severity":"critical","summary":"Redis out of configured maxmemory (instance {{ $labels.instance }})","description":"Redis is running out of configured maxmemory (> 90%) on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["redis_memory_max_bytes","redis_memory_used_bytes"],"line":85},{"kind":"alert","alert":"RedisTooManyConnections","expr":"redis_connected_clients / redis_config_maxclients * 100 > 90 and redis_config_maxclients > 0","for":"2m","severity":"warning","summary":"Redis too many connections (instance {{ $labels.instance }})","description":"Redis connections > 90% of maxclients on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["redis_config_maxclients","redis_connected_clients"],"line":94},{"kind":"alert","alert":"RedisRejectedConnections","expr":"increase(redis_rejected_connections_total[1m]) > 5","for":"0m","severity":"warning","summary":"Redis rejected connections (instance {{ $labels.instance }})","description":"Some connections to Redis were rejected on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["redis_rejected_connections_total"],"line":103},{"kind":"alert","alert":"RedisReplicaLagHigh","expr":"redis_connected_slave_lag_seconds > 30","for":"5m","severity":"warning","summary":"Redis replica {{ $labels.slave_ip }} lagging on {{ $labels.instance }}","description":"Replica {{ $labels.slave_ip }}:{{ $labels.slave_port }} of {{ $labels.instance }} is {{ $value | humanizeDuration }} behind. It is connected, so the disconnection alerts stay quiet, but it serves stale reads and would lose that much data if promoted.","runbook_url":"","metric_refs":["redis_connected_slave_lag_seconds"],"line":112},{"kind":"alert","alert":"RedisEvictingKeys","expr":"increase(redis_evicted_keys_total[5m]) > 0","for":"5m","severity":"warning","summary":"Redis is evicting keys on {{ $labels.instance }}","description":"{{ $value | printf \"%.0f\" }} keys evicted in 5m on {{ $labels.instance }}. maxmemory is being enforced and data is being discarded - not a cache-by-design fleet, so this is loss rather than housekeeping.","runbook_url":"","metric_refs":["redis_evicted_keys_total"],"line":130}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/keydb.rules.yml"},{"file":"ksql.rules.yml","name":"ksql.rules","category":"Messaging & streaming","rules":[{"kind":"alert","alert":"KsqldbExporter","expr":"# Two distinct failure modes: a failing scrape (up == 0, exporter crashed\n# or host unreachable) and the target vanishing from service discovery,\n# where no up series exists to be 0. Both branches are grouped\n# identically so the alert carries the same labels either way.\ngroup by (job, instance) (up{job=\"ksqldb\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=\"ksqldb\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=\"ksqldb\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"Job {{ $labels.job }} having an issue on {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"KsqlServerDown","expr":"sum(ksql_ksql_engine_query_stats_liveness_indicator < 1) by (instance,job)","for":"2m","severity":"critical","summary":"KSQL {{ $labels.instance }} having an issue","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":["ksql_ksql_engine_query_stats_liveness_indicator"],"line":27},{"kind":"alert","alert":"KsqlQueriesNotRunning","expr":"sum(ksql_ksql_engine_query_stats_not_running_queries > 0) by (instance,job)","for":"2m","severity":"critical","summary":"KSQL queries down on {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has some down queries, please check why","runbook_url":"","metric_refs":["ksql_ksql_engine_query_stats_not_running_queries"],"line":35},{"kind":"alert","alert":"KsqlQueriesAreRebalancing","expr":"sum(ksql_ksql_engine_query_stats_rebalancing_queries > 0) by (instance,job)","for":"5m","severity":"warning","summary":"KSQL queries are rebalancing on {{ $labels.instance }} for 5 minutes","description":"{{ $labels.instance }} has rebalancing queries for 5 minutes, it could be normal","runbook_url":"","metric_refs":["ksql_ksql_engine_query_stats_rebalancing_queries"],"line":43},{"kind":"alert","alert":"KSQLErrorQueries","expr":"ksql_ksql_engine_query_stats_error_queries > 0","for":"1m","severity":"critical","summary":"KSQL queries in ERROR state on {{ $labels.ksql_cluster }}","description":"{{ $value }} KSQL query(ies) are in ERROR state on cluster {{ $labels.ksql_cluster }}. Data processing is stopped for those queries.","runbook_url":"","metric_refs":["ksql_ksql_engine_query_stats_error_queries"],"line":53},{"kind":"alert","alert":"KSQLPendingErrorQueries","expr":"ksql_ksql_engine_query_stats_pending_error_queries > 0","for":"1m","severity":"warning","summary":"KSQL queries in PENDING_ERROR state on {{ $labels.ksql_cluster }}","description":"{{ $value }} KSQL query(ies) are in PENDING_ERROR state on cluster {{ $labels.ksql_cluster }}.","runbook_url":"","metric_refs":["ksql_ksql_engine_query_stats_pending_error_queries"],"line":62},{"kind":"alert","alert":"KSQLHighErrorRate","expr":"ksql_ksql_engine_query_stats_error_rate > 0.05","for":"5m","severity":"warning","summary":"KSQL cluster {{ $labels.ksql_cluster }} high error rate","description":"KSQL cluster {{ $labels.ksql_cluster }} is failing to process more than 5% of consumed messages.","runbook_url":"","metric_refs":["ksql_ksql_engine_query_stats_error_rate"],"line":71},{"kind":"alert","alert":"KafkaConsumerLagHigh","expr":"kafka_consumer_consumer_fetch_manager_metrics_records_lag_max > 10000","for":"5m","severity":"warning","summary":"Kafka consumer {{ $labels.client_id }} high lag on {{ $labels.topic }}","description":"Kafka consumer {{ $labels.client_id }} has lag above 10000 records on topic {{ $labels.topic }} partition {{ $labels.partition }}.","runbook_url":"","metric_refs":["kafka_consumer_consumer_fetch_manager_metrics_records_lag_max"],"line":80}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/ksql.rules.yml"},{"file":"kubernetes.rules.yml","name":"kubernetes.rules","category":"Kubernetes","rules":[{"kind":"alert","alert":"KubernetesNodeNotReady","expr":"kube_node_status_condition{condition=\"Ready\",status=\"true\"} == 0","for":"10m","severity":"critical","summary":"Kubernetes node not ready ({{ $labels.node }})","description":"Node {{ $labels.node }} has been unready for more than 10 minutes","runbook_url":"","metric_refs":["kube_node_status_condition"],"line":6},{"kind":"alert","alert":"KubernetesNodeMemoryPressure","expr":"kube_node_status_condition{condition=\"MemoryPressure\",status=\"true\"} == 1","for":"2m","severity":"critical","summary":"Kubernetes node memory pressure ({{ $labels.node }})","description":"Node {{ $labels.node }} has MemoryPressure condition","runbook_url":"","metric_refs":["kube_node_status_condition"],"line":15},{"kind":"alert","alert":"KubernetesNodeDiskPressure","expr":"kube_node_status_condition{condition=\"DiskPressure\",status=\"true\"} == 1","for":"2m","severity":"critical","summary":"Kubernetes node disk pressure ({{ $labels.node }})","description":"Node {{ $labels.node }} has DiskPressure condition","runbook_url":"","metric_refs":["kube_node_status_condition"],"line":24},{"kind":"alert","alert":"KubernetesNodeNetworkUnavailable","expr":"kube_node_status_condition{condition=\"NetworkUnavailable\",status=\"true\"} == 1","for":"2m","severity":"critical","summary":"Kubernetes node network unavailable ({{ $labels.node }})","description":"Node {{ $labels.node }} has NetworkUnavailable condition","runbook_url":"","metric_refs":["kube_node_status_condition"],"line":33},{"kind":"alert","alert":"KubernetesNodeOutOfPodCapacity","expr":"sum by (node) (\n  (kube_pod_status_phase{phase=\"Running\"} == 1)\n  + on(uid, instance) group_left(node) (0 * kube_pod_info)\n) / sum by (node) (kube_node_status_allocatable{resource=\"pods\"}) * 100 > 90\n","for":"2m","severity":"warning","summary":"Kubernetes node out of pod capacity ({{ $labels.node }})","description":"Node {{ $labels.node }} is using {{ $value | printf \"%.0f\" }}% of pod capacity","runbook_url":"","metric_refs":["kube_node_status_allocatable","kube_pod_info","kube_pod_status_phase"],"line":42},{"kind":"alert","alert":"KubernetesNodeSchedulingDisabled","expr":"kube_node_spec_unschedulable == 1","for":"30m","severity":"warning","summary":"Kubernetes node scheduling disabled ({{ $labels.node }})","description":"Node {{ $labels.node }} has been unschedulable for more than 30 minutes","runbook_url":"","metric_refs":["kube_node_spec_unschedulable"],"line":55},{"kind":"alert","alert":"KubernetesKubeletScrapeCoverageDegraded","expr":"count by (dc, k8s_cluster_name) (\n  kube_node_info\n  unless on (dc, k8s_cluster_name, node)\n  up{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\"}\n) >= 2\nand on (dc, k8s_cluster_name)\navg_over_time(\n  (\n    count by (dc, k8s_cluster_name) (\n      kube_node_info\n      unless on (dc, k8s_cluster_name, node)\n      up{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\"}\n    )\n    / count by (dc, k8s_cluster_name) (kube_node_info)\n  )[30m:1m]\n) > 0.05\n","for":"15m","severity":"warning","summary":"Kubernetes kubelet scrape coverage degraded","description":"{{ $value }} nodes have no kubelet cAdvisor scrape target, so per-pod CPU and memory metrics are missing for them","runbook_url":"","metric_refs":["k8s_cluster_name","kube_node_info"],"line":64},{"kind":"alert","alert":"KubernetesPodNotHealthy","expr":"sum by (namespace, pod) (kube_pod_status_phase{phase=~\"Pending|Unknown\"}) > 0","for":"15m","severity":"critical","summary":"Kubernetes pod not healthy ({{ $labels.namespace }}/{{ $labels.pod }})","description":"Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-running state for more than 15 minutes","runbook_url":"","metric_refs":["kube_pod_status_phase"],"line":94},{"kind":"alert","alert":"KubernetesPodCrashLooping","expr":"max_over_time(kube_pod_container_status_waiting_reason{reason=\"CrashLoopBackOff\"}[5m]) == 1","for":"10m","severity":"warning","summary":"Kubernetes pod crash looping ({{ $labels.namespace }}/{{ $labels.pod }})","description":"Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container }} has been in CrashLoopBackOff for more than 10 minutes","runbook_url":"","metric_refs":["kube_pod_container_status_waiting_reason"],"line":105},{"kind":"alert","alert":"KubernetesContainerOomKiller","expr":"(kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1)\nand ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason=\"OOMKilled\"}[10m]) == 1\n","for":"0m","severity":"warning","summary":"Kubernetes container OOMKilled ({{ $labels.namespace }}/{{ $labels.pod }})","description":"Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes","runbook_url":"","metric_refs":["kube_pod_container_status_last_terminated_reason","kube_pod_container_status_restarts_total"],"line":116},{"kind":"alert","alert":"KubernetesNamespaceCpuRequestUnderused","expr":"avg_over_time(kube_namespace_cpu5m_request_ratio[6h]) < 0.1\nand on (dc, k8s_cluster_name, namespace)\n(\n  sum by (dc, k8s_cluster_name, namespace) (kube_namespace_cpu_request)\n  - sum by (dc, k8s_cluster_name, namespace) (avg_over_time(kube_container_cpu_usage_5m[6h]))\n  > 0.5\n)\n","for":"6h","severity":"warning","summary":"Kubernetes namespace CPU request underused ({{ $labels.namespace }})","description":"Namespace {{ $labels.namespace }} has averaged {{ $value | humanizePercentage }} of its CPU request over the last 6h - consider lowering requests","runbook_url":"","metric_refs":["k8s_cluster_name","kube_container_cpu_usage_5m","kube_namespace_cpu5m_request_ratio","kube_namespace_cpu_request"],"line":132},{"kind":"alert","alert":"KubernetesNamespaceMemoryRequestUnderused","expr":"avg_over_time(kube_namespace_memory_request_ratio[6h]) < 0.1\nand on (dc, k8s_cluster_name, namespace)\n(\n  sum by (dc, k8s_cluster_name, namespace) (kube_namespace_memory_request)\n  - sum by (dc, k8s_cluster_name, namespace) (avg_over_time(kube_container_memory_usage[6h]))\n  > 1024 * 1024 * 1024\n)\n","for":"6h","severity":"warning","summary":"Kubernetes namespace memory request underused ({{ $labels.namespace }})","description":"Namespace {{ $labels.namespace }} has averaged {{ $value | humanizePercentage }} of its memory request over the last 6h - consider lowering requests","runbook_url":"","metric_refs":["k8s_cluster_name","kube_container_memory_usage","kube_namespace_memory_request","kube_namespace_memory_request_ratio"],"line":148},{"kind":"alert","alert":"KubernetesDeploymentReplicasMismatch","expr":"kube_deployment_spec_replicas != kube_deployment_status_replicas_available","for":"10m","severity":"warning","summary":"Kubernetes deployment replicas mismatch ({{ $labels.namespace }}/{{ $labels.deployment }})","description":"Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has replica count mismatch for more than 10 minutes","runbook_url":"","metric_refs":["kube_deployment_spec_replicas","kube_deployment_status_replicas_available"],"line":166},{"kind":"alert","alert":"KubernetesDeploymentGenerationMismatch","expr":"kube_deployment_status_observed_generation != kube_deployment_metadata_generation","for":"10m","severity":"critical","summary":"Kubernetes deployment generation mismatch ({{ $labels.namespace }}/{{ $labels.deployment }})","description":"Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has failed and has not been rolled back","runbook_url":"","metric_refs":["kube_deployment_metadata_generation","kube_deployment_status_observed_generation"],"line":175},{"kind":"alert","alert":"KubernetesStatefulsetDown","expr":"kube_statefulset_replicas != kube_statefulset_status_replicas_ready > 0","for":"10m","severity":"critical","summary":"Kubernetes StatefulSet down ({{ $labels.namespace }}/{{ $labels.statefulset }})","description":"StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has replicas down","runbook_url":"","metric_refs":["kube_statefulset_replicas","kube_statefulset_status_replicas_ready"],"line":184},{"kind":"alert","alert":"KubernetesStatefulsetReplicasMismatch","expr":"kube_statefulset_status_replicas_ready != kube_statefulset_status_replicas","for":"10m","severity":"warning","summary":"Kubernetes StatefulSet replicas mismatch ({{ $labels.namespace }}/{{ $labels.statefulset }})","description":"StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} does not match the expected number of replicas","runbook_url":"","metric_refs":["kube_statefulset_status_replicas","kube_statefulset_status_replicas_ready"],"line":195},{"kind":"alert","alert":"KubernetesStatefulsetGenerationMismatch","expr":"kube_statefulset_status_observed_generation != kube_statefulset_metadata_generation","for":"10m","severity":"critical","summary":"Kubernetes StatefulSet generation mismatch ({{ $labels.namespace }}/{{ $labels.statefulset }})","description":"StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has failed and has not been rolled back","runbook_url":"","metric_refs":["kube_statefulset_metadata_generation","kube_statefulset_status_observed_generation"],"line":204},{"kind":"alert","alert":"KubernetesStatefulsetUpdateNotRolledOut","expr":"max without (revision) (kube_statefulset_status_current_revision unless kube_statefulset_status_update_revision)\n* (kube_statefulset_replicas != kube_statefulset_status_replicas_updated)\nand\n# Both conditions above are true by definition during any rolling update.\n# Only alert once the rollout has stopped making progress.\n(changes(kube_statefulset_status_replicas_updated[5m]) == 0)\n","for":"10m","severity":"warning","summary":"Kubernetes StatefulSet update not rolled out ({{ $labels.namespace }}/{{ $labels.statefulset }})","description":"StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been fully rolled out","runbook_url":"","metric_refs":["kube_statefulset_replicas","kube_statefulset_status_current_revision","kube_statefulset_status_replicas_updated","kube_statefulset_status_update_revision"],"line":213},{"kind":"alert","alert":"KubernetesReplicasetReplicasMismatch","expr":"kube_replicaset_spec_replicas != kube_replicaset_status_ready_replicas","for":"10m","severity":"warning","summary":"Kubernetes ReplicaSet replicas mismatch ({{ $labels.namespace }}/{{ $labels.replicaset }})","description":"ReplicaSet {{ $labels.namespace }}/{{ $labels.replicaset }} replicas mismatch for more than 10 minutes","runbook_url":"","metric_refs":["kube_replicaset_spec_replicas","kube_replicaset_status_ready_replicas"],"line":228},{"kind":"alert","alert":"KubernetesDaemonsetRolloutStuck","expr":"(kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled * 100 < 100\nand kube_daemonset_status_desired_number_scheduled > 0)\nor kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled > 0\n","for":"10m","severity":"warning","summary":"Kubernetes DaemonSet rollout stuck ({{ $labels.namespace }}/{{ $labels.daemonset }})","description":"Some pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled or not ready","runbook_url":"","metric_refs":["kube_daemonset_status_current_number_scheduled","kube_daemonset_status_desired_number_scheduled","kube_daemonset_status_number_ready"],"line":239},{"kind":"alert","alert":"KubernetesDaemonsetMisscheduled","expr":"kube_daemonset_status_number_misscheduled > 0","for":"1m","severity":"critical","summary":"Kubernetes DaemonSet misscheduled ({{ $labels.namespace }}/{{ $labels.daemonset }})","description":"{{ $value }} pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running on nodes where they should not","runbook_url":"","metric_refs":["kube_daemonset_status_number_misscheduled"],"line":251},{"kind":"alert","alert":"KubernetesJobFailed","expr":"kube_job_status_failed > 0 and kube_job_status_succeeded == 0","for":"0m","severity":"warning","summary":"Kubernetes job failed ({{ $labels.namespace }}/{{ $labels.job_name }})","description":"Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete","runbook_url":"","metric_refs":["kube_job_status_failed","kube_job_status_succeeded"],"line":262},{"kind":"alert","alert":"KubernetesJobNotStarting","expr":"kube_job_status_active == 0\nand kube_job_status_failed == 0\nand kube_job_status_succeeded == 0\nand (time() - kube_job_status_start_time) > 600\n","for":"0m","severity":"warning","summary":"Kubernetes job not starting ({{ $labels.namespace }}/{{ $labels.job_name }})","description":"Job {{ $labels.namespace }}/{{ $labels.job_name }} did not start for 10 minutes","runbook_url":"","metric_refs":["kube_job_status_active","kube_job_status_failed","kube_job_status_start_time","kube_job_status_succeeded"],"line":273},{"kind":"alert","alert":"KubernetesJobSlowCompletion","expr":"kube_job_spec_completions\n- kube_job_status_succeeded\n- sum without (reason) (kube_job_status_failed)\n> 0\n","for":"12h","severity":"critical","summary":"Kubernetes job slow completion ({{ $labels.namespace }}/{{ $labels.job_name }})","description":"Job {{ $labels.namespace }}/{{ $labels.job_name }} did not complete within 12 hours","runbook_url":"","metric_refs":["kube_job_spec_completions","kube_job_status_failed","kube_job_status_succeeded"],"line":286},{"kind":"alert","alert":"KubernetesCronjobFailing","expr":"(\n  # Scheduled more recently than it last succeeded.\n  (kube_cronjob_status_last_schedule_time > kube_cronjob_status_last_successful_time)\n  or\n  # Or never succeeded at all: with no last_successful_time series the\n  # comparison above returns nothing, so the worst case -- a CronJob\n  # that has never once worked -- went undetected.\n  (\n    kube_cronjob_status_last_schedule_time > 0\n    unless\n    kube_cronjob_status_last_successful_time\n  )\n)\nAND (kube_cronjob_status_active == 0)\nAND (kube_cronjob_spec_suspend == 0)\n","for":"5m","severity":"critical","summary":"Kubernetes CronJob failing ({{ $labels.namespace }}/{{ $labels.cronjob }})","description":"CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} last schedule did not complete successfully","runbook_url":"","metric_refs":["kube_cronjob_spec_suspend","kube_cronjob_status_active","kube_cronjob_status_last_schedule_time","kube_cronjob_status_last_successful_time","last_successful_time"],"line":303},{"kind":"alert","alert":"KubernetesJobRunningTooLong","expr":"(time() - (kube_job_status_start_time unless kube_job_status_completion_time)) > 3600\n","for":"0m","severity":"warning","summary":"Kubernetes Job running too long ({{ $labels.namespace }}/{{ $labels.job_name }})","description":"Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than 1 hour to complete","runbook_url":"","metric_refs":["kube_job_status_completion_time","kube_job_status_start_time"],"line":329},{"kind":"alert","alert":"KubernetesPersistentvolumeclaimPending","expr":"kube_persistentvolumeclaim_status_phase{phase=\"Pending\"} == 1","for":"15m","severity":"warning","summary":"Kubernetes PVC pending ({{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }})","description":"PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} has been pending for more than 15 minutes","runbook_url":"","metric_refs":["kube_persistentvolumeclaim_status_phase"],"line":344},{"kind":"alert","alert":"KubernetesVolumeUsageHigh","expr":"kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 15 and kubelet_volume_stats_capacity_bytes > 0","for":"5m","severity":"warning","summary":"Kubernetes volume 85% full ({{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }})","description":"Volume {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is more than 85% used ({{ $value | printf \"%.1f\" }}% free). Expand it before KubernetesVolumeOutOfDiskSpace fires.","runbook_url":"","metric_refs":["kubelet_volume_stats_available_bytes","kubelet_volume_stats_capacity_bytes"],"line":355},{"kind":"alert","alert":"KubernetesVolumeOutOfDiskSpace","expr":"kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10 and kubelet_volume_stats_capacity_bytes > 0","for":"2m","severity":"warning","summary":"Kubernetes volume out of disk space ({{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }})","description":"Volume {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} has less than 10% free space ({{ $value | printf \"%.1f\" }}%)","runbook_url":"","metric_refs":["kubelet_volume_stats_available_bytes","kubelet_volume_stats_capacity_bytes"],"line":364},{"kind":"alert","alert":"KubernetesVolumeFullInFourDays","expr":"kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes < 0.15\nand predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0\n","for":"0m","severity":"critical","summary":"Kubernetes volume full in 4 days ({{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }})","description":"Volume {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is predicted to be full within 4 days","runbook_url":"","metric_refs":["kubelet_volume_stats_available_bytes","kubelet_volume_stats_capacity_bytes"],"line":373},{"kind":"alert","alert":"KubernetesPersistentvolumeError","expr":"kube_persistentvolume_status_phase{phase=~\"Failed|Pending\"} > 0","for":"0m","severity":"critical","summary":"Kubernetes PersistentVolume error ({{ $labels.persistentvolume }})","description":"PersistentVolume {{ $labels.persistentvolume }} is in {{ $labels.phase }} state","runbook_url":"","metric_refs":["kube_persistentvolume_status_phase"],"line":386},{"kind":"alert","alert":"KubernetesHpaScaleInability","expr":"(kube_horizontalpodautoscaler_spec_max_replicas - kube_horizontalpodautoscaler_status_desired_replicas)\n* on (horizontalpodautoscaler, namespace)\n(kube_horizontalpodautoscaler_status_condition{condition=\"ScalingLimited\", status=\"true\"} == 1) == 0\n","for":"2m","severity":"warning","summary":"Kubernetes HPA cannot scale ({{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }})","description":"HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has reached its max replicas and is ScalingLimited","runbook_url":"","metric_refs":["kube_horizontalpodautoscaler_spec_max_replicas","kube_horizontalpodautoscaler_status_condition","kube_horizontalpodautoscaler_status_desired_replicas"],"line":397},{"kind":"alert","alert":"KubernetesHpaMetricsUnavailability","expr":"kube_horizontalpodautoscaler_status_condition{status=\"false\", condition=\"ScalingActive\"} == 1","for":"15m","severity":"warning","summary":"Kubernetes HPA metrics unavailable ({{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }})","description":"HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} cannot collect metrics - autoscaling is inactive","runbook_url":"","metric_refs":["kube_horizontalpodautoscaler_status_condition"],"line":409},{"kind":"alert","alert":"KubernetesApiServerErrors","expr":"sum(rate(apiserver_request_total{job=\"apiserver\",code=~\"5..\"}[5m])) by (instance, job)\n/ sum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (instance, job) * 100 > 3\nand sum(rate(apiserver_request_total{job=\"apiserver\"}[5m])) by (instance, job) > 0\n","for":"2m","severity":"critical","summary":"Kubernetes API server error rate high ({{ $labels.instance }})","description":"Kubernetes API server is experiencing {{ $value | printf \"%.2f\" }}% error rate on {{ $labels.instance }}","runbook_url":"","metric_refs":["apiserver_request_total"],"line":422},{"kind":"alert","alert":"KubernetesApiClientErrors","expr":"(sum(rate(rest_client_requests_total{code=~\"5..\"}[5m])) by (instance, job)\n/ sum(rate(rest_client_requests_total[5m])) by (instance, job)) * 100 > 1\nand sum(rate(rest_client_requests_total[5m])) by (instance, job) > 0\n","for":"2m","severity":"critical","summary":"Kubernetes API client errors ({{ $labels.instance }})","description":"API client on {{ $labels.instance }} is experiencing {{ $value | printf \"%.2f\" }}% error rate","runbook_url":"","metric_refs":["rest_client_requests_total"],"line":434},{"kind":"alert","alert":"KubernetesApiServerLatency","expr":"histogram_quantile(0.99,\n  sum by (verb, le) (\n    rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\",verb!~\"(?:CONNECT|WATCHLIST|WATCH|PROXY|LIST)\"}[10m])\n  )\n) > 1\n","for":"2m","severity":"warning","summary":"Kubernetes API server high latency ({{ $labels.instance }})","description":"API server p99 latency is {{ $value | printf \"%.2f\" }}s for {{ $labels.verb }} {{ $labels.resource }}","runbook_url":"","metric_refs":["apiserver_request_duration_seconds_bucket"],"line":448},{"kind":"alert","alert":"KubernetesClientCertificateExpiresNextWeek","expr":"apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0\nand on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 7*24*60*60\n","for":"0m","severity":"warning","summary":"Kubernetes client certificate expires next week","description":"A client certificate used to authenticate to the apiserver expires within 7 days","runbook_url":"","metric_refs":["apiserver_client_certificate_expiration_seconds_bucket","apiserver_client_certificate_expiration_seconds_count"],"line":469},{"kind":"alert","alert":"KubernetesClientCertificateExpiresSoon","expr":"apiserver_client_certificate_expiration_seconds_count{job=\"apiserver\"} > 0\nand on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job=\"apiserver\"}[5m]))) < 24*60*60\n","for":"0m","severity":"critical","summary":"Kubernetes client certificate expires soon","description":"A client certificate used to authenticate to the apiserver expires within 24 hours","runbook_url":"","metric_refs":["apiserver_client_certificate_expiration_seconds_bucket","apiserver_client_certificate_expiration_seconds_count"],"line":480}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/kubernetes.rules.yml"},{"file":"memcached.rules.yml","name":"memcached.rules","category":"Databases","rules":[{"kind":"alert","alert":"MemcachedExporterDown","expr":"group by (job, instance) (up{job=~\".*memcached.*\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=~\".*memcached.*\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=~\".*memcached.*\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"Memcached exporter down - {{ $labels.job }} @ {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"MemcachedDown","expr":"memcached_up == 0","for":"2m","severity":"critical","summary":"Memcached down on {{ $labels.instance }}","description":"The exporter on {{ $labels.instance }} cannot reach memcached.","runbook_url":"","metric_refs":["memcached_up"],"line":19},{"kind":"alert","alert":"MemcachedConnectionsSaturation","expr":"memcached_current_connections / (memcached_max_connections > 0) > 0.85\n","for":"5m","severity":"critical","summary":"Memcached connections near limit on {{ $labels.instance }} ({{ $value | humanizePercentage }})","description":"More than 85% of max_connections in use on {{ $labels.instance }}. memcached rejects new connections at the limit rather than queueing them.","runbook_url":"","metric_refs":["memcached_current_connections","memcached_max_connections"],"line":29},{"kind":"alert","alert":"MemcachedMemorySaturation","expr":"memcached_current_bytes / (memcached_limit_bytes > 0) > 0.90\n","for":"10m","severity":"warning","summary":"Memcached memory near limit on {{ $labels.instance }} ({{ $value | humanizePercentage }})","description":"Stored data is above 90% of the configured limit on {{ $labels.instance }}. Expect evictions.","runbook_url":"","metric_refs":["memcached_current_bytes","memcached_limit_bytes"],"line":41},{"kind":"alert","alert":"MemcachedEvictionsHigh","expr":"rate(memcached_items_evicted_total[5m]) > 10","for":"15m","severity":"warning","summary":"Memcached evicting {{ $value | humanize }} items/s on {{ $labels.instance }}","description":"Sustained eviction on {{ $labels.instance }} means the working set no longer fits. Consider raising the memory limit.","runbook_url":"","metric_refs":["memcached_items_evicted_total"],"line":51},{"kind":"alert","alert":"MemcachedLowHitRate","expr":"sum by (instance, job) (rate(memcached_commands_total{command=\"get\",status=\"hit\"}[5m]))\n/\n(sum by (instance, job) (rate(memcached_commands_total{command=\"get\"}[5m])) > 10)\n< 0.50\n","for":"15m","severity":"warning","summary":"Memcached hit rate low on {{ $labels.instance }} ({{ $value | humanizePercentage }})","description":"Fewer than half of gets are served from cache on {{ $labels.instance }} at more than 10 req/s. The cache is not earning its keep.","runbook_url":"","metric_refs":["memcached_commands_total"],"line":62}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/memcached.rules.yml"},{"file":"mongodb.rules.yml","name":"mongodb.rules","category":"Databases","rules":[{"kind":"alert","alert":"MongodbExporterDown","expr":"group by (job, instance) (up{job=~\".*mongodb.*\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=~\".*mongodb.*\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=~\".*mongodb.*\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"MongoDB exporter down - {{ $labels.job }} @ {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":8},{"kind":"alert","alert":"MongodbDown","expr":"mongodb_up == 0","for":"2m","severity":"critical","summary":"MongoDB down on {{ $labels.instance }}","description":"The exporter on {{ $labels.instance }} cannot reach mongod.","runbook_url":"","metric_refs":["mongodb_up"],"line":24},{"kind":"alert","alert":"MongodbConnectionsSaturation","expr":"mongodb_ss_connections{conn_type=\"current\"}\n/ ignoring(conn_type) (\n    mongodb_ss_connections{conn_type=\"current\"}\n    + ignoring(conn_type) mongodb_ss_connections{conn_type=\"available\"}\n    > 0\n  )\n> 0.80\n","for":"10m","severity":"critical","summary":"MongoDB connections near limit on {{ $labels.instance }} ({{ $value | humanizePercentage }})","description":"More than 80% of the available connection slots are in use on {{ $labels.instance }}.","runbook_url":"","metric_refs":["conn_type","mongodb_ss_connections"],"line":33},{"kind":"alert","alert":"MongodbHighReadQueue","expr":"mongodb_ss_globalLock_currentQueue{count_type=\"readers\"} > 100","for":"10m","severity":"warning","summary":"MongoDB read queue on {{ $labels.instance }} ({{ $value | humanize }} waiting)","description":"{{ $value | humanize }} readers are queued on the global lock on {{ $labels.instance }}.","runbook_url":"","metric_refs":["mongodb_ss_globalLock_currentQueue"],"line":51},{"kind":"alert","alert":"MongodbHighWriteQueue","expr":"mongodb_ss_globalLock_currentQueue{count_type=\"writers\"} > 100","for":"10m","severity":"warning","summary":"MongoDB write queue on {{ $labels.instance }} ({{ $value | humanize }} waiting)","description":"{{ $value | humanize }} writers are queued on the global lock on {{ $labels.instance }}.","runbook_url":"","metric_refs":["mongodb_ss_globalLock_currentQueue"],"line":61},{"kind":"alert","alert":"MongodbWiredTigerCacheSaturation","expr":"mongodb_ss_wt_cache_bytes_currently_in_the_cache\n/ (mongodb_ss_wt_cache_maximum_bytes_configured > 0)\n> 0.95\n","for":"15m","severity":"warning","summary":"MongoDB WiredTiger cache at {{ $value | humanizePercentage }} on {{ $labels.instance }}","description":"The WiredTiger cache on {{ $labels.instance }} is above 95% full. Eviction pressure will show up as read latency.","runbook_url":"","metric_refs":["mongodb_ss_wt_cache_bytes_currently_in_the_cache","mongodb_ss_wt_cache_maximum_bytes_configured"],"line":70}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/mongodb.rules.yml"},{"file":"mysql.rules.yml","name":"mysql.rules","category":"Databases","rules":[{"kind":"alert","alert":"MySQLReplicationDown","expr":"sum(mysql_slave_status_slave_sql_running + mysql_slave_status_slave_io_running) by (instance,master_host,job) < 2","for":"2m","severity":"critical","summary":"MySQL Replication is not running {{$labels.instance}}","description":"MySQL replication seems down on {{$labels.instance}}","runbook_url":"","metric_refs":["master_host","mysql_slave_status_slave_io_running","mysql_slave_status_slave_sql_running"],"line":3},{"kind":"alert","alert":"MysqlExporter","expr":"# Two distinct failure modes: a failing scrape (up == 0, exporter crashed\n# or host unreachable) and the target vanishing from service discovery,\n# where no up series exists to be 0. Both branches are grouped\n# identically so the alert carries the same labels either way.\ngroup by (job, instance) (up{job=~\".*mysql-exporter\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=~\".*mysql-exporter\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=~\".*mysql-exporter\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"Job {{ $labels.job }} having an issue on {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":11},{"kind":"alert","alert":"MysqlDown","expr":"mysql_up == 0","for":"2m","severity":"critical","summary":"Mysql having an issue on {{ $labels.instance }}","description":"Mysql on {{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":["mysql_up"],"line":30},{"kind":"alert","alert":"MysqlConnectionErrorsCritical","expr":"sum by (job,instance,error) (\n  rate(mysql_global_status_connection_errors_total{error=~\"max_connections|internal\"}[5m])\n) > 0\n","for":"2m","severity":"critical","summary":"MySQL connection errors ({{ $labels.error }}) on {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} is rejecting connections ({{ $labels.error }}) - clients are being refused","runbook_url":"","metric_refs":["mysql_global_status_connection_errors_total"],"line":43},{"kind":"alert","alert":"MysqlConnectionErrors","expr":"sum by (job,instance,error) (\n  rate(mysql_global_status_connection_errors_total{error!~\"max_connections|internal\"}[5m])\n) > 0\n","for":"5m","severity":"warning","summary":"MySQL connection errors ({{ $labels.error }}) on {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has connection errors of type {{ $labels.error }} ({{ $value }}/s)","runbook_url":"","metric_refs":["mysql_global_status_connection_errors_total"],"line":54},{"kind":"alert","alert":"GaleraWrongSize","expr":"sum(mysql_global_status_wsrep_cluster_size) by (job,instance) < 3","for":"2m","severity":"critical","summary":"{{ $labels.instance }} of job {{ $labels.job }} missing Galera instances {{ $value }}","description":"This node sees a Galera cluster of {{ $value }} members, fewer than the expected 3. One more loss drops it below quorum and it stops accepting writes.","runbook_url":"","metric_refs":["mysql_global_status_wsrep_cluster_size"],"line":67},{"kind":"alert","alert":"GaleraNodeUnhealthy","expr":"sum(mysql_global_status_wsrep_cluster_status) by (job,instance) != 1","for":"2m","severity":"critical","summary":"{{ $labels.instance }} of job {{ $labels.job }} unhealthy node","description":"wsrep_cluster_status is not Primary: this node sits in a non-primary component and refuses writes. Usually a network partition or a quorum loss.","runbook_url":"","metric_refs":["mysql_global_status_wsrep_cluster_status"],"line":79},{"kind":"alert","alert":"GaleraOutOfSync","expr":"mysql_global_status_wsrep_local_state != 4 and mysql_global_variables_wsrep_desync == 0","for":"5m","severity":"warning","summary":"Galera cluster node out of sync.","description":"{{$labels.job}} on {{$labels.instance}} is not in sync ({{$value}} != 4).","runbook_url":"","metric_refs":["mysql_global_status_wsrep_local_state","mysql_global_variables_wsrep_desync"],"line":87},{"kind":"alert","alert":"GaleraDonorFallingBehind","expr":"mysql_global_status_wsrep_local_state == 2 and mysql_global_status_wsrep_local_recv_queue > 100","for":"5m","severity":"warning","summary":"Galera cluster donor node falling behind.","description":"{{$labels.job}} on {{$labels.instance}} is a donor (hotbackup) and is falling behind (queue size {{$value}}).","runbook_url":"","metric_refs":["mysql_global_status_wsrep_local_recv_queue","mysql_global_status_wsrep_local_state"],"line":95},{"kind":"alert","alert":"GaleraNotReady","expr":"mysql_global_status_wsrep_ready != 1","for":"5m","severity":"critical","summary":"Galera cluster node not ready","description":"{{$labels.job}} on {{$labels.instance}} is not ready.","runbook_url":"","metric_refs":["mysql_global_status_wsrep_ready"],"line":103},{"kind":"alert","alert":"MySQLInnoDBLogWaits","expr":"(\n  rate(mysql_global_status_innodb_log_waits[15m])\n  /\n  rate(mysql_global_status_innodb_log_write_requests[15m])\n) > 0.01\nand\nrate(mysql_global_status_innodb_log_write_requests[15m]) > 1\n","for":"5m","severity":"warning","summary":"MySQL InnoDB log writes stalling (instance {{ $labels.instance }})","description":"More than 1% of InnoDB log write requests are waiting for disk on {{ $labels.instance }} ({{ $value | humanizePercentage }}) - consider increasing innodb_log_buffer_size","runbook_url":"","metric_refs":["mysql_global_status_innodb_log_waits","mysql_global_status_innodb_log_write_requests"],"line":111},{"kind":"alert","alert":"MysqlTooManyConnections","expr":"max_over_time(mysql_global_status_threads_connected[1m]) / mysql_global_variables_max_connections * 100 > 80 and mysql_global_variables_max_connections > 0","for":"2m","severity":"warning","summary":"MySQL too many connections (instance {{ $labels.instance }})","description":"More than 80% of MySQL connections are in use on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["mysql_global_status_threads_connected","mysql_global_variables_max_connections"],"line":127},{"kind":"alert","alert":"MysqlHighThreadsRunning","expr":"max_over_time(mysql_global_status_threads_running[1m]) / mysql_global_variables_max_connections * 100 > 60 and mysql_global_variables_max_connections > 0","for":"2m","severity":"warning","summary":"MySQL high threads running (instance {{ $labels.instance }})","description":"More than 60% of MySQL connections are in running state on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["mysql_global_status_threads_running","mysql_global_variables_max_connections"],"line":136},{"kind":"alert","alert":"MysqlHighPreparedStatementsUtilization","expr":"max_over_time(mysql_global_status_prepared_stmt_count[1m]) / mysql_global_variables_max_prepared_stmt_count * 100 > 80 and mysql_global_variables_max_prepared_stmt_count > 0","for":"2m","severity":"warning","summary":"MySQL high prepared statements utilization (instance {{ $labels.instance }})","description":"High utilization of prepared statements (> 80%) on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["mysql_global_status_prepared_stmt_count","mysql_global_variables_max_prepared_stmt_count"],"line":145},{"kind":"alert","alert":"MysqlSlowQueries","expr":"increase(mysql_global_status_slow_queries[1m]) > 0","for":"2m","severity":"warning","summary":"MySQL slow queries (instance {{ $labels.instance }})","description":"MySQL server has new slow queries ({{ $value }} in the last minute) on {{ $labels.instance }}","runbook_url":"","metric_refs":["mysql_global_status_slow_queries"],"line":154},{"kind":"alert","alert":"MysqlTooManyOpenFiles","expr":"mysql_global_status_innodb_num_open_files / mysql_global_variables_open_files_limit * 100 > 75 and mysql_global_variables_open_files_limit > 0","for":"2m","severity":"warning","summary":"MySQL too many open files (instance {{ $labels.instance }})","description":"MySQL open files > 75% of open_files_limit on {{ $labels.instance }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["mysql_global_status_innodb_num_open_files","mysql_global_variables_open_files_limit"],"line":163},{"kind":"alert","alert":"MysqlInnodbForceRecoveryEnabled","expr":"mysql_global_variables_innodb_force_recovery != 0","for":"2m","severity":"warning","summary":"MySQL InnoDB force recovery enabled (instance {{ $labels.instance }})","description":"innodb_force_recovery is enabled on {{ $labels.instance }} - should only be set temporarily for crash recovery","runbook_url":"","metric_refs":["mysql_global_variables_innodb_force_recovery"],"line":172},{"kind":"alert","alert":"MysqlInnodbHistoryLenTooLong","expr":"mysql_info_schema_innodb_metrics_transaction_trx_rseg_history_len > 50000","for":"2m","severity":"warning","summary":"MySQL InnoDB history_len too long (instance {{ $labels.instance }})","description":"InnoDB undo log (history_len) is too long ({{ $value }}) on {{ $labels.instance }} - long-running transactions suspected","runbook_url":"","metric_refs":["mysql_info_schema_innodb_metrics_transaction_trx_rseg_history_len"],"line":181}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/mysql.rules.yml"},{"file":"nvme.rules.yml","name":"nvme.rules","category":"Hosts & hardware","rules":[{"kind":"alert","alert":"NVMEDown","expr":"# Two distinct failure modes: a failing scrape (up == 0, exporter crashed\n# or host unreachable) and the target vanishing from service discovery,\n# where no up series exists to be 0. Both branches are grouped\n# identically so the alert carries the same labels either way.\ngroup by (job, instance) (up{job=\"nvme_exporter\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=\"nvme_exporter\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=\"nvme_exporter\"}[2m]))\n)\n","for":"5m","severity":"critical","summary":"NVME Exporter Unreachable {{$labels.instance}}","description":"nvme_exporter seems down on {{$labels.instance}}","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"NVMEMediaError","expr":"sum(rate(nvme_media_errors[1m])) by (instance,device,job) > 0","for":"5m","severity":"critical","summary":"Critical Usage for {{ $labels.device }}","description":"You should change the NVME {{ $labels.device }} on {{ $labels.instance }}","runbook_url":"","metric_refs":["nvme_media_errors"],"line":22},{"kind":"alert","alert":"NVMEUsage","expr":"nvme_percent_used > 40","for":"1m","severity":"warning","summary":"High NVMe wear level for {{ $labels.device }} on {{ $labels.instance }}","description":"NVMe {{ $labels.device }} on {{ $labels.instance }} wear level is above 40% - plan for replacement.","runbook_url":"","metric_refs":["nvme_percent_used"],"line":30},{"kind":"alert","alert":"NVMEUsageCritical","expr":"nvme_percent_used > 60","for":"1m","severity":"critical","summary":"Critical NVMe wear level for {{ $labels.device }} on {{ $labels.instance }}","description":"NVMe {{ $labels.device }} on {{ $labels.instance }} wear level is above 60% - replace soon.","runbook_url":"","metric_refs":["nvme_percent_used"],"line":38},{"kind":"alert","alert":"NVMECriticalWarning","expr":"nvme_critical_warning > 0","for":"0m","severity":"critical","summary":"NVMe critical warning on {{ $labels.device }}","description":"NVMe controller has active critical warnings on {{ $labels.device }}.","runbook_url":"","metric_refs":["nvme_critical_warning"],"line":47},{"kind":"alert","alert":"NVMEAvailSpareLow","expr":"nvme_avail_spare < nvme_spare_thresh","for":"0m","severity":"critical","summary":"NVMe spare capacity low on {{ $labels.device }}","description":"NVMe spare capacity below manufacturer threshold on {{ $labels.device }} ({{ $value }}%).","runbook_url":"","metric_refs":["nvme_avail_spare","nvme_spare_thresh"],"line":56},{"kind":"alert","alert":"NVMEUnsafeShutdowns","expr":"increase(nvme_unsafe_shutdowns[24h]) > 0","for":"0m","severity":"warning","summary":"NVMe unsafe shutdown on {{ $labels.device }}","description":"NVMe device {{ $labels.device }} had an unsafe shutdown in the last 24h.","runbook_url":"","metric_refs":["nvme_unsafe_shutdowns"],"line":65},{"kind":"alert","alert":"NVMEErrorLogEntries","expr":"increase(nvme_num_err_log_entries[1h]) > 0","for":"0m","severity":"warning","summary":"NVMe error log entries on {{ $labels.device }}","description":"NVMe device {{ $labels.device }} logged new error entries in the past hour.","runbook_url":"","metric_refs":["nvme_num_err_log_entries"],"line":74},{"kind":"alert","alert":"NVMEHighTemperature","expr":"nvme_temperature > 158","for":"5m","severity":"warning","summary":"NVMe high temperature on {{ $labels.device }}","description":"NVMe device {{ $labels.device }} temperature above 70C ({{ $value }}F). Check cooling.","runbook_url":"","metric_refs":["nvme_temperature"],"line":83}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/nvme.rules.yml"},{"file":"php.rules.yml","name":"php.rules","category":"Web & load balancing","rules":[{"kind":"alert","alert":"PHPFPMExporter","expr":"# Two distinct failure modes: a failing scrape (up == 0, exporter crashed\n# or host unreachable) and the target vanishing from service discovery,\n# where no up series exists to be 0. Both branches are grouped\n# identically so the alert carries the same labels either way.\ngroup by (job, instance) (up{job=\"phpfpm\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=\"phpfpm\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=\"phpfpm\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"Job {{ $labels.job }} having an issue on {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"PHPFPMDown","expr":"sum(phpfpm_up) by (instance,job,pool) == 0","for":"2m","severity":"critical","summary":"PHPFPM Exporter is UP but PHPFPM seems down for {{ $labels.instance }}","description":"Pool {{ $labels.pool }} reports phpfpm_up == 0 while the exporter scrapes fine: the FPM status page is unreachable or the pool is down. Requests to this pool fail.","runbook_url":"","metric_refs":["phpfpm_up"],"line":22},{"kind":"alert","alert":"PHPFPMConnectionLimit","expr":"sum(rate(phpfpm_max_children_reached[2m])) by (instance,job) > 0","for":"0s","severity":"critical","summary":"High connection usage on PHPFPM for {{ $labels.instance }} for all pools","description":"Pools on {{ $labels.instance }} hit pm.max_children: new requests wait in the listen backlog. Raise max_children if memory allows, or find the slow requests holding workers.","runbook_url":"","metric_refs":["phpfpm_max_children_reached"],"line":33},{"kind":"alert","alert":"PHPFPMQueueSaturation","expr":"phpfpm_listen_queue / (phpfpm_listen_queue_length > 0) > 0.8","for":"2m","severity":"critical","summary":"PHP-FPM queue saturation on pool {{ $labels.pool }}","description":"PHP-FPM queue above 80% full on pool {{ $labels.pool }}. New requests will be rejected soon.","runbook_url":"","metric_refs":["phpfpm_listen_queue","phpfpm_listen_queue_length"],"line":42},{"kind":"alert","alert":"PHPFPMProcessSaturation","expr":"phpfpm_active_processes / phpfpm_total_processes > 0.9","for":"5m","severity":"warning","summary":"PHP-FPM process saturation on pool {{ $labels.pool }}","description":"90% of PHP-FPM workers busy on pool {{ $labels.pool }}. Pool may become saturated.","runbook_url":"","metric_refs":["phpfpm_active_processes","phpfpm_total_processes"],"line":53},{"kind":"alert","alert":"PHPFPMSlowRequests","expr":"(\n  increase(phpfpm_slow_requests[5m])\n  /\n  increase(phpfpm_accepted_connections[5m])\n) > 0.01\nand\nincrease(phpfpm_accepted_connections[5m]) > 10\n","for":"0m","severity":"warning","summary":"PHP-FPM slow requests on pool {{ $labels.pool }}","description":"More than 1% of requests are slow on pool {{ $labels.pool }} ({{ $value | humanizePercentage }})","runbook_url":"","metric_refs":["phpfpm_accepted_connections","phpfpm_slow_requests"],"line":62},{"kind":"alert","alert":"PHPFPMScrapeFailed","expr":"increase(phpfpm_scrape_failures[5m]) > 0","for":"0m","severity":"critical","summary":"PHP-FPM exporter scrape failed for pool {{ $labels.pool }}","description":"PHP-FPM exporter cannot reach pool {{ $labels.pool }}.","runbook_url":"","metric_refs":["phpfpm_scrape_failures"],"line":78}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/php.rules.yml"},{"file":"ping.rules.yml","name":"ping.rules","category":"Network, DNS & TLS","rules":[{"kind":"alert","alert":"PingHighPacketLoss","expr":"ping_loss_ratio > 0.1","for":"2m","severity":"critical","summary":"Ping high packet loss (target {{ $labels.target }})","description":"Packet loss is above 10% for target {{ $labels.target }} on {{ $labels.ip }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["ping_loss_ratio"],"line":3},{"kind":"alert","alert":"PingPacketLossTotal","expr":"ping_loss_ratio >= 1","for":"1m","severity":"critical","summary":"Ping packet loss total (target {{ $labels.target }})","description":"100% packet loss detected for target {{ $labels.target }} on {{ $labels.ip }}","runbook_url":"","metric_refs":["ping_loss_ratio"],"line":12},{"kind":"alert","alert":"PingHighLatency","expr":"ping_rtt_mean_ms > 200","for":"5m","severity":"warning","summary":"Ping high latency (target {{ $labels.target }})","description":"Mean RTT is above 200ms for target {{ $labels.target }} on {{ $labels.ip }}\n  VALUE = {{ $value }}ms","runbook_url":"","metric_refs":["ping_rtt_mean_ms"],"line":21},{"kind":"alert","alert":"PingExporterDown","expr":"ping_up == 0","for":"2m","severity":"critical","summary":"Ping exporter down (instance {{ $labels.instance }})","description":"Ping exporter is down on {{ $labels.instance }}","runbook_url":"","metric_refs":["ping_up"],"line":30}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/ping.rules.yml"},{"file":"postgresql.rules.yml","name":"postgresql.rules","category":"Databases","rules":[{"kind":"alert","alert":"PostgresExporter","expr":"# Two distinct failure modes: a failing scrape (up == 0, exporter crashed\n# or host unreachable) and the target vanishing from service discovery,\n# where no up series exists to be 0. Both branches are grouped\n# identically so the alert carries the same labels either way.\ngroup by (job, instance) (up{job=~\".*postgres.*\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=~\".*postgres.*\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=~\".*postgres.*\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"Job {{ $labels.job }} having an issue on {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"PostgreSQLDown","expr":"sum(pg_up) by (instance,job) == 0","for":"1m","severity":"critical","summary":"{{ $labels.job }}/{{ $labels.instance }} cant be scraped","description":"{{ $labels.job }}/{{ $labels.instance }} cant be scraped, assume its down. Check if PGSQL is UP or user `prometheus` exists","runbook_url":"","metric_refs":["pg_up"],"line":23},{"kind":"alert","alert":"PostgreSQLMaxConnectionsReached","expr":"sum by (instance,job) (pg_stat_activity_count) >=\n(sum by (instance,job) (pg_settings_max_connections)-\nsum by (instance,job) (pg_settings_superuser_reserved_connections))\n","for":"1m","severity":"critical","summary":"{{ $labels.instance }} has maxed out Postgres connections.","description":"{{ $labels.instance }} is exceeding the currently configured maximum Postgres connection limit (current value: {{ $value }}). Services may be degraded - please take immediate action (you probably need to increase max_connections in the configuration)","runbook_url":"","metric_refs":["pg_settings_max_connections","pg_settings_superuser_reserved_connections","pg_stat_activity_count"],"line":31},{"kind":"alert","alert":"PostgreSQLHighConnections","expr":"sum by (instance,job) (pg_stat_activity_count) >\n(\n  sum by (instance,job) (pg_settings_max_connections)\n  -\n  sum by (instance,job) (pg_settings_superuser_reserved_connections)\n) * 0.8\n","for":"5m","severity":"warning","summary":"{{ $labels.instance }} is over 80% of max Postgres connections.","description":"{{ $labels.instance }} is exceeding 80% of the currently configured maximum PostgreSQL connection limit (current value: {{ $value }}s). Please check utilization graphs and confirm if this is normal service growth, abuse or an otherwise temporary condition or if new resources need to be provisioned (or the limits increased, which is mostly likely).","runbook_url":"","metric_refs":["pg_settings_max_connections","pg_settings_superuser_reserved_connections","pg_stat_activity_count"],"line":42},{"kind":"alert","alert":"PostgreSQLSlowQueries","expr":"max by (datname,instance) (\n  pg_stat_activity_max_tx_duration{datname!~\"template.*\"}\n) > 10\n","for":"5m","severity":"warning","summary":"PostgreSQL high number of slow queries on {{ $labels.instance }} for database {{ $labels.datname }}","description":"PostgreSQL high number of slow queries {{ $labels.instance }} for database {{ $labels.datname }} with a value of {{ $value }}","runbook_url":"","metric_refs":["pg_stat_activity_max_tx_duration"],"line":56},{"kind":"alert","alert":"PostgreSQLHighQPS","expr":"avg by (datname,instance) (\n  irate(\n    pg_stat_database_xact_commit{datname!~\"template.*\"}[5m]\n  )\n  +\n  irate(\n    pg_stat_database_xact_rollback{datname!~\"template.*\"}[5m]\n  )\n) > 8000\n","for":"5m","severity":"warning","summary":"PostgreSQL high number of queries per second on {{ $labels.instance }} for database {{ $labels.datname }}","description":"PostgreSQL high number of queries per second on {{ $labels.instance }} for database {{ $labels.datname }} with a value of {{ $value }}","runbook_url":"","metric_refs":["pg_stat_database_xact_commit","pg_stat_database_xact_rollback"],"line":67},{"kind":"alert","alert":"PostgreSQLHighQPSCritical","expr":"avg by (datname,instance) (\n  irate(\n    pg_stat_database_xact_commit{datname!~\"template.*\"}[5m]\n  )\n  +\n  irate(\n    pg_stat_database_xact_rollback{datname!~\"template.*\"}[5m]\n  )\n) > 10000\n","for":"5m","severity":"critical","summary":"PostgreSQL critical number of queries per second on {{ $labels.instance }} for database {{ $labels.datname }}","description":"PostgreSQL very high number of queries per second on {{ $labels.instance }} for database {{ $labels.datname }} with a value of {{ $value }}","runbook_url":"","metric_refs":["pg_stat_database_xact_commit","pg_stat_database_xact_rollback"],"line":84},{"kind":"alert","alert":"PostgreSQLRepliDown","expr":"sum(pg_replication_slots_active) by (instance,job,slot_name) < 1","for":"5m","severity":"critical","summary":"PostgreSQL replication down for {{ $labels.instance }} / {{ $labels.slot_name}} ","description":"Replication slot {{ $labels.slot_name }} is inactive: its consumer is disconnected. The primary retains WAL for the slot, so disk usage grows until it reconnects or the slot is dropped.","runbook_url":"","metric_refs":["pg_replication_slots_active","slot_name"],"line":101},{"kind":"alert","alert":"PostgresqlDeadLocks","expr":"(\n  increase(pg_stat_database_deadlocks{datname!~\"template.*|postgres\",datid!=\"0\"}[5m])\n  /\n  increase(pg_stat_database_xact_commit{datname!~\"template.*|postgres\",datid!=\"0\"}[5m])\n) > 0.001\nand\nincrease(pg_stat_database_xact_commit{datname!~\"template.*|postgres\",datid!=\"0\"}[5m]) > 50\n","for":"0m","severity":"warning","summary":"PostgreSQL dead locks (instance {{ $labels.instance }})","description":"Deadlock rate above 0.1% of commits on {{ $labels.instance }}/{{ $labels.datname }} over 5m ({{ $value | humanizePercentage }})","runbook_url":"","metric_refs":["pg_stat_database_deadlocks","pg_stat_database_xact_commit"],"line":114},{"kind":"alert","alert":"PostgresqlHighRollbackRate","expr":"sum by (namespace,datname,instance) (rate(pg_stat_database_xact_rollback{datname!~\"template.*|postgres\",datid!=\"0\"}[3m]))\n/ (\n  sum by (namespace,datname,instance) (rate(pg_stat_database_xact_rollback{datname!~\"template.*|postgres\",datid!=\"0\"}[3m]))\n  + sum by (namespace,datname,instance) (rate(pg_stat_database_xact_commit{datname!~\"template.*|postgres\",datid!=\"0\"}[3m]))\n) > 0.02\nand (\n  sum by (namespace,datname,instance) (rate(pg_stat_database_xact_rollback{datname!~\"template.*|postgres\",datid!=\"0\"}[3m]))\n  + sum by (namespace,datname,instance) (rate(pg_stat_database_xact_commit{datname!~\"template.*|postgres\",datid!=\"0\"}[3m]))\n) > 0\n","for":"0m","severity":"warning","summary":"PostgreSQL high rollback rate (instance {{ $labels.instance }})","description":"Ratio of rollbacks vs commits > 2% on {{ $labels.instance }}/{{ $labels.datname }}\n  VALUE = {{ $value }}","runbook_url":"","metric_refs":["pg_stat_database_xact_commit","pg_stat_database_xact_rollback"],"line":133},{"kind":"alert","alert":"PostgresqlTooManyDeadTuples","expr":"((pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1 and (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup) > 0","for":"2m","severity":"warning","summary":"PostgreSQL too many dead tuples (instance {{ $labels.instance }})","description":"Dead tuples ratio >= 10% on {{ $labels.instance }} table {{ $labels.relname }} - VACUUM may be needed","runbook_url":"","metric_refs":["pg_stat_user_tables_n_dead_tup","pg_stat_user_tables_n_live_tup"],"line":151},{"kind":"alert","alert":"PostgresqlTooManyLocksAcquired","expr":"sum by (server, instance, job) (pg_locks_count)\n/ ((pg_settings_max_locks_per_transaction * pg_settings_max_connections) > 0)\n> 0.20\n","for":"2m","severity":"critical","summary":"PostgreSQL too many locks acquired (instance {{ $labels.instance }})","description":"More than 20% of max_locks_per_transaction used on {{ $labels.instance }} - consider increasing max_locks_per_transaction","runbook_url":"","metric_refs":["pg_locks_count","pg_settings_max_connections","pg_settings_max_locks_per_transaction"],"line":160},{"kind":"alert","alert":"PostgresqlTableNotAutoVacuumed","expr":"((pg_stat_user_tables_n_tup_del + pg_stat_user_tables_n_tup_upd + pg_stat_user_tables_n_tup_hot_upd) > pg_settings_autovacuum_vacuum_threshold)\nand (time() - pg_stat_user_tables_last_autovacuum) > 60 * 60 * 24 * 10\n","for":"0m","severity":"warning","summary":"PostgreSQL table not auto vacuumed (instance {{ $labels.instance }})","description":"Table {{ $labels.relname }} has not been auto vacuumed for 10 days on {{ $labels.instance }}","runbook_url":"","metric_refs":["pg_settings_autovacuum_vacuum_threshold","pg_stat_user_tables_last_autovacuum","pg_stat_user_tables_n_tup_del","pg_stat_user_tables_n_tup_hot_upd","pg_stat_user_tables_n_tup_upd"],"line":174},{"kind":"alert","alert":"PostgresqlUnusedReplicationSlot","expr":"(pg_replication_slots_active == 0) and (pg_replication_is_replica == 0)","for":"1m","severity":"warning","summary":"PostgreSQL unused replication slot (instance {{ $labels.instance }})","description":"Inactive replication slot on {{ $labels.instance }} - will prevent WAL cleanup and may fill disk","runbook_url":"","metric_refs":["pg_replication_is_replica","pg_replication_slots_active"],"line":185},{"kind":"alert","alert":"PostgresqlBloatIndexHigh","expr":"pg_bloat_btree_bloat_pct > 80 and on (idxname) (pg_bloat_btree_real_size > 100000000)","for":"1h","severity":"warning","summary":"PostgreSQL bloat index high (instance {{ $labels.instance }})","description":"Index {{ $labels.idxname }} bloat > 80% on {{ $labels.instance }} - run: REINDEX INDEX CONCURRENTLY {{ $labels.idxname }}","runbook_url":"","metric_refs":["pg_bloat_btree_bloat_pct","pg_bloat_btree_real_size"],"line":194},{"kind":"alert","alert":"PostgresqlBloatTableHigh","expr":"pg_bloat_table_bloat_pct > 80 and on (relname) (pg_bloat_table_real_size > 200000000)","for":"1h","severity":"warning","summary":"PostgreSQL bloat table high (instance {{ $labels.instance }})","description":"Table {{ $labels.relname }} bloat > 80% on {{ $labels.instance }} - run: VACUUM {{ $labels.relname }}","runbook_url":"","metric_refs":["pg_bloat_table_bloat_pct","pg_bloat_table_real_size"],"line":203},{"kind":"alert","alert":"PostgresqlInvalidIndex","expr":"pg_general_index_info_pg_relation_size{indexrelname=~\".*ccnew.*\"}","for":"6h","severity":"warning","summary":"PostgreSQL invalid index (instance {{ $labels.instance }})","description":"Table {{ $labels.relname }} has invalid index {{ $labels.indexrelname }} on {{ $labels.instance }} - run: DROP INDEX {{ $labels.indexrelname }}","runbook_url":"","metric_refs":["pg_general_index_info_pg_relation_size"],"line":212},{"kind":"alert","alert":"PostgresExporterScrapeFailed","expr":"pg_exporter_last_scrape_error == 1","for":"2m","severity":"critical","summary":"PostgreSQL exporter scrape failed (instance {{ $labels.instance }})","description":"postgres_exporter cannot scrape {{ $labels.instance }}","runbook_url":"","metric_refs":["pg_exporter_last_scrape_error"],"line":221},{"kind":"alert","alert":"PostgresReplicationLagHigh","expr":"(pg_replication_lag or pg_replication_lag_seconds) > 30","for":"5m","severity":"warning","summary":"PostgreSQL replication lag high (server {{ $labels.server }})","description":"Replication lag on {{ $labels.server }} is {{ $value | humanizeDuration }}. Replica is falling behind.","runbook_url":"","metric_refs":["pg_replication_lag","pg_replication_lag_seconds"],"line":230},{"kind":"alert","alert":"PostgresReplicationLagCritical","expr":"(pg_replication_lag or pg_replication_lag_seconds) > 300","for":"2m","severity":"critical","summary":"PostgreSQL replication lag critical (server {{ $labels.server }})","description":"Replication lag on {{ $labels.server }} is {{ $value | humanizeDuration }}. Replica is severely behind.","runbook_url":"","metric_refs":["pg_replication_lag","pg_replication_lag_seconds"],"line":242},{"kind":"alert","alert":"PostgresWALArchiveFailing","expr":"increase(pg_stat_archiver_failed_count[5m]) > 0","for":"0m","severity":"critical","summary":"PostgreSQL WAL archive failing (server {{ $labels.server }})","description":"WAL archiving failures on {{ $labels.server }}. PITR recovery may be broken.","runbook_url":"","metric_refs":["pg_stat_archiver_failed_count"],"line":254},{"kind":"alert","alert":"PostgresWALArchiveStale","expr":"pg_stat_archiver_last_archive_age > 3600","for":"5m","severity":"warning","summary":"PostgreSQL WAL archive stale (server {{ $labels.server }})","description":"Last successful WAL archive on {{ $labels.server }} was {{ $value | humanizeDuration }} ago. Check archive destination.","runbook_url":"","metric_refs":["pg_stat_archiver_last_archive_age"],"line":263},{"kind":"alert","alert":"PostgresConnectionsSaturation","expr":"sum by (server, instance, job) (pg_stat_database_numbackends)\n/ (pg_settings_max_connections > 0)\n> 0.8\n","for":"5m","severity":"warning","summary":"PostgreSQL connections saturation (server {{ $labels.server }})","description":"PostgreSQL connection usage above 80% of max_connections on {{ $labels.server }}.","runbook_url":"","metric_refs":["pg_settings_max_connections","pg_stat_database_numbackends"],"line":272},{"kind":"alert","alert":"PostgresChecksumFailures","expr":"increase(pg_stat_database_checksum_failures[5m]) > 0","for":"0m","severity":"critical","summary":"PostgreSQL checksum failures (database {{ $labels.datname }})","description":"Data checksum failure detected on {{ $labels.datname }} - possible disk corruption.","runbook_url":"","metric_refs":["pg_stat_database_checksum_failures"],"line":290},{"kind":"alert","alert":"PostgresTempFilesHigh","expr":"(\n  rate(pg_stat_database_temp_bytes[5m]) * 3600\n  / on(datname) group_left() pg_database_size_bytes\n) > 0.50\n","for":"10m","severity":"warning","summary":"PostgreSQL temp files high (database {{ $labels.datname }})","description":"Database {{ $labels.datname }} projected to write more than 50% of its size in temp files per hour ({{ $value | humanizePercentage }}) - query memory pressure, check work_mem.","runbook_url":"","metric_refs":["pg_database_size_bytes","pg_stat_database_temp_bytes"],"line":299},{"kind":"alert","alert":"PostgresBackendFsyncHigh","expr":"rate(pg_stat_bgwriter_buffers_backend_fsync_total[5m]) > 0","for":"5m","severity":"warning","summary":"PostgreSQL backend fsync high (instance {{ $labels.instance }})","description":"Backend processes are performing fsync themselves on {{ $labels.instance }} - bgwriter cannot keep up.","runbook_url":"","metric_refs":["pg_stat_bgwriter_buffers_backend_fsync_total"],"line":312},{"kind":"alert","alert":"PostgresDatabaseSizeGrowthHigh","expr":"(\n  predict_linear(pg_database_size_bytes[6h], 86400) - pg_database_size_bytes\n) / pg_database_size_bytes > 0.20\nand pg_database_size_bytes > 0\n","for":"0m","severity":"warning","summary":"PostgreSQL database size growth high (database {{ $labels.datname }})","description":"Database {{ $labels.datname }} projected to grow by more than 20% of its current size in 24h ({{ $value | humanizePercentage }})","runbook_url":"","metric_refs":["pg_database_size_bytes"],"line":321},{"kind":"alert","alert":"PostgresqlStandbyQueryConflicts","expr":"increase(pg_stat_database_conflicts[1h]) > 0","for":"5m","severity":"warning","summary":"Postgres standby cancelled queries on {{ $labels.datname }} @ {{ $labels.instance }}","description":"{{ $value | printf \"%.0f\" }} queries were cancelled on {{ $labels.datname }} @ {{ $labels.instance }} in the last hour by conflicts with WAL recovery. Clients saw errors; the standby itself looks healthy. Consider hot_standby_feedback or max_standby_streaming_delay.","runbook_url":"","metric_refs":["pg_stat_database_conflicts"],"line":335}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/postgresql.rules.yml"},{"file":"prometheus.records.yml","name":"prometheus.records","category":"Observability","rules":[{"kind":"record","record":"name:tsdb_series:count","expr":"label_replace(\n  count by (__name__) ({__name__=~\".+\", __name__!~\"name:tsdb_series:count|tsdb_series:total\"}),\n  \"name\", \"$1\", \"__name__\", \"(.+)\"\n)\n","metric_refs":[],"line":15},{"kind":"record","record":"tsdb_series:total","expr":"sum(name:tsdb_series:count)","metric_refs":["name:tsdb_series:count"],"line":27}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/prometheus.records.yml"},{"file":"prometheus.rules.yml","name":"prometheus.rules","category":"Observability","rules":[{"kind":"alert","alert":"PushGatewayDown","expr":"# Two distinct failure modes: a failing scrape (up == 0, exporter crashed\n# or host unreachable) and the target vanishing from service discovery,\n# where no up series exists to be 0. Both branches are grouped\n# identically so the alert carries the same labels either way.\ngroup by (job, instance) (up{job=\"prometheus-pushgateway\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=\"prometheus-pushgateway\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=\"prometheus-pushgateway\"}[2m]))\n)\n","for":"2m","severity":"critical","summary":"Job {{ $labels.job }} having an issue on {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"PrometheusConfigReloadFailed","expr":"prometheus_config_last_reload_successful == 0","for":"10m","severity":"warning","summary":"reload of Prometheus configuration has failed for {{ $labels.instance }}","description":"The last configuration reload failed: Prometheus keeps running the previous config, so recent scrape and rule changes are not applied. Its logs show the parse error.","runbook_url":"","metric_refs":["prometheus_config_last_reload_successful"],"line":22},{"kind":"alert","alert":"PrometheusErrorSendingAlerts","expr":"rate(prometheus_notifications_errors_total[5m]) / rate(prometheus_notifications_sent_total[5m]) > 0.03","for":"10m","severity":"warning","summary":"Errors while sending alerts from Prometheus {{ $labels.instance }} to Alertmanager {{ $labels.alertmanager }}","description":"More than 3% of notifications to {{ $labels.alertmanager }} fail: alerts may not reach Alertmanager. Check that it is reachable from {{ $labels.instance }}.","runbook_url":"","metric_refs":["prometheus_notifications_errors_total","prometheus_notifications_sent_total"],"line":30},{"kind":"alert","alert":"PrometheusJobMissing","expr":"absent(up{job=\"prometheus\"})","for":"5m","severity":"warning","summary":"Prometheus job missing (job {{ $labels.job }})","description":"A Prometheus job has disappeared","runbook_url":"","metric_refs":[],"line":38},{"kind":"alert","alert":"PrometheusTargetMissing","expr":"up{job=~\"prometheus|alertmanager|.*pushgateway\"} == 0 unless on(job) (sum by (job) (up) == 0)","for":"1m","severity":"critical","summary":"Prometheus target missing (instance {{ $labels.instance }})","description":"A Prometheus target has disappeared. An exporter might be crashed.","runbook_url":"","metric_refs":[],"line":46},{"kind":"alert","alert":"PrometheusAllTargetsMissing","expr":"sum by (job) (up) == 0","for":"1m","severity":"critical","summary":"Prometheus all targets missing (job {{ $labels.job }})","description":"A Prometheus job does not have living target anymore.","runbook_url":"","metric_refs":[],"line":59},{"kind":"alert","alert":"PrometheusAlertManagerJobMissing","expr":"absent(up{job=\"alertmanager\"})","for":"5m","severity":"warning","summary":"Prometheus AlertManager job missing (job {{ $labels.job }})","description":"A Prometheus AlertManager job has disappeared","runbook_url":"","metric_refs":[],"line":67},{"kind":"alert","alert":"PrometheusTooManyRestarts","expr":"changes(process_start_time_seconds{job=~\"prometheus|.*pushgateway|alertmanager\"}[15m]) > 2","for":"0m","severity":"warning","summary":"Prometheus too many restarts (instance {{ $labels.instance }})","description":"Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.","runbook_url":"","metric_refs":["process_start_time_seconds"],"line":75},{"kind":"alert","alert":"PrometheusAlertmanagerConfigurationReloadFailure","expr":"alertmanager_config_last_reload_successful != 1","for":"5m","severity":"warning","summary":"Prometheus AlertManager configuration reload failure (instance {{ $labels.instance }})","description":"AlertManager configuration reload error","runbook_url":"","metric_refs":["alertmanager_config_last_reload_successful"],"line":83},{"kind":"alert","alert":"PrometheusAlertmanagerConfigNotSynced","expr":"count by (job) (count_values by (job) (\"config_hash\", alertmanager_config_hash)) > 1","for":"15m","severity":"warning","summary":"Prometheus AlertManager config not synced (instance {{ $labels.instance }})","description":"Configurations of AlertManager cluster instances are out of sync","runbook_url":"","metric_refs":["alertmanager_config_hash","config_hash"],"line":91},{"kind":"alert","alert":"PrometheusNotConnectedToAlertmanager","expr":"prometheus_notifications_alertmanagers_discovered < 1","for":"5m","severity":"critical","summary":"Prometheus not connected to alertmanager (instance {{ $labels.instance }})","description":"Prometheus cannot connect the alertmanager","runbook_url":"","metric_refs":["prometheus_notifications_alertmanagers_discovered"],"line":102},{"kind":"alert","alert":"PrometheusRuleEvaluationFailures","expr":"increase(prometheus_rule_evaluation_failures_total[3m]) > 0","for":"0m","severity":"critical","summary":"Prometheus rule evaluation failures (instance {{ $labels.instance }})","description":"Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.","runbook_url":"","metric_refs":["prometheus_rule_evaluation_failures_total"],"line":111},{"kind":"alert","alert":"PrometheusTemplateTextExpansionFailures","expr":"increase(prometheus_template_text_expansion_failures_total[3m]) > 0","for":"0m","severity":"critical","summary":"Prometheus template text expansion failures (instance {{ $labels.instance }})","description":"Prometheus encountered {{ $value }} template text expansion failures","runbook_url":"","metric_refs":["prometheus_template_text_expansion_failures_total"],"line":119},{"kind":"alert","alert":"PrometheusRuleEvaluationSlow","expr":"prometheus_rule_group_last_duration_seconds > prometheus_rule_group_interval_seconds","for":"5m","severity":"warning","summary":"Prometheus rule evaluation slow (instance {{ $labels.instance }})","description":"Prometheus rule evaluation took more time than the scheduled interval. It indicates a slower storage backend access or too complex query.","runbook_url":"","metric_refs":["prometheus_rule_group_interval_seconds","prometheus_rule_group_last_duration_seconds"],"line":127},{"kind":"alert","alert":"PrometheusRuleGroupIterationsMissed","expr":"sum by (job, instance, rule_group) (rate(prometheus_rule_group_iterations_missed_total[5m])) > 0","for":"15m","severity":"warning","summary":"Prometheus rule group missing iterations (instance {{ $labels.instance }})","description":"Rule group {{ $labels.rule_group }} takes longer than its interval to evaluate and skips iterations, so its recording rules and alerts have gaps.","runbook_url":"","metric_refs":["prometheus_rule_group_iterations_missed_total","rule_group"],"line":135},{"kind":"alert","alert":"PrometheusNotificationsBacklog","expr":"min_over_time(prometheus_notifications_queue_length[10m]) > 0","for":"0m","severity":"warning","summary":"Prometheus notifications backlog (instance {{ $labels.instance }})","description":"The Prometheus notification queue has not been empty for 10 minutes","runbook_url":"","metric_refs":["prometheus_notifications_queue_length"],"line":145},{"kind":"alert","alert":"PrometheusAlertmanagerNotificationFailing","expr":"rate(alertmanager_notifications_failed_total[3m]) > 0.05","for":"5m","severity":"critical","summary":"Prometheus AlertManager notification failing (instance {{ $labels.instance }})","description":"Alertmanager is failing sending notifications ({{ $value | humanize }} failures/s)","runbook_url":"","metric_refs":["alertmanager_notifications_failed_total"],"line":153},{"kind":"alert","alert":"PrometheusTargetEmpty","expr":"prometheus_sd_discovered_targets == 0","for":"30m","severity":"warning","summary":"Prometheus target empty (instance {{ $labels.instance }}, config {{ $labels.config }})","description":"SD config {{ $labels.config }} on {{ $labels.instance }} has discovered no target for 30 minutes","runbook_url":"","metric_refs":["prometheus_sd_discovered_targets"],"line":161},{"kind":"alert","alert":"PrometheusTargetScrapingSlow","expr":"prometheus_target_interval_length_seconds{quantile=\"0.5\"} / on (interval, instance, job) prometheus_target_interval_length_seconds{quantile=\"0.01\"} > 1.05","for":"5m","severity":"warning","summary":"Prometheus target scraping slow (instance {{ $labels.instance }})","description":"Prometheus is scraping exporters slowly since it exceeded the requested interval time. Your Prometheus server is under-provisioned.","runbook_url":"","metric_refs":["prometheus_target_interval_length_seconds"],"line":171},{"kind":"alert","alert":"PrometheusLargeScrape","expr":"increase(prometheus_target_scrapes_exceeded_sample_limit_total[10m]) > 10","for":"5m","severity":"warning","summary":"Prometheus large scrape (instance {{ $labels.instance }})","description":"Prometheus has many scrapes that exceed the sample limit","runbook_url":"","metric_refs":["prometheus_target_scrapes_exceeded_sample_limit_total"],"line":182},{"kind":"alert","alert":"PrometheusTargetScrapeDuplicate","expr":"increase(prometheus_target_scrapes_sample_duplicate_timestamp_total[5m]) > 3","for":"0m","severity":"warning","summary":"Prometheus target scrape duplicate (instance {{ $labels.instance }})","description":"Prometheus has many samples rejected due to duplicate timestamps but different values ({{ $value }} samples)","runbook_url":"","metric_refs":["prometheus_target_scrapes_sample_duplicate_timestamp_total"],"line":190},{"kind":"alert","alert":"PrometheusTsdbCheckpointCreationFailures","expr":"increase(prometheus_tsdb_checkpoint_creations_failed_total[5m]) > 0","for":"0m","severity":"critical","summary":"Prometheus TSDB checkpoint creation failures (instance {{ $labels.instance }})","description":"Prometheus encountered {{ $value }} checkpoint creation failures","runbook_url":"","metric_refs":["prometheus_tsdb_checkpoint_creations_failed_total"],"line":198},{"kind":"alert","alert":"PrometheusTsdbCheckpointDeletionFailures","expr":"increase(prometheus_tsdb_checkpoint_deletions_failed_total[5m]) > 0","for":"0m","severity":"critical","summary":"Prometheus TSDB checkpoint deletion failures (instance {{ $labels.instance }})","description":"Prometheus encountered {{ $value }} checkpoint deletion failures","runbook_url":"","metric_refs":["prometheus_tsdb_checkpoint_deletions_failed_total"],"line":206},{"kind":"alert","alert":"PrometheusTsdbCompactionsFailed","expr":"increase(prometheus_tsdb_compactions_failed_total[5m]) > 0","for":"0m","severity":"critical","summary":"Prometheus TSDB compactions failed (instance {{ $labels.instance }})","description":"Prometheus encountered {{ $value }} TSDB compactions failures","runbook_url":"","metric_refs":["prometheus_tsdb_compactions_failed_total"],"line":214},{"kind":"alert","alert":"PrometheusTsdbHeadTruncationsFailed","expr":"increase(prometheus_tsdb_head_truncations_failed_total[5m]) > 0","for":"0m","severity":"critical","summary":"Prometheus TSDB head truncations failed (instance {{ $labels.instance }})","description":"Prometheus encountered {{ $value }} TSDB head truncation failures","runbook_url":"","metric_refs":["prometheus_tsdb_head_truncations_failed_total"],"line":222},{"kind":"alert","alert":"PrometheusTsdbReloadFailures","expr":"increase(prometheus_tsdb_reloads_failures_total[5m]) > 0","for":"0m","severity":"critical","summary":"Prometheus TSDB reload failures (instance {{ $labels.instance }})","description":"Prometheus encountered {{ $value }} TSDB reload failures","runbook_url":"","metric_refs":["prometheus_tsdb_reloads_failures_total"],"line":230},{"kind":"alert","alert":"PrometheusTsdbWalCorruptions","expr":"increase(prometheus_tsdb_wal_corruptions_total[5m]) > 0","for":"0m","severity":"critical","summary":"Prometheus TSDB WAL corruptions (instance {{ $labels.instance }})","description":"Prometheus encountered {{ $value }} TSDB WAL corruptions","runbook_url":"","metric_refs":["prometheus_tsdb_wal_corruptions_total"],"line":238},{"kind":"alert","alert":"PrometheusTsdbMmapChunkCorruptions","expr":"increase(prometheus_tsdb_mmap_chunk_corruptions_total[5m]) > 0","for":"0m","severity":"critical","summary":"Prometheus TSDB mmap chunk corruptions (instance {{ $labels.instance }})","description":"Prometheus encountered {{ $value }} corrupted mmapped head chunks, which is unrecoverable local data loss","runbook_url":"","metric_refs":["prometheus_tsdb_mmap_chunk_corruptions_total"],"line":246},{"kind":"alert","alert":"PrometheusTsdbWalWritesFailed","expr":"increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0","for":"0m","severity":"critical","summary":"Prometheus TSDB WAL writes failed (instance {{ $labels.instance }})","description":"Prometheus failed {{ $value }} WAL writes in the last 5 minutes. Check the free space and health of its data volume.","runbook_url":"","metric_refs":["prometheus_tsdb_wal_writes_failed_total"],"line":255},{"kind":"alert","alert":"PrometheusTsdbWalTruncationsFailed","expr":"increase(prometheus_tsdb_wal_truncations_failed_total[5m]) > 0","for":"0m","severity":"critical","summary":"Prometheus TSDB WAL truncations failed (instance {{ $labels.instance }})","description":"Prometheus encountered {{ $value }} TSDB WAL truncation failures","runbook_url":"","metric_refs":["prometheus_tsdb_wal_truncations_failed_total"],"line":264},{"kind":"alert","alert":"PrometheusTimeseriesCardinality","expr":"name:tsdb_series:count > 10000\nand\nname:tsdb_series:count / scalar(tsdb_series:total) > 0.10\n","for":"15m","severity":"warning","summary":"Prometheus timeseries cardinality - {{ $labels.name }}","description":"The \"{{ $labels.name }}\" metric accounts for {{ $value }} series, over 10% of the whole TSDB","runbook_url":"","metric_refs":["name:tsdb_series:count","tsdb_series:total"],"line":272},{"kind":"alert","alert":"PrometheusRemoteWriteDroppedSamples","expr":"rate(prometheus_remote_storage_samples_dropped_total{reason!=\"dropped_series\"}[5m]) > 0","for":"5m","severity":"critical","summary":"Prometheus remote write dropping samples (instance {{ $labels.instance }})","description":"Prometheus is dropping samples for remote_name={{ $labels.remote_name }}, reason={{ $labels.reason }} ({{ $value | humanize }} samples/s lost)","runbook_url":"","metric_refs":["prometheus_remote_storage_samples_dropped_total"],"line":288},{"kind":"alert","alert":"PrometheusRemoteWriteFailed","expr":"rate(prometheus_remote_storage_samples_failed_total[5m]) > 0","for":"5m","severity":"critical","summary":"Prometheus remote write failing (instance {{ $labels.instance }})","description":"Prometheus failed to send samples to remote_name={{ $labels.remote_name }} ({{ $value | humanize }} failures/s)","runbook_url":"","metric_refs":["prometheus_remote_storage_samples_failed_total"],"line":304},{"kind":"alert","alert":"PrometheusRemoteWriteShardsMax","expr":"prometheus_remote_storage_shards >= prometheus_remote_storage_shards_max","for":"5m","severity":"warning","summary":"Prometheus remote write shards at max (instance {{ $labels.instance }})","description":"Prometheus remote write queue for remote_name={{ $labels.remote_name }} is running at max shards ({{ $value }}). The queue is saturated and samples may be dropped.","runbook_url":"","metric_refs":["prometheus_remote_storage_shards","prometheus_remote_storage_shards_max"],"line":312},{"kind":"alert","alert":"PrometheusRemoteWriteBehind","expr":"(\n  max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds[5m])\n  - ignoring (remote_name, url) group_right\n  max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds[5m])\n) > 300\n","for":"15m","severity":"critical","summary":"Prometheus remote write behind (instance {{ $labels.instance }})","description":"Prometheus remote write to remote_name={{ $labels.remote_name }} is {{ $value | humanizeDuration }} behind the newest ingested sample. The remote storage is missing recent data.","runbook_url":"","metric_refs":["prometheus_remote_storage_highest_timestamp_in_seconds","prometheus_remote_storage_queue_highest_sent_timestamp_seconds","remote_name"],"line":320},{"kind":"alert","alert":"PrometheusQueryEngineBusy","expr":"prometheus_engine_queries >= prometheus_engine_queries_concurrent_max","for":"1m","severity":"warning","summary":"Prometheus query engine at concurrency limit (instance {{ $labels.instance }})","description":"Prometheus is running {{ $value }} concurrent queries, which equals the maximum allowed. New queries may be rejected.","runbook_url":"","metric_refs":["prometheus_engine_queries","prometheus_engine_queries_concurrent_max"],"line":335},{"kind":"alert","alert":"PrometheusTsdbOutOfOrderSamples","expr":"rate(prometheus_tsdb_out_of_order_samples_total[5m]) > 0","for":"5m","severity":"warning","summary":"Prometheus TSDB out-of-order samples (instance {{ $labels.instance }})","description":"Prometheus is receiving out-of-order samples ({{ $value | humanize }}/s). This usually indicates clock skew or a buggy exporter.","runbook_url":"","metric_refs":["prometheus_tsdb_out_of_order_samples_total"],"line":343},{"kind":"alert","alert":"PrometheusTsdbOutOfBoundSamples","expr":"rate(prometheus_tsdb_out_of_bound_samples_total[5m]) > 0","for":"5m","severity":"warning","summary":"Prometheus TSDB out-of-bound samples (instance {{ $labels.instance }})","description":"Prometheus is rejecting samples with timestamps too old or too far in the future ({{ $value | humanize }}/s). Check target clocks.","runbook_url":"","metric_refs":["prometheus_tsdb_out_of_bound_samples_total"],"line":351},{"kind":"alert","alert":"PrometheusTargetLimitExceeded","expr":"increase(prometheus_target_scrape_pool_exceeded_target_limit_total[5m]) > 0","for":"0m","severity":"warning","summary":"Prometheus scrape pool target limit exceeded (instance {{ $labels.instance }})","description":"Prometheus is silently dropping targets in job {{ $labels.instance }} because the target limit was exceeded.","runbook_url":"","metric_refs":["prometheus_target_scrape_pool_exceeded_target_limit_total"],"line":359},{"kind":"alert","alert":"PrometheusScrapedBodySizeExceeded","expr":"rate(prometheus_target_scrapes_exceeded_body_size_limit_total[5m]) > 0","for":"5m","severity":"warning","summary":"Prometheus scrape body size limit exceeded (instance {{ $labels.instance }})","description":"Prometheus is truncating scrape responses that exceed the body size limit ({{ $value | humanize }} scrapes/s). The exporter may be too verbose.","runbook_url":"","metric_refs":["prometheus_target_scrapes_exceeded_body_size_limit_total"],"line":369},{"kind":"alert","alert":"PrometheusSDFailedConfigs","expr":"prometheus_sd_failed_configs > 0","for":"1m","severity":"warning","summary":"Prometheus service discovery has failed configs (instance {{ $labels.instance }})","description":"{{ $value }} service discovery config(s) are failing (name={{ $labels.name }}). Targets from these configs will not be scraped.","runbook_url":"","metric_refs":["prometheus_sd_failed_configs"],"line":377},{"kind":"alert","alert":"PrometheusNotificationsDropped","expr":"sum without (alertmanager) (rate(prometheus_notifications_dropped_total[5m])) * 60 > 0","for":"5m","severity":"critical","summary":"Prometheus is dropping alert notifications (instance {{ $labels.instance }})","description":"{{ $value | humanize }} alerts/min are being dropped on the way to Alertmanager by {{ $labels.instance }}. Alerts are firing and nobody is being told.","runbook_url":"","metric_refs":["prometheus_notifications_dropped_total"],"line":387},{"kind":"alert","alert":"Watchdog","expr":"vector(1)","for":"0m","severity":"warning","summary":"Watchdog heartbeat, this alert always fires","description":"Always-firing heartbeat that proves the Prometheus to Alertmanager pipeline works. Route it to a dead man's switch; if it stops arriving, alerting is down.","runbook_url":"","metric_refs":[],"line":407},{"kind":"alert","alert":"PrometheusOperatorReconcileErrors","expr":"sum by (controller) (increase(prometheus_operator_reconcile_errors_total[1h])) > 0","for":"30m","severity":"warning","summary":"Prometheus Operator {{ $labels.controller }} controller cannot reconcile","description":"{{ $value | humanize }} reconcile errors in the last hour on the {{ $labels.controller }} controller: new monitors, rules and remote writes are not applied. The reason is in the status conditions of the resource.","runbook_url":"","metric_refs":["prometheus_operator_reconcile_errors_total"],"line":417},{"kind":"alert","alert":"PrometheusOperatorNotReady","expr":"min by (controller) (max_over_time(prometheus_operator_ready[5m])) == 0","for":"10m","severity":"warning","summary":"Prometheus Operator {{ $labels.controller }} controller not ready","description":"The {{ $labels.controller }} controller of the Prometheus Operator is not ready: changes to its resources are not applied.","runbook_url":"","metric_refs":["prometheus_operator_ready"],"line":427},{"kind":"alert","alert":"PrometheusOperatorRejectedResources","expr":"max by (controller, resource) (prometheus_operator_managed_resources{state=\"rejected\"}) > 0","for":"10m","severity":"warning","summary":"Prometheus Operator rejects {{ $labels.resource }} objects","description":"{{ $value }} {{ $labels.resource }} objects are rejected by the {{ $labels.controller }} controller and not loaded. The operator logs give the reason.","runbook_url":"","metric_refs":["prometheus_operator_managed_resources"],"line":435}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/prometheus.rules.yml"},{"file":"promtail.rules.yml","name":"promtail.rules","category":"Observability","rules":[{"kind":"alert","alert":"PromtailRequestErrors","expr":"100 * sum(rate(promtail_request_duration_seconds_count{status_code=~\"5..|-1\"}[5m])) by (namespace, job, host, instance)\n/ sum(rate(promtail_request_duration_seconds_count[5m])) by (namespace, job, host, instance)\n> 10\nand sum(rate(promtail_request_duration_seconds_count[5m])) by (namespace, job, host, instance) > 0\n","for":"5m","severity":"critical","summary":"Promtail request errors (instance {{ $labels.instance }})","description":"{{ $labels.job }} -> {{ $labels.host }} is experiencing {{ printf \"%.2f\" $value }}% errors","runbook_url":"","metric_refs":["promtail_request_duration_seconds_count"],"line":3},{"kind":"alert","alert":"PromtailRequestLatency","expr":"histogram_quantile(0.99, sum(rate(promtail_request_duration_seconds_bucket[5m])) by (namespace, job, host, le)) > 1\n","for":"5m","severity":"critical","summary":"Promtail request latency (instance {{ $labels.instance }})","description":"{{ $labels.job }} -> {{ $labels.host }} p99 latency is {{ printf \"%.2f\" $value }}s","runbook_url":"","metric_refs":["promtail_request_duration_seconds_bucket"],"line":24},{"kind":"alert","alert":"PromtailDroppedEntries","expr":"rate(promtail_dropped_entries_total[5m]) > 0","for":"2m","severity":"critical","summary":"Promtail dropped entries (instance {{ $labels.instance }})","description":"Promtail is dropping log entries that cannot be sent to Loki on {{ $labels.instance }} (reason={{ $labels.reason }}).","runbook_url":"","metric_refs":["promtail_dropped_entries_total"],"line":34},{"kind":"alert","alert":"PromtailBatchRetries","expr":"rate(promtail_batch_retries_total[5m]) > 0.1","for":"5m","severity":"warning","summary":"Promtail batch retries (instance {{ $labels.instance }})","description":"Promtail is retrying batches frequently on {{ $labels.instance }}. Loki may be slow or unreachable.","runbook_url":"","metric_refs":["promtail_batch_retries_total"],"line":43},{"kind":"alert","alert":"PromtailConfigReloadFailed","expr":"increase(promtail_config_reload_fail_total[5m]) > 0","for":"0m","severity":"critical","summary":"Promtail config reload failed (instance {{ $labels.instance }})","description":"Promtail config reload failed on {{ $labels.instance }}.","runbook_url":"","metric_refs":["promtail_config_reload_fail_total"],"line":52},{"kind":"alert","alert":"PromtailNoActiveScrapeTargets","expr":"promtail_targets_active_total == 0","for":"5m","severity":"warning","summary":"Promtail no active scrape targets (instance {{ $labels.instance }})","description":"Promtail has no active scrape targets on {{ $labels.instance }}. Log collection may have stopped.","runbook_url":"","metric_refs":["promtail_targets_active_total"],"line":61}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/promtail.rules.yml"},{"file":"rabbitmq.rules.yml","name":"rabbitmq.rules","category":"Messaging & streaming","rules":[{"kind":"alert","alert":"RabbitmqDown","expr":"group by (job, instance) (up{job=~\".*rabbitmq.*\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=~\".*rabbitmq.*\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=~\".*rabbitmq.*\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"RabbitMQ down - {{ $labels.job }} @ {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":6},{"kind":"alert","alert":"RabbitmqMemoryAlarm","expr":"rabbitmq_alarms_memory_used_watermark == 1","for":"0m","severity":"critical","summary":"RabbitMQ memory alarm on {{ $labels.instance }} - publishers blocked","description":"RabbitMQ on {{ $labels.instance }} has crossed its memory watermark and is blocking all publishing connections.","runbook_url":"","metric_refs":["rabbitmq_alarms_memory_used_watermark"],"line":27},{"kind":"alert","alert":"RabbitmqDiskAlarm","expr":"rabbitmq_alarms_free_disk_space_watermark == 1","for":"0m","severity":"critical","summary":"RabbitMQ disk alarm on {{ $labels.instance }} - publishers blocked","description":"Free disk space on {{ $labels.instance }} has crossed the watermark. RabbitMQ is blocking all publishing connections.","runbook_url":"","metric_refs":["rabbitmq_alarms_free_disk_space_watermark"],"line":36},{"kind":"alert","alert":"RabbitmqFileDescriptorsSaturation","expr":"rabbitmq_process_open_fds / (rabbitmq_process_max_fds > 0) > 0.80\n","for":"10m","severity":"critical","summary":"RabbitMQ file descriptors near limit on {{ $labels.instance }} ({{ $value | humanizePercentage }})","description":"More than 80% of the file descriptor limit is in use on {{ $labels.instance }}. Connections will be dropped at the limit.","runbook_url":"","metric_refs":["rabbitmq_process_max_fds","rabbitmq_process_open_fds"],"line":47},{"kind":"alert","alert":"RabbitmqErlangProcessesSaturation","expr":"rabbitmq_erlang_processes_used / (rabbitmq_erlang_processes_limit > 0) > 0.80\n","for":"10m","severity":"warning","summary":"RabbitMQ Erlang processes near limit on {{ $labels.instance }} ({{ $value | humanizePercentage }})","description":"More than 80% of the Erlang process limit is in use on {{ $labels.instance }}.","runbook_url":"","metric_refs":["rabbitmq_erlang_processes_limit","rabbitmq_erlang_processes_used"],"line":59},{"kind":"alert","alert":"RabbitmqMemoryNearLimit","expr":"rabbitmq_process_resident_memory_bytes / (rabbitmq_resident_memory_limit_bytes > 0) > 0.80\n","for":"10m","severity":"warning","summary":"RabbitMQ memory at {{ $value | humanizePercentage }} of its limit on {{ $labels.instance }}","description":"RabbitMQ on {{ $labels.instance }} is approaching the memory watermark. Publishers are blocked once it is crossed.","runbook_url":"","metric_refs":["rabbitmq_process_resident_memory_bytes","rabbitmq_resident_memory_limit_bytes"],"line":69},{"kind":"alert","alert":"RabbitmqQueueBacklog","expr":"rabbitmq_queue_messages_ready > 10000","for":"15m","severity":"warning","summary":"RabbitMQ backlog of {{ $value | humanize }} ready messages on {{ $labels.instance }}","description":"Messages are accumulating faster than consumers drain them on {{ $labels.instance }}.","runbook_url":"","metric_refs":["rabbitmq_queue_messages_ready"],"line":83},{"kind":"alert","alert":"RabbitmqUnackedMessagesStuck","expr":"rabbitmq_queue_messages_unacked > 1000","for":"30m","severity":"warning","summary":"RabbitMQ has {{ $value | humanize }} unacknowledged messages on {{ $labels.instance }}","description":"Messages have been delivered but not acknowledged for 30 minutes on {{ $labels.instance }}. A consumer is likely stuck.","runbook_url":"","metric_refs":["rabbitmq_queue_messages_unacked"],"line":94}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/rabbitmq.rules.yml"},{"file":"stackdriver.rules.yml","name":"stackdriver.rules","category":"Observability","rules":[{"kind":"alert","alert":"StackdriverExporterScrapeError","expr":"stackdriver_monitoring_last_scrape_error > 0","for":"5m","severity":"critical","summary":"Stackdriver exporter scrape error (project {{ $labels.project_id }})","description":"Stackdriver exporter failed to scrape GCP Monitoring API for project {{ $labels.project_id }}","runbook_url":"","metric_refs":["stackdriver_monitoring_last_scrape_error"],"line":3},{"kind":"alert","alert":"StackdriverExporterSlowScrape","expr":"stackdriver_monitoring_last_scrape_duration_seconds > 300","for":"5m","severity":"warning","summary":"Stackdriver exporter slow scrape (project {{ $labels.project_id }})","description":"Stackdriver scrape for project {{ $labels.project_id }} is taking more than 5 minutes ({{ $value }}s)","runbook_url":"","metric_refs":["stackdriver_monitoring_last_scrape_duration_seconds"],"line":12},{"kind":"alert","alert":"StackdriverExporterScrapeErrorsIncreasing","expr":"increase(stackdriver_monitoring_scrape_errors_total[15m]) > 5","for":"0m","severity":"warning","summary":"Stackdriver exporter scrape errors increasing (project {{ $labels.project_id }})","description":"{{ $value }} scrape errors in the last 15 minutes for project {{ $labels.project_id }}","runbook_url":"","metric_refs":["stackdriver_monitoring_scrape_errors_total"],"line":21},{"kind":"alert","alert":"StackdriverExporterScrapeStale","expr":"time() - stackdriver_monitoring_last_scrape_timestamp > 600","for":"0m","severity":"warning","summary":"Stackdriver exporter scrape stale (project {{ $labels.project_id }})","description":"No successful scrape for project {{ $labels.project_id }} in the last 10 minutes","runbook_url":"","metric_refs":["stackdriver_monitoring_last_scrape_timestamp"],"line":30},{"kind":"alert","alert":"StackdriverExporterHighApiCalls","expr":"rate(stackdriver_monitoring_api_calls_total[5m]) * 60 > 100","for":"0m","severity":"warning","summary":"Stackdriver exporter high API calls (project {{ $labels.project_id }})","description":"{{ $value }} API calls/min for project {{ $labels.project_id }} - may hit GCP quota","runbook_url":"","metric_refs":["stackdriver_monitoring_api_calls_total"],"line":39}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/stackdriver.rules.yml"},{"file":"strimzi.rules.yml","name":"strimzi.rules","category":"Other","rules":[{"kind":"alert","alert":"StrimziCaCertificateNotRenewed","expr":"max by (exported_namespace, namespace, resource_namespace, cluster, type) (strimzi_certificate_expiration_timestamp_ms) / 1000 - time() < 14 * 86400","for":"1h","severity":"warning","summary":"Strimzi {{ $labels.type }} of Kafka {{ $labels.cluster }} expires in {{ $value | humanizeDuration }}","description":"The CA entered its renewal window and was not renewed. The cluster operator is most likely stuck on a failed reconciliation: check its logs. Once it expires, brokers and clients lose TLS.","runbook_url":"","metric_refs":["exported_namespace","resource_namespace","strimzi_certificate_expiration_timestamp_ms"],"line":5},{"kind":"alert","alert":"StrimziCaCertificateExpiringSoon","expr":"max by (exported_namespace, namespace, resource_namespace, cluster, type) (strimzi_certificate_expiration_timestamp_ms) / 1000 - time() < 7 * 86400","for":"15m","severity":"warning","summary":"Strimzi {{ $labels.type }} of Kafka {{ $labels.cluster }} expires in {{ $value | humanizeDuration }}","description":"Less than 7 days before the CA expires. Unblock the cluster operator reconciliation, or renew the CA manually with the strimzi.io/force-renew annotation on its Secret.","runbook_url":"","metric_refs":["exported_namespace","resource_namespace","strimzi_certificate_expiration_timestamp_ms"],"line":14},{"kind":"alert","alert":"StrimziCaCertificateExpiringImminently","expr":"max by (exported_namespace, namespace, resource_namespace, cluster, type) (strimzi_certificate_expiration_timestamp_ms) / 1000 - time() < 1 * 86400","for":"5m","severity":"critical","summary":"Strimzi {{ $labels.type }} of Kafka {{ $labels.cluster }} expires in {{ $value | humanizeDuration }}","description":"Less than 1 day before the CA expires. Unblock the cluster operator reconciliation, or renew the CA manually with the strimzi.io/force-renew annotation on its Secret.","runbook_url":"","metric_refs":["exported_namespace","resource_namespace","strimzi_certificate_expiration_timestamp_ms"],"line":23},{"kind":"alert","alert":"StrimziKafkaReconciliationStuck","expr":"sum by (exported_namespace, namespace) (increase(strimzi_reconciliations_successful_total{kind=\"Kafka\"}[1h])) == 0\nand on (exported_namespace, namespace)\nsum by (exported_namespace, namespace) (strimzi_resources{kind=\"Kafka\"}) > 0\n","for":"1h","severity":"critical","summary":"No successful Strimzi reconciliation of the Kafka in {{ if $labels.exported_namespace }}{{ $labels.exported_namespace }}{{ else }}{{ $labels.namespace }}{{ end }} for 2 hours","description":"The cluster operator reconciles every 2 minutes and none succeeded for 2 hours. CA renewal and certificate regeneration are steps of that reconciliation: left stuck, the certificates expire.","runbook_url":"","metric_refs":["exported_namespace","strimzi_reconciliations_successful_total","strimzi_resources"],"line":34},{"kind":"alert","alert":"StrimziOperatorDown","expr":"max by (namespace) (kube_deployment_status_replicas_available{deployment=\"strimzi-cluster-operator\"}) < 1","for":"10m","severity":"critical","summary":"Strimzi cluster operator has no available replica in {{ $labels.namespace }}","description":"Kafka clusters keep serving but are no longer reconciled: no rolling updates, no certificate renewal.","runbook_url":"","metric_refs":["kube_deployment_status_replicas_available"],"line":46}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/strimzi.rules.yml"},{"file":"thanos.rules.yml","name":"thanos.rules","category":"Other","rules":[{"kind":"alert","alert":"ThanosComponentDown","expr":"up{job=~\".*thanos.*\"} == 0","for":"20m","severity":"critical","summary":"Thanos component {{ $labels.job }} is down","description":"Thanos target {{ $labels.job }} ({{ $labels.instance }}) has failed its scrape for 20 minutes. The process is down or unreachable.","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"ThanosStatusUnhealthy","expr":"thanos_status{check=\"healthy\"} == 0","for":"30m","severity":"critical","summary":"Thanos {{ $labels.component }} reports unhealthy","description":"Thanos {{ $labels.component }} on {{ $labels.instance }} has been reporting an unhealthy status for 30 minutes. The process runs but flagged an internal error.","runbook_url":"","metric_refs":["thanos_status"],"line":12},{"kind":"alert","alert":"ThanosQueryIsDown","expr":"sum by (job) (up{job=~\".*thanos.*query.*\"}) == 0","for":"20m","severity":"critical","summary":"Thanos Query {{ $labels.job }} has no healthy instance","description":"Every instance of {{ $labels.job }} has been down for 20 minutes. All queries fail until one returns.","runbook_url":"","metric_refs":[],"line":21},{"kind":"alert","alert":"ThanosQueryNoStoreConnections","expr":"sum by (job, instance) (thanos_store_nodes_grpc_connections{job=~\".*thanos.*query.*\"}) == 0","for":"20m","severity":"critical","summary":"Thanos Query {{ $labels.instance }} has no store connection","description":"Thanos Query {{ $labels.instance }} has no gRPC connection to any Store API. It returns no data until connections are restored.","runbook_url":"","metric_refs":["thanos_store_nodes_grpc_connections"],"line":30},{"kind":"alert","alert":"ThanosQueryHttpErrorRateHigh","expr":"sum by (job, handler) (rate(http_requests_total{job=~\".*thanos.*query.*\", handler=~\"query|query_range\", code=~\"5..\"}[5m]))\n/\nsum by (job, handler) (rate(http_requests_total{job=~\".*thanos.*query.*\", handler=~\"query|query_range\"}[5m]))\n* 100 > 5\n","for":"20m","severity":"critical","summary":"Thanos Query {{ $labels.job }}: 5xx on {{ $labels.handler }} requests","description":"{{ $value | humanize }}% of {{ $labels.handler }} requests handled by {{ $labels.job }} return a 5xx.","runbook_url":"","metric_refs":["http_requests_total"],"line":39},{"kind":"alert","alert":"ThanosQueryGrpcServerErrorRate","expr":"sum by (job) (rate(grpc_server_handled_total{job=~\".*thanos.*query.*\", grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[5m]))\n/\nsum by (job) (rate(grpc_server_started_total{job=~\".*thanos.*query.*\"}[5m]))\n* 100 > 5\n","for":"20m","severity":"warning","summary":"Thanos Query {{ $labels.job }}: gRPC server errors","description":"{{ $value | humanize }}% of the gRPC requests served by {{ $labels.job }} fail.","runbook_url":"","metric_refs":["grpc_server_handled_total","grpc_server_started_total"],"line":52},{"kind":"alert","alert":"ThanosQueryGrpcClientErrorRate","expr":"sum by (job) (rate(grpc_client_handled_total{job=~\".*thanos.*query.*\", grpc_code!=\"OK\"}[15m]))\n/\nsum by (job) (rate(grpc_client_started_total{job=~\".*thanos.*query.*\"}[15m]))\n* 100 > 10\n","for":"20m","severity":"warning","summary":"Thanos Query {{ $labels.job }}: gRPC client errors towards stores","description":"{{ $value | humanize }}% of the gRPC requests sent by {{ $labels.job }} to its store endpoints fail. Store gateways or receivers may be unhealthy.","runbook_url":"","metric_refs":["grpc_client_handled_total","grpc_client_started_total"],"line":65},{"kind":"alert","alert":"ThanosQueryInstantLatencyHigh","expr":"histogram_quantile(0.99,\n  sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\".*thanos.*query.*\", handler=\"query\"}[5m]))\n) > 40\n","for":"20m","severity":"warning","summary":"Thanos Query {{ $labels.job }}: slow instant queries","description":"p99 latency of instant queries on {{ $labels.job }} is {{ $value | humanizeDuration }}.","runbook_url":"","metric_refs":["http_request_duration_seconds_bucket"],"line":79},{"kind":"alert","alert":"ThanosQueryRangeLatencyHigh","expr":"histogram_quantile(0.99,\n  sum by (job, le) (rate(http_request_duration_seconds_bucket{job=~\".*thanos.*query.*\", handler=\"query_range\"}[5m]))\n) > 90\n","for":"20m","severity":"warning","summary":"Thanos Query {{ $labels.job }}: slow range queries","description":"p99 latency of range queries on {{ $labels.job }} is {{ $value | humanizeDuration }}.","runbook_url":"","metric_refs":["http_request_duration_seconds_bucket"],"line":91},{"kind":"alert","alert":"ThanosStoreGrpcErrorRate","expr":"sum by (job) (rate(grpc_server_handled_total{job=~\".*thanos.*store.*\", grpc_code=~\"Unknown|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[5m]))\n/\nsum by (job) (rate(grpc_server_started_total{job=~\".*thanos.*store.*\"}[5m]))\n* 100 > 5\n","for":"20m","severity":"warning","summary":"Thanos Store {{ $labels.job }}: gRPC errors","description":"{{ $value | humanize }}% of the gRPC requests served by {{ $labels.job }} fail. Queriers get errors or partial results for historical ranges.","runbook_url":"","metric_refs":["grpc_server_handled_total","grpc_server_started_total"],"line":103},{"kind":"alert","alert":"ThanosStoreBucketHighOperationFailures","expr":"sum by (job, bucket) (rate(thanos_objstore_bucket_operation_failures_total{job=~\".*thanos.*store.*\"}[5m]))\n/\nsum by (job, bucket) (rate(thanos_objstore_bucket_operations_total{job=~\".*thanos.*store.*\"}[5m]))\n* 100 > 5\n","for":"20m","severity":"warning","summary":"Thanos Store {{ $labels.job }}: object storage failures on {{ $labels.bucket }}","description":"{{ $value | humanize }}% of the operations of {{ $labels.job }} against bucket {{ $labels.bucket }} fail. Check bucket availability, credentials and network.","runbook_url":"","metric_refs":["thanos_objstore_bucket_operation_failures_total","thanos_objstore_bucket_operations_total"],"line":116},{"kind":"alert","alert":"ThanosStoreBlockLoadFailures","expr":"sum by (job) (rate(thanos_bucket_store_block_load_failures_total{job=~\".*thanos.*store.*\"}[5m])) > 0","for":"20m","severity":"warning","summary":"Thanos Store {{ $labels.job }}: block load failures","description":"{{ $labels.job }} fails to load blocks from object storage. Some time ranges return incomplete results.","runbook_url":"","metric_refs":["thanos_bucket_store_block_load_failures_total"],"line":129},{"kind":"alert","alert":"ThanosReceiveHttpErrorRateHigh","expr":"sum by (job) (rate(http_requests_total{job=~\".*thanos.*receive.*\", handler=\"receive\", code=~\"5..\"}[5m]))\n/\nsum by (job) (rate(http_requests_total{job=~\".*thanos.*receive.*\", handler=\"receive\"}[5m]))\n* 100 > 5\n","for":"20m","severity":"warning","summary":"Thanos Receive {{ $labels.job }}: 5xx on remote write","description":"{{ $value | humanize }}% of the write requests received by {{ $labels.job }} return a 5xx. Remote write senders are losing or retrying data.","runbook_url":"","metric_refs":["http_requests_total"],"line":138},{"kind":"alert","alert":"ThanosReceiveHighForwardFailureRate","expr":"sum by (job) (rate(thanos_receive_forward_requests_total{job=~\".*thanos.*receive.*\", result=\"error\"}[5m]))\n/\nsum by (job) (rate(thanos_receive_forward_requests_total{job=~\".*thanos.*receive.*\"}[5m]))\n* 100 > 20\n","for":"20m","severity":"warning","summary":"Thanos Receive {{ $labels.job }}: forward failures","description":"{{ $value | humanize }}% of the requests forwarded by {{ $labels.job }} to other receivers fail. Check connectivity between replicas.","runbook_url":"","metric_refs":["thanos_receive_forward_requests_total"],"line":151},{"kind":"alert","alert":"ThanosReceiveConfigReloadFailure","expr":"avg by (job, instance) (thanos_receive_config_last_reload_successful{job=~\".*thanos.*receive.*\"}) != 1","for":"20m","severity":"warning","summary":"Thanos Receive {{ $labels.instance }}: hashring reload failed","description":"The last hashring configuration reload of {{ $labels.instance }} failed. Routing changes have not taken effect.","runbook_url":"","metric_refs":["thanos_receive_config_last_reload_successful"],"line":164},{"kind":"alert","alert":"ThanosReceiveHeadCompactionStalled","expr":"max by (job, instance) (\n  ((time() - prometheus_tsdb_head_min_time{job=~\".*thanos.*receive.*\"} / 1000) / 3600) > 0\n) > 6\n","for":"30m","severity":"critical","summary":"Thanos Receive {{ $labels.instance }}: head block not compacted","description":"The head block of {{ $labels.instance }} spans {{ $value | humanize }}h. Compaction has stalled and memory keeps growing until an OOM kill.","runbook_url":"","metric_refs":["prometheus_tsdb_head_min_time"],"line":175},{"kind":"alert","alert":"ThanosRuleQueueIsDroppingAlerts","expr":"sum by (job, instance) (rate(thanos_alert_queue_alerts_dropped_total{job=~\".*thanos.*rule.*\"}[5m])) > 0","for":"20m","severity":"critical","summary":"Thanos Rule {{ $labels.instance }}: alert queue dropping alerts","description":"{{ $labels.instance }} drops alerts from its queue. Alertmanager may not receive every firing alert.","runbook_url":"","metric_refs":["thanos_alert_queue_alerts_dropped_total"],"line":187},{"kind":"alert","alert":"ThanosRuleSenderIsFailingAlerts","expr":"sum by (job, instance) (rate(thanos_alert_sender_alerts_dropped_total{job=~\".*thanos.*rule.*\"}[5m])) > 0","for":"20m","severity":"critical","summary":"Thanos Rule {{ $labels.instance }}: failing to send alerts","description":"{{ $labels.instance }} cannot deliver alerts to any Alertmanager. Firing alerts are dropped.","runbook_url":"","metric_refs":["thanos_alert_sender_alerts_dropped_total"],"line":196},{"kind":"alert","alert":"ThanosRuleHighRuleEvaluationFailures","expr":"sum by (job, instance) (rate(prometheus_rule_evaluation_failures_total{job=~\".*thanos.*rule.*\"}[5m]))\n/\nsum by (job, instance) (rate(prometheus_rule_evaluations_total{job=~\".*thanos.*rule.*\"}[5m]))\n* 100 > 5\n","for":"20m","severity":"critical","summary":"Thanos Rule {{ $labels.instance }}: rule evaluation failures","description":"{{ $value | humanize }}% of the rule evaluations of {{ $labels.instance }} fail. Alerts and recording rules are stale or missing.","runbook_url":"","metric_refs":["prometheus_rule_evaluation_failures_total","prometheus_rule_evaluations_total"],"line":205},{"kind":"alert","alert":"ThanosNoRuleEvaluations","expr":"sum by (job, instance) (rate(prometheus_rule_evaluations_total{job=~\".*thanos.*rule.*\"}[5m])) <= 0\nand\nsum by (job, instance) (thanos_rule_loaded_rules{job=~\".*thanos.*rule.*\"}) > 0\n","for":"20m","severity":"critical","summary":"Thanos Rule {{ $labels.instance }}: no rule evaluations","description":"{{ $labels.instance }} has rules loaded but evaluated none for 20 minutes. It is stuck or cannot reach its query endpoints.","runbook_url":"","metric_refs":["prometheus_rule_evaluations_total","thanos_rule_loaded_rules"],"line":218},{"kind":"alert","alert":"ThanosRuleConfigReloadFailure","expr":"avg by (job, instance) (thanos_rule_config_last_reload_successful{job=~\".*thanos.*rule.*\"}) != 1","for":"20m","severity":"warning","summary":"Thanos Rule {{ $labels.instance }}: config reload failed","description":"{{ $labels.instance }} could not reload its rule files. New or changed rule groups do not take effect.","runbook_url":"","metric_refs":["thanos_rule_config_last_reload_successful"],"line":230},{"kind":"alert","alert":"ThanosShipperNoUploads","expr":"sum by (job, instance) (increase(thanos_shipper_uploads_total{job=~\".*thanos.*\"}[6h])) == 0","for":"30m","severity":"warning","summary":"Thanos Shipper {{ $labels.instance }}: no block uploaded in 6h","description":"{{ $labels.instance }} has shipped no block to object storage for 6 hours. The bucket is unreachable or block cutting has stalled.","runbook_url":"","metric_refs":["thanos_shipper_uploads_total"],"line":240},{"kind":"alert","alert":"ThanosShipperUploadFailures","expr":"sum by (job, instance) (rate(thanos_shipper_upload_failures_total{job=~\".*thanos.*\"}[5m])) > 0","for":"20m","severity":"warning","summary":"Thanos Shipper {{ $labels.instance }}: block upload failures","description":"{{ $labels.instance }} fails to upload TSDB blocks to object storage. Check credentials, connectivity and logs.","runbook_url":"","metric_refs":["thanos_shipper_upload_failures_total"],"line":249},{"kind":"alert","alert":"ThanosShipperCorruptedBlocks","expr":"sum by (job, instance) (increase(thanos_shipper_corrupted_blocks_total{job=~\".*thanos.*\"}[1h])) > 0","for":"20m","severity":"warning","summary":"Thanos Shipper {{ $labels.instance }}: corrupted blocks","description":"{{ $labels.instance }} detected corrupted TSDB blocks in the last hour. Data may be lost, check TSDB integrity.","runbook_url":"","metric_refs":["thanos_shipper_corrupted_blocks_total"],"line":258},{"kind":"alert","alert":"ThanosBlockMetaSyncFailures","expr":"sum by (job, instance) (rate(thanos_blocks_meta_sync_failures_total{job=~\".*thanos.*\"}[5m])) > 0","for":"20m","severity":"warning","summary":"Thanos {{ $labels.job }}: block metadata sync failures","description":"{{ $labels.instance }} fails to sync block metadata from object storage and works on a stale view of the available blocks.","runbook_url":"","metric_refs":["thanos_blocks_meta_sync_failures_total"],"line":267},{"kind":"alert","alert":"ThanosCompactHalted","expr":"thanos_compact_halted{job=~\".*thanos.*compact.*\"} == 1","for":"5m","severity":"critical","summary":"Thanos Compact {{ $labels.instance }} halted","description":"{{ $labels.instance }} halted after an unexpected error and no longer compacts or downsamples. It needs manual investigation, usually a corrupted or overlapping block.","runbook_url":"","metric_refs":["thanos_compact_halted"],"line":276}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/thanos.rules.yml"},{"file":"varnish.rules.yml","name":"varnish.rules","category":"Web & load balancing","rules":[{"kind":"alert","alert":"VarnishExporterDown","expr":"group by (job, instance) (up{job=~\".*varnish.*\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=~\".*varnish.*\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=~\".*varnish.*\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"Varnish exporter down - {{ $labels.job }} @ {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"VarnishDown","expr":"varnish_up == 0","for":"2m","severity":"critical","summary":"Varnish down on {{ $labels.instance }}","description":"The exporter on {{ $labels.instance }} cannot read Varnish's shared memory. The child process is likely dead.","runbook_url":"","metric_refs":["varnish_up"],"line":19},{"kind":"alert","alert":"VarnishBackendUnhealthy","expr":"increase(varnish_main_backend_unhealthy[5m]) > 0","for":"5m","severity":"critical","summary":"Varnish backends going unhealthy on {{ $labels.instance }}","description":"{{ $value | humanize }} backend health transitions to unhealthy in 5 minutes on {{ $labels.instance }}.","runbook_url":"","metric_refs":["varnish_main_backend_unhealthy"],"line":30},{"kind":"alert","alert":"VarnishBackendFailures","expr":"rate(varnish_main_backend_fail[5m]) > 0.1","for":"10m","severity":"critical","summary":"Varnish backend fetch failures on {{ $labels.instance }} ({{ $value | humanize }}/s)","description":"Varnish on {{ $labels.instance }} is failing to reach its backends. Clients are getting errors, not stale cache.","runbook_url":"","metric_refs":["varnish_main_backend_fail"],"line":41},{"kind":"alert","alert":"VarnishThreadsLimited","expr":"increase(varnish_main_threads_limited[5m]) > 0","for":"10m","severity":"critical","summary":"Varnish hitting its thread limit on {{ $labels.instance }}","description":"Varnish on {{ $labels.instance }} wanted more worker threads than thread_pool_max allows. Requests are queueing.","runbook_url":"","metric_refs":["varnish_main_threads_limited"],"line":52},{"kind":"alert","alert":"VarnishLowCacheHitRate","expr":"sum by (instance, job) (rate(varnish_main_cache_hit[5m]))\n/\n(\n  sum by (instance, job) (\n    rate(varnish_main_cache_hit[5m]) + rate(varnish_main_cache_miss[5m])\n  ) > 10\n)\n< 0.60\n","for":"15m","severity":"warning","summary":"Varnish cache hit rate low on {{ $labels.instance }} ({{ $value | humanizePercentage }})","description":"Fewer than 60% of lookups are served from cache on {{ $labels.instance }} at more than 10 req/s. Check TTLs and Vary headers.","runbook_url":"","metric_refs":["varnish_main_cache_hit","varnish_main_cache_miss"],"line":63},{"kind":"alert","alert":"VarnishLRUNuking","expr":"rate(varnish_main_n_lru_nuked[5m]) > 1","for":"15m","severity":"warning","summary":"Varnish evicting {{ $value | humanize }} objects/s on {{ $labels.instance }}","description":"Varnish on {{ $labels.instance }} is nuking live objects from cache to free space. The working set no longer fits in the configured storage.","runbook_url":"","metric_refs":["varnish_main_n_lru_nuked"],"line":81}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/varnish.rules.yml"},{"file":"zookeeper.rules.yml","name":"zookeeper.rules","category":"Messaging & streaming","rules":[{"kind":"alert","alert":"ZookeeperExporter","expr":"# Two distinct failure modes: a failing scrape (up == 0, exporter crashed\n# or host unreachable) and the target vanishing from service discovery,\n# where no up series exists to be 0. Both branches are grouped\n# identically so the alert carries the same labels either way.\ngroup by (job, instance) (up{job=\"zookeeper\"} == 0)\nor\n(\n  group by (job, instance) (last_over_time(up{job=\"zookeeper\"}[1h]))\n  unless\n  group by (job, instance) (last_over_time(up{job=\"zookeeper\"}[5m]))\n)\n","for":"2m","severity":"critical","summary":"Job {{ $labels.job }} having an issue on {{ $labels.instance }}","description":"{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.","runbook_url":"","metric_refs":[],"line":3},{"kind":"alert","alert":"ZookeeperWrongQuorum","expr":"min(zookeeper_QuorumSize) by (instance,job) < 3","for":"2m","severity":"critical","summary":"{{ $labels.instance }} of job {{ $labels.job }} has an insufficient Quorum","description":"The ensemble reports a quorum size of {{ $value }}, below 3. One more node loss prevents it from electing a leader.","runbook_url":"","metric_refs":["zookeeper_QuorumSize"],"line":22},{"kind":"alert","alert":"ZookeeperNoLeader","expr":"sum(zookeeper_Leader) by (job) == 0","for":"1m","severity":"critical","summary":"ZooKeeper no leader","description":"No ZooKeeper node is currently the leader. The ensemble is unavailable.","runbook_url":"","metric_refs":["zookeeper_Leader"],"line":31},{"kind":"alert","alert":"ZookeeperHighAvgLatency","expr":"zookeeper_AvgRequestLatency > 100","for":"5m","severity":"warning","summary":"ZooKeeper high average latency (instance {{ $labels.instance }})","description":"ZooKeeper average request latency above 100ms on {{ $labels.instance }} (replicaId={{ $labels.replicaId }}). Client operations may time out.","runbook_url":"","metric_refs":["zookeeper_AvgRequestLatency"],"line":42},{"kind":"alert","alert":"ZookeeperHighMaxLatency","expr":"zookeeper_MaxRequestLatency > 500","for":"5m","severity":"warning","summary":"ZooKeeper high max latency (instance {{ $labels.instance }})","description":"ZooKeeper max request latency above 500ms on {{ $labels.instance }}.","runbook_url":"","metric_refs":["zookeeper_MaxRequestLatency"],"line":51},{"kind":"alert","alert":"ZookeeperOutstandingRequests","expr":"zookeeper_OutstandingRequests > 10","for":"5m","severity":"warning","summary":"ZooKeeper outstanding requests (instance {{ $labels.instance }})","description":"ZooKeeper has {{ $value }} outstanding requests on {{ $labels.instance }} - node may be overloaded.","runbook_url":"","metric_refs":["zookeeper_OutstandingRequests"],"line":60},{"kind":"alert","alert":"ZookeeperJVMDeadlock","expr":"jvm_threads_deadlocked_monitor > 0","for":"0m","severity":"critical","summary":"ZooKeeper JVM deadlock (instance {{ $labels.instance }})","description":"JVM thread deadlock detected in ZooKeeper on {{ $labels.instance }}. Restart required.","runbook_url":"","metric_refs":["jvm_threads_deadlocked_monitor"],"line":69},{"kind":"alert","alert":"ZookeeperJMXScrapeFailed","expr":"jmx_scrape_error != 0","for":"2m","severity":"critical","summary":"ZooKeeper JMX scrape failed (instance {{ $labels.instance }})","description":"JMX scrape failed on ZooKeeper {{ $labels.instance }}. Metrics are incomplete.","runbook_url":"","metric_refs":["jmx_scrape_error"],"line":78}],"github_url":"https://github.com/PixiBixi/prometheus-rules/blob/main/rules/zookeeper.rules.yml"}],"metrics":{"aerospike_namespace_client_read_success":["counter","client read success","aerospike"],"aerospike_namespace_client_read_timeout":["counter","client read timeout","aerospike"],"aerospike_namespace_client_write_success":["counter","client write success","aerospike"],"aerospike_namespace_client_write_timeout":["counter","client write timeout","aerospike"],"aerospike_namespace_clock_skew_stop_writes":["gauge","clock skew stop writes","aerospike"],"aerospike_namespace_dead_partitions":["gauge","dead partitions","aerospike"],"aerospike_namespace_device_available_pct":["gauge","device available pct","aerospike"],"aerospike_namespace_high_water_memory_pct":["gauge","high water memory pct","aerospike"],"aerospike_namespace_memory_free_pct":["gauge","memory free pct","aerospike"],"aerospike_namespace_migrate_rx_partitions_remaining":["gauge","migrate rx partitions remaining","aerospike"],"aerospike_namespace_migrate_tx_partitions_remaining":["gauge","migrate tx partitions remaining","aerospike"],"aerospike_namespace_stop_writes":["gauge","stop writes","aerospike"],"aerospike_namespace_stop_writes_pct":["gauge","stop writes pct","aerospike"],"aerospike_namespace_unavailable_partitions":["gauge","unavailable partitions","aerospike"],"aerospike_node_stats_cluster_clock_skew_ms":["gauge","cluster clock skew ms","aerospike"],"aerospike_node_stats_cluster_clock_skew_stop_writes_sec":["gauge","cluster clock skew stop writes sec","aerospike"],"aerospike_node_stats_heap_efficiency_pct":["gauge","heap efficiency pct","aerospike"],"aerospike_node_stats_system_free_mem_pct":["gauge","system free mem pct","aerospike"],"aerospike_node_up":["gauge","Aerospike node active status","aerospike"],"aerospike_xdr_lag":["gauge","lag","aerospike"],"apache_accesses_total":["counter","Current total apache accesses (*)","apache_exporter"],"apache_duration_ms_total":["counter","Total duration of all registered requests in ms","apache_exporter"],"apache_processes":["gauge","Apache process count","apache_exporter"],"apache_scoreboard":["gauge","Apache scoreboard statuses","apache_exporter"],"apache_up":["gauge","Could the apache server be reached","apache_exporter"],"apache_uptime_seconds_total":["counter","Current uptime in seconds (*)","apache_exporter"],"apache_workers":["gauge","Apache worker statuses","apache_exporter"],"process_max_fds":["gauge","Maximum number of open file descriptors.","apache_exporter"],"process_open_fds":["gauge","Number of open file descriptors.","apache_exporter"],"process_start_time_seconds":["gauge","Start time of the process since unix epoch in seconds.","apache_exporter"],"rollout_info":["gauge","Information about rollout.","argo-rollouts"],"rollout_reconcile_error":["counter","Error occurring during the rollout","argo-rollouts"],"argocd_app_info":["gauge","Information about application.","argocd"],"argocd_app_sync_total":["counter","Number of application syncs.","argocd"],"argocd_cluster_connection_status":["gauge","The k8s cluster current connection status.","argocd"],"argocd_git_lsremote_fail_total":["counter","Number of git ls-remote requests failures by repo server","argocd"],"blackbox_exporter_config_last_reload_success_timestamp_seconds":["gauge","Timestamp of the last successful configuration reload.","blackbox_exporter"],"blackbox_exporter_config_last_reload_successful":["gauge","Blackbox exporter config loaded successfully.","blackbox_exporter"],"blackbox_module_unknown_total":["counter","Count of unknown modules requested by probes","blackbox_exporter"],"certmanager_certificate_expiration_timestamp_seconds":["gauge","The timestamp after which the certificate expires, expressed in Unix Epoch Time.","cert-manager"],"certmanager_certificate_ready_status":["gauge","The ready status of the certificate.","cert-manager"],"certmanager_certificate_renewal_timestamp_seconds":["gauge","The timestamp after which the certificate should be renewed, expressed in Unix Epoch Time.","cert-manager"],"certmanager_clusterissuer_ready_status":["gauge","The ready status of the ClusterIssuer.","cert-manager"],"certmanager_controller_sync_call_count":["counter","The number of sync() calls made by a controller. Label: controller (fixed small set of controller names).","cert-manager"],"certmanager_issuer_ready_status":["gauge","The ready status of the Issuer.","cert-manager"],"certmanager_controller_sync_error_count":["counter","The number of errors encountered during controller sync().","cert-manager"],"certmanager_http_acme_client_request_count":["counter","The number of requests made by the ACME client.","cert-manager"],"certmanager_http_acme_client_request_duration_seconds_count":["","","cert-manager"],"controller_runtime_reconcile_errors_total":["counter","Total number of reconciliation errors per controller","controller-runtime"],"controller_runtime_reconcile_panics_total":["counter","Total number of reconciliation panics per controller","controller-runtime"],"controller_runtime_webhook_requests_total":["counter","Total number of admission requests by HTTP status code.","controller-runtime"],"workqueue_unfinished_work_seconds":["gauge","How many seconds of work has been done that is in progress and hasn't been observed by work_duration. Large values indicate stuck threads. One can deduce the number of stuck threads by observing the rate at which this increases.","controller-runtime"],"rest_client_requests_total":["counter","Number of HTTP requests, partitioned by status code, method, and host.","controller-runtime"],"haproxy_process_pool_failures_total":["counter","Number of failed pool allocations since this worker was started","haproxy_exporter_2x"],"haproxy_process_max_connections":["gauge","Hard limit on the number of per-process connections (configured or imposed by Ulimit-n)","haproxy_exporter_2x"],"haproxy_process_hard_max_connections":["gauge","Hard limit on the number of per-process connections (imposed by Memmax_MB or Ulimit-n)","haproxy_exporter_2x"],"haproxy_process_current_connections":["gauge","Current number of connections on this worker process","haproxy_exporter_2x"],"haproxy_process_max_ssl_connections":["gauge","Hard limit on the number of per-process SSL endpoints (front+back), 0=unlimited","haproxy_exporter_2x"],"haproxy_process_current_ssl_connections":["gauge","Current number of SSL endpoints on this worker process (front+back)","haproxy_exporter_2x"],"haproxy_process_current_connection_rate":["gauge","Number of front connections created on this worker process over the last second","haproxy_exporter_2x"],"haproxy_process_limit_connection_rate":["gauge","Hard limit for ConnRate (global.maxconnrate)","haproxy_exporter_2x"],"haproxy_process_current_session_rate":["gauge","Number of sessions created on this worker process over the last second","haproxy_exporter_2x"],"haproxy_process_limit_session_rate":["gauge","Hard limit for SessRate (global.maxsessrate)","haproxy_exporter_2x"],"haproxy_process_idle_time_percent":["gauge","Percentage of last second spent waiting in the current worker thread","haproxy_exporter_2x"],"haproxy_process_stopping":["gauge","1 if the worker process is currently stopping, otherwise zero","haproxy_exporter_2x"],"haproxy_process_dropped_logs_total":["counter","Total number of dropped logs for current worker process since started","haproxy_exporter_2x"],"haproxy_process_failed_resolutions":["counter","Total number of failed DNS resolutions in current worker process since started","haproxy_exporter_2x"],"haproxy_frontend_current_sessions":["gauge","Number of current sessions on the frontend, backend or server","haproxy_exporter_2x"],"haproxy_frontend_limit_sessions":["gauge","Frontend/listener/server's maxconn, backend's fullconn","haproxy_exporter_2x"],"haproxy_frontend_request_errors_total":["counter","Total number of invalid requests since process started","haproxy_exporter_2x"],"haproxy_frontend_status":["gauge","Current status of the service, per state label value.","haproxy_exporter_2x"],"haproxy_frontend_http_responses_total":["counter","Total number of HTTP responses with status 100-199 returned by this object since the worker process started","haproxy_exporter_2x"],"haproxy_frontend_http_requests_total":["counter","Total number of HTTP requests processed by this object since the worker process started","haproxy_exporter_2x"],"haproxy_frontend_connections_total":["counter","Total number of new connections accepted on this frontend since the worker process started","haproxy_exporter_2x"],"haproxy_frontend_denied_connections_total":["counter","Total number of incoming connections blocked on a listener/frontend by a tcp-request connection rule since the worker process started","haproxy_exporter_2x"],"haproxy_backend_current_queue":["gauge","Number of current queued connections","haproxy_exporter_2x"],"haproxy_backend_current_sessions":["gauge","Number of current sessions on the frontend, backend or server","haproxy_exporter_2x"],"haproxy_backend_limit_sessions":["gauge","Frontend/listener/server's maxconn, backend's fullconn","haproxy_exporter_2x"],"haproxy_backend_connection_errors_total":["counter","Total number of failed connections to server since the worker process started","haproxy_exporter_2x"],"haproxy_backend_retry_warnings_total":["counter","Total number of server connection retries since the worker process started","haproxy_exporter_2x"],"haproxy_backend_redispatch_warnings_total":["counter","Total number of server redispatches due to connection failures since the worker process started","haproxy_exporter_2x"],"haproxy_backend_status":["gauge","Current status of the service, per state label value.","haproxy_exporter_2x"],"haproxy_backend_active_servers":["gauge","Total number of active UP servers with a non-zero weight","haproxy_exporter_2x"],"haproxy_backend_backup_servers":["gauge","Total number of backup UP servers with a non-zero weight","haproxy_exporter_2x"],"haproxy_backend_check_up_down_total":["counter","Total number of failed checks causing UP to DOWN server transitions, per server/backend, since the worker process started","haproxy_exporter_2x"],"haproxy_backend_downtime_seconds_total":["counter","Total time spent in DOWN state, for server or backend","haproxy_exporter_2x"],"haproxy_backend_http_responses_total":["counter","Total number of HTTP responses with status 100-199 returned by this object since the worker process started","haproxy_exporter_2x"],"haproxy_backend_client_aborts_total":["counter","Total number of requests or connections aborted by the client since the worker process started","haproxy_exporter_2x"],"haproxy_backend_server_aborts_total":["counter","Total number of requests or connections aborted by the server since the worker process started","haproxy_exporter_2x"],"haproxy_backend_queue_time_average_seconds":["gauge","Avg. queue time for last 1024 successful connections.","haproxy_exporter_2x"],"haproxy_backend_response_time_average_seconds":["gauge","Avg. response time for last 1024 successful connections.","haproxy_exporter_2x"],"haproxy_backend_connection_attempts_total":["counter","Total number of outgoing connection attempts on this backend/server since the worker process started","haproxy_exporter_2x"],"haproxy_backend_connection_reuses_total":["counter","Total number of reused connection on this backend/server since the worker process started","haproxy_exporter_2x"],"haproxy_backend_internal_errors_total":["counter","Total number of internal errors since process started","haproxy_exporter_2x"],"haproxy_backend_agg_server_check_status":["gauge","Backend's aggregated gauge of servers' state check status","haproxy_exporter_2x"],"haproxy_server_current_queue":["gauge","Number of current queued connections","haproxy_exporter_2x"],"haproxy_server_current_sessions":["gauge","Number of current sessions on the frontend, backend or server","haproxy_exporter_2x"],"haproxy_server_limit_sessions":["gauge","Frontend/listener/server's maxconn, backend's fullconn","haproxy_exporter_2x"],"haproxy_server_sessions_total":["counter","Total number of sessions since process started","haproxy_exporter_2x"],"haproxy_server_bytes_in_total":["counter","Total number of request bytes since process started","haproxy_exporter_2x"],"haproxy_server_bytes_out_total":["counter","Total number of response bytes since process started","haproxy_exporter_2x"],"haproxy_server_connection_errors_total":["counter","Total number of failed connections to server since the worker process started","haproxy_exporter_2x"],"haproxy_server_response_errors_total":["counter","Total number of invalid responses since the worker process started","haproxy_exporter_2x"],"haproxy_server_status":["gauge","Current status of the service, per state label value.","haproxy_exporter_2x"],"haproxy_server_check_failures_total":["counter","Total number of failed individual health checks per server/backend, since the worker process started","haproxy_exporter_2x"],"haproxy_server_downtime_seconds_total":["counter","Total time spent in DOWN state, for server or backend","haproxy_exporter_2x"],"haproxy_server_http_responses_total":["counter","Total number of HTTP responses with status 100-199 returned by this object since the worker process started","haproxy_exporter_2x"],"haproxy_server_connection_attempts_total":["counter","Total number of outgoing connection attempts on this backend/server since the worker process started","haproxy_exporter_2x"],"haproxy_sticktable_size":["gauge","Stick table size.","haproxy_exporter_2x"],"haproxy_sticktable_used":["gauge","Number of entries used in this stick table.","haproxy_exporter_2x"],"ipsec_status":["gauge","ipsec status value","ipsec_exporter"],"kafka_consumergroup_current_offset":["gauge","Current Offset of a ConsumerGroup at Topic/Partition","kafka_consumer_exporter"],"kafka_consumergroup_current_offset_sum":["gauge","Current Offset of a ConsumerGroup at Topic for all partitions","kafka_consumer_exporter"],"kafka_consumergroup_lag":["gauge","Current Approximate Lag of a ConsumerGroup at Topic/Partition","kafka_consumer_exporter"],"kafka_consumergroup_lag_sum":["gauge","Current Approximate Lag of a ConsumerGroup at Topic for all partitions","kafka_consumer_exporter"],"jvm_threads_current":["gauge","Current thread count of a JVM","kafka_exporter"],"jvm_threads_deadlocked":["gauge","Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers","kafka_exporter"],"jvm_threads_deadlocked_monitor":["gauge","Cycles of JVM-threads that are in deadlock waiting to acquire object monitors","kafka_exporter"],"jvm_threads_state":["gauge","Current count of threads by state","kafka_exporter"],"jvm_buffer_pool_used_bytes":["gauge","Used bytes of a given JVM buffer pool.","kafka_exporter"],"jvm_buffer_pool_capacity_bytes":["gauge","Bytes capacity of a given JVM buffer pool.","kafka_exporter"],"kafka_network_requestmetrics_totaltimems":["gauge","Attribute exposed for management kafka.network:name=TotalTimeMs,type=RequestMetrics,attribute=50thPercentile","kafka_exporter"],"kafka_server_replicamanager_reassigningpartitions":["gauge","Attribute exposed for management kafka.server:name=ReassigningPartitions,type=ReplicaManager,attribute=Value","kafka_exporter"],"kafka_network_socketserver_networkprocessoravgidlepercent":["gauge","Attribute exposed for management kafka.network:name=NetworkProcessorAvgIdlePercent,type=SocketServer,attribute=Value","kafka_exporter"],"kafka_controller_controllerstats_uncleanleaderelections_total":["counter","Attribute exposed for management kafka.controller:name=UncleanLeaderElectionsPerSec,type=ControllerStats,attribute=Count","kafka_exporter"],"kafka_server_replicamanager_underminisrpartitioncount":["gauge","Attribute exposed for management kafka.server:name=UnderMinIsrPartitionCount,type=ReplicaManager,attribute=Value","kafka_exporter"],"kafka_server_replicamanager_offlinereplicacount":["gauge","Attribute exposed for management kafka.server:name=OfflineReplicaCount,type=ReplicaManager,attribute=Value","kafka_exporter"],"kafka_controller_kafkacontroller_activecontrollercount":["gauge","Attribute exposed for management kafka.controller:name=ActiveControllerCount,type=KafkaController,attribute=Value","kafka_exporter"],"kafka_controller_kafkacontroller_offlinepartitionscount":["gauge","Attribute exposed for management kafka.controller:name=OfflinePartitionsCount,type=KafkaController,attribute=Value","kafka_exporter"],"kafka_network_requestmetrics_totaltimems_count":["counter","Attribute exposed for management kafka.network:name=TotalTimeMs,type=RequestMetrics,attribute=Count","kafka_exporter"],"kafka_server_brokertopicmetrics_failedproducerequests_total":["counter","Attribute exposed for management kafka.server:name=FailedProduceRequestsPerSec,type=BrokerTopicMetrics,attribute=Count","kafka_exporter"],"kafka_log_logcleaner_deadthreadcount":["gauge","Attribute exposed for management kafka.log:name=DeadThreadCount,type=LogCleaner,attribute=Value","kafka_exporter"],"kafka_server_brokertopicmetrics_totalproducerequests_total":["counter","Attribute exposed for management kafka.server:name=TotalProduceRequestsPerSec,type=BrokerTopicMetrics,attribute=Count","kafka_exporter"],"kafka_server_replicamanager_underreplicatedpartitions":["gauge","Attribute exposed for management kafka.server:name=UnderReplicatedPartitions,type=ReplicaManager,attribute=Value","kafka_exporter"],"kafka_server_replicamanager_atminisrpartitioncount":["gauge","Attribute exposed for management kafka.server:name=AtMinIsrPartitionCount,type=ReplicaManager,attribute=Value","kafka_exporter"],"kafka_log_logmanager_offlinelogdirectorycount":["gauge","Attribute exposed for management kafka.log:name=OfflineLogDirectoryCount,type=LogManager,attribute=Value","kafka_exporter"],"kafka_controller_kafkacontroller_fencedbrokercount":["gauge","Attribute exposed for management kafka.controller:name=FencedBrokerCount,type=KafkaController,attribute=Value","kafka_exporter"],"kafka_server_kafkarequesthandlerpool_requesthandleravgidle_percent":["gauge","Attribute exposed for management kafka.server:name=RequestHandlerAvgIdlePercent,type=KafkaRequestHandlerPool,attribute=MeanRate","kafka_exporter"],"jmx_scrape_error":["gauge","Non-zero if this scrape failed.","kafka_exporter"],"jvm_memory_bytes_used":["gauge","Used bytes of a given JVM memory area.","kafka_exporter"],"jvm_memory_bytes_max":["gauge","Max (bytes) of a given JVM memory area.","kafka_exporter"],"jvm_gc_collection_seconds_count":["summary","Time spent in a given JVM garbage collector in seconds.","kafka_exporter"],"jvm_gc_collection_seconds_sum":["","","kafka_exporter"],"keepalived_advertisements_interval_errors_total":["counter","Advertisement interval errors","keepalived_exporter"],"keepalived_advertisements_received_total":["counter","Advertisements received","keepalived_exporter"],"keepalived_authentication_failure_total":["counter","Authentication failure","keepalived_exporter"],"keepalived_authentication_invalid_total":["counter","Authentication invalid","keepalived_exporter"],"keepalived_become_master_total":["counter","Became master","keepalived_exporter"],"keepalived_ip_ttl_errors_total":["counter","TTL errors","keepalived_exporter"],"keepalived_packet_length_errors_total":["counter","Packet length errors","keepalived_exporter"],"keepalived_up":["gauge","Status","keepalived_exporter"],"keepalived_vrrp_state":["gauge","State of vrrp","keepalived_exporter"],"redis_config_maxclients":["gauge","config_maxclients metric","keydb_exporter"],"redis_connected_clients":["gauge","connected_clients metric","keydb_exporter"],"redis_connected_slave_lag_seconds":["gauge","Lag of connected slave","keydb_exporter"],"redis_connected_slaves":["gauge","connected_slaves metric","keydb_exporter"],"redis_evicted_keys_total":["counter","evicted_keys_total metric","keydb_exporter"],"redis_instance_info":["gauge","Information about the Redis instance","keydb_exporter"],"redis_memory_max_bytes":["gauge","memory_max_bytes metric","keydb_exporter"],"redis_memory_used_bytes":["gauge","memory_used_bytes metric","keydb_exporter"],"redis_rdb_last_save_timestamp_seconds":["gauge","rdb_last_save_timestamp_seconds metric","keydb_exporter"],"redis_rejected_connections_total":["counter","rejected_connections_total metric","keydb_exporter"],"redis_up":["gauge","Information about the Redis instance","keydb_exporter"],"redis_cluster_slots_fail":["gauge","cluster_slots_fail metric","keydb_exporter"],"redis_cluster_state":["gauge","cluster_state metric","keydb_exporter"],"memcached_commands_total":["counter","Total number of all requests broken down by command (get, set, etc.) and status.","memcached_exporter"],"memcached_current_bytes":["gauge","Current number of bytes used to store items.","memcached_exporter"],"memcached_current_connections":["gauge","Current number of open connections.","memcached_exporter"],"memcached_items_evicted_total":["counter","Total number of valid items removed from cache to free memory for new items.","memcached_exporter"],"memcached_limit_bytes":["gauge","Number of bytes this server is allowed to use for storage.","memcached_exporter"],"memcached_max_connections":["gauge","Maximum number of clients allowed.","memcached_exporter"],"memcached_up":["gauge","Could the memcached server be reached.","memcached_exporter"],"mongodb_ss_connections":["untyped","serverStatus.connections","mongodb_exporter"],"mongodb_ss_globalLock_currentQueue":["untyped","serverStatus.globalLock.currentQueue","mongodb_exporter"],"mongodb_ss_wt_cache_bytes_currently_in_the_cache":["untyped","serverStatus.wiredTiger.cache.bytes currently in the cache","mongodb_exporter"],"mongodb_ss_wt_cache_maximum_bytes_configured":["untyped","serverStatus.wiredTiger.cache.maximum bytes configured","mongodb_exporter"],"mongodb_up":["gauge","Whether MongoDB is up.","mongodb_exporter"],"mysql_global_status_connection_errors_total":["counter","Total number of MySQL connection errors.","mysqld_exporter"],"mysql_global_status_innodb_log_waits":["untyped","Generic metric from SHOW GLOBAL STATUS.","mysqld_exporter"],"mysql_global_status_innodb_log_write_requests":["untyped","Generic metric from SHOW GLOBAL STATUS.","mysqld_exporter"],"mysql_global_status_innodb_num_open_files":["untyped","Generic metric from SHOW GLOBAL STATUS.","mysqld_exporter"],"mysql_global_status_prepared_stmt_count":["untyped","Generic metric from SHOW GLOBAL STATUS.","mysqld_exporter"],"mysql_global_status_slow_queries":["untyped","Generic metric from SHOW GLOBAL STATUS.","mysqld_exporter"],"mysql_global_status_threads_connected":["untyped","Generic metric from SHOW GLOBAL STATUS.","mysqld_exporter"],"mysql_global_status_threads_running":["untyped","Generic metric from SHOW GLOBAL STATUS.","mysqld_exporter"],"mysql_global_status_wsrep_cluster_size":["untyped","Generic metric from SHOW GLOBAL STATUS.","mysqld_exporter"],"mysql_global_status_wsrep_cluster_status":["untyped","Generic metric from SHOW GLOBAL STATUS.","mysqld_exporter"],"mysql_global_status_wsrep_local_recv_queue":["untyped","Generic metric from SHOW GLOBAL STATUS.","mysqld_exporter"],"mysql_global_status_wsrep_local_state":["untyped","Generic metric from SHOW GLOBAL STATUS.","mysqld_exporter"],"mysql_global_status_wsrep_ready":["untyped","Generic metric from SHOW GLOBAL STATUS.","mysqld_exporter"],"mysql_global_variables_innodb_force_recovery":["gauge","Generic gauge metric from SHOW GLOBAL VARIABLES.","mysqld_exporter"],"mysql_global_variables_max_connections":["gauge","Generic gauge metric from SHOW GLOBAL VARIABLES.","mysqld_exporter"],"mysql_global_variables_max_prepared_stmt_count":["gauge","Generic gauge metric from SHOW GLOBAL VARIABLES.","mysqld_exporter"],"mysql_global_variables_open_files_limit":["gauge","Generic gauge metric from SHOW GLOBAL VARIABLES.","mysqld_exporter"],"mysql_global_variables_wsrep_desync":["gauge","Generic gauge metric from SHOW GLOBAL VARIABLES.","mysqld_exporter"],"mysql_info_schema_innodb_metrics_transaction_trx_rseg_history_len":["gauge","Length of the TRX_RSEG_HISTORY list","mysqld_exporter"],"mysql_up":["gauge","Whether the MySQL server is up.","mysqld_exporter"],"node_cpu_core_throttles_total":["counter","Number of times this CPU core has been throttled.","node_exporter"],"node_cpu_frequency_max_hertz":["gauge","Maximum cpu thread frequency in hertz.","node_exporter"],"node_cpu_scaling_frequency_hertz":["gauge","Current scaled CPU thread frequency in hertz.","node_exporter"],"node_cpu_seconds_total":["counter","Seconds the CPUs spent in each mode.","node_exporter"],"node_disk_io_time_seconds_total":["counter","Total seconds spent doing I/Os.","node_exporter"],"node_filefd_allocated":["gauge","File descriptor statistics: allocated.","node_exporter"],"node_filefd_maximum":["gauge","File descriptor statistics: maximum.","node_exporter"],"node_filesystem_avail_bytes":["gauge","Filesystem space available to non-root users in bytes.","node_exporter"],"node_filesystem_device_error":["gauge","Whether an error occurred while getting statistics for the given device.","node_exporter"],"node_filesystem_files":["gauge","Filesystem total file nodes.","node_exporter"],"node_filesystem_files_free":["gauge","Filesystem total free file nodes.","node_exporter"],"node_filesystem_readonly":["gauge","Filesystem read-only status.","node_exporter"],"node_filesystem_size_bytes":["gauge","Filesystem size in bytes.","node_exporter"],"node_hwmon_temp_celsius":["gauge","Hardware monitor for temperature (input)","node_exporter"],"node_hwmon_temp_crit_celsius":["gauge","Hardware monitor for temperature (crit)","node_exporter"],"node_hwmon_temp_max_celsius":["gauge","Hardware monitor for temperature (max)","node_exporter"],"node_md_disks":["gauge","Number of active/failed/spare disks of device.","node_exporter"],"node_md_disks_required":["gauge","Total number of disks of device.","node_exporter"],"node_md_state":["gauge","Indicates the state of md-device.","node_exporter"],"node_memory_Buffers_bytes":["gauge","Memory information field Buffers_bytes.","node_exporter"],"node_memory_Cached_bytes":["gauge","Memory information field Cached_bytes.","node_exporter"],"node_memory_HardwareCorrupted_bytes":["gauge","Memory information field HardwareCorrupted_bytes.","node_exporter"],"node_memory_MemAvailable_bytes":["gauge","Memory information field MemAvailable_bytes.","node_exporter"],"node_memory_MemFree_bytes":["gauge","Memory information field MemFree_bytes.","node_exporter"],"node_memory_MemTotal_bytes":["gauge","Memory information field MemTotal_bytes.","node_exporter"],"node_memory_Slab_bytes":["gauge","Memory information field Slab_bytes.","node_exporter"],"node_memory_SwapFree_bytes":["gauge","Memory information field SwapFree_bytes.","node_exporter"],"node_memory_SwapTotal_bytes":["gauge","Memory information field SwapTotal_bytes.","node_exporter"],"node_netstat_TcpExt_ListenDrops":["untyped","Statistic TcpExtListenDrops.","node_exporter"],"node_netstat_TcpExt_ListenOverflows":["untyped","Statistic TcpExtListenOverflows.","node_exporter"],"node_netstat_Tcp_OutSegs":["untyped","Statistic TcpOutSegs.","node_exporter"],"node_netstat_Tcp_PassiveOpens":["untyped","Statistic TcpPassiveOpens.","node_exporter"],"node_netstat_Tcp_RetransSegs":["untyped","Statistic TcpRetransSegs.","node_exporter"],"node_netstat_Udp_RcvbufErrors":["untyped","Statistic UdpRcvbufErrors.","node_exporter"],"node_network_carrier":["gauge","Network device property: carrier","node_exporter"],"node_network_carrier_changes_total":["counter","Network device property: carrier_changes_total","node_exporter"],"node_network_receive_bytes_total":["counter","Network device statistic receive_bytes.","node_exporter"],"node_network_receive_drop_total":["counter","Network device statistic receive_drop.","node_exporter"],"node_network_receive_errs_total":["counter","Network device statistic receive_errs.","node_exporter"],"node_network_receive_packets_total":["counter","Network device statistic receive_packets.","node_exporter"],"node_network_speed_bytes":["gauge","Network device property: speed_bytes","node_exporter"],"node_network_transmit_bytes_total":["counter","Network device statistic transmit_bytes.","node_exporter"],"node_network_transmit_errs_total":["counter","Network device statistic transmit_errs.","node_exporter"],"node_network_transmit_packets_total":["counter","Network device statistic transmit_packets.","node_exporter"],"node_network_up":["gauge","Value is 1 if operstate is 'up', 0 otherwise.","node_exporter"],"node_nf_conntrack_entries":["gauge","Number of currently allocated flow entries for connection tracking.","node_exporter"],"node_nf_conntrack_entries_limit":["gauge","Maximum size of connection tracking table.","node_exporter"],"node_procs_blocked":["gauge","Number of processes blocked waiting for I/O to complete.","node_exporter"],"node_sockstat_TCP_alloc":["gauge","Number of TCP sockets in state alloc.","node_exporter"],"node_sockstat_TCP_orphan":["gauge","Number of TCP sockets in state orphan.","node_exporter"],"node_softnet_dropped_total":["counter","Number of dropped packets","node_exporter"],"node_systemd_system_running":["gauge","Whether the system is operational (see 'systemctl is-system-running')","node_exporter"],"node_systemd_unit_state":["gauge","Systemd unit","node_exporter"],"node_textfile_mtime_seconds":["gauge","Unixtime mtime of textfiles successfully read.","node_exporter"],"node_textfile_scrape_error":["gauge","1 if there was an error opening or reading a file, 0 otherwise","node_exporter"],"node_timex_offset_seconds":["gauge","Time offset in between local system and reference clock.","node_exporter"],"node_timex_sync_status":["gauge","Is clock synchronized to a reliable server (1 = yes, 0 = no).","node_exporter"],"node_vmstat_oom_kill":["untyped","/proc/vmstat information field oom_kill.","node_exporter"],"node_vmstat_pgfault":["untyped","/proc/vmstat information field pgfault.","node_exporter"],"node_vmstat_pgmajfault":["untyped","/proc/vmstat information field pgmajfault.","node_exporter"],"node_vmstat_pswpin":["untyped","/proc/vmstat information field pswpin.","node_exporter"],"node_vmstat_pswpout":["untyped","/proc/vmstat information field pswpout.","node_exporter"],"server_metrics":["untyped","Metric read from /home/node_exporter/server_metrics.prom","node_exporter"],"nvme_avail_spare":["gauge","Normalized percentage of remaining spare capacity available","nvme_exporter"],"nvme_critical_warning":["gauge","Critical warnings for the state of the controller","nvme_exporter"],"nvme_media_errors":["counter","Number of unrecovered data integrity errors","nvme_exporter"],"nvme_num_err_log_entries":["counter","Lifetime number of error log entries","nvme_exporter"],"nvme_percent_used":["gauge","Vendor specific estimate of the percentage of life used","nvme_exporter"],"nvme_spare_thresh":["gauge","Async event completion may occur when avail spare < threshold","nvme_exporter"],"nvme_temperature":["gauge","Temperature in degrees fahrenheit","nvme_exporter"],"nvme_unsafe_shutdowns":["counter","Number of unsafe shutdowns","nvme_exporter"],"phpfpm_accepted_connections":["counter","The number of requests accepted by the pool.","phpfpm_exporter"],"phpfpm_active_processes":["gauge","The number of active processes.","phpfpm_exporter"],"phpfpm_listen_queue":["gauge","The number of requests in the queue of pending connections.","phpfpm_exporter"],"phpfpm_listen_queue_length":["gauge","The size of the socket queue of pending connections.","phpfpm_exporter"],"phpfpm_max_children_reached":["counter","The number of times, the process limit has been reached, when pm tries to start more children (works only for pm 'dynamic' and 'ondemand').","phpfpm_exporter"],"phpfpm_scrape_failures":["counter","The number of failures scraping from PHP-FPM.","phpfpm_exporter"],"phpfpm_slow_requests":["counter","The number of requests that exceeded your 'request_slowlog_timeout' value.","phpfpm_exporter"],"phpfpm_total_processes":["gauge","The number of idle + active processes.","phpfpm_exporter"],"phpfpm_up":["gauge","Could PHP-FPM be reached?","phpfpm_exporter"],"ping_loss_ratio":["gauge","Packet loss from 0.0 to 1.0","ping_exporter"],"ping_rtt_mean_ms":["gauge","Mean round trip time in millis (deprecated)","ping_exporter"],"ping_up":["gauge","ping_exporter version","ping_exporter"],"pg_database_size_bytes":["gauge","Disk space used by the database","postgres_exporter"],"pg_exporter_last_scrape_error":["gauge","Whether the last scrape of metrics from PostgreSQL resulted in an error (1 for error, 0 for success).","postgres_exporter"],"pg_locks_count":["gauge","Number of locks","postgres_exporter"],"pg_replication_is_replica":["gauge","Indicates if the server is a replica","postgres_exporter"],"pg_replication_lag_seconds":["gauge","Replication lag behind master in seconds","postgres_exporter"],"pg_replication_slots_active":["gauge","Flag indicating if the slot is active","postgres_exporter"],"pg_settings_autovacuum_vacuum_threshold":["gauge","Server Parameter: autovacuum_vacuum_threshold","postgres_exporter"],"pg_settings_max_connections":["gauge","Server Parameter: max_connections","postgres_exporter"],"pg_settings_max_locks_per_transaction":["gauge","Server Parameter: max_locks_per_transaction","postgres_exporter"],"pg_settings_superuser_reserved_connections":["gauge","Server Parameter: superuser_reserved_connections","postgres_exporter"],"pg_stat_activity_count":["gauge","number of connections in this state","postgres_exporter"],"pg_stat_activity_max_tx_duration":["gauge","max duration in seconds any active transaction has been running","postgres_exporter"],"pg_stat_archiver_failed_count":["counter","Number of failed attempts for archiving WAL files","postgres_exporter"],"pg_stat_archiver_last_archive_age":["gauge","Time in seconds since last WAL segment was successfully archived","postgres_exporter"],"pg_stat_bgwriter_buffers_backend_fsync_total":["counter","Number of times a backend had to execute its own fsync call (normally the background writer handles those even when the backend does its own write)","postgres_exporter"],"pg_stat_database_conflicts":["counter","Number of queries canceled due to conflicts with recovery in this database. (Conflicts occur only on standby servers; see pg_stat_database_conflicts for details.)","postgres_exporter"],"pg_stat_database_deadlocks":["counter","Number of deadlocks detected in this database","postgres_exporter"],"pg_stat_database_numbackends":["gauge","Number of backends currently connected to this database. This is the only column in this view that returns a value reflecting current state; all other columns return the accumulated values since the last reset.","postgres_exporter"],"pg_stat_database_temp_bytes":["counter","Total amount of data written to temporary files by queries in this database. All temporary files are counted, regardless of why the temporary file was created, and regardless of the log_temp_files setting.","postgres_exporter"],"pg_stat_database_xact_commit":["counter","Number of transactions in this database that have been committed","postgres_exporter"],"pg_stat_database_xact_rollback":["counter","Number of transactions in this database that have been rolled back","postgres_exporter"],"pg_stat_user_tables_last_autovacuum":["gauge","Last time at which this table was vacuumed by the autovacuum daemon","postgres_exporter"],"pg_stat_user_tables_n_dead_tup":["gauge","Estimated number of dead rows","postgres_exporter"],"pg_stat_user_tables_n_live_tup":["gauge","Estimated number of live rows","postgres_exporter"],"pg_stat_user_tables_n_tup_del":["counter","Number of rows deleted","postgres_exporter"],"pg_stat_user_tables_n_tup_hot_upd":["counter","Number of rows HOT updated (i.e., with no separate index update required)","postgres_exporter"],"pg_stat_user_tables_n_tup_upd":["counter","Number of rows updated","postgres_exporter"],"pg_up":["gauge","Whether the last scrape of metrics from PostgreSQL was able to connect to the server (1 for yes, 0 for no).","postgres_exporter"],"pg_replication_lag":["gauge","Replication lag behind master in seconds","postgres_exporter"],"pg_stat_database_checksum_failures":["untyped","Unknown metric from pg_stat_database","postgres_exporter"],"prometheus_config_last_reload_successful":["gauge","Whether the last configuration reload attempt was successful.","prometheus"],"prometheus_engine_queries":["gauge","The current number of queries being executed or waiting.","prometheus"],"prometheus_engine_queries_concurrent_max":["gauge","The max number of concurrent queries.","prometheus"],"prometheus_notifications_alertmanagers_discovered":["gauge","The number of alertmanagers discovered and active.","prometheus"],"prometheus_notifications_dropped_total":["counter","Total number of alerts dropped due to errors when sending to Alertmanager.","prometheus"],"prometheus_notifications_errors_total":["counter","Total number of sent alerts affected by errors.","prometheus"],"prometheus_notifications_queue_length":["gauge","The number of alert notifications in the queue.","prometheus"],"prometheus_notifications_sent_total":["counter","Total number of alerts sent.","prometheus"],"prometheus_remote_storage_highest_timestamp_in_seconds":["gauge","Highest timestamp that has come into the remote storage via the Appender interface, in seconds since epoch. Initialized to 0 when no data has been received yet. Deprecated, check prometheus_remote_storage_queue_highest_timestamp_seconds which is more accurate.","prometheus"],"prometheus_remote_storage_queue_highest_sent_timestamp_seconds":["gauge","Highest timestamp successfully sent by this queue, in seconds since epoch. Initialized to 0 when no data has been sent yet.","prometheus"],"prometheus_remote_storage_samples_dropped_total":["counter","Total number of samples which were dropped after being read from the WAL before being sent via remote write, either via relabelling, due to being too old or unintentionally because of an unknown reference ID.","prometheus"],"prometheus_remote_storage_samples_failed_total":["counter","Total number of samples which failed on send to remote storage, non-recoverable errors.","prometheus"],"prometheus_remote_storage_shards":["gauge","The number of shards used for parallel sending to the remote storage.","prometheus"],"prometheus_remote_storage_shards_max":["gauge","The maximum number of shards that the queue is allowed to run.","prometheus"],"prometheus_rule_evaluation_failures_total":["counter","The total number of rule evaluation failures.","prometheus"],"prometheus_rule_evaluations_total":["counter","The total number of rule evaluations.","prometheus"],"prometheus_rule_group_interval_seconds":["gauge","The interval of a rule group.","prometheus"],"prometheus_rule_group_iterations_missed_total":["counter","The total number of rule group evaluations missed due to slow rule group evaluation.","prometheus"],"prometheus_rule_group_last_duration_seconds":["gauge","The duration of the last rule group evaluation.","prometheus"],"prometheus_sd_discovered_targets":["gauge","Current number of discovered targets.","prometheus"],"prometheus_sd_failed_configs":["gauge","Current number of service discovery configurations that failed to load.","prometheus"],"prometheus_target_interval_length_seconds":["summary","Actual intervals between scrapes.","prometheus"],"prometheus_target_interval_length_seconds_sum":["","","prometheus"],"prometheus_target_interval_length_seconds_count":["","","prometheus"],"prometheus_target_scrape_pool_exceeded_target_limit_total":["counter","Total number of times scrape pools hit the target limit, during sync or config reload.","prometheus"],"prometheus_target_scrapes_exceeded_body_size_limit_total":["counter","Total number of scrapes that hit the body size limit","prometheus"],"prometheus_target_scrapes_exceeded_sample_limit_total":["counter","Total number of scrapes that hit the sample limit and were rejected.","prometheus"],"prometheus_target_scrapes_sample_duplicate_timestamp_total":["counter","Total number of samples rejected due to duplicate timestamps but different values.","prometheus"],"prometheus_template_text_expansion_failures_total":["counter","The total number of template text expansion failures.","prometheus"],"prometheus_tsdb_checkpoint_creations_failed_total":["counter","Total number of checkpoint creations that failed.","prometheus"],"prometheus_tsdb_checkpoint_deletions_failed_total":["counter","Total number of checkpoint deletions that failed.","prometheus"],"prometheus_tsdb_compactions_failed_total":["counter","Total number of compactions that failed for the partition.","prometheus"],"prometheus_tsdb_head_min_time":["gauge","Minimum time bound of the head block. The unit is decided by the library consumer.","prometheus"],"prometheus_tsdb_head_truncations_failed_total":["counter","Total number of head truncations that failed.","prometheus"],"prometheus_tsdb_mmap_chunk_corruptions_total":["counter","Total number of memory-mapped chunk corruptions.","prometheus"],"prometheus_tsdb_out_of_bound_samples_total":["counter","Total number of out of bound samples ingestion failed attempts with out of order support disabled.","prometheus"],"prometheus_tsdb_out_of_order_samples_total":["counter","Total number of out of order samples ingestion failed attempts due to out of order being disabled.","prometheus"],"prometheus_tsdb_reloads_failures_total":["counter","Number of times the database failed to reloadBlocks block data from disk.","prometheus"],"prometheus_tsdb_wal_corruptions_total":["counter","Total number of WAL corruptions.","prometheus"],"prometheus_tsdb_wal_truncations_failed_total":["counter","Total number of write log truncations that failed.","prometheus"],"prometheus_tsdb_wal_writes_failed_total":["counter","Total number of write log writes that failed.","prometheus"],"promtail_batch_retries_total":["counter","Number of times batches has had to be retried.","promtail_exporter"],"promtail_config_reload_fail_total":["counter","Number of reload fail times.","promtail_exporter"],"promtail_dropped_entries_total":["counter","Number of log entries dropped because failed to be sent to the ingester after all retries.","promtail_exporter"],"promtail_request_duration_seconds_bucket":["histogram","Duration of send requests.","promtail_exporter"],"promtail_targets_active_total":["gauge","Number of active total.","promtail_exporter"],"rabbitmq_process_open_fds":["gauge","Open file descriptors","rabbitmq_exporter"],"rabbitmq_process_resident_memory_bytes":["gauge","Memory used in bytes","rabbitmq_exporter"],"rabbitmq_erlang_processes_used":["gauge","Erlang processes used","rabbitmq_exporter"],"rabbitmq_process_max_fds":["gauge","Open file descriptors limit","rabbitmq_exporter"],"rabbitmq_resident_memory_limit_bytes":["gauge","Memory high watermark in bytes","rabbitmq_exporter"],"rabbitmq_erlang_processes_limit":["gauge","Erlang processes limit","rabbitmq_exporter"],"rabbitmq_queue_messages_ready":["gauge","Messages ready to be delivered to consumers","rabbitmq_exporter"],"rabbitmq_queue_messages_unacked":["gauge","Messages delivered to consumers but not yet acknowledged","rabbitmq_exporter"],"rabbitmq_alarms_free_disk_space_watermark":["untyped","is 1 if free disk space watermark alarm is in effect","rabbitmq_exporter"],"rabbitmq_alarms_memory_used_watermark":["untyped","is 1 if VM memory watermark alarm is in effect","rabbitmq_exporter"],"node_cpu_average":["gauge","The percentage of CPU utilization.","rds_enhanced"],"rdsosmetrics_tasks_blocked":["gauge","The number of tasks that are blocked.","rds_enhanced"],"stackdriver_monitoring_api_calls_total":["counter","Total number of Google Stackdriver Monitoring API calls made.","stackdriver_exporter"],"stackdriver_monitoring_last_scrape_duration_seconds":["gauge","Duration of the last metrics scrape from Google Stackdriver Monitoring.","stackdriver_exporter"],"stackdriver_monitoring_last_scrape_error":["gauge","Whether the last metrics scrape from Google Stackdriver Monitoring resulted in an error (1 for error, 0 for success).","stackdriver_exporter"],"stackdriver_monitoring_last_scrape_timestamp":["gauge","Number of seconds since 1970 since last metrics scrape from Google Stackdriver Monitoring.","stackdriver_exporter"],"stackdriver_monitoring_scrape_errors_total":["counter","Total number of Google Stackdriver Monitoring metrics scrape errors.","stackdriver_exporter"],"strimzi_certificate_expiration_timestamp_ms":["gauge","Time in milliseconds when the certificate expires","strimzi"],"strimzi_reconciliations_successful_total":["counter","Number of reconciliations done by the operator for individual resources which were successful","strimzi"],"strimzi_resources":["gauge","Number of custom resources the operator sees","strimzi"],"thanos_status":["gauge","Represents status (0 indicates failure, 1 indicates success) of the component.","thanos"],"http_requests_total":["counter","Tracks the number of HTTP requests.","thanos"],"http_request_duration_seconds_bucket":["histogram","Tracks the latencies for HTTP requests.","thanos"],"grpc_server_started_total":["counter","Total number of RPCs started on the server.","thanos"],"grpc_server_handled_total":["counter","Total number of RPCs completed on the server, regardless of success or failure.","thanos"],"grpc_client_started_total":["counter","Total number of RPCs started on the client.","thanos"],"grpc_client_handled_total":["counter","Total number of RPCs completed by the client, regardless of success or failure.","thanos"],"thanos_objstore_bucket_operations_total":["counter","Total number of all attempted operations against a bucket.","thanos"],"thanos_objstore_bucket_operation_failures_total":["counter","Total number of operations against a bucket that failed, but were not expected to fail in certain way from caller perspective. Those errors have to be investigated.","thanos"],"thanos_blocks_meta_sync_failures_total":["counter","Total blocks metadata synchronization failures","thanos"],"thanos_bucket_store_block_load_failures_total":["counter","Total number of failed remote block loading attempts.","thanos"],"thanos_receive_forward_requests_total":["counter","The number of forward requests.","thanos"],"thanos_receive_config_last_reload_successful":["gauge","Whether the last hashring configuration file reload attempt was successful.","thanos"],"thanos_shipper_uploads_total":["counter","Total number of uploaded blocks","thanos"],"thanos_shipper_upload_failures_total":["counter","Total number of block upload failures","thanos"],"thanos_shipper_corrupted_blocks_total":["counter","Total number of corrupted blocks","thanos"],"thanos_alert_queue_alerts_dropped_total":["counter","Total number of alerts that were dropped from the queue.","thanos"],"thanos_alert_sender_alerts_dropped_total":["counter","Total number of alerts dropped in case of all sends to alertmanagers failed.","thanos"],"thanos_rule_config_last_reload_successful":["gauge","Whether the last configuration reload attempt was successful.","thanos"],"thanos_rule_loaded_rules":["gauge","Loaded rules partitioned by file and group.","thanos"],"thanos_compact_halted":["gauge","Set to 1 if the compactor halted due to an unexpected error.","thanos"],"varnish_main_backend_fail":["counter","Backend conn. failures","varnish_exporter"],"varnish_main_backend_unhealthy":["counter","Backend conn. not attempted","varnish_exporter"],"varnish_main_cache_hit":["counter","Cache hits","varnish_exporter"],"varnish_main_cache_miss":["counter","Cache misses","varnish_exporter"],"varnish_main_n_lru_nuked":["counter","Number of LRU nuked objects","varnish_exporter"],"varnish_main_threads_limited":["counter","Threads hit max","varnish_exporter"],"varnish_up":["gauge","Was the last scrape of varnish successful.","varnish_exporter"],"zookeeper_MaxRequestLatency":["gauge","org.apache.ZooKeeperService:name=null,type=null,attribute=MaxRequestLatency","zookeeper_exporter"],"zookeeper_AvgRequestLatency":["gauge","org.apache.ZooKeeperService:name=null,type=null,attribute=AvgRequestLatency","zookeeper_exporter"],"zookeeper_Leader":["gauge","org.apache.ZooKeeperService:name=null,type=null,attribute=Leader","zookeeper_exporter"],"zookeeper_OutstandingRequests":["gauge","org.apache.ZooKeeperService:name=null,type=null,attribute=OutstandingRequests","zookeeper_exporter"],"zookeeper_QuorumSize":["gauge","org.apache.ZooKeeperService:name=null,type=null,attribute=QuorumSize","zookeeper_exporter"]}}