diff --git a/k8s/monitoring/values-production.yaml b/k8s/monitoring/values-production.yaml index 81c33d8..4162ed4 100644 --- a/k8s/monitoring/values-production.yaml +++ b/k8s/monitoring/values-production.yaml @@ -161,6 +161,12 @@ kube-prometheus-stack: - match: alertname: WebrtcBeError receiver: 'email-and-slack-notifications' + - match: + alertname: PodOOMKilled + receiver: 'slack-notifications-oom' + - match: + alertname: PodCrashLoopBackOff + receiver: 'slack-notifications' receivers: - name: 'null' - name: 'email-alerts' @@ -172,7 +178,21 @@ kube-prometheus-stack: - api_url: 'https://hooks.slack.com/services/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' channel: '#video-cluster-prd-alerts' send_resolved: true - title: '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] Monitoring Event Notification' + title: '[PRODUCTION] [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] Monitoring Event Notification' + text: >- + {{ range .Alerts }} + *Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}` + *Description:* {{ .Annotations.description }} + *Details:* + {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}` + {{ end }} + {{ end }} + - name: 'slack-notifications-oom' + slack_configs: + - api_url: 'https://hooks.slack.com/services/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' + channel: '#video-cluster-prd-alerts' + send_resolved: false + title: '[PRODUCTION] [OOM KILLED] Monitoring Event Notification' text: >- {{ range .Alerts }} *Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}` diff --git a/k8s/monitoring/values-staging.yaml b/k8s/monitoring/values-staging.yaml index ebb92b3..663a00b 100644 --- a/k8s/monitoring/values-staging.yaml +++ b/k8s/monitoring/values-staging.yaml @@ -172,6 +172,12 @@ kube-prometheus-stack: - match: alertname: WebrtcBeError receiver: 'email-and-slack-notifications' + - match: + alertname: PodOOMKilled + receiver: 'slack-notifications-oom' + - match: + alertname: PodCrashLoopBackOff + receiver: 'slack-notifications' receivers: - name: 'null' - name: 'email-alerts' @@ -183,7 +189,21 @@ kube-prometheus-stack: - api_url: 'https://hooks.slack.com/services/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' channel: '#video-cluster-staging-alerts' send_resolved: true - title: '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] Monitoring Event Notification' + title: '[STAGING] [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] Monitoring Event Notification' + text: >- + {{ range .Alerts }} + *Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}` + *Description:* {{ .Annotations.description }} + *Details:* + {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}` + {{ end }} + {{ end }} + - name: 'slack-notifications-oom' + slack_configs: + - api_url: 'https://hooks.slack.com/services/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' + channel: '#video-cluster-staging-alerts' + send_resolved: false + title: '[STAGING] [OOM KILLED] Monitoring Event Notification' text: >- {{ range .Alerts }} *Alert:* {{ .Annotations.summary }} - `{{ .Labels.severity }}`