MSF-17779: Monitor undesired LCM pod status

In case LCM pods can start due to insufficient resources

Author: Chien Minh Do

k8s/charts/lcm-bricks/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v1
 name: lcm-bricks
 description: LCM Bricks
-version: 2.0.3
+version: 2.0.6

k8s/charts/lcm-bricks/templates/prometheus/alertingRules.yaml

@@ -92,4 +92,14 @@ data:
         annotations:
           description: "We are hitting CPU limit in LCM namespace."
           summary: "We are hitting CPU limit in LCM namespace."
-
+      - alert: "[LCM] POD is in undesirable state on cluster={{ .Values.clusterId }}"
+        expr: kube_pod_status_phase{namespace='{{ .Release.Namespace }}', phase!~"Running|Succeeded|Failed"} > 0
+        for: 5m
+        labels:
+          cluster_id: {{ .Values.clusterId }}
+          severity: critical
+          team: lcm
+        annotations:
+          description: "POD {{`{{ $labels.pod }}`}} is not in desirable state"
+          summary: "POD is not in desirable state"
+          runbook: "https://confluence.intgdc.com/display/plat/Generic+runbooks#Genericrunbooks-Podisinundesirablestate"