Infra: JenkinsJobHealthExporter

- Integration of Jenkins Job checker

Signed-off-by: pmikus <pmikus@cisco.com>
Change-Id: I822039cb64a3a352b49314ddab7c6099af3fe644

3 files changed, 87 insertions, 42 deletions

diff --git a/terraform-ci-infra/1n_nmd/alertmanager/conf/nomad/alertmanager.hcl b/terraform-ci-infra/1n_nmd/alertmanager/conf/nomad/alertmanager.hcl
index fafe623c85..40d84e337a 100644
--- a/terraform-ci-infra/1n_nmd/alertmanager/conf/nomad/alertmanager.hcl
+++ b/terraform-ci-infra/1n_nmd/alertmanager/conf/nomad/alertmanager.hcl
@@ -160,10 +160,6 @@ job "${job_name}" {
         left_delimiter  = "{{{"
         right_delimiter = "}}}"
         data            = <<EOH
-global:
-  # The API URL to use for Slack notifications.
-  slack_api_url: 'https://hooks.slack.com/services/${slack_api_key}'
-
 # The directory from which notification templates are read.
 templates:
 - '/etc/alertmanager/template/*.tmpl'
@@ -185,7 +181,7 @@ templates:
 
 # The root route on which each incoming alert enters.
 route:
-  receiver: '${default_receiver}'
+  receiver: '${slack_default_receiver}'
 
   # The labels by which incoming alerts are grouped together. For example,
   # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
@@ -217,18 +213,21 @@ route:
   # overwritten on each.
   # The child route trees.
   routes:
-  # This routes performs a regular expression match on alert labels to
-  # catch alerts that are related to a list of services.
+  - match_re:
+      alertname: JenkinsJob.*
+    receiver: ${slack_jenkins_receiver}
+    routes:
+    - match:
+        severity: critical
+      receiver: '${slack_jenkins_receiver}'
+
   - match_re:
       service: .*
-    receiver: ${default_receiver}
-    # The service has a sub-route for critical alerts, any alerts
-    # that do not match, i.e. severity != critical, fall-back to the
-    # parent node and are sent to 'team-X-mails'
+    receiver: ${slack_default_receiver}
     routes:
     - match:
         severity: critical
-      receiver: '${default_receiver}'
+      receiver: '${slack_default_receiver}'
 
 # Inhibition rules allow to mute a set of alerts given that another alert is
 # firing.
@@ -239,17 +238,42 @@ inhibit_rules:
     severity: 'critical'
   target_match:
     severity: 'warning'
-  # Apply inhibition if the alertname is the same.
-  # CAUTION:
-  #   If all label names listed in `equal` are missing
-  #   from both the source and target alerts,
-  #   the inhibition rule will apply!
-  equal: ['alertname', 'cluster', 'service']
+  equal: ['alertname', 'instance']
 
 receivers:
-- name: '${default_receiver}'
+- name: '${slack_jenkins_receiver}'
+  slack_configs:
+  - api_url: 'https://hooks.slack.com/services/${slack_jenkins_api_key}'
+    channel: '#${slack_jenkins_channel}'
+    send_resolved: true
+    icon_url: https://avatars3.githubusercontent.com/u/3380462
+    title: |-
+     [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}
+     {{- if gt (len .CommonLabels) (len .GroupLabels) -}}
+       {{" "}}(
+       {{- with .CommonLabels.Remove .GroupLabels.Names }}
+         {{- range $index, $label := .SortedPairs -}}
+           {{ if $index }}, {{ end }}
+           {{- $label.Name }}="{{ $label.Value -}}"
+         {{- end }}
+       {{- end -}}
+       )
+     {{- end }}
+    text: >-
+     {{ range .Alerts -}}
+     *Alert:* {{ .Annotations.summary }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }}
+
+     *Description:* {{ .Annotations.description }}
+
+     *Details:*
+       {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}`
+       {{ end }}
+     {{ end }}
+
+- name: '${slack_default_receiver}'
   slack_configs:
-  - channel: '#${slack_channel}'
+  - api_url: 'https://hooks.slack.com/services/${slack_default_api_key}'
+    channel: '#${slack_default_channel}'
     send_resolved: true
     icon_url: https://avatars3.githubusercontent.com/u/3380462
     title: |-
diff --git a/terraform-ci-infra/1n_nmd/alertmanager/main.tf b/terraform-ci-infra/1n_nmd/alertmanager/main.tf
index 8307601669..9525aabc0c 100644
--- a/terraform-ci-infra/1n_nmd/alertmanager/main.tf
+++ b/terraform-ci-infra/1n_nmd/alertmanager/main.tf
@@ -14,20 +14,23 @@ locals {
 data "template_file" "nomad_job_alertmanager" {
   template         = file("${path.module}/conf/nomad/alertmanager.hcl")
   vars             = {
-    datacenters        = local.datacenters
-    url                = local.alertmanager_url
-    job_name           = var.alertmanager_job_name
-    use_canary         = var.alertmanager_use_canary
-    group_count        = var.alertmanager_group_count
-    service_name       = var.alertmanager_service_name
-    use_vault_provider = var.alertmanager_vault_secret.use_vault_provider
-    version            = var.alertmanager_version
-    cpu                = var.alertmanager_cpu
-    mem                = var.alertmanager_mem
-    port               = var.alertmanager_port
-    slack_api_key      = var.alertmanager_slack_api_key
-    slack_channel      = var.alertmanager_slack_channel
-    default_receiver   = var.alertmanager_default_receiver
+    datacenters            = local.datacenters
+    url                    = local.alertmanager_url
+    job_name               = var.alertmanager_job_name
+    use_canary             = var.alertmanager_use_canary
+    group_count            = var.alertmanager_group_count
+    service_name           = var.alertmanager_service_name
+    use_vault_provider     = var.alertmanager_vault_secret.use_vault_provider
+    version                = var.alertmanager_version
+    cpu                    = var.alertmanager_cpu
+    mem                    = var.alertmanager_mem
+    port                   = var.alertmanager_port
+    slack_jenkins_api_key  = var.alertmanager_slack_jenkins_api_key
+    slack_jenkins_channel  = var.alertmanager_slack_jenkins_channel
+    slack_jenkins_receiver = var.alertmanager_slack_jenkins_receiver
+    slack_default_api_key  = var.alertmanager_slack_default_api_key
+    slack_default_channel  = var.alertmanager_slack_default_channel
+    slack_default_receiver = var.alertmanager_slack_default_receiver
   }
 }
 
diff --git a/terraform-ci-infra/1n_nmd/alertmanager/variables.tf b/terraform-ci-infra/1n_nmd/alertmanager/variables.tf
index 152530b96d..ffedf24f3d 100644
--- a/terraform-ci-infra/1n_nmd/alertmanager/variables.tf
+++ b/terraform-ci-infra/1n_nmd/alertmanager/variables.tf
@@ -65,20 +65,38 @@ variable "alertmanager_port" {
   default     = 9093
 }
 
-variable "alertmanager_default_receiver" {
-  description = "Alertmanager default receiver"
+variable "alertmanager_slack_jenkins_api_key" {
+  description = "Alertmanager jenkins slack API key"
   type        = string
-  default     = "default-receiver"
+  default     = "XXXXXXXXX/XXXXXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXX"
+}
+
+variable "alertmanager_slack_jenkins_receiver" {
+  description = "Alertmanager jenkins slack receiver"
+  type        = string
+  default     = "jenkins-slack-receiver"
+}
+
+variable "alertmanager_slack_jenkins_channel" {
+  description = "Alertmanager jenkins slack channel"
+  type        = string
+  default     = "jenkins-channel"
 }
 
-variable "alertmanager_slack_api_key" {
-  description = "Alertmanager slack API key"
+variable "alertmanager_slack_default_api_key" {
+  description = "Alertmanager default slack API key"
   type        = string
   default     = "XXXXXXXXX/XXXXXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXX"
 }
 
-variable "alertmanager_slack_channel" {
-  description = "Alertmanager slack channel"
+variable "alertmanager_slack_default_receiver" {
+  description = "Alertmanager default slack receiver"
+  type        = string
+  default     = "default-slack-receiver"
+}
+
+variable "alertmanager_slack_default_channel" {
+  description = "Alertmanager default slack channel"
   type        = string
-  default     = "slack-channel"
+  default     = "default-channel"
 }
\ No newline at end of file