diff options
Diffstat (limited to 'terraform-ci-infra/1n_nmd/alertmanager')
3 files changed, 87 insertions, 42 deletions
diff --git a/terraform-ci-infra/1n_nmd/alertmanager/conf/nomad/alertmanager.hcl b/terraform-ci-infra/1n_nmd/alertmanager/conf/nomad/alertmanager.hcl index fafe623c85..40d84e337a 100644 --- a/terraform-ci-infra/1n_nmd/alertmanager/conf/nomad/alertmanager.hcl +++ b/terraform-ci-infra/1n_nmd/alertmanager/conf/nomad/alertmanager.hcl @@ -160,10 +160,6 @@ job "${job_name}" { left_delimiter = "{{{" right_delimiter = "}}}" data = <<EOH -global: - # The API URL to use for Slack notifications. - slack_api_url: 'https://hooks.slack.com/services/${slack_api_key}' - # The directory from which notification templates are read. templates: - '/etc/alertmanager/template/*.tmpl' @@ -185,7 +181,7 @@ templates: # The root route on which each incoming alert enters. route: - receiver: '${default_receiver}' + receiver: '${slack_default_receiver}' # The labels by which incoming alerts are grouped together. For example, # multiple alerts coming in for cluster=A and alertname=LatencyHigh would @@ -217,18 +213,21 @@ route: # overwritten on each. # The child route trees. routes: - # This routes performs a regular expression match on alert labels to - # catch alerts that are related to a list of services. + - match_re: + alertname: JenkinsJob.* + receiver: ${slack_jenkins_receiver} + routes: + - match: + severity: critical + receiver: '${slack_jenkins_receiver}' + - match_re: service: .* - receiver: ${default_receiver} - # The service has a sub-route for critical alerts, any alerts - # that do not match, i.e. severity != critical, fall-back to the - # parent node and are sent to 'team-X-mails' + receiver: ${slack_default_receiver} routes: - match: severity: critical - receiver: '${default_receiver}' + receiver: '${slack_default_receiver}' # Inhibition rules allow to mute a set of alerts given that another alert is # firing. @@ -239,17 +238,42 @@ inhibit_rules: severity: 'critical' target_match: severity: 'warning' - # Apply inhibition if the alertname is the same. - # CAUTION: - # If all label names listed in `equal` are missing - # from both the source and target alerts, - # the inhibition rule will apply! - equal: ['alertname', 'cluster', 'service'] + equal: ['alertname', 'instance'] receivers: -- name: '${default_receiver}' +- name: '${slack_jenkins_receiver}' + slack_configs: + - api_url: 'https://hooks.slack.com/services/${slack_jenkins_api_key}' + channel: '#${slack_jenkins_channel}' + send_resolved: true + icon_url: https://avatars3.githubusercontent.com/u/3380462 + title: |- + [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }} + {{- if gt (len .CommonLabels) (len .GroupLabels) -}} + {{" "}}( + {{- with .CommonLabels.Remove .GroupLabels.Names }} + {{- range $index, $label := .SortedPairs -}} + {{ if $index }}, {{ end }} + {{- $label.Name }}="{{ $label.Value -}}" + {{- end }} + {{- end -}} + ) + {{- end }} + text: >- + {{ range .Alerts -}} + *Alert:* {{ .Annotations.summary }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }} + + *Description:* {{ .Annotations.description }} + + *Details:* + {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}` + {{ end }} + {{ end }} + +- name: '${slack_default_receiver}' slack_configs: - - channel: '#${slack_channel}' + - api_url: 'https://hooks.slack.com/services/${slack_default_api_key}' + channel: '#${slack_default_channel}' send_resolved: true icon_url: https://avatars3.githubusercontent.com/u/3380462 title: |- diff --git a/terraform-ci-infra/1n_nmd/alertmanager/main.tf b/terraform-ci-infra/1n_nmd/alertmanager/main.tf index 8307601669..9525aabc0c 100644 --- a/terraform-ci-infra/1n_nmd/alertmanager/main.tf +++ b/terraform-ci-infra/1n_nmd/alertmanager/main.tf @@ -14,20 +14,23 @@ locals { data "template_file" "nomad_job_alertmanager" { template = file("${path.module}/conf/nomad/alertmanager.hcl") vars = { - datacenters = local.datacenters - url = local.alertmanager_url - job_name = var.alertmanager_job_name - use_canary = var.alertmanager_use_canary - group_count = var.alertmanager_group_count - service_name = var.alertmanager_service_name - use_vault_provider = var.alertmanager_vault_secret.use_vault_provider - version = var.alertmanager_version - cpu = var.alertmanager_cpu - mem = var.alertmanager_mem - port = var.alertmanager_port - slack_api_key = var.alertmanager_slack_api_key - slack_channel = var.alertmanager_slack_channel - default_receiver = var.alertmanager_default_receiver + datacenters = local.datacenters + url = local.alertmanager_url + job_name = var.alertmanager_job_name + use_canary = var.alertmanager_use_canary + group_count = var.alertmanager_group_count + service_name = var.alertmanager_service_name + use_vault_provider = var.alertmanager_vault_secret.use_vault_provider + version = var.alertmanager_version + cpu = var.alertmanager_cpu + mem = var.alertmanager_mem + port = var.alertmanager_port + slack_jenkins_api_key = var.alertmanager_slack_jenkins_api_key + slack_jenkins_channel = var.alertmanager_slack_jenkins_channel + slack_jenkins_receiver = var.alertmanager_slack_jenkins_receiver + slack_default_api_key = var.alertmanager_slack_default_api_key + slack_default_channel = var.alertmanager_slack_default_channel + slack_default_receiver = var.alertmanager_slack_default_receiver } } diff --git a/terraform-ci-infra/1n_nmd/alertmanager/variables.tf b/terraform-ci-infra/1n_nmd/alertmanager/variables.tf index 152530b96d..ffedf24f3d 100644 --- a/terraform-ci-infra/1n_nmd/alertmanager/variables.tf +++ b/terraform-ci-infra/1n_nmd/alertmanager/variables.tf @@ -65,20 +65,38 @@ variable "alertmanager_port" { default = 9093 } -variable "alertmanager_default_receiver" { - description = "Alertmanager default receiver" +variable "alertmanager_slack_jenkins_api_key" { + description = "Alertmanager jenkins slack API key" type = string - default = "default-receiver" + default = "XXXXXXXXX/XXXXXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXX" +} + +variable "alertmanager_slack_jenkins_receiver" { + description = "Alertmanager jenkins slack receiver" + type = string + default = "jenkins-slack-receiver" +} + +variable "alertmanager_slack_jenkins_channel" { + description = "Alertmanager jenkins slack channel" + type = string + default = "jenkins-channel" } -variable "alertmanager_slack_api_key" { - description = "Alertmanager slack API key" +variable "alertmanager_slack_default_api_key" { + description = "Alertmanager default slack API key" type = string default = "XXXXXXXXX/XXXXXXXXXXX/XXXXXXXXXXXXXXXXXXXXXXXX" } -variable "alertmanager_slack_channel" { - description = "Alertmanager slack channel" +variable "alertmanager_slack_default_receiver" { + description = "Alertmanager default slack receiver" + type = string + default = "default-slack-receiver" +} + +variable "alertmanager_slack_default_channel" { + description = "Alertmanager default slack channel" type = string - default = "slack-channel" + default = "default-channel" }
\ No newline at end of file |