Infra: Backend infra upgrade

Signed-off-by: pmikus <pmikus@cisco.com>
Change-Id: If1e659339f0d25ebcaab4388745c62aa0852abb3

16 files changed, 151 insertions, 103 deletions

diff --git a/fdio.infra.ansible/roles/consul/defaults/main.yaml b/fdio.infra.ansible/roles/consul/defaults/main.yaml
index 786554eb58..b875c88c74 100644
--- a/fdio.infra.ansible/roles/consul/defaults/main.yaml
+++ b/fdio.infra.ansible/roles/consul/defaults/main.yaml
@@ -3,14 +3,9 @@
 
 # Inst - Prerequisites.
 packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}"
-
 packages_base:
-  - "cgroup-bin"
   - "curl"
-  - "git"
-  - "libcgroup1"
   - "unzip"
-  - "htop"
 packages_by_distro:
   ubuntu:
     - []
@@ -20,7 +15,7 @@ packages_by_arch:
   x86_64:
     - []
 
-# Inst - Download Consul.
+# Inst - Consul Map.
 consul_architecture_map:
   amd64: "amd64"
   x86_64: "amd64"
@@ -29,7 +24,7 @@ consul_architecture_map:
   32-bit: "386"
   64-bit: "amd64"
 consul_architecture: "{{ consul_architecture_map[ansible_architecture] }}"
-consul_version: "1.8.6"
+consul_version: "1.9.5"
 consul_pkg: "consul_{{ consul_version }}_linux_{{ consul_architecture }}.zip"
 consul_zip_url: "https://releases.hashicorp.com/consul/{{ consul_version }}/{{ consul_pkg }}"
 
@@ -52,20 +47,17 @@ systemd_resolved_state: "stopped"
 # Conf - User and group.
 consul_group: "consul"
 consul_group_state: "present"
-consul_manage_group: true
-consul_manage_user: true
 consul_user: "consul"
-consul_user_groups: [ docker, nomad, consul, root ]
 consul_user_state: "present"
 
 # Conf - nomad.d/consul.hcl
 consul_nomad_integration: true
 consul_certificates:
-  - src: "{{ vault_consul_v1_ca_file }}"
+  - src: "{{ file_consul_ca_pem }}"
     dest: "{{ consul_ca_file }}"
-  - src: "{{ vault_consul_v1_cert_file }}"
+  - src: "{{ file_consul_server_0_pem }}"
     dest: "{{ consul_cert_file }}"
-  - src: "{{ vault_consul_v1_key_file }}"
+  - src: "{{ file_consul_server_0_key_pem }}"
     dest: "{{ consul_key_file }}"
 
 consul_auto_advertise: true
@@ -92,7 +84,8 @@ consul_encrypt: ""
 consul_ca_file: "{{ consul_ssl_dir }}/ca.pem"
 consul_cert_file: "{{ consul_ssl_dir }}/consul.pem"
 consul_key_file: "{{ consul_ssl_dir }}/consul-key.pem"
-consul_ui: true
+consul_ui_config:
+  enabled: true
 consul_recursors:
   - 1.1.1.1
   - 8.8.8.8
@@ -107,4 +100,4 @@ consul_port_serf_wan: 8302
 consul_port_server: 8300
 
 # Conf - services.json
-consul_services: false
\ No newline at end of file
+consul_services: false
diff --git a/fdio.infra.ansible/roles/consul/handlers/main.yaml b/fdio.infra.ansible/roles/consul/handlers/main.yaml
index 338baea74e..d0e0c598a9 100644
--- a/fdio.infra.ansible/roles/consul/handlers/main.yaml
+++ b/fdio.infra.ansible/roles/consul/handlers/main.yaml
@@ -14,10 +14,3 @@
     enabled: true
     name: "consul"
     state: "{{ consul_restart_handler_state }}"
-
-- name: Stop Systemd-resolved
-  systemd:
-    daemon_reload: true
-    enabled: false
-    name: "systemd-resolved"
-    state: "{{ systemd_resolved_state }}"
\ No newline at end of file
diff --git a/fdio.infra.ansible/roles/consul/meta/main.yaml b/fdio.infra.ansible/roles/consul/meta/main.yaml
index 4ada8efad6..5fb7e185c0 100644
--- a/fdio.infra.ansible/roles/consul/meta/main.yaml
+++ b/fdio.infra.ansible/roles/consul/meta/main.yaml
@@ -7,3 +7,16 @@
 # info: 1.0 - added role
 
 dependencies: [ ]
+galaxy_info:
+  role_name: consul
+  author: fd.io
+  description: Hashicrop Consul.
+  company: none
+  license: "license (Apache)"
+  min_ansible_version: 2.9
+  platforms:
+    - name: Ubuntu
+      versions:
+        - focal
+  galaxy_tags:
+    - consul
diff --git a/fdio.infra.ansible/roles/consul/tasks/main.yaml b/fdio.infra.ansible/roles/consul/tasks/main.yaml
index 99ac52da44..834d7f1798 100644
--- a/fdio.infra.ansible/roles/consul/tasks/main.yaml
+++ b/fdio.infra.ansible/roles/consul/tasks/main.yaml
@@ -3,7 +3,7 @@
 
 - name: Inst - Update Package Cache (APT)
   apt:
-    update_cache: yes
+    update_cache: true
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
@@ -21,8 +21,6 @@
   group:
     name: "{{ consul_group }}"
     state: "{{ consul_group_state }}"
-  when:
-    - consul_manage_group | bool
   tags:
     - consul-conf-user
 
@@ -30,11 +28,8 @@
   user:
     name: "{{ consul_user }}"
     group: "{{ consul_group }}"
-    groups: "{{ consul_user_groups }}"
     state: "{{ consul_user_state }}"
     system: true
-  when:
-    - consul_manage_user | bool
   tags:
     - consul-conf-user
 
@@ -167,6 +162,15 @@
   tags:
     - consul-conf
 
+- name: Conf - Stop Systemd-resolved
+  systemd:
+    daemon_reload: true
+    enabled: false
+    name: "systemd-resolved"
+    state: "{{ systemd_resolved_state }}"
+  tags:
+    - consul-conf
+
 - name: Conf - System.d Script
   template:
     src: "consul_systemd.service.j2"
@@ -174,9 +178,10 @@
     owner: "root"
     group: "root"
     mode: 0644
-#  notify:
-#    - "Restart Consul"
-#    - "Stop Systemd-resolved"
-#    - "Restart Nomad"
+  notify:
+    - "Restart Consul"
+    - "Restart Nomad"
   tags:
     - consul-conf
+
+- meta: flush_handlers
diff --git a/fdio.infra.ansible/roles/consul/templates/base.hcl.j2 b/fdio.infra.ansible/roles/consul/templates/base.hcl.j2
index 536c48d847..dab43fb3fc 100644
--- a/fdio.infra.ansible/roles/consul/templates/base.hcl.j2
+++ b/fdio.infra.ansible/roles/consul/templates/base.hcl.j2
@@ -14,14 +14,14 @@ server = {{ consul_node_server | bool | lower }}
 encrypt = "{{ consul_encrypt }}"
 {% if consul_node_server | bool == True %}
 bootstrap_expect = {{ consul_bootstrap_expect }}
-verify_incoming = true
-verify_outgoing = true
-verify_server_hostname = true
+verify_incoming = false
+verify_outgoing = false
+verify_server_hostname = false
 ca_file = "{{ consul_ca_file }}"
 cert_file = "{{ consul_cert_file }}"
 key_file = "{{ consul_key_file }}"
 auto_encrypt {
-  allow_tls = true
+  allow_tls = false
 }
 {% else %}
 verify_incoming = false
@@ -36,7 +36,20 @@ auto_encrypt {
 retry_join = [ {% for ip_port in consul_retry_servers -%} "{{ ip_port }}"{% if not loop.last %}, {% endif %}{%- endfor -%} ]
 {%- endif %}
 
-ui = {{ consul_ui | bool | lower }}
+{% if consul_ui_config -%}
+ui_config {
+{% for key, value in consul_ui_config.items() %}
+  {%- if value|bool %}
+  {{ key }} = {{ value | bool | lower }}
+  {%- elif value|string or value == "" %}
+  {{ key }} = "{{ value }}"
+  {%- else %}
+  {{ key }} = {{ value }}
+  {%- endif %}
+{% endfor %}
+
+}
+{%- endif %}
 
 {% if consul_recursors -%}
 recursors = [ {% for server in consul_recursors -%} "{{ server }}"{% if not loop.last %}, {% endif %}{%- endfor -%} ]
diff --git a/fdio.infra.ansible/roles/consul/templates/consul_systemd.service.j2 b/fdio.infra.ansible/roles/consul/templates/consul_systemd.service.j2
index 8e1ef1310d..16874f213e 100644
--- a/fdio.infra.ansible/roles/consul/templates/consul_systemd.service.j2
+++ b/fdio.infra.ansible/roles/consul/templates/consul_systemd.service.j2
@@ -1,21 +1,18 @@
 [Unit]
-Description=Consul Service
-Documentation=https://www.nomadproject.io/docs/
-Wants=network-online.target
+Description="HashiCorp Consul - A service mesh solution"
+Documentation=https://www.consul.io/
+Requires=network-online.target
 After=network-online.target
 
 [Service]
-# TODO: Decrease privilege
-ExecReload=/bin/kill -SIGHUP $MAINPID
-ExecStart={{ consul_bin_dir }}/consul agent -config-dir {{ consul_config_dir }}
-KillSignal=SIGTERM
-LimitNOFILE=infinity
-LimitNPROC=infinity
-Restart=on-failure
-RestartSec=1
 User=root
 Group=root
-Environment="GOMAXPROCS=2"
+ExecStart={{ consul_bin_dir }}/consul agent -config-dir={{ consul_config_dir }}
+ExecReload=/bin/kill --signal HUP $MAINPID
+KillMode=process
+KillSignal=SIGTERM
+Restart=on-failure
+LimitNOFILE=infinity
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/fdio.infra.ansible/roles/docker/defaults/main.yaml b/fdio.infra.ansible/roles/docker/defaults/main.yaml
index 8343558238..df9d2c92b6 100644
--- a/fdio.infra.ansible/roles/docker/defaults/main.yaml
+++ b/fdio.infra.ansible/roles/docker/defaults/main.yaml
@@ -27,7 +27,7 @@ docker_yum_gpg_key: https://download.docker.com/linux/centos/gpg
 
 # A list of users who will be added to the docker group.
 docker_users:
-  - "testuser"
+  - "{{ ansible_user }}"
 
 # Proxy settings.
 docker_daemon_environment_http:
diff --git a/fdio.infra.ansible/roles/nomad/defaults/main.yaml b/fdio.infra.ansible/roles/nomad/defaults/main.yaml
index 864890c11e..da9a872f83 100644
--- a/fdio.infra.ansible/roles/nomad/defaults/main.yaml
+++ b/fdio.infra.ansible/roles/nomad/defaults/main.yaml
@@ -3,14 +3,10 @@
 
 # Inst - Prerequisites.
 packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}"
-
 packages_base:
-  - "cgroup-bin"
   - "curl"
   - "git"
-  - "libcgroup1"
   - "unzip"
-  - "htop"
 packages_by_distro:
   ubuntu:
     - []
@@ -20,7 +16,7 @@ packages_by_arch:
   x86_64:
     - []
 
-# Inst - Download Nomad.
+# Inst - Nomad Map.
 nomad_architecture_map:
   amd64: "amd64"
   x86_64: "amd64"
@@ -29,7 +25,7 @@ nomad_architecture_map:
   32-bit: "386"
   64-bit: "amd64"
 nomad_architecture: "{{ nomad_architecture_map[ansible_architecture] }}"
-nomad_version: "0.12.0"
+nomad_version: "1.0.4"
 nomad_pkg: "nomad_{{ nomad_version }}_linux_{{ nomad_architecture }}.zip"
 nomad_zip_url: "https://releases.hashicorp.com/nomad/{{ nomad_version }}/{{ nomad_pkg }}"
 
@@ -49,10 +45,7 @@ nomad_restart_handler_state: "restarted"
 # Conf - User and group.
 nomad_group: "nomad"
 nomad_group_state: "present"
-nomad_manage_group: true
-nomad_manage_user: true
 nomad_user: "nomad"
-nomad_user_groups: [ docker, nomad, root ]
 nomad_user_state: "present"
 
 # Conf - base.hcl
@@ -71,14 +64,16 @@ nomad_cert_file: "{{ nomad_ssl_dir }}/nomad.pem"
 nomad_http: false
 nomad_key_file: "{{ nomad_ssl_dir }}/nomad-key.pem"
 nomad_rpc: false
+nomad_verify_https_client: false
+nomad_verify_server_hostname: false
 
 # Conf - client.hcl
 nomad_certificates:
-  - src: "{{ vault_nomad_ca_file }}"
+  - src: "{{ file_nomad_ca_pem }}"
     dest: "{{ nomad_ca_file }}"
-  - src: "{{ vault_nomad_cert_file }}"
+  - src: "{{ file_nomad_client_pem }}"
     dest: "{{ nomad_cert_file }}"
-  - src: "{{ vault_nomad_key_file }}"
+  - src: "{{ file_nomad_client_key_pem }}"
     dest: "{{ nomad_key_file }}"
 nomad_node_class: ""
 nomad_no_host_uuid: true
@@ -97,7 +92,6 @@ nomad_collection_interval: 60s
 nomad_use_node_name: false
 nomad_publish_allocation_metrics: true
 nomad_publish_node_metrics: true
-nomad_backwards_compatible_metrics: false
 nomad_telemetry_provider_parameters:
   prometheus_metrics: true
 
diff --git a/fdio.infra.ansible/roles/nomad/handlers/main.yaml b/fdio.infra.ansible/roles/nomad/handlers/main.yaml
index f0bcee9142..8ef4d80353 100644
--- a/fdio.infra.ansible/roles/nomad/handlers/main.yaml
+++ b/fdio.infra.ansible/roles/nomad/handlers/main.yaml
@@ -7,4 +7,3 @@
     enabled: true
     name: "nomad"
     state: "{{ nomad_restart_handler_state }}"
-
diff --git a/fdio.infra.ansible/roles/nomad/meta/main.yaml b/fdio.infra.ansible/roles/nomad/meta/main.yaml
index 9fc40d9ae1..7036087739 100644
--- a/fdio.infra.ansible/roles/nomad/meta/main.yaml
+++ b/fdio.infra.ansible/roles/nomad/meta/main.yaml
@@ -1,9 +1,23 @@
 ---
 # file: roles/nomad/meta/main.yaml
 
-# desc: Install nomad from stable branch and configure service.
+# desc: Install nomad from repo and configure service.
 # inst: Nomad
 # conf: ?
 # info: 1.0 - added role
 
 dependencies: [ docker ]
+
+galaxy_info:
+  role_name: nomad
+  author: fd.io
+  description: Hashicorp Nomad.
+  company: none
+  license: "license (Apache)"
+  min_ansible_version: 2.9
+  platforms:
+    - name: Ubuntu
+      versions:
+        - focal
+  galaxy_tags:
+    - nomad
diff --git a/fdio.infra.ansible/roles/nomad/tasks/main.yaml b/fdio.infra.ansible/roles/nomad/tasks/main.yaml
index 54e80513b8..63025a6ead 100644
--- a/fdio.infra.ansible/roles/nomad/tasks/main.yaml
+++ b/fdio.infra.ansible/roles/nomad/tasks/main.yaml
@@ -3,7 +3,7 @@
 
 - name: Inst - Update Package Cache (APT)
   apt:
-    update_cache: yes
+    update_cache: true
     cache_valid_time: 3600
   when:
     - ansible_distribution|lower == 'ubuntu'
@@ -20,9 +20,7 @@
 - name: Conf - Add Nomad Group
   group:
     name: "{{ nomad_group }}"
-    state: "{{ nomad_group_state }}"
-  when:
-    - nomad_manage_group | bool
+    state: "{{ nomad_user_state }}"
   tags:
     - nomad-conf-user
 
@@ -30,11 +28,8 @@
   user:
     name: "{{ nomad_user }}"
     group: "{{ nomad_group }}"
-    groups: "{{ nomad_user_groups }}"
-    state: "{{ nomad_user_state }}"
+    state: "{{ nomad_group_state }}"
     system: true
-  when:
-    - nomad_manage_user | bool
   tags:
     - nomad-conf-user
 
@@ -186,7 +181,9 @@
     owner: "root"
     group: "root"
     mode: 0644
-#  notify:
-#    - "Restart Nomad"
+  notify:
+    - "Restart Nomad"
   tags:
     - nomad-conf
+
+- meta: flush_handlers
diff --git a/fdio.infra.ansible/roles/nomad/templates/nomad_systemd.service.j2 b/fdio.infra.ansible/roles/nomad/templates/nomad_systemd.service.j2
index 2a87c65063..61f07df5b6 100644
--- a/fdio.infra.ansible/roles/nomad/templates/nomad_systemd.service.j2
+++ b/fdio.infra.ansible/roles/nomad/templates/nomad_systemd.service.j2
@@ -1,21 +1,28 @@
 [Unit]
-Description=Nomad Service
-Documentation=https://www.nomadproject.io/docs/
+Description=Nomad
+Documentation=https://nomadproject.io/docs/
 Wants=network-online.target
 After=network-online.target
 
+# When using Nomad with Consul it is not necessary to start Consul first. These
+# lines start Consul before Nomad as an optimization to avoid Nomad logging
+# that Consul is unavailable at startup.
+#Wants=consul.service
+#After=consul.service
+
 [Service]
-# TODO: Decrease privilege
-ExecReload=/bin/kill -SIGHUP $MAINPID
-ExecStart={{ nomad_bin_dir }}/nomad agent -config={{ nomad_config_dir }}
-KillSignal=SIGTERM
+ExecReload=/bin/kill -HUP $MAINPID
+ExecStart={{ nomad_bin_dir }}/nomad agent -config {{ nomad_config_dir }}
+KillMode=process
+KillSignal=SIGINT
 LimitNOFILE=infinity
 LimitNPROC=infinity
 Restart=on-failure
-RestartSec=1
-User=root
-Group=root
-Environment="GOMAXPROCS=2"
+RestartSec=2
+StartLimitBurst=3
+StartLimitInterval=10
+TasksMax=infinity
+OOMScoreAdjust=-1000
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2
index 256c6999e9..7b62f76976 100644
--- a/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2
+++ b/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2
@@ -16,5 +16,4 @@ telemetry {
     use_node_name = {{ nomad_use_node_name | bool | lower }}
     publish_allocation_metrics = {{ nomad_publish_allocation_metrics | bool | lower }}
     publish_node_metrics = {{ nomad_publish_node_metrics | bool | lower }}
-    backwards_compatible_metrics = {{ nomad_backwards_compatible_metrics | bool | lower }}
 }
diff --git a/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2
index 650765f1b1..46dc1fe6b1 100644
--- a/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2
+++ b/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2
@@ -5,6 +5,8 @@
 tls {
     http = {{ nomad_http | bool | lower }}
     rpc = {{ nomad_rpc | bool | lower }}
+    verify_server_hostname = {{ nomad_verify_server_hostname | bool | lower }}
+    verify_https_client = {{ nomad_verify_https_client | bool | lower }}
     ca_file = "{{ nomad_ca_file }}"
     cert_file = "{{ nomad_cert_file }}"
     key_file = "{{ nomad_key_file }}"
diff --git a/fdio.infra.ansible/roles/prometheus_exporter/tasks/ubuntu_focal.yaml b/fdio.infra.ansible/roles/prometheus_exporter/tasks/ubuntu_focal.yaml
new file mode 100644
index 0000000000..3d7064355e
--- /dev/null
+++ b/fdio.infra.ansible/roles/prometheus_exporter/tasks/ubuntu_focal.yaml
@@ -0,0 +1,33 @@
+---
+# file: roles/prometheus_exporter/tasks/ubuntu_focal.yaml
+
+- name: Inst - Update Package Cache (APT)
+  apt:
+    update_cache: yes
+    cache_valid_time: 3600
+  tags:
+    - prometheus-inst-prerequisites
+
+- name: Inst - Prerequisites
+  package:
+    name: "init-system-helpers"
+    default_release: "focal-backports"
+    state: latest
+  tags:
+    - prometheus-inst-prerequisites
+
+- name: Inst - Prometheus Node Exporter
+  apt:
+    deb: "{{ ne_packages }}"
+  notify:
+    - "Restart Prometheus Node Exporter"
+  tags:
+    - prometheus-inst-node-exporter
+
+- name: Inst - Prometheus Blackbox Exporter
+  apt:
+    deb: "{{ be_packages }}"
+  notify:
+    - "Restart Prometheus Blackbox Exporter"
+  tags:
+    - prometheus-inst-blackbox-exporter
\ No newline at end of file
diff --git a/fdio.infra.ansible/roles/user_add/tasks/main.yaml b/fdio.infra.ansible/roles/user_add/tasks/main.yaml
index f980aff84d..e2ef63db48 100644
--- a/fdio.infra.ansible/roles/user_add/tasks/main.yaml
+++ b/fdio.infra.ansible/roles/user_add/tasks/main.yaml
@@ -22,27 +22,16 @@
   with_subelements:
     - "{{ users }}"
     - ssh_key
-    - skip_missing: yes
+    - skip_missing: true
   tags:
     - user-add-conf
 
-- name: Conf - Allow Password Login
+- name: Conf - Disable Password Login
   lineinfile:
     dest: "/etc/ssh/sshd_config"
-    regexp: "^PasswordAuthentication no"
-    line: "PasswordAuthentication yes"
+    regexp: "^PasswordAuthentication yes"
+    line: "PasswordAuthentication no"
   notify:
     - "Restart SSHd"
   tags:
     - user-add-conf
-
-- name: Conf - Add Visudo Entry
-  lineinfile:
-    dest: "/etc/sudoers"
-    state: present
-    line: "{{ item.username }} ALL=(ALL) NOPASSWD: ALL"
-    validate: "visudo -cf %s"
-  with_items: "{{ users }}"
-  tags:
-    - user-add-conf
-