Infra: Cleanup Nomad configs

Signed-off-by: pmikus <pmikus@cisco.com>
Change-Id: Ia5c9f0902551de1a63144e6f56dfa6db2895b0b2

9 files changed, 259 insertions, 110 deletions

diff --git a/fdio.infra.ansible/roles/nomad/defaults/main.yaml b/fdio.infra.ansible/roles/nomad/defaults/main.yaml
index 01f020ad81..2ace6b22d5 100644
--- a/fdio.infra.ansible/roles/nomad/defaults/main.yaml
+++ b/fdio.infra.ansible/roles/nomad/defaults/main.yaml
@@ -1,7 +1,7 @@
 ---
 # file: roles/nomad/defaults/main.yaml
 
-# Inst - Prerequisites.
+# Prerequisites
 packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}"
 packages_base:
   - "curl"
@@ -16,7 +16,8 @@ packages_by_arch:
   x86_64:
     - []
 
-# Inst - Nomad Map.
+# Package
+nomad_version: "{{ lookup('env','NOMAD_VERSION') | default('1.0.4', true) }}"
 nomad_architecture_map:
   amd64: "amd64"
   x86_64: "amd64"
@@ -25,63 +26,49 @@ nomad_architecture_map:
   32-bit: "386"
   64-bit: "amd64"
 nomad_architecture: "{{ nomad_architecture_map[ansible_architecture] }}"
-nomad_version: "1.0.4"
-nomad_pkg: "nomad_{{ nomad_version }}_linux_{{ nomad_architecture }}.zip"
-nomad_zip_url: "https://releases.hashicorp.com/nomad/{{ nomad_version }}/{{ nomad_pkg }}"
+nomad_pkg: "nomad_{{ nomad_version }}_linux_{{nomad_architecture}}.zip"
+nomad_zip_url: "https://releases.hashicorp.com/nomad/{{ nomad_version }}/nomad_{{ nomad_version }}_linux_{{nomad_architecture}}.zip"
+nomad_checksum_file_url: "https://releases.hashicorp.com/nomad/{{ nomad_version }}/nomad_{{ nomad_version}}_SHA256SUMS"
+nomad_podman_enable: false
+nomad_podman_version: "{{ lookup('env','NOMAD_PODMAN_VERSION') | default('0.1.0', true) }}"
+nomad_podman_pkg: "nomad-driver-podman_{{ nomad_podman_version }}_linux_{{nomad_architecture}}.zip"
+nomad_podman_url: "https://releases.hashicorp.com/nomad-driver-podman/{{ nomad_podman_version }}"
+nomad_podman_zip_url: "{{ nomad_podman_url }}/{{ nomad_podman_pkg }}"
+nomad_podman_checksum_file_url: "{{ nomad_podman_url }}/nomad-driver-podman_{{ nomad_podman_version }}_SHA256SUMS"
 
-# Inst - System paths.
+# Paths
+nomad_inst_dir: "/opt"
 nomad_bin_dir: "/usr/local/bin"
 nomad_config_dir: "/etc/nomad.d"
 nomad_data_dir: "/var/nomad"
-nomad_inst_dir: "/opt"
+nomad_plugin_dir: "{{ nomad_data_dir }}/plugins"
 nomad_lockfile: "/var/lock/subsys/nomad"
 nomad_run_dir: "/var/run/nomad"
 nomad_ssl_dir: "/etc/nomad.d/ssl"
 
-# Conf - Service.
-nomad_node_role: "both"
+# Initialization and startup script templates
 nomad_restart_handler_state: "restarted"
 
-# Conf - User and group.
+# System user and group
 nomad_group: "nomad"
 nomad_group_state: "present"
 nomad_user: "nomad"
 nomad_user_state: "present"
 
-# Conf - base.hcl
-nomad_bind_addr: "0.0.0.0"
+# Nomad settings
 nomad_datacenter: "dc1"
-nomad_disable_update_check: true
-nomad_enable_debug: false
-nomad_log_level: "INFO"
-nomad_name: "{{ inventory_hostname }}"
 nomad_region: "global"
+nomad_log_level: "INFO"
 nomad_syslog_enable: true
+nomad_iface: "{{ lookup('env','NOMAD_IFACE') | default(ansible_default_ipv4.interface, true) }}"
+nomad_node_name: "{{ inventory_hostname }}"
+nomad_node_role: "{{ lookup('env','NOMAD_NODE_ROLE') | default('client', true) }}"
+nomad_leave_on_terminate: true
+nomad_leave_on_interrupt: false
+nomad_disable_update_check: true
+nomad_enable_debug: false
 
-# Conf - tls.hcl
-nomad_ca_file: "{{ nomad_ssl_dir }}/ca.pem"
-nomad_cert_file: "{{ nomad_ssl_dir }}/nomad.pem"
-nomad_http: false
-nomad_key_file: "{{ nomad_ssl_dir }}/nomad-key.pem"
-nomad_rpc: false
-nomad_verify_https_client: false
-nomad_verify_server_hostname: false
-
-# Conf - client.hcl
-nomad_certificates:
-  - src: "{{ file_nomad_ca_pem }}"
-    dest: "{{ nomad_ca_file }}"
-  - src: "{{ file_nomad_client_pem }}"
-    dest: "{{ nomad_cert_file }}"
-  - src: "{{ file_nomad_client_key_pem }}"
-    dest: "{{ nomad_key_file }}"
-nomad_node_class: ""
-nomad_no_host_uuid: true
-nomad_options: {}
-nomad_servers: []
-nomad_volumes: []
-
-# Conf - server.hcl
+# Server settings
 nomad_bootstrap_expect: 2
 nomad_encrypt: ""
 nomad_retry_join: true
@@ -90,25 +77,127 @@ nomad_retry_join: true
 nomad_node_gc_threshold: "24h"
 # Specifies the interval between the job garbage collections. Only jobs who have
 # been terminal for at least job_gc_threshold will be collected.
-nomad_job_gc_interval: "1m"
+nomad_job_gc_interval: "10m"
 # Specifies the minimum time a job must be in the terminal state before it is
 # eligible for garbage collection.
-nomad_job_gc_threshold: "1m"
+nomad_job_gc_threshold: "4h"
 # Specifies the minimum time an evaluation must be in the terminal state before
 # it is eligible for garbage collection.
-nomad_eval_gc_threshold: "1m"
+nomad_eval_gc_threshold: "1h"
 # Specifies the minimum time a deployment must be in the terminal state before
 # it is eligible for garbage collection.
-nomad_deployment_gc_threshold: "1m"
-
-# Conf - telemetry.hcl
-nomad_disable_hostname: false
-nomad_collection_interval: 60s
-nomad_use_node_name: false
-nomad_publish_allocation_metrics: true
-nomad_publish_node_metrics: true
-nomad_telemetry_provider_parameters:
-  prometheus_metrics: true
-
-# Conf - custom.hcl
-# empty
+nomad_deployment_gc_threshold: "1h"
+nomad_encrypt_enable: "{{ lookup('env','NOMAD_ENCRYPT_ENABLE') | default('false', true) }}"
+nomad_raft_protocol: 2
+
+# Client settings
+nomad_certificates:
+  - src: "{{ file_nomad_ca_pem }}"
+    dest: "{{ nomad_ca_file }}"
+  - src: "{{ file_nomad_client_pem }}"
+    dest: "{{ nomad_cert_file }}"
+  - src: "{{ file_nomad_client_key_pem }}"
+    dest: "{{ nomad_key_file }}"
+nomad_node_class: ""
+nomad_no_host_uuid: true
+nomad_max_kill_timeout: "30s"
+nomad_gc_interval: "1m"
+nomad_gc_disk_usage_threshold: 80
+nomad_gc_inode_usage_threshold: 70
+nomad_gc_parallel_destroys: 2
+nomad_reserved:
+  cpu: "{{ nomad_reserved_cpu | default('0', true) }}"
+  memory: "{{ nomad_reserved_memory | default('0', true) }}"
+  disk: "{{ nomad_reserved_disk | default('0', true) }}"
+  ports: "{{ nomad_reserved_ports | default('22', true) }}"
+nomad_volumes: []
+nomad_options: {}
+nomad_meta: {}
+nomad_chroot_env: false
+nomad_plugins: {}
+
+# Addresses
+nomad_bind_address: "{{ hostvars[inventory_hostname]['ansible_'+ nomad_iface ]['ipv4']['address'] }}"
+nomad_advertise_address: "{{ hostvars[inventory_hostname]['ansible_' + nomad_iface]['ipv4']['address'] }}"
+
+# Ports
+nomad_ports:
+  http: "{{ nomad_ports_http | default('4646', true) }}"
+  rpc: "{{ nomad_ports_rpc | default('4647', true) }}"
+  serf: "{{ nomad_ports_serf | default('4648', true) }}"
+
+# Servers
+nomad_group_name: "nomad"
+nomad_servers: "\
+  {% if nomad_use_consul==false %}\
+    {% set _nomad_servers = [] %}\
+    {% for host in groups[nomad_group_name] %}\
+      {% set _nomad_node_role = hostvars[host]['nomad_node_role'] | default('client', true) %}\
+      {% if ( _nomad_node_role == 'server' or _nomad_node_role == 'both') %}\
+        {% if _nomad_servers.append(host) %}{% endif %}\
+      {% endif %}\
+    {% endfor %}\
+    {{ _nomad_servers }}\
+    {% else %}\
+    []\
+  {% endif %}"
+nomad_gather_server_facts: false
+
+# Consul
+nomad_use_consul: true
+nomad_consul_address: "localhost:8500"
+nomad_consul_token: ""
+nomad_consul_servers_service_name: "nomad"
+nomad_consul_clients_service_name: "nomad-client"
+nomad_consul_tags: {}
+
+# ACLs
+nomad_acl_enabled: "{{ lookup('env', 'NOMAD_ACL_ENABLED') | default('no', true) }}"
+nomad_acl_token_ttl: "30s"
+nomad_acl_policy_ttl: "30s"
+nomad_acl_replication_token: ""
+
+# Vault
+nomad_vault_enabled: "{{ lookup('env', 'NOMAD_VAULT_ENABLED') | default('no', true) }}"
+nomad_vault_address: "{{ vault_address | default('0.0.0.0', true) }}"
+nomad_vault_allow_unauthenticated: true
+nomad_vault_create_from_role: ""
+nomad_vault_task_token_ttl: ""
+nomad_vault_ca_file: ""
+nomad_vault_ca_path: ""
+nomad_vault_cert_file: ""
+nomad_vault_key_file: ""
+nomad_vault_tls_server_name: ""
+nomad_vault_tls_skip_verify: false
+nomad_vault_token: ""
+nomad_vault_namespace: ""
+
+# Docker
+nomad_docker_enable: "{{ lookup('env','NOMAD_DOCKER_ENABLE') | default('false', true) }}"
+nomad_docker_dmsetup: true
+
+# TLS
+nomad_tls_enable: true
+nomad_ca_file: "{{ nomad_ssl_dir }}/ca.pem"
+nomad_cert_file: "{{ nomad_ssl_dir }}/nomad.pem"
+nomad_key_file: "{{ nomad_ssl_dir }}/nomad-key.pem"
+nomad_http: false
+nomad_rpc: false
+nomad_rpc_upgrade_mode: false
+nomad_verify_server_hostname: false
+nomad_verify_https_client: false
+
+# Conf - autopilot.hcl
+nomad_autopilot_cleanup_dead_servers: true
+nomad_autopilot_last_contact_threshold: "200ms"
+nomad_autopilot_max_trailing_logs: 250
+nomad_autopilot_server_stabilization_time: "10s"
+
+# Telemetry
+nomad_telemetry: true
+nomad_telemetry_disable_hostname: false
+nomad_telemetry_collection_interval: 60s
+nomad_telemetry_use_node_name: false
+nomad_telemetry_publish_allocation_metrics: true
+nomad_telemetry_publish_node_metrics: true
+nomad_telemetry_prometheus_metrics: true
diff --git a/fdio.infra.ansible/roles/nomad/tasks/main.yaml b/fdio.infra.ansible/roles/nomad/tasks/main.yaml
index 63025a6ead..997b1e9c91 100644
--- a/fdio.infra.ansible/roles/nomad/tasks/main.yaml
+++ b/fdio.infra.ansible/roles/nomad/tasks/main.yaml
@@ -150,15 +150,13 @@
   tags:
     - nomad-conf
 
-- name: Conf - Custom Configuration
+- name: Conf - Consul Configuration
   template:
-    src: custom.json.j2
-    dest: "{{ nomad_config_dir }}/custom.json"
+    src: consul.hcl.j2
+    dest: "{{ nomad_config_dir }}/consul.hcl"
     owner: "{{ nomad_user }}"
     group: "{{ nomad_group }}"
     mode: 0644
-  when:
-    - nomad_config_custom is defined
   tags:
     - nomad-conf
 
diff --git a/fdio.infra.ansible/roles/nomad/templates/base.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/base.hcl.j2
index 7badecf9e0..cd7fb54f9c 100644
--- a/fdio.infra.ansible/roles/nomad/templates/base.hcl.j2
+++ b/fdio.infra.ansible/roles/nomad/templates/base.hcl.j2
@@ -1,11 +1,26 @@
-name = "{{ nomad_name }}"
+name = "{{ nomad_node_name }}"
 region = "{{ nomad_region }}"
 datacenter = "{{ nomad_datacenter }}"
 
-bind_addr = "{{ nomad_bind_addr }}"
-data_dir = "{{ nomad_data_dir }}"
-
-enable_syslog = {{ nomad_syslog_enable | bool | lower }}
 enable_debug = {{ nomad_enable_debug | bool | lower }}
 disable_update_check = {{ nomad_disable_update_check | bool | lower }}
+
+bind_addr = "{{ nomad_bind_address }}"
+advertise {
+    http = "{{ nomad_advertise_address }}:{{ nomad_ports.http }}"
+    rpc = "{{ nomad_advertise_address }}:{{ nomad_ports.rpc }}"
+    serf = "{{ nomad_advertise_address }}:{{ nomad_ports.serf }}"
+}
+ports {
+    http = {{ nomad_ports['http'] }}
+    rpc = {{ nomad_ports['rpc'] }}
+    serf = {{ nomad_ports['serf'] }}
+}
+
+data_dir = "{{ nomad_data_dir }}"
+
 log_level = "{{ nomad_log_level }}"
+enable_syslog = {{ nomad_syslog_enable | bool | lower }}
+
+leave_on_terminate = {{ nomad_leave_on_terminate | bool | lower }}
+leave_on_interrupt = {{ nomad_leave_on_interrupt | bool | lower }}
diff --git a/fdio.infra.ansible/roles/nomad/templates/client.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/client.hcl.j2
index f245697a22..f82f38a4e4 100644
--- a/fdio.infra.ansible/roles/nomad/templates/client.hcl.j2
+++ b/fdio.infra.ansible/roles/nomad/templates/client.hcl.j2
@@ -1,14 +1,44 @@
 client {
     enabled = {{ nomad_node_client | bool | lower }}
-    no_host_uuid = {{ nomad_no_host_uuid | bool | lower }}
+
     node_class = "{{ nomad_node_class }}"
+    no_host_uuid = {{ nomad_no_host_uuid | bool | lower }}
+
+{% if nomad_use_consul == False %}
+    {% if nomad_servers -%}
+        servers = [ {% for ip_port in nomad_servers -%} "{{ ip_port }}" {% if not loop.last %},{% endif %}{%- endfor -%} ]
+    {% endif -%}
+{% endif %}
 
+    {% if nomad_network_interface is defined -%}
+    network_interface = "{{ nomad_network_interface }}"
+    {% endif -%}
+    {% if nomad_network_speed is defined -%}
+    network_speed = "{{ nomad_network_speed }}"
+    {% endif -%}
     {% if nomad_cpu_total_compute is defined -%}
     cpu_total_compute = {{ nomad_cpu_total_compute }}
     {% endif -%}
 
-    {% if nomad_servers -%}
-    servers = [ {% for ip_port in nomad_servers -%} "{{ ip_port }}" {% if not loop.last %},{% endif %}{%- endfor -%} ]
+    reserved {
+        cpu = {{ nomad_reserved['cpu'] }}
+        memory = {{ nomad_reserved['memory'] }}
+        disk = {{ nomad_reserved['disk'] }}
+    }
+
+    {% for nomad_host_volume in nomad_volumes -%}
+    host_volume "{{ nomad_host_volume.name }}" {
+        path = "{{ nomad_host_volume.path }}"
+        read_only = {{ nomad_host_volume.read_only | bool | lower }}
+    }
+    {% endfor %}
+
+    {% if nomad_chroot_env != False -%}
+    chroot_env = {
+    {% for key, value in nomad_chroot_env.items() %}
+    "{{ key }}" = "{{ value }}"
+    {% endfor -%}
+    }
     {% endif %}
 
     {% if nomad_options -%}
@@ -19,13 +49,12 @@ client {
     }
     {% endif %}
 
-    {% if nomad_volumes -%}
-    {% for volume in nomad_volumes -%}
-    host_volume "{{ volume.name }}" {
-        path = "{{ volume.path }}"
-        read_only = {{ volume.read_only | bool | lower }}
-    }
+    {% if nomad_meta -%}
+    meta = {
+    {% for key, value in nomad_meta.items() %}
+    "{{ key }}" = "{{ value }}"
     {% endfor -%}
+    }
     {% endif %}
 
 }
diff --git a/fdio.infra.ansible/roles/nomad/templates/consul.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/consul.hcl.j2
new file mode 100644
index 0000000000..6d30676ca0
--- /dev/null
+++ b/fdio.infra.ansible/roles/nomad/templates/consul.hcl.j2
@@ -0,0 +1,18 @@
+{% if nomad_use_consul | bool == True %}
+consul {
+    # The address to the Consul agent.
+    address = "{{ nomad_consul_address }}"
+    token = "{{ nomad_consul_token }}"
+    # The service name to register the server and client with Consul.
+    server_service_name = "{{ nomad_consul_servers_service_name }}"
+    client_service_name = "{{ nomad_consul_clients_service_name }}"
+    tags = {{ nomad_consul_tags | to_json }}
+
+    # Enables automatically registering the services.
+    auto_advertise = true
+
+    # Enabling the server and client to bootstrap using Consul.
+    server_auto_join = true
+    client_auto_join = true
+}
+{% endif %}
\ No newline at end of file
diff --git a/fdio.infra.ansible/roles/nomad/templates/custom.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/custom.hcl.j2
deleted file mode 100644
index 37ff6f3496..0000000000
--- a/fdio.infra.ansible/roles/nomad/templates/custom.hcl.j2
+++ /dev/null
@@ -1,5 +0,0 @@
-{% if nomad_config_custom -%}
-{{ nomad_config_custom | to_nice_json }}
-{% else %}
-{}
-{% endif %}
diff --git a/fdio.infra.ansible/roles/nomad/templates/server.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/server.hcl.j2
index 5ccf45c1ac..663ee3a549 100644
--- a/fdio.infra.ansible/roles/nomad/templates/server.hcl.j2
+++ b/fdio.infra.ansible/roles/nomad/templates/server.hcl.j2
@@ -5,7 +5,29 @@ server {
         bootstrap_expect = {{ nomad_bootstrap_expect }}
     {%- endif %}
 
-    encrypt = "{{ nomad_encrypt }}"
+    {% if nomad_authoritative_region is defined %}
+        authoritative_region = "{{ nomad_authoritative_region }}"
+    {% endif %}
+
+{% if nomad_use_consul == False %}
+    {% if nomad_retry_join | bool -%}
+        retry_join = [
+            {%- set comma = joiner(",") -%}
+            {% for server in nomad_servers -%}
+                {{ comma() }}"{{ hostvars[server]['nomad_advertise_address'] | ipwrap }}"
+            {%- endfor -%} ]
+        retry_max = {{ nomad_retry_max }}
+        retry_interval = "{{ nomad_retry_interval }}"
+    {% else -%}
+        start_join = [
+            {%- set comma = joiner(",") -%}
+            {% for server in nomad_servers -%}
+                {{ comma() }}"{{ hostvars[server]['nomad_advertise_address'] | ipwrap }}"
+            {%- endfor -%} ]
+    {%- endif %}
+{% endif %}
+
+    encrypt = "{{ nomad_encrypt | default('') }}"
 
     {% if nomad_node_gc_threshold -%}
         node_gc_threshold = "{{ nomad_node_gc_threshold }}"
@@ -27,10 +49,4 @@ server {
         deployment_gc_threshold = "{{ nomad_deployment_gc_threshold }}"
     {%- endif %}
 
-    {% if nomad_retry_join | bool -%}
-    server_join {
-        retry_join = [ {% for ip_port in nomad_retry_servers -%} "{{ ip_port }}" {% if not loop.last %},{% endif %}{%- endfor -%} ]
-    }
-    {%- endif %}
-
 }
diff --git a/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2
index 7b62f76976..4ad5330d1b 100644
--- a/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2
+++ b/fdio.infra.ansible/roles/nomad/templates/telemetry.hcl.j2
@@ -1,19 +1,10 @@
+{% if nomad_telemetry | bool == True %}
 telemetry {
-    # Telemetry provider parameters
-    {% for key, value in nomad_telemetry_provider_parameters.items() -%}
-    {% if value|bool -%}
-    {{ key }} = {{ value | bool | lower }}
-    {% elif value|string or value == "" -%}
-    {{ key }} = "{{ value }}"
-    {% else %}
-    {{ key }} = {{ value }}
-    {% endif -%}
-    {% endfor -%}
-
-    # Common parameters
-    disable_hostname = {{ nomad_disable_hostname | bool | lower }}
-    collection_interval = "{{ nomad_collection_interval }}"
-    use_node_name = {{ nomad_use_node_name | bool | lower }}
-    publish_allocation_metrics = {{ nomad_publish_allocation_metrics | bool | lower }}
-    publish_node_metrics = {{ nomad_publish_node_metrics | bool | lower }}
+    disable_hostname = "{{ nomad_telemetry_disable_hostname | default(false) | bool | lower }}"
+    collection_interval = "{{ nomad_telemetry_collection_interval | default("1s") }}"
+    use_node_name = "{{ nomad_telemetry_use_node_name | default(false) | bool | lower }}"
+    publish_allocation_metrics = "{{ nomad_telemetry_publish_allocation_metrics | default(false) | bool | lower }}"
+    publish_node_metrics = "{{ nomad_telemetry_publish_node_metrics | default(false) | bool | lower }}"
+    prometheus_metrics = "{{ nomad_telemetry_prometheus_metrics | default(false) | bool | lower }}"
 }
+{% endif %}
diff --git a/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2 b/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2
index 46dc1fe6b1..ceccdc8be5 100644
--- a/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2
+++ b/fdio.infra.ansible/roles/nomad/templates/tls.hcl.j2
@@ -1,14 +1,12 @@
-{% if ( nomad_ca_file ) and
-      ( nomad_cert_file ) and
-      ( nomad_key_file )
-%}
+{% if nomad_tls_enable | bool %}
 tls {
     http = {{ nomad_http | bool | lower }}
     rpc = {{ nomad_rpc | bool | lower }}
-    verify_server_hostname = {{ nomad_verify_server_hostname | bool | lower }}
-    verify_https_client = {{ nomad_verify_https_client | bool | lower }}
     ca_file = "{{ nomad_ca_file }}"
     cert_file = "{{ nomad_cert_file }}"
     key_file = "{{ nomad_key_file }}"
+    rpc_upgrade_mode = {{ nomad_rpc_upgrade_mode | bool | lower }}
+    verify_server_hostname = {{ nomad_verify_server_hostname | bool | lower }}
+    verify_https_client = {{ nomad_verify_https_client | bool | lower }}
 }
 {% endif %}