Infra: Ansible Nomad

Signed-off-by: pmikus <pmikus@cisco.com>
Change-Id: Icc03bf62d8639f603ca90aa6ab849a46dc911d56

14 files changed, 404 insertions, 16 deletions

diff --git a/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml
index e91442718e..2fb6e2e213 100644
--- a/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml
@@ -51,3 +51,10 @@
   tags:
     - install-mellanox
 
+- name: Mellanox Install - FIX qemu-system removal
+  package:
+    name: "qemu-system"
+    state: latest
+    update_cache: true
+  tags:
+    - install-mellanox
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/defaults/main.yaml
new file mode 100644
index 0000000000..ad8b36ee66
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/defaults/main.yaml
@@ -0,0 +1,103 @@
+---
+# file: roles/nomad/defaults/main.yaml
+
+# Inst - Prerequisites.
+packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}"
+
+packages_base:
+  - "cgroup-bin"
+  - "curl"
+  - "git"
+  - "libcgroup1"
+  - "unzip"
+packages_by_distro:
+  ubuntu:
+    - []
+packages_by_arch:
+  aarch64:
+    - []
+  x86_64:
+    - []
+
+# Inst - Download Nomad.
+nomad_architecture_map:
+  amd64: "amd64"
+  x86_64: "amd64"
+  armv7l: "arm"
+  aarch64: "arm64"
+  32-bit: "386"
+  64-bit: "amd64"
+nomad_architecture: "{{ nomad_architecture_map[ansible_architecture] }}"
+nomad_version: "0.10.2"
+nomad_pkg: "nomad_{{ nomad_version }}_linux_{{ nomad_architecture }}.zip"
+nomad_zip_url: "https://releases.hashicorp.com/nomad/{{ nomad_version }}/{{ nomad_pkg }}"
+
+# Inst - System paths.
+nomad_bin_dir: "/usr/local/bin"
+nomad_config_dir: "/etc/nomad.d"
+nomad_data_dir: "/var/nomad"
+nomad_inst_dir: "/opt"
+nomad_lockfile: "/var/lock/subsys/nomad"
+nomad_run_dir: "/var/run/nomad"
+nomad_ssl_dir: "/etc/nomad.d/ssl"
+
+# Conf - Service.
+nomad_node_role: "both"
+nomad_restart_handler_state: "restarted"
+
+# Conf - User and group.
+nomad_group: "nomad"
+nomad_group_state: "present"
+nomad_manage_group: true
+nomad_manage_user: true
+nomad_user: "nomad"
+nomad_user_groups: [ docker, nomad, root ]
+nomad_user_state: "present"
+
+# Conf - base.hcl
+nomad_bind_addr: "0.0.0.0"
+nomad_datacenter: "dc1"
+nomad_disable_update_check: true
+nomad_enable_debug: false
+nomad_log_level: "INFO"
+nomad_name: "{{ inventory_hostname }}"
+nomad_region: "global"
+nomad_syslog_enable: true
+
+# Conf - base.hcl (TLS)
+nomad_ca_file: "{{ nomad_ssl_dir }}/ca.pem"
+nomad_cert_file: "{{ nomad_ssl_dir }}/nomad.pem"
+nomad_http: false
+nomad_key_file: "{{ nomad_ssl_dir }}/nomad-key.pem"
+nomad_rpc: false
+
+# Conf - client.hcl
+nomad_certificates:
+  - src: "{{ vault_nomad_ca_file }}"
+    dest: "{{ nomad_ca_file }}"
+  - src: "{{ vault_nomad_cert_file }}"
+    dest: "{{ nomad_cert_file }}"
+  - src: "{{ vault_nomad_key_file }}"
+    dest: "{{ nomad_key_file }}"
+nomad_node_class: ""
+nomad_no_host_uuid: true
+nomad_options: {}
+nomad_servers: []
+
+# Conf - custom.hcl
+# empty
+
+# Conf - server.hcl
+#nomad_retry_max: 0
+#nomad_retry_join: false
+#nomad_retry_interval: "30s"
+#nomad_rejoin_after_leave: false
+#nomad_enabled_schedulers:
+#  - service
+#  - batch
+#  - system
+#nomad_num_schedulers: "{{ ansible_processor_vcpus }}"
+#nomad_node_gc_threshold: "24h"
+#nomad_job_gc_threshold: "4h"
+#nomad_eval_gc_threshold: "1h"
+#nomad_encrypt: ""
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/handlers/main.yaml
new file mode 100644
index 0000000000..f0bcee9142
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/handlers/main.yaml
@@ -0,0 +1,10 @@
+---
+# file roles/nomad/handlers/main.yaml
+
+- name: Restart Nomad
+  systemd:
+    daemon_reload: true
+    enabled: true
+    name: "nomad"
+    state: "{{ nomad_restart_handler_state }}"
+
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/meta/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/meta/main.yaml
new file mode 100644
index 0000000000..9fc40d9ae1
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/meta/main.yaml
@@ -0,0 +1,9 @@
+---
+# file: roles/nomad/meta/main.yaml
+
+# desc: Install nomad from stable branch and configure service.
+# inst: Nomad
+# conf: ?
+# info: 1.0 - added role
+
+dependencies: [ docker ]
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml
new file mode 100644
index 0000000000..9093afd08e
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml
@@ -0,0 +1,164 @@
+---
+# file: roles/nomad/tasks/main.yaml
+
+- name: Inst - Prerequisites
+  package:
+    name: "{{ packages | flatten(levels=1) }}"
+    state: latest
+    update_cache: true
+  tags:
+    - nomad-inst-prerequisites
+
+- name: Conf - Add Nomad Group
+  group:
+    name: "{{ nomad_group }}"
+    state: "{{ nomad_group_state }}"
+  when:
+    - nomad_manage_group | bool
+  tags:
+    - nomad-conf-user
+
+- name: Conf - Add Nomad user
+  user:
+    name: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    groups: "{{ nomad_user_groups }}"
+    state: "{{ nomad_user_state }}"
+    system: true
+  when:
+    - nomad_manage_user | bool
+  tags:
+    - nomad-conf-user
+
+- name: Inst - Download Nomad
+  get_url:
+    url: "{{ nomad_zip_url }}"
+    dest: "{{ nomad_inst_dir }}/{{ nomad_pkg }}"
+  tags:
+    - nomad-inst-package
+
+- name: Inst - Unarchive Nomad
+  unarchive:
+    src: "{{ nomad_inst_dir }}/{{ nomad_pkg }}"
+    dest: "{{ nomad_inst_dir }}/"
+    creates: "{{ nomad_inst_dir }}/nomad"
+    remote_src: true
+  tags:
+    - nomad-inst-package
+
+- name: Inst - Nomad
+  copy:
+    src: "{{ nomad_inst_dir }}/nomad"
+    dest: "{{ nomad_bin_dir }}"
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    force: true
+    mode: 0755
+    remote_src: true
+  tags:
+    - nomad-inst-package
+
+- name: Inst - Cleanup
+  file:
+    path: "{{ nomad_inst_dir }}/nomad"
+    state: "absent"
+  tags:
+    - nomad-inst-package
+
+- name: Conf - Create Directories "{{ nomad_data_dir }}"
+  file:
+    dest: "{{ nomad_data_dir }}"
+    state: directory
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+  tags:
+    - nomad-conf
+
+- name: Conf - Create Directories "{{ nomad_ssl_dir }}"
+  file:
+    dest: "{{ nomad_ssl_dir }}"
+    state: directory
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+  tags:
+    - nomad-conf
+
+- name: Conf - Create Config Directory
+  file:
+    dest: "{{ nomad_config_dir }}"
+    state: directory
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    mode: 0755
+  tags:
+    - nomad-conf
+
+- name: Conf - Base Configuration
+  template:
+    src: base.hcl.j2
+    dest: "{{ nomad_config_dir }}/base.hcl"
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    mode: 0644
+  tags:
+    - nomad-conf
+
+- name: Conf - Server Configuration
+  template:
+    src: server.hcl.j2
+    dest: "{{ nomad_config_dir }}/server.hcl"
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    mode: 0644
+  when:
+    - nomad_node_server | bool
+  tags:
+    - nomad-conf
+
+- name: Conf - Client Configuration
+  template:
+    src: client.hcl.j2
+    dest: "{{ nomad_config_dir }}/client.hcl"
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    mode: 0644
+  when:
+    - nomad_node_client | bool
+  tags:
+    - nomad-conf
+
+- name: Conf - Custom Configuration
+  template:
+    src: custom.json.j2
+    dest: "{{ nomad_config_dir }}/custom.json"
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    mode: 0644
+  when:
+    - nomad_config_custom is defined
+  tags:
+    - nomad-conf
+
+- name: Conf - Copy Certificates And Keys
+  copy:
+    content: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    mode: 0600
+  no_log: true
+  loop: "{{ nomad_certificates | flatten(levels=1) }}"
+  tags:
+    - nomad-conf
+
+- name: Conf - System.d Script
+  template:
+    src: "nomad_systemd.service.j2"
+    dest: "/lib/systemd/system/nomad.service"
+    owner: "root"
+    group: "root"
+    mode: 0644
+  notify:
+    - "Restart Nomad"
+  tags:
+    - nomad-conf
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/base.hcl.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/base.hcl.j2
new file mode 100644
index 0000000000..0e2b60c6d4
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/base.hcl.j2
@@ -0,0 +1,24 @@
+name = "{{ nomad_name }}"
+region = "{{ nomad_region }}"
+datacenter = "{{ nomad_datacenter }}"
+
+bind_addr = "{{ nomad_bind_addr }}"
+data_dir = "{{ nomad_data_dir }}"
+
+enable_syslog = {{ nomad_syslog_enable | bool | lower }}
+enable_debug = {{ nomad_enable_debug | bool | lower }}
+disable_update_check = {{ nomad_disable_update_check | bool | lower }}
+log_level = "{{ nomad_log_level }}"
+
+{% if ( nomad_ca_file ) and
+      ( nomad_cert_file ) and
+      ( nomad_key_file )
+%}
+tls {
+    http = {{ nomad_http | bool | lower }}
+    rpc = {{ nomad_rpc | bool | lower }}
+    ca_file = "{{ nomad_ca_file }}"
+    cert_file = "{{ nomad_cert_file }}"
+    key_file = "{{ nomad_key_file }}"
+}
+{% endif %}
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/client.hcl.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/client.hcl.j2
new file mode 100644
index 0000000000..91fd1c947f
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/client.hcl.j2
@@ -0,0 +1,18 @@
+client {
+    enabled = {{ nomad_node_client | bool | lower }}
+    no_host_uuid = {{ nomad_no_host_uuid | bool | lower }}
+    node_class = "{{ nomad_node_class }}"
+
+    {% if nomad_servers -%}
+    servers = [ {{ nomad_servers }} ]
+    {% endif %}
+
+    {% if nomad_options -%}
+    options = {
+    {% for key, value in nomad_options.items() %}
+    "{{ key }}" = "{{ value }}"
+    {% endfor -%}
+    }
+    {% endif %}
+
+}
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/custom.hcl.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/custom.hcl.j2
new file mode 100644
index 0000000000..37ff6f3496
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/custom.hcl.j2
@@ -0,0 +1,5 @@
+{% if nomad_config_custom -%}
+{{ nomad_config_custom | to_nice_json }}
+{% else %}
+{}
+{% endif %}
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/nomad_systemd.service.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/nomad_systemd.service.j2
new file mode 100644
index 0000000000..d0beed24b8
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/nomad_systemd.service.j2
@@ -0,0 +1,18 @@
+[Unit]
+Description=nomad agent
+Wants=network.target
+After=nomad.service
+
+[Service]
+# TODO: Decrease privilege
+User=root
+Group=root
+Environment="GOMAXPROCS=2"
+ExecStart={{ nomad_bin_dir }}/nomad agent -config={{ nomad_config_dir }}
+ExecReload=/bin/kill -9 $MAINPID
+KillSignal=SIGTERM
+Restart=on-failure
+RestartSec=1
+
+[Install]
+WantedBy=default.target
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/server.hcl.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/server.hcl.j2
new file mode 100644
index 0000000000..c28a43c5d2
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/server.hcl.j2
@@ -0,0 +1,4 @@
+server {
+  enabled          = false
+  encrypt = "Y4T+5JGx1C3l2NFBBvkTWQ=="
+}
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/vars/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/vars/main.yaml
new file mode 100644
index 0000000000..a72222c992
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/vars/main.yaml
@@ -0,0 +1,5 @@
+---
+# file: roles/nomad/vars/main.yaml
+
+nomad_node_client: "{{ (nomad_node_role == 'client') or (nomad_node_role == 'both') }}"
+nomad_node_server: "{{ (nomad_node_role == 'server') or (nomad_node_role == 'both') }}"
diff --git a/resources/tools/testbed-setup/ansible/roles/user_add/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/user_add/defaults/main.yaml
index 9e5aaf272c..56f5098f12 100644
--- a/resources/tools/testbed-setup/ansible/roles/user_add/defaults/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/user_add/defaults/main.yaml
@@ -1,5 +1,11 @@
 ---
 # file: roles/user_add/defaults/main.yaml
 
-user_pass: "$6$zpBUdQ4q$P2zKclumvCndWujgP/qQ8eMk3YZk7ESAom04Fqp26hJH2jWkMXEX..jqxzMdDLJKiDaDHIaSkQMVjHzd3cRLs1"
+# Default shell for a user if none is specified.
+users_shell: /bin/bash
 
+# Default create home dirs for new users.
+users_create_homedirs: true
+
+# Default list of users to create.
+users: []
diff --git a/resources/tools/testbed-setup/ansible/roles/user_add/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/user_add/handlers/main.yaml
index e6b7d82844..960f573b48 100644
--- a/resources/tools/testbed-setup/ansible/roles/user_add/handlers/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/user_add/handlers/main.yaml
@@ -1,10 +1,7 @@
 ---
 # file: roles/user_add/handlers/main.yaml
 
-- name: Restart sshd
+- name: Restart SSHd
   service:
     name: sshd
     state: restarted
-  tags:
-    - restart-sshd
-
diff --git a/resources/tools/testbed-setup/ansible/roles/user_add/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/user_add/tasks/main.yaml
index 2672996202..8323284261 100644
--- a/resources/tools/testbed-setup/ansible/roles/user_add/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/user_add/tasks/main.yaml
@@ -1,31 +1,49 @@
 ---
 # file: roles/user_add/tasks/main.yaml
 
-- name: Add testuser account
+- name: Conf - Add User
   user:
-    name: "testuser"
+    append: "{{ item.append | default(omit) }}"
+    createhome: "{{ 'yes' if users_create_homedirs else 'no' }}"
+    generate_ssh_key: "{{ item.generate_ssh_key | default(omit) }}"
+    group: "{{ item.group | default(item.username) }}"
+    groups: "{{ item.groups | join(',') if 'groups' in item else '' }}"
+    name: "{{ item.username }}"
+    password: "{{ item.password if item.password is defined else '!' }}"
+    shell: "{{ item.shell if item.shell is defined else users_shell }}"
     state: present
-    shell: "/bin/bash"
-    password: "{{ user_pass }}"
+  with_items: "{{ users }}"
   tags:
-    - add-user
+    - user-add-conf
 
-- name: Allow password login
+- name: Conf - SSH keys
+  authorized_key:
+    user: "{{ item.0.username }}"
+    key: "{{ item.1 }}"
+  with_subelements:
+    - "{{ users }}"
+    - ssh_key
+    - skip_missing: yes
+  tags:
+    - user-add-conf
+
+- name: Conf - Allow Password Login
   lineinfile:
     dest: "/etc/ssh/sshd_config"
     regexp: "^PasswordAuthentication no"
     line: "PasswordAuthentication yes"
   notify:
-    - "Restart sshd"
+    - "Restart SSHd"
   tags:
-    - allow-password-login
+    - user-add-conf
 
-- name: Add visudo entry
+- name: Conf - Add Visudo Entry
   lineinfile:
     dest: "/etc/sudoers"
     state: present
-    line: "testuser ALL=(ALL) NOPASSWD: ALL"
+    line: "{{ item.username }} ALL=(ALL) NOPASSWD: ALL"
     validate: "visudo -cf %s"
+  with_items: "{{ users }}"
   tags:
-    - allow-sudo
+    - user-add-conf