Infra: Ansible Nomad

Signed-off-by: pmikus <pmikus@cisco.com>
Change-Id: Icc03bf62d8639f603ca90aa6ab849a46dc911d56

21 files changed, 915 insertions, 183 deletions

diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml
new file mode 100644
index 0000000000..aca60b645e
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml
@@ -0,0 +1,29 @@
+---
+# file: host_vars/10.30.51.28.yaml
+
+hostname: "s41-nomad"
+inventory_cimc_hostname: '10.30.50.28'
+# User managmeent.
+users:
+  - username: localadmin
+    groups: [adm, sudo]
+    password: "$6$FIsbVDQR$5D0wgufOd2FtnmOiRNsGlgg6Loh.0x3dWSj72DSQnqisSyE9DROfgSgA6s0yxDwz4Jd5SRTXiTKuRYuSQ5POI1"
+    ssh_key:
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com"
+  - username: testuser
+    groups: [adm, sudo]
+    password: "$6$zpBUdQ4q$P2zKclumvCndWujgP/qQ8eMk3YZk7ESAom04Fqp26hJH2jWkMXEX..jqxzMdDLJKiDaDHIaSkQMVjHzd3cRLs1"
+    ssh_key:
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com"
+
+# Nomad settings.
+nomad_datacenter: "yul1"
+nomad_name: "s40-nomad-{{ ansible_architecture }}"
+nomad_node_role: "client"
+nomad_node_class: "builder"
+nomad_options:
+  driver.raw_exec.enable: 1
+  docker.cleanup.image: false
+  docker.privileged.enabled: true
+  driver.whitelist: "docker,raw_exec,exec"
+nomad_servers: "\"10.30.51.32:4647\", \"10.30.51.33:4647\""
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml
new file mode 100644
index 0000000000..2eb2dfc88c
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml
@@ -0,0 +1,30 @@
+---
+# file: host_vars/10.30.51.29.yaml
+
+hostname: "s39-nomad"
+inventory_cimc_hostname: "10.30.50.29"
+
+# User managmeent.
+users:
+  - username: localadmin
+    groups: [adm, sudo]
+    password: "$6$FIsbVDQR$5D0wgufOd2FtnmOiRNsGlgg6Loh.0x3dWSj72DSQnqisSyE9DROfgSgA6s0yxDwz4Jd5SRTXiTKuRYuSQ5POI1"
+    ssh_key:
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com"
+  - username: testuser
+    groups: [adm, sudo]
+    password: "$6$zpBUdQ4q$P2zKclumvCndWujgP/qQ8eMk3YZk7ESAom04Fqp26hJH2jWkMXEX..jqxzMdDLJKiDaDHIaSkQMVjHzd3cRLs1"
+    ssh_key:
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com"
+
+# Nomad settings.
+nomad_datacenter: "yul1"
+nomad_name: "s39-nomad-{{ ansible_architecture }}"
+nomad_node_role: "client"
+nomad_node_class: "builder"
+nomad_options:
+  driver.raw_exec.enable: 1
+  docker.cleanup.image: false
+  docker.privileged.enabled: true
+  driver.whitelist: "docker,raw_exec,exec"
+nomad_servers: "\"10.30.51.32:4647\", \"10.30.51.33:4647\""
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml
new file mode 100644
index 0000000000..3aed1c8b52
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml
@@ -0,0 +1,30 @@
+---
+# file: host_vars/10.30.51.30.yaml
+
+hostname: "s40-nomad"
+inventory_cimc_hostname: '10.30.50.30'
+
+# User managmeent.
+users:
+  - username: localadmin
+    groups: [adm, sudo]
+    password: "$6$FIsbVDQR$5D0wgufOd2FtnmOiRNsGlgg6Loh.0x3dWSj72DSQnqisSyE9DROfgSgA6s0yxDwz4Jd5SRTXiTKuRYuSQ5POI1"
+    ssh_key:
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com"
+  - username: testuser
+    groups: [adm, sudo]
+    password: "$6$zpBUdQ4q$P2zKclumvCndWujgP/qQ8eMk3YZk7ESAom04Fqp26hJH2jWkMXEX..jqxzMdDLJKiDaDHIaSkQMVjHzd3cRLs1"
+    ssh_key:
+      - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com"
+
+# Nomad settings.
+nomad_datacenter: "yul1"
+nomad_name: "s40-nomad-{{ ansible_architecture }}"
+nomad_node_role: "both"
+nomad_node_class: "builder"
+nomad_options:
+  driver.raw_exec.enable: 1
+  docker.cleanup.image: false
+  docker.privileged.enabled: true
+  driver.whitelist: "docker,raw_exec,exec"
+nomad_servers: "\"10.30.51.32:4647\", \"10.30.51.33:4647\""
diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts
index 0402c68109..52f0076082 100644
--- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts
+++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts
@@ -45,3 +45,8 @@ all:
         10.30.51.50: #s1-t11-sut1 - skylake
         10.30.51.51: #s2-t12-sut1 - skylake
         10.30.51.69: #s27-t13-sut1 - thunderx2
+    nomad:
+      hosts:
+        10.30.51.28: #s41-nomad - haswell
+        10.30.51.29: #s39-nomad - haswell
+        10.30.51.30: #s40-nomad - haswell
diff --git a/resources/tools/testbed-setup/ansible/nomad.yaml b/resources/tools/testbed-setup/ansible/nomad.yaml
new file mode 100644
index 0000000000..b09245cfdb
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/nomad.yaml
@@ -0,0 +1,14 @@
+---
+# file: nomad.yaml
+
+- hosts: nomad
+  remote_user: testuser
+  become: yes
+  become_user: root
+  roles:
+    - role: user_add
+      tags: user_add
+    - role: docker
+      tags: docker
+    - role: nomad
+      tags: nomad
diff --git a/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml
index e91442718e..2fb6e2e213 100644
--- a/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml
@@ -51,3 +51,10 @@
   tags:
     - install-mellanox
 
+- name: Mellanox Install - FIX qemu-system removal
+  package:
+    name: "qemu-system"
+    state: latest
+    update_cache: true
+  tags:
+    - install-mellanox
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/defaults/main.yaml
new file mode 100644
index 0000000000..ad8b36ee66
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/defaults/main.yaml
@@ -0,0 +1,103 @@
+---
+# file: roles/nomad/defaults/main.yaml
+
+# Inst - Prerequisites.
+packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}"
+
+packages_base:
+  - "cgroup-bin"
+  - "curl"
+  - "git"
+  - "libcgroup1"
+  - "unzip"
+packages_by_distro:
+  ubuntu:
+    - []
+packages_by_arch:
+  aarch64:
+    - []
+  x86_64:
+    - []
+
+# Inst - Download Nomad.
+nomad_architecture_map:
+  amd64: "amd64"
+  x86_64: "amd64"
+  armv7l: "arm"
+  aarch64: "arm64"
+  32-bit: "386"
+  64-bit: "amd64"
+nomad_architecture: "{{ nomad_architecture_map[ansible_architecture] }}"
+nomad_version: "0.10.2"
+nomad_pkg: "nomad_{{ nomad_version }}_linux_{{ nomad_architecture }}.zip"
+nomad_zip_url: "https://releases.hashicorp.com/nomad/{{ nomad_version }}/{{ nomad_pkg }}"
+
+# Inst - System paths.
+nomad_bin_dir: "/usr/local/bin"
+nomad_config_dir: "/etc/nomad.d"
+nomad_data_dir: "/var/nomad"
+nomad_inst_dir: "/opt"
+nomad_lockfile: "/var/lock/subsys/nomad"
+nomad_run_dir: "/var/run/nomad"
+nomad_ssl_dir: "/etc/nomad.d/ssl"
+
+# Conf - Service.
+nomad_node_role: "both"
+nomad_restart_handler_state: "restarted"
+
+# Conf - User and group.
+nomad_group: "nomad"
+nomad_group_state: "present"
+nomad_manage_group: true
+nomad_manage_user: true
+nomad_user: "nomad"
+nomad_user_groups: [ docker, nomad, root ]
+nomad_user_state: "present"
+
+# Conf - base.hcl
+nomad_bind_addr: "0.0.0.0"
+nomad_datacenter: "dc1"
+nomad_disable_update_check: true
+nomad_enable_debug: false
+nomad_log_level: "INFO"
+nomad_name: "{{ inventory_hostname }}"
+nomad_region: "global"
+nomad_syslog_enable: true
+
+# Conf - base.hcl (TLS)
+nomad_ca_file: "{{ nomad_ssl_dir }}/ca.pem"
+nomad_cert_file: "{{ nomad_ssl_dir }}/nomad.pem"
+nomad_http: false
+nomad_key_file: "{{ nomad_ssl_dir }}/nomad-key.pem"
+nomad_rpc: false
+
+# Conf - client.hcl
+nomad_certificates:
+  - src: "{{ vault_nomad_ca_file }}"
+    dest: "{{ nomad_ca_file }}"
+  - src: "{{ vault_nomad_cert_file }}"
+    dest: "{{ nomad_cert_file }}"
+  - src: "{{ vault_nomad_key_file }}"
+    dest: "{{ nomad_key_file }}"
+nomad_node_class: ""
+nomad_no_host_uuid: true
+nomad_options: {}
+nomad_servers: []
+
+# Conf - custom.hcl
+# empty
+
+# Conf - server.hcl
+#nomad_retry_max: 0
+#nomad_retry_join: false
+#nomad_retry_interval: "30s"
+#nomad_rejoin_after_leave: false
+#nomad_enabled_schedulers:
+#  - service
+#  - batch
+#  - system
+#nomad_num_schedulers: "{{ ansible_processor_vcpus }}"
+#nomad_node_gc_threshold: "24h"
+#nomad_job_gc_threshold: "4h"
+#nomad_eval_gc_threshold: "1h"
+#nomad_encrypt: ""
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/handlers/main.yaml
new file mode 100644
index 0000000000..f0bcee9142
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/handlers/main.yaml
@@ -0,0 +1,10 @@
+---
+# file roles/nomad/handlers/main.yaml
+
+- name: Restart Nomad
+  systemd:
+    daemon_reload: true
+    enabled: true
+    name: "nomad"
+    state: "{{ nomad_restart_handler_state }}"
+
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/meta/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/meta/main.yaml
new file mode 100644
index 0000000000..9fc40d9ae1
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/meta/main.yaml
@@ -0,0 +1,9 @@
+---
+# file: roles/nomad/meta/main.yaml
+
+# desc: Install nomad from stable branch and configure service.
+# inst: Nomad
+# conf: ?
+# info: 1.0 - added role
+
+dependencies: [ docker ]
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml
new file mode 100644
index 0000000000..9093afd08e
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml
@@ -0,0 +1,164 @@
+---
+# file: roles/nomad/tasks/main.yaml
+
+- name: Inst - Prerequisites
+  package:
+    name: "{{ packages | flatten(levels=1) }}"
+    state: latest
+    update_cache: true
+  tags:
+    - nomad-inst-prerequisites
+
+- name: Conf - Add Nomad Group
+  group:
+    name: "{{ nomad_group }}"
+    state: "{{ nomad_group_state }}"
+  when:
+    - nomad_manage_group | bool
+  tags:
+    - nomad-conf-user
+
+- name: Conf - Add Nomad user
+  user:
+    name: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    groups: "{{ nomad_user_groups }}"
+    state: "{{ nomad_user_state }}"
+    system: true
+  when:
+    - nomad_manage_user | bool
+  tags:
+    - nomad-conf-user
+
+- name: Inst - Download Nomad
+  get_url:
+    url: "{{ nomad_zip_url }}"
+    dest: "{{ nomad_inst_dir }}/{{ nomad_pkg }}"
+  tags:
+    - nomad-inst-package
+
+- name: Inst - Unarchive Nomad
+  unarchive:
+    src: "{{ nomad_inst_dir }}/{{ nomad_pkg }}"
+    dest: "{{ nomad_inst_dir }}/"
+    creates: "{{ nomad_inst_dir }}/nomad"
+    remote_src: true
+  tags:
+    - nomad-inst-package
+
+- name: Inst - Nomad
+  copy:
+    src: "{{ nomad_inst_dir }}/nomad"
+    dest: "{{ nomad_bin_dir }}"
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    force: true
+    mode: 0755
+    remote_src: true
+  tags:
+    - nomad-inst-package
+
+- name: Inst - Cleanup
+  file:
+    path: "{{ nomad_inst_dir }}/nomad"
+    state: "absent"
+  tags:
+    - nomad-inst-package
+
+- name: Conf - Create Directories "{{ nomad_data_dir }}"
+  file:
+    dest: "{{ nomad_data_dir }}"
+    state: directory
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+  tags:
+    - nomad-conf
+
+- name: Conf - Create Directories "{{ nomad_ssl_dir }}"
+  file:
+    dest: "{{ nomad_ssl_dir }}"
+    state: directory
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+  tags:
+    - nomad-conf
+
+- name: Conf - Create Config Directory
+  file:
+    dest: "{{ nomad_config_dir }}"
+    state: directory
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    mode: 0755
+  tags:
+    - nomad-conf
+
+- name: Conf - Base Configuration
+  template:
+    src: base.hcl.j2
+    dest: "{{ nomad_config_dir }}/base.hcl"
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    mode: 0644
+  tags:
+    - nomad-conf
+
+- name: Conf - Server Configuration
+  template:
+    src: server.hcl.j2
+    dest: "{{ nomad_config_dir }}/server.hcl"
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    mode: 0644
+  when:
+    - nomad_node_server | bool
+  tags:
+    - nomad-conf
+
+- name: Conf - Client Configuration
+  template:
+    src: client.hcl.j2
+    dest: "{{ nomad_config_dir }}/client.hcl"
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    mode: 0644
+  when:
+    - nomad_node_client | bool
+  tags:
+    - nomad-conf
+
+- name: Conf - Custom Configuration
+  template:
+    src: custom.json.j2
+    dest: "{{ nomad_config_dir }}/custom.json"
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    mode: 0644
+  when:
+    - nomad_config_custom is defined
+  tags:
+    - nomad-conf
+
+- name: Conf - Copy Certificates And Keys
+  copy:
+    content: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    owner: "{{ nomad_user }}"
+    group: "{{ nomad_group }}"
+    mode: 0600
+  no_log: true
+  loop: "{{ nomad_certificates | flatten(levels=1) }}"
+  tags:
+    - nomad-conf
+
+- name: Conf - System.d Script
+  template:
+    src: "nomad_systemd.service.j2"
+    dest: "/lib/systemd/system/nomad.service"
+    owner: "root"
+    group: "root"
+    mode: 0644
+  notify:
+    - "Restart Nomad"
+  tags:
+    - nomad-conf
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/base.hcl.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/base.hcl.j2
new file mode 100644
index 0000000000..0e2b60c6d4
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/base.hcl.j2
@@ -0,0 +1,24 @@
+name = "{{ nomad_name }}"
+region = "{{ nomad_region }}"
+datacenter = "{{ nomad_datacenter }}"
+
+bind_addr = "{{ nomad_bind_addr }}"
+data_dir = "{{ nomad_data_dir }}"
+
+enable_syslog = {{ nomad_syslog_enable | bool | lower }}
+enable_debug = {{ nomad_enable_debug | bool | lower }}
+disable_update_check = {{ nomad_disable_update_check | bool | lower }}
+log_level = "{{ nomad_log_level }}"
+
+{% if ( nomad_ca_file ) and
+      ( nomad_cert_file ) and
+      ( nomad_key_file )
+%}
+tls {
+    http = {{ nomad_http | bool | lower }}
+    rpc = {{ nomad_rpc | bool | lower }}
+    ca_file = "{{ nomad_ca_file }}"
+    cert_file = "{{ nomad_cert_file }}"
+    key_file = "{{ nomad_key_file }}"
+}
+{% endif %}
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/client.hcl.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/client.hcl.j2
new file mode 100644
index 0000000000..91fd1c947f
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/client.hcl.j2
@@ -0,0 +1,18 @@
+client {
+    enabled = {{ nomad_node_client | bool | lower }}
+    no_host_uuid = {{ nomad_no_host_uuid | bool | lower }}
+    node_class = "{{ nomad_node_class }}"
+
+    {% if nomad_servers -%}
+    servers = [ {{ nomad_servers }} ]
+    {% endif %}
+
+    {% if nomad_options -%}
+    options = {
+    {% for key, value in nomad_options.items() %}
+    "{{ key }}" = "{{ value }}"
+    {% endfor -%}
+    }
+    {% endif %}
+
+}
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/custom.hcl.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/custom.hcl.j2
new file mode 100644
index 0000000000..37ff6f3496
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/custom.hcl.j2
@@ -0,0 +1,5 @@
+{% if nomad_config_custom -%}
+{{ nomad_config_custom | to_nice_json }}
+{% else %}
+{}
+{% endif %}
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/nomad_systemd.service.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/nomad_systemd.service.j2
new file mode 100644
index 0000000000..d0beed24b8
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/nomad_systemd.service.j2
@@ -0,0 +1,18 @@
+[Unit]
+Description=nomad agent
+Wants=network.target
+After=nomad.service
+
+[Service]
+# TODO: Decrease privilege
+User=root
+Group=root
+Environment="GOMAXPROCS=2"
+ExecStart={{ nomad_bin_dir }}/nomad agent -config={{ nomad_config_dir }}
+ExecReload=/bin/kill -9 $MAINPID
+KillSignal=SIGTERM
+Restart=on-failure
+RestartSec=1
+
+[Install]
+WantedBy=default.target
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/server.hcl.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/server.hcl.j2
new file mode 100644
index 0000000000..c28a43c5d2
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/server.hcl.j2
@@ -0,0 +1,4 @@
+server {
+  enabled          = false
+  encrypt = "Y4T+5JGx1C3l2NFBBvkTWQ=="
+}
diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/vars/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/vars/main.yaml
new file mode 100644
index 0000000000..a72222c992
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/nomad/vars/main.yaml
@@ -0,0 +1,5 @@
+---
+# file: roles/nomad/vars/main.yaml
+
+nomad_node_client: "{{ (nomad_node_role == 'client') or (nomad_node_role == 'both') }}"
+nomad_node_server: "{{ (nomad_node_role == 'server') or (nomad_node_role == 'both') }}"
diff --git a/resources/tools/testbed-setup/ansible/roles/user_add/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/user_add/defaults/main.yaml
index 9e5aaf272c..56f5098f12 100644
--- a/resources/tools/testbed-setup/ansible/roles/user_add/defaults/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/user_add/defaults/main.yaml
@@ -1,5 +1,11 @@
 ---
 # file: roles/user_add/defaults/main.yaml
 
-user_pass: "$6$zpBUdQ4q$P2zKclumvCndWujgP/qQ8eMk3YZk7ESAom04Fqp26hJH2jWkMXEX..jqxzMdDLJKiDaDHIaSkQMVjHzd3cRLs1"
+# Default shell for a user if none is specified.
+users_shell: /bin/bash
 
+# Default create home dirs for new users.
+users_create_homedirs: true
+
+# Default list of users to create.
+users: []
diff --git a/resources/tools/testbed-setup/ansible/roles/user_add/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/user_add/handlers/main.yaml
index e6b7d82844..960f573b48 100644
--- a/resources/tools/testbed-setup/ansible/roles/user_add/handlers/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/user_add/handlers/main.yaml
@@ -1,10 +1,7 @@
 ---
 # file: roles/user_add/handlers/main.yaml
 
-- name: Restart sshd
+- name: Restart SSHd
   service:
     name: sshd
     state: restarted
-  tags:
-    - restart-sshd
-
diff --git a/resources/tools/testbed-setup/ansible/roles/user_add/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/user_add/tasks/main.yaml
index 2672996202..8323284261 100644
--- a/resources/tools/testbed-setup/ansible/roles/user_add/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/user_add/tasks/main.yaml
@@ -1,31 +1,49 @@
 ---
 # file: roles/user_add/tasks/main.yaml
 
-- name: Add testuser account
+- name: Conf - Add User
   user:
-    name: "testuser"
+    append: "{{ item.append | default(omit) }}"
+    createhome: "{{ 'yes' if users_create_homedirs else 'no' }}"
+    generate_ssh_key: "{{ item.generate_ssh_key | default(omit) }}"
+    group: "{{ item.group | default(item.username) }}"
+    groups: "{{ item.groups | join(',') if 'groups' in item else '' }}"
+    name: "{{ item.username }}"
+    password: "{{ item.password if item.password is defined else '!' }}"
+    shell: "{{ item.shell if item.shell is defined else users_shell }}"
     state: present
-    shell: "/bin/bash"
-    password: "{{ user_pass }}"
+  with_items: "{{ users }}"
   tags:
-    - add-user
+    - user-add-conf
 
-- name: Allow password login
+- name: Conf - SSH keys
+  authorized_key:
+    user: "{{ item.0.username }}"
+    key: "{{ item.1 }}"
+  with_subelements:
+    - "{{ users }}"
+    - ssh_key
+    - skip_missing: yes
+  tags:
+    - user-add-conf
+
+- name: Conf - Allow Password Login
   lineinfile:
     dest: "/etc/ssh/sshd_config"
     regexp: "^PasswordAuthentication no"
     line: "PasswordAuthentication yes"
   notify:
-    - "Restart sshd"
+    - "Restart SSHd"
   tags:
-    - allow-password-login
+    - user-add-conf
 
-- name: Add visudo entry
+- name: Conf - Add Visudo Entry
   lineinfile:
     dest: "/etc/sudoers"
     state: present
-    line: "testuser ALL=(ALL) NOPASSWD: ALL"
+    line: "{{ item.username }} ALL=(ALL) NOPASSWD: ALL"
     validate: "visudo -cf %s"
+  with_items: "{{ users }}"
   tags:
-    - allow-sudo
+    - user-add-conf
 
diff --git a/resources/tools/testbed-setup/ansible/site.yaml b/resources/tools/testbed-setup/ansible/site.yaml
index f2f68c6bb7..6ea7fa1d0f 100644
--- a/resources/tools/testbed-setup/ansible/site.yaml
+++ b/resources/tools/testbed-setup/ansible/site.yaml
@@ -9,3 +9,6 @@
 
 - import_playbook: vpp_device.yaml
   tags: vpp-device
+
+- import_playbook: nomad.yaml
+  tags: nomad
diff --git a/resources/tools/testbed-setup/ansible/vault.yml b/resources/tools/testbed-setup/ansible/vault.yml
index 56f15d84d9..c2dfbc9476 100644
--- a/resources/tools/testbed-setup/ansible/vault.yml
+++ b/resources/tools/testbed-setup/ansible/vault.yml
@@ -1,168 +1,401 @@
 $ANSIBLE_VAULT;1.1;AES256
-65646237323436373833326439303135393936363466383764663164336365623832653531376234
-6438303331653966383964383663396536663761666563320a633065656366336466643330303936
-33653539343061343839353736666330326434383430616638666338306332326636316138656164
-6336303733653563340a613062363264393031633339363263396631663562666262663136393239
-34393763663737666338373136633239666332376464616465376237353732356334336338636231
-64383237313136363461323734383665306537383566313366636562346562383364636663636364
-62626136653666636264643030333332666131663230643531303932333233643638356366613363
-61613933386664373066626466336330633332613566633535346237623135303238643965373963
-37386335303934306531653036636130313036386533343437313034616663623462386330663662
-66663238353665666363353939373066633933353539373335336239383762376562623465633166
-30343034643863306132323936323231356233306463656130353536396366343938643165386161
-65613230336237383963623638613437623136613530353439653133373466313434623530653666
-34623866636265323030366230356632663934393035646339646462633566396231633038653134
-66316333383336653238333737396262613535623266343733343830653531346432323238333535
-34333739313730633037363232616161313231626166333462383764396235326637353732316335
-39376161373031656236623939376333343530356361643134366563316662373938346338633265
-31366633346136663239353338623562316532316164636638393634303630613261303238633731
-31396236326235323962613937363134666537653231663836653362396532616235633939396134
-64313235613463623263303233373931366337393333393462343035313530363836333466616238
-35363866653563346231613361303162616361363839663738393565643666383765313333363136
-65333563666532383264366132643064626638373961633465613932616637616438623938653036
-62666330633330623632646636623262393330383166666139393164336237303738343132653166
-36613836386565346263663966613139346463313036366138313465363064356563393761346164
-31323131613132303339343162386266303561623064383465386138623933643832343263363364
-64653330643161343263653361356436653332316561643863386334303231616665326364336439
-62396564336332383939353932313236346664316266323139643461623062663031343462383432
-61306437663838353334626365336537663563316264346634623336633331643330363963633838
-39333662623237633565353139396632633034313463303264313362646639356439356232376333
-33376265386438663164626262313630303662396537316135646562363438633163613339323639
-65376266333037303066346533353739303465353937323864663862303832363261393066383363
-66316433663533396438663839336563646265666335656364333966383763363563383437366138
-66393563313835633532613735366561653433343164346563383430323164383333616338386466
-61656365333431646238646534306130366535623266386432303035363337653865633561643137
-34613162616131346361306539653330346637386631646335383233353136353133396637666339
-37356631376638323761363261303438653636353539336535646263666266643338626530363563
-62613961343461333531366162313536656532333763633861373931336165396238393763656561
-36366631363731353837663634316666323436396163636333373037346331386362393633323835
-32306533376366633363653363376666313963306639613366653562363762636236363064353733
-30633832383330313661666135646531346330393535613834323431303334646632336239343532
-34623330663630373562373432643334613664316165303736666432373164613764353536366535
-35346464633635636162663230386363356465306164616234396331626537633462623433313637
-32396431343666643663383931353133353137633431346531313263643934643039363734383738
-31333438666165353739356633653432613061303965353665323665626330633764386339343639
-31383763386465343836666266363338663265386262633062343434383139336162393035623562
-66333034343264643036386130346234396433633238343838663230303630613663363137323362
-32306264303864323438353363636664313935383565636135653264663538313432376639643663
-62323161623532656433643238613939363933396334353438663235626632663936346536323964
-39313865356262373463656638646631653934366532383834643266323530316135356235366539
-63643361353864653437353532383134343861633566623331363561323732323639313738386231
-33643931363535393765363036343334323464303436626332313064626338356565396631633761
-66306530356336363062663763356337333164303564363630303866333030303337303562623738
-34636230643233353437643763623435323466393632623939353533303163633763383835343535
-36386230313230383532376639356564646139613338633437363161666261643833376263666263
-66323762373839316131343834623532313232646132343834326363663731313566333331646137
-64663331646338333765383236363032366166316464366431323137363263346636616265353366
-34336530303132316639663036396438336466656361393632303636656265386435393134643938
-30346663326632376638363433363866366163626266633966333530626631346330323038666635
-39333135306362663364373532386662623435343136346564386666613137643663303732373831
-37383032383866623565353966396533363234653665346332343833383234316138333032626161
-34633836613630333066353732393534373338376432363061326538613336613665336461653666
-65343461653835623365376338326137303262353239333263353231663836313139613333306534
-64643237623262303532396163353939303661643762383436626664663765353763383834383663
-62386338333338616130353936316130336633626165393737303162316131313266633762653830
-30636635333963663439356530663638663336393230383237626165343037666563306663336364
-31653931313730303930383663646666613766303464376532663866363362336535346130633961
-63353661366664653164663730373165353839336635323136306266346363356561656535306239
-62646631396365346534326164326564386464386636373632646435613434383161343166383363
-31646639353038626563636631383438333832623834343233306335316266663734343636643438
-37346531646137383438393238393137356366663363363962636635353561656532653231666438
-31353339663736386638313135636136666364306562323134313064626438353037336531613065
-33336530656339636231393236393033643930396432653637326436643135363639396661663764
-37373835626531373831303837613231643336333861663934306364386135616561653264616439
-63633465653637336561373836646436383566353033636237313230393831306535383464643161
-34326164326633353033336435333630353836626530303830656231353061393038393735616339
-32313039343961623636373336616564666230343961383935616536616231376361393338326530
-33373338383234663130373563636162393966623036363531623461633631656161613166366261
-39363163666463316435363938623135326361363466346266333833396135353839306662303939
-38383431386139326530376537333264346264373565303735363861656637373234333965613639
-63336635316632353130356138666465356334623033356339636132393366366265313934643335
-32306435653666663035323463623239633939616464666132356361613035313861356262336632
-38303032666537623761633737633564383332336132663061653037353239623836356163316564
-33653131353436313565653531363762353336373363316464343365346265636439353261363134
-39313363323635653264343131653666623963366366616266663566636239373232333766663962
-63393630643166393761313839616236373631383166613564633564613235613338323031333762
-61326134353938393764343063363764366633663462383533633734386363313266326161303762
-35386637356466653037626262613439376165616463366338346338646563353637376130313438
-61656638393239343039613064633438393063623232663031643839323734643432636633303264
-63336536333063633262376239646536636435623863336236323836363939633737613232313564
-30303165303235653664626133333436393836366437383463313530326666656363393139393662
-36623438633230353563343731633035626565343963373630316132393564363230333866366239
-61633363356531663338613236343466393834666433366139313636653730303966656633373135
-64613966323833393465303339353637363766323965373631343337333564386138383630303935
-39643132313135396461323361643265393064363966393234663662396132316639616230326265
-33346463373935373833643764623231363134636337336365383038656437326333303138623566
-64663562316166356161353830666231356635653332366238353135323465653739306638353261
-35636430656164353963303435303736373833336439653133666166616332666436393637646537
-39373335333337343931353535363534636434316130376635353831386461643536623835313332
-31646564373533333031353365626433613762636463613336616633343230643764306334643662
-36653833336236663834323833663533653439653731323435336665306133646432376437613761
-33306533663135333532323838653130633361653033383334386435343832663434383031396637
-37373662373239363136396332633762616137326233646264373661383464333132343963383338
-30336331613439346138643531643333356135316538653331353530353232643537313661386566
-32633031613236366264626466633033336431346134326338656534303634356238653164366532
-34633466643235303362663130356232636233626666353161663561623634313966326630343466
-32306364613634623930656538336634336430306235356536663361343264366136306230313738
-62353436353638656664353438663361313235356365396336356464353832306238626162396330
-61323262303337383239646238396364623634386635643764353330346433663665383561326339
-61616164613931363337656264353439373033366133303835393637623265323032653361303961
-38353665643264396439616363323761626538643465333463363032616333306238326332303231
-34336461396664643332323335363266323036373036666235656165653736393836393365303530
-38386564346231623238313030396339666262383638666539653138356139316664333364313031
-31346133343731663234653465383961396465616432313432653466303337383332316133323662
-61363731313662313035373136613761653030613139363133316331356532393133393332613039
-31613934346365633935373761303566656231303766303130363337626431396661343335643463
-32376235353466353831346265316336363961386132643531393831373830343230366531303933
-35646535386132643964393832363239353637383838336336653230353036663461653935666463
-33333938636237636135396637616666353036313332353666303166623531353838323361336364
-35656132616130363133366231323164323635313632353835623236373638353237386535346336
-31323661636366346232616563366230336562326132646665636562616561383665643561393232
-62623864323533656636326264626230666633666563663833386266616233313638653836623438
-32393162396234353262356232666462643466653461633532343638323533656633306335623565
-38316330323438383837646265323832613933316135326234383864333737653735666438323965
-61326266646663363162303036343139623265303331623239356466333430356431333664376339
-61636631316436656337353833336630373036383736353166653434393433333664376562346631
-34306662363935613633323539326164666130363538643038366265373734643961656465623830
-39633862383636643939336232323832613832383465666239343236363566343363323836616661
-65643364663538316138383038626161303331653832656337303933356664383435376336313832
-30313832653064316239356661323138383762383864613233333530623131376237356136373932
-34636531316364326563373931306635323964616563643563616634643164373765336132613330
-30653862373966333630613635393234356338346138326135306635373666613533626135396437
-65623865666634653665656364393532633766326661313136356461316661323663363838633164
-31386234633438636165373139643337636532363036643632366366636236613164336638663839
-61326462303234326561376636613463316432653762656236396330303465303065633534356433
-62616438336338646437666564336166386333623030306136613236653334306634633666363237
-30333333343032636333383238643732373034646362373831386162316431383235326132363539
-35623737636461303935323962373362323235356666353561663234373730383262343464653265
-61333138326466303230363936383862313334633130336639396630653233623738366261626133
-64356635643030353263623936623766653438666466393736386365383431666136303666663164
-64633932393362356638646333623266353938383564376238643465623165666330313936336566
-64613333633862316234363830346264306565666236393735346466316364306365636164643937
-34613335343532343161303439616464623331383133313133616630366464383838643030633366
-38396230613332306131323434636336636535306437643332343937626336353438343434653539
-32316536386563623537366236653639663138386238623637633836323538316336386365366162
-62346534386332646430656132653761306164653732323461303763363736636330366561666564
-35623536343139666339613961396633363562333637383230626566313133666237643366323930
-35626231323364336531353733313931306163643733386565623536643766303535666666616536
-30636231633330376136306134383531643833663962306433663064383538646133396334616364
-33333630343263353736626330623637393731376138363730323963373661643534623533643863
-31363966613963646334303737616562613537616661383461363136373936623433666364393039
-61306139383531646630653461333231373963626538656166396431666130313361373265666339
-34333762396666353330366334396465353731613734383265336365396634313835666363666531
-63393831336466626334393834323165333165383161306531616532316565316633306538333566
-65353337343862333065393861313464383632666664376266373365613066323037616566393035
-64326135353038313636333163663562646332633066316232653532623265336437356362376364
-32656361313332613261353232396164633332616163343333623236333639613938326633323132
-61653432343436353539383463346365353231353935646431353437313437313139336433383365
-37383763613437636435376265343039376266623534653539613438353335363937316137323335
-39396537316662613362636263363736646332396233666362613933633464343762346332366339
-63376335316261363837346238303361616534306237366632323933333965366632303235646636
-31306263653766323935306139333930376339366534363634646266623831356333353566623137
-39393561643630383262336432386266303531666630356433613233623334663834616264633064
-34626335383166373933393461353738333863643230343834356166653538653135343937376137
-31633537386133313335343164623833383833303632376565643536616363643830376462346231
-38353431653330366435623062303634663332613038616563336364656333346537663365326161
-65323437623132376235343234666263383566383134366566313633623431333834303762353138
-34623338316131333062363635636361623336643332373462636537656635353066623531306262
-313666303466366361313532616333623131
+37383464316330306537633837623331383864353039623636613630623538346261306366316563
+6566663934656231656464396231336633393361323332640a663864623739376637303066393665
+35343837306337653834396663666139363535313662376262366463316435666261303662316230
+3830336532363861330a306138313961343931383438363663323162356166636461306234303165
+64656563613034333139633234326165323231346366323030663137393330396231666531656539
+34326263666435656534643664396661366330383466653334323239653533346266363866633436
+30633134313436623631346139393737393266386139653665306261643732383165363364346264
+31336161633530653731633530366633663161356539663563643633666361626531333639303134
+65393665383265323632366335316166653935323632366662613133323130363665663835393236
+32316530663333353436376637336534333431303739653535623766313236373261316339646632
+66333735366363333934353464343366343661396632353932396564646435613033623030646331
+37346137663765616231376266373764613331663131643266336538386262643762383533666665
+36663237386363326234623266643465306366353863303564346632353237636462376663343938
+65643566396639646566626330376138663830643434636133363637616635623733383930663564
+38303863336339303861303339363433306331396466343434323763336165666434316633363731
+30336234343166643964663362363930363331336436393863646464346431663262623731333532
+34336130333861356238346565336338663339653461386566383036613139363562356532666562
+32303331626536386263653232623230316262633632626462323034343136636137666662313962
+65613462643932636363343635643534353664386333313363396261646563653836636638653134
+32303239643032626136373833643838613862646538633234333062653063643962373132653330
+37646633303337333561333832393661313333633139396662663036613565343365316431633162
+64373462656362373161356332383033666261333861333637323436666432363663396163373534
+61343630623637313937616137666363643933633261383163643535396139386336623235353261
+38373263653637393130303762383738386339633162303137643332333137363338306132383566
+65623839653739316538613961376138336437663466343836643762653762373036316234393564
+37613565303233626139336236646265356337383436363664613935333064643064663261356435
+61316663373939303038633434663833393638393732343836333334323634646430646361626139
+31636364613762363937663763333339353263333530643761376631313861646330633366623763
+38313633373736613939396635666232373163313239633336663636353261393731383364316265
+62386365303761646339383434663331306364666263666532353836613161393532646538333362
+34623963653866613365323632323332323432353437336362366238316239663563616535326264
+30363132613339613933356634366237303064653235633039656239313738346436383738353030
+62616237386536333663376538353839626137343338626334373634663339303864373564636435
+31316133643962306434323234316532376562363133333164663237313164633633663236373334
+33613131656239336338623636373334363130346363366436626237366662326263646630663564
+31353762373430626330353836643266383433393064636264623234636264306331306264623266
+38643737323263663836393261643962333762363932363832623531623866623766646166373565
+66306535366230633366366437393536356130636433613264326163303663316332663135323966
+35383662336537663464653135336638626530306139363665343631656565653161346634316132
+62336634336666383335303930623837643638666364633464303536623634666636373961326366
+37326234353336646533346631353166303132633835346361393337316433386330373931323962
+31663830643664613631663561333638336635333636663937323038313830303163393364383438
+32346434653732653630316531646334646664346330663762353135336462643431623665346633
+65633539333935633465653861316336343339346138383565373830626630326564366661393832
+30303930343735616463613166333636396665666262616332646135613731346330623139616666
+34623861643162393062613537643064633532313234653535643436623632326537343866616164
+66626631353539666438316663323765353035396666373236636235373566356561663331633334
+31646337343061663565373461613761663437313338303464313866643737353162303863333837
+31666535653137363461636331356331666535643339666135303232626631613132366230393333
+34653131626461393236666338363262376133353535323165303531653535653732336235386232
+61386238633064623230316465376233353236346465613731653437323835333136313165643262
+62633065356131396530383263343838303331383836303261343763396635343234666664636531
+65643138333831666432383937336265376566333862623363386163333464363961353430336239
+38633664623464326163656134393364656238323139343038356662363766633333613065346461
+65616565646262313766363064613333666135373263643566343435343032653964626566323962
+34623633623938373837653062376362373431663937643833633563663538386561343863366236
+35663033356265393465633934346436356561653136333835346339353236343638316430623434
+31616431366633393031353433303365656438343635363634366361383030633133346264356261
+61656530313932303434306532373930366135636633373736623664386639353763333634653531
+61613461373732616333383965323063313039383832616361323336643230633765353062376633
+32633735376534316262316265393465616231303634656539343261633533313263323262323536
+32353039613164323761386235623664353162653338313962303465663534623762353238653634
+36633734666532623436353033663030633437626131663639376364333364343263326163636437
+35343535336232613861363334653262633936383138313937346236343538313237323230626465
+32613439613464313432653861613564353838386137353964323761306636383061363532346232
+62643735633939343064333162303764353533353562363938353363313266613231653964303330
+36396533323435653765373564316661633066373363323633343234653165623266393730333032
+38353334353030396165616263336335643463343565343762646665326636656232656166373665
+30346165366331386430656637633063666663623362663633643564323933376633623733313038
+65303338313161316431636563623335366531613862363930396536616161636132353930303035
+36326532653966653766366338356562323036343637376362313133313039646533333264356663
+39343464616435666330316361613835626639643333376465393931653732636636326266316463
+63326233633362636666313730353030653531613266653734633163613937643462656138346537
+66646434313332336436363633346438313866373837623838303238633233653134363465623130
+37633838653061626430313934666561323361366365613766373232323663643337333735333631
+30653534616230306139343530666665663962326263323465363035363932323735376636633639
+64343136303366313465666363666435356139366537336231326665386130643532353864383663
+66316232633463333739663537343837653835633363653737386262316538306564353363366238
+61653665346139316137306136363666396461646561383233313436353065333864333162353765
+66663936376535326236336261326235343734626139376136373762306465396232313538393633
+39356137643433666461306534343030626235306366393662306232633930633735646335333538
+64613037303939333630636135316465666335316233323363623730323938313065626466363430
+64383037316234343066636337616466303261323166396665373466303465356136313861623163
+39396532323832613461383831663139656339393932323232663639643339373961613539646565
+38306162396163626531346265333066343462633161326666346430356662306665663539356632
+63346662396337363036353261343035646332303062366334353438303136366538613032643632
+34316335383537333165386561303536396630386535313231363537303533346133343964343135
+33653831323032653838306231323862613734396464663161363764346238646237643365643664
+65313065613330363161303864663164626563376331313165393236376531343962383139333830
+65613631656233383261613861373538633230653436353962333162663165313264636135323330
+64363461323730316634313766636533363131383238616363366431633330376333303737663966
+30316163336361626133373165346231613461626535333432653835616664313536653863386430
+34393238396366303438653831376634653839396531313936663230373336386137313934666563
+34383030623739656634386632303033316133333734656364613131353230343639373630313138
+39373830633161656265633032353538343331663535306535326134646664393736623563393563
+30663036643037636261613633636439646637333230383237653238373264353134333838333839
+35366239616339353031623565613237366230663732633530636266393965316130353036383064
+39623238626337366437396630653764376133323130323234376561303734313732303865323830
+64643161306233633766613231346365653766666132616466326534383862333162666437626237
+66623234616266373139633635333931313631356138386431663563633833316138346633383137
+35343334383337626635306664386239653764373138386563333061626435356338323761336331
+66393430636133396662326666303832336162366166326436353834613034663639323531393738
+65356137373334353837663234383338323634306162323533353930303762646639323639366231
+33336533393731313431306165653963303736393264323136616437386661396438623237373165
+32646337336539356431336530373566656631626161313864306138613933663436333966636236
+63373530343337363334656637363436323039393336633736653537366265626237343434313834
+62303537663436306538356638323330386265626563333638353661336530363136373236343033
+39363165383464336661626632313866383037643332393265316632313435383333626239356332
+66343863366634643836636432656630313639393764346639333236376232353436663362363761
+61626637323865656431333830343239636536653538313432393235323563656539666532306433
+66643730613237613936383863336638303064356437326536373764326230363439396332366136
+39623762326638623365383064313861643933373564663134636638343166643839393363626561
+62663466303732613932626236393131663730643166613964376639656433333835333130633830
+30363634323961383765656137343663383633666233393230353032633064393062326539306537
+38663365623263343262326238636331333034303533633931353335656363633866663162373735
+62376435336531653230653033636235623864303438626130623464653838663261353338366233
+66323865363436383861663735616335353337333462336265656134633036616130613031333139
+37346433653337303164376664643431636638333336623832376463373061386263346439366236
+65653163363964386663373836653566633632393632663931336332343330626539323963346332
+62623865383434326333373532383063313335316131353262653031626664323332363162383730
+61613463383263333264363466623836306233653064633637343736393437633733613434636261
+37393962633265623432636661323739326462313339326266313736636130376431656530356236
+36373531343964316661633164313164316231386366626462626164303361623833303937343961
+36353834323931323361393433313633333866623435303931346665306134623435616235343365
+65396462636237353832313562393831363733393566613233396661623434323763623930316137
+65316434313935393932326332373132313938633237326566636437623763616131343237383338
+64356164373737383436373733616461343035333130356362313334393861346166326532663539
+34636161323862636435343663366662306436316463333331326332326236323066343430396664
+31653363366330303737323063373939626662356433623139626432363866366465633165353832
+66616163313433633266373738643963323438313832316238616464373432353434663431653833
+37663734356539333735373666393836363332316538663330336136333262363262653961336536
+34313636336434646663366163666365386362363835376361653361333437323331313863663837
+30333832333435643630653438633465663033343662313236306632643435323332363663373339
+63613031653261636630383166393033396665363333393238333438323137613435336430626461
+34613434363733363933343431363864643562383932663162303361663965323035386135303134
+62303935396432343466656662393332396136323363633362666530383962366232366365646263
+66303766626565333037336364363934663965366565663839396530646132646239626165663563
+61353461373565333232343763653461353362383536383035616562643465346262333835303330
+38396363396662313734356337363535343562623163303532383361656565626465623633343538
+32626664333461326137393036313666353832623264333238663135643033343861313737613632
+35363832306363626535316564326330393032306531333836323361656462303137323765663737
+64613230356433366561333636363630663435366234313730376538613134393765303438316364
+33366535316366343833393238303239376662356231656138306664323930383139393434383836
+34613130363265366561633030613862376338373730653561623236636232333535626335346138
+63613638363565623837386261623838363539363062316336303335383836386565343963356439
+34343562353765316365336564323434666639313634393332633765613061636665383965303830
+34656466656165336662663231663361303166393736323461333031613433613532346433343565
+38366632653666643038633634366537346235303866386235356661336235646665306335303561
+34613561623336313739633764666661616130383537396566613433366166316130663735636265
+36376436346461346331383361353462376633623063313334363130653930663032373531323835
+63303564366536613330376136643130663762366231366162323538363432633834306365313333
+66636337623739623132373137643531356162353231636365386533643337366661316666653838
+35393635313036323233656136393231396633316531646265626562363964383464623532663437
+39336434373562363165363230323064393738656338383633366166623963366630663932343039
+32613866346563313835366534626433613633383132393763343037623066393965666634363366
+66333261363762303732626134366335363933303162373733653266316338626330613833366636
+31613832343836363038353433343338333262313164343034303738303661623964633830363462
+30303339636534633034323331303864373132353838613939643734346661636166346334653034
+39303063356634333332356638616364373362653034396139356139376566326338663236336332
+35376330623237386434633963306238346361323236336565393964373964633336333334316163
+37636636306531323131393334396537646662393334613766393837326566363964316434653838
+39383738356234383232613132316164656535393637623139656434363862366264366237396533
+30383161613439313335396531353264343066383662616239343165623236313736373861366465
+30626464303634393539353763633333386463316266643763313363326462306136343164313839
+61393963313433396663306435653134363038616161373632623732333939336261313464356161
+36326638343963613566646435653731643761346533333337353766323534616535346335313964
+34663330626532613432343362306630383663333531383961613562366334623836626361363966
+65353136393932343264323533356330383937303435303830336538653634666439323833303932
+35363431343339303961393237326130643861326134633466343537396638646139363034616661
+35613033383565303036306138663733653839376638333063306365393735353235626562373932
+30343964386633633731633038393834373233393562393933636634613832353631323831393062
+64303236326463373766653334663063333039633335613134353238386165656332353166666361
+37313063366333323730303664616530393962633036656462376463663034663566393934616430
+32303363303833643232353463663937633938383465653030333362306266616465363062343334
+37366664616333346134353839343136356266356135316336373361343166613963333631306130
+33663235356331616236383038363164326430323530663432666134353035623665626539363962
+64336164653661636139636263393264313437313739313763356135333766646335626539613738
+37656333343462376538323836663437306333636637396430373036303965323137653165343436
+37616533626431353764306633386331386637366464333437356134646262396535653837313561
+66303064366533353238663530353266636631363730623034633862306538396464353332613063
+30303336313435616430396361383166663433633137356539393265343833333535636565353466
+66333665646135616361636265366165653865346136306436303131376265306566666465303835
+33303237336134396562366434323862323762383139376561353865363866306538316238633435
+35366431316333396135373830636665386461643235393732623263636331643234666137336461
+63613462663836336161366364373437356232306134356432323162636565333837613733373536
+38303432313337346431376364303464333339623134343330313630666635313162303466316262
+35643236316465653538653261323131323739343032646339663463343365366235636338363763
+39666130383562333439306466653435313766663238386231383962626566636533333265376133
+30326464643232323262373732393663343930393265373532356139333230313637653531393338
+61383334656539353738393832316239306265313931383666363532363136386230613537336661
+35323234366639343730623732353433633137656563306161616463323432303061353365383632
+66643732363165653138633638323039306531646362353131336133663964396361323830623236
+33643733613436383331316136353462366166653236373632656162663138643337666436656530
+64633833393538393333623465396232653330613562323562636539313166613664336236646363
+62343236366466363633653766373364343762306439656262366430366139343937376637643538
+35336330383730393738333165313663323564333134333436316131633936643035303838386262
+37306632643732343734613334353463383362646364333861363935303262656634373737326431
+36386466663633646438383364336634306532656234356334393161623264383336663233386634
+36306636623035613532336562343461633166313262633434393636666362366566316566366532
+37616362653364376165373764656263316630343861326131343636383965386666613165346264
+38363338373334393536393165316337333230656664336332646635633237333835343837363263
+31623231336263363036336331383936613232356138313830336439353932326437373935306434
+33633138663631323566393130366632303033643236373134383838333432643736323265373434
+30336433383439353662653438393231316565393663656338633564346661343838366231336666
+65363735383663323732393639303834623962353230346530393532323761656634363131383230
+66306261663538633938313363643636616232366532366435323833626535396138353233306462
+66646434373835333631353932626436616665313163356235333165393432353864653262636466
+33313864363563623666363733316261303233623331353935303538336439336361383732383036
+62333830383966643964326339663437303664326266323266353331623364303231646161376638
+63346138343462313033663466613162343263383633343962383764316336343336636335653164
+66316363396336306330363937316362393863383934303837316561333138306139653333333434
+33356261613838383634633733326234303736366163316638363539653738383462393636646437
+62316236373439363735393566353065636633653534383665363433333235333833356138363566
+32363430393563346239396264343162663233326134643964626662386231316566363565313566
+62376462346166613934396238646263643431623132313566383538323336336162636439613135
+35643266313264356135373262363766613765653735333634313835393861383163353234663738
+32646237643664626266356331613932383034333038366230356266633231353930326161313537
+33666663616565346639396436326530356539356464383232333863393932306463333131336633
+36376530313865373062303638333364393964666662623062396636313234643937373332393061
+32663565313361386232643831373834346436373063633734623136613337336561346238333233
+61376562376234666639616662316237613965336539633438313833396632353166656664376630
+31353266386439376263656235643463396133343933656364646664373262643365353634363635
+65396663653961326463653833323263636363306666653632306637383531383431376364656530
+34346462373330336434326338346462323734366235656630373666383336396234363965343630
+30376330323562643137326562616338376537623334396462313764356363356337393834626632
+65363735383565626636306237343161653334656233653464306164376666353830303537386264
+63366430396635633639363031303464326435303138376561333338653564383065313865623664
+61346664646639353035643437626235643432656663323364343338323961623837326663396135
+39616265633933313465363331613137333563373434616236326231303331303735383330613532
+66343366666234313238626535633666363761666337653339616635643263623435386265643261
+38306239643230653836656134646263646265323936326462653337343066613163383062356336
+34356530646139636631656338613331353131353830626365633532653464386365323762326334
+63653531643763616162623431353535333665353733643935313737343638323565393735353637
+30616538343332343461343833343030336365333566623964333830343465383664316135643364
+65366230373837346130653864636237303464376537373632666236356539383736326166663461
+65626661616235653538653062326264666265623739336566316236643963656464653237326265
+34623537336337643266383438626233663733313435303736316163653536326436656638333431
+37316639313965306235346431613536383366613362636266363563373766633063633961616332
+61633138373939626234333264393438393337636364636233643932376466393661333532396333
+30373931613938623931613036623336636638316461643438613963643462663337383630643932
+66386366353239396234383736333331643961663136623438333164656565393537613765343161
+64633235396661363534643963393564323932616133623737396132613862373639383064393232
+65363363336132383966656635313033303034306532666230326234376437316239393035646663
+39386230623639353934326533303864633863356436626135613139346439336131396639396261
+63363664333866356163666637663732373237383431306634356166303732326132633463313063
+62386432336361663237333963636533633565613538633334306434633239383932343365656435
+35386161646637363234343739623230343765323131653861303438393466613635333535343465
+37313761663637303039343762653861396337363132633535363866376637313263326432643434
+66363932333735613462343331303939336539313530373730633835356466356561336339306564
+32383031383931613266643538313534643334613331623438313130353438383133336238373237
+33633561653865633931353664323865333965633363666531363931623737346336376636626133
+33373566663262363735353933396563356165663962663036373461333131636563306365363731
+37633339356263366335303930373530623938373232393365663433613836303837306330643634
+34313631623336633161313830306633616537333137313237363266383934646133633835353232
+36353935363765366262363061363131313063613930376631613435313339353662313632356564
+30313064336439663837376437393738643339643166366335383131656233353263633035383233
+66366161393232316361306332353435653764373064383462333137313532316166393339333435
+30343062386530363835386530316634633934323239326161376530616131663937646339353437
+36623334386438363839363161323166313338353037333234613939613762313762663761636530
+64643830353561653732303332376233343535353832333632373864616334313566626238656366
+65323233363661643339363133303930306432663861353264373435623237626130383232313434
+61356263303433373566373864616537616431383037306230356139666436633439376237626531
+34633362303566626261346237353036626261623964353563336566316266623664343866386238
+64353536326534613539396136366338303230306236373531383935323230633938303438376664
+30636238633430626563303031363665616433386162366331343439333034643537303135653532
+36636238366261313331663065656233353463643062373132376637353935626365633635363366
+64656635373233353131653430633131346132353265383334383230333630383338313365356331
+38383663313461303639613963356134366665356634346630626361653966396239376334356335
+65343530636230353061343063623162616661636430653234656433626161346133316138373033
+38383731376562633539396261626637653337373232626638643836613563326664333165653231
+36353466613937623832353038323034623935333438656661323232373031653834636539326632
+61393137636265343337336564323763383435343730666439623536323364613332343962393433
+30356264323461326565393437343037616630653237363434623831626437663937646235373066
+38396164643934356636323065393937393265613464393738303735376537646232333333373165
+34346535396634346338376564353239623434303835306230343730613335643235306534383565
+37666234363031363463633039366639616161343032653063303234336432343839336433323830
+38316237646632303761656633306665393361376330393839663432353439303730313335653365
+61343161646364393332316534353135366639313063393137303239383332376362303131353566
+61376130313739346566643433353362386164393034333161633834386538366563316534636436
+38636136306239633038393037643865363337393164633066326463613466383937663232306433
+36366136376235383935323761323362356561313731356262653364343265323938633464333134
+35623237343466306530313834356130376635396163373365613734373161663964633535663362
+39313338626662343231366462646633613136616464373736383339316561383765333463333436
+30363535336363386239626161663939346639643061663861346130373535313234616636323965
+37346465303532306434303662623164363830363633326238343565646331636636636437393536
+61633539376437623434616366346439616266343666653566633664646230316239313866333162
+37306665633434663064383430363361656432656361633431633931646637376561333164643165
+36303039323239363865323737626636343364323932306366626336363234346635656330643734
+64643133303539393061366236346436663966323265353539316166353861333635663530366563
+65646132333035666163303562356438333735663863613738653232613739323432633165333964
+65613834653865376332303566386237343334326438353138316532396232613031393764393139
+35323966636633323764356135613539393066326430353631353964636563643132326137653033
+65616339363861316132316338356230363538663765353063323334366465313362373139333233
+36396432376131626666653335336637396363633266323630636632363665366533636634623861
+66623736623433316138376364636463643063316532396533326137336564323933616363616338
+62323761383036383734303035656432663765303638656264373235313937653132366330636639
+62623437333838303536383437333237343562366237316534643336383161626532353262316461
+61383632326536343631666535396661613233346265373063376562333162666135393265343666
+32373532333465646233616237653561313739366462313832383431313430306134396533643737
+62356262626137646566396262643334343664613932663033366233313662313033323836343565
+34366534633739633131376330373431346232373739343634383836313233343936336366313035
+38376434613865616636376565343065343832313834343030653463386261323766303232313636
+39643935663063613865646131363531363533333365326635396161313334383230623266313162
+33333837373038623636313965623131383162303338623630343632616161333232306131383035
+66336165613862383235386330616334316139626336326262306264643461643261613038346333
+32636435353465653065326430353835333633643362303464383935353964363237616563663338
+34613361646437373939323633346233373735326363653861313661316362393832653133383162
+32323665653662616631346335386363646231616639653533363936653535346530373338663336
+31393236303761323335303038623939303434353865303830626436343237666535646161623866
+36323132376536336239306166363330313730626530313932316662373535313334393234346235
+33313564363434393537613261333831393133353662346631303738363430353565353666366235
+65353665363438356136393936646639353332623031616439303839613266373266663266373562
+65363134323962613837663435323966343965373738376330623039396663356336616664313139
+38373763333362633766303436396239633761323032393837376237313161643232373461313163
+62646531303066303138346136346536313264356330396332303366383963333834396535643136
+30623661643438383234393862353464313365313562393132626334633364613366656361316237
+34353335356433656538303538376337316436326461613130316630363132656534346462303330
+63356330376566363831386232626262313139326364353866643530656336623134336662393065
+33666636646462366635353630623065326161313831303933316362646137393538336537623161
+66353661303034373036386436383562323331666334396236346334313932346338343634633431
+39623330653765303231383439393434616338343739386536646435343764616366306335316338
+61396563306532306131663863316236346531623264376132643731613532613531333265666438
+30643134643837633966366139313138663334636438366433616466396561373731333763393436
+37643230623563303532333262333031626633396636353332323562303762346236653561356135
+62303637633466666130636431336230333733613061326263383338323664326662616561393033
+34303231343735643564636364373336666136336465393037663863366537663266343563643436
+33336432613839346266636366373463663563356336633538643963636431323261303361323834
+39616365396335303238666533366233663461393837646265396433626464323863373463366439
+63643930616361366632333462613731363065306366386234353335653563393131336339383633
+31656631633933616637643439393737643862303063363231326231633834376138613766373565
+65393161613733353039656162323136653565306438376166326639323030316366623065626134
+36303736346638323732663539336263616536346233376363383130333832616635666138376432
+34303437373837666133623963393836346165343736656539633736616531613936386264643137
+63663839363932633961386363383865643132386130636433306166343463383636396639316664
+35326462663132346261626231383365356339346466376133336664333164346434663162633735
+33663230336138306365636265373462313962393037363864313161343364306332313735663938
+64663164643563346532643335313862653438666330363839346362353236363430656465386164
+31373461316534323036383562353035363532616366356262323033613237323334623665356430
+64663537643861383636353336626563396338386436383837633566646530356633643938366364
+65356230316261363633363566343633663631316563636364633661383163353633623933613764
+31373031376165616332373239353065616663386136303639663034386334643265336435353332
+33363836356639343336636332363662633764353532353266386139626566373239666339376638
+38323037366536306634636366336438656565383162623236646331336565363065313136623539
+39643036616465663734306239373965656361303039663337356630313439636661346531363561
+38656266323230653166313164323066323632336433343336346234653061646663306531396132
+62636466306465386530353066336439336162663562356233366630383738623732323339653036
+39306536323531343961616164343636626266633763376662396138363130313038356539646561
+64633362356332656637643339373831373935306564356533626338643433646537313564336235
+33316366343834343332306633353363366432643138343839646335653536373238316339353335
+34633731363833346438666564333733333034363766633831346539343238623165356563353934
+38396539356461636536633366326231366364356666623965306234346238353834663233376236
+66366630376537653563316263346330616634316466373861393736303661373338393066326132
+32386337363064333462396535356662323161313437653232623932613265383530303330613535
+62616135363866363664616130663738613130303866373661396261346362323262623239333135
+63373330316535613166326538366664393565653761366536656266656362323838396261623261
+37393534333262643663313233666335623835333135306539353034333733626163393631333037
+61386665366233333032656331656264636539353034643634663533613038323364613233343034
+35346636373263336266303864306635343064663062313730383661376137613331316634383838
+64303461613961656631626236383330316663326231306261336563303638326163333064643738
+38396465643934653237323864633931653537366163353136313766323831323066643962306335
+62323965613466356632636530383838363935643236626362316631653839626461313736656138
+62393761393231366564386231323731313064396135343966373736353936333065636639623863
+39323431383635366237613661656437653937666631326366626539346234326435366464616231
+63613435373665353462653862646232316238393233386537623630396661623136326666303337
+34366237313761616162336139343935613537643164333465303865313361666432633565306130
+35613036343638643337366530373365393137396438396261656133393830636137343938326634
+30356662616330343864333032613037333062333538376263623933376339393465356534343339
+31663363633839363132303531653035383835393665636466653836313635386364386263623063
+39396133373439616434323631653534373163633261613432303035383235303739383138613465
+35303561633761616363663961613161663435303936313461396435376432343239613330616530
+39666534356437306536363462656535326337313433623334623963396337646530663961656464
+30646162653139336533623365343961316338326435653562316639326539333862373732346563
+31353265383738393531373539353039343339353134396439623636623738616564363934623063
+62363736326331633232376465363361636533313335373661613933313662383761616237633837
+61373364616462336530366264306665313565353134323538353332343261323863653834303832
+62616132386437386237356232373334356661383262386238613334613462343537333764326338
+35306464316630353837353462663061646139366533623036363461303232623830383633643534
+35336234326261656265623138356461396638356338373466366563393932633137623132303136
+33326336666439626233323961353563623764663332646165306335373938363738663939396132
+66636433313665333737306362326236366161343236643036636239333630373238353139373431
+39613235346632326133333465633138633531613233376430383030363764656538613831663331
+66396630363330366462633939623564366165396137373232373937313132313763626336373936
+63313262336465303634353066303864663035393435663863306366353365346435373862363738
+64316264366332363563306164343466646132616530636664643762386439646562373761656335
+37356239373265336630386636623733653537356662653066643338373735653339326231306230
+65353433363961363437303332373537343134356663383165623135393664373539633432636335
+61643831346234636661303833626265363831343762373933363337643635643532376635393766
+37326466376639663663616534633662336635633431396633313336643863646238373735636137
+64303861353763343463333530326432613533383565616536303035613133653566656262363739
+31383936666132613962333837353839316264376666313862626635313831346432373165326135
+39363231636634343765653039643933373161313965663130383165363835346137616432383937
+30363665633766636662343639636236376262333839383631643264333233393432616466633438
+66646238373933363435363362353730643763396662303534316266643130313332356163376631
+36663766636430356631613131656132396232313136646339393232623331623961313763343431
+38656239666465613566363037323963393931356239633130636666616139323462623835326537
+31353431323630623139356134663333333261333939343131633163306161653435323737613761
+64393466363039633632313533616338636335333134323230346364353139356533343135373765
+66343166393838643439306564663234623362313433613163323237353166626563363666373838
+36386432333335326665353435306132346434623230316234383537353263613638333935626431
+61663839663563303133