From c180acb883d659136a96fa989c85d9b045eea467 Mon Sep 17 00:00:00 2001 From: pmikus Date: Wed, 20 May 2020 10:07:36 +0000 Subject: Infra: Ansible Nomad Signed-off-by: pmikus Change-Id: Icc03bf62d8639f603ca90aa6ab849a46dc911d56 --- .../lf_inventory/host_vars/10.30.51.28.yaml | 29 ++ .../lf_inventory/host_vars/10.30.51.29.yaml | 30 ++ .../lf_inventory/host_vars/10.30.51.30.yaml | 30 ++ .../ansible/inventories/lf_inventory/hosts | 5 + resources/tools/testbed-setup/ansible/nomad.yaml | 14 + .../ansible/roles/mellanox/tasks/main.yaml | 7 + .../ansible/roles/nomad/defaults/main.yaml | 103 ++++ .../ansible/roles/nomad/handlers/main.yaml | 10 + .../ansible/roles/nomad/meta/main.yaml | 9 + .../ansible/roles/nomad/tasks/main.yaml | 164 ++++++ .../ansible/roles/nomad/templates/base.hcl.j2 | 24 + .../ansible/roles/nomad/templates/client.hcl.j2 | 18 + .../ansible/roles/nomad/templates/custom.hcl.j2 | 5 + .../roles/nomad/templates/nomad_systemd.service.j2 | 18 + .../ansible/roles/nomad/templates/server.hcl.j2 | 4 + .../ansible/roles/nomad/vars/main.yaml | 5 + .../ansible/roles/user_add/defaults/main.yaml | 8 +- .../ansible/roles/user_add/handlers/main.yaml | 5 +- .../ansible/roles/user_add/tasks/main.yaml | 40 +- resources/tools/testbed-setup/ansible/site.yaml | 3 + resources/tools/testbed-setup/ansible/vault.yml | 567 +++++++++++++++------ 21 files changed, 915 insertions(+), 183 deletions(-) create mode 100644 resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml create mode 100644 resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml create mode 100644 resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml create mode 100644 resources/tools/testbed-setup/ansible/nomad.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/nomad/defaults/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/nomad/handlers/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/nomad/meta/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml create mode 100644 resources/tools/testbed-setup/ansible/roles/nomad/templates/base.hcl.j2 create mode 100644 resources/tools/testbed-setup/ansible/roles/nomad/templates/client.hcl.j2 create mode 100644 resources/tools/testbed-setup/ansible/roles/nomad/templates/custom.hcl.j2 create mode 100644 resources/tools/testbed-setup/ansible/roles/nomad/templates/nomad_systemd.service.j2 create mode 100644 resources/tools/testbed-setup/ansible/roles/nomad/templates/server.hcl.j2 create mode 100644 resources/tools/testbed-setup/ansible/roles/nomad/vars/main.yaml (limited to 'resources/tools/testbed-setup/ansible') diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml new file mode 100644 index 0000000000..aca60b645e --- /dev/null +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.28.yaml @@ -0,0 +1,29 @@ +--- +# file: host_vars/10.30.51.28.yaml + +hostname: "s41-nomad" +inventory_cimc_hostname: '10.30.50.28' +# User managmeent. +users: + - username: localadmin + groups: [adm, sudo] + password: "$6$FIsbVDQR$5D0wgufOd2FtnmOiRNsGlgg6Loh.0x3dWSj72DSQnqisSyE9DROfgSgA6s0yxDwz4Jd5SRTXiTKuRYuSQ5POI1" + ssh_key: + - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com" + - username: testuser + groups: [adm, sudo] + password: "$6$zpBUdQ4q$P2zKclumvCndWujgP/qQ8eMk3YZk7ESAom04Fqp26hJH2jWkMXEX..jqxzMdDLJKiDaDHIaSkQMVjHzd3cRLs1" + ssh_key: + - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com" + +# Nomad settings. +nomad_datacenter: "yul1" +nomad_name: "s40-nomad-{{ ansible_architecture }}" +nomad_node_role: "client" +nomad_node_class: "builder" +nomad_options: + driver.raw_exec.enable: 1 + docker.cleanup.image: false + docker.privileged.enabled: true + driver.whitelist: "docker,raw_exec,exec" +nomad_servers: "\"10.30.51.32:4647\", \"10.30.51.33:4647\"" diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml new file mode 100644 index 0000000000..2eb2dfc88c --- /dev/null +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.29.yaml @@ -0,0 +1,30 @@ +--- +# file: host_vars/10.30.51.29.yaml + +hostname: "s39-nomad" +inventory_cimc_hostname: "10.30.50.29" + +# User managmeent. +users: + - username: localadmin + groups: [adm, sudo] + password: "$6$FIsbVDQR$5D0wgufOd2FtnmOiRNsGlgg6Loh.0x3dWSj72DSQnqisSyE9DROfgSgA6s0yxDwz4Jd5SRTXiTKuRYuSQ5POI1" + ssh_key: + - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com" + - username: testuser + groups: [adm, sudo] + password: "$6$zpBUdQ4q$P2zKclumvCndWujgP/qQ8eMk3YZk7ESAom04Fqp26hJH2jWkMXEX..jqxzMdDLJKiDaDHIaSkQMVjHzd3cRLs1" + ssh_key: + - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com" + +# Nomad settings. +nomad_datacenter: "yul1" +nomad_name: "s39-nomad-{{ ansible_architecture }}" +nomad_node_role: "client" +nomad_node_class: "builder" +nomad_options: + driver.raw_exec.enable: 1 + docker.cleanup.image: false + docker.privileged.enabled: true + driver.whitelist: "docker,raw_exec,exec" +nomad_servers: "\"10.30.51.32:4647\", \"10.30.51.33:4647\"" diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml new file mode 100644 index 0000000000..3aed1c8b52 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/host_vars/10.30.51.30.yaml @@ -0,0 +1,30 @@ +--- +# file: host_vars/10.30.51.30.yaml + +hostname: "s40-nomad" +inventory_cimc_hostname: '10.30.50.30' + +# User managmeent. +users: + - username: localadmin + groups: [adm, sudo] + password: "$6$FIsbVDQR$5D0wgufOd2FtnmOiRNsGlgg6Loh.0x3dWSj72DSQnqisSyE9DROfgSgA6s0yxDwz4Jd5SRTXiTKuRYuSQ5POI1" + ssh_key: + - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com" + - username: testuser + groups: [adm, sudo] + password: "$6$zpBUdQ4q$P2zKclumvCndWujgP/qQ8eMk3YZk7ESAom04Fqp26hJH2jWkMXEX..jqxzMdDLJKiDaDHIaSkQMVjHzd3cRLs1" + ssh_key: + - "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEAgObJFDIMmPwQhhkjAynvlbwpM5yeSewyaE7vTLaFf4uFz4vmsE2hFf6B2xXHUGLVwoVfk91UeK7LOGrdDpoDDHzvPZXj5NmZI+WiWax5y2pQZNkcSZws0ENCeEc4hPwc4veJ1JmhokF4Bsmu14HyFMaFUhM8897jtJwsh+9fLA/no0iPGaQqEtRUQhkV+P4jCEPoY0qdRZAzVw/rY4EGAMhsJe3EJmyj63OfrrkG3+hvSLFo5pDxHQr3pZd/c6ukI7xMef48PosAvGCm3oxzb/Gu9PZIGuHLczY+tCnzCkY7MO7E+IWgjXrUAfYwSWz8XmFmA9LLe26DT5jkcK8hGQ== pmikus@cisco.com" + +# Nomad settings. +nomad_datacenter: "yul1" +nomad_name: "s40-nomad-{{ ansible_architecture }}" +nomad_node_role: "both" +nomad_node_class: "builder" +nomad_options: + driver.raw_exec.enable: 1 + docker.cleanup.image: false + docker.privileged.enabled: true + driver.whitelist: "docker,raw_exec,exec" +nomad_servers: "\"10.30.51.32:4647\", \"10.30.51.33:4647\"" diff --git a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts index 0402c68109..52f0076082 100644 --- a/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts +++ b/resources/tools/testbed-setup/ansible/inventories/lf_inventory/hosts @@ -45,3 +45,8 @@ all: 10.30.51.50: #s1-t11-sut1 - skylake 10.30.51.51: #s2-t12-sut1 - skylake 10.30.51.69: #s27-t13-sut1 - thunderx2 + nomad: + hosts: + 10.30.51.28: #s41-nomad - haswell + 10.30.51.29: #s39-nomad - haswell + 10.30.51.30: #s40-nomad - haswell diff --git a/resources/tools/testbed-setup/ansible/nomad.yaml b/resources/tools/testbed-setup/ansible/nomad.yaml new file mode 100644 index 0000000000..b09245cfdb --- /dev/null +++ b/resources/tools/testbed-setup/ansible/nomad.yaml @@ -0,0 +1,14 @@ +--- +# file: nomad.yaml + +- hosts: nomad + remote_user: testuser + become: yes + become_user: root + roles: + - role: user_add + tags: user_add + - role: docker + tags: docker + - role: nomad + tags: nomad diff --git a/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml index e91442718e..2fb6e2e213 100644 --- a/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/mellanox/tasks/main.yaml @@ -51,3 +51,10 @@ tags: - install-mellanox +- name: Mellanox Install - FIX qemu-system removal + package: + name: "qemu-system" + state: latest + update_cache: true + tags: + - install-mellanox diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/defaults/main.yaml new file mode 100644 index 0000000000..ad8b36ee66 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/nomad/defaults/main.yaml @@ -0,0 +1,103 @@ +--- +# file: roles/nomad/defaults/main.yaml + +# Inst - Prerequisites. +packages: "{{ packages_base + packages_by_distro[ansible_distribution | lower] + packages_by_arch[ansible_machine] }}" + +packages_base: + - "cgroup-bin" + - "curl" + - "git" + - "libcgroup1" + - "unzip" +packages_by_distro: + ubuntu: + - [] +packages_by_arch: + aarch64: + - [] + x86_64: + - [] + +# Inst - Download Nomad. +nomad_architecture_map: + amd64: "amd64" + x86_64: "amd64" + armv7l: "arm" + aarch64: "arm64" + 32-bit: "386" + 64-bit: "amd64" +nomad_architecture: "{{ nomad_architecture_map[ansible_architecture] }}" +nomad_version: "0.10.2" +nomad_pkg: "nomad_{{ nomad_version }}_linux_{{ nomad_architecture }}.zip" +nomad_zip_url: "https://releases.hashicorp.com/nomad/{{ nomad_version }}/{{ nomad_pkg }}" + +# Inst - System paths. +nomad_bin_dir: "/usr/local/bin" +nomad_config_dir: "/etc/nomad.d" +nomad_data_dir: "/var/nomad" +nomad_inst_dir: "/opt" +nomad_lockfile: "/var/lock/subsys/nomad" +nomad_run_dir: "/var/run/nomad" +nomad_ssl_dir: "/etc/nomad.d/ssl" + +# Conf - Service. +nomad_node_role: "both" +nomad_restart_handler_state: "restarted" + +# Conf - User and group. +nomad_group: "nomad" +nomad_group_state: "present" +nomad_manage_group: true +nomad_manage_user: true +nomad_user: "nomad" +nomad_user_groups: [ docker, nomad, root ] +nomad_user_state: "present" + +# Conf - base.hcl +nomad_bind_addr: "0.0.0.0" +nomad_datacenter: "dc1" +nomad_disable_update_check: true +nomad_enable_debug: false +nomad_log_level: "INFO" +nomad_name: "{{ inventory_hostname }}" +nomad_region: "global" +nomad_syslog_enable: true + +# Conf - base.hcl (TLS) +nomad_ca_file: "{{ nomad_ssl_dir }}/ca.pem" +nomad_cert_file: "{{ nomad_ssl_dir }}/nomad.pem" +nomad_http: false +nomad_key_file: "{{ nomad_ssl_dir }}/nomad-key.pem" +nomad_rpc: false + +# Conf - client.hcl +nomad_certificates: + - src: "{{ vault_nomad_ca_file }}" + dest: "{{ nomad_ca_file }}" + - src: "{{ vault_nomad_cert_file }}" + dest: "{{ nomad_cert_file }}" + - src: "{{ vault_nomad_key_file }}" + dest: "{{ nomad_key_file }}" +nomad_node_class: "" +nomad_no_host_uuid: true +nomad_options: {} +nomad_servers: [] + +# Conf - custom.hcl +# empty + +# Conf - server.hcl +#nomad_retry_max: 0 +#nomad_retry_join: false +#nomad_retry_interval: "30s" +#nomad_rejoin_after_leave: false +#nomad_enabled_schedulers: +# - service +# - batch +# - system +#nomad_num_schedulers: "{{ ansible_processor_vcpus }}" +#nomad_node_gc_threshold: "24h" +#nomad_job_gc_threshold: "4h" +#nomad_eval_gc_threshold: "1h" +#nomad_encrypt: "" diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/handlers/main.yaml new file mode 100644 index 0000000000..f0bcee9142 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/nomad/handlers/main.yaml @@ -0,0 +1,10 @@ +--- +# file roles/nomad/handlers/main.yaml + +- name: Restart Nomad + systemd: + daemon_reload: true + enabled: true + name: "nomad" + state: "{{ nomad_restart_handler_state }}" + diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/meta/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/meta/main.yaml new file mode 100644 index 0000000000..9fc40d9ae1 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/nomad/meta/main.yaml @@ -0,0 +1,9 @@ +--- +# file: roles/nomad/meta/main.yaml + +# desc: Install nomad from stable branch and configure service. +# inst: Nomad +# conf: ? +# info: 1.0 - added role + +dependencies: [ docker ] diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml new file mode 100644 index 0000000000..9093afd08e --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/nomad/tasks/main.yaml @@ -0,0 +1,164 @@ +--- +# file: roles/nomad/tasks/main.yaml + +- name: Inst - Prerequisites + package: + name: "{{ packages | flatten(levels=1) }}" + state: latest + update_cache: true + tags: + - nomad-inst-prerequisites + +- name: Conf - Add Nomad Group + group: + name: "{{ nomad_group }}" + state: "{{ nomad_group_state }}" + when: + - nomad_manage_group | bool + tags: + - nomad-conf-user + +- name: Conf - Add Nomad user + user: + name: "{{ nomad_user }}" + group: "{{ nomad_group }}" + groups: "{{ nomad_user_groups }}" + state: "{{ nomad_user_state }}" + system: true + when: + - nomad_manage_user | bool + tags: + - nomad-conf-user + +- name: Inst - Download Nomad + get_url: + url: "{{ nomad_zip_url }}" + dest: "{{ nomad_inst_dir }}/{{ nomad_pkg }}" + tags: + - nomad-inst-package + +- name: Inst - Unarchive Nomad + unarchive: + src: "{{ nomad_inst_dir }}/{{ nomad_pkg }}" + dest: "{{ nomad_inst_dir }}/" + creates: "{{ nomad_inst_dir }}/nomad" + remote_src: true + tags: + - nomad-inst-package + +- name: Inst - Nomad + copy: + src: "{{ nomad_inst_dir }}/nomad" + dest: "{{ nomad_bin_dir }}" + owner: "{{ nomad_user }}" + group: "{{ nomad_group }}" + force: true + mode: 0755 + remote_src: true + tags: + - nomad-inst-package + +- name: Inst - Cleanup + file: + path: "{{ nomad_inst_dir }}/nomad" + state: "absent" + tags: + - nomad-inst-package + +- name: Conf - Create Directories "{{ nomad_data_dir }}" + file: + dest: "{{ nomad_data_dir }}" + state: directory + owner: "{{ nomad_user }}" + group: "{{ nomad_group }}" + tags: + - nomad-conf + +- name: Conf - Create Directories "{{ nomad_ssl_dir }}" + file: + dest: "{{ nomad_ssl_dir }}" + state: directory + owner: "{{ nomad_user }}" + group: "{{ nomad_group }}" + tags: + - nomad-conf + +- name: Conf - Create Config Directory + file: + dest: "{{ nomad_config_dir }}" + state: directory + owner: "{{ nomad_user }}" + group: "{{ nomad_group }}" + mode: 0755 + tags: + - nomad-conf + +- name: Conf - Base Configuration + template: + src: base.hcl.j2 + dest: "{{ nomad_config_dir }}/base.hcl" + owner: "{{ nomad_user }}" + group: "{{ nomad_group }}" + mode: 0644 + tags: + - nomad-conf + +- name: Conf - Server Configuration + template: + src: server.hcl.j2 + dest: "{{ nomad_config_dir }}/server.hcl" + owner: "{{ nomad_user }}" + group: "{{ nomad_group }}" + mode: 0644 + when: + - nomad_node_server | bool + tags: + - nomad-conf + +- name: Conf - Client Configuration + template: + src: client.hcl.j2 + dest: "{{ nomad_config_dir }}/client.hcl" + owner: "{{ nomad_user }}" + group: "{{ nomad_group }}" + mode: 0644 + when: + - nomad_node_client | bool + tags: + - nomad-conf + +- name: Conf - Custom Configuration + template: + src: custom.json.j2 + dest: "{{ nomad_config_dir }}/custom.json" + owner: "{{ nomad_user }}" + group: "{{ nomad_group }}" + mode: 0644 + when: + - nomad_config_custom is defined + tags: + - nomad-conf + +- name: Conf - Copy Certificates And Keys + copy: + content: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: "{{ nomad_user }}" + group: "{{ nomad_group }}" + mode: 0600 + no_log: true + loop: "{{ nomad_certificates | flatten(levels=1) }}" + tags: + - nomad-conf + +- name: Conf - System.d Script + template: + src: "nomad_systemd.service.j2" + dest: "/lib/systemd/system/nomad.service" + owner: "root" + group: "root" + mode: 0644 + notify: + - "Restart Nomad" + tags: + - nomad-conf diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/base.hcl.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/base.hcl.j2 new file mode 100644 index 0000000000..0e2b60c6d4 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/base.hcl.j2 @@ -0,0 +1,24 @@ +name = "{{ nomad_name }}" +region = "{{ nomad_region }}" +datacenter = "{{ nomad_datacenter }}" + +bind_addr = "{{ nomad_bind_addr }}" +data_dir = "{{ nomad_data_dir }}" + +enable_syslog = {{ nomad_syslog_enable | bool | lower }} +enable_debug = {{ nomad_enable_debug | bool | lower }} +disable_update_check = {{ nomad_disable_update_check | bool | lower }} +log_level = "{{ nomad_log_level }}" + +{% if ( nomad_ca_file ) and + ( nomad_cert_file ) and + ( nomad_key_file ) +%} +tls { + http = {{ nomad_http | bool | lower }} + rpc = {{ nomad_rpc | bool | lower }} + ca_file = "{{ nomad_ca_file }}" + cert_file = "{{ nomad_cert_file }}" + key_file = "{{ nomad_key_file }}" +} +{% endif %} diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/client.hcl.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/client.hcl.j2 new file mode 100644 index 0000000000..91fd1c947f --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/client.hcl.j2 @@ -0,0 +1,18 @@ +client { + enabled = {{ nomad_node_client | bool | lower }} + no_host_uuid = {{ nomad_no_host_uuid | bool | lower }} + node_class = "{{ nomad_node_class }}" + + {% if nomad_servers -%} + servers = [ {{ nomad_servers }} ] + {% endif %} + + {% if nomad_options -%} + options = { + {% for key, value in nomad_options.items() %} + "{{ key }}" = "{{ value }}" + {% endfor -%} + } + {% endif %} + +} diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/custom.hcl.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/custom.hcl.j2 new file mode 100644 index 0000000000..37ff6f3496 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/custom.hcl.j2 @@ -0,0 +1,5 @@ +{% if nomad_config_custom -%} +{{ nomad_config_custom | to_nice_json }} +{% else %} +{} +{% endif %} diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/nomad_systemd.service.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/nomad_systemd.service.j2 new file mode 100644 index 0000000000..d0beed24b8 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/nomad_systemd.service.j2 @@ -0,0 +1,18 @@ +[Unit] +Description=nomad agent +Wants=network.target +After=nomad.service + +[Service] +# TODO: Decrease privilege +User=root +Group=root +Environment="GOMAXPROCS=2" +ExecStart={{ nomad_bin_dir }}/nomad agent -config={{ nomad_config_dir }} +ExecReload=/bin/kill -9 $MAINPID +KillSignal=SIGTERM +Restart=on-failure +RestartSec=1 + +[Install] +WantedBy=default.target diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/templates/server.hcl.j2 b/resources/tools/testbed-setup/ansible/roles/nomad/templates/server.hcl.j2 new file mode 100644 index 0000000000..c28a43c5d2 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/nomad/templates/server.hcl.j2 @@ -0,0 +1,4 @@ +server { + enabled = false + encrypt = "Y4T+5JGx1C3l2NFBBvkTWQ==" +} diff --git a/resources/tools/testbed-setup/ansible/roles/nomad/vars/main.yaml b/resources/tools/testbed-setup/ansible/roles/nomad/vars/main.yaml new file mode 100644 index 0000000000..a72222c992 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/nomad/vars/main.yaml @@ -0,0 +1,5 @@ +--- +# file: roles/nomad/vars/main.yaml + +nomad_node_client: "{{ (nomad_node_role == 'client') or (nomad_node_role == 'both') }}" +nomad_node_server: "{{ (nomad_node_role == 'server') or (nomad_node_role == 'both') }}" diff --git a/resources/tools/testbed-setup/ansible/roles/user_add/defaults/main.yaml b/resources/tools/testbed-setup/ansible/roles/user_add/defaults/main.yaml index 9e5aaf272c..56f5098f12 100644 --- a/resources/tools/testbed-setup/ansible/roles/user_add/defaults/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/user_add/defaults/main.yaml @@ -1,5 +1,11 @@ --- # file: roles/user_add/defaults/main.yaml -user_pass: "$6$zpBUdQ4q$P2zKclumvCndWujgP/qQ8eMk3YZk7ESAom04Fqp26hJH2jWkMXEX..jqxzMdDLJKiDaDHIaSkQMVjHzd3cRLs1" +# Default shell for a user if none is specified. +users_shell: /bin/bash +# Default create home dirs for new users. +users_create_homedirs: true + +# Default list of users to create. +users: [] diff --git a/resources/tools/testbed-setup/ansible/roles/user_add/handlers/main.yaml b/resources/tools/testbed-setup/ansible/roles/user_add/handlers/main.yaml index e6b7d82844..960f573b48 100644 --- a/resources/tools/testbed-setup/ansible/roles/user_add/handlers/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/user_add/handlers/main.yaml @@ -1,10 +1,7 @@ --- # file: roles/user_add/handlers/main.yaml -- name: Restart sshd +- name: Restart SSHd service: name: sshd state: restarted - tags: - - restart-sshd - diff --git a/resources/tools/testbed-setup/ansible/roles/user_add/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/user_add/tasks/main.yaml index 2672996202..8323284261 100644 --- a/resources/tools/testbed-setup/ansible/roles/user_add/tasks/main.yaml +++ b/resources/tools/testbed-setup/ansible/roles/user_add/tasks/main.yaml @@ -1,31 +1,49 @@ --- # file: roles/user_add/tasks/main.yaml -- name: Add testuser account +- name: Conf - Add User user: - name: "testuser" + append: "{{ item.append | default(omit) }}" + createhome: "{{ 'yes' if users_create_homedirs else 'no' }}" + generate_ssh_key: "{{ item.generate_ssh_key | default(omit) }}" + group: "{{ item.group | default(item.username) }}" + groups: "{{ item.groups | join(',') if 'groups' in item else '' }}" + name: "{{ item.username }}" + password: "{{ item.password if item.password is defined else '!' }}" + shell: "{{ item.shell if item.shell is defined else users_shell }}" state: present - shell: "/bin/bash" - password: "{{ user_pass }}" + with_items: "{{ users }}" tags: - - add-user + - user-add-conf -- name: Allow password login +- name: Conf - SSH keys + authorized_key: + user: "{{ item.0.username }}" + key: "{{ item.1 }}" + with_subelements: + - "{{ users }}" + - ssh_key + - skip_missing: yes + tags: + - user-add-conf + +- name: Conf - Allow Password Login lineinfile: dest: "/etc/ssh/sshd_config" regexp: "^PasswordAuthentication no" line: "PasswordAuthentication yes" notify: - - "Restart sshd" + - "Restart SSHd" tags: - - allow-password-login + - user-add-conf -- name: Add visudo entry +- name: Conf - Add Visudo Entry lineinfile: dest: "/etc/sudoers" state: present - line: "testuser ALL=(ALL) NOPASSWD: ALL" + line: "{{ item.username }} ALL=(ALL) NOPASSWD: ALL" validate: "visudo -cf %s" + with_items: "{{ users }}" tags: - - allow-sudo + - user-add-conf diff --git a/resources/tools/testbed-setup/ansible/site.yaml b/resources/tools/testbed-setup/ansible/site.yaml index f2f68c6bb7..6ea7fa1d0f 100644 --- a/resources/tools/testbed-setup/ansible/site.yaml +++ b/resources/tools/testbed-setup/ansible/site.yaml @@ -9,3 +9,6 @@ - import_playbook: vpp_device.yaml tags: vpp-device + +- import_playbook: nomad.yaml + tags: nomad diff --git a/resources/tools/testbed-setup/ansible/vault.yml b/resources/tools/testbed-setup/ansible/vault.yml index 56f15d84d9..c2dfbc9476 100644 --- a/resources/tools/testbed-setup/ansible/vault.yml +++ b/resources/tools/testbed-setup/ansible/vault.yml @@ -1,168 +1,401 @@ $ANSIBLE_VAULT;1.1;AES256 -65646237323436373833326439303135393936363466383764663164336365623832653531376234 -6438303331653966383964383663396536663761666563320a633065656366336466643330303936 -33653539343061343839353736666330326434383430616638666338306332326636316138656164 -6336303733653563340a613062363264393031633339363263396631663562666262663136393239 -34393763663737666338373136633239666332376464616465376237353732356334336338636231 -64383237313136363461323734383665306537383566313366636562346562383364636663636364 -62626136653666636264643030333332666131663230643531303932333233643638356366613363 -61613933386664373066626466336330633332613566633535346237623135303238643965373963 -37386335303934306531653036636130313036386533343437313034616663623462386330663662 -66663238353665666363353939373066633933353539373335336239383762376562623465633166 -30343034643863306132323936323231356233306463656130353536396366343938643165386161 -65613230336237383963623638613437623136613530353439653133373466313434623530653666 -34623866636265323030366230356632663934393035646339646462633566396231633038653134 -66316333383336653238333737396262613535623266343733343830653531346432323238333535 -34333739313730633037363232616161313231626166333462383764396235326637353732316335 -39376161373031656236623939376333343530356361643134366563316662373938346338633265 -31366633346136663239353338623562316532316164636638393634303630613261303238633731 -31396236326235323962613937363134666537653231663836653362396532616235633939396134 -64313235613463623263303233373931366337393333393462343035313530363836333466616238 -35363866653563346231613361303162616361363839663738393565643666383765313333363136 -65333563666532383264366132643064626638373961633465613932616637616438623938653036 -62666330633330623632646636623262393330383166666139393164336237303738343132653166 -36613836386565346263663966613139346463313036366138313465363064356563393761346164 -31323131613132303339343162386266303561623064383465386138623933643832343263363364 -64653330643161343263653361356436653332316561643863386334303231616665326364336439 -62396564336332383939353932313236346664316266323139643461623062663031343462383432 -61306437663838353334626365336537663563316264346634623336633331643330363963633838 -39333662623237633565353139396632633034313463303264313362646639356439356232376333 -33376265386438663164626262313630303662396537316135646562363438633163613339323639 -65376266333037303066346533353739303465353937323864663862303832363261393066383363 -66316433663533396438663839336563646265666335656364333966383763363563383437366138 -66393563313835633532613735366561653433343164346563383430323164383333616338386466 -61656365333431646238646534306130366535623266386432303035363337653865633561643137 -34613162616131346361306539653330346637386631646335383233353136353133396637666339 -37356631376638323761363261303438653636353539336535646263666266643338626530363563 -62613961343461333531366162313536656532333763633861373931336165396238393763656561 -36366631363731353837663634316666323436396163636333373037346331386362393633323835 -32306533376366633363653363376666313963306639613366653562363762636236363064353733 -30633832383330313661666135646531346330393535613834323431303334646632336239343532 -34623330663630373562373432643334613664316165303736666432373164613764353536366535 -35346464633635636162663230386363356465306164616234396331626537633462623433313637 -32396431343666643663383931353133353137633431346531313263643934643039363734383738 -31333438666165353739356633653432613061303965353665323665626330633764386339343639 -31383763386465343836666266363338663265386262633062343434383139336162393035623562 -66333034343264643036386130346234396433633238343838663230303630613663363137323362 -32306264303864323438353363636664313935383565636135653264663538313432376639643663 -62323161623532656433643238613939363933396334353438663235626632663936346536323964 -39313865356262373463656638646631653934366532383834643266323530316135356235366539 -63643361353864653437353532383134343861633566623331363561323732323639313738386231 -33643931363535393765363036343334323464303436626332313064626338356565396631633761 -66306530356336363062663763356337333164303564363630303866333030303337303562623738 -34636230643233353437643763623435323466393632623939353533303163633763383835343535 -36386230313230383532376639356564646139613338633437363161666261643833376263666263 -66323762373839316131343834623532313232646132343834326363663731313566333331646137 -64663331646338333765383236363032366166316464366431323137363263346636616265353366 -34336530303132316639663036396438336466656361393632303636656265386435393134643938 -30346663326632376638363433363866366163626266633966333530626631346330323038666635 -39333135306362663364373532386662623435343136346564386666613137643663303732373831 -37383032383866623565353966396533363234653665346332343833383234316138333032626161 -34633836613630333066353732393534373338376432363061326538613336613665336461653666 -65343461653835623365376338326137303262353239333263353231663836313139613333306534 -64643237623262303532396163353939303661643762383436626664663765353763383834383663 -62386338333338616130353936316130336633626165393737303162316131313266633762653830 -30636635333963663439356530663638663336393230383237626165343037666563306663336364 -31653931313730303930383663646666613766303464376532663866363362336535346130633961 -63353661366664653164663730373165353839336635323136306266346363356561656535306239 -62646631396365346534326164326564386464386636373632646435613434383161343166383363 -31646639353038626563636631383438333832623834343233306335316266663734343636643438 -37346531646137383438393238393137356366663363363962636635353561656532653231666438 -31353339663736386638313135636136666364306562323134313064626438353037336531613065 -33336530656339636231393236393033643930396432653637326436643135363639396661663764 -37373835626531373831303837613231643336333861663934306364386135616561653264616439 -63633465653637336561373836646436383566353033636237313230393831306535383464643161 -34326164326633353033336435333630353836626530303830656231353061393038393735616339 -32313039343961623636373336616564666230343961383935616536616231376361393338326530 -33373338383234663130373563636162393966623036363531623461633631656161613166366261 -39363163666463316435363938623135326361363466346266333833396135353839306662303939 -38383431386139326530376537333264346264373565303735363861656637373234333965613639 -63336635316632353130356138666465356334623033356339636132393366366265313934643335 -32306435653666663035323463623239633939616464666132356361613035313861356262336632 -38303032666537623761633737633564383332336132663061653037353239623836356163316564 -33653131353436313565653531363762353336373363316464343365346265636439353261363134 -39313363323635653264343131653666623963366366616266663566636239373232333766663962 -63393630643166393761313839616236373631383166613564633564613235613338323031333762 -61326134353938393764343063363764366633663462383533633734386363313266326161303762 -35386637356466653037626262613439376165616463366338346338646563353637376130313438 -61656638393239343039613064633438393063623232663031643839323734643432636633303264 -63336536333063633262376239646536636435623863336236323836363939633737613232313564 -30303165303235653664626133333436393836366437383463313530326666656363393139393662 -36623438633230353563343731633035626565343963373630316132393564363230333866366239 -61633363356531663338613236343466393834666433366139313636653730303966656633373135 -64613966323833393465303339353637363766323965373631343337333564386138383630303935 -39643132313135396461323361643265393064363966393234663662396132316639616230326265 -33346463373935373833643764623231363134636337336365383038656437326333303138623566 -64663562316166356161353830666231356635653332366238353135323465653739306638353261 -35636430656164353963303435303736373833336439653133666166616332666436393637646537 -39373335333337343931353535363534636434316130376635353831386461643536623835313332 -31646564373533333031353365626433613762636463613336616633343230643764306334643662 -36653833336236663834323833663533653439653731323435336665306133646432376437613761 -33306533663135333532323838653130633361653033383334386435343832663434383031396637 -37373662373239363136396332633762616137326233646264373661383464333132343963383338 -30336331613439346138643531643333356135316538653331353530353232643537313661386566 -32633031613236366264626466633033336431346134326338656534303634356238653164366532 -34633466643235303362663130356232636233626666353161663561623634313966326630343466 -32306364613634623930656538336634336430306235356536663361343264366136306230313738 -62353436353638656664353438663361313235356365396336356464353832306238626162396330 -61323262303337383239646238396364623634386635643764353330346433663665383561326339 -61616164613931363337656264353439373033366133303835393637623265323032653361303961 -38353665643264396439616363323761626538643465333463363032616333306238326332303231 -34336461396664643332323335363266323036373036666235656165653736393836393365303530 -38386564346231623238313030396339666262383638666539653138356139316664333364313031 -31346133343731663234653465383961396465616432313432653466303337383332316133323662 -61363731313662313035373136613761653030613139363133316331356532393133393332613039 -31613934346365633935373761303566656231303766303130363337626431396661343335643463 -32376235353466353831346265316336363961386132643531393831373830343230366531303933 -35646535386132643964393832363239353637383838336336653230353036663461653935666463 -33333938636237636135396637616666353036313332353666303166623531353838323361336364 -35656132616130363133366231323164323635313632353835623236373638353237386535346336 -31323661636366346232616563366230336562326132646665636562616561383665643561393232 -62623864323533656636326264626230666633666563663833386266616233313638653836623438 -32393162396234353262356232666462643466653461633532343638323533656633306335623565 -38316330323438383837646265323832613933316135326234383864333737653735666438323965 -61326266646663363162303036343139623265303331623239356466333430356431333664376339 -61636631316436656337353833336630373036383736353166653434393433333664376562346631 -34306662363935613633323539326164666130363538643038366265373734643961656465623830 -39633862383636643939336232323832613832383465666239343236363566343363323836616661 -65643364663538316138383038626161303331653832656337303933356664383435376336313832 -30313832653064316239356661323138383762383864613233333530623131376237356136373932 -34636531316364326563373931306635323964616563643563616634643164373765336132613330 -30653862373966333630613635393234356338346138326135306635373666613533626135396437 -65623865666634653665656364393532633766326661313136356461316661323663363838633164 -31386234633438636165373139643337636532363036643632366366636236613164336638663839 -61326462303234326561376636613463316432653762656236396330303465303065633534356433 -62616438336338646437666564336166386333623030306136613236653334306634633666363237 -30333333343032636333383238643732373034646362373831386162316431383235326132363539 -35623737636461303935323962373362323235356666353561663234373730383262343464653265 -61333138326466303230363936383862313334633130336639396630653233623738366261626133 -64356635643030353263623936623766653438666466393736386365383431666136303666663164 -64633932393362356638646333623266353938383564376238643465623165666330313936336566 -64613333633862316234363830346264306565666236393735346466316364306365636164643937 -34613335343532343161303439616464623331383133313133616630366464383838643030633366 -38396230613332306131323434636336636535306437643332343937626336353438343434653539 -32316536386563623537366236653639663138386238623637633836323538316336386365366162 -62346534386332646430656132653761306164653732323461303763363736636330366561666564 -35623536343139666339613961396633363562333637383230626566313133666237643366323930 -35626231323364336531353733313931306163643733386565623536643766303535666666616536 -30636231633330376136306134383531643833663962306433663064383538646133396334616364 -33333630343263353736626330623637393731376138363730323963373661643534623533643863 -31363966613963646334303737616562613537616661383461363136373936623433666364393039 -61306139383531646630653461333231373963626538656166396431666130313361373265666339 -34333762396666353330366334396465353731613734383265336365396634313835666363666531 -63393831336466626334393834323165333165383161306531616532316565316633306538333566 -65353337343862333065393861313464383632666664376266373365613066323037616566393035 -64326135353038313636333163663562646332633066316232653532623265336437356362376364 -32656361313332613261353232396164633332616163343333623236333639613938326633323132 -61653432343436353539383463346365353231353935646431353437313437313139336433383365 -37383763613437636435376265343039376266623534653539613438353335363937316137323335 -39396537316662613362636263363736646332396233666362613933633464343762346332366339 -63376335316261363837346238303361616534306237366632323933333965366632303235646636 -31306263653766323935306139333930376339366534363634646266623831356333353566623137 -39393561643630383262336432386266303531666630356433613233623334663834616264633064 -34626335383166373933393461353738333863643230343834356166653538653135343937376137 -31633537386133313335343164623833383833303632376565643536616363643830376462346231 -38353431653330366435623062303634663332613038616563336364656333346537663365326161 -65323437623132376235343234666263383566383134366566313633623431333834303762353138 -34623338316131333062363635636361623336643332373462636537656635353066623531306262 -313666303466366361313532616333623131 +37383464316330306537633837623331383864353039623636613630623538346261306366316563 +6566663934656231656464396231336633393361323332640a663864623739376637303066393665 +35343837306337653834396663666139363535313662376262366463316435666261303662316230 +3830336532363861330a306138313961343931383438363663323162356166636461306234303165 +64656563613034333139633234326165323231346366323030663137393330396231666531656539 +34326263666435656534643664396661366330383466653334323239653533346266363866633436 +30633134313436623631346139393737393266386139653665306261643732383165363364346264 +31336161633530653731633530366633663161356539663563643633666361626531333639303134 +65393665383265323632366335316166653935323632366662613133323130363665663835393236 +32316530663333353436376637336534333431303739653535623766313236373261316339646632 +66333735366363333934353464343366343661396632353932396564646435613033623030646331 +37346137663765616231376266373764613331663131643266336538386262643762383533666665 +36663237386363326234623266643465306366353863303564346632353237636462376663343938 +65643566396639646566626330376138663830643434636133363637616635623733383930663564 +38303863336339303861303339363433306331396466343434323763336165666434316633363731 +30336234343166643964663362363930363331336436393863646464346431663262623731333532 +34336130333861356238346565336338663339653461386566383036613139363562356532666562 +32303331626536386263653232623230316262633632626462323034343136636137666662313962 +65613462643932636363343635643534353664386333313363396261646563653836636638653134 +32303239643032626136373833643838613862646538633234333062653063643962373132653330 +37646633303337333561333832393661313333633139396662663036613565343365316431633162 +64373462656362373161356332383033666261333861333637323436666432363663396163373534 +61343630623637313937616137666363643933633261383163643535396139386336623235353261 +38373263653637393130303762383738386339633162303137643332333137363338306132383566 +65623839653739316538613961376138336437663466343836643762653762373036316234393564 +37613565303233626139336236646265356337383436363664613935333064643064663261356435 +61316663373939303038633434663833393638393732343836333334323634646430646361626139 +31636364613762363937663763333339353263333530643761376631313861646330633366623763 +38313633373736613939396635666232373163313239633336663636353261393731383364316265 +62386365303761646339383434663331306364666263666532353836613161393532646538333362 +34623963653866613365323632323332323432353437336362366238316239663563616535326264 +30363132613339613933356634366237303064653235633039656239313738346436383738353030 +62616237386536333663376538353839626137343338626334373634663339303864373564636435 +31316133643962306434323234316532376562363133333164663237313164633633663236373334 +33613131656239336338623636373334363130346363366436626237366662326263646630663564 +31353762373430626330353836643266383433393064636264623234636264306331306264623266 +38643737323263663836393261643962333762363932363832623531623866623766646166373565 +66306535366230633366366437393536356130636433613264326163303663316332663135323966 +35383662336537663464653135336638626530306139363665343631656565653161346634316132 +62336634336666383335303930623837643638666364633464303536623634666636373961326366 +37326234353336646533346631353166303132633835346361393337316433386330373931323962 +31663830643664613631663561333638336635333636663937323038313830303163393364383438 +32346434653732653630316531646334646664346330663762353135336462643431623665346633 +65633539333935633465653861316336343339346138383565373830626630326564366661393832 +30303930343735616463613166333636396665666262616332646135613731346330623139616666 +34623861643162393062613537643064633532313234653535643436623632326537343866616164 +66626631353539666438316663323765353035396666373236636235373566356561663331633334 +31646337343061663565373461613761663437313338303464313866643737353162303863333837 +31666535653137363461636331356331666535643339666135303232626631613132366230393333 +34653131626461393236666338363262376133353535323165303531653535653732336235386232 +61386238633064623230316465376233353236346465613731653437323835333136313165643262 +62633065356131396530383263343838303331383836303261343763396635343234666664636531 +65643138333831666432383937336265376566333862623363386163333464363961353430336239 +38633664623464326163656134393364656238323139343038356662363766633333613065346461 +65616565646262313766363064613333666135373263643566343435343032653964626566323962 +34623633623938373837653062376362373431663937643833633563663538386561343863366236 +35663033356265393465633934346436356561653136333835346339353236343638316430623434 +31616431366633393031353433303365656438343635363634366361383030633133346264356261 +61656530313932303434306532373930366135636633373736623664386639353763333634653531 +61613461373732616333383965323063313039383832616361323336643230633765353062376633 +32633735376534316262316265393465616231303634656539343261633533313263323262323536 +32353039613164323761386235623664353162653338313962303465663534623762353238653634 +36633734666532623436353033663030633437626131663639376364333364343263326163636437 +35343535336232613861363334653262633936383138313937346236343538313237323230626465 +32613439613464313432653861613564353838386137353964323761306636383061363532346232 +62643735633939343064333162303764353533353562363938353363313266613231653964303330 +36396533323435653765373564316661633066373363323633343234653165623266393730333032 +38353334353030396165616263336335643463343565343762646665326636656232656166373665 +30346165366331386430656637633063666663623362663633643564323933376633623733313038 +65303338313161316431636563623335366531613862363930396536616161636132353930303035 +36326532653966653766366338356562323036343637376362313133313039646533333264356663 +39343464616435666330316361613835626639643333376465393931653732636636326266316463 +63326233633362636666313730353030653531613266653734633163613937643462656138346537 +66646434313332336436363633346438313866373837623838303238633233653134363465623130 +37633838653061626430313934666561323361366365613766373232323663643337333735333631 +30653534616230306139343530666665663962326263323465363035363932323735376636633639 +64343136303366313465666363666435356139366537336231326665386130643532353864383663 +66316232633463333739663537343837653835633363653737386262316538306564353363366238 +61653665346139316137306136363666396461646561383233313436353065333864333162353765 +66663936376535326236336261326235343734626139376136373762306465396232313538393633 +39356137643433666461306534343030626235306366393662306232633930633735646335333538 +64613037303939333630636135316465666335316233323363623730323938313065626466363430 +64383037316234343066636337616466303261323166396665373466303465356136313861623163 +39396532323832613461383831663139656339393932323232663639643339373961613539646565 +38306162396163626531346265333066343462633161326666346430356662306665663539356632 +63346662396337363036353261343035646332303062366334353438303136366538613032643632 +34316335383537333165386561303536396630386535313231363537303533346133343964343135 +33653831323032653838306231323862613734396464663161363764346238646237643365643664 +65313065613330363161303864663164626563376331313165393236376531343962383139333830 +65613631656233383261613861373538633230653436353962333162663165313264636135323330 +64363461323730316634313766636533363131383238616363366431633330376333303737663966 +30316163336361626133373165346231613461626535333432653835616664313536653863386430 +34393238396366303438653831376634653839396531313936663230373336386137313934666563 +34383030623739656634386632303033316133333734656364613131353230343639373630313138 +39373830633161656265633032353538343331663535306535326134646664393736623563393563 +30663036643037636261613633636439646637333230383237653238373264353134333838333839 +35366239616339353031623565613237366230663732633530636266393965316130353036383064 +39623238626337366437396630653764376133323130323234376561303734313732303865323830 +64643161306233633766613231346365653766666132616466326534383862333162666437626237 +66623234616266373139633635333931313631356138386431663563633833316138346633383137 +35343334383337626635306664386239653764373138386563333061626435356338323761336331 +66393430636133396662326666303832336162366166326436353834613034663639323531393738 +65356137373334353837663234383338323634306162323533353930303762646639323639366231 +33336533393731313431306165653963303736393264323136616437386661396438623237373165 +32646337336539356431336530373566656631626161313864306138613933663436333966636236 +63373530343337363334656637363436323039393336633736653537366265626237343434313834 +62303537663436306538356638323330386265626563333638353661336530363136373236343033 +39363165383464336661626632313866383037643332393265316632313435383333626239356332 +66343863366634643836636432656630313639393764346639333236376232353436663362363761 +61626637323865656431333830343239636536653538313432393235323563656539666532306433 +66643730613237613936383863336638303064356437326536373764326230363439396332366136 +39623762326638623365383064313861643933373564663134636638343166643839393363626561 +62663466303732613932626236393131663730643166613964376639656433333835333130633830 +30363634323961383765656137343663383633666233393230353032633064393062326539306537 +38663365623263343262326238636331333034303533633931353335656363633866663162373735 +62376435336531653230653033636235623864303438626130623464653838663261353338366233 +66323865363436383861663735616335353337333462336265656134633036616130613031333139 +37346433653337303164376664643431636638333336623832376463373061386263346439366236 +65653163363964386663373836653566633632393632663931336332343330626539323963346332 +62623865383434326333373532383063313335316131353262653031626664323332363162383730 +61613463383263333264363466623836306233653064633637343736393437633733613434636261 +37393962633265623432636661323739326462313339326266313736636130376431656530356236 +36373531343964316661633164313164316231386366626462626164303361623833303937343961 +36353834323931323361393433313633333866623435303931346665306134623435616235343365 +65396462636237353832313562393831363733393566613233396661623434323763623930316137 +65316434313935393932326332373132313938633237326566636437623763616131343237383338 +64356164373737383436373733616461343035333130356362313334393861346166326532663539 +34636161323862636435343663366662306436316463333331326332326236323066343430396664 +31653363366330303737323063373939626662356433623139626432363866366465633165353832 +66616163313433633266373738643963323438313832316238616464373432353434663431653833 +37663734356539333735373666393836363332316538663330336136333262363262653961336536 +34313636336434646663366163666365386362363835376361653361333437323331313863663837 +30333832333435643630653438633465663033343662313236306632643435323332363663373339 +63613031653261636630383166393033396665363333393238333438323137613435336430626461 +34613434363733363933343431363864643562383932663162303361663965323035386135303134 +62303935396432343466656662393332396136323363633362666530383962366232366365646263 +66303766626565333037336364363934663965366565663839396530646132646239626165663563 +61353461373565333232343763653461353362383536383035616562643465346262333835303330 +38396363396662313734356337363535343562623163303532383361656565626465623633343538 +32626664333461326137393036313666353832623264333238663135643033343861313737613632 +35363832306363626535316564326330393032306531333836323361656462303137323765663737 +64613230356433366561333636363630663435366234313730376538613134393765303438316364 +33366535316366343833393238303239376662356231656138306664323930383139393434383836 +34613130363265366561633030613862376338373730653561623236636232333535626335346138 +63613638363565623837386261623838363539363062316336303335383836386565343963356439 +34343562353765316365336564323434666639313634393332633765613061636665383965303830 +34656466656165336662663231663361303166393736323461333031613433613532346433343565 +38366632653666643038633634366537346235303866386235356661336235646665306335303561 +34613561623336313739633764666661616130383537396566613433366166316130663735636265 +36376436346461346331383361353462376633623063313334363130653930663032373531323835 +63303564366536613330376136643130663762366231366162323538363432633834306365313333 +66636337623739623132373137643531356162353231636365386533643337366661316666653838 +35393635313036323233656136393231396633316531646265626562363964383464623532663437 +39336434373562363165363230323064393738656338383633366166623963366630663932343039 +32613866346563313835366534626433613633383132393763343037623066393965666634363366 +66333261363762303732626134366335363933303162373733653266316338626330613833366636 +31613832343836363038353433343338333262313164343034303738303661623964633830363462 +30303339636534633034323331303864373132353838613939643734346661636166346334653034 +39303063356634333332356638616364373362653034396139356139376566326338663236336332 +35376330623237386434633963306238346361323236336565393964373964633336333334316163 +37636636306531323131393334396537646662393334613766393837326566363964316434653838 +39383738356234383232613132316164656535393637623139656434363862366264366237396533 +30383161613439313335396531353264343066383662616239343165623236313736373861366465 +30626464303634393539353763633333386463316266643763313363326462306136343164313839 +61393963313433396663306435653134363038616161373632623732333939336261313464356161 +36326638343963613566646435653731643761346533333337353766323534616535346335313964 +34663330626532613432343362306630383663333531383961613562366334623836626361363966 +65353136393932343264323533356330383937303435303830336538653634666439323833303932 +35363431343339303961393237326130643861326134633466343537396638646139363034616661 +35613033383565303036306138663733653839376638333063306365393735353235626562373932 +30343964386633633731633038393834373233393562393933636634613832353631323831393062 +64303236326463373766653334663063333039633335613134353238386165656332353166666361 +37313063366333323730303664616530393962633036656462376463663034663566393934616430 +32303363303833643232353463663937633938383465653030333362306266616465363062343334 +37366664616333346134353839343136356266356135316336373361343166613963333631306130 +33663235356331616236383038363164326430323530663432666134353035623665626539363962 +64336164653661636139636263393264313437313739313763356135333766646335626539613738 +37656333343462376538323836663437306333636637396430373036303965323137653165343436 +37616533626431353764306633386331386637366464333437356134646262396535653837313561 +66303064366533353238663530353266636631363730623034633862306538396464353332613063 +30303336313435616430396361383166663433633137356539393265343833333535636565353466 +66333665646135616361636265366165653865346136306436303131376265306566666465303835 +33303237336134396562366434323862323762383139376561353865363866306538316238633435 +35366431316333396135373830636665386461643235393732623263636331643234666137336461 +63613462663836336161366364373437356232306134356432323162636565333837613733373536 +38303432313337346431376364303464333339623134343330313630666635313162303466316262 +35643236316465653538653261323131323739343032646339663463343365366235636338363763 +39666130383562333439306466653435313766663238386231383962626566636533333265376133 +30326464643232323262373732393663343930393265373532356139333230313637653531393338 +61383334656539353738393832316239306265313931383666363532363136386230613537336661 +35323234366639343730623732353433633137656563306161616463323432303061353365383632 +66643732363165653138633638323039306531646362353131336133663964396361323830623236 +33643733613436383331316136353462366166653236373632656162663138643337666436656530 +64633833393538393333623465396232653330613562323562636539313166613664336236646363 +62343236366466363633653766373364343762306439656262366430366139343937376637643538 +35336330383730393738333165313663323564333134333436316131633936643035303838386262 +37306632643732343734613334353463383362646364333861363935303262656634373737326431 +36386466663633646438383364336634306532656234356334393161623264383336663233386634 +36306636623035613532336562343461633166313262633434393636666362366566316566366532 +37616362653364376165373764656263316630343861326131343636383965386666613165346264 +38363338373334393536393165316337333230656664336332646635633237333835343837363263 +31623231336263363036336331383936613232356138313830336439353932326437373935306434 +33633138663631323566393130366632303033643236373134383838333432643736323265373434 +30336433383439353662653438393231316565393663656338633564346661343838366231336666 +65363735383663323732393639303834623962353230346530393532323761656634363131383230 +66306261663538633938313363643636616232366532366435323833626535396138353233306462 +66646434373835333631353932626436616665313163356235333165393432353864653262636466 +33313864363563623666363733316261303233623331353935303538336439336361383732383036 +62333830383966643964326339663437303664326266323266353331623364303231646161376638 +63346138343462313033663466613162343263383633343962383764316336343336636335653164 +66316363396336306330363937316362393863383934303837316561333138306139653333333434 +33356261613838383634633733326234303736366163316638363539653738383462393636646437 +62316236373439363735393566353065636633653534383665363433333235333833356138363566 +32363430393563346239396264343162663233326134643964626662386231316566363565313566 +62376462346166613934396238646263643431623132313566383538323336336162636439613135 +35643266313264356135373262363766613765653735333634313835393861383163353234663738 +32646237643664626266356331613932383034333038366230356266633231353930326161313537 +33666663616565346639396436326530356539356464383232333863393932306463333131336633 +36376530313865373062303638333364393964666662623062396636313234643937373332393061 +32663565313361386232643831373834346436373063633734623136613337336561346238333233 +61376562376234666639616662316237613965336539633438313833396632353166656664376630 +31353266386439376263656235643463396133343933656364646664373262643365353634363635 +65396663653961326463653833323263636363306666653632306637383531383431376364656530 +34346462373330336434326338346462323734366235656630373666383336396234363965343630 +30376330323562643137326562616338376537623334396462313764356363356337393834626632 +65363735383565626636306237343161653334656233653464306164376666353830303537386264 +63366430396635633639363031303464326435303138376561333338653564383065313865623664 +61346664646639353035643437626235643432656663323364343338323961623837326663396135 +39616265633933313465363331613137333563373434616236326231303331303735383330613532 +66343366666234313238626535633666363761666337653339616635643263623435386265643261 +38306239643230653836656134646263646265323936326462653337343066613163383062356336 +34356530646139636631656338613331353131353830626365633532653464386365323762326334 +63653531643763616162623431353535333665353733643935313737343638323565393735353637 +30616538343332343461343833343030336365333566623964333830343465383664316135643364 +65366230373837346130653864636237303464376537373632666236356539383736326166663461 +65626661616235653538653062326264666265623739336566316236643963656464653237326265 +34623537336337643266383438626233663733313435303736316163653536326436656638333431 +37316639313965306235346431613536383366613362636266363563373766633063633961616332 +61633138373939626234333264393438393337636364636233643932376466393661333532396333 +30373931613938623931613036623336636638316461643438613963643462663337383630643932 +66386366353239396234383736333331643961663136623438333164656565393537613765343161 +64633235396661363534643963393564323932616133623737396132613862373639383064393232 +65363363336132383966656635313033303034306532666230326234376437316239393035646663 +39386230623639353934326533303864633863356436626135613139346439336131396639396261 +63363664333866356163666637663732373237383431306634356166303732326132633463313063 +62386432336361663237333963636533633565613538633334306434633239383932343365656435 +35386161646637363234343739623230343765323131653861303438393466613635333535343465 +37313761663637303039343762653861396337363132633535363866376637313263326432643434 +66363932333735613462343331303939336539313530373730633835356466356561336339306564 +32383031383931613266643538313534643334613331623438313130353438383133336238373237 +33633561653865633931353664323865333965633363666531363931623737346336376636626133 +33373566663262363735353933396563356165663962663036373461333131636563306365363731 +37633339356263366335303930373530623938373232393365663433613836303837306330643634 +34313631623336633161313830306633616537333137313237363266383934646133633835353232 +36353935363765366262363061363131313063613930376631613435313339353662313632356564 +30313064336439663837376437393738643339643166366335383131656233353263633035383233 +66366161393232316361306332353435653764373064383462333137313532316166393339333435 +30343062386530363835386530316634633934323239326161376530616131663937646339353437 +36623334386438363839363161323166313338353037333234613939613762313762663761636530 +64643830353561653732303332376233343535353832333632373864616334313566626238656366 +65323233363661643339363133303930306432663861353264373435623237626130383232313434 +61356263303433373566373864616537616431383037306230356139666436633439376237626531 +34633362303566626261346237353036626261623964353563336566316266623664343866386238 +64353536326534613539396136366338303230306236373531383935323230633938303438376664 +30636238633430626563303031363665616433386162366331343439333034643537303135653532 +36636238366261313331663065656233353463643062373132376637353935626365633635363366 +64656635373233353131653430633131346132353265383334383230333630383338313365356331 +38383663313461303639613963356134366665356634346630626361653966396239376334356335 +65343530636230353061343063623162616661636430653234656433626161346133316138373033 +38383731376562633539396261626637653337373232626638643836613563326664333165653231 +36353466613937623832353038323034623935333438656661323232373031653834636539326632 +61393137636265343337336564323763383435343730666439623536323364613332343962393433 +30356264323461326565393437343037616630653237363434623831626437663937646235373066 +38396164643934356636323065393937393265613464393738303735376537646232333333373165 +34346535396634346338376564353239623434303835306230343730613335643235306534383565 +37666234363031363463633039366639616161343032653063303234336432343839336433323830 +38316237646632303761656633306665393361376330393839663432353439303730313335653365 +61343161646364393332316534353135366639313063393137303239383332376362303131353566 +61376130313739346566643433353362386164393034333161633834386538366563316534636436 +38636136306239633038393037643865363337393164633066326463613466383937663232306433 +36366136376235383935323761323362356561313731356262653364343265323938633464333134 +35623237343466306530313834356130376635396163373365613734373161663964633535663362 +39313338626662343231366462646633613136616464373736383339316561383765333463333436 +30363535336363386239626161663939346639643061663861346130373535313234616636323965 +37346465303532306434303662623164363830363633326238343565646331636636636437393536 +61633539376437623434616366346439616266343666653566633664646230316239313866333162 +37306665633434663064383430363361656432656361633431633931646637376561333164643165 +36303039323239363865323737626636343364323932306366626336363234346635656330643734 +64643133303539393061366236346436663966323265353539316166353861333635663530366563 +65646132333035666163303562356438333735663863613738653232613739323432633165333964 +65613834653865376332303566386237343334326438353138316532396232613031393764393139 +35323966636633323764356135613539393066326430353631353964636563643132326137653033 +65616339363861316132316338356230363538663765353063323334366465313362373139333233 +36396432376131626666653335336637396363633266323630636632363665366533636634623861 +66623736623433316138376364636463643063316532396533326137336564323933616363616338 +62323761383036383734303035656432663765303638656264373235313937653132366330636639 +62623437333838303536383437333237343562366237316534643336383161626532353262316461 +61383632326536343631666535396661613233346265373063376562333162666135393265343666 +32373532333465646233616237653561313739366462313832383431313430306134396533643737 +62356262626137646566396262643334343664613932663033366233313662313033323836343565 +34366534633739633131376330373431346232373739343634383836313233343936336366313035 +38376434613865616636376565343065343832313834343030653463386261323766303232313636 +39643935663063613865646131363531363533333365326635396161313334383230623266313162 +33333837373038623636313965623131383162303338623630343632616161333232306131383035 +66336165613862383235386330616334316139626336326262306264643461643261613038346333 +32636435353465653065326430353835333633643362303464383935353964363237616563663338 +34613361646437373939323633346233373735326363653861313661316362393832653133383162 +32323665653662616631346335386363646231616639653533363936653535346530373338663336 +31393236303761323335303038623939303434353865303830626436343237666535646161623866 +36323132376536336239306166363330313730626530313932316662373535313334393234346235 +33313564363434393537613261333831393133353662346631303738363430353565353666366235 +65353665363438356136393936646639353332623031616439303839613266373266663266373562 +65363134323962613837663435323966343965373738376330623039396663356336616664313139 +38373763333362633766303436396239633761323032393837376237313161643232373461313163 +62646531303066303138346136346536313264356330396332303366383963333834396535643136 +30623661643438383234393862353464313365313562393132626334633364613366656361316237 +34353335356433656538303538376337316436326461613130316630363132656534346462303330 +63356330376566363831386232626262313139326364353866643530656336623134336662393065 +33666636646462366635353630623065326161313831303933316362646137393538336537623161 +66353661303034373036386436383562323331666334396236346334313932346338343634633431 +39623330653765303231383439393434616338343739386536646435343764616366306335316338 +61396563306532306131663863316236346531623264376132643731613532613531333265666438 +30643134643837633966366139313138663334636438366433616466396561373731333763393436 +37643230623563303532333262333031626633396636353332323562303762346236653561356135 +62303637633466666130636431336230333733613061326263383338323664326662616561393033 +34303231343735643564636364373336666136336465393037663863366537663266343563643436 +33336432613839346266636366373463663563356336633538643963636431323261303361323834 +39616365396335303238666533366233663461393837646265396433626464323863373463366439 +63643930616361366632333462613731363065306366386234353335653563393131336339383633 +31656631633933616637643439393737643862303063363231326231633834376138613766373565 +65393161613733353039656162323136653565306438376166326639323030316366623065626134 +36303736346638323732663539336263616536346233376363383130333832616635666138376432 +34303437373837666133623963393836346165343736656539633736616531613936386264643137 +63663839363932633961386363383865643132386130636433306166343463383636396639316664 +35326462663132346261626231383365356339346466376133336664333164346434663162633735 +33663230336138306365636265373462313962393037363864313161343364306332313735663938 +64663164643563346532643335313862653438666330363839346362353236363430656465386164 +31373461316534323036383562353035363532616366356262323033613237323334623665356430 +64663537643861383636353336626563396338386436383837633566646530356633643938366364 +65356230316261363633363566343633663631316563636364633661383163353633623933613764 +31373031376165616332373239353065616663386136303639663034386334643265336435353332 +33363836356639343336636332363662633764353532353266386139626566373239666339376638 +38323037366536306634636366336438656565383162623236646331336565363065313136623539 +39643036616465663734306239373965656361303039663337356630313439636661346531363561 +38656266323230653166313164323066323632336433343336346234653061646663306531396132 +62636466306465386530353066336439336162663562356233366630383738623732323339653036 +39306536323531343961616164343636626266633763376662396138363130313038356539646561 +64633362356332656637643339373831373935306564356533626338643433646537313564336235 +33316366343834343332306633353363366432643138343839646335653536373238316339353335 +34633731363833346438666564333733333034363766633831346539343238623165356563353934 +38396539356461636536633366326231366364356666623965306234346238353834663233376236 +66366630376537653563316263346330616634316466373861393736303661373338393066326132 +32386337363064333462396535356662323161313437653232623932613265383530303330613535 +62616135363866363664616130663738613130303866373661396261346362323262623239333135 +63373330316535613166326538366664393565653761366536656266656362323838396261623261 +37393534333262643663313233666335623835333135306539353034333733626163393631333037 +61386665366233333032656331656264636539353034643634663533613038323364613233343034 +35346636373263336266303864306635343064663062313730383661376137613331316634383838 +64303461613961656631626236383330316663326231306261336563303638326163333064643738 +38396465643934653237323864633931653537366163353136313766323831323066643962306335 +62323965613466356632636530383838363935643236626362316631653839626461313736656138 +62393761393231366564386231323731313064396135343966373736353936333065636639623863 +39323431383635366237613661656437653937666631326366626539346234326435366464616231 +63613435373665353462653862646232316238393233386537623630396661623136326666303337 +34366237313761616162336139343935613537643164333465303865313361666432633565306130 +35613036343638643337366530373365393137396438396261656133393830636137343938326634 +30356662616330343864333032613037333062333538376263623933376339393465356534343339 +31663363633839363132303531653035383835393665636466653836313635386364386263623063 +39396133373439616434323631653534373163633261613432303035383235303739383138613465 +35303561633761616363663961613161663435303936313461396435376432343239613330616530 +39666534356437306536363462656535326337313433623334623963396337646530663961656464 +30646162653139336533623365343961316338326435653562316639326539333862373732346563 +31353265383738393531373539353039343339353134396439623636623738616564363934623063 +62363736326331633232376465363361636533313335373661613933313662383761616237633837 +61373364616462336530366264306665313565353134323538353332343261323863653834303832 +62616132386437386237356232373334356661383262386238613334613462343537333764326338 +35306464316630353837353462663061646139366533623036363461303232623830383633643534 +35336234326261656265623138356461396638356338373466366563393932633137623132303136 +33326336666439626233323961353563623764663332646165306335373938363738663939396132 +66636433313665333737306362326236366161343236643036636239333630373238353139373431 +39613235346632326133333465633138633531613233376430383030363764656538613831663331 +66396630363330366462633939623564366165396137373232373937313132313763626336373936 +63313262336465303634353066303864663035393435663863306366353365346435373862363738 +64316264366332363563306164343466646132616530636664643762386439646562373761656335 +37356239373265336630386636623733653537356662653066643338373735653339326231306230 +65353433363961363437303332373537343134356663383165623135393664373539633432636335 +61643831346234636661303833626265363831343762373933363337643635643532376635393766 +37326466376639663663616534633662336635633431396633313336643863646238373735636137 +64303861353763343463333530326432613533383565616536303035613133653566656262363739 +31383936666132613962333837353839316264376666313862626635313831346432373165326135 +39363231636634343765653039643933373161313965663130383165363835346137616432383937 +30363665633766636662343639636236376262333839383631643264333233393432616466633438 +66646238373933363435363362353730643763396662303534316266643130313332356163376631 +36663766636430356631613131656132396232313136646339393232623331623961313763343431 +38656239666465613566363037323963393931356239633130636666616139323462623835326537 +31353431323630623139356134663333333261333939343131633163306161653435323737613761 +64393466363039633632313533616338636335333134323230346364353139356533343135373765 +66343166393838643439306564663234623362313433613163323237353166626563363666373838 +36386432333335326665353435306132346434623230316234383537353263613638333935626431 +61663839663563303133 -- cgit 1.2.3-korg