diff options
| -rw-r--r-- | resources/tools/terraform/1n_nmd/prod_storage/prod-nginx.nomad | 17 | ||||
| -rw-r--r-- | resources/tools/terraform/1n_nmd/prod_vpp_device/prod_csit_shim.nomad (renamed from resources/tools/terraform/1n_nmd/prod_vpp_device/prod_csit_shim_amd.nomad) | 91 | ||||
| -rw-r--r-- | resources/tools/terraform/1n_nmd/prod_vpp_device/prod_csit_shim_arm.nomad | 100 | ||||
| -rw-r--r-- | resources/tools/terraform/1n_nmd/prod_vpp_device/resources.tf | 9 | ||||
| -rw-r--r-- | resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/Dockerfile | 61 | ||||
| -rw-r--r-- | resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/files/badkeypub | 1 | ||||
| -rw-r--r-- | resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/files/sshconfig | 3 | ||||
| -rw-r--r-- | resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/files/wrapdocker | 113 | ||||
| -rw-r--r-- | resources/tools/testbed-setup/ansible/roles/csit_shim_image/tasks/main.yaml | 32 | ||||
| -rw-r--r-- | resources/tools/testbed-setup/ansible/vpp_device.yaml | 2 |
10 files changed, 307 insertions, 122 deletions
diff --git a/resources/tools/terraform/1n_nmd/prod_storage/prod-nginx.nomad b/resources/tools/terraform/1n_nmd/prod_storage/prod-nginx.nomad index 6c153ffd03..72d321d3b1 100644 --- a/resources/tools/terraform/1n_nmd/prod_storage/prod-nginx.nomad +++ b/resources/tools/terraform/1n_nmd/prod_storage/prod-nginx.nomad @@ -22,19 +22,19 @@ job "prod-nginx" { # The "max_parallel" parameter specifies the maximum number of updates to # perform in parallel. In this case, this specifies to update a single task # at a time. - max_parallel = 0 + max_parallel = 0 # The "min_healthy_time" parameter specifies the minimum time the allocation # must be in the healthy state before it is marked as healthy and unblocks # further allocations from being updated. - min_healthy_time = "10s" + min_healthy_time = "10s" # The "healthy_deadline" parameter specifies the deadline in which the # allocation must be marked as healthy after which the allocation is # automatically transitioned to unhealthy. Transitioning to unhealthy will # fail the deployment and potentially roll back the job if "auto_revert" is # set to true. - healthy_deadline = "3m" + healthy_deadline = "3m" # The "progress_deadline" parameter specifies the deadline in which an # allocation must be marked as healthy. The deadline begins when the first @@ -47,7 +47,7 @@ job "prod-nginx" { # The "auto_revert" parameter specifies if the job should auto-revert to the # last stable job on deployment failure. A job is marked as stable if all the # allocations as part of its deployment were marked healthy. - auto_revert = false + auto_revert = false # The "canary" parameter specifies that changes to the job that would result # in destructive updates should create the specified number of canaries @@ -58,9 +58,16 @@ job "prod-nginx" { # Further, setting "canary" equal to the count of the task group allows # blue/green deployments. When the job is updated, a full set of the new # version is deployed and upon promotion the old version is stopped. - canary = 0 + canary = 0 } + reschedule { + delay = "2m" + delay_function = "constant" + unlimited = true + } + + # The "group" stanza defines a series of tasks that should be co-located on # the same Nomad client. Any task within a group will be placed on the same # client. diff --git a/resources/tools/terraform/1n_nmd/prod_vpp_device/prod_csit_shim_amd.nomad b/resources/tools/terraform/1n_nmd/prod_vpp_device/prod_csit_shim.nomad index 1ba24696a4..328f503a0b 100644 --- a/resources/tools/terraform/1n_nmd/prod_vpp_device/prod_csit_shim_amd.nomad +++ b/resources/tools/terraform/1n_nmd/prod_vpp_device/prod_csit_shim.nomad @@ -1,4 +1,4 @@ -job "prod-csit-shim-amd" { +job "prod-csit-shim" { # The "region" parameter specifies the region in which to execute the job. # If omitted, this inherits the default region name of "global". # region = "global" @@ -18,11 +18,6 @@ job "prod-csit-shim-amd" { # type = "system" - constraint { - attribute = "${node.class}" - value = "csit" - } - # The "group" stanza defines a series of tasks that should be co-located on # the same Nomad client. Any task within a group will be placed on the same # client. @@ -32,11 +27,87 @@ job "prod-csit-shim-amd" { # # https://www.nomadproject.io/docs/job-specification/group.html # - group "prod-group1-csit-amd" { + group "prod-group1-csit-shim-amd" { + # The "count" parameter specifies the number of the task groups that should + # be running under this group. This value must be non-negative and defaults + # to 1. + count = 1 + + constraint { + attribute = "${node.class}" + value = "csit" + } + + restart { + interval = "1m" + attempts = 3 + delay = "15s" + mode = "delay" + } + + # The "task" stanza creates an individual unit of work, such as a Docker + # container, web application, or batch processing. + # + # For more information and examples on the "task" stanza, please see + # the online documentation at: + # + # https://www.nomadproject.io/docs/job-specification/task.html + # + task "prod-task1-csit-shim-amd" { + # The "driver" parameter specifies the task driver that should be used to + # run the task. + driver = "docker" + + # The "config" stanza specifies the driver configuration, which is passed + # directly to the driver to start the task. The details of configurations + # are specific to each driver, so please see specific driver + # documentation for more information. + config { + image = "csit_shim-ubuntu1804:local" + network_mode = "host" + pid_mode = "host" + volumes = [ + "/var/run/docker.sock:/var/run/docker.sock" + ] + privileged = true + } + + # The "resources" stanza describes the requirements a task needs to + # execute. Resource requirements include memory, network, cpu, and more. + # This ensures the task will execute on a machine that contains enough + # resource capacity. + # + # For more information and examples on the "resources" stanza, please see + # the online documentation at: + # + # https://www.nomadproject.io/docs/job-specification/resources.html + # + resources { + cpu = 100 + memory = 128 + network { + mbits = 10 + port "ssh" { + static = 6022 + } + port "ssh2" { + static = 6023 + } + } + } + } + } + + group "prod-group1-csit-shim-arm" { # The "count" parameter specifies the number of the task groups that should # be running under this group. This value must be non-negative and defaults # to 1. - count = 2 + count = 1 + + constraint { + attribute = "${node.class}" + value = "csitarm" + } restart { interval = "1m" @@ -53,7 +124,7 @@ job "prod-csit-shim-amd" { # # https://www.nomadproject.io/docs/job-specification/task.html # - task "prod-task1-csit-amd" { + task "prod-task1-csit-shim-arm" { # The "driver" parameter specifies the task driver that should be used to # run the task. driver = "docker" @@ -63,7 +134,7 @@ job "prod-csit-shim-amd" { # are specific to each driver, so please see specific driver # documentation for more information. config { - image = "snergster/csit-shim" + image = "csit_shim-ubuntu1804:local" network_mode = "host" pid_mode = "host" volumes = [ diff --git a/resources/tools/terraform/1n_nmd/prod_vpp_device/prod_csit_shim_arm.nomad b/resources/tools/terraform/1n_nmd/prod_vpp_device/prod_csit_shim_arm.nomad deleted file mode 100644 index ba7b656d37..0000000000 --- a/resources/tools/terraform/1n_nmd/prod_vpp_device/prod_csit_shim_arm.nomad +++ /dev/null @@ -1,100 +0,0 @@ -job "prod-csit-shim-arm" { - # The "region" parameter specifies the region in which to execute the job. - # If omitted, this inherits the default region name of "global". - # region = "global" - # - # The "datacenters" parameter specifies the list of datacenters which should - # be considered when placing this task. This must be provided. - datacenters = [ "yul1" ] - - # The "type" parameter controls the type of job, which impacts the scheduler's - # decision on placement. This configuration is optional and defaults to - # "service". For a full list of job types and their differences, please see - # the online documentation. - # - # For more information, please see the online documentation at: - # - # https://www.nomadproject.io/docs/jobspec/schedulers.html - # - type = "system" - - constraint { - attribute = "${node.class}" - value = "csitarm" - } - - # The "group" stanza defines a series of tasks that should be co-located on - # the same Nomad client. Any task within a group will be placed on the same - # client. - # - # For more information and examples on the "group" stanza, please see - # the online documentation at: - # - # https://www.nomadproject.io/docs/job-specification/group.html - # - group "prod-group1-csit-arm" { - # The "count" parameter specifies the number of the task groups that should - # be running under this group. This value must be non-negative and defaults - # to 1. - count = 2 - - restart { - interval = "1m" - attempts = 3 - delay = "15s" - mode = "delay" - } - - # The "task" stanza creates an individual unit of work, such as a Docker - # container, web application, or batch processing. - # - # For more information and examples on the "task" stanza, please see - # the online documentation at: - # - # https://www.nomadproject.io/docs/job-specification/task.html - # - task "prod-task1-csit-arm" { - # The "driver" parameter specifies the task driver that should be used to - # run the task. - driver = "docker" - - # The "config" stanza specifies the driver configuration, which is passed - # directly to the driver to start the task. The details of configurations - # are specific to each driver, so please see specific driver - # documentation for more information. - config { - image = "snergster/csit-arm-shim" - network_mode = "host" - pid_mode = "host" - volumes = [ - "/var/run/docker.sock:/var/run/docker.sock" - ] - privileged = true - } - - # The "resources" stanza describes the requirements a task needs to - # execute. Resource requirements include memory, network, cpu, and more. - # This ensures the task will execute on a machine that contains enough - # resource capacity. - # - # For more information and examples on the "resources" stanza, please see - # the online documentation at: - # - # https://www.nomadproject.io/docs/job-specification/resources.html - # - resources { - cpu = 100 - memory = 128 - network { - mbits = 10 - port "ssh" { - static = 6022 - } - port "ssh2" { - static = 6023 - } - } - } - } - } -}
\ No newline at end of file diff --git a/resources/tools/terraform/1n_nmd/prod_vpp_device/resources.tf b/resources/tools/terraform/1n_nmd/prod_vpp_device/resources.tf index 8dc101c12a..dace9094f2 100644 --- a/resources/tools/terraform/1n_nmd/prod_vpp_device/resources.tf +++ b/resources/tools/terraform/1n_nmd/prod_vpp_device/resources.tf @@ -1,9 +1,4 @@ -resource "nomad_job" "prod_csit_shim_arm" { +resource "nomad_job" "prod_csit_shim" { provider = nomad - jobspec = file("${path.module}/prod_csit_shim_arm.nomad") -} - -resource "nomad_job" "prod_csit_shim_amd" { - provider = nomad - jobspec = file("${path.module}/prod_csit_shim_amd.nomad") + jobspec = file("${path.module}/prod_csit_shim.nomad") }
\ No newline at end of file diff --git a/resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/Dockerfile b/resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/Dockerfile new file mode 100644 index 0000000000..2b2e1eae55 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/Dockerfile @@ -0,0 +1,61 @@ +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:18.04 +LABEL Description="CSIT vpp-device ubuntu 18.04 shim image" +LABEL Version="master" + +# Setup the environment +ENV DEBIAN_FRONTEND=noninteractive +ENV NOTVISIBLE "in users profile" +RUN echo "export VISIBLE=now" >> /etc/profile + +ADD files/wrapdocker /usr/local/bin/wrapdocker +RUN chmod +x /usr/local/bin/wrapdocker + +# Install packages and Docker +RUN apt-get -q update \ + && apt-get install -y -qq \ + bash \ + curl \ + iproute2 \ + locales \ + ssh \ + sudo \ + tzdata \ + uuid-runtime \ + && curl -fsSL https://get.docker.com | sh \ + && rm -rf /var/lib/apt/lists/* + +# Configure locales +RUN locale-gen en_US + +RUN mkdir /var/run/sshd +RUN echo 'root:Csit1234' | chpasswd +RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config + +# SSH login fix. Otherwise user is kicked off after login +RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +# Need volume for sidecar docker launches +VOLUME /var/lib/docker + +# SSH to listen on port 6022 in shim +RUN echo 'Port 6022' >>/etc/ssh/sshd_config +RUN echo 'Port 6023' >>/etc/ssh/sshd_config +ADD files/badkeypub /root/.ssh/authorized_keys +ADD files/sshconfig /root/.ssh/config + +# Start sshd by default +EXPOSE 22 +CMD ["/usr/sbin/sshd", "-D"]
\ No newline at end of file diff --git a/resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/files/badkeypub b/resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/files/badkeypub new file mode 100644 index 0000000000..4530b66b05 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/files/badkeypub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCyUNd/iRk5Ajw4ZBB0gXyjzecEzQHh/MctgvHGJjasqJDkwYyZBrunUorOZ3n82W8VGdd5+eNINCWOM/ERjuaHjnutfade+ocPgZRdk+kEgTvetDVNWIgBd0PMVcnp57jJfx7CZVqTNgGeVQ8OJ2RbJGeOb/EKApQI74IPkAfc0PSieSw5gC0eqEOHb39Awgp0ycrzsUHF/OEicfCmo+6vvrMGenDe7frKUoTKYMWs7l3DOyFC8NaOxhGD3J1Ne5u3A/r4w6mN1HVI0rFwIcoms+t0B4lb2ODWKZiZikQdn8/eqwsmbSEZZsWN3FkshgjPS83+dNqVwB6pPY5Yqte7 ejk@bhima.local
\ No newline at end of file diff --git a/resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/files/sshconfig b/resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/files/sshconfig new file mode 100644 index 0000000000..e7bd90757e --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/files/sshconfig @@ -0,0 +1,3 @@ +Host 172.17.0.* + StrictHostKeyChecking no + UserKnownHostsFile=/dev/null
\ No newline at end of file diff --git a/resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/files/wrapdocker b/resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/files/wrapdocker new file mode 100644 index 0000000000..d13f8b7c5e --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/csit_shim_image/files/files/wrapdocker @@ -0,0 +1,113 @@ +#!/bin/bash + +# Ensure that all nodes in /dev/mapper correspond to mapped devices currently loaded by the device-mapper kernel driver +dmsetup mknodes + +# First, make sure that cgroups are mounted correctly. +CGROUP=/sys/fs/cgroup +: {LOG:=stdio} + +[ -d $CGROUP ] || + mkdir $CGROUP + +mountpoint -q $CGROUP || + mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || { + echo "Could not make a tmpfs mount. Did you use --privileged?" + exit 1 + } + +if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security +then + mount -t securityfs none /sys/kernel/security || { + echo "Could not mount /sys/kernel/security." + echo "AppArmor detection and --privileged mode might break." + } +fi + +# Mount the cgroup hierarchies exactly as they are in the parent system. +for SUBSYS in $(cut -d: -f2 /proc/1/cgroup) +do + [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS + mountpoint -q $CGROUP/$SUBSYS || + mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS + + # The two following sections address a bug which manifests itself + # by a cryptic "lxc-start: no ns_cgroup option specified" when + # trying to start containers withina container. + # The bug seems to appear when the cgroup hierarchies are not + # mounted on the exact same directories in the host, and in the + # container. + + # Named, control-less cgroups are mounted with "-o name=foo" + # (and appear as such under /proc/<pid>/cgroup) but are usually + # mounted on a directory named "foo" (without the "name=" prefix). + # Systemd and OpenRC (and possibly others) both create such a + # cgroup. To avoid the aforementioned bug, we symlink "foo" to + # "name=foo". This shouldn't have any adverse effect. + echo $SUBSYS | grep -q ^name= && { + NAME=$(echo $SUBSYS | sed s/^name=//) + ln -s $SUBSYS $CGROUP/$NAME + } + + # Likewise, on at least one system, it has been reported that + # systemd would mount the CPU and CPU accounting controllers + # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu" + # but on a directory called "cpu,cpuacct" (note the inversion + # in the order of the groups). This tries to work around it. + [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct +done + +# Note: as I write those lines, the LXC userland tools cannot setup +# a "sub-container" properly if the "devices" cgroup is not in its +# own hierarchy. Let's detect this and issue a warning. +grep -q :devices: /proc/1/cgroup || + echo "WARNING: the 'devices' cgroup should be in its own hierarchy." +grep -qw devices /proc/1/cgroup || + echo "WARNING: it looks like the 'devices' cgroup is not mounted." + +# Now, close extraneous file descriptors. +pushd /proc/self/fd >/dev/null +for FD in * +do + case "$FD" in + # Keep stdin/stdout/stderr + [012]) + ;; + # Nuke everything else + *) + eval exec "$FD>&-" + ;; + esac +done +popd >/dev/null + + +# If a pidfile is still around (for example after a container restart), +# delete it so that docker can start. +rm -rf /var/run/docker.pid + +# If we were given a PORT environment variable, start as a simple daemon; +# otherwise, spawn a shell as well +if [ "$PORT" ] +then + exec dockerd -H 0.0.0.0:$PORT -H unix:///var/run/docker.sock \ + $DOCKER_DAEMON_ARGS +else + if [ "$LOG" == "file" ] + then + dockerd $DOCKER_DAEMON_ARGS &>/var/log/docker.log & + else + dockerd $DOCKER_DAEMON_ARGS & + fi + (( timeout = 60 + SECONDS )) + until docker info >/dev/null 2>&1 + do + if (( SECONDS >= timeout )); then + echo 'Timed out trying to connect to internal docker host.' >&2 + break + fi + sleep 1 + done + [[ $1 ]] && exec "$@" + exec bash --login +fi
\ No newline at end of file diff --git a/resources/tools/testbed-setup/ansible/roles/csit_shim_image/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/csit_shim_image/tasks/main.yaml new file mode 100644 index 0000000000..bdba4f6563 --- /dev/null +++ b/resources/tools/testbed-setup/ansible/roles/csit_shim_image/tasks/main.yaml @@ -0,0 +1,32 @@ +--- +# file: roles/csit_shim_image/tasks/main.yaml + +- name: Create a directory if it does not exist + file: + path: "{{ item }}" + state: "directory" + mode: 0755 + with_items: + - "/opt/csit-shim/" + - "/opt/csit-shim/files" + tags: csit-shim-image + +- name: Copy Build Items + copy: + src: "{{ item }}" + dest: "/opt/csit-shim/{{ item }}" + owner: "root" + group: "root" + mode: 0655 + with_items: + - "Dockerfile" + - "files/badkeypub" + - "files/sshconfig" + - "files/wrapdocker" + tags: csit-shim-image + +- name: Build CSIT shim Docker Image + shell: "docker build -t csit_shim-ubuntu1804:local ." + args: + chdir: "/opt/csit-shim" + tags: csit-shim-image
\ No newline at end of file diff --git a/resources/tools/testbed-setup/ansible/vpp_device.yaml b/resources/tools/testbed-setup/ansible/vpp_device.yaml index 363d1b190a..ac42b8cafe 100644 --- a/resources/tools/testbed-setup/ansible/vpp_device.yaml +++ b/resources/tools/testbed-setup/ansible/vpp_device.yaml @@ -24,5 +24,7 @@ tags: kernel_vm - role: csit_sut_image tags: csit_sut_image + - role: csit_shim_image + tags: csit_shim_image - role: cleanup tags: cleanup |