Infra: Ansible docker images cleanup

Signed-off-by: pmikus <pmikus@cisco.com>
Change-Id: I8d67b8ad5db5c0a7c9b3fa892e1e66dab2f666d0

10 files changed, 188 insertions, 74 deletions

diff --git a/resources/tools/testbed-setup/ansible/nomad.yaml b/resources/tools/testbed-setup/ansible/nomad.yaml
index 88d5165e2f..ae9de1e524 100644
--- a/resources/tools/testbed-setup/ansible/nomad.yaml
+++ b/resources/tools/testbed-setup/ansible/nomad.yaml
@@ -27,4 +27,6 @@
     - role: jenkins_job_health_exporter
       tags: jenkins_job_health_exporter
     - role: cadvisor
-      tags: cadvisor
\ No newline at end of file
+      tags: cadvisor
+    - role: cleanup
+      tags: cleanup
\ No newline at end of file
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/clean_images.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/clean_images.yaml
new file mode 100644
index 0000000000..e030acbff2
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/clean_images.yaml
@@ -0,0 +1,36 @@
+---
+# file: roles/cleanup/tasks/clean_images.yaml
+
+- name: Clean Docker Images
+  block:
+  - name: Clean Images - Prefetch Docker Images
+    cron:
+      name: "Prefetch docker image {{ item }}"
+      minute: "10"
+      hour: "7"
+      job: "/usr/bin/docker pull {{ item }}"
+    loop:
+      "{{ images_to_prefetch_by_arch[ansible_machine] }}"
+    tags:
+      - prefetch-docker-images
+
+  - name: Clean Images - Remove Dangling Docker Images
+    cron:
+      name: "Remove dangling docker images"
+      minute: "10"
+      hour: "5"
+      weekday: "7"
+      job: "/usr/bin/docker rmi $(/usr/bin/docker images --filter 'dangling=true' -q)"
+    tags:
+      - remove-docker-images-dangling
+
+  # TODO: Disabled until all images will be in registry
+  #- name: Clean Images - Prune Docker Images
+  #  cron:
+  #    name: "Prune docker images"
+  #    minute: "10"
+  #    hour: "6"
+  #    weekday: 7
+  #    job: "/usr/bin/docker image prune --all --force"
+  #  tags:
+  #    - prune-docker-images
\ No newline at end of file
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml
index ad4fb37681..25fd48e420 100644
--- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_containers.yaml
@@ -3,16 +3,18 @@
 
 - name: Kill Docker Containers
   block:
-  - name: Kill container - Get Running Docker Containers
+  - name: Kill Container - Get Running Docker Containers
     shell: "docker ps -aq"
     register: running_containers
     changed_when: no
-    tags: kill-containers
+    tags:
+      - kill-containers
 
-  - name: Kill container - Remove All Docker Containers
+  - name: Kill Container - Remove All Docker Containers
     shell: "docker rm --force {{ item }}"
     with_items: "{{ running_containers.stdout_lines }}"
-    tags: kill-containers
+    tags:
+      - kill-containers
 
   rescue:
   - name: Restart Docker Daemon
@@ -22,16 +24,18 @@
 
 - name: Kill LXC Containers
   block:
-  - name: Kill container - Get Running LXC Containers
+  - name: Kill Container - Get Running LXC Containers
     shell: "lxc-ls"
     register: running_containers
     changed_when: no
-    tags: kill-containers
+    tags:
+      - kill-containers
 
-  - name: Kill container - Remove All LXC Containers
+  - name: Kill Container - Remove All LXC Containers
     shell: "lxc-destroy --force -n {{ item }}"
     with_items: "{{ running_containers.stdout_lines }}"
-    tags: kill-containers
+    tags:
+      - kill-containers
 
   rescue:
   - fail:
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml
index 30a9f459b5..c7cee37485 100644
--- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/kill_process.yaml
@@ -1,32 +1,36 @@
 ---
 # file: roles/cleanup/tasks/kill_process.yaml
 
-- name: Kill process - {{ process }}
+- name: Kill Process - {{ process }}
   block:
-  - name: Kill process - Get pid of {{ process }}
+  - name: Get PID Of {{ process }}
     shell: "ps -ef | grep -v grep | grep -w {{ process }} | awk '{print $2}'"
-    when: >
-      process is defined and process != ""
+    when:
+      - process is defined and process != ""
     register: running_processes
-    tags: kill-process
+    tags:
+      - kill-process
 
-  - name: Kill process - Safe kill {{ process }}
+  - name: Safe Kill {{ process }}
     shell: "kill {{ item }}"
     with_items: "{{ running_processes.stdout_lines }}"
-    tags: kill-process
+    tags:
+      - kill-process
 
   - wait_for:
       path: "/proc/{{ item }}/status"
-      state: absent
+      state: "absent"
     with_items: "{{ running_processes.stdout_lines }}"
     ignore_errors: yes
     register: killed_processes
-    tags: kill-process
+    tags:
+      - kill-process
 
-  - name: Kill process - Force kill {{ process }}
+  - name: Kill Process - Force Kill {{ process }}
     shell: "kill -9 {{ item }}"
     with_items: "{{ killed_processes.results | select('failed') | map(attribute='item') | list }}"
-    tags: kill-process
+    tags:
+      - kill-process
 
   rescue:
   - fail:
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/main.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/main.yaml
index 64a55c4672..eeda0139b3 100644
--- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/main.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/main.yaml
@@ -14,18 +14,30 @@
 #     - vpp_device
 #         - Run tasks on vpp_device servers only.
 #         - Reset SRIOV
+#         - Docker image cleanup
+#     - nomad
+#         - Docker image cleanup
 
 - name: tg specific
   include_tasks: tg.yaml
   when: "'tg' in group_names"
-  tags: cleanup
+  tags:
+    - cleanup
 
 - name: sut specific
   include_tasks: sut.yaml
   when: "'sut' in group_names"
-  tags: cleanup
+  tags:
+    - cleanup
 
 - name: vpp_device specific
   include_tasks: vpp_device.yaml
   when: "'vpp_device' in group_names"
-  tags: cleanup
+  tags:
+    - cleanup
+
+- name: nomad specific
+  include_tasks: nomad.yaml
+  when: "'nomad' in group_names"
+  tags:
+    - cleanup
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/nomad.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/nomad.yaml
new file mode 100644
index 0000000000..3c5bf6462d
--- /dev/null
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/nomad.yaml
@@ -0,0 +1,22 @@
+---
+# file: roles/cleanup/tasks/nomad.yaml
+
+- name: Host Cleanup
+  block:
+  - name: Clean Images
+    import_tasks: clean_images.yaml
+    vars:
+      images_to_prefetch_by_arch:
+        aarch64:
+          - "fdiotools/builder-ubuntu2004:prod-aarch64"
+          - "fdiotools/builder-ubuntu1804:prod-aarch64"
+          - "fdiotools/builder-centos8:prod-aarch64"
+        x86_64:
+          - "fdiotools/builder-ubuntu2004:prod-x86_64"
+          - "fdiotools/builder-ubuntu1804:prod-x86_64"
+          - "fdiotools/builder-debian10:prod-x86_64"
+          - "fdiotools/builder-debian9:prod-x86_64"
+          - "fdiotools/builder-centos8:prod-x86_64"
+          - "fdiotools/builder-centos7:prod-x86_64"
+    tags:
+      - clean-images
\ No newline at end of file
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/remove_package.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/remove_package.yaml
index 0c8816fe29..302b43c99a 100644
--- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/remove_package.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/remove_package.yaml
@@ -1,19 +1,21 @@
 ---
 # file: roles/cleanup/tasks/remove_package.yaml
 
-- name: Remove package - Fix corrupted apt
-  shell: 'dpkg --configure -a'
-  when: >
-    ansible_distribution == 'Ubuntu'
-  tags: remove-package
+- name: Remove Package - Fix Corrupted APT
+  shell: "dpkg --configure -a"
+  when:
+    - ansible_distribution == 'Ubuntu'
+  tags:
+    - remove-package
 
-- name: Remove package - {{ package }}
+- name: Remove Package - {{ package }}
   apt:
-    name: '{{ package }}'
+    name: "{{ package }}"
     force: yes
     purge: yes
-    state: absent
+    state: "absent"
   failed_when: no
-  when: >
-    ansible_distribution == 'Ubuntu'
-  tags: remove-package
+  when:
+    - ansible_distribution == 'Ubuntu'
+  tags:
+    - remove-package
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml
index 53a65dd608..d80a35b1cb 100644
--- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/sut.yaml
@@ -1,72 +1,83 @@
 ---
 # file: roles/cleanup/tasks/sut.yaml
 
-- name: Host cleanup
+- name: Host Cleanup
   block:
-    - name: Kill processes - qemu
+    - name: Kill Processes - Qemu
       import_tasks: kill_process.yaml
       vars:
         process: "qemu"
-      tags: kill-process
+      tags:
+        - kill-process
 
-    - name: Kill processes - l3fwd
+    - name: Kill Processes - L3fwd
       import_tasks: kill_process.yaml
       vars:
         process: "l3fwd"
-      tags: kill-process
+      tags:
+        - kill-process
 
-    - name: Kill processes - testpmd
+    - name: Kill Processes - Testpmd
       import_tasks: kill_process.yaml
       vars:
         process: "testpmd"
-      tags: kill-process
+      tags:
+        - kill-process
 
-    - name: Kill processes - iperf3
+    - name: Kill Processes - iPerf3
       import_tasks: kill_process.yaml
       vars:
         process: "iperf3"
-      tags: kill-process
+      tags:
+        - kill-process
 
-    - name: Kill processes - vpp_echo
+    - name: Kill Processes - vpp_echo
       import_tasks: kill_process.yaml
       vars:
         process: "vpp_echo"
-      tags: kill-process
+      tags:
+        - kill-process
 
-    - name: Find file or dir - Core zip file
+    - name: Find File Or Dir - Core Zip File
       find:
         paths: "/tmp/"
         patterns: "*tar.lzo.lrz.xz*"
       register: files_to_delete
-      tags: remove-file-dir
+      tags:
+        - remove-file-dir
 
-    - name: Remove file or dir - Core zip file
+    - name: Remove File Or Dir - Core Zip File
       file:
         path: "{{ item.path }}"
         state: absent
       with_items: "{{ files_to_delete.files }}"
-      tags: remove-file-dir
+      tags:
+        - remove-file-dir
 
-    - name: Find file or dir - Core dump file
+    - name: Find File Or Dir - Core Dump File
       find:
         paths: "/tmp/"
         patterns: "*core*"
       register: files_to_delete
-      tags: remove-file-dir
+      tags:
+        - remove-file-dir
 
-    - name: Remove file or dir - Core dump file
+    - name: Remove File Or Dir - Core Dump File
       file:
         path: "{{ item.path }}"
         state: absent
       with_items: "{{ files_to_delete.files }}"
-      tags: remove-file-dir
+      tags:
+        - remove-file-dir
 
-    - name: Kill containers - Remove all containers
+    - name: Kill Containers - Remove All Containers
       import_tasks: kill_containers.yaml
-      tags: kill-containers
+      tags:
+        - kill-containers
 
-    - name: Remove packages - Remove VPP
+    - name: Remove Packages - Remove VPP
       import_tasks: remove_package.yaml
       vars:
         package: "*vpp*"
-      tags: remove-package
+      tags:
+        - remove-package
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/tg.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/tg.yaml
index 9ac83bc9fc..fa2d2d2819 100644
--- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/tg.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/tg.yaml
@@ -1,9 +1,13 @@
 ---
 # file: roles/cleanup/tasks/tg.yaml
 
-- name: Kill processes - TRex
-  import_tasks: kill_process.yaml
-  vars:
-      process: "_t-rex"
-  when: docker_tg is undefined
-  tags: kill-process
+- name: Host Cleanup
+  block:
+    - name: Kill Processes - TRex
+      import_tasks: kill_process.yaml
+      vars:
+        process: "_t-rex"
+      when:
+        - docker_tg is undefined
+      tags:
+        - kill-process
diff --git a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/vpp_device.yaml b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/vpp_device.yaml
index f5e6ea5488..41c4b29d37 100644
--- a/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/vpp_device.yaml
+++ b/resources/tools/testbed-setup/ansible/roles/cleanup/tasks/vpp_device.yaml
@@ -1,15 +1,32 @@
 ---
 # file: roles/cleanup/tasks/vpp_device.yaml
 
-- name: Reset vpp_device binary
-  copy:
-   src: 'files/reset_vppdevice.sh'
-   dest: '/usr/local/bin'
-   owner: 'root'
-   group: 'root'
-   mode: '744'
-  tags: reset-sriov
+- name: Host Cleanup
+  block:
+  - name: Reset vpp_device Binary
+    copy:
+     src: "files/reset_vppdevice.sh"
+     dest: "/usr/local/bin"
+     owner: "root"
+     group: "root"
+     mode: "744"
+    tags:
+      - reset-sriov
 
-- name: Reset vpp_device
-  raw: 'reset_vppdevice.sh --force'
-  tags: reset-sriov
+  - name: Clean Images
+    import_tasks: clean_images.yaml
+    vars:
+      images_to_prefetch_by_arch:
+        aarch64:
+          - "fdiotools/builder-ubuntu2004:prod-aarch64"
+          - "fdiotools/builder-ubuntu1804:prod-aarch64"
+          - "fdiotools/builder-centos8:prod-aarch64"
+        x86_64:
+          - "fdiotools/builder-ubuntu2004:prod-x86_64"
+          - "fdiotools/builder-ubuntu1804:prod-x86_64"
+          - "fdiotools/builder-debian10:prod-x86_64"
+          - "fdiotools/builder-debian9:prod-x86_64"
+          - "fdiotools/builder-centos8:prod-x86_64"
+          - "fdiotools/builder-centos7:prod-x86_64"
+    tags:
+      - clean-images
\ No newline at end of file