From 90bb3970ecfb54a3fe273695fe6431e716ac1f3e Mon Sep 17 00:00:00 2001 From: Mauro Sardara Date: Tue, 2 Jun 2020 19:16:29 +0200 Subject: Test commands for release repo cleanup. Signed-off-by: Mauro Sardara Change-Id: I71bee2717bbcc3021fd85020f2059839ea5a930c Signed-off-by: Mauro Sardara --- jjb/ci-management/package-list.sh | 50 +++++++ jjb/ci-management/packagecloud-repo-cleanup.sh | 161 +++++++++++++++++++++++ jjb/ci-management/packagecloud-repo-cleanup.yaml | 140 ++++++++++++++++++++ 3 files changed, 351 insertions(+) create mode 100644 jjb/ci-management/package-list.sh create mode 100644 jjb/ci-management/packagecloud-repo-cleanup.sh create mode 100644 jjb/ci-management/packagecloud-repo-cleanup.yaml diff --git a/jjb/ci-management/package-list.sh b/jjb/ci-management/package-list.sh new file mode 100644 index 00000000..6172b7b5 --- /dev/null +++ b/jjb/ci-management/package-list.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +export PACKAGE_LIST_COMMON="libhicnctrl \ +libhicnctrl-memif \ +hicn-collectd-plugins \ +hicn-apps \ +hicn-light \ +hicn-apps-memif \ +libhicntransport-memif \ +libhicn \ +libhicntransport \ +hicn-plugin \ +facemgr \ +hicn-utils-memif \ +hicn-utils \ +hicn-sysrepo-plugin \ +hicn-extra-plugin \ +libparc \ +libparc-doc \ +longbow \ +longbow-doc" + +export PACKAGE_LIST_UBUNTU="libhicnctrl-dev \ +libhicn-ctrl-dev \ +libhicnctrl-memif-dev \ +libhicntransport-memif-dev \ +libhicn-dev \ +libhicntransport-dev \ +hicn-plugin-dev \ +libdash \ +libdash-dev \ +libdash-doc \ +libparc-dev \ +libmemif-dev \ +longbow-dev" + +export PACKAGE_LIST_CENTOS="libhicnctrl-devel \ +libhicnctrl-memif-devel \ +libhicntransport-memif-devel \ +libhicn-devel \ +libhicntransport-devel \ +hicn-plugin-devel \ +libparc-devel \ +libmemif-devel \ +longbow-devel" + +export VERSION_WHITELIST="19.01-227 \ +19.08-289 \ +20.01-73 \ +20.01-114" diff --git a/jjb/ci-management/packagecloud-repo-cleanup.sh b/jjb/ci-management/packagecloud-repo-cleanup.sh new file mode 100644 index 00000000..8fdfe118 --- /dev/null +++ b/jjb/ci-management/packagecloud-repo-cleanup.sh @@ -0,0 +1,161 @@ +#!/bin/bash +set -euo pipefail + +# Number of packages to keep. +N_PACKAGES=5 + +PACKAGECLOUD_REPO_DEB="https://packagecloud.io/install/repositories/fdio/${STREAM}/script.deb.sh" +PACKAGECLOUD_REPO_RPM="https://packagecloud.io/install/repositories/fdio/${STREAM}/script.rpm.sh" + +FACTER_OS=$(/usr/bin/facter operatingsystem) +PACKAGE_LIST="" +VERSION_REGEX="[0-9]+.[0-9]+[-_][0-9]+[-_]release(-1)?|[0-9]+.[0-9]+[-_][0-9]+~g[[:graph:]]+" + +declare -A FUNCTIONS + +echo_err () { + >&2 echo ${@} + exit 1 +} + +contains() { + [[ ${1} =~ (^|[[:space:]])${2}($|[[:space:]]) ]] && return 1 || return 0 +} + +check_version_whitelist() { + if [[ ${1} =~ ([0-9]+).([0-9]+)[-_]([0-9]+).+ ]]; then + MAJOR=${BASH_REMATCH[1]} + MINOR=${BASH_REMATCH[2]} + REVISION=${BASH_REMATCH[3]} + VER="${MAJOR}.${MINOR}-${REVISION}" + + if contains "${VERSION_WHITELIST}" ${VER}; then + return 1; + fi + fi + + return 0 +} + +# Params +# $1: Package list +build_package_blacklist_ubuntu () { + PACKAGE_LIST=${@} + OUTPUT_LIST="" + ARCH=$(dpkg --print-architecture) + + for package in ${PACKAGE_LIST}; do + OUTPUT=$(apt-cache policy ${package} 2> /dev/null) + + if [[ ${?} -ne 0 || -z "${OUTPUT}" ]]; then + continue + fi + + # N_PACKAGES + 2 is justified by the fact thathe output of apt-cache policy is in the form: + # apt-cache policy libhicn + # libhicn: + # Installed: (none) + # Candidate: 20.05-11-release + # Version table: + # 20.05-11-release 500 + # + # The first version candidate is printed twice. + # So we need to exclude the "Candidate: 20.05-11-release" (+1). + # Also `tail -n +M` starts to print from the Mth line, being 1 the first line. + # So to exclude the Mth line an additional +1 is needed. + VERSIONS="$(echo ${OUTPUT} | grep -E -o ${VERSION_REGEX} | tail -n +$((N_PACKAGES + 2)))" + + for version in ${VERSIONS}; do + if ! check_version_whitelist ${version}; then + OUTPUT_LIST+="${package}_${version}_${ARCH}.deb " + fi + done + done + + echo ${OUTPUT_LIST} +} + +# Params +# $1: Package list +build_package_blacklist_centos () { + PACKAGE_LIST=${@} + OUTPUT_LIST="" + VERSIONS="" + ARCH=$(uname -m) + + for package in ${PACKAGE_LIST}; do + OUTPUT=$(yum --showduplicates list ${package} 2> /dev/null) + if [[ ${?} -ne 0 || -z "${OUTPUT}" ]]; then + continue + fi + + VERSIONS="$(echo ${OUTPUT} | grep -Eo "${VERSION_REGEX}" | head -n -${N_PACKAGES})" + + for version in ${VERSIONS}; do + if ! check_version_whitelist ${version}; then + OUTPUT_LIST+="${package}-${version}.${ARCH}.rpm " + fi + done + done + + echo ${OUTPUT_LIST} +} + +promote_attic_repo_centos () { + FACTER_OSMAJREL=$(/usr/bin/facter operatingsystemmajrelease) + FACTER_ARCH=$(/usr/bin/facter architecture) + + for package in ${@}; do + echo package_cloud promote \ + ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ \ + ${package} ${PCIO_CO}/attic/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ + package_cloud promote \ + ${PCIO_CO}/${STREAM}/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ \ + ${package} ${PCIO_CO}/attic/el/${FACTER_OSMAJREL}/os/${FACTER_ARCH}/ + done +} + +promote_attic_repo_ubuntu () { + FACTER_LSBNAME=$(/usr/bin/facter lsbdistcodename) + + for package in ${@}; do + echo package_cloud promote ${PCIO_CO}/${STREAM}/ubuntu/${FACTER_LSBNAME}/main/ \ + ${package} ${PCIO_CO}/attic/ubuntu/${FACTER_LSBNAME}/main/ + package_cloud promote ${PCIO_CO}/${STREAM}/ubuntu/${FACTER_LSBNAME}/main/ \ + ${package} ${PCIO_CO}/attic/ubuntu/${FACTER_LSBNAME}/main/ + done +} + +promote_to_attic_repo () { + ${FUNCTIONS["promote_attic_repo"]} ${@} +} + +setup_fdio_repo () { + case "${FACTER_OS}" in + Ubuntu) + curl -s ${PACKAGECLOUD_REPO_DEB} | sudo bash + FUNCTIONS["package_blacklist"]="build_package_blacklist_ubuntu" + FUNCTIONS["promote_attic_repo"]="promote_attic_repo_ubuntu" + PACKAGE_LIST="${PACKAGE_LIST_COMMON} ${PACKAGE_LIST_UBUNTU}" + ;; + CentOS) + curl -s ${PACKAGECLOUD_REPO_RPM} | sudo bash + FUNCTIONS["package_blacklist"]="build_package_blacklist_centos" + FUNCTIONS["promote_attic_repo"]="promote_attic_repo_centos" + PACKAGE_LIST="${PACKAGE_LIST_COMMON} ${PACKAGE_LIST_CENTOS}" + ;; + *) + echo_err "Distribution ${FACTER_OS} is not supported." + ;; + esac +} + +# Params +# $1: Package list +build_package_blacklist () { + ${FUNCTIONS["package_blacklist"]} ${@} +} + +setup_fdio_repo +PACKAGES_TO_PROMOTE=$(build_package_blacklist ${PACKAGE_LIST}) +promote_to_attic_repo ${PACKAGES_TO_PROMOTE} \ No newline at end of file diff --git a/jjb/ci-management/packagecloud-repo-cleanup.yaml b/jjb/ci-management/packagecloud-repo-cleanup.yaml new file mode 100644 index 00000000..88966c52 --- /dev/null +++ b/jjb/ci-management/packagecloud-repo-cleanup.yaml @@ -0,0 +1,140 @@ +- project: + name: packagecloud-repo-cleanup + jobs: + - 'packagecloud-repo-cleanup-release-{os}' + - 'packagecloud-repo-cleanup-hicn-{os}' + project: 'ci-management' + os: + - ubuntu1804: + repo-os-part: 'ubuntu.bionic.main' + - ubuntu1804arm: + repo-os-part: 'ubuntu-arm.bionic.main' + - ubuntu1604: + repo-os-part: 'ubuntu.xenial.main' + - centos7: + repo-os-part: 'centos7' + +- job-template: + name: 'packagecloud-repo-cleanup-release-{os}' + + project-type: freestyle + node: '{os}-us' + concurrent: true + + build-discarder: + daysToKeep: '{build-days-to-keep}' + numToKeep: 100 + artifactDaysToKeep: '{build-artifact-days-to-keep}' + artifactNumToKeep: '{build-artifact-num-to-keep}' + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - os-parameter: + os: '{os}' + - stream-parameter: + stream: 'release' + - string: + name: ARCHIVE_ARTIFACTS + default: '{archive-artifacts}' + description: Artifacts to archive to the logs server. + + wrappers: + - fdio-infra-wrappers: + build-timeout: 120 + + triggers: + - gerrit: + server-name: 'Primary' + trigger-on: + - comment-added-contains-event: + comment-contains-value: 'release-cleanup' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + + builders: + - config-file-provider: + files: + - file-id: '.packagecloud' + target: '/root/.packagecloud' + - config-file-provider: + files: + - file-id: 'packagecloud_api' + target: '/root/packagecloud_api' + - shell: + !include-raw-escape: + - package-list.sh + - packagecloud-repo-cleanup.sh + + publishers: + - fdio-infra-shiplogs: + maven-version: 'mvn36' + +- job-template: + name: 'packagecloud-repo-cleanup-hicn-{os}' + + project-type: freestyle + node: '{os}-us' + concurrent: true + + build-discarder: + daysToKeep: '{build-days-to-keep}' + numToKeep: 100 + artifactDaysToKeep: '{build-artifact-days-to-keep}' + artifactNumToKeep: '{build-artifact-num-to-keep}' + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - os-parameter: + os: '{os}' + - stream-parameter: + stream: 'hicn' + - string: + name: ARCHIVE_ARTIFACTS + default: '{archive-artifacts}' + description: Artifacts to archive to the logs server. + + wrappers: + - fdio-infra-wrappers: + build-timeout: 120 + + triggers: + - timed: '@weekly' + - gerrit: + server-name: 'Primary' + trigger-on: + - comment-added-contains-event: + comment-contains-value: 'release-cleanup' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + + builders: + - config-file-provider: + files: + - file-id: '.packagecloud' + target: '/root/.packagecloud' + - config-file-provider: + files: + - file-id: 'packagecloud_api' + target: '/root/packagecloud_api' + - shell: + !include-raw-escape: + - package-list.sh + - packagecloud-repo-cleanup.sh + + publishers: + - fdio-infra-shiplogs: + maven-version: 'mvn36' -- cgit 1.2.3-korg