From a63e80093c52d4893e5709860919d0f27cd8c122 Mon Sep 17 00:00:00 2001 From: Vanessa Rene Valderrama Date: Thu, 13 Oct 2016 13:54:18 -0500 Subject: Convert Vagrant to Packer This converts all of the in use Vagrant definitions to Packer and allows us to build images and allow automated testing of images. Change-Id: Ic87cce4aa674146952efad325a42b095086426d0 Signed-off-by: Vanessa Rene Valderrama --- packer/provision/basebuild.sh | 190 ++++++++++++++++++++++++++++++++++++++ packer/provision/baseline.sh | 168 +++++++++++++++++++++++++++++++++ packer/provision/null_data.sh | 4 + packer/provision/rh-user_data.sh | 4 + packer/provision/system_reseal.sh | 38 ++++++++ 5 files changed, 404 insertions(+) create mode 100644 packer/provision/basebuild.sh create mode 100644 packer/provision/baseline.sh create mode 100644 packer/provision/null_data.sh create mode 100644 packer/provision/rh-user_data.sh create mode 100644 packer/provision/system_reseal.sh (limited to 'packer/provision') diff --git a/packer/provision/basebuild.sh b/packer/provision/basebuild.sh new file mode 100644 index 000000000..f43f3857d --- /dev/null +++ b/packer/provision/basebuild.sh @@ -0,0 +1,190 @@ +#!/bin/bash + +# vim: ts=4 sw=4 sts=4 et tw=72 : + +rh_systems() { + + # RH Install build tools + echo "---> Installing RH build tools $(date +'%Y%m%dT%H%M%S')" + RH_TOOLS_PKGS="@development redhat-lsb glibc-static java-1.8.0-openjdk-devel yum-utils openssl-devel apr-devel indent" + yum install -y ${RH_TOOLS_PKGS} + + # Memory leakage checks + yum install -y valgrind + + # RH Install Python dependencies + ###REMOVED mysql-devel + echo "---> Installing RH Python dependencies $(date +'%Y%m%dT%H%M%S')" + RH_PYTHON_PKGS="python-devel python-virtualenv python-setuptools python-pip openssl-devel" + yum install -y ${RH_PYTHON_PKGS} + + # RH Install Documentation packages + ###Removed python-pyparsing + echo "---> Installing RH documentation packages $(date +'%Y%m%dT%H%M%S')" + RH_DOC_PKGS="doxygen graphviz python-jinja2 asciidoc dblatex source-highlight python-sphinx" + yum install -y install ${RH_DOC_PKGS} + + # RH Install GCC packages + echo "---> Installing RH GCC packages $(date +'%Y%m%dT%H%M%S')" + RH_GCC_PKGS="cpp gcc c++ cmake" + yum install -y ${RH_GCC_PKGS} + + # RH Install components to build Ganglia modules + echo "---> Installing RH components $(date +'%Y%m%dT%H%M%S')" + RH_GANGLIA_MODS="libconfuse-devel ganglia-devel mock" + yum install -y --enablerepo=epel ${RH_GANGLIA_MODS} + + # RH Install module for VPP project + echo uio_pci_generic >> /etc/modules + + # RH Install VPP packages to shorten build times + echo "---> Installing VPP dependencies $(date +'%Y%m%dT%H%M%S')" + RH_VPP_PKGS="curl autoconf automake bison ccache dkms git libtool libconfuse-dev git-review cscope" + yum install -y ${RH_VPP_PKGS} + + yum groupinstall "Development Tools" + # RH Install TLDK dependencies + RH_TLKD_PKGS="libpcap-devel libcap-devel" + yum install -y ${RH_TLKD_PKGS} + + # RH Install debuginfo packages + #echo "---> Installing debug packages $(date +'%Y%m%dT%H%M%S')" + #RH_DEBUG_PKGS="glibc openssl-libs zlib" + #debuginfo-install -y ${RH_DEBUG_PKGS} + + # # RH Packer builds happen from the centos flavor images + # PACKERDIR=$(mktemp -d) + # # disable double quote checking + # # shellcheck disable=SC2086 + # cd $PACKERDIR + # wget https://releases.hashicorp.com/packer/0.10.1/packer_0.10.1_linux_amd64.zip + # unzip packer_0.10.1_linux_amd64.zip -d /usr/local/bin/ + # # rename packer to avoid conflicts with cracklib + # mv /usr/local/bin/packer /usr/local/bin/packer.io + + # # cleanup from the installation + # # disable double quote checking + # # shellcheck disable=SC2086 + # rm -rf $PACKERDIR + # # cleanup from previous install process + # if [ -d /tmp/packer ] + # then + # rm -rf /tmp/packer + # fi +} + +ubuntu_systems() { + + # DEB cloud packages + echo "---> Installing cloud packages $(date +'%Y%m%dT%H%M%S')" + CLOUD_PKGS="cloud-initramfs-dyn-netconf cloud-initramfs-growroot cloud-initramfs-rescuevol" + apt install -y ${CLOUD_PKGS} + + # DEB Install Python dependencies + echo "---> Installing Python dependencies $(date +'%Y%m%dT%H%M%S')" + PYTHON_PKGS="python-dev python-virtualenv python-setuptools python-pip libssl-dev libmysqlclient-dev python2.7" + apt install -y ${PYTHON_PKGS} + + # Memory leakage checks + apt install -y valgrind + + # DEB Install Documentation packages + echo "---> Installing documentation packages $(date +'%Y%m%dT%H%M%S')" + DOC_PKGS="doxygen graphviz python-pyparsing python-jinja2 asciidoc dblatex source-highlight" + apt install -y ${DOC_PKGS} + + # DEB Install the correct version of toolchain packages + echo "---> Installing latest toolchain packages from PPA $(date +'%Y%m%dT%H%M%S')" + + # DEB Install PPA packages + echo "---> Adding '$1' PPA $(date +'%Y%m%dT%H%M%S')" + dpkg -l software-properties-common > /dev/null 2>&1 || software-properties-common + + listfile=$(perl -e "print(q{$1} =~ m{^ppa:(.+)/ppa})")-ppa-${CODENAME}.list + if [ ! -f /etc/apt/sources.list.d/${listfile} ] + then + do_retry sudo apt-add-repository -y $1 + fi + + #Retry to prevent timeout failure + echo "---> Updating package index $(date +'%Y%m%dT%H%M%S')" + do_retry sudo apt-get update + echo "<--- Updating package index $(date +'%Y%m%dT%H%M%S')" + echo "<--- Adding '$1' PPA $(date +'%Y%m%dT%H%M%S')" + + # DEB Install GCC packages + echo "---> Installing GCC-5 packages $(date +'%Y%m%dT%H%M%S')" + GCC_PKGS="cpp gcc g++ cmake" + sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + sudo apt-get update + apt install -y ${GCC_PKGS} + + # DEB Install VPP packages to shorten build times + echo "---> Installing VPP DEB_DEPENDS packages $(date +'%Y%m%dT%H%M%S')" + VPP_PKGS="curl build-essential autoconf automake bison libssl-dev ccache debhelper dkms git libtool libganglia1-dev libapr1-dev dh-systemd libconfuse-dev git-review exuberant-ctags cscope indent" + apt install -y ${VPP_PKGS} + + # DEB Install latest kernel and uio + echo "---> Installing kernel image and header packages $(date +'%Y%m%dT%H%M%S')" + DEB_PKGS="linux-image-extra-virtual linux-headers-virtual linux-headers-`uname -r`" + apt install -y ${DEB_PKGS} + + #Configuring thirdparty Nexus repo + echo "deb [trusted=yes] https://nexus.fd.io/content/repositories/thirdparty ./" > /etc/apt/sources.list.d/FD.io.thirdparty.list + apt-get update + + # DEB Install deb_dpdk packages to shorten build times + ###REMOVED sphinx-rtd-theme + echo "---> Installing deb_dpdk packages $(date +'%Y%m%dT%H%M%S')" + DEB_DPDK_PKGS="google-mock lsb-release dpkg-dev debian-xcontrol devscripts pristine-tar dh-python python-sphinx libpcap0.8-dev libstdc++5 python-scapy inkscape libxen-dev libxenstore3.0 python-sphinx-rtd-theme" + apt install -y ${DEB_DPDK_PKGS} + + sudo apt install -y libcap-dev libpcap-dev + + TEXLIVE_PKGS="texlive-fonts-recommended tex-common texlive-base texlive-binaries texlive-pictures texlive-latex-recommended preview-latex-style texlive-latex-extra" + apt install -y ${TEXLIVE_PKGS} + echo "<--- Installing deb_dpdk packages $(date +'%Y%m%dT%H%M%S')" + + # DEB Manipulation tools, edits debugger, and LSB + echo "---> Installing tools packages $(date +'%Y%m%dT%H%M%S')" + TOOL_PKGS="iproute2 ethtool vlan bridge-utils vim gdb lsb-release" + apt install -y ${TOOL_PKGS} + + # DEB Clean up packages for a smaller image + apt-get update + + # DEB Updating CA certificates + echo "---> Forcing CA certificate update $(date +'%Y%m%dT%H%M%S')" + sudo update-ca-certificates -f > /dev/null 2>&1 + echo "<--- Forcing CA certificate update $(date +'%Y%m%dT%H%M%S')" +} + +all_systems() { + + echo 'Configure keep alive to prevent timeout during testing' + local SSH_CFG=/etc/ssh/ssh_config + echo "TCPKeepAlive true" | sudo tee -a ${SSH_CFG} >/dev/null 2>&1 + echo "ServerAliveCountMax 30" | sudo tee -a ${SSH_CFG} >/dev/null 2>&1 + echo "ServerAliveInterval 10" | sudo tee -a ${SSH_CFG} >/dev/null 2>&1 + +} + +echo "---> Detecting OS" +ORIGIN=$(facter operatingsystem | tr '[:upper:]' '[:lower:]') + +case "${ORIGIN}" in + fedora|centos|redhat) + echo "---> RH type system detected" + rh_systems + ;; + ubuntu) + echo "---> Ubuntu system detected" + ubuntu_systems + ;; + *) + echo "---> Unknown operating system" + ;; +esac + +# execute steps for all systems +all_systems diff --git a/packer/provision/baseline.sh b/packer/provision/baseline.sh new file mode 100644 index 000000000..36783554d --- /dev/null +++ b/packer/provision/baseline.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +# vim: ts=4 sw=4 sts=4 et tw=72 : + +rh_systems() { + # Handle the occurance where SELINUX is actually disabled + SELINUX=$(grep -E '^SELINUX=(disabled|permissive|enforcing)$' /etc/selinux/config) + MODE=$(echo "$SELINUX" | cut -f 2 -d '=') + case "$MODE" in + permissive) + echo "************************************" + echo "** SYSTEM ENTERING ENFORCING MODE **" + echo "************************************" + # make sure that the filesystem is properly labelled. + # it could be not fully labeled correctly if it was just switched + # from disabled, the autorelabel misses some things + # skip relabelling on /dev as it will generally throw errors + restorecon -R -e /dev / + + # enable enforcing mode from the very start + setenforce enforcing + + # configure system for enforcing mode on next boot + sed -i 's/SELINUX=permissive/SELINUX=enforcing/' /etc/selinux/config + ;; + disabled) + sed -i 's/SELINUX=disabled/SELINUX=permissive/' /etc/selinux/config + touch /.autorelabel + + echo "*******************************************" + echo "** SYSTEM REQUIRES A RESTART FOR SELINUX **" + echo "*******************************************" + ;; + enforcing) + echo "*********************************" + echo "** SYSTEM IS IN ENFORCING MODE **" + echo "*********************************" + ;; + esac + + echo "---> Updating operating system" + yum clean all -q + yum install -y -q deltarpm + yum update -y -q + + # add in components we need or want on systems + echo "---> Installing base packages" + yum install -y -q @base https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + # separate group installs from package installs since a non-existing + # group with dnf based systems (F21+) will fail the install if such + # a group does not exist + yum install -y -q unzip xz puppet git perl-XML-XPath wget make + + # All of our systems require Java (because of Jenkins) + # Install all versions of the OpenJDK devel but force 1.7.0 to be the + # default + + echo "---> Configuring OpenJDK" + yum install -y -q 'java-*-openjdk-devel' + + FACTER_OS=$(/usr/bin/facter operatingsystem) + FACTER_OSVER=$(/usr/bin/facter operatingsystemrelease) + case "$FACTER_OS" in + Fedora) + if [ "$FACTER_OSVER" -ge "21" ] + then + echo "---> not modifying java alternatives as OpenJDK 1.7.0 does not exist" + else + alternatives --set java /usr/lib/jvm/jre-1.7.0-openjdk.x86_64/bin/java + alternatives --set java_sdk_openjdk /usr/lib/jvm/java-1.7.0-openjdk.x86_64 + fi + ;; + *) + alternatives --set java /usr/lib/jvm/jre-1.7.0-openjdk.x86_64/bin/java + alternatives --set java_sdk_openjdk /usr/lib/jvm/java-1.7.0-openjdk.x86_64 + ;; + esac +} + +ubuntu_systems() { + # Ignore SELinux since slamming that onto Ubuntu leads to + # frustration + + export DEBIAN_FRONTEND=noninteractive + cat <> /etc/apt/apt.conf +APT { + Get { + Assume-Yes "true"; + allow-change-held-packages "true"; + allow-downgrades "true"; + allow-remove-essential "true"; + }; +}; + +Dpkg::Options { + "--force-confdef"; + "--force-confold"; +}; + +EOF + + echo "---> Updating operating system" + apt-get update -qq > /dev/null + apt-get upgrade -qq > /dev/null + + # add in stuff we know we need + echo "---> Installing base packages" + apt-get install -qq unzip xz-utils puppet git libxml-xpath-perl make wget > /dev/null + + # install Java 7 + echo "---> Configuring OpenJDK" + apt-get install -qq openjdk-7-jdk > /dev/null + + # make jdk8 available + add-apt-repository -y ppa:openjdk-r/ppa > /dev/null + apt-get update -qq > /dev/null + # We need to force openjdk-8-jdk to install + apt-get install -qq openjdk-8-jdk > /dev/null + + # make sure that we still default to openjdk 7 + update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java + update-alternatives --set javac /usr/lib/jvm/java-7-openjdk-amd64/bin/javac +} + +all_systems() { + # Allow jenkins access to update-alternatives command to switch java version + cat </etc/sudoers.d/89-jenkins-user-defaults +Defaults:jenkins !requiretty +jenkins ALL = NOPASSWD: /usr/bin/update-alternatives +EOF + + # Do any Distro specific installations here + echo "Checking distribution" + FACTER_OS=$(/usr/bin/facter operatingsystem) + case "$FACTER_OS" in + *) + echo "---> $FACTER_OS found" + echo "No extra steps for $FACTER_OS" + ;; + esac +} + +echo "---> Attempting to detect OS" +# upstream cloud images use the distro name as the initial user +ORIGIN=$(if [ -e /etc/redhat-release ] + then + echo redhat + else + echo ubuntu + fi) +#ORIGIN=$(logname) + +case "${ORIGIN}" in + fedora|centos|redhat) + echo "---> RH type system detected" + rh_systems + ;; + ubuntu) + echo "---> Ubuntu system detected" + ubuntu_systems + ;; + *) + echo "---> Unknown operating system" + ;; +esac + +# execute steps for all systems +all_systems diff --git a/packer/provision/null_data.sh b/packer/provision/null_data.sh new file mode 100644 index 000000000..3fa6a3c9f --- /dev/null +++ b/packer/provision/null_data.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# vi: ts=4 sw=4 sts=4 et : + +# Nothing to do for Ubuntu specific provisioning diff --git a/packer/provision/rh-user_data.sh b/packer/provision/rh-user_data.sh new file mode 100644 index 000000000..6bddb2449 --- /dev/null +++ b/packer/provision/rh-user_data.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# vi: ts=4 sw=4 sts=4 et : + +/bin/sed -i 's/ requiretty/ !requiretty/' /etc/sudoers; diff --git a/packer/provision/system_reseal.sh b/packer/provision/system_reseal.sh new file mode 100644 index 000000000..f8bc7dc5f --- /dev/null +++ b/packer/provision/system_reseal.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# vim: sw=2 ts=2 sts=2 et : + +rm -rf /etc/Pegasus/*.cnf /etc/Pegasus/*.crt /etc/Pegasus/*.csr \ + /etc/Pegasus/*.pem /etc/Pegasus/*.srl /root/anaconda-ks.cfg \ + /root/anaconda-post.log /root/initial-setup-ks.cfg /root/install.log \ + /root/install.log.syslog /var/cache/fontconfig/* /var/cache/gdm/* \ + /var/cache/man/* /var/lib/AccountService/users/* /var/lib/fprint/* \ + /var/lib/logrotate.status /var/log/*.log* /var/log/BackupPC/LOG \ + /var/log/ConsoleKit/* /var/log/anaconda.syslog /var/log/anaconda/* \ + /var/log/apache2/*_log /var/log/apache2/*_log-* /var/log/apt/* \ + /var/log/aptitude* /var/log/audit/* /var/log/btmp* /var/log/ceph/*.log \ + /var/log/chrony/*.log /var/log/cron* /var/log/cups/*_log /var/log/debug* \ + /var/log/dmesg* /var/log/exim4/* /var/log/faillog* /var/log/gdm/* \ + /var/log/glusterfs/*glusterd.vol.log /var/log/glusterfs/glusterfs.log \ + /var/log/httpd/*log /var/log/installer/* /var/log/jetty/jetty-console.log \ + /var/log/journal/* /var/log/lastlog* /var/log/libvirt/libvirtd.log \ + /var/log/libvirt/lxc/*.log /var/log/libvirt/qemu/*.log \ + /var/log/libvirt/uml/*.log /var/log/lightdm/* /var/log/mail/* \ + /var/log/maillog* /var/log/messages* /var/log/ntp /var/log/ntpstats/* \ + /var/log/ppp/connect-errors /var/log/rhsm/* /var/log/sa/* /var/log/secure* \ + /var/log/setroubleshoot/*.log /var/log/spooler* /var/log/squid/*.log \ + /var/log/syslog* /var/log/tallylog* /var/log/tuned/tuned.log /var/log/wtmp* \ + /var/named/data/named.run + +rm -rf ~/.viminfo /etc/ssh/ssh*key* /root/.ssh/* + +# kill any cloud-init related bits +rm -rf /var/lib/cloud/* + +# clean-up any manual packer uploads +rm -rf /tmp/packer + +# Force a system sync and sleep to get around any SSD issues +echo "Forcing sync and sleep for 10sec" +sync +sleep 10 -- cgit 1.2.3-korg