summaryrefslogtreecommitdiffstats
path: root/docker/scripts/lib_apt.sh
blob: 725aedcca564a2592c89210f5a685097955d7920 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
# lib_apt.sh - Docker build script apt library.
#              For import only.

# Copyright (c) 2021 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Don't import more than once.
if [ -n "$(alias lib_apt_imported 2> /dev/null)" ] ; then
    return 0
fi
alias lib_apt_imported=true

export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"}
. "$CIMAN_DOCKER_SCRIPTS/lib_common.sh"
. "$CIMAN_DOCKER_SCRIPTS/lib_csit.sh"

dump_apt_package_list() {
    branchname="$(echo $branch | sed -e 's,/,_,')"
    dpkg -l > \
         "$DOCKER_BUILD_LOG_DIR/$FDIOTOOLS_IMAGENAME-$branchname-apt-packages.log"
}

apt_install_packages() {
    apt-get install -y --allow-downgrades --allow-remove-essential \
            --allow-change-held-packages $@
}

# Used for older OS distro's which are incompatible
# with modern distro cmake vesrion
apt_override_cmake_install_with_pip3_version() {
    local os_cmake="/usr/bin/cmake"
    local os_cmake_ver="$($os_cmake --version | head -1)"
    local pip3_cmake="/usr/local/bin/cmake"

    python3 -m pip --disable-pip-version-check install cmake || true
    local pip3_cmake_ver="$($pip3_cmake --version | head -1)"
    echo_log "Overriding $OS_NAME '$os_cmake_ver' with '$pip3_cmake_ver'!"
    apt-get remove -y cmake --autoremove || true
    update-alternatives --quiet --remove-all cmake || true
    update-alternatives --quiet --install "$os_cmake" cmake "$pip3_cmake" 100
    echo_log "Default cmake ($(which cmake)) version: '$(cmake --version | head -1)'!"
}

generate_apt_dockerfile_common() {
    local executor_class="$1"
    local executor_image="$2"
        debian_docker_inst_sed="| sed -e 's/has_rootless_extras="1"//g' | sh
"
    cat <<EOF >>"$DOCKERFILE"

# Create download dir to cache external tarballs
WORKDIR $DOCKER_DOWNLOADS_DIR

# Copy-in temporary build tree containing
# ci-management, vpp, & csit git repos
WORKDIR $DOCKER_BUILD_DIR
COPY . .

# Build Environment Variables
ENV DEBIAN_FRONTEND="noninteractive"
ENV FDIOTOOLS_IMAGE="$executor_image"
ENV FDIOTOOLS_EXECUTOR_CLASS="$executor_class"
ENV CIMAN_ROOT="$DOCKER_CIMAN_ROOT"
ENV PATH="\$PATH:$DOCKER_CIMAN_ROOT/docker/scripts"

# Configure locales
RUN apt-get update -qq \\
  && apt-get install -y \\
        apt-utils \\
        locales \\
  && sed -i 's/# \(en_US\.UTF-8 .*\)/\1/' /etc/locale.gen \\
  && locale-gen en_US.UTF-8 \\
  && dpkg-reconfigure --frontend=noninteractive locales \\
  && update-locale LANG=en_US.UTF-8 \\
  && TZ=Etc/UTC && ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone \\
  && rm -r /var/lib/apt/lists/*
ENV LANG="en_US.UTF-8" LANGUAGE="en_US" LC_ALL="en_US.UTF-8"

# Install baseline packages (minimum build & utils).
#
# ci-management global-jjb requirements:
#        facter
#        python3-pip
#        python3-venv
#    for lftools:
#        xmlstarlet
#        libxml2-dev
#        libxslt-dev
#   from packer/provision/baseline.sh:
#        unzip
#        xz-utils
#        git
#        git-review
#        libxml2-dev
#        libxml-xpath-perl
#        libxslt-dev
#        make
#        wget
#        jq
#
# Python build from source requirements:
#        build-essential
#
# TODO:  Fix broken project requirement install targets
#        graphviz         for 'make bootstrap-doxygen' (VPP)
#        doxygen          for 'make doxygen' (VPP)
#        enchant          for 'make docs' (VPP)
#        libffi-dev       for python cffi install (Ubuntu20.04/VPP/aarch64)
#        liblapack-dev    for python numpy/scipy (CSIT/aarch64)
#        libopenblas-dev  for python numpy/scipy (CSIT/aarch64)
#        libpcap-dev      for python pypcap install (CSIT)
#        sshpass          for CSIT jobs
#
#        From .../csit/resources/tools/presentation/run_report_*.sh:
#        libxml2
#        libxml2-dev
#        libxslt-dev
#        build-essential
#        zlib1g-dev
#        unzip
#        xvrb
#        texlive-latex-recommended
#        texlive-fonts-recommended
#        texlive-fonts-extra
#        texlive-latex-extra
#        latexmk
#        wkhtmltopdf
#        inkscape
#
RUN apt-get update -qq \\
  && apt-get install -y \\
             apt-transport-https \\
             curl \\
             ca-certificates \\
             default-jdk \\
             default-jre \\
             dnsutils \\
             doxygen \\
             enchant \\
             emacs \\
             facter \\
             gawk \\
             gdb \\
             gfortran \\
             git \\
             git-review \\
             gnupg-agent \\
             graphviz \\
             inkscape \\
             iproute2 \\
             iputils-clockdiff \\
             iputils-ping \\
             iputils-tracepath \\
             jq \\
             latexmk \\
             libffi-dev \\
             liblapack-dev \\
             libopenblas-dev \\
             libpcap-dev \\
             libxml2 \\
             libxml2-dev \\
             libxml-xpath-perl \\
             libxslt-dev \\
             make \\
             python3-pip \\
             python3-venv \\
             rsync \\
             ruby-dev \\
             software-properties-common \\
             sshpass \\
             sudo \\
             texlive-fonts-extra \\
             texlive-fonts-recommended \\
             texlive-latex-extra \\
             texlive-latex-recommended \\
             traceroute \\
             tree \\
             unzip \\
             vim \\
             wget \\
             wkhtmltopdf \\
             xmlstarlet \\
             xvfb \\
             xz-utils \\
             zlib1g-dev \\
  && curl -L https://packagecloud.io/fdio/master/gpgkey | apt-key add - \\
  && curl -s https://packagecloud.io/install/repositories/fdio/master/script.deb.sh | bash \\
EOF
    # Hack to prevent failure on debian-9 build
    head $DOCKERFILE
    if grep -qe 'debian:9' "$DOCKERFILE" ; then
        echo "  && curl -fsSL https://get.docker.com | sed -e 's/has_rootless_extras=\"1\"//g' | sh \ " >>"$DOCKERFILE"
    else
        echo "  && curl -fsSL https://get.docker.com | sh \ " >>"$DOCKERFILE"
    fi

    cat <<EOF >>"$DOCKERFILE"
  && rm -r /var/lib/apt/lists/*

# Install packages for all project branches
#
RUN apt-get update -qq \\
  && dbld_vpp_install_packages.sh \\
  && dbld_csit_install_packages.sh \\
  && rm -r /var/lib/apt/lists/*
EOF
}

generate_apt_dockerfile_clean() {
    cat <<EOF >>"$DOCKERFILE"

# Clean up copy-in build tree
RUN dbld_dump_build_logs.sh \\
  && rm -rf "/tmp/*" "$DOCKER_BUILD_FILES_DIR" "/root/.ccache"
EOF
}

# Generate 'builder' class apt dockerfile
builder_generate_apt_dockerfile() {
    local executor_class="$1"
    local executor_os_name="$2"
    local executor_image="$3"
    local vpp_install_skip_sysctl_envvar="";

    if grep -q "debian-9"  <<< "$executor_os_name" ; then
        # Workaround to VPP package installation failure on debian-9
        vpp_install_skip_sysctl_envvar="ENV VPP_INSTALL_SKIP_SYSCTL=1"
    fi
    generate_apt_dockerfile_common $executor_class $executor_image
    csit_builder_generate_docker_build_files
    cat <<EOF >>"$DOCKERFILE"

# Install LF-IT requirements
ENV LF_VENV="/root/lf-venv"
RUN apt-get update -qq \\
  && dbld_lfit_requirements.sh \\
  && rm -r /var/lib/apt/lists/*

# Install packagecloud requirements
RUN gem install rake package_cloud \\
  && curl -s https://packagecloud.io/install/repositories/fdio/master/script.deb.sh | bash

# Install CSIT ssh requirements
# TODO: Verify why badkey is required & figure out how to avoid it.
COPY files/badkey /root/.ssh/id_rsa
COPY files/sshconfig /root/.ssh/config

# CI Runtime Environment
WORKDIR /
$vpp_install_skip_sysctl_envvar
ENV VPP_ZOMBIE_NOCHECK="1"
ENV CCACHE_DIR="/scratch/ccache"
ENV CCACHE_MAXSIZE="10G"
EOF
    generate_apt_dockerfile_clean
}

# Generate 'csit_dut' class apt dockerfile
csit_dut_generate_apt_dockerfile() {
    local executor_class="$1"
    local executor_os_name="$2"
    local executor_image="$3"

    csit_dut_generate_docker_build_files
    generate_apt_dockerfile_common "$executor_class" "$executor_image"
    cat <<EOF >>"$DOCKERFILE"

# Install csit_dut specific packages
RUN apt-get update -qq \\
  && apt-get install -y \\
             net-tools \\
             openssh-server \\
             pciutils \\
             rsyslog \\
             supervisor \\
  && rm -r /var/lib/apt/lists/*

# Fix permissions
RUN chown root:syslog /var/log \\
  && chmod 755 /etc/default

# Create directory structure
RUN mkdir -p /var/run/sshd

# SSH settings
RUN echo 'root:Csit1234' | chpasswd \\
  && sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config \\
  && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd

EXPOSE 2222

COPY files/supervisord.conf /etc/supervisor/supervisord.conf

CMD ["sh", "-c", "rm -f /dev/shm/db /dev/shm/global_vm /dev/shm/vpe-api; /usr/bin/supervisord -c /etc/supervisor/supervisord.conf; /usr/sbin/sshd -D -p 2222"]
EOF
    generate_apt_dockerfile_clean
}

# Generate 'csit_shim' class apt dockerfile
csit_shim_generate_apt_dockerfile() {
    local executor_class="$1"
    local executor_os_name="$2"
    local executor_image="$3"

    csit_shim_generate_docker_build_files
    cat <<EOF >>"$DOCKERFILE"

# Copy-in temporary build tree containing
# ci-management, vpp, & csit git repos
WORKDIR $DOCKER_BUILD_DIR
COPY . .

# Build Environment Variables
ENV DEBIAN_FRONTEND="noninteractive"
ENV FDIOTOOLS_IMAGE="$executor_image"
ENV FDIOTOOLS_EXECUTOR_CLASS="$executor_class"
ENV CIMAN_ROOT="$DOCKER_CIMAN_ROOT"
ENV PATH="\$PATH:$DOCKER_CIMAN_ROOT/docker/scripts"

# Configure locales & timezone
RUN apt-get update -qq \\
  && apt-get install -y \\
             apt-utils \\
             locales \\
  && sed -i 's/# \(en_US\.UTF-8 .*\)/\1/' /etc/locale.gen \\
  && locale-gen en_US.UTF-8 \\
  && dpkg-reconfigure --frontend=noninteractive locales \\
  && update-locale LANG=en_US.UTF-8 \\
  && TZ=Etc/UTC && ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone \\
  && rm -r /var/lib/apt/lists/*
ENV LANG=en_US.UTF-8 LANGUAGE=en_US LC_ALL=en_US.UTF-8

COPY files/wrapdocker /usr/local/bin/wrapdocker
RUN chmod +x /usr/local/bin/wrapdocker

# Install packages and Docker
RUN apt-get update -qq \\
  && apt-get install -y  \\
             bash \\
             curl \\
             iproute2 \\
             locales \\
             ssh \\
             sudo \\
             tzdata \\
             uuid-runtime \\
  && curl -fsSL https://get.docker.com | sh \\
  && rm -rf /var/lib/apt/lists/*

RUN mkdir /var/run/sshd
RUN echo 'root:Csit1234' | chpasswd
RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config

# SSH login fix. Otherwise user is kicked off after login
RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd

# Need volume for sidecar docker launches
VOLUME /var/lib/docker

# SSH to listen on port 6022 in shim
RUN echo 'Port 6022' >>/etc/ssh/sshd_config
RUN echo 'Port 6023' >>/etc/ssh/sshd_config

# TODO: Verify why badkeypub is required & figure out how to avoid it.
COPY files/badkeypub /root/.ssh/authorized_keys
COPY files/sshconfig /root/.ssh/config

# Clean up copy-in build tree
RUN rm -rf /tmp/* $DOCKER_BUILD_FILES_DIR

# Start sshd by default
EXPOSE 22
CMD ["/usr/sbin/sshd", "-D"]
EOF
}

generate_apt_dockerfile() {
    local executor_class="$1"
    local executor_os_name="$2"
    local from_image="$3"
    local executor_image="$4"

    cat <<EOF  >"$DOCKERIGNOREFILE"
**/__pycache__
*.pyc
EOF
    cat <<EOF  >"$DOCKERFILE"
FROM $from_image AS ${executor_class}-executor-image
LABEL Description="FD.io CI '$executor_class' executor docker image for $executor_os_name/$OS_ARCH"
LABEL Vendor="fd.io"
LABEL Version="$DOCKER_TAG"
EOF
    ${executor_class}_generate_apt_dockerfile "$executor_class" \
        "$executor_os_name" "$executor_image"
}