diff options
Diffstat (limited to 'jjb/scripts/publish_library_py.sh')
| -rw-r--r-- | jjb/scripts/publish_library_py.sh | 318 |
1 files changed, 318 insertions, 0 deletions
diff --git a/jjb/scripts/publish_library_py.sh b/jjb/scripts/publish_library_py.sh new file mode 100644 index 00000000..1cbeb23c --- /dev/null +++ b/jjb/scripts/publish_library_py.sh @@ -0,0 +1,318 @@ +#!/bin/bash + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "---> publish_library_py.sh" + +set -exuo pipefail + +PYTHON_SCRIPT="/w/workspace/publish_library.py" + +pip3 install boto3 +mkdir -p $(dirname "$PYTHON_SCRIPT") + +cat >$PYTHON_SCRIPT <<'END_OF_PYTHON_SCRIPT' +#!/usr/bin/python3 + +"""S3 publish library.""" + +import glob +import gzip +import logging +import os +import shutil +import subprocess +import sys +import tempfile + +import boto3 +from botocore.exceptions import ClientError +import requests +import six + + +logging.basicConfig( + format=u"%(levelname)s: %(message)s", + stream=sys.stdout, + level=logging.INFO +) +logging.getLogger(u"botocore").setLevel(logging.INFO) + + +FILE_TYPE = { + u"xml": u"application/xml", + u"html": u"text/html", + u"txt": u"text/plain", + u"log": u"text/plain", + u"css": u"text/css", + u"md": u"text/markdown", + u"rst": u"text/x-rst", + u"csv": u"text/csv", + u"svg": u"image/svg+xml", + u"jpg": u"image/jpeg", + u"png": u"image/png", + u"gif": u"image/gif", + u"js": u"application/javascript", + u"pdf": u"application/pdf", + u"json": u"application/json", + u"otf": u"font/otf", + u"ttf": u"font/ttf", + u"woff": u"font/woff", + u"woff2": u"font/woff2" +} + + +def compress_text(src_dpath): + """Compress all text files in directory. + + :param src_dpath: Input dir path. + :type src_dpath: str + """ + save_dir = os.getcwd() + os.chdir(src_dpath) + + compress_types = [ + "**/*.html", + "**/*.log", + "**/*.txt", + "**/*.xml", + "**/*.json" + ] + paths = [] + for _type in compress_types: + search = os.path.join(src_dpath, _type) + paths.extend(glob.glob(search, recursive=True)) + + for _file in paths: + # glob may follow symlink paths that open can't find + if os.path.exists(_file): + gz_file = u"{}.gz".format(_file) + with open(_file, "rb") as src, gzip.open(gz_file, "wb") as dest: + shutil.copyfileobj(src, dest) + os.remove(_file) + + os.chdir(save_dir) + + +def copy_archives(workspace): + """Copy files or directories in a $WORKSPACE/archives to the current + directory. + + :params workspace: Workspace directery with archives directory. + :type workspace: str + """ + archives_dir = os.path.join(workspace, u"archives") + dest_dir = os.getcwd() + + logging.debug(u"Copying files from " + archives_dir + u" to " + dest_dir) + + if os.path.exists(archives_dir): + if os.path.isfile(archives_dir): + logging.error(u"Target is a file, not a directory.") + raise RuntimeError(u"Not a directory.") + else: + logging.debug("Archives dir {} does exist.".format(archives_dir)) + for item in os.listdir(archives_dir): + src = os.path.join(archives_dir, item) + dst = os.path.join(dest_dir, item) + try: + if os.path.isdir(src): + shutil.copytree(src, dst, symlinks=False, ignore=None) + else: + shutil.copy2(src, dst) + except shutil.Error as e: + logging.error(e) + raise RuntimeError(u"Could not copy " + src) + else: + logging.error(u"Archives dir does not exist.") + raise RuntimeError(u"Missing directory " + archives_dir) + + +def upload(s3_resource, s3_bucket, src_fpath, s3_path): + """Upload single file to destination bucket. + + :param s3_resource: S3 storage resource. + :param s3_bucket: S3 bucket name. + :param src_fpath: Input file path. + :param s3_path: Destination file path on remote storage. + :type s3_resource: Object + :type s3_bucket: str + :type src_fpath: str + :type s3_path: str + """ + def is_gzip_file(filepath): + with open(filepath, u"rb") as test_f: + return test_f.read(2) == b"\x1f\x8b" + + if os.path.isdir(src_fpath): + return + if os.path.isfile(src_fpath): + file_name, file_extension = os.path.splitext(src_fpath) + content_encoding = u"" + content_type = u"application/octet-stream" + if is_gzip_file(src_fpath): + file_name, file_extension = os.path.splitext(file_name) + content_encoding = "gzip" + content_type = FILE_TYPE.get( + file_extension.strip("."), + u"application/octet-stream" + ) + + extra_args = dict() + extra_args[u"ContentType"] = content_type + if content_encoding: + extra_args[u"ContentEncoding"] = content_encoding + + try: + s3_resource.Bucket(s3_bucket).upload_file( + src_fpath, s3_path, ExtraArgs=extra_args + ) + logging.info(u"Successfully uploaded to " + s3_path) + except ClientError as e: + logging.error(e) + + +def upload_recursive(s3_resource, s3_bucket, src_fpath, s3_path): + """Recursively uploads input folder to destination. + + Example: + - s3_bucket: logs.fd.io + - src_fpath: /workspace/archives. + - s3_path: /hostname/job/id/ + + :param s3_resource: S3 storage resource. + :param s3_bucket: S3 bucket name. + :param src_fpath: Input folder path. + :param s3_path: S3 destination path. + :type s3_resource: Object + :type s3_bucket: str + :type src_fpath: str + :type s3_path: str + """ + for path, _, files in os.walk(src_fpath): + for file in files: + _path = path.replace(src_fpath, u"") + _src_fpath = path + u"/" + file + _s3_path = os.path.normpath(s3_path + u"/" + _path + u"/" + file) + upload( + s3_resource=s3_resource, + s3_bucket=s3_bucket, + src_fpath=_src_fpath, + s3_path=_s3_path + ) + + +def deploy_docs(s3_bucket, s3_path, docs_dir): + """Ship docs dir content to S3 bucket. Requires the s3 bucket to exist. + + :param s3_bucket: Name of S3 bucket. Eg: lf-project-date + :param s3_path: Path on S3 bucket to place the docs. Eg: + csit/${GERRIT_BRANCH}/report + :param docs_dir: Directory in which to recursively upload content. + :type s3_bucket: Object + :type s3_path: str + :type docs_dir: str + """ + try: + s3_resource = boto3.resource( + u"s3", + endpoint_url=os.environ[u"AWS_ENDPOINT_URL"] + ) + except KeyError: + s3_resource = boto3.resource( + u"s3" + ) + + upload_recursive( + s3_resource=s3_resource, + s3_bucket=s3_bucket, + src_fpath=docs_dir, + s3_path=s3_path + ) + + +def deploy_s3(s3_bucket, s3_path, build_url, workspace): + """Add logs and archives to temp directory to be shipped to S3 bucket. + Fetches logs and system information and pushes them and archives to S3 + for log archiving. + Requires the s3 bucket to exist. + + :param s3_bucket: Name of S3 bucket. Eg: lf-project-date + :param s3_path: Path on S3 bucket place the logs and archives. Eg: + $JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER + :param build_url: URL of the Jenkins build. Jenkins typically provides this + via the $BUILD_URL environment variable. + :param workspace: Directory in which to search, typically in Jenkins this is + $WORKSPACE + :type s3_bucket: Object + :type s3_path: str + :type build_url: str + :type workspace: str + """ + try: + s3_resource = boto3.resource( + u"s3", + endpoint_url=os.environ[u"AWS_ENDPOINT_URL"] + ) + except KeyError: + s3_resource = boto3.resource( + u"s3" + ) + + previous_dir = os.getcwd() + work_dir = tempfile.mkdtemp(prefix="backup-s3.") + os.chdir(work_dir) + + # Copy archive files to tmp dir. + copy_archives(workspace) + + # Create additional build logs. + with open(u"_build-details.log", u"w+") as f: + f.write(u"build-url: " + build_url) + + # Magic string used to trim console logs at the appropriate level during + # wget. + MAGIC_STRING = u"-----END_OF_BUILD-----" + logging.info(MAGIC_STRING) + + resp = requests.get(build_url + u"/consoleText") + with open(u"console.log", u"w+", encoding=u"utf-8") as f: + f.write( + six.text_type(resp.content.decode(u"utf-8").split(MAGIC_STRING)[0]) + ) + + query = u"time=HH:mm:ss&appendLog" + resp = requests.get(build_url + u"/timestamps?" + query) + with open(u"console-timestamp.log", u"w+", encoding=u"utf-8") as f: + f.write( + six.text_type(resp.content.decode(u"utf-8").split(MAGIC_STRING)[0]) + ) + + compress_text(work_dir) + + upload_recursive( + s3_resource=s3_resource, + s3_bucket=s3_bucket, + src_fpath=work_dir, + s3_path=s3_path + ) + + os.chdir(previous_dir) + shutil.rmtree(work_dir) + + +if __name__ == u"__main__": + globals()[sys.argv[1]](*sys.argv[2:]) + +END_OF_PYTHON_SCRIPT |