diff options
Diffstat (limited to 'fdio.infra.terraform/1n_nmd/tools')
-rwxr-xr-x | fdio.infra.terraform/1n_nmd/tools/artifacts.py | 138 | ||||
-rwxr-xr-x | fdio.infra.terraform/1n_nmd/tools/artifacts_download.py | 47 |
2 files changed, 185 insertions, 0 deletions
diff --git a/fdio.infra.terraform/1n_nmd/tools/artifacts.py b/fdio.infra.terraform/1n_nmd/tools/artifacts.py new file mode 100755 index 0000000000..36bef7c5c6 --- /dev/null +++ b/fdio.infra.terraform/1n_nmd/tools/artifacts.py @@ -0,0 +1,138 @@ +#!/usr/bin/python3 + +# Copyright (c) 2020 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Storage utilities library.""" + +import argparse +import gzip +import os +from mimetypes import MimeTypes + +from boto3 import resource +from botocore.client import Config + + +ENDPOINT_URL = u"http://storage.service.consul:9000" +AWS_ACCESS_KEY_ID = u"storage" +AWS_SECRET_ACCESS_KEY = u"Storage1234" +REGION_NAME = u"yul1" +COMPRESS_MIME = ( + u"text/html", + u"text/xml", + u"application/octet-stream" +) + + +def compress(src_fpath): + """Compress a single file. + + :param src_fpath: Input file path. + :type src_fpath: str + """ + with open(src_fpath, u"rb") as orig_file: + with gzip.open(f"{src_fpath}.gz", u"wb") as zipped_file: + zipped_file.writelines(orig_file) + + +def upload(storage, bucket, src_fpath, dst_fpath): + """Upload single file to destination bucket. + + :param storage: S3 storage resource. + :param bucket: S3 bucket name. + :param src_fpath: Input file path. + :param dst_fpath: Destination file path on remote storage. + :type storage: Object + :type bucket: str + :type src_fpath: str + :type dst_fpath: str + """ + mime = MimeTypes().guess_type(src_fpath)[0] + if not mime: + mime = "application/octet-stream" + + if mime in COMPRESS_MIME and bucket in "logs": + compress(src_fpath) + src_fpath = f"{src_fpath}.gz" + dst_fpath = f"{dst_fpath}.gz" + + storage.Bucket(f"{bucket}.fd.io").upload_file( + src_fpath, + dst_fpath, + ExtraArgs={ + u"ContentType": mime + } + ) + print(f"https://{bucket}.nginx.service.consul/{dst_fpath}") + + +def upload_recursive(storage, bucket, src_fpath): + """Recursively uploads input folder to destination. + + Example: + - bucket: logs + - src_fpath: /home/user + - dst_fpath: logs.fd.io/home/user + + :param storage: S3 storage resource. + :param bucket: S3 bucket name. + :param src_fpath: Input folder path. + :type storage: Object + :type bucket: str + :type src_fpath: str + """ + for path, _, files in os.walk(src_fpath): + for file in files: + _path = path.replace(src_fpath, u"") + _dir = src_fpath[1:] if src_fpath[0] == "/" else src_fpath + _dst_fpath = os.path.normpath(f"{_dir}/{_path}/{file}") + _src_fpath = os.path.join(path, file) + upload(storage, bucket, _src_fpath, _dst_fpath) + + +def main(): + """Main function for storage manipulation.""" + + parser = argparse.ArgumentParser() + parser.add_argument( + u"-d", u"--dir", required=True, type=str, + help=u"Directory to upload to storage." + ) + parser.add_argument( + u"-b", u"--bucket", required=True, type=str, + help=u"Target bucket on storage." + ) + args = parser.parse_args() + + # Create main storage resource. + storage = resource( + u"s3", + endpoint_url=ENDPOINT_URL, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + config=Config( + signature_version=u"s3v4" + ), + region_name=REGION_NAME + ) + + upload_recursive( + storage=storage, + bucket=args.bucket, + src_fpath=args.dir + ) + + +if __name__ == u"__main__": + main() diff --git a/fdio.infra.terraform/1n_nmd/tools/artifacts_download.py b/fdio.infra.terraform/1n_nmd/tools/artifacts_download.py new file mode 100755 index 0000000000..2af895e8b9 --- /dev/null +++ b/fdio.infra.terraform/1n_nmd/tools/artifacts_download.py @@ -0,0 +1,47 @@ +#!/usr/bin/python3 + +# Copyright (c) 2021 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Storage utilities library.""" + +from boto3 import resource +from botocore.client import Config + + +ENDPOINT_URL = u"http://storage.service.consul:9000" +AWS_ACCESS_KEY_ID = u"storage" +AWS_SECRET_ACCESS_KEY = u"Storage1234" +REGION_NAME = u"yul1" +LOGS_BUCKET = f"logs.fd.io" + + +if __name__ == u"__main__": + """Main function for storage manipulation.""" + + # Create main storage resource. + storage = resource( + u"s3", + endpoint_url=ENDPOINT_URL, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + config=Config( + signature_version=u"s3v4" + ), + region_name=REGION_NAME + ) + + storage.Bucket(LOGS_BUCKET).download_file( + "/vex-yul-rot-jenkins-1/csit-vpp-perf-report-iterative-2101-3n-skx/47/archives/output_info.xml.gz", + "output.xml.gz" + )
\ No newline at end of file |