aboutsummaryrefslogtreecommitdiffstats
path: root/terraform-ci-infra/1n_nmd/tools
diff options
context:
space:
mode:
authorpmikus <pmikus@cisco.com>2021-03-07 08:57:13 +0000
committerPeter Mikus <pmikus@cisco.com>2021-03-11 07:07:03 +0000
commitc318223fcd266c0ee2982e803c44e193c2023054 (patch)
tree4559bf49a5ab183d0c3a75a883edf40e7802222f /terraform-ci-infra/1n_nmd/tools
parent221e2f4da7cb27954525d973d930cb8db4601c8f (diff)
Infra: Switch csit-shim to fdiotools
+ use /u/fdiotools + use ubuntu 20.04 Signed-off-by: pmikus <pmikus@cisco.com> Change-Id: I091e63a0d9e50de203b1527c7500b3864a616af6
Diffstat (limited to 'terraform-ci-infra/1n_nmd/tools')
-rwxr-xr-xterraform-ci-infra/1n_nmd/tools/artifacts.py138
-rwxr-xr-xterraform-ci-infra/1n_nmd/tools/artifacts_download.py47
2 files changed, 0 insertions, 185 deletions
diff --git a/terraform-ci-infra/1n_nmd/tools/artifacts.py b/terraform-ci-infra/1n_nmd/tools/artifacts.py
deleted file mode 100755
index 36bef7c5c6..0000000000
--- a/terraform-ci-infra/1n_nmd/tools/artifacts.py
+++ /dev/null
@@ -1,138 +0,0 @@
-#!/usr/bin/python3
-
-# Copyright (c) 2020 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Storage utilities library."""
-
-import argparse
-import gzip
-import os
-from mimetypes import MimeTypes
-
-from boto3 import resource
-from botocore.client import Config
-
-
-ENDPOINT_URL = u"http://storage.service.consul:9000"
-AWS_ACCESS_KEY_ID = u"storage"
-AWS_SECRET_ACCESS_KEY = u"Storage1234"
-REGION_NAME = u"yul1"
-COMPRESS_MIME = (
- u"text/html",
- u"text/xml",
- u"application/octet-stream"
-)
-
-
-def compress(src_fpath):
- """Compress a single file.
-
- :param src_fpath: Input file path.
- :type src_fpath: str
- """
- with open(src_fpath, u"rb") as orig_file:
- with gzip.open(f"{src_fpath}.gz", u"wb") as zipped_file:
- zipped_file.writelines(orig_file)
-
-
-def upload(storage, bucket, src_fpath, dst_fpath):
- """Upload single file to destination bucket.
-
- :param storage: S3 storage resource.
- :param bucket: S3 bucket name.
- :param src_fpath: Input file path.
- :param dst_fpath: Destination file path on remote storage.
- :type storage: Object
- :type bucket: str
- :type src_fpath: str
- :type dst_fpath: str
- """
- mime = MimeTypes().guess_type(src_fpath)[0]
- if not mime:
- mime = "application/octet-stream"
-
- if mime in COMPRESS_MIME and bucket in "logs":
- compress(src_fpath)
- src_fpath = f"{src_fpath}.gz"
- dst_fpath = f"{dst_fpath}.gz"
-
- storage.Bucket(f"{bucket}.fd.io").upload_file(
- src_fpath,
- dst_fpath,
- ExtraArgs={
- u"ContentType": mime
- }
- )
- print(f"https://{bucket}.nginx.service.consul/{dst_fpath}")
-
-
-def upload_recursive(storage, bucket, src_fpath):
- """Recursively uploads input folder to destination.
-
- Example:
- - bucket: logs
- - src_fpath: /home/user
- - dst_fpath: logs.fd.io/home/user
-
- :param storage: S3 storage resource.
- :param bucket: S3 bucket name.
- :param src_fpath: Input folder path.
- :type storage: Object
- :type bucket: str
- :type src_fpath: str
- """
- for path, _, files in os.walk(src_fpath):
- for file in files:
- _path = path.replace(src_fpath, u"")
- _dir = src_fpath[1:] if src_fpath[0] == "/" else src_fpath
- _dst_fpath = os.path.normpath(f"{_dir}/{_path}/{file}")
- _src_fpath = os.path.join(path, file)
- upload(storage, bucket, _src_fpath, _dst_fpath)
-
-
-def main():
- """Main function for storage manipulation."""
-
- parser = argparse.ArgumentParser()
- parser.add_argument(
- u"-d", u"--dir", required=True, type=str,
- help=u"Directory to upload to storage."
- )
- parser.add_argument(
- u"-b", u"--bucket", required=True, type=str,
- help=u"Target bucket on storage."
- )
- args = parser.parse_args()
-
- # Create main storage resource.
- storage = resource(
- u"s3",
- endpoint_url=ENDPOINT_URL,
- aws_access_key_id=AWS_ACCESS_KEY_ID,
- aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
- config=Config(
- signature_version=u"s3v4"
- ),
- region_name=REGION_NAME
- )
-
- upload_recursive(
- storage=storage,
- bucket=args.bucket,
- src_fpath=args.dir
- )
-
-
-if __name__ == u"__main__":
- main()
diff --git a/terraform-ci-infra/1n_nmd/tools/artifacts_download.py b/terraform-ci-infra/1n_nmd/tools/artifacts_download.py
deleted file mode 100755
index 2af895e8b9..0000000000
--- a/terraform-ci-infra/1n_nmd/tools/artifacts_download.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/python3
-
-# Copyright (c) 2021 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Storage utilities library."""
-
-from boto3 import resource
-from botocore.client import Config
-
-
-ENDPOINT_URL = u"http://storage.service.consul:9000"
-AWS_ACCESS_KEY_ID = u"storage"
-AWS_SECRET_ACCESS_KEY = u"Storage1234"
-REGION_NAME = u"yul1"
-LOGS_BUCKET = f"logs.fd.io"
-
-
-if __name__ == u"__main__":
- """Main function for storage manipulation."""
-
- # Create main storage resource.
- storage = resource(
- u"s3",
- endpoint_url=ENDPOINT_URL,
- aws_access_key_id=AWS_ACCESS_KEY_ID,
- aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
- config=Config(
- signature_version=u"s3v4"
- ),
- region_name=REGION_NAME
- )
-
- storage.Bucket(LOGS_BUCKET).download_file(
- "/vex-yul-rot-jenkins-1/csit-vpp-perf-report-iterative-2101-3n-skx/47/archives/output_info.xml.gz",
- "output.xml.gz"
- ) \ No newline at end of file