aboutsummaryrefslogtreecommitdiffstats
path: root/fdio.infra.terraform/1n_nmd/tools
diff options
context:
space:
mode:
Diffstat (limited to 'fdio.infra.terraform/1n_nmd/tools')
-rwxr-xr-xfdio.infra.terraform/1n_nmd/tools/artifacts.py138
-rwxr-xr-xfdio.infra.terraform/1n_nmd/tools/artifacts_download.py47
2 files changed, 185 insertions, 0 deletions
diff --git a/fdio.infra.terraform/1n_nmd/tools/artifacts.py b/fdio.infra.terraform/1n_nmd/tools/artifacts.py
new file mode 100755
index 0000000000..36bef7c5c6
--- /dev/null
+++ b/fdio.infra.terraform/1n_nmd/tools/artifacts.py
@@ -0,0 +1,138 @@
+#!/usr/bin/python3
+
+# Copyright (c) 2020 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Storage utilities library."""
+
+import argparse
+import gzip
+import os
+from mimetypes import MimeTypes
+
+from boto3 import resource
+from botocore.client import Config
+
+
+ENDPOINT_URL = u"http://storage.service.consul:9000"
+AWS_ACCESS_KEY_ID = u"storage"
+AWS_SECRET_ACCESS_KEY = u"Storage1234"
+REGION_NAME = u"yul1"
+COMPRESS_MIME = (
+ u"text/html",
+ u"text/xml",
+ u"application/octet-stream"
+)
+
+
+def compress(src_fpath):
+ """Compress a single file.
+
+ :param src_fpath: Input file path.
+ :type src_fpath: str
+ """
+ with open(src_fpath, u"rb") as orig_file:
+ with gzip.open(f"{src_fpath}.gz", u"wb") as zipped_file:
+ zipped_file.writelines(orig_file)
+
+
+def upload(storage, bucket, src_fpath, dst_fpath):
+ """Upload single file to destination bucket.
+
+ :param storage: S3 storage resource.
+ :param bucket: S3 bucket name.
+ :param src_fpath: Input file path.
+ :param dst_fpath: Destination file path on remote storage.
+ :type storage: Object
+ :type bucket: str
+ :type src_fpath: str
+ :type dst_fpath: str
+ """
+ mime = MimeTypes().guess_type(src_fpath)[0]
+ if not mime:
+ mime = "application/octet-stream"
+
+ if mime in COMPRESS_MIME and bucket in "logs":
+ compress(src_fpath)
+ src_fpath = f"{src_fpath}.gz"
+ dst_fpath = f"{dst_fpath}.gz"
+
+ storage.Bucket(f"{bucket}.fd.io").upload_file(
+ src_fpath,
+ dst_fpath,
+ ExtraArgs={
+ u"ContentType": mime
+ }
+ )
+ print(f"https://{bucket}.nginx.service.consul/{dst_fpath}")
+
+
+def upload_recursive(storage, bucket, src_fpath):
+ """Recursively uploads input folder to destination.
+
+ Example:
+ - bucket: logs
+ - src_fpath: /home/user
+ - dst_fpath: logs.fd.io/home/user
+
+ :param storage: S3 storage resource.
+ :param bucket: S3 bucket name.
+ :param src_fpath: Input folder path.
+ :type storage: Object
+ :type bucket: str
+ :type src_fpath: str
+ """
+ for path, _, files in os.walk(src_fpath):
+ for file in files:
+ _path = path.replace(src_fpath, u"")
+ _dir = src_fpath[1:] if src_fpath[0] == "/" else src_fpath
+ _dst_fpath = os.path.normpath(f"{_dir}/{_path}/{file}")
+ _src_fpath = os.path.join(path, file)
+ upload(storage, bucket, _src_fpath, _dst_fpath)
+
+
+def main():
+ """Main function for storage manipulation."""
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ u"-d", u"--dir", required=True, type=str,
+ help=u"Directory to upload to storage."
+ )
+ parser.add_argument(
+ u"-b", u"--bucket", required=True, type=str,
+ help=u"Target bucket on storage."
+ )
+ args = parser.parse_args()
+
+ # Create main storage resource.
+ storage = resource(
+ u"s3",
+ endpoint_url=ENDPOINT_URL,
+ aws_access_key_id=AWS_ACCESS_KEY_ID,
+ aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
+ config=Config(
+ signature_version=u"s3v4"
+ ),
+ region_name=REGION_NAME
+ )
+
+ upload_recursive(
+ storage=storage,
+ bucket=args.bucket,
+ src_fpath=args.dir
+ )
+
+
+if __name__ == u"__main__":
+ main()
diff --git a/fdio.infra.terraform/1n_nmd/tools/artifacts_download.py b/fdio.infra.terraform/1n_nmd/tools/artifacts_download.py
new file mode 100755
index 0000000000..2af895e8b9
--- /dev/null
+++ b/fdio.infra.terraform/1n_nmd/tools/artifacts_download.py
@@ -0,0 +1,47 @@
+#!/usr/bin/python3
+
+# Copyright (c) 2021 Cisco and/or its affiliates.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Storage utilities library."""
+
+from boto3 import resource
+from botocore.client import Config
+
+
+ENDPOINT_URL = u"http://storage.service.consul:9000"
+AWS_ACCESS_KEY_ID = u"storage"
+AWS_SECRET_ACCESS_KEY = u"Storage1234"
+REGION_NAME = u"yul1"
+LOGS_BUCKET = f"logs.fd.io"
+
+
+if __name__ == u"__main__":
+ """Main function for storage manipulation."""
+
+ # Create main storage resource.
+ storage = resource(
+ u"s3",
+ endpoint_url=ENDPOINT_URL,
+ aws_access_key_id=AWS_ACCESS_KEY_ID,
+ aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
+ config=Config(
+ signature_version=u"s3v4"
+ ),
+ region_name=REGION_NAME
+ )
+
+ storage.Bucket(LOGS_BUCKET).download_file(
+ "/vex-yul-rot-jenkins-1/csit-vpp-perf-report-iterative-2101-3n-skx/47/archives/output_info.xml.gz",
+ "output.xml.gz"
+ ) \ No newline at end of file