From 80a4b408b1bec92f51813c5b758d7b2739f665ed Mon Sep 17 00:00:00 2001 From: pmikus Date: Tue, 7 Sep 2021 07:01:36 +0000 Subject: Terraform: Cleanup + Use terraform fmt to format Signed-off-by: pmikus Change-Id: I32bd2f81c5838722506d44a8ff8ab48da204643a --- fdio.infra.terraform/1n_nmd/.terraform.lock.hcl | 58 ------- fdio.infra.terraform/1n_nmd/alertmanager/main.tf | 10 +- .../1n_nmd/alertmanager/variables.tf | 4 +- fdio.infra.terraform/1n_nmd/aws/main.tf | 18 +-- fdio.infra.terraform/1n_nmd/aws/providers.tf | 4 +- fdio.infra.terraform/1n_nmd/aws/variables.tf | 6 +- fdio.infra.terraform/1n_nmd/grafana/main.tf | 8 +- fdio.infra.terraform/1n_nmd/grafana/variables.tf | 4 +- fdio.infra.terraform/1n_nmd/main.tf | 172 ++++++++++----------- fdio.infra.terraform/1n_nmd/minio/main.tf | 20 +-- fdio.infra.terraform/1n_nmd/minio/variables.tf | 20 +-- fdio.infra.terraform/1n_nmd/nginx/main.tf | 8 +- fdio.infra.terraform/1n_nmd/nginx/variables.tf | 2 +- fdio.infra.terraform/1n_nmd/prometheus/main.tf | 10 +- .../1n_nmd/prometheus/variables.tf | 4 +- fdio.infra.terraform/1n_nmd/providers.tf | 4 +- fdio.infra.terraform/1n_nmd/tools/artifacts.py | 138 ----------------- fdio.infra.terraform/1n_nmd/vpp_device/main.tf | 8 +- .../1n_nmd/vpp_device/variables.tf | 2 +- 19 files changed, 152 insertions(+), 348 deletions(-) delete mode 100644 fdio.infra.terraform/1n_nmd/.terraform.lock.hcl delete mode 100755 fdio.infra.terraform/1n_nmd/tools/artifacts.py (limited to 'fdio.infra.terraform/1n_nmd') diff --git a/fdio.infra.terraform/1n_nmd/.terraform.lock.hcl b/fdio.infra.terraform/1n_nmd/.terraform.lock.hcl deleted file mode 100644 index 3a2e4ef85f..0000000000 --- a/fdio.infra.terraform/1n_nmd/.terraform.lock.hcl +++ /dev/null @@ -1,58 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/nomad" { - version = "1.4.11" - constraints = "~> 1.4.9" - hashes = [ - "h1:ElEvgyMfWoWyQbB6c51rGTjQlZKWf3QOvf5NhX/Vuyw=", - "zh:150d0ab25241a42f2ac5008878e0106c0887eec15181a40bee1433b87f01b8ed", - "zh:1d4ccda0729f12060e7f4ce5c6d83042d6d38ba2e546b68722ccb74832793b0c", - "zh:2964652181f59097aa1126f4b215b9232702b1a56df3e017e6b5683d5615714b", - "zh:42843e68bca24280e84ec600ae5d8f085fb26bdcdb4c0ccff2139ed81e1cb8c1", - "zh:4c6d90d40a360d84bc84c9af35c64056237537fa0f8118bf890fcf4e71f7b0f6", - "zh:51771ce42a99d7d4f5a4306367eee4cea72391c07f1f1c55c3c4a5c6a9eca53d", - "zh:6ab2389f1be6bb39d4966c253bf4fc77348e90f7e07ed2abb5ec5c90a4bbb615", - "zh:9b109254ea7ca6a5b0ede33b406cf5fed779f05672891bbd1cc3255c9cb17663", - "zh:a38c929d4fd03193cce94178c0fbaa1f7f09e93223ac71dc77c834d429b1c7c9", - "zh:bdc9bc10a1ecb5ae3da651df1709bf9d5474f25e310b73bdf32c86417674d32b", - ] -} - -provider "registry.terraform.io/hashicorp/template" { - version = "2.1.2" - constraints = "~> 2.1.2" - hashes = [ - "h1:8NcPRk3yxQtUlAT/YGfjBEJ76rQI2ljARYeIEjhtWho=", - "zh:149e4bf47ac21b67f6567767afcd29caaf0b0ca43714748093a00a2a98cd17a8", - "zh:2ff61a5eb7550e0df2baefccea78a8b621faef76154aad7ddf9c85c1d69f7ebf", - "zh:3b2d9a9f80754eb0a250a80e0dfdef385501697850a54ead744d1615e60fe648", - "zh:545b93c818035aac59f4a821644276c123a74aa210b1221974d832a6009df201", - "zh:5508512a522152a302591b399512fa736d8f57088c85ca74f7e00014db3a8c26", - "zh:701b56016a6db814ade171877375a2429b45979f97c2d112e4f2103f0433eb08", - "zh:90fc08165958538d8a099f17282c615d5b13f86bb215af33e2ca7551bf81996f", - "zh:affa6d409060c01a610854a395970d76701d0b07696e1ed6776b3f3b58014104", - "zh:b66ffed670bf0ed6714fa4ac26444a8e22f71ec6da134faf0b1f77fb2c13c666", - "zh:bb3d87db22f0ac56717eadde39690e3e27c1c01b10d0ecbe2e6e39f1e5c4d808", - "zh:c54b9693c9f348591432aabc808cbe1786bcda1cb70d312ef62a24545a14f945", - "zh:e7c8f8506cee5fa28f842714857d412a2b09e61127a0efe2a164c2f3d9bf2619", - ] -} - -provider "registry.terraform.io/hashicorp/vault" { - version = "2.16.0" - constraints = ">= 2.14.0" - hashes = [ - "h1:h27r8aZ5nwRfEelTQnJoA8s3TndJYPI7+3Df1DXIhXk=", - "zh:13dde74fac618ee0281bad60a60966a85d4a59c8420b15fd6499996fa1bc99b3", - "zh:1daad9da6c82f43cbd07bf1cfedf3c6960fb2f96bc59f94fd75d361065b8c51a", - "zh:68075d8e1824b745267ce9e4ef693b202b9282561811de6ccf7298935f482128", - "zh:86df4a4405413d575cd72985483163e62539afbd659fddef59fc637875b707e2", - "zh:8f8306ada4c1c44945ce5205e4f1cfbf5e3d46a9da2f3a1d0be17d32e4935845", - "zh:9eb75febcd6fcca9885a6f5e93293a200b2effbe31f47d265cc4d1346d42d29e", - "zh:a658b55b239bc7ad59a2bf55e7abbfe5f0111d37dd68b5d4bb947eee93969092", - "zh:af10679c241bd0e0168f57c24e839fd24c747f3e84b7bb6de3cd791471709249", - "zh:ee3030f36846de45450be088aa4c2b1f69246b2ecf40d7ea6a15a7f09ac5e5d0", - "zh:efe6cc23f77336604358e627b0b565c1421a97376e510a9cdaaf849524944713", - ] -} diff --git a/fdio.infra.terraform/1n_nmd/alertmanager/main.tf b/fdio.infra.terraform/1n_nmd/alertmanager/main.tf index 9525aabc0c..b7ab5dce92 100644 --- a/fdio.infra.terraform/1n_nmd/alertmanager/main.tf +++ b/fdio.infra.terraform/1n_nmd/alertmanager/main.tf @@ -1,5 +1,5 @@ locals { - datacenters = join(",", var.nomad_datacenters) + datacenters = join(",", var.nomad_datacenters) alertmanager_url = join("", [ @@ -12,8 +12,8 @@ locals { } data "template_file" "nomad_job_alertmanager" { - template = file("${path.module}/conf/nomad/alertmanager.hcl") - vars = { + template = file("${path.module}/conf/nomad/alertmanager.hcl") + vars = { datacenters = local.datacenters url = local.alertmanager_url job_name = var.alertmanager_job_name @@ -35,6 +35,6 @@ data "template_file" "nomad_job_alertmanager" { } resource "nomad_job" "nomad_job_alertmanager" { - jobspec = data.template_file.nomad_job_alertmanager.rendered - detach = false + jobspec = data.template_file.nomad_job_alertmanager.rendered + detach = false } \ No newline at end of file diff --git a/fdio.infra.terraform/1n_nmd/alertmanager/variables.tf b/fdio.infra.terraform/1n_nmd/alertmanager/variables.tf index ffedf24f3d..e24ceb64c6 100644 --- a/fdio.infra.terraform/1n_nmd/alertmanager/variables.tf +++ b/fdio.infra.terraform/1n_nmd/alertmanager/variables.tf @@ -2,7 +2,7 @@ variable "nomad_datacenters" { description = "Nomad data centers" type = list(string) - default = [ "dc1" ] + default = ["dc1"] } # Alermanager @@ -38,7 +38,7 @@ variable "alertmanager_use_canary" { variable "alertmanager_vault_secret" { description = "Set of properties to be able to fetch secret from vault" - type = object({ + type = object({ use_vault_provider = bool, vault_kv_policy_name = string, vault_kv_path = string, diff --git a/fdio.infra.terraform/1n_nmd/aws/main.tf b/fdio.infra.terraform/1n_nmd/aws/main.tf index 8084eb4e33..be7eb7c577 100644 --- a/fdio.infra.terraform/1n_nmd/aws/main.tf +++ b/fdio.infra.terraform/1n_nmd/aws/main.tf @@ -1,18 +1,18 @@ resource "vault_aws_secret_backend" "aws" { - access_key = var.aws_access_key - secret_key = var.aws_secret_key - path = "${var.name}-path" + access_key = var.aws_access_key + secret_key = var.aws_secret_key + path = "${var.name}-path" default_lease_ttl_seconds = "43200" max_lease_ttl_seconds = "43200" } resource "vault_aws_secret_backend_role" "admin" { - backend = vault_aws_secret_backend.aws.path - name = "${var.name}-role" - credential_type = "iam_user" + backend = vault_aws_secret_backend.aws.path + name = "${var.name}-role" + credential_type = "iam_user" - policy_document = <= 200 && var.minio_resource_proxy.memory >= 128 @@ -156,11 +156,11 @@ variable "minio_buckets" { variable "minio_upstreams" { description = "List of upstream services (list of object with service_name, port)" - type = list(object({ + type = list(object({ service_name = string, port = number, })) - default = [] + default = [] } variable "mc_extra_commands" { diff --git a/fdio.infra.terraform/1n_nmd/nginx/main.tf b/fdio.infra.terraform/1n_nmd/nginx/main.tf index 025fcb6b8b..fbd48bf645 100644 --- a/fdio.infra.terraform/1n_nmd/nginx/main.tf +++ b/fdio.infra.terraform/1n_nmd/nginx/main.tf @@ -3,8 +3,8 @@ locals { } data "template_file" "nomad_job_nginx" { - template = file("${path.module}/conf/nomad/nginx.hcl") - vars = { + template = file("${path.module}/conf/nomad/nginx.hcl") + vars = { job_name = var.nginx_job_name datacenters = local.datacenters use_host_volume = var.nginx_use_host_volume @@ -13,6 +13,6 @@ data "template_file" "nomad_job_nginx" { } resource "nomad_job" "nomad_job_nginx" { - jobspec = data.template_file.nomad_job_nginx.rendered - detach = false + jobspec = data.template_file.nomad_job_nginx.rendered + detach = false } \ No newline at end of file diff --git a/fdio.infra.terraform/1n_nmd/nginx/variables.tf b/fdio.infra.terraform/1n_nmd/nginx/variables.tf index 0262014049..589ccee94a 100644 --- a/fdio.infra.terraform/1n_nmd/nginx/variables.tf +++ b/fdio.infra.terraform/1n_nmd/nginx/variables.tf @@ -2,7 +2,7 @@ variable "nomad_datacenters" { description = "Nomad data centers" type = list(string) - default = [ "dc1" ] + default = ["dc1"] } variable "nomad_host_volume" { diff --git a/fdio.infra.terraform/1n_nmd/prometheus/main.tf b/fdio.infra.terraform/1n_nmd/prometheus/main.tf index 9506ba3941..0c504c9ca4 100644 --- a/fdio.infra.terraform/1n_nmd/prometheus/main.tf +++ b/fdio.infra.terraform/1n_nmd/prometheus/main.tf @@ -1,5 +1,5 @@ locals { - datacenters = join(",", var.nomad_datacenters) + datacenters = join(",", var.nomad_datacenters) prometheus_url = join("", [ @@ -12,8 +12,8 @@ locals { } data "template_file" "nomad_job_prometheus" { - template = file("${path.module}/conf/nomad/prometheus.hcl") - vars = { + template = file("${path.module}/conf/nomad/prometheus.hcl") + vars = { datacenters = local.datacenters url = local.prometheus_url job_name = var.prometheus_job_name @@ -32,6 +32,6 @@ data "template_file" "nomad_job_prometheus" { } resource "nomad_job" "nomad_job_prometheus" { - jobspec = data.template_file.nomad_job_prometheus.rendered - detach = false + jobspec = data.template_file.nomad_job_prometheus.rendered + detach = false } \ No newline at end of file diff --git a/fdio.infra.terraform/1n_nmd/prometheus/variables.tf b/fdio.infra.terraform/1n_nmd/prometheus/variables.tf index 55ffa33856..befd9412cb 100644 --- a/fdio.infra.terraform/1n_nmd/prometheus/variables.tf +++ b/fdio.infra.terraform/1n_nmd/prometheus/variables.tf @@ -2,7 +2,7 @@ variable "nomad_datacenters" { description = "Nomad data centers" type = list(string) - default = [ "dc1" ] + default = ["dc1"] } variable "nomad_host_volume" { @@ -44,7 +44,7 @@ variable "prometheus_use_canary" { variable "prometheus_vault_secret" { description = "Set of properties to be able to fetch secret from vault" - type = object({ + type = object({ use_vault_provider = bool, vault_kv_policy_name = string, vault_kv_path = string, diff --git a/fdio.infra.terraform/1n_nmd/providers.tf b/fdio.infra.terraform/1n_nmd/providers.tf index c7e0ad8bf3..c3b9ec275d 100644 --- a/fdio.infra.terraform/1n_nmd/providers.tf +++ b/fdio.infra.terraform/1n_nmd/providers.tf @@ -16,6 +16,6 @@ terraform { } provider "nomad" { - address = var.nomad_provider_address - alias = "yul1" + address = var.nomad_provider_address + alias = "yul1" } \ No newline at end of file diff --git a/fdio.infra.terraform/1n_nmd/tools/artifacts.py b/fdio.infra.terraform/1n_nmd/tools/artifacts.py deleted file mode 100755 index 8c0a24af72..0000000000 --- a/fdio.infra.terraform/1n_nmd/tools/artifacts.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/python3 - -# Copyright (c) 2021 Cisco and/or its affiliates. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Storage utilities library.""" - -import argparse -import gzip -import os -from mimetypes import MimeTypes - -from boto3 import resource -from botocore.client import Config - - -ENDPOINT_URL = u"http://storage.service.consul:9000" -AWS_ACCESS_KEY_ID = u"storage" -AWS_SECRET_ACCESS_KEY = u"Storage1234" -REGION_NAME = u"yul1" -COMPRESS_MIME = ( - u"text/html", - u"text/xml", - u"application/octet-stream" -) - - -def compress(src_fpath): - """Compress a single file. - - :param src_fpath: Input file path. - :type src_fpath: str - """ - with open(src_fpath, u"rb") as orig_file: - with gzip.open(f"{src_fpath}.gz", u"wb") as zipped_file: - zipped_file.writelines(orig_file) - - -def upload(storage, bucket, src_fpath, dst_fpath): - """Upload single file to destination bucket. - - :param storage: S3 storage resource. - :param bucket: S3 bucket name. - :param src_fpath: Input file path. - :param dst_fpath: Destination file path on remote storage. - :type storage: Object - :type bucket: str - :type src_fpath: str - :type dst_fpath: str - """ - mime = MimeTypes().guess_type(src_fpath)[0] - if not mime: - mime = "application/octet-stream" - - if mime in COMPRESS_MIME and bucket in "logs": - compress(src_fpath) - src_fpath = f"{src_fpath}.gz" - dst_fpath = f"{dst_fpath}.gz" - - storage.Bucket(f"{bucket}.fd.io").upload_file( - src_fpath, - dst_fpath, - ExtraArgs={ - u"ContentType": mime - } - ) - print(f"https://{bucket}.nginx.service.consul/{dst_fpath}") - - -def upload_recursive(storage, bucket, src_fpath): - """Recursively uploads input folder to destination. - - Example: - - bucket: logs - - src_fpath: /home/user - - dst_fpath: logs.fd.io/home/user - - :param storage: S3 storage resource. - :param bucket: S3 bucket name. - :param src_fpath: Input folder path. - :type storage: Object - :type bucket: str - :type src_fpath: str - """ - for path, _, files in os.walk(src_fpath): - for file in files: - _path = path.replace(src_fpath, u"") - _dir = src_fpath[1:] if src_fpath[0] == "/" else src_fpath - _dst_fpath = os.path.normpath(f"{_dir}/{_path}/{file}") - _src_fpath = os.path.join(path, file) - upload(storage, bucket, _src_fpath, _dst_fpath) - - -def main(): - """Main function for storage manipulation.""" - - parser = argparse.ArgumentParser() - parser.add_argument( - u"-d", u"--dir", required=True, type=str, - help=u"Directory to upload to storage." - ) - parser.add_argument( - u"-b", u"--bucket", required=True, type=str, - help=u"Target bucket on storage." - ) - args = parser.parse_args() - - # Create main storage resource. - storage = resource( - u"s3", - endpoint_url=ENDPOINT_URL, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - config=Config( - signature_version=u"s3v4" - ), - region_name=REGION_NAME - ) - - upload_recursive( - storage=storage, - bucket=args.bucket, - src_fpath=args.dir - ) - - -if __name__ == u"__main__": - main() diff --git a/fdio.infra.terraform/1n_nmd/vpp_device/main.tf b/fdio.infra.terraform/1n_nmd/vpp_device/main.tf index 80c02ec316..89b28ce385 100644 --- a/fdio.infra.terraform/1n_nmd/vpp_device/main.tf +++ b/fdio.infra.terraform/1n_nmd/vpp_device/main.tf @@ -3,8 +3,8 @@ locals { } data "template_file" "nomad_job_csit_shim" { - template = file("${path.module}/conf/nomad/csit_shim.hcl") - vars = { + template = file("${path.module}/conf/nomad/csit_shim.hcl") + vars = { datacenters = local.datacenters job_name = var.csit_shim_job_name group_count = var.csit_shim_group_count @@ -16,6 +16,6 @@ data "template_file" "nomad_job_csit_shim" { } resource "nomad_job" "nomad_job_csit_shim" { - jobspec = data.template_file.nomad_job_csit_shim.rendered - detach = false + jobspec = data.template_file.nomad_job_csit_shim.rendered + detach = false } \ No newline at end of file diff --git a/fdio.infra.terraform/1n_nmd/vpp_device/variables.tf b/fdio.infra.terraform/1n_nmd/vpp_device/variables.tf index 893968bfc0..401be66f27 100644 --- a/fdio.infra.terraform/1n_nmd/vpp_device/variables.tf +++ b/fdio.infra.terraform/1n_nmd/vpp_device/variables.tf @@ -2,7 +2,7 @@ variable "nomad_datacenters" { description = "Nomad data centers" type = list(string) - default = [ "dc1" ] + default = ["dc1"] } # CSIT SHIM -- cgit 1.2.3-korg