diff options
Diffstat (limited to 'fdio.infra.terraform')
31 files changed, 166 insertions, 176 deletions
diff --git a/fdio.infra.terraform/terraform-aws-1n-aws-c5n/variables.tf b/fdio.infra.terraform/terraform-aws-1n-aws-c5n/variables.tf index 7b054d5aa4..43002d1b8a 100644 --- a/fdio.infra.terraform/terraform-aws-1n-aws-c5n/variables.tf +++ b/fdio.infra.terraform/terraform-aws-1n-aws-c5n/variables.tf @@ -38,7 +38,7 @@ variable "placement_group_strategy" { variable "tg_ami" { description = "AMI to use for the instance." type = string - default = "ami-04f5fbbd9880e691a" + default = "ami-06ea2ad776e2a8461" } variable "tg_associate_public_ip_address" { diff --git a/fdio.infra.terraform/terraform-aws-1n-c6in/variables.tf b/fdio.infra.terraform/terraform-aws-1n-c6in/variables.tf index efdbe912e2..923ae7fb74 100644 --- a/fdio.infra.terraform/terraform-aws-1n-c6in/variables.tf +++ b/fdio.infra.terraform/terraform-aws-1n-c6in/variables.tf @@ -38,7 +38,7 @@ variable "placement_group_strategy" { variable "tg_ami" { description = "AMI to use for the instance." type = string - default = "ami-04f5fbbd9880e691a" + default = "ami-06ea2ad776e2a8461" } variable "tg_associate_public_ip_address" { diff --git a/fdio.infra.terraform/terraform-aws-2n-aws-c5n/variables.tf b/fdio.infra.terraform/terraform-aws-2n-aws-c5n/variables.tf index a498645df9..f85ae03dc2 100644 --- a/fdio.infra.terraform/terraform-aws-2n-aws-c5n/variables.tf +++ b/fdio.infra.terraform/terraform-aws-2n-aws-c5n/variables.tf @@ -38,7 +38,7 @@ variable "placement_group_strategy" { variable "tg_ami" { description = "AMI to use for the instance." type = string - default = "ami-04f5fbbd9880e691a" + default = "ami-06ea2ad776e2a8461" } variable "tg_associate_public_ip_address" { diff --git a/fdio.infra.terraform/terraform-aws-2n-c6in/variables.tf b/fdio.infra.terraform/terraform-aws-2n-c6in/variables.tf index b901fca8d9..fea82b0a5c 100644 --- a/fdio.infra.terraform/terraform-aws-2n-c6in/variables.tf +++ b/fdio.infra.terraform/terraform-aws-2n-c6in/variables.tf @@ -50,7 +50,7 @@ variable "placement_group_strategy" { variable "tg_ami" { description = "AMI to use for the instance." type = string - default = "ami-0c18f927fc6ac6c52" + default = "ami-06ea2ad776e2a8461" } variable "tg_associate_public_ip_address" { diff --git a/fdio.infra.terraform/terraform-aws-2n-c7gn/variables.tf b/fdio.infra.terraform/terraform-aws-2n-c7gn/variables.tf index 0cc9252c68..891b2054c0 100644 --- a/fdio.infra.terraform/terraform-aws-2n-c7gn/variables.tf +++ b/fdio.infra.terraform/terraform-aws-2n-c7gn/variables.tf @@ -38,7 +38,7 @@ variable "placement_group_strategy" { variable "tg_ami" { description = "AMI to use for the instance." type = string - default = "ami-04f5fbbd9880e691a" + default = "ami-09a82dc14ad8658ce" } variable "tg_associate_public_ip_address" { @@ -74,7 +74,7 @@ variable "tg_source_dest_check" { variable "sut1_ami" { description = "AMI to use for the instance." type = string - default = "ami-0c0f39556bbb626b2" + default = "ami-0f79fc4ee6157a1b4" } variable "sut1_associate_public_ip_address" { diff --git a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/variables.tf b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/variables.tf index b706d77f00..9002e55b8b 100644 --- a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/variables.tf +++ b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/variables.tf @@ -1,7 +1,7 @@ variable "region" { description = "AWS Region." type = string - default = "eu-north-1" + default = "us-east-1" } variable "vault_provider_address" { @@ -30,7 +30,7 @@ variable "vault_name" { variable "application_bucket" { description = "The name of the bucket." type = string - default = "elasticbeanstalk-eu-north-1-407116685360" + default = "elasticbeanstalk-us-east-1-407116685360" } variable "application_description" { @@ -42,7 +42,7 @@ variable "application_description" { variable "application_name" { description = "Name of the Beanstalk Application." type = string - default = "fdio-csit-dash-app-m7g" + default = "fdio-csit-dash-app-m8g" } variable "application_source" { diff --git a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/versions.tf b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/versions.tf index cd4761e34c..95a01e3a64 100644 --- a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/versions.tf +++ b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/versions.tf @@ -2,10 +2,10 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.31.0" + version = ">= 5.84.0" } vault = { - version = ">= 3.23.0" + version = ">= 4.6.0" } } required_version = ">= 1.4.2" diff --git a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-env/main.tf b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-env/main.tf index 9f115cd6c6..53413c55dc 100644 --- a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-env/main.tf +++ b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-env/main.tf @@ -1,5 +1,6 @@ -data "vault_generic_secret" "fdio_docs" { - path = "kv/secret/data/etl/fdio_docs" +data "vault_kv_secret_v2" "csit_docs" { + mount = "kv" + name = "etl/csit_docs" } data "vault_aws_access_credentials" "creds" { @@ -11,8 +12,8 @@ module "elastic_beanstalk_application" { source = "../terraform-aws-elastic-beanstalk-application" # application - application_description = "FD.io CDASH M7G" - application_name = "fdio-csit-dash-app-m7g" + application_description = "FD.io CDASH M8G" + application_name = "fdio-csit-dash-app-m8g" appversion_lifecycle_service_role_arn = "arn:aws:iam::407116685360:role/aws-service-role/elasticbeanstalk.amazonaws.com/AWSServiceRoleForElasticBeanstalk" appversion_lifecycle_max_count = 10 appversion_lifecycle_delete_source_from_s3 = false @@ -22,7 +23,7 @@ module "elastic_beanstalk_environment" { source = "../terraform-aws-elastic-beanstalk-environment" # environment - application_name = "fdio-csit-dash-app-m7g" + application_name = "fdio-csit-dash-app-m8g" # vpc vpc_cidr_block = "10.0.0.0/16" @@ -31,22 +32,22 @@ module "elastic_beanstalk_environment" { vpc_instance_tenancy = "default" # subnet - subnet_a_availability_zone = "eu-north-1a" + subnet_a_availability_zone = "us-east-1a" subnet_a_cidr_block = "10.0.0.0/20" - subnet_b_availability_zone = "eu-north-1b" + subnet_b_availability_zone = "us-east-1b" subnet_b_cidr_block = "10.0.16.0/20" # environment environment_application = module.elastic_beanstalk_application.application_name environment_description = module.elastic_beanstalk_application.application_description - environment_name = "fdio-csit-dash-env-m7g" - environment_solution_stack_name = "64bit Amazon Linux 2023 v4.0.6 running Python 3.11" + environment_name = "fdio-csit-dash-env-m8g" + environment_solution_stack_name = "64bit Amazon Linux 2023 v4.3.2 running Python 3.12" environment_tier = "WebServer" environment_wait_for_ready_timeout = "25m" environment_version_label = "" # aws:ec2:instances - instances_instance_types = "m7g.2xlarge" + instances_instance_types = "m8g.2xlarge" # aws:ec2:vpc associate_public_ip_address = true @@ -57,7 +58,7 @@ module "elastic_beanstalk_environment" { # aws:elasticbeanstalk:environment environment_loadbalancer_type = "application" - environment_loadbalancer_ssl_certificate_id = "arn:aws:acm:eu-north-1:407116685360:certificate/3ef3c6ae-f1d4-49f0-a8cd-5d090991bf73" + environment_loadbalancer_ssl_certificate_id = "arn:aws:acm:us-east-1:407116685360:certificate/3439b8aa-4f07-496f-a2de-e38159050c85" # aws:elasticbeanstalk:environment:process:default environment_process_default_healthcheck_interval = 10 @@ -117,8 +118,8 @@ module "elastic_beanstalk_environment" { # aws:elasticbeanstalk:application:environment environment_variables = { - "AWS_ACCESS_KEY_ID" = data.vault_generic_secret.fdio_docs.data["access_key"] - "AWS_SECRET_ACCESS_KEY" = data.vault_generic_secret.fdio_docs.data["secret_key"] - "AWS_DEFAULT_REGION" = data.vault_generic_secret.fdio_docs.data["region"] + "AWS_ACCESS_KEY_ID" = data.vault_kv_secret_v2.csit_docs.data["access_key"] + "AWS_SECRET_ACCESS_KEY" = data.vault_kv_secret_v2.csit_docs.data["secret_key"] + "AWS_DEFAULT_REGION" = data.vault_kv_secret_v2.csit_docs.data["region"] } } diff --git a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-env/variables.tf b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-env/variables.tf index d7ff1d19ff..968cd4c962 100644 --- a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-env/variables.tf +++ b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-env/variables.tf @@ -1,7 +1,7 @@ variable "region" { description = "AWS Region." type = string - default = "eu-north-1" + default = "us-east-1" } variable "vault_provider_address" { diff --git a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-env/versions.tf b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-env/versions.tf index 153890e01b..356c0aa582 100644 --- a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-env/versions.tf +++ b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-env/versions.tf @@ -2,16 +2,16 @@ terraform { backend "consul" { address = "10.30.51.24:8500" scheme = "http" - path = "terraform/dash_m7g" + path = "terraform/dash_m8g" } required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.31.0" + version = ">= 5.83.1" } vault = { - version = ">= 3.23.0" + version = ">= 4.6.0" } } - required_version = ">= 1.4.2" + required_version = ">= 1.10.4" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2410.hcl.tftpl deleted file mode 100644 index 96fef71e17..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2410.hcl.tftpl +++ /dev/null @@ -1,59 +0,0 @@ -job "${job_name}" { - datacenters = ["${datacenters}"] - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_device_rls2410.py" - destination = "local/" - } - artifact { - source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_device.json" - destination = "local/" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_device_rls2410.py" - ] - work_dir = "/local" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack.hcl.tftpl index 974e915101..8aa364388d 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2410.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack.hcl.tftpl @@ -21,7 +21,7 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_hoststack_rls2410.py" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_hoststack.py" destination = "local/" } artifact { @@ -37,7 +37,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_hoststack_rls2410.py" + "coverage_hoststack.py" ] work_dir = "/local" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr.hcl.tftpl index c2c282752b..1a2db579a3 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2410.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr.hcl.tftpl @@ -21,7 +21,7 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_mrr_rls2410.py" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_mrr.py" destination = "local/" } artifact { @@ -37,7 +37,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_mrr_rls2410.py" + "coverage_mrr.py" ] work_dir = "/local" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr.hcl.tftpl index ee4771b0b3..b294c390cd 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2410.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr.hcl.tftpl @@ -21,7 +21,7 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_ndrpdr_rls2410.py" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_ndrpdr.py" destination = "local/" } artifact { @@ -37,7 +37,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_ndrpdr_rls2410.py" + "coverage_ndrpdr.py" ] work_dir = "/local" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf.hcl.tftpl index d31ebd81de..83bbceb109 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2410.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf.hcl.tftpl @@ -21,7 +21,7 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_reconf_rls2410.py" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_reconf.py" destination = "local/" } artifact { @@ -37,7 +37,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_reconf_rls2410.py" + "coverage_reconf.py" ] work_dir = "/local" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak.hcl.tftpl index 9d71dd8b9b..871e4f79a3 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2410.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak.hcl.tftpl @@ -21,7 +21,7 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_soak_rls2410.py" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_soak.py" destination = "local/" } artifact { @@ -37,7 +37,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "coverage_soak_rls2410.py" + "coverage_soak.py" ] work_dir = "/local" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack.hcl.tftpl index 79397eda20..c7a9f60730 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2410.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack.hcl.tftpl @@ -21,7 +21,7 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_hoststack_rls2410.py" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_hoststack.py" destination = "local/" } artifact { @@ -37,7 +37,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_hoststack_rls2410.py" + "iterative_hoststack.py" ] work_dir = "/local" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr.hcl.tftpl index c1c298d4a4..3274596680 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2410.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr.hcl.tftpl @@ -21,7 +21,7 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_mrr_rls2410.py" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_mrr.py" destination = "local/" } artifact { @@ -37,7 +37,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_mrr_rls2410.py" + "iterative_mrr.py" ] work_dir = "/local" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr.hcl.tftpl index 42202f30e2..0d9f4bb250 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2410.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr.hcl.tftpl @@ -21,7 +21,7 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_ndrpdr_rls2410.py" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_ndrpdr.py" destination = "local/" } artifact { @@ -37,7 +37,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_ndrpdr_rls2410.py" + "iterative_ndrpdr.py" ] work_dir = "/local" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf.hcl.tftpl index 4a7773ca46..5045edbb33 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2410.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf.hcl.tftpl @@ -21,7 +21,7 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_reconf_rls2410.py" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_reconf.py" destination = "local/" } artifact { @@ -37,7 +37,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_reconf_rls2410.py" + "iterative_reconf.py" ] work_dir = "/local" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak.hcl.tftpl index 3759f4bb46..c569a43b2f 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2410.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak.hcl.tftpl @@ -21,7 +21,7 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_soak_rls2410.py" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_soak.py" destination = "local/" } artifact { @@ -37,7 +37,7 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_soak_rls2410.py" + "iterative_soak.py" ] work_dir = "/local" } diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf index 57baa24276..b9027a8ceb 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf @@ -3,9 +3,9 @@ data "vault_kv_secret_v2" "fdio_logs" { name = "etl/fdio_logs" } -data "vault_kv_secret_v2" "fdio_docs" { +data "vault_kv_secret_v2" "csit_docs" { mount = "kv" - name = "etl/fdio_docs" + name = "etl/csit_docs" } module "etl" { @@ -18,9 +18,9 @@ module "etl" { aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data.access_key aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data.secret_key aws_default_region = data.vault_kv_secret_v2.fdio_logs.data.region - out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data.access_key - out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data.secret_key - out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data.region + out_aws_access_key_id = data.vault_kv_secret_v2.csit_docs.data.access_key + out_aws_secret_access_key = data.vault_kv_secret_v2.csit_docs.data.secret_key + out_aws_default_region = data.vault_kv_secret_v2.csit_docs.data.region cron = "0 30 0 * * * *" datacenters = ["yul1"] job_name = each.key diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf index e8ddeb086c..d0d9fa171f 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf @@ -55,51 +55,47 @@ variable "nomad_jobs" { memory = 50000 }, { - job_name = "etl-trending-hoststack" + job_name = "etl-iterative-hoststack" memory = 50000 }, { - job_name = "etl-iterative-hoststack-rls2410" + job_name = "etl-iterative-mrr" memory = 50000 }, { - job_name = "etl-iterative-mrr-rls2410" + job_name = "etl-iterative-ndrpdr" memory = 50000 }, { - job_name = "etl-iterative-ndrpdr-rls2410" + job_name = "etl-iterative-reconf" memory = 50000 }, { - job_name = "etl-iterative-reconf-rls2410" + job_name = "etl-iterative-soak" memory = 50000 }, { - job_name = "etl-iterative-soak-rls2410" + job_name = "etl-coverage-hoststack" memory = 50000 }, { - job_name = "etl-coverage-device-rls2410" + job_name = "etl-coverage-mrr" memory = 50000 }, { - job_name = "etl-coverage-hoststack-rls2410" + job_name = "etl-coverage-ndrpdr" memory = 50000 }, { - job_name = "etl-coverage-mrr-rls2410" + job_name = "etl-coverage-reconf" memory = 50000 }, { - job_name = "etl-coverage-ndrpdr-rls2410" + job_name = "etl-coverage-soak" memory = 50000 }, { - job_name = "etl-coverage-reconf-rls2410" - memory = 50000 - }, - { - job_name = "etl-coverage-soak-rls2410" + job_name = "etl-trending-hoststack" memory = 50000 }, { diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf index ffe25bb42e..cf67cc25c7 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf @@ -10,8 +10,8 @@ terraform { version = ">= 2.3.0" } vault = { - version = ">= 4.3.0" + version = ">= 4.6.0" } } - required_version = ">= 1.5.4" + required_version = ">= 1.10.4" } diff --git a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/main.tf b/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/main.tf deleted file mode 100644 index 08c3ca8b73..0000000000 --- a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/main.tf +++ /dev/null @@ -1,23 +0,0 @@ -module "fdio-logs" { - # fdio logs iam - source = "../" - name = "dynamic-aws-creds-vault-fdio-logs" - aws_access_key = var.aws_access_key - aws_secret_key = var.aws_secret_key -} - -module "fdio-docs" { - # fdio docs iam - source = "../" - name = "dynamic-aws-creds-vault-fdio-docs" - aws_access_key = var.aws_access_key - aws_secret_key = var.aws_secret_key -} - -module "fdio-csit-jenkins" { - # fdio csit jenkins iam - source = "../" - name = "dynamic-aws-creds-vault-fdio-csit-jenkins" - aws_access_key = var.aws_access_key - aws_secret_key = var.aws_secret_key -} diff --git a/fdio.infra.terraform/terraform-vault-aws-secret-backend/main.tf b/fdio.infra.terraform/terraform-vault-aws-secret-backend/main.tf index 814121986f..6a2d42e681 100644 --- a/fdio.infra.terraform/terraform-vault-aws-secret-backend/main.tf +++ b/fdio.infra.terraform/terraform-vault-aws-secret-backend/main.tf @@ -1,4 +1,4 @@ -resource "vault_aws_secret_backend" "aws" { +resource "vault_aws_secret_backend" "aws_secret_backend" { access_key = var.aws_access_key secret_key = var.aws_secret_key path = "${var.name}-path" @@ -7,34 +7,18 @@ resource "vault_aws_secret_backend" "aws" { max_lease_ttl_seconds = "0" } -resource "vault_aws_secret_backend_role" "admin" { - backend = vault_aws_secret_backend.aws.path +resource "vault_aws_secret_backend_role" "aws_secret_backend_role" { + backend = vault_aws_secret_backend.aws_secret_backend.path name = "${var.name}-role" credential_type = "iam_user" - policy_document = <<EOF -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "iam:*", - "ec2:*", - "s3:*", - "elasticbeanstalk:*" - ], - "Resource": "*" - } - ] -} -EOF + policy_document = var.policy_document } output "backend" { - value = vault_aws_secret_backend.aws.path + value = vault_aws_secret_backend.aws_secret_backend.path } output "role" { - value = vault_aws_secret_backend_role.admin.name + value = vault_aws_secret_backend_role.aws_secret_backend_role.name } diff --git a/fdio.infra.terraform/terraform-vault-aws-secret-backend/variables.tf b/fdio.infra.terraform/terraform-vault-aws-secret-backend/variables.tf index 2545345185..d7a2f4a987 100644 --- a/fdio.infra.terraform/terraform-vault-aws-secret-backend/variables.tf +++ b/fdio.infra.terraform/terraform-vault-aws-secret-backend/variables.tf @@ -15,3 +15,8 @@ variable "name" { description = "Vault path" type = string } + +variable "policy_document" { + description = "AWS policy document" + type = string +} diff --git a/fdio.infra.terraform/terraform-vault-fdio-creds/main.tf b/fdio.infra.terraform/terraform-vault-fdio-creds/main.tf new file mode 100644 index 0000000000..4469bb131c --- /dev/null +++ b/fdio.infra.terraform/terraform-vault-fdio-creds/main.tf @@ -0,0 +1,86 @@ +module "fdio-logs" { + # fdio logs iam + source = "../terraform-vault-aws-secret-backend" + name = "dynamic-aws-creds-vault-fdio-logs" + aws_access_key = var.aws_access_key + aws_secret_key = var.aws_secret_key + policy_document = jsonencode({ + Statement = [ + { + Action = [ + "iam:*", + "ec2:*", + "s3:*", + "elasticbeanstalk:*", + "ssm:*", + "cloudformation:*", + "logs:*", + "elasticloadbalancing:*", + "autoscaling:*", + "cloudwatch:*" + ] + Effect = "Allow" + Resource = "*" + }, + ] + Version = "2012-10-17" + }) +} + +module "csit-cdash" { + # csit cdash iam + source = "../terraform-vault-aws-secret-backend" + name = "dynamic-aws-creds-vault-cdash" + aws_access_key = var.aws_access_key + aws_secret_key = var.aws_secret_key + policy_document = jsonencode({ + Statement = [ + { + Action = [ + "iam:*", + "ec2:*", + "s3:*", + "elasticbeanstalk:*", + "ssm:*", + "cloudformation:*", + "logs:*", + "elasticloadbalancing:*", + "autoscaling:*", + "cloudwatch:*" + ] + Effect = "Allow" + Resource = "*" + }, + ] + Version = "2012-10-17" + }) +} + +module "fdio-csit-jenkins" { + # fdio csit jenkins iam + source = "../terraform-vault-aws-secret-backend" + name = "dynamic-aws-creds-vault-fdio-csit-jenkins" + aws_access_key = var.aws_access_key + aws_secret_key = var.aws_secret_key + policy_document = jsonencode({ + Statement = [ + { + Action = [ + "iam:*", + "ec2:*", + "s3:*", + "elasticbeanstalk:*", + "ssm:*", + "cloudformation:*", + "logs:*", + "elasticloadbalancing:*", + "autoscaling:*", + "cloudwatch:*" + ] + Effect = "Allow" + Resource = "*" + }, + ] + Version = "2012-10-17" + }) +} diff --git a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/providers.tf b/fdio.infra.terraform/terraform-vault-fdio-creds/providers.tf index 102fd31b87..102fd31b87 100644 --- a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/providers.tf +++ b/fdio.infra.terraform/terraform-vault-fdio-creds/providers.tf diff --git a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/variables.tf b/fdio.infra.terraform/terraform-vault-fdio-creds/variables.tf index b1f64eccf2..b1f64eccf2 100644 --- a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/variables.tf +++ b/fdio.infra.terraform/terraform-vault-fdio-creds/variables.tf diff --git a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/versions.tf b/fdio.infra.terraform/terraform-vault-fdio-creds/versions.tf index c573731d65..c573731d65 100644 --- a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/versions.tf +++ b/fdio.infra.terraform/terraform-vault-fdio-creds/versions.tf |