diff options
Diffstat (limited to 'fdio.infra.terraform')
52 files changed, 824 insertions, 923 deletions
diff --git a/fdio.infra.terraform/terraform-aws-1n-aws-c5n/providers.tf b/fdio.infra.terraform/terraform-aws-1n-aws-c5n/providers.tf index 2482ca2839..485f0ebe30 100644 --- a/fdio.infra.terraform/terraform-aws-1n-aws-c5n/providers.tf +++ b/fdio.infra.terraform/terraform-aws-1n-aws-c5n/providers.tf @@ -7,5 +7,5 @@ provider "aws" { provider "vault" { address = "http://10.30.51.24:8200" skip_tls_verify = true - token = "s.4z5PsufFwV3sHbCzK9Y2Cojd" + token = "hvs.bzHw4ZHsz9B0019P8I73yS6l" }
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-aws-1n-c6in/providers.tf b/fdio.infra.terraform/terraform-aws-1n-c6in/providers.tf index 2482ca2839..485f0ebe30 100644 --- a/fdio.infra.terraform/terraform-aws-1n-c6in/providers.tf +++ b/fdio.infra.terraform/terraform-aws-1n-c6in/providers.tf @@ -7,5 +7,5 @@ provider "aws" { provider "vault" { address = "http://10.30.51.24:8200" skip_tls_verify = true - token = "s.4z5PsufFwV3sHbCzK9Y2Cojd" + token = "hvs.bzHw4ZHsz9B0019P8I73yS6l" }
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-aws-2n-aws-c5n/providers.tf b/fdio.infra.terraform/terraform-aws-2n-aws-c5n/providers.tf index 2482ca2839..485f0ebe30 100644 --- a/fdio.infra.terraform/terraform-aws-2n-aws-c5n/providers.tf +++ b/fdio.infra.terraform/terraform-aws-2n-aws-c5n/providers.tf @@ -7,5 +7,5 @@ provider "aws" { provider "vault" { address = "http://10.30.51.24:8200" skip_tls_verify = true - token = "s.4z5PsufFwV3sHbCzK9Y2Cojd" + token = "hvs.bzHw4ZHsz9B0019P8I73yS6l" }
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-aws-2n-c6gn/providers.tf b/fdio.infra.terraform/terraform-aws-2n-c6gn/providers.tf index d0e7490d38..1a98cb50df 100644 --- a/fdio.infra.terraform/terraform-aws-2n-c6gn/providers.tf +++ b/fdio.infra.terraform/terraform-aws-2n-c6gn/providers.tf @@ -7,5 +7,5 @@ provider "aws" { provider "vault" { address = "http://vault.service.consul:8200" skip_tls_verify = true - token = "s.4z5PsufFwV3sHbCzK9Y2Cojd" + token = "hvs.bzHw4ZHsz9B0019P8I73yS6l" }
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-aws-2n-c6in/providers.tf b/fdio.infra.terraform/terraform-aws-2n-c6in/providers.tf index d0e7490d38..1a98cb50df 100644 --- a/fdio.infra.terraform/terraform-aws-2n-c6in/providers.tf +++ b/fdio.infra.terraform/terraform-aws-2n-c6in/providers.tf @@ -7,5 +7,5 @@ provider "aws" { provider "vault" { address = "http://vault.service.consul:8200" skip_tls_verify = true - token = "s.4z5PsufFwV3sHbCzK9Y2Cojd" + token = "hvs.bzHw4ZHsz9B0019P8I73yS6l" }
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-aws-2n-c6in/variables.tf b/fdio.infra.terraform/terraform-aws-2n-c6in/variables.tf index 51af9587d9..4835b7afba 100644 --- a/fdio.infra.terraform/terraform-aws-2n-c6in/variables.tf +++ b/fdio.infra.terraform/terraform-aws-2n-c6in/variables.tf @@ -50,7 +50,7 @@ variable "placement_group_strategy" { variable "tg_ami" { description = "AMI to use for the instance." type = string - default = "ami-07430bfa17fd4e597" + default = "ami-0285a978d8bd77dcb" } variable "tg_associate_public_ip_address" { @@ -86,7 +86,7 @@ variable "tg_source_dest_check" { variable "sut1_ami" { description = "AMI to use for the instance." type = string - default = "ami-0a890555652963ec2" + default = "ami-0f3ecd7f56fc232bd" } variable "sut1_associate_public_ip_address" { diff --git a/fdio.infra.terraform/terraform-aws-2n-c7gn/providers.tf b/fdio.infra.terraform/terraform-aws-2n-c7gn/providers.tf index d0e7490d38..1a98cb50df 100644 --- a/fdio.infra.terraform/terraform-aws-2n-c7gn/providers.tf +++ b/fdio.infra.terraform/terraform-aws-2n-c7gn/providers.tf @@ -7,5 +7,5 @@ provider "aws" { provider "vault" { address = "http://vault.service.consul:8200" skip_tls_verify = true - token = "s.4z5PsufFwV3sHbCzK9Y2Cojd" + token = "hvs.bzHw4ZHsz9B0019P8I73yS6l" }
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-aws-2n-c7gn/variables.tf b/fdio.infra.terraform/terraform-aws-2n-c7gn/variables.tf index 2a80b86936..1b9d375c24 100644 --- a/fdio.infra.terraform/terraform-aws-2n-c7gn/variables.tf +++ b/fdio.infra.terraform/terraform-aws-2n-c7gn/variables.tf @@ -5,7 +5,7 @@ variable "vault-name" { variable "region" { description = "AWS Region." type = string - default = "eu-west-1" + default = "us-east-1" } variable "resource_prefix" { @@ -38,7 +38,7 @@ variable "placement_group_strategy" { variable "tg_ami" { description = "AMI to use for the instance." type = string - default = "ami-0b5aa26a6e4192705" + default = "ami-0243e38bc8ac1f8a3" } variable "tg_associate_public_ip_address" { @@ -74,7 +74,7 @@ variable "tg_source_dest_check" { variable "sut1_ami" { description = "AMI to use for the instance." type = string - default = "ami-08930f71bd0be1085" + default = "ami-0c0f39556bbb626b2" } variable "sut1_associate_public_ip_address" { diff --git a/fdio.infra.terraform/terraform-aws-3n-aws-c5n/providers.tf b/fdio.infra.terraform/terraform-aws-3n-aws-c5n/providers.tf index 2482ca2839..485f0ebe30 100644 --- a/fdio.infra.terraform/terraform-aws-3n-aws-c5n/providers.tf +++ b/fdio.infra.terraform/terraform-aws-3n-aws-c5n/providers.tf @@ -7,5 +7,5 @@ provider "aws" { provider "vault" { address = "http://10.30.51.24:8200" skip_tls_verify = true - token = "s.4z5PsufFwV3sHbCzK9Y2Cojd" + token = "hvs.bzHw4ZHsz9B0019P8I73yS6l" }
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-aws-3n-c6gn/providers.tf b/fdio.infra.terraform/terraform-aws-3n-c6gn/providers.tf index 2482ca2839..485f0ebe30 100644 --- a/fdio.infra.terraform/terraform-aws-3n-c6gn/providers.tf +++ b/fdio.infra.terraform/terraform-aws-3n-c6gn/providers.tf @@ -7,5 +7,5 @@ provider "aws" { provider "vault" { address = "http://10.30.51.24:8200" skip_tls_verify = true - token = "s.4z5PsufFwV3sHbCzK9Y2Cojd" + token = "hvs.bzHw4ZHsz9B0019P8I73yS6l" }
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-aws-3n-c6in/providers.tf b/fdio.infra.terraform/terraform-aws-3n-c6in/providers.tf index 2482ca2839..485f0ebe30 100644 --- a/fdio.infra.terraform/terraform-aws-3n-c6in/providers.tf +++ b/fdio.infra.terraform/terraform-aws-3n-c6in/providers.tf @@ -7,5 +7,5 @@ provider "aws" { provider "vault" { address = "http://10.30.51.24:8200" skip_tls_verify = true - token = "s.4z5PsufFwV3sHbCzK9Y2Cojd" + token = "hvs.bzHw4ZHsz9B0019P8I73yS6l" }
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-aws-3n-c6in/variables.tf b/fdio.infra.terraform/terraform-aws-3n-c6in/variables.tf index 3255cab3f3..084a9da390 100644 --- a/fdio.infra.terraform/terraform-aws-3n-c6in/variables.tf +++ b/fdio.infra.terraform/terraform-aws-3n-c6in/variables.tf @@ -38,7 +38,7 @@ variable "placement_group_strategy" { variable "tg_ami" { description = "AMI to use for the instance." type = string - default = "ami-07430bfa17fd4e597" + default = "ami-0285a978d8bd77dcb" } variable "tg_associate_public_ip_address" { @@ -74,7 +74,7 @@ variable "tg_source_dest_check" { variable "sut1_ami" { description = "AMI to use for the instance." type = string - default = "ami-0a890555652963ec2" + default = "ami-0f3ecd7f56fc232bd" } variable "sut1_associate_public_ip_address" { diff --git a/fdio.infra.terraform/terraform-aws-3n-c7gn/providers.tf b/fdio.infra.terraform/terraform-aws-3n-c7gn/providers.tf index 2482ca2839..485f0ebe30 100644 --- a/fdio.infra.terraform/terraform-aws-3n-c7gn/providers.tf +++ b/fdio.infra.terraform/terraform-aws-3n-c7gn/providers.tf @@ -7,5 +7,5 @@ provider "aws" { provider "vault" { address = "http://10.30.51.24:8200" skip_tls_verify = true - token = "s.4z5PsufFwV3sHbCzK9Y2Cojd" + token = "hvs.bzHw4ZHsz9B0019P8I73yS6l" }
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/variables.tf b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/variables.tf index 653a92b970..b706d77f00 100644 --- a/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/variables.tf +++ b/fdio.infra.terraform/terraform-aws-fdio-csit-dash-app-base/variables.tf @@ -19,7 +19,7 @@ variable "vault_provider_skip_tls_verify" { variable "vault_provider_token" { description = "Vault root token." type = string - default = "s.4z5PsufFwV3sHbCzK9Y2Cojd" + default = "hvs.bzHw4ZHsz9B0019P8I73yS6l" } variable "vault_name" { diff --git a/fdio.infra.terraform/terraform-nomad-nomad-job/main.tf b/fdio.infra.terraform/terraform-nomad-nomad-job/main.tf new file mode 100644 index 0000000000..fc604fec7d --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-nomad-job/main.tf @@ -0,0 +1,33 @@ +locals { + datacenters = join(",", var.datacenters) + envs = join("\n", concat([], var.envs)) +} + +resource "nomad_job" "nomad_job" { + jobspec = templatefile( + "${path.cwd}/conf/nomad/${var.job_name}.hcl.tftpl", + { + aws_access_key_id = var.aws_access_key_id, + aws_secret_access_key = var.aws_secret_access_key, + aws_default_region = var.aws_default_region + cpu = var.cpu, + cron = var.cron, + datacenters = local.datacenters, + envs = local.envs, + image = var.image, + job_name = var.job_name, + memory = var.memory, + out_aws_access_key_id = var.out_aws_access_key_id, + out_aws_secret_access_key = var.out_aws_secret_access_key, + out_aws_default_region = var.out_aws_default_region + prohibit_overlap = var.prohibit_overlap, + time_zone = var.time_zone, + type = var.type, + use_vault_provider = var.vault_secret.use_vault_provider, + vault_kv_policy_name = var.vault_secret.vault_kv_policy_name, + vault_kv_path = var.vault_secret.vault_kv_path, + vault_kv_field_access_key = var.vault_secret.vault_kv_field_access_key, + vault_kv_field_secret_key = var.vault_secret.vault_kv_field_secret_key + }) + detach = false +} diff --git a/fdio.infra.terraform/terraform-nomad-nomad-job/variables.tf b/fdio.infra.terraform/terraform-nomad-nomad-job/variables.tf new file mode 100644 index 0000000000..86d1b45753 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-nomad-job/variables.tf @@ -0,0 +1,115 @@ +# Nomad +variable "datacenters" { + description = "Specifies the list of DCs to be considered placing this task." + type = list(string) + default = ["dc1"] +} + +# ETL +variable "aws_access_key_id" { + description = "AWS access key." + type = string + default = "aws" +} + +variable "aws_secret_access_key" { + description = "AWS secret key" + type = string + default = "aws" +} + +variable "aws_default_region" { + description = "AWS region" + type = string + default = "aws" +} + +variable "cpu" { + description = "Specifies the CPU required to run this task in MHz." + type = number + default = 10000 +} + +variable "cron" { + description = "Specifies a cron expression configuring the interval to launch." + type = string + default = "@daily" +} + +variable "envs" { + description = "Specifies ETL environment variables." + type = list(string) + default = [] +} + +variable "image" { + description = "Specifies the Docker image to run." + type = string + default = "pmikus/docker-ubuntu-focal-aws-glue:latest" +} + +variable "job_name" { + description = "Specifies a name for the job." + type = string + default = "etl" +} + +variable "memory" { + description = "Specifies the memory required in MB." + type = number + default = 50000 +} + +variable "out_aws_access_key_id" { + description = "AWS access key." + type = string + default = "aws" +} + +variable "out_aws_secret_access_key" { + description = "AWS secret key" + type = string + default = "aws" +} + +variable "out_aws_default_region" { + description = "AWS region" + type = string + default = "aws" +} + +variable "prohibit_overlap" { + description = "Specifies if this job should wait until previous completed." + type = bool + default = true +} + +variable "time_zone" { + description = "Specifies the time zone to evaluate the next launch interval." + type = string + default = "UTC" +} + +variable "type" { + description = "Specifies the Nomad scheduler to use." + type = string + default = "batch" +} + +variable "vault_secret" { + type = object({ + use_vault_provider = bool, + vault_kv_policy_name = string, + vault_kv_path = string, + vault_kv_field_access_key = string, + vault_kv_field_secret_key = string + }) + description = "Set of properties to be able to fetch secret from vault." + default = { + use_vault_provider = true + vault_kv_policy_name = "kv" + vault_kv_path = "data/etl" + vault_kv_field_access_key = "access_key" + vault_kv_field_secret_key = "secret_key" + } +} diff --git a/fdio.infra.terraform/terraform-nomad-nomad-job/versions.tf b/fdio.infra.terraform/terraform-nomad-nomad-job/versions.tf new file mode 100644 index 0000000000..f40435fe77 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-nomad-job/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + nomad = { + source = "hashicorp/nomad" + version = ">= 1.4.20" + } + } + required_version = ">= 1.5.4" +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2402.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2402.hcl.tftpl deleted file mode 100644 index cc0b1df8b5..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2402.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_device_rls2402.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -}
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2410.hcl.tftpl new file mode 100644 index 0000000000..96fef71e17 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2410.hcl.tftpl @@ -0,0 +1,59 @@ +job "${job_name}" { + datacenters = ["${datacenters}"] + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_device_rls2410.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_device.json" + destination = "local/" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "coverage_device_rls2410.py" + ] + work_dir = "/local" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2402.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2402.hcl.tftpl deleted file mode 100644 index 95d7a4c46e..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2402.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_hoststack_rls2402.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -}
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2410.hcl.tftpl new file mode 100644 index 0000000000..974e915101 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2410.hcl.tftpl @@ -0,0 +1,59 @@ +job "${job_name}" { + datacenters = ["${datacenters}"] + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_hoststack_rls2410.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_hoststack.json" + destination = "local/" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "coverage_hoststack_rls2410.py" + ] + work_dir = "/local" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2402.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2402.hcl.tftpl deleted file mode 100644 index 3bab9264fa..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2402.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_mrr_rls2402.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -}
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2402.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2410.hcl.tftpl index 74478c59f7..c2c282752b 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2402.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2410.hcl.tftpl @@ -1,5 +1,5 @@ job "${job_name}" { - datacenters = "${datacenters}" + datacenters = ["${datacenters}"] type = "${type}" periodic { cron = "${cron}" @@ -21,8 +21,12 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_mrr_rls2410.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_mrr.json" + destination = "local/" } driver = "docker" config { @@ -33,9 +37,9 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_hoststack_rls2402.py" + "coverage_mrr_rls2410.py" ] - work_dir = "/local/csit/csit.infra.etl" + work_dir = "/local" } env { AWS_ACCESS_KEY_ID = "${aws_access_key_id}" diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2402.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2402.hcl.tftpl deleted file mode 100644 index 6142219546..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2402.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_ndrpdr_rls2402.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -}
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2410.hcl.tftpl new file mode 100644 index 0000000000..ee4771b0b3 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2410.hcl.tftpl @@ -0,0 +1,59 @@ +job "${job_name}" { + datacenters = ["${datacenters}"] + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_ndrpdr_rls2410.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_ndrpdr.json" + destination = "local/" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "coverage_ndrpdr_rls2410.py" + ] + work_dir = "/local" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2402.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2402.hcl.tftpl deleted file mode 100644 index b474e75217..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2402.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_reconf_rls2402.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -}
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2410.hcl.tftpl new file mode 100644 index 0000000000..d31ebd81de --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2410.hcl.tftpl @@ -0,0 +1,59 @@ +job "${job_name}" { + datacenters = ["${datacenters}"] + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_reconf_rls2410.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_reconf.json" + destination = "local/" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "coverage_reconf_rls2410.py" + ] + work_dir = "/local" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2402.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2402.hcl.tftpl deleted file mode 100644 index 0352e1e879..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2402.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_soak_rls2402.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -}
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2402.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2410.hcl.tftpl index e6bd87b8ed..9d71dd8b9b 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2402.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2410.hcl.tftpl @@ -1,5 +1,5 @@ job "${job_name}" { - datacenters = "${datacenters}" + datacenters = ["${datacenters}"] type = "${type}" periodic { cron = "${cron}" @@ -21,8 +21,12 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_soak_rls2410.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_soak.json" + destination = "local/" } driver = "docker" config { @@ -33,9 +37,9 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_mrr_rls2402.py" + "coverage_soak_rls2410.py" ] - work_dir = "/local/csit/csit.infra.etl" + work_dir = "/local" } env { AWS_ACCESS_KEY_ID = "${aws_access_key_id}" diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2410.hcl.tftpl new file mode 100644 index 0000000000..79397eda20 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2410.hcl.tftpl @@ -0,0 +1,59 @@ +job "${job_name}" { + datacenters = ["${datacenters}"] + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_hoststack_rls2410.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_hoststack.json" + destination = "local/" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "iterative_hoststack_rls2410.py" + ] + work_dir = "/local" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2402.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2410.hcl.tftpl index 4a40321377..c1c298d4a4 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2402.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2410.hcl.tftpl @@ -1,5 +1,5 @@ job "${job_name}" { - datacenters = "${datacenters}" + datacenters = ["${datacenters}"] type = "${type}" periodic { cron = "${cron}" @@ -21,8 +21,12 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_mrr_rls2410.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_mrr.json" + destination = "local/" } driver = "docker" config { @@ -33,9 +37,9 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_ndrpdr_rls2402.py" + "iterative_mrr_rls2410.py" ] - work_dir = "/local/csit/csit.infra.etl" + work_dir = "/local" } env { AWS_ACCESS_KEY_ID = "${aws_access_key_id}" diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2410.hcl.tftpl new file mode 100644 index 0000000000..42202f30e2 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2410.hcl.tftpl @@ -0,0 +1,59 @@ +job "${job_name}" { + datacenters = ["${datacenters}"] + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_ndrpdr_rls2410.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_ndrpdr.json" + destination = "local/" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "iterative_ndrpdr_rls2410.py" + ] + work_dir = "/local" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2410.hcl.tftpl new file mode 100644 index 0000000000..4a7773ca46 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2410.hcl.tftpl @@ -0,0 +1,59 @@ +job "${job_name}" { + datacenters = ["${datacenters}"] + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_reconf_rls2410.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_reconf.json" + destination = "local/" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "iterative_reconf_rls2410.py" + ] + work_dir = "/local" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2402.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2402.hcl.tftpl deleted file mode 100644 index c4ad363879..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2402.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "iterative_soak_rls2402.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2402.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2410.hcl.tftpl index 670dd37a11..3759f4bb46 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2402.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2410.hcl.tftpl @@ -1,5 +1,5 @@ job "${job_name}" { - datacenters = "${datacenters}" + datacenters = ["${datacenters}"] type = "${type}" periodic { cron = "${cron}" @@ -21,8 +21,12 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_soak_rls2410.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_soak.json" + destination = "local/" } driver = "docker" config { @@ -33,9 +37,9 @@ job "${job_name}" { "--executor-memory", "20g", "--executor-cores", "2", "--master", "local[2]", - "iterative_reconf_rls2402.py" + "iterative_soak_rls2410.py" ] - work_dir = "/local/csit/csit.infra.etl" + work_dir = "/local" } env { AWS_ACCESS_KEY_ID = "${aws_access_key_id}" diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-stats.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-stats.hcl.tftpl index 86ca584de7..6634018988 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-stats.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-stats.hcl.tftpl @@ -1,5 +1,5 @@ job "${job_name}" { - datacenters = "${datacenters}" + datacenters = ["${datacenters}"] type = "${type}" periodic { cron = "${cron}" @@ -21,8 +21,12 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/stats.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/stats_sra.json" + destination = "local/" } driver = "docker" config { @@ -33,7 +37,7 @@ job "${job_name}" { "--executor-memory", "10g", "stats.py" ] - work_dir = "/local/csit/csit.infra.etl" + work_dir = "/local" } env { AWS_ACCESS_KEY_ID = "${aws_access_key_id}" diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-hoststack.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-hoststack.hcl.tftpl index 24aa4095d2..b8f7cc27c4 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-hoststack.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-hoststack.hcl.tftpl @@ -1,5 +1,5 @@ job "${job_name}" { - datacenters = "${datacenters}" + datacenters = ["${datacenters}"] type = "${type}" periodic { cron = "${cron}" @@ -21,8 +21,12 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_hoststack.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_hoststack.json" + destination = "local/" } driver = "docker" config { @@ -33,7 +37,7 @@ job "${job_name}" { "--executor-memory", "30g", "trending_hoststack.py" ] - work_dir = "/local/csit/csit.infra.etl" + work_dir = "/local" } env { AWS_ACCESS_KEY_ID = "${aws_access_key_id}" @@ -50,4 +54,4 @@ job "${job_name}" { } } } -}
\ No newline at end of file +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-mrr.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-mrr.hcl.tftpl index 47d6149eed..d3b301d5b3 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-mrr.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-mrr.hcl.tftpl @@ -1,5 +1,5 @@ job "${job_name}" { - datacenters = "${datacenters}" + datacenters = ["${datacenters}"] type = "${type}" periodic { cron = "${cron}" @@ -21,8 +21,12 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_mrr.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_mrr.json" + destination = "local/" } driver = "docker" config { @@ -33,7 +37,7 @@ job "${job_name}" { "--executor-memory", "30g", "trending_mrr.py" ] - work_dir = "/local/csit/csit.infra.etl" + work_dir = "/local" } env { AWS_ACCESS_KEY_ID = "${aws_access_key_id}" @@ -50,4 +54,4 @@ job "${job_name}" { } } } -}
\ No newline at end of file +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl index 8cd40f537e..53a0aa2393 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl @@ -1,5 +1,5 @@ job "${job_name}" { - datacenters = "${datacenters}" + datacenters = ["${datacenters}"] type = "${type}" periodic { cron = "${cron}" @@ -21,8 +21,12 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_ndrpdr.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_ndrpdr.json" + destination = "local/" } driver = "docker" config { @@ -35,7 +39,7 @@ job "${job_name}" { "--master", "local[2]", "trending_ndrpdr.py" ] - work_dir = "/local/csit/csit.infra.etl" + work_dir = "/local" } env { AWS_ACCESS_KEY_ID = "${aws_access_key_id}" @@ -52,4 +56,4 @@ job "${job_name}" { } } } -}
\ No newline at end of file +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-soak.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-soak.hcl.tftpl index 6d77a898df..b18ce527bc 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-soak.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-soak.hcl.tftpl @@ -1,5 +1,5 @@ job "${job_name}" { - datacenters = "${datacenters}" + datacenters = ["${datacenters}"] type = "${type}" periodic { cron = "${cron}" @@ -21,8 +21,12 @@ job "${job_name}" { } task "${job_name}" { artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_soak.py" + destination = "local/" + } + artifact { + source = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_soak.json" + destination = "local/" } driver = "docker" config { @@ -35,7 +39,7 @@ job "${job_name}" { "--master", "local[2]", "trending_soak.py" ] - work_dir = "/local/csit/csit.infra.etl" + work_dir = "/local" } env { AWS_ACCESS_KEY_ID = "${aws_access_key_id}" diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf deleted file mode 100644 index aac81d9b78..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf +++ /dev/null @@ -1,283 +0,0 @@ -data "vault_generic_secret" "fdio_logs" { - path = "kv/secret/data/etl/fdio_logs" -} - -data "vault_generic_secret" "fdio_docs" { - path = "kv/secret/data/etl/fdio_docs" -} - -module "etl-stats" { - providers = { - nomad = nomad.yul1 - } - source = "../" - - aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] - aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] - aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] - out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] - out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] - out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "0 30 0 * * * *" - datacenters = ["yul1"] - job_name = "etl-stats" -} - -module "etl-trending-hoststack" { - providers = { - nomad = nomad.yul1 - } - source = "../" - - aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] - aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] - aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] - out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] - out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] - out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "0 30 0 * * * *" - datacenters = ["yul1"] - job_name = "etl-trending-hoststack" -} - -module "etl-trending-mrr" { - providers = { - nomad = nomad.yul1 - } - source = "../" - - aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] - aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] - aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] - out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] - out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] - out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "0 30 0 * * * *" - datacenters = ["yul1"] - job_name = "etl-trending-mrr" - memory = 60000 -} - -module "etl-trending-ndrpdr" { - providers = { - nomad = nomad.yul1 - } - source = "../" - - aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] - aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] - aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] - out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] - out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] - out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "0 30 0 * * * *" - datacenters = ["yul1"] - job_name = "etl-trending-ndrpdr" - memory = 60000 -} - -module "etl-trending-soak" { - providers = { - nomad = nomad.yul1 - } - source = "../" - - aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] - aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] - aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] - out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] - out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] - out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] - cron = "0 30 0 * * * *" - datacenters = ["yul1"] - job_name = "etl-trending-soak" - memory = 60000 -} - -#module "etl-iterative-hoststack-rls2402" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-iterative-hoststack-rls2402" -#} -# -#module "etl-iterative-mrr-rls2402" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-iterative-mrr-rls2402" -#} -# -#module "etl-iterative-ndrpdr-rls2402" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-iterative-ndrpdr-rls2402" -#} -# -#module "etl-iterative-reconf-rls2402" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-iterative-reconf-rls2402" -#} -# -#module "etl-iterative-soak-rls2402" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-iterative-soak-rls2402" -#} -# -#module "etl-coverage-device-rls2402" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-coverage-device-rls2402" -#} -# -#module "etl-coverage-hoststack-rls2402" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-coverage-hoststack-rls2402" -#} -# -#module "etl-coverage-mrr-rls2402" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-coverage-mrr-rls2402" -#} -# -#module "etl-coverage-ndrpdr-rls2402" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-coverage-ndrpdr-rls2402" -#} -# -#module "etl-coverage-reconf-rls2402" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-coverage-reconf-rls2402" -#} -# -#module "etl-coverage-soak-rls2402" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_generic_secret.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_generic_secret.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_generic_secret.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_generic_secret.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_generic_secret.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_generic_secret.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-coverage-soak-rls2402" -#} -#
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/variables.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/variables.tf deleted file mode 100644 index db24bdf0fa..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/variables.tf +++ /dev/null @@ -1,47 +0,0 @@ -variable "nomad_acl" { - description = "Nomad ACLs enabled/disabled." - type = bool - default = false -} - -variable "nomad_provider_address" { - description = "FD.io Nomad cluster address." - type = string - default = "http://10.30.51.23:4646" -} - -variable "nomad_provider_ca_file" { - description = "A local file path to a PEM-encoded certificate authority." - type = string - default = "/etc/nomad.d/ssl/nomad-ca.pem" -} - -variable "nomad_provider_cert_file" { - description = "A local file path to a PEM-encoded certificate." - type = string - default = "/etc/nomad.d/ssl/nomad-cli.pem" -} - -variable "nomad_provider_key_file" { - description = "A local file path to a PEM-encoded private key." - type = string - default = "/etc/nomad.d/ssl/nomad-cli-key.pem" -} - -variable "vault_provider_address" { - description = "Vault cluster address." - type = string - default = "http://10.30.51.23:8200" -} - -variable "vault_provider_skip_tls_verify" { - description = "Verification of the Vault server's TLS certificate." - type = bool - default = false -} - -variable "vault_provider_token" { - description = "Vault root token." - type = string - sensitive = true -} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf deleted file mode 100644 index 0c05e76d65..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf +++ /dev/null @@ -1,17 +0,0 @@ -terraform { - backend "consul" { - address = "10.30.51.23:8500" - scheme = "http" - path = "terraform/etl" - } - required_providers { - nomad = { - source = "hashicorp/nomad" - version = ">= 1.4.20" - } - vault = { - version = ">= 3.12.0" - } - } - required_version = ">= 1.5.4" -} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf index cd6a9a52ff..57baa24276 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf @@ -1,33 +1,28 @@ -locals { - datacenters = join(",", var.datacenters) - envs = join("\n", concat([], var.envs)) +data "vault_kv_secret_v2" "fdio_logs" { + mount = "kv" + name = "etl/fdio_logs" } -resource "nomad_job" "nomad_job" { - jobspec = templatefile( - "${path.module}/conf/nomad/${var.job_name}.hcl.tftpl", - { - aws_access_key_id = var.aws_access_key_id, - aws_secret_access_key = var.aws_secret_access_key, - aws_default_region = var.aws_default_region - cpu = var.cpu, - cron = var.cron, - datacenters = local.datacenters, - envs = local.envs, - image = var.image, - job_name = var.job_name, - memory = var.memory, - out_aws_access_key_id = var.out_aws_access_key_id, - out_aws_secret_access_key = var.out_aws_secret_access_key, - out_aws_default_region = var.out_aws_default_region - prohibit_overlap = var.prohibit_overlap, - time_zone = var.time_zone, - type = var.type, - use_vault_provider = var.vault_secret.use_vault_provider, - vault_kv_policy_name = var.vault_secret.vault_kv_policy_name, - vault_kv_path = var.vault_secret.vault_kv_path, - vault_kv_field_access_key = var.vault_secret.vault_kv_field_access_key, - vault_kv_field_secret_key = var.vault_secret.vault_kv_field_secret_key - }) - detach = false +data "vault_kv_secret_v2" "fdio_docs" { + mount = "kv" + name = "etl/fdio_docs" } + +module "etl" { + for_each = { for job in var.nomad_jobs : job.job_name => job } + providers = { + nomad = nomad.yul1 + } + source = "../terraform-nomad-nomad-job" + + aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data.access_key + aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data.secret_key + aws_default_region = data.vault_kv_secret_v2.fdio_logs.data.region + out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data.access_key + out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data.secret_key + out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data.region + cron = "0 30 0 * * * *" + datacenters = ["yul1"] + job_name = each.key + memory = each.value.memory +}
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/providers.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/providers.tf index c6617da02b..c6617da02b 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/providers.tf +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/providers.tf diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf index f6d318e855..e8ddeb086c 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf @@ -1,115 +1,118 @@ -# Nomad -variable "datacenters" { - description = "Specifies the list of DCs to be considered placing this task." - type = list(string) - default = ["dc1"] -} - -# ETL -variable "aws_access_key_id" { - description = "AWS access key." - type = string - default = "aws" -} - -variable "aws_secret_access_key" { - description = "AWS secret key" - type = string - default = "aws" -} - -variable "aws_default_region" { - description = "AWS region" - type = string - default = "aws" -} - -variable "cpu" { - description = "Specifies the CPU required to run this task in MHz." - type = number - default = 10000 -} - -variable "cron" { - description = "Specifies a cron expression configuring the interval to launch." - type = string - default = "@daily" -} - -variable "envs" { - description = "Specifies ETL environment variables." - type = list(string) - default = [] +variable "nomad_acl" { + description = "Nomad ACLs enabled/disabled." + type = bool + default = false } -variable "image" { - description = "Specifies the Docker image to run." +variable "nomad_provider_address" { + description = "FD.io Nomad cluster address." type = string - default = "pmikus/docker-ubuntu-focal-aws-glue:latest" + default = "http://10.30.51.23:4646" } -variable "job_name" { - description = "Specifies a name for the job." +variable "nomad_provider_ca_file" { + description = "A local file path to a PEM-encoded certificate authority." type = string - default = "etl" -} - -variable "memory" { - description = "Specifies the memory required in MB." - type = number - default = 50000 + default = "/etc/nomad.d/ssl/nomad-ca.pem" } -variable "out_aws_access_key_id" { - description = "AWS access key." +variable "nomad_provider_cert_file" { + description = "A local file path to a PEM-encoded certificate." type = string - default = "aws" + default = "/etc/nomad.d/ssl/nomad.pem" } -variable "out_aws_secret_access_key" { - description = "AWS secret key" +variable "nomad_provider_key_file" { + description = "A local file path to a PEM-encoded private key." type = string - default = "aws" + default = "/etc/nomad.d/ssl/nomad-key.pem" } -variable "out_aws_default_region" { - description = "AWS region" +variable "vault_provider_address" { + description = "Vault cluster address." type = string - default = "aws" + default = "http://10.30.51.23:8200" } -variable "prohibit_overlap" { - description = "Specifies if this job should wait until previous completed." +variable "vault_provider_skip_tls_verify" { + description = "Verification of the Vault server's TLS certificate." type = bool - default = true + default = false } -variable "time_zone" { - description = "Specifies the time zone to evaluate the next launch interval." +variable "vault_provider_token" { + description = "Vault root token." type = string - default = "UTC" -} - -variable "type" { - description = "Specifies the Nomad scheduler to use." - type = string - default = "batch" -} - -variable "vault_secret" { - type = object({ - use_vault_provider = bool, - vault_kv_policy_name = string, - vault_kv_path = string, - vault_kv_field_access_key = string, - vault_kv_field_secret_key = string - }) - description = "Set of properties to be able to fetch secret from vault." - default = { - use_vault_provider = false - vault_kv_policy_name = "kv" - vault_kv_path = "secret/data/etl" - vault_kv_field_access_key = "access_key" - vault_kv_field_secret_key = "secret_key" - } -} + sensitive = true +} + +variable "nomad_jobs" { + description = "List of ETL jobs" + type = list(map(any)) + default = [ + { + job_name = "etl-stats" + memory = 50000 + }, + { + job_name = "etl-trending-hoststack" + memory = 50000 + }, + { + job_name = "etl-iterative-hoststack-rls2410" + memory = 50000 + }, + { + job_name = "etl-iterative-mrr-rls2410" + memory = 50000 + }, + { + job_name = "etl-iterative-ndrpdr-rls2410" + memory = 50000 + }, + { + job_name = "etl-iterative-reconf-rls2410" + memory = 50000 + }, + { + job_name = "etl-iterative-soak-rls2410" + memory = 50000 + }, + { + job_name = "etl-coverage-device-rls2410" + memory = 50000 + }, + { + job_name = "etl-coverage-hoststack-rls2410" + memory = 50000 + }, + { + job_name = "etl-coverage-mrr-rls2410" + memory = 50000 + }, + { + job_name = "etl-coverage-ndrpdr-rls2410" + memory = 50000 + }, + { + job_name = "etl-coverage-reconf-rls2410" + memory = 50000 + }, + { + job_name = "etl-coverage-soak-rls2410" + memory = 50000 + }, + { + job_name = "etl-trending-mrr" + memory = 60000 + }, + { + job_name = "etl-trending-ndrpdr" + memory = 60000 + }, + { + job_name = "etl-trending-soak" + memory = 60000 + } + ] +}
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf index f40435fe77..ffe25bb42e 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf @@ -1,8 +1,16 @@ terraform { + backend "consul" { + address = "10.30.51.23:8500" + scheme = "http" + path = "terraform/etl" + } required_providers { nomad = { source = "hashicorp/nomad" - version = ">= 1.4.20" + version = ">= 2.3.0" + } + vault = { + version = ">= 4.3.0" } } required_version = ">= 1.5.4" diff --git a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/main.tf b/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/main.tf index 4473dafda8..08c3ca8b73 100644 --- a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/main.tf +++ b/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/main.tf @@ -2,16 +2,22 @@ module "fdio-logs" { # fdio logs iam source = "../" name = "dynamic-aws-creds-vault-fdio-logs" + aws_access_key = var.aws_access_key + aws_secret_key = var.aws_secret_key } module "fdio-docs" { # fdio docs iam source = "../" name = "dynamic-aws-creds-vault-fdio-docs" + aws_access_key = var.aws_access_key + aws_secret_key = var.aws_secret_key } module "fdio-csit-jenkins" { # fdio csit jenkins iam source = "../" name = "dynamic-aws-creds-vault-fdio-csit-jenkins" + aws_access_key = var.aws_access_key + aws_secret_key = var.aws_secret_key } diff --git a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/variables.tf b/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/variables.tf index d3d728a49d..b1f64eccf2 100644 --- a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/variables.tf +++ b/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/variables.tf @@ -1,7 +1,7 @@ variable "vault_provider_address" { description = "Vault cluster address." type = string - default = "http://10.30.51.23:8200" + default = "http://10.30.51.26:8200" } variable "vault_provider_skip_tls_verify" { @@ -15,3 +15,15 @@ variable "vault_provider_token" { type = string sensitive = true } + +variable "aws_access_key" { + description = "AWS access key" + type = string + sensitive = true +} + +variable "aws_secret_key" { + description = "AWS secret key" + type = string + sensitive = true +}
\ No newline at end of file diff --git a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/versions.tf b/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/versions.tf index 4c93000093..c573731d65 100644 --- a/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/versions.tf +++ b/fdio.infra.terraform/terraform-vault-aws-secret-backend/fdio/versions.tf @@ -1,12 +1,12 @@ terraform { backend "consul" { - address = "10.30.51.23:8500" + address = "10.30.51.26:8500" scheme = "http" path = "terraform/aws-secret-backend" } required_providers { vault = { - version = ">= 3.12.0" + version = "4.3.0" } } required_version = ">= 1.5.4" diff --git a/fdio.infra.terraform/terraform-vault-aws-secret-backend/main.tf b/fdio.infra.terraform/terraform-vault-aws-secret-backend/main.tf index a65c390792..814121986f 100644 --- a/fdio.infra.terraform/terraform-vault-aws-secret-backend/main.tf +++ b/fdio.infra.terraform/terraform-vault-aws-secret-backend/main.tf @@ -19,7 +19,10 @@ resource "vault_aws_secret_backend_role" "admin" { { "Effect": "Allow", "Action": [ - "iam:*", "ec2:*" + "iam:*", + "ec2:*", + "s3:*", + "elasticbeanstalk:*" ], "Resource": "*" } diff --git a/fdio.infra.terraform/terraform-vault-aws-secret-backend/versions.tf b/fdio.infra.terraform/terraform-vault-aws-secret-backend/versions.tf index 996288568d..f084df8654 100644 --- a/fdio.infra.terraform/terraform-vault-aws-secret-backend/versions.tf +++ b/fdio.infra.terraform/terraform-vault-aws-secret-backend/versions.tf @@ -1,7 +1,7 @@ terraform { required_providers { vault = { - version = ">=2.22.1" + version = "4.3.0" } } required_version = ">= 1.1.4" |