diff options
Diffstat (limited to 'fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio')
4 files changed, 0 insertions, 369 deletions
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf deleted file mode 100644 index 026ab168d9..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/main.tf +++ /dev/null @@ -1,292 +0,0 @@ -data "vault_kv_secret_v2" "fdio_logs" { - mount = "kv" - name = "etl/fdio_logs" -} - -data "vault_kv_secret_v2" "fdio_docs" { - mount = "kv" - name = "etl/fdio_docs" -} - -#data "vault_kv_secret_v2" "fdio_logs" { -# path = "kv/data/etl/fdio_logs" -#} -# -#data "vault_kv_secret_v2" "fdio_docs" { -# path = "kv/data/etl/fdio_docs" -#} - -module "etl-stats" { - providers = { - nomad = nomad.yul1 - } - source = "../" - - aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data.access_key - aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data.secret_key - aws_default_region = data.vault_kv_secret_v2.fdio_logs.data.region - out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data.access_key - out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data.secret_key - out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data.region - cron = "0 30 0 * * * *" - datacenters = ["yul1"] - job_name = "etl-stats" -} - -module "etl-trending-hoststack" { - providers = { - nomad = nomad.yul1 - } - source = "../" - - aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data.access_key - aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data.secret_key - aws_default_region = data.vault_kv_secret_v2.fdio_logs.data.region - out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data.access_key - out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data.secret_key - out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data.region - cron = "0 30 0 * * * *" - datacenters = ["yul1"] - job_name = "etl-trending-hoststack" -} - -module "etl-trending-mrr" { - providers = { - nomad = nomad.yul1 - } - source = "../" - - aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data.access_key - aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data.secret_key - aws_default_region = data.vault_kv_secret_v2.fdio_logs.data.region - out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data.access_key - out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data.secret_key - out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data.region - cron = "0 30 0 * * * *" - datacenters = ["yul1"] - job_name = "etl-trending-mrr" - memory = 60000 -} - -module "etl-trending-ndrpdr" { - providers = { - nomad = nomad.yul1 - } - source = "../" - - aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data.access_key - aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data.secret_key - aws_default_region = data.vault_kv_secret_v2.fdio_logs.data.region - out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data.access_key - out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data.secret_key - out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data.region - cron = "0 30 0 * * * *" - datacenters = ["yul1"] - job_name = "etl-trending-ndrpdr" - memory = 60000 -} - -module "etl-trending-soak" { - providers = { - nomad = nomad.yul1 - } - source = "../" - - aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data.access_key - aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data.secret_key - aws_default_region = data.vault_kv_secret_v2.fdio_logs.data.region - out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data.access_key - out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data.secret_key - out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data.region - cron = "0 30 0 * * * *" - datacenters = ["yul1"] - job_name = "etl-trending-soak" - memory = 60000 -} - -#module "etl-iterative-hoststack-rls2406" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_kv_secret_v2.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-iterative-hoststack-rls2406" -#} -# -#module "etl-iterative-mrr-rls2406" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_kv_secret_v2.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-iterative-mrr-rls2406" -#} -# -#module "etl-iterative-ndrpdr-rls2406" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_kv_secret_v2.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-iterative-ndrpdr-rls2406" -#} -# -#module "etl-iterative-reconf-rls2406" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_kv_secret_v2.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-iterative-reconf-rls2406" -#} -# -#module "etl-iterative-soak-rls2406" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_kv_secret_v2.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-iterative-soak-rls2406" -#} -# -#module "etl-coverage-device-rls2406" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_kv_secret_v2.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-coverage-device-rls2406" -#} -# -#module "etl-coverage-hoststack-rls2406" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_kv_secret_v2.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-coverage-hoststack-rls2406" -#} -# -#module "etl-coverage-mrr-rls2406" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_kv_secret_v2.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-coverage-mrr-rls2406" -#} -# -#module "etl-coverage-ndrpdr-rls2406" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_kv_secret_v2.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-coverage-ndrpdr-rls2406" -#} -# -#module "etl-coverage-reconf-rls2406" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_kv_secret_v2.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-coverage-reconf-rls2406" -#} -# -#module "etl-coverage-soak-rls2406" { -# providers = { -# nomad = nomad.yul1 -# } -# source = "../" -# -# aws_access_key_id = data.vault_kv_secret_v2.fdio_logs.data["access_key"] -# aws_secret_access_key = data.vault_kv_secret_v2.fdio_logs.data["secret_key"] -# aws_default_region = data.vault_kv_secret_v2.fdio_logs.data["region"] -# out_aws_access_key_id = data.vault_kv_secret_v2.fdio_docs.data["access_key"] -# out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data["secret_key"] -# out_aws_default_region = data.vault_kv_secret_v2.fdio_docs.data["region"] -# cron = "0 30 0 * * * *" -# datacenters = ["yul1"] -# job_name = "etl-coverage-soak-rls2406" -#} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/providers.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/providers.tf deleted file mode 100644 index c6617da02b..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/providers.tf +++ /dev/null @@ -1,13 +0,0 @@ -provider "nomad" { - address = var.nomad_provider_address - alias = "yul1" - # ca_file = var.nomad_provider_ca_file - # cert_file = var.nomad_provider_cert_file - # key_file = var.nomad_provider_key_file -} - -provider "vault" { - address = var.vault_provider_address - skip_tls_verify = var.vault_provider_skip_tls_verify - token = var.vault_provider_token -} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/variables.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/variables.tf deleted file mode 100644 index 60298d4c99..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/variables.tf +++ /dev/null @@ -1,47 +0,0 @@ -variable "nomad_acl" { - description = "Nomad ACLs enabled/disabled." - type = bool - default = false -} - -variable "nomad_provider_address" { - description = "FD.io Nomad cluster address." - type = string - default = "http://10.30.51.23:4646" -} - -variable "nomad_provider_ca_file" { - description = "A local file path to a PEM-encoded certificate authority." - type = string - default = "/etc/nomad.d/ssl/nomad-ca.pem" -} - -variable "nomad_provider_cert_file" { - description = "A local file path to a PEM-encoded certificate." - type = string - default = "/etc/nomad.d/ssl/nomad.pem" -} - -variable "nomad_provider_key_file" { - description = "A local file path to a PEM-encoded private key." - type = string - default = "/etc/nomad.d/ssl/nomad-key.pem" -} - -variable "vault_provider_address" { - description = "Vault cluster address." - type = string - default = "http://10.30.51.23:8200" -} - -variable "vault_provider_skip_tls_verify" { - description = "Verification of the Vault server's TLS certificate." - type = bool - default = false -} - -variable "vault_provider_token" { - description = "Vault root token." - type = string - sensitive = true -} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf deleted file mode 100644 index ffe25bb42e..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/fdio/versions.tf +++ /dev/null @@ -1,17 +0,0 @@ -terraform { - backend "consul" { - address = "10.30.51.23:8500" - scheme = "http" - path = "terraform/etl" - } - required_providers { - nomad = { - source = "hashicorp/nomad" - version = ">= 2.3.0" - } - vault = { - version = ">= 4.3.0" - } - } - required_version = ">= 1.5.4" -} |