feat(terraform): Refactor ETLoper-240930

Signed-off-by: Peter Mikus <peter.mikus@protonmail.ch>
Change-Id: I86711ef80304d72a701ef84737f503ee52659dc5

1 files changed, 25 insertions, 30 deletions

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf
index cd6a9a52ff..57baa24276 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf
@@ -1,33 +1,28 @@
-locals {
-  datacenters = join(",", var.datacenters)
-  envs        = join("\n", concat([], var.envs))
+data "vault_kv_secret_v2" "fdio_logs" {
+  mount = "kv"
+  name  = "etl/fdio_logs"
 }
 
-resource "nomad_job" "nomad_job" {
-  jobspec = templatefile(
-    "${path.module}/conf/nomad/${var.job_name}.hcl.tftpl",
-    {
-      aws_access_key_id         = var.aws_access_key_id,
-      aws_secret_access_key     = var.aws_secret_access_key,
-      aws_default_region        = var.aws_default_region
-      cpu                       = var.cpu,
-      cron                      = var.cron,
-      datacenters               = local.datacenters,
-      envs                      = local.envs,
-      image                     = var.image,
-      job_name                  = var.job_name,
-      memory                    = var.memory,
-      out_aws_access_key_id     = var.out_aws_access_key_id,
-      out_aws_secret_access_key = var.out_aws_secret_access_key,
-      out_aws_default_region    = var.out_aws_default_region
-      prohibit_overlap          = var.prohibit_overlap,
-      time_zone                 = var.time_zone,
-      type                      = var.type,
-      use_vault_provider        = var.vault_secret.use_vault_provider,
-      vault_kv_policy_name      = var.vault_secret.vault_kv_policy_name,
-      vault_kv_path             = var.vault_secret.vault_kv_path,
-      vault_kv_field_access_key = var.vault_secret.vault_kv_field_access_key,
-      vault_kv_field_secret_key = var.vault_secret.vault_kv_field_secret_key
-  })
-  detach = false
+data "vault_kv_secret_v2" "fdio_docs" {
+  mount = "kv"
+  name  = "etl/fdio_docs"
 }
+
+module "etl" {
+  for_each = { for job in var.nomad_jobs : job.job_name => job }
+  providers = {
+    nomad = nomad.yul1
+  }
+  source = "../terraform-nomad-nomad-job"
+
+  aws_access_key_id         = data.vault_kv_secret_v2.fdio_logs.data.access_key
+  aws_secret_access_key     = data.vault_kv_secret_v2.fdio_logs.data.secret_key
+  aws_default_region        = data.vault_kv_secret_v2.fdio_logs.data.region
+  out_aws_access_key_id     = data.vault_kv_secret_v2.fdio_docs.data.access_key
+  out_aws_secret_access_key = data.vault_kv_secret_v2.fdio_docs.data.secret_key
+  out_aws_default_region    = data.vault_kv_secret_v2.fdio_docs.data.region
+  cron                      = "0 30 0 * * * *"
+  datacenters               = ["yul1"]
+  job_name                  = each.key
+  memory                    = each.value.memory
+}
\ No newline at end of file