14 files changed, 38 insertions, 96 deletions

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2410.hcl.tftpl
deleted file mode 100644
index 96fef71e17..0000000000
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2410.hcl.tftpl
+++ /dev/null
@@ -1,59 +0,0 @@
-job "${job_name}" {
-  datacenters = ["${datacenters}"]
-  type        = "${type}"
-  periodic {
-    cron             = "${cron}"
-    prohibit_overlap = "${prohibit_overlap}"
-    time_zone        = "${time_zone}"
-  }
-  group "${job_name}" {
-    restart {
-      mode = "fail"
-    }
-    constraint {
-      attribute = "$${attr.cpu.arch}"
-      operator  = "!="
-      value     = "arm64"
-    }
-    constraint {
-      attribute = "$${node.class}"
-      value     = "builder"
-    }
-    task "${job_name}" {
-      artifact {
-        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_device_rls2410.py"
-        destination = "local/"
-      }
-      artifact {
-        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_device.json"
-        destination = "local/"
-      }
-      driver = "docker"
-      config {
-        image   = "${image}"
-        command = "gluesparksubmit"
-        args = [
-          "--driver-memory", "20g",
-          "--executor-memory", "20g",
-          "--executor-cores", "2",
-          "--master", "local[2]",
-          "coverage_device_rls2410.py"
-        ]
-        work_dir = "/local"
-      }
-      env {
-        AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
-        AWS_SECRET_ACCESS_KEY     = "${aws_secret_access_key}"
-        AWS_DEFAULT_REGION        = "${aws_default_region}"
-        OUT_AWS_ACCESS_KEY_ID     = "${out_aws_access_key_id}"
-        OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
-        OUT_AWS_DEFAULT_REGION    = "${out_aws_default_region}"
-        ${ envs }
-      }
-      resources {
-        cpu    = ${cpu}
-        memory = ${memory}
-      }
-    }
-  }
-}
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack.hcl.tftpl
index 974e915101..8aa364388d 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack.hcl.tftpl
@@ -21,7 +21,7 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_hoststack_rls2410.py"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_hoststack.py"
         destination = "local/"
       }
       artifact {
@@ -37,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "coverage_hoststack_rls2410.py"
+          "coverage_hoststack.py"
         ]
         work_dir = "/local"
       }
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr.hcl.tftpl
index c2c282752b..1a2db579a3 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr.hcl.tftpl
@@ -21,7 +21,7 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_mrr_rls2410.py"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_mrr.py"
         destination = "local/"
       }
       artifact {
@@ -37,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "coverage_mrr_rls2410.py"
+          "coverage_mrr.py"
         ]
         work_dir = "/local"
       }
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr.hcl.tftpl
index ee4771b0b3..b294c390cd 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr.hcl.tftpl
@@ -21,7 +21,7 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_ndrpdr_rls2410.py"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_ndrpdr.py"
         destination = "local/"
       }
       artifact {
@@ -37,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "coverage_ndrpdr_rls2410.py"
+          "coverage_ndrpdr.py"
         ]
         work_dir = "/local"
       }
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf.hcl.tftpl
index d31ebd81de..83bbceb109 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf.hcl.tftpl
@@ -21,7 +21,7 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_reconf_rls2410.py"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_reconf.py"
         destination = "local/"
       }
       artifact {
@@ -37,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "coverage_reconf_rls2410.py"
+          "coverage_reconf.py"
         ]
         work_dir = "/local"
       }
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak.hcl.tftpl
index 9d71dd8b9b..871e4f79a3 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak.hcl.tftpl
@@ -21,7 +21,7 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_soak_rls2410.py"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/coverage_soak.py"
         destination = "local/"
       }
       artifact {
@@ -37,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "coverage_soak_rls2410.py"
+          "coverage_soak.py"
         ]
         work_dir = "/local"
       }
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack.hcl.tftpl
index 79397eda20..c7a9f60730 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack.hcl.tftpl
@@ -21,7 +21,7 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_hoststack_rls2410.py"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_hoststack.py"
         destination = "local/"
       }
       artifact {
@@ -37,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "iterative_hoststack_rls2410.py"
+          "iterative_hoststack.py"
         ]
         work_dir = "/local"
       }
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr.hcl.tftpl
index c1c298d4a4..3274596680 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr.hcl.tftpl
@@ -21,7 +21,7 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_mrr_rls2410.py"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_mrr.py"
         destination = "local/"
       }
       artifact {
@@ -37,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "iterative_mrr_rls2410.py"
+          "iterative_mrr.py"
         ]
         work_dir = "/local"
       }
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr.hcl.tftpl
index 42202f30e2..0d9f4bb250 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr.hcl.tftpl
@@ -21,7 +21,7 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_ndrpdr_rls2410.py"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_ndrpdr.py"
         destination = "local/"
       }
       artifact {
@@ -37,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "iterative_ndrpdr_rls2410.py"
+          "iterative_ndrpdr.py"
         ]
         work_dir = "/local"
       }
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf.hcl.tftpl
index 4a7773ca46..5045edbb33 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf.hcl.tftpl
@@ -21,7 +21,7 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_reconf_rls2410.py"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_reconf.py"
         destination = "local/"
       }
       artifact {
@@ -37,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "iterative_reconf_rls2410.py"
+          "iterative_reconf.py"
         ]
         work_dir = "/local"
       }
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2410.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak.hcl.tftpl
index 3759f4bb46..c569a43b2f 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2410.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak.hcl.tftpl
@@ -21,7 +21,7 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_soak_rls2410.py"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/iterative_soak.py"
         destination = "local/"
       }
       artifact {
@@ -37,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "20g",
           "--executor-cores", "2",
           "--master", "local[2]",
-          "iterative_soak_rls2410.py"
+          "iterative_soak.py"
         ]
         work_dir = "/local"
       }
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf
index 57baa24276..da3a47dcd6 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/main.tf
@@ -8,6 +8,11 @@ data "vault_kv_secret_v2" "fdio_docs" {
   name  = "etl/fdio_docs"
 }
 
+data "vault_kv_secret_v2" "csit_docs" {
+  mount = "kv"
+  name  = "etl/csit_docs"
+}
+
 module "etl" {
   for_each = { for job in var.nomad_jobs : job.job_name => job }
   providers = {
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf
index e8ddeb086c..d0d9fa171f 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/variables.tf
@@ -55,51 +55,47 @@ variable "nomad_jobs" {
       memory = 50000
     },
     {
-      job_name = "etl-trending-hoststack"
+      job_name = "etl-iterative-hoststack"
       memory = 50000
     },
     {
-      job_name = "etl-iterative-hoststack-rls2410"
+      job_name = "etl-iterative-mrr"
       memory = 50000
     },
     {
-      job_name = "etl-iterative-mrr-rls2410"
+      job_name = "etl-iterative-ndrpdr"
       memory = 50000
     },
     {
-      job_name = "etl-iterative-ndrpdr-rls2410"
+      job_name = "etl-iterative-reconf"
       memory = 50000
     },
     {
-      job_name = "etl-iterative-reconf-rls2410"
+      job_name = "etl-iterative-soak"
       memory = 50000
     },
     {
-      job_name = "etl-iterative-soak-rls2410"
+      job_name = "etl-coverage-hoststack"
       memory = 50000
     },
     {
-      job_name = "etl-coverage-device-rls2410"
+      job_name = "etl-coverage-mrr"
       memory = 50000
     },
     {
-      job_name = "etl-coverage-hoststack-rls2410"
+      job_name = "etl-coverage-ndrpdr"
       memory = 50000
     },
     {
-      job_name = "etl-coverage-mrr-rls2410"
+      job_name = "etl-coverage-reconf"
       memory = 50000
     },
     {
-      job_name = "etl-coverage-ndrpdr-rls2410"
+      job_name = "etl-coverage-soak"
       memory = 50000
     },
     {
-      job_name = "etl-coverage-reconf-rls2410"
-      memory = 50000
-    },
-    {
-      job_name = "etl-coverage-soak-rls2410"
+      job_name = "etl-trending-hoststack"
       memory = 50000
     },
     {
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf b/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf
index ffe25bb42e..cf67cc25c7 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/versions.tf
@@ -10,8 +10,8 @@ terraform {
       version = ">= 2.3.0"
     }
     vault = {
-      version = ">= 4.3.0"
+      version = ">= 4.6.0"
     }
   }
-  required_version = ">= 1.5.4"
+  required_version = ">= 1.10.4"
 }