feat(terraform): Update ETL

Signed-off-by: Peter Mikus <peter.mikus@protonmail.ch>
Change-Id: I840cbc5a95e9af41af1811fc28db71e6b0d19d1e

5 files changed, 43 insertions, 23 deletions

diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-stats.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-stats.hcl.tftpl
index 86ca584de7..6634018988 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-stats.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-stats.hcl.tftpl
@@ -1,5 +1,5 @@
 job "${job_name}" {
-  datacenters = "${datacenters}"
+  datacenters = ["${datacenters}"]
   type        = "${type}"
   periodic {
     cron             = "${cron}"
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/stats.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/stats_sra.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -33,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "10g",
           "stats.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-hoststack.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-hoststack.hcl.tftpl
index 24aa4095d2..b8f7cc27c4 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-hoststack.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-hoststack.hcl.tftpl
@@ -1,5 +1,5 @@
 job "${job_name}" {
-  datacenters = "${datacenters}"
+  datacenters = ["${datacenters}"]
   type        = "${type}"
   periodic {
     cron             = "${cron}"
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_hoststack.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_hoststack.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -33,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "30g",
           "trending_hoststack.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
@@ -50,4 +54,4 @@ job "${job_name}" {
       }
     }
   }
-}
\ No newline at end of file
+}
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-mrr.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-mrr.hcl.tftpl
index 47d6149eed..d3b301d5b3 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-mrr.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-mrr.hcl.tftpl
@@ -1,5 +1,5 @@
 job "${job_name}" {
-  datacenters = "${datacenters}"
+  datacenters = ["${datacenters}"]
   type        = "${type}"
   periodic {
     cron             = "${cron}"
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_mrr.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_mrr.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -33,7 +37,7 @@ job "${job_name}" {
           "--executor-memory", "30g",
           "trending_mrr.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
@@ -50,4 +54,4 @@ job "${job_name}" {
       }
     }
   }
-}
\ No newline at end of file
+}
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl
index 8cd40f537e..53a0aa2393 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-ndrpdr.hcl.tftpl
@@ -1,5 +1,5 @@
 job "${job_name}" {
-  datacenters = "${datacenters}"
+  datacenters = ["${datacenters}"]
   type        = "${type}"
   periodic {
     cron             = "${cron}"
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_ndrpdr.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_ndrpdr.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "trending_ndrpdr.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"
@@ -52,4 +56,4 @@ job "${job_name}" {
       }
     }
   }
-}
\ No newline at end of file
+}
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-soak.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-soak.hcl.tftpl
index 6d77a898df..b18ce527bc 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-soak.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-trending-soak.hcl.tftpl
@@ -1,5 +1,5 @@
 job "${job_name}" {
-  datacenters = "${datacenters}"
+  datacenters = ["${datacenters}"]
   type        = "${type}"
   periodic {
     cron             = "${cron}"
@@ -21,8 +21,12 @@ job "${job_name}" {
     }
     task "${job_name}" {
       artifact {
-        source      = "git::https://github.com/FDio/csit"
-        destination = "local/csit"
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_soak.py"
+        destination = "local/"
+      }
+      artifact {
+        source      = "https://raw.githubusercontent.com/FDio/csit/master/csit.infra.etl/trending_soak.json"
+        destination = "local/"
       }
       driver = "docker"
       config {
@@ -35,7 +39,7 @@ job "${job_name}" {
           "--master", "local[2]",
           "trending_soak.py"
         ]
-        work_dir = "/local/csit/csit.infra.etl"
+        work_dir = "/local"
       }
       env {
         AWS_ACCESS_KEY_ID         = "${aws_access_key_id}"