From b0b7f8dfdd37d45de0f3fcf758798e3bfbb9381d Mon Sep 17 00:00:00 2001 From: pmikus Date: Thu, 17 Aug 2023 12:26:28 +0000 Subject: feat(terraform): ETL for 2310 Signed-off-by: pmikus Change-Id: I3013b8be6a7f06d2f1f3b8320e7cb6f057a47491 --- .../nomad/etl-coverage-device-rls2306.hcl.tftpl | 55 ---------------------- .../nomad/etl-coverage-device-rls2310.hcl.tftpl | 55 ++++++++++++++++++++++ .../nomad/etl-coverage-hoststack-rls2306.hcl.tftpl | 55 ---------------------- .../nomad/etl-coverage-hoststack-rls2310.hcl.tftpl | 55 ++++++++++++++++++++++ .../conf/nomad/etl-coverage-mrr-rls2306.hcl.tftpl | 55 ---------------------- .../conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl | 55 ++++++++++++++++++++++ .../nomad/etl-coverage-ndrpdr-rls2306.hcl.tftpl | 55 ---------------------- .../nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl | 55 ++++++++++++++++++++++ .../nomad/etl-coverage-reconf-rls2306.hcl.tftpl | 55 ---------------------- .../nomad/etl-coverage-reconf-rls2310.hcl.tftpl | 55 ++++++++++++++++++++++ .../conf/nomad/etl-coverage-soak-rls2306.hcl.tftpl | 55 ---------------------- .../conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl | 55 ++++++++++++++++++++++ .../etl-iterative-hoststack-rls2306.hcl.tftpl | 55 ---------------------- .../etl-iterative-hoststack-rls2310.hcl.tftpl | 55 ++++++++++++++++++++++ .../conf/nomad/etl-iterative-mrr-rls2306.hcl.tftpl | 55 ---------------------- .../conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl | 55 ++++++++++++++++++++++ .../nomad/etl-iterative-ndrpdr-rls2306.hcl.tftpl | 55 ---------------------- .../nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl | 55 ++++++++++++++++++++++ .../nomad/etl-iterative-reconf-rls2306.hcl.tftpl | 55 ---------------------- .../nomad/etl-iterative-reconf-rls2310.hcl.tftpl | 55 ++++++++++++++++++++++ .../nomad/etl-iterative-soak-rls2306.hcl.tftpl | 55 ---------------------- .../nomad/etl-iterative-soak-rls2310.hcl.tftpl | 55 ++++++++++++++++++++++ 22 files changed, 605 insertions(+), 605 deletions(-) delete mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2306.hcl.tftpl create mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2310.hcl.tftpl delete mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2306.hcl.tftpl create mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2310.hcl.tftpl delete mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2306.hcl.tftpl create mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl delete mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2306.hcl.tftpl create mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl delete mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2306.hcl.tftpl create mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2310.hcl.tftpl delete mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2306.hcl.tftpl create mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl delete mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2306.hcl.tftpl create mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2310.hcl.tftpl delete mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2306.hcl.tftpl create mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl delete mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2306.hcl.tftpl create mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl delete mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2306.hcl.tftpl create mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2310.hcl.tftpl delete mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2306.hcl.tftpl create mode 100644 fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2310.hcl.tftpl (limited to 'fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad') diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2306.hcl.tftpl deleted file mode 100644 index c769659bc6..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2306.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_device_rls2306.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -} \ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2310.hcl.tftpl new file mode 100644 index 0000000000..217197667a --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-device-rls2310.hcl.tftpl @@ -0,0 +1,55 @@ +job "${job_name}" { + datacenters = "${datacenters}" + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "git::https://github.com/FDio/csit" + destination = "local/csit" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "coverage_device_rls2310.py" + ] + work_dir = "/local/csit/csit.infra.etl" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} \ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2306.hcl.tftpl deleted file mode 100644 index 3567f0258e..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2306.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_hoststack_rls2306.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -} \ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2310.hcl.tftpl new file mode 100644 index 0000000000..6aa5268df1 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-hoststack-rls2310.hcl.tftpl @@ -0,0 +1,55 @@ +job "${job_name}" { + datacenters = "${datacenters}" + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "git::https://github.com/FDio/csit" + destination = "local/csit" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "coverage_hoststack_rls2310.py" + ] + work_dir = "/local/csit/csit.infra.etl" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} \ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2306.hcl.tftpl deleted file mode 100644 index edfd995980..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2306.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_mrr_rls2306.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -} \ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl new file mode 100644 index 0000000000..6358dbe690 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-mrr-rls2310.hcl.tftpl @@ -0,0 +1,55 @@ +job "${job_name}" { + datacenters = "${datacenters}" + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "git::https://github.com/FDio/csit" + destination = "local/csit" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "coverage_mrr_rls2310.py" + ] + work_dir = "/local/csit/csit.infra.etl" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} \ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2306.hcl.tftpl deleted file mode 100644 index 298853b15e..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2306.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_ndrpdr_rls2306.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -} \ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl new file mode 100644 index 0000000000..c2400ad444 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-ndrpdr-rls2310.hcl.tftpl @@ -0,0 +1,55 @@ +job "${job_name}" { + datacenters = "${datacenters}" + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "git::https://github.com/FDio/csit" + destination = "local/csit" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "coverage_ndrpdr_rls2310.py" + ] + work_dir = "/local/csit/csit.infra.etl" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} \ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2306.hcl.tftpl deleted file mode 100644 index 82630ba64d..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2306.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_reconf_rls2306.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -} \ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2310.hcl.tftpl new file mode 100644 index 0000000000..3bea4e77b4 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-reconf-rls2310.hcl.tftpl @@ -0,0 +1,55 @@ +job "${job_name}" { + datacenters = "${datacenters}" + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "git::https://github.com/FDio/csit" + destination = "local/csit" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "coverage_reconf_rls2310.py" + ] + work_dir = "/local/csit/csit.infra.etl" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} \ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2306.hcl.tftpl deleted file mode 100644 index f32a7cf024..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2306.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_soak_rls2306.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -} \ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl new file mode 100644 index 0000000000..8ca898be37 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-coverage-soak-rls2310.hcl.tftpl @@ -0,0 +1,55 @@ +job "${job_name}" { + datacenters = "${datacenters}" + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "git::https://github.com/FDio/csit" + destination = "local/csit" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "coverage_soak_rls2310.py" + ] + work_dir = "/local/csit/csit.infra.etl" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} \ No newline at end of file diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2306.hcl.tftpl deleted file mode 100644 index c4ebf032d8..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2306.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "iterative_hoststack_rls2306.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2310.hcl.tftpl new file mode 100644 index 0000000000..285fcac9c0 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-hoststack-rls2310.hcl.tftpl @@ -0,0 +1,55 @@ +job "${job_name}" { + datacenters = "${datacenters}" + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "git::https://github.com/FDio/csit" + destination = "local/csit" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "iterative_hoststack_rls2310.py" + ] + work_dir = "/local/csit/csit.infra.etl" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2306.hcl.tftpl deleted file mode 100644 index d4e0f9598f..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2306.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "iterative_mrr_rls2306.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl new file mode 100644 index 0000000000..764dfe7147 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-mrr-rls2310.hcl.tftpl @@ -0,0 +1,55 @@ +job "${job_name}" { + datacenters = "${datacenters}" + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "git::https://github.com/FDio/csit" + destination = "local/csit" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "iterative_mrr_rls2310.py" + ] + work_dir = "/local/csit/csit.infra.etl" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2306.hcl.tftpl deleted file mode 100644 index e18ebbc6e2..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2306.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "iterative_ndrpdr_rls2306.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl new file mode 100644 index 0000000000..5ea75a49d9 --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-ndrpdr-rls2310.hcl.tftpl @@ -0,0 +1,55 @@ +job "${job_name}" { + datacenters = "${datacenters}" + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "git::https://github.com/FDio/csit" + destination = "local/csit" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "iterative_ndrpdr_rls2310.py" + ] + work_dir = "/local/csit/csit.infra.etl" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2306.hcl.tftpl deleted file mode 100644 index 46b5ba2c80..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2306.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "iterative_reconf_rls2306.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2310.hcl.tftpl new file mode 100644 index 0000000000..17ad83e25a --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-reconf-rls2310.hcl.tftpl @@ -0,0 +1,55 @@ +job "${job_name}" { + datacenters = "${datacenters}" + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "git::https://github.com/FDio/csit" + destination = "local/csit" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "iterative_reconf_rls2310.py" + ] + work_dir = "/local/csit/csit.infra.etl" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2306.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2306.hcl.tftpl deleted file mode 100644 index 7df5aef88f..0000000000 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2306.hcl.tftpl +++ /dev/null @@ -1,55 +0,0 @@ -job "${job_name}" { - datacenters = "${datacenters}" - type = "${type}" - periodic { - cron = "${cron}" - prohibit_overlap = "${prohibit_overlap}" - time_zone = "${time_zone}" - } - group "${job_name}" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "iterative_soak_rls2306.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } -} diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2310.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2310.hcl.tftpl new file mode 100644 index 0000000000..69753701ce --- /dev/null +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl-iterative-soak-rls2310.hcl.tftpl @@ -0,0 +1,55 @@ +job "${job_name}" { + datacenters = "${datacenters}" + type = "${type}" + periodic { + cron = "${cron}" + prohibit_overlap = "${prohibit_overlap}" + time_zone = "${time_zone}" + } + group "${job_name}" { + restart { + mode = "fail" + } + constraint { + attribute = "$${attr.cpu.arch}" + operator = "!=" + value = "arm64" + } + constraint { + attribute = "$${node.class}" + value = "builder" + } + task "${job_name}" { + artifact { + source = "git::https://github.com/FDio/csit" + destination = "local/csit" + } + driver = "docker" + config { + image = "${image}" + command = "gluesparksubmit" + args = [ + "--driver-memory", "20g", + "--executor-memory", "20g", + "--executor-cores", "2", + "--master", "local[2]", + "iterative_soak_rls2310.py" + ] + work_dir = "/local/csit/csit.infra.etl" + } + env { + AWS_ACCESS_KEY_ID = "${aws_access_key_id}" + AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" + AWS_DEFAULT_REGION = "${aws_default_region}" + OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" + OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" + OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" + ${ envs } + } + resources { + cpu = ${cpu} + memory = ${memory} + } + } + } +} -- cgit 1.2.3-korg