diff options
author | 2022-11-25 15:33:34 +0000 | |
---|---|---|
committer | 2022-11-28 12:26:00 +0000 | |
commit | 833915316fc17b05bfb47639397fd7f251b89979 (patch) | |
tree | e141b5cf013af8133a215426f69bab25b550c564 /fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl | |
parent | 4b4912b829496b2e156dd2125f6a5f609e63c2a2 (diff) |
feat(etl): Add telemetry, remove release
Signed-off-by: pmikus <peter.mikus@protonmail.ch>
Change-Id: I26471a041c83ffc5d221b2b6d69b4443a40f3de8
Diffstat (limited to 'fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl')
-rw-r--r-- | fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl | 476 |
1 files changed, 238 insertions, 238 deletions
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl index 0abb0e5d51..1fb2f22e43 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl @@ -140,242 +140,242 @@ job "${job_name}" { } } } - group "${job_name}-rls2210" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}-coverage-device" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_device_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - task "${job_name}-coverage-mrr" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_mrr_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - task "${job_name}-coverage-ndrpdr" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_ndrpdr_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - task "${job_name}-coverage-soak" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_soak_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - task "${job_name}-iterative-mrr" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "iterative_mrr_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - task "${job_name}-iterative-ndrpdr" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "iterative_ndrpdr_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - task "${job_name}-iterative-soak" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "iterative_soak_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } +# group "${job_name}-rls2210" { +# restart { +# mode = "fail" +# } +# constraint { +# attribute = "$${attr.cpu.arch}" +# operator = "!=" +# value = "arm64" +# } +# constraint { +# attribute = "$${node.class}" +# value = "builder" +# } +# task "${job_name}-coverage-device" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "coverage_device_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# task "${job_name}-coverage-mrr" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "coverage_mrr_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# task "${job_name}-coverage-ndrpdr" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "coverage_ndrpdr_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# task "${job_name}-coverage-soak" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "coverage_soak_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# task "${job_name}-iterative-mrr" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "iterative_mrr_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# task "${job_name}-iterative-ndrpdr" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "iterative_ndrpdr_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# task "${job_name}-iterative-soak" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "iterative_soak_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# } } |