From 833915316fc17b05bfb47639397fd7f251b89979 Mon Sep 17 00:00:00 2001 From: pmikus Date: Fri, 25 Nov 2022 15:33:34 +0000 Subject: feat(etl): Add telemetry, remove release Signed-off-by: pmikus Change-Id: I26471a041c83ffc5d221b2b6d69b4443a40f3de8 --- .../conf/nomad/etl.hcl.tftpl | 476 ++++++++++----------- 1 file changed, 238 insertions(+), 238 deletions(-) (limited to 'fdio.infra.terraform') diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl index 0abb0e5d51..1fb2f22e43 100644 --- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl +++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl @@ -140,242 +140,242 @@ job "${job_name}" { } } } - group "${job_name}-rls2210" { - restart { - mode = "fail" - } - constraint { - attribute = "$${attr.cpu.arch}" - operator = "!=" - value = "arm64" - } - constraint { - attribute = "$${node.class}" - value = "builder" - } - task "${job_name}-coverage-device" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_device_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - task "${job_name}-coverage-mrr" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_mrr_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - task "${job_name}-coverage-ndrpdr" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_ndrpdr_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - task "${job_name}-coverage-soak" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "coverage_soak_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - task "${job_name}-iterative-mrr" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "iterative_mrr_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - task "${job_name}-iterative-ndrpdr" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "iterative_ndrpdr_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - task "${job_name}-iterative-soak" { - artifact { - source = "git::https://github.com/FDio/csit" - destination = "local/csit" - } - driver = "docker" - config { - image = "${image}" - command = "gluesparksubmit" - args = [ - "--driver-memory", "20g", - "--executor-memory", "20g", - "--executor-cores", "2", - "--master", "local[2]", - "iterative_soak_rls2210.py" - ] - work_dir = "/local/csit/csit.infra.etl" - } - env { - AWS_ACCESS_KEY_ID = "${aws_access_key_id}" - AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" - AWS_DEFAULT_REGION = "${aws_default_region}" - OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" - OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" - OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" - ${ envs } - } - resources { - cpu = ${cpu} - memory = ${memory} - } - } - } +# group "${job_name}-rls2210" { +# restart { +# mode = "fail" +# } +# constraint { +# attribute = "$${attr.cpu.arch}" +# operator = "!=" +# value = "arm64" +# } +# constraint { +# attribute = "$${node.class}" +# value = "builder" +# } +# task "${job_name}-coverage-device" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "coverage_device_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# task "${job_name}-coverage-mrr" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "coverage_mrr_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# task "${job_name}-coverage-ndrpdr" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "coverage_ndrpdr_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# task "${job_name}-coverage-soak" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "coverage_soak_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# task "${job_name}-iterative-mrr" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "iterative_mrr_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# task "${job_name}-iterative-ndrpdr" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "iterative_ndrpdr_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# task "${job_name}-iterative-soak" { +# artifact { +# source = "git::https://github.com/FDio/csit" +# destination = "local/csit" +# } +# driver = "docker" +# config { +# image = "${image}" +# command = "gluesparksubmit" +# args = [ +# "--driver-memory", "20g", +# "--executor-memory", "20g", +# "--executor-cores", "2", +# "--master", "local[2]", +# "iterative_soak_rls2210.py" +# ] +# work_dir = "/local/csit/csit.infra.etl" +# } +# env { +# AWS_ACCESS_KEY_ID = "${aws_access_key_id}" +# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}" +# AWS_DEFAULT_REGION = "${aws_default_region}" +# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}" +# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}" +# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}" +# ${ envs } +# } +# resources { +# cpu = ${cpu} +# memory = ${memory} +# } +# } +# } } -- cgit 1.2.3-korg