aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--csit.infra.etl/trending_mrr.json10
-rw-r--r--csit.infra.etl/trending_ndrpdr.json10
-rw-r--r--csit.infra.etl/trending_soak.json10
-rw-r--r--fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl476
4 files changed, 268 insertions, 238 deletions
diff --git a/csit.infra.etl/trending_mrr.json b/csit.infra.etl/trending_mrr.json
index 4e222d33d5..d3d7746451 100644
--- a/csit.infra.etl/trending_mrr.json
+++ b/csit.infra.etl/trending_mrr.json
@@ -130,6 +130,16 @@
},
{
"metadata": {},
+ "name": "telemetry",
+ "nullable": true,
+ "type": {
+ "containsNull": true,
+ "elementType": "string",
+ "type": "array"
+ }
+ },
+ {
+ "metadata": {},
"name": "test_id",
"nullable": true,
"type": "string"
diff --git a/csit.infra.etl/trending_ndrpdr.json b/csit.infra.etl/trending_ndrpdr.json
index fd833aa84c..22cd505671 100644
--- a/csit.infra.etl/trending_ndrpdr.json
+++ b/csit.infra.etl/trending_ndrpdr.json
@@ -658,6 +658,16 @@
},
{
"metadata": {},
+ "name": "telemetry",
+ "nullable": true,
+ "type": {
+ "containsNull": true,
+ "elementType": "string",
+ "type": "array"
+ }
+ },
+ {
+ "metadata": {},
"name": "test_id",
"nullable": true,
"type": "string"
diff --git a/csit.infra.etl/trending_soak.json b/csit.infra.etl/trending_soak.json
index 819d3142d3..1aba81483e 100644
--- a/csit.infra.etl/trending_soak.json
+++ b/csit.infra.etl/trending_soak.json
@@ -200,6 +200,16 @@
},
{
"metadata": {},
+ "name": "telemetry",
+ "nullable": true,
+ "type": {
+ "containsNull": true,
+ "elementType": "string",
+ "type": "array"
+ }
+ },
+ {
+ "metadata": {},
"name": "test_id",
"nullable": true,
"type": "string"
diff --git a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl
index 0abb0e5d51..1fb2f22e43 100644
--- a/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl
+++ b/fdio.infra.terraform/terraform-nomad-pyspark-etl/conf/nomad/etl.hcl.tftpl
@@ -140,242 +140,242 @@ job "${job_name}" {
}
}
}
- group "${job_name}-rls2210" {
- restart {
- mode = "fail"
- }
- constraint {
- attribute = "$${attr.cpu.arch}"
- operator = "!="
- value = "arm64"
- }
- constraint {
- attribute = "$${node.class}"
- value = "builder"
- }
- task "${job_name}-coverage-device" {
- artifact {
- source = "git::https://github.com/FDio/csit"
- destination = "local/csit"
- }
- driver = "docker"
- config {
- image = "${image}"
- command = "gluesparksubmit"
- args = [
- "--driver-memory", "20g",
- "--executor-memory", "20g",
- "--executor-cores", "2",
- "--master", "local[2]",
- "coverage_device_rls2210.py"
- ]
- work_dir = "/local/csit/csit.infra.etl"
- }
- env {
- AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
- AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
- AWS_DEFAULT_REGION = "${aws_default_region}"
- OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
- OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
- OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
- ${ envs }
- }
- resources {
- cpu = ${cpu}
- memory = ${memory}
- }
- }
- task "${job_name}-coverage-mrr" {
- artifact {
- source = "git::https://github.com/FDio/csit"
- destination = "local/csit"
- }
- driver = "docker"
- config {
- image = "${image}"
- command = "gluesparksubmit"
- args = [
- "--driver-memory", "20g",
- "--executor-memory", "20g",
- "--executor-cores", "2",
- "--master", "local[2]",
- "coverage_mrr_rls2210.py"
- ]
- work_dir = "/local/csit/csit.infra.etl"
- }
- env {
- AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
- AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
- AWS_DEFAULT_REGION = "${aws_default_region}"
- OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
- OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
- OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
- ${ envs }
- }
- resources {
- cpu = ${cpu}
- memory = ${memory}
- }
- }
- task "${job_name}-coverage-ndrpdr" {
- artifact {
- source = "git::https://github.com/FDio/csit"
- destination = "local/csit"
- }
- driver = "docker"
- config {
- image = "${image}"
- command = "gluesparksubmit"
- args = [
- "--driver-memory", "20g",
- "--executor-memory", "20g",
- "--executor-cores", "2",
- "--master", "local[2]",
- "coverage_ndrpdr_rls2210.py"
- ]
- work_dir = "/local/csit/csit.infra.etl"
- }
- env {
- AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
- AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
- AWS_DEFAULT_REGION = "${aws_default_region}"
- OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
- OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
- OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
- ${ envs }
- }
- resources {
- cpu = ${cpu}
- memory = ${memory}
- }
- }
- task "${job_name}-coverage-soak" {
- artifact {
- source = "git::https://github.com/FDio/csit"
- destination = "local/csit"
- }
- driver = "docker"
- config {
- image = "${image}"
- command = "gluesparksubmit"
- args = [
- "--driver-memory", "20g",
- "--executor-memory", "20g",
- "--executor-cores", "2",
- "--master", "local[2]",
- "coverage_soak_rls2210.py"
- ]
- work_dir = "/local/csit/csit.infra.etl"
- }
- env {
- AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
- AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
- AWS_DEFAULT_REGION = "${aws_default_region}"
- OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
- OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
- OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
- ${ envs }
- }
- resources {
- cpu = ${cpu}
- memory = ${memory}
- }
- }
- task "${job_name}-iterative-mrr" {
- artifact {
- source = "git::https://github.com/FDio/csit"
- destination = "local/csit"
- }
- driver = "docker"
- config {
- image = "${image}"
- command = "gluesparksubmit"
- args = [
- "--driver-memory", "20g",
- "--executor-memory", "20g",
- "--executor-cores", "2",
- "--master", "local[2]",
- "iterative_mrr_rls2210.py"
- ]
- work_dir = "/local/csit/csit.infra.etl"
- }
- env {
- AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
- AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
- AWS_DEFAULT_REGION = "${aws_default_region}"
- OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
- OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
- OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
- ${ envs }
- }
- resources {
- cpu = ${cpu}
- memory = ${memory}
- }
- }
- task "${job_name}-iterative-ndrpdr" {
- artifact {
- source = "git::https://github.com/FDio/csit"
- destination = "local/csit"
- }
- driver = "docker"
- config {
- image = "${image}"
- command = "gluesparksubmit"
- args = [
- "--driver-memory", "20g",
- "--executor-memory", "20g",
- "--executor-cores", "2",
- "--master", "local[2]",
- "iterative_ndrpdr_rls2210.py"
- ]
- work_dir = "/local/csit/csit.infra.etl"
- }
- env {
- AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
- AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
- AWS_DEFAULT_REGION = "${aws_default_region}"
- OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
- OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
- OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
- ${ envs }
- }
- resources {
- cpu = ${cpu}
- memory = ${memory}
- }
- }
- task "${job_name}-iterative-soak" {
- artifact {
- source = "git::https://github.com/FDio/csit"
- destination = "local/csit"
- }
- driver = "docker"
- config {
- image = "${image}"
- command = "gluesparksubmit"
- args = [
- "--driver-memory", "20g",
- "--executor-memory", "20g",
- "--executor-cores", "2",
- "--master", "local[2]",
- "iterative_soak_rls2210.py"
- ]
- work_dir = "/local/csit/csit.infra.etl"
- }
- env {
- AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
- AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
- AWS_DEFAULT_REGION = "${aws_default_region}"
- OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
- OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
- OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
- ${ envs }
- }
- resources {
- cpu = ${cpu}
- memory = ${memory}
- }
- }
- }
+# group "${job_name}-rls2210" {
+# restart {
+# mode = "fail"
+# }
+# constraint {
+# attribute = "$${attr.cpu.arch}"
+# operator = "!="
+# value = "arm64"
+# }
+# constraint {
+# attribute = "$${node.class}"
+# value = "builder"
+# }
+# task "${job_name}-coverage-device" {
+# artifact {
+# source = "git::https://github.com/FDio/csit"
+# destination = "local/csit"
+# }
+# driver = "docker"
+# config {
+# image = "${image}"
+# command = "gluesparksubmit"
+# args = [
+# "--driver-memory", "20g",
+# "--executor-memory", "20g",
+# "--executor-cores", "2",
+# "--master", "local[2]",
+# "coverage_device_rls2210.py"
+# ]
+# work_dir = "/local/csit/csit.infra.etl"
+# }
+# env {
+# AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
+# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
+# AWS_DEFAULT_REGION = "${aws_default_region}"
+# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
+# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
+# ${ envs }
+# }
+# resources {
+# cpu = ${cpu}
+# memory = ${memory}
+# }
+# }
+# task "${job_name}-coverage-mrr" {
+# artifact {
+# source = "git::https://github.com/FDio/csit"
+# destination = "local/csit"
+# }
+# driver = "docker"
+# config {
+# image = "${image}"
+# command = "gluesparksubmit"
+# args = [
+# "--driver-memory", "20g",
+# "--executor-memory", "20g",
+# "--executor-cores", "2",
+# "--master", "local[2]",
+# "coverage_mrr_rls2210.py"
+# ]
+# work_dir = "/local/csit/csit.infra.etl"
+# }
+# env {
+# AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
+# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
+# AWS_DEFAULT_REGION = "${aws_default_region}"
+# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
+# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
+# ${ envs }
+# }
+# resources {
+# cpu = ${cpu}
+# memory = ${memory}
+# }
+# }
+# task "${job_name}-coverage-ndrpdr" {
+# artifact {
+# source = "git::https://github.com/FDio/csit"
+# destination = "local/csit"
+# }
+# driver = "docker"
+# config {
+# image = "${image}"
+# command = "gluesparksubmit"
+# args = [
+# "--driver-memory", "20g",
+# "--executor-memory", "20g",
+# "--executor-cores", "2",
+# "--master", "local[2]",
+# "coverage_ndrpdr_rls2210.py"
+# ]
+# work_dir = "/local/csit/csit.infra.etl"
+# }
+# env {
+# AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
+# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
+# AWS_DEFAULT_REGION = "${aws_default_region}"
+# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
+# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
+# ${ envs }
+# }
+# resources {
+# cpu = ${cpu}
+# memory = ${memory}
+# }
+# }
+# task "${job_name}-coverage-soak" {
+# artifact {
+# source = "git::https://github.com/FDio/csit"
+# destination = "local/csit"
+# }
+# driver = "docker"
+# config {
+# image = "${image}"
+# command = "gluesparksubmit"
+# args = [
+# "--driver-memory", "20g",
+# "--executor-memory", "20g",
+# "--executor-cores", "2",
+# "--master", "local[2]",
+# "coverage_soak_rls2210.py"
+# ]
+# work_dir = "/local/csit/csit.infra.etl"
+# }
+# env {
+# AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
+# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
+# AWS_DEFAULT_REGION = "${aws_default_region}"
+# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
+# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
+# ${ envs }
+# }
+# resources {
+# cpu = ${cpu}
+# memory = ${memory}
+# }
+# }
+# task "${job_name}-iterative-mrr" {
+# artifact {
+# source = "git::https://github.com/FDio/csit"
+# destination = "local/csit"
+# }
+# driver = "docker"
+# config {
+# image = "${image}"
+# command = "gluesparksubmit"
+# args = [
+# "--driver-memory", "20g",
+# "--executor-memory", "20g",
+# "--executor-cores", "2",
+# "--master", "local[2]",
+# "iterative_mrr_rls2210.py"
+# ]
+# work_dir = "/local/csit/csit.infra.etl"
+# }
+# env {
+# AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
+# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
+# AWS_DEFAULT_REGION = "${aws_default_region}"
+# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
+# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
+# ${ envs }
+# }
+# resources {
+# cpu = ${cpu}
+# memory = ${memory}
+# }
+# }
+# task "${job_name}-iterative-ndrpdr" {
+# artifact {
+# source = "git::https://github.com/FDio/csit"
+# destination = "local/csit"
+# }
+# driver = "docker"
+# config {
+# image = "${image}"
+# command = "gluesparksubmit"
+# args = [
+# "--driver-memory", "20g",
+# "--executor-memory", "20g",
+# "--executor-cores", "2",
+# "--master", "local[2]",
+# "iterative_ndrpdr_rls2210.py"
+# ]
+# work_dir = "/local/csit/csit.infra.etl"
+# }
+# env {
+# AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
+# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
+# AWS_DEFAULT_REGION = "${aws_default_region}"
+# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
+# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
+# ${ envs }
+# }
+# resources {
+# cpu = ${cpu}
+# memory = ${memory}
+# }
+# }
+# task "${job_name}-iterative-soak" {
+# artifact {
+# source = "git::https://github.com/FDio/csit"
+# destination = "local/csit"
+# }
+# driver = "docker"
+# config {
+# image = "${image}"
+# command = "gluesparksubmit"
+# args = [
+# "--driver-memory", "20g",
+# "--executor-memory", "20g",
+# "--executor-cores", "2",
+# "--master", "local[2]",
+# "iterative_soak_rls2210.py"
+# ]
+# work_dir = "/local/csit/csit.infra.etl"
+# }
+# env {
+# AWS_ACCESS_KEY_ID = "${aws_access_key_id}"
+# AWS_SECRET_ACCESS_KEY = "${aws_secret_access_key}"
+# AWS_DEFAULT_REGION = "${aws_default_region}"
+# OUT_AWS_ACCESS_KEY_ID = "${out_aws_access_key_id}"
+# OUT_AWS_SECRET_ACCESS_KEY = "${out_aws_secret_access_key}"
+# OUT_AWS_DEFAULT_REGION = "${out_aws_default_region}"
+# ${ envs }
+# }
+# resources {
+# cpu = ${cpu}
+# memory = ${memory}
+# }
+# }
+# }
}