aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPeter Mikus <pmikus@cisco.com>2022-05-25 09:02:29 +0200
committerPeter Mikus <pmikus@cisco.com>2022-05-25 09:02:29 +0200
commit4c6b808980195b3f474607b9412d0917dedac05a (patch)
tree10125495649c0a7bb1b2f7a8bf92a92e30693f5a
parenta201f3d3a5fdb9ccb86d57664cfd1de4bf484d36 (diff)
feat(etl): report 2206
Signed-off-by: Peter Mikus <pmikus@cisco.com> Change-Id: I61fc4187ecd9948c2a8091229c5670f9c73e82ae
-rw-r--r--csit.infra.etl/coverage_device.json12
-rw-r--r--csit.infra.etl/coverage_mrr.json12
-rw-r--r--csit.infra.etl/coverage_ndrpdr.json12
-rw-r--r--csit.infra.etl/coverage_rls2206.py (renamed from csit.infra.etl/coverage_rls2202.py)4
-rw-r--r--csit.infra.etl/coverage_soak.json12
-rw-r--r--csit.infra.etl/iterative_mrr.json12
-rw-r--r--csit.infra.etl/iterative_ndrpdr.json12
-rw-r--r--csit.infra.etl/iterative_rls2206.py (renamed from csit.infra.etl/iterative_rls2202.py)4
-rw-r--r--csit.infra.etl/iterative_soak.json12
9 files changed, 88 insertions, 4 deletions
diff --git a/csit.infra.etl/coverage_device.json b/csit.infra.etl/coverage_device.json
index 12989b30fa..2a4327c05f 100644
--- a/csit.infra.etl/coverage_device.json
+++ b/csit.infra.etl/coverage_device.json
@@ -32,6 +32,18 @@
},
{
"metadata": {},
+ "name": "tg_type",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
+ "name": "tg_version",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
"name": "start_time",
"nullable": true,
"type": "timestamp"
diff --git a/csit.infra.etl/coverage_mrr.json b/csit.infra.etl/coverage_mrr.json
index 13b80ec82e..531fcfc62f 100644
--- a/csit.infra.etl/coverage_mrr.json
+++ b/csit.infra.etl/coverage_mrr.json
@@ -32,6 +32,18 @@
},
{
"metadata": {},
+ "name": "tg_type",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
+ "name": "tg_version",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
"name": "result",
"nullable": true,
"type": {
diff --git a/csit.infra.etl/coverage_ndrpdr.json b/csit.infra.etl/coverage_ndrpdr.json
index f188321bfb..37c45820ea 100644
--- a/csit.infra.etl/coverage_ndrpdr.json
+++ b/csit.infra.etl/coverage_ndrpdr.json
@@ -32,6 +32,18 @@
},
{
"metadata": {},
+ "name": "tg_type",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
+ "name": "tg_version",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
"name": "result",
"nullable": true,
"type": {
diff --git a/csit.infra.etl/coverage_rls2202.py b/csit.infra.etl/coverage_rls2206.py
index 97b0a12b88..4df7898fe1 100644
--- a/csit.infra.etl/coverage_rls2202.py
+++ b/csit.infra.etl/coverage_rls2206.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-coverage-2202" in path]
+filtered_paths = [path for path in paths if "report-coverage-2206" in path]
for schema_name in ["mrr", "ndrpdr", "soak", "device"]:
out_sdf = process_json_to_dataframe(schema_name, filtered_paths)
@@ -156,7 +156,7 @@ for schema_name in ["mrr", "ndrpdr", "soak", "device"]:
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2202",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2206",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
diff --git a/csit.infra.etl/coverage_soak.json b/csit.infra.etl/coverage_soak.json
index 59eaec2e9d..5691e8bf5b 100644
--- a/csit.infra.etl/coverage_soak.json
+++ b/csit.infra.etl/coverage_soak.json
@@ -32,6 +32,18 @@
},
{
"metadata": {},
+ "name": "tg_type",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
+ "name": "tg_version",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
"name": "result",
"nullable": true,
"type": {
diff --git a/csit.infra.etl/iterative_mrr.json b/csit.infra.etl/iterative_mrr.json
index 13b80ec82e..531fcfc62f 100644
--- a/csit.infra.etl/iterative_mrr.json
+++ b/csit.infra.etl/iterative_mrr.json
@@ -32,6 +32,18 @@
},
{
"metadata": {},
+ "name": "tg_type",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
+ "name": "tg_version",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
"name": "result",
"nullable": true,
"type": {
diff --git a/csit.infra.etl/iterative_ndrpdr.json b/csit.infra.etl/iterative_ndrpdr.json
index f188321bfb..37c45820ea 100644
--- a/csit.infra.etl/iterative_ndrpdr.json
+++ b/csit.infra.etl/iterative_ndrpdr.json
@@ -32,6 +32,18 @@
},
{
"metadata": {},
+ "name": "tg_type",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
+ "name": "tg_version",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
"name": "result",
"nullable": true,
"type": {
diff --git a/csit.infra.etl/iterative_rls2202.py b/csit.infra.etl/iterative_rls2206.py
index 13b6f4272c..f4625cf9a8 100644
--- a/csit.infra.etl/iterative_rls2202.py
+++ b/csit.infra.etl/iterative_rls2206.py
@@ -141,7 +141,7 @@ paths = wr.s3.list_objects(
ignore_empty=True
)
-filtered_paths = [path for path in paths if "report-iterative-2202" in path]
+filtered_paths = [path for path in paths if "report-iterative-2206" in path]
for schema_name in ["mrr", "ndrpdr", "soak"]:
out_sdf = process_json_to_dataframe(schema_name, filtered_paths)
@@ -156,7 +156,7 @@ for schema_name in ["mrr", "ndrpdr", "soak"]:
try:
wr.s3.to_parquet(
df=out_sdf.toPandas(),
- path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2202",
+ path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2206",
dataset=True,
partition_cols=["test_type", "year", "month", "day"],
compression="snappy",
diff --git a/csit.infra.etl/iterative_soak.json b/csit.infra.etl/iterative_soak.json
index 59eaec2e9d..5691e8bf5b 100644
--- a/csit.infra.etl/iterative_soak.json
+++ b/csit.infra.etl/iterative_soak.json
@@ -32,6 +32,18 @@
},
{
"metadata": {},
+ "name": "tg_type",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
+ "name": "tg_version",
+ "nullable": true,
+ "type": "string"
+ },
+ {
+ "metadata": {},
"name": "result",
"nullable": true,
"type": {