test(etl): Manual run

Signed-off-by: Peter Mikus <peter.mikus@protonmail.ch>
Change-Id: Ibeeb187332e956d1f86267e90dc25a573d3999a1

11 files changed, 100 insertions, 34 deletions

diff --git a/csit.infra.etl/coverage_device_rls2410.py b/csit.infra.etl/coverage_device_rls2410.py
index 48746daabb..1ffba9076f 100644
--- a/csit.infra.etl/coverage_device_rls2410.py
+++ b/csit.infra.etl/coverage_device_rls2410.py
@@ -41,9 +41,15 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
+#LAST_MODIFIED_END=utc.localize(
+#    datetime.strptime(
+#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+#        "%Y-%m-%d"
+#    )
+#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+        f"{datetime.now().year}-9-29",
         "%Y-%m-%d"
     )
 )
@@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("device", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit(datetime.now().month)) \
-    .withColumn("day", lit(datetime.now().day)) \
+    .withColumn("month", lit("9")) \
+    .withColumn("day", lit("29")) \
     .repartition(1)
 
 try:
diff --git a/csit.infra.etl/coverage_hoststack_rls2410.py b/csit.infra.etl/coverage_hoststack_rls2410.py
index 080e998b90..af12d83224 100644
--- a/csit.infra.etl/coverage_hoststack_rls2410.py
+++ b/csit.infra.etl/coverage_hoststack_rls2410.py
@@ -41,9 +41,15 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
+#LAST_MODIFIED_END=utc.localize(
+#    datetime.strptime(
+#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+#        "%Y-%m-%d"
+#    )
+#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+        f"{datetime.now().year}-9-29",
         "%Y-%m-%d"
     )
 )
@@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit(datetime.now().month)) \
-    .withColumn("day", lit(datetime.now().day)) \
+    .withColumn("month", lit("9")) \
+    .withColumn("day", lit("29")) \
     .repartition(1)
 
 try:
diff --git a/csit.infra.etl/coverage_mrr_rls2410.py b/csit.infra.etl/coverage_mrr_rls2410.py
index 9400d8d7fb..c06b306e6f 100644
--- a/csit.infra.etl/coverage_mrr_rls2410.py
+++ b/csit.infra.etl/coverage_mrr_rls2410.py
@@ -41,9 +41,15 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
+#LAST_MODIFIED_END=utc.localize(
+#    datetime.strptime(
+#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+#        "%Y-%m-%d"
+#    )
+#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+        f"{datetime.now().year}-9-29",
         "%Y-%m-%d"
     )
 )
@@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("mrr", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit(datetime.now().month)) \
-    .withColumn("day", lit(datetime.now().day)) \
+    .withColumn("month", lit("9")) \
+    .withColumn("day", lit("29")) \
     .repartition(1)
 
 try:
diff --git a/csit.infra.etl/coverage_ndrpdr_rls2410.py b/csit.infra.etl/coverage_ndrpdr_rls2410.py
index 18b7627aa9..303a207f12 100644
--- a/csit.infra.etl/coverage_ndrpdr_rls2410.py
+++ b/csit.infra.etl/coverage_ndrpdr_rls2410.py
@@ -41,9 +41,15 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
+#LAST_MODIFIED_END=utc.localize(
+#    datetime.strptime(
+#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+#        "%Y-%m-%d"
+#    )
+#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+        f"{datetime.now().year}-9-29",
         "%Y-%m-%d"
     )
 )
@@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit(datetime.now().month)) \
-    .withColumn("day", lit(datetime.now().day)) \
+    .withColumn("month", lit("9")) \
+    .withColumn("day", lit("29")) \
     .repartition(1)
 
 try:
diff --git a/csit.infra.etl/coverage_reconf_rls2410.py b/csit.infra.etl/coverage_reconf_rls2410.py
index 4e596c24c6..4d45073b19 100644
--- a/csit.infra.etl/coverage_reconf_rls2410.py
+++ b/csit.infra.etl/coverage_reconf_rls2410.py
@@ -41,9 +41,15 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
+#LAST_MODIFIED_END=utc.localize(
+#    datetime.strptime(
+#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+#        "%Y-%m-%d"
+#    )
+#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+        f"{datetime.now().year}-9-29",
         "%Y-%m-%d"
     )
 )
@@ -139,6 +145,7 @@ paths = wr.s3.list_objects(
     last_modified_end=LAST_MODIFIED_END,
     ignore_suffix=IGNORE_SUFFIX,
     ignore_empty=True
+)
 
 filtered_paths = [path for path in paths if "report-coverage-2410" in path]
 
@@ -146,8 +153,8 @@ out_sdf = process_json_to_dataframe("reconf", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit(datetime.now().month)) \
-    .withColumn("day", lit(datetime.now().day)) \
+    .withColumn("month", lit("9")) \
+    .withColumn("day", lit("29")) \
     .repartition(1)
 
 try:
@@ -158,7 +165,6 @@ try:
     )
 except KeyError:
     boto3_session = session.Session()
-)
 
 try:
     wr.s3.to_parquet(
diff --git a/csit.infra.etl/coverage_soak_rls2410.py b/csit.infra.etl/coverage_soak_rls2410.py
index 5dc9fb7ed0..79a46a531f 100644
--- a/csit.infra.etl/coverage_soak_rls2410.py
+++ b/csit.infra.etl/coverage_soak_rls2410.py
@@ -41,9 +41,15 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
+#LAST_MODIFIED_END=utc.localize(
+#    datetime.strptime(
+#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+#        "%Y-%m-%d"
+#    )
+#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+        f"{datetime.now().year}-9-29",
         "%Y-%m-%d"
     )
 )
@@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("soak", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit(datetime.now().month)) \
-    .withColumn("day", lit(datetime.now().day)) \
+    .withColumn("month", lit("9")) \
+    .withColumn("day", lit("29")) \
     .repartition(1)
 
 try:
diff --git a/csit.infra.etl/iterative_hoststack_rls2410.py b/csit.infra.etl/iterative_hoststack_rls2410.py
index 6d9e3d633f..743579fa24 100644
--- a/csit.infra.etl/iterative_hoststack_rls2410.py
+++ b/csit.infra.etl/iterative_hoststack_rls2410.py
@@ -41,9 +41,15 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
+#LAST_MODIFIED_END=utc.localize(
+#    datetime.strptime(
+#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+#        "%Y-%m-%d"
+#    )
+#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+        f"{datetime.now().year}-9-29",
         "%Y-%m-%d"
     )
 )
@@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("hoststack", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit(datetime.now().month)) \
-    .withColumn("day", lit(datetime.now().day)) \
+    .withColumn("month", lit("9")) \
+    .withColumn("day", lit("29")) \
     .repartition(1)
 
 try:
diff --git a/csit.infra.etl/iterative_mrr_rls2410.py b/csit.infra.etl/iterative_mrr_rls2410.py
index d74e6a90f3..431991428b 100644
--- a/csit.infra.etl/iterative_mrr_rls2410.py
+++ b/csit.infra.etl/iterative_mrr_rls2410.py
@@ -41,9 +41,15 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
+#LAST_MODIFIED_END=utc.localize(
+#    datetime.strptime(
+#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+#        "%Y-%m-%d"
+#    )
+#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+        f"{datetime.now().year}-9-29",
         "%Y-%m-%d"
     )
 )
@@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("mrr", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit(datetime.now().month)) \
-    .withColumn("day", lit(datetime.now().day)) \
+    .withColumn("month", lit("9")) \
+    .withColumn("day", lit("29")) \
     .repartition(1)
 
 try:
diff --git a/csit.infra.etl/iterative_ndrpdr_rls2410.py b/csit.infra.etl/iterative_ndrpdr_rls2410.py
index 0530dc8ef5..50f103ae60 100644
--- a/csit.infra.etl/iterative_ndrpdr_rls2410.py
+++ b/csit.infra.etl/iterative_ndrpdr_rls2410.py
@@ -41,9 +41,15 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
+#LAST_MODIFIED_END=utc.localize(
+#    datetime.strptime(
+#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+#        "%Y-%m-%d"
+#    )
+#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+        f"{datetime.now().year}-9-29",
         "%Y-%m-%d"
     )
 )
@@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit(datetime.now().month)) \
-    .withColumn("day", lit(datetime.now().day)) \
+    .withColumn("month", lit("9")) \
+    .withColumn("day", lit("29")) \
     .repartition(1)
 
 try:
diff --git a/csit.infra.etl/iterative_reconf_rls2410.py b/csit.infra.etl/iterative_reconf_rls2410.py
index 41d2aedcc1..6e4693d898 100644
--- a/csit.infra.etl/iterative_reconf_rls2410.py
+++ b/csit.infra.etl/iterative_reconf_rls2410.py
@@ -41,9 +41,15 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
+#LAST_MODIFIED_END=utc.localize(
+#    datetime.strptime(
+#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+#        "%Y-%m-%d"
+#    )
+#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+        f"{datetime.now().year}-9-29",
         "%Y-%m-%d"
     )
 )
@@ -148,8 +154,8 @@ out_sdf.show(truncate=False)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit(datetime.now().month)) \
-    .withColumn("day", lit(datetime.now().day)) \
+    .withColumn("month", lit("9")) \
+    .withColumn("day", lit("29")) \
     .repartition(1)
 
 try:
diff --git a/csit.infra.etl/iterative_soak_rls2410.py b/csit.infra.etl/iterative_soak_rls2410.py
index f581c7c1e9..b4123ba247 100644
--- a/csit.infra.etl/iterative_soak_rls2410.py
+++ b/csit.infra.etl/iterative_soak_rls2410.py
@@ -41,9 +41,15 @@ IGNORE_SUFFIX=[
     "setup.output.info.json.gz",
     "teardown.output.info.json.gz"
 ]
+#LAST_MODIFIED_END=utc.localize(
+#    datetime.strptime(
+#        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+#        "%Y-%m-%d"
+#    )
+#)
 LAST_MODIFIED_END=utc.localize(
     datetime.strptime(
-        f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
+        f"{datetime.now().year}-9-29",
         "%Y-%m-%d"
     )
 )
@@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("soak", filtered_paths)
 out_sdf.printSchema()
 out_sdf = out_sdf \
     .withColumn("year", lit(datetime.now().year)) \
-    .withColumn("month", lit(datetime.now().month)) \
-    .withColumn("day", lit(datetime.now().day)) \
+    .withColumn("month", lit("9")) \
+    .withColumn("day", lit("29")) \
     .repartition(1)
 
 try: