From 44ba652225bcf37dc3748ab62636a9a141421227 Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Tue, 1 Oct 2024 08:49:00 +0200 Subject: revert(etl): Manual Signed-off-by: Peter Mikus Change-Id: I42ab6a9db31dabcb4928798cb1ef77dd7f37deb5 --- csit.infra.etl/coverage_device_rls2410.py | 12 +++--------- csit.infra.etl/coverage_hoststack_rls2410.py | 12 +++--------- csit.infra.etl/coverage_mrr_rls2410.py | 12 +++--------- csit.infra.etl/coverage_ndrpdr_rls2410.py | 12 +++--------- csit.infra.etl/coverage_reconf_rls2410.py | 12 +++--------- csit.infra.etl/coverage_soak_rls2410.py | 12 +++--------- csit.infra.etl/iterative_hoststack_rls2410.py | 12 +++--------- csit.infra.etl/iterative_mrr_rls2410.py | 12 +++--------- csit.infra.etl/iterative_ndrpdr_rls2410.py | 12 +++--------- csit.infra.etl/iterative_reconf_rls2410.py | 12 +++--------- csit.infra.etl/iterative_soak_rls2410.py | 12 +++--------- 11 files changed, 33 insertions(+), 99 deletions(-) diff --git a/csit.infra.etl/coverage_device_rls2410.py b/csit.infra.etl/coverage_device_rls2410.py index 3a0a632f67..48746daabb 100644 --- a/csit.infra.etl/coverage_device_rls2410.py +++ b/csit.infra.etl/coverage_device_rls2410.py @@ -41,15 +41,9 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] -#LAST_MODIFIED_END=utc.localize( -# datetime.strptime( -# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", -# "%Y-%m-%d" -# ) -#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-9-27", + f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", "%Y-%m-%d" ) ) @@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("device", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit("9")) \ - .withColumn("day", lit("27")) \ + .withColumn("month", lit(datetime.now().month)) \ + .withColumn("day", lit(datetime.now().day)) \ .repartition(1) try: diff --git a/csit.infra.etl/coverage_hoststack_rls2410.py b/csit.infra.etl/coverage_hoststack_rls2410.py index abed41694c..080e998b90 100644 --- a/csit.infra.etl/coverage_hoststack_rls2410.py +++ b/csit.infra.etl/coverage_hoststack_rls2410.py @@ -41,15 +41,9 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] -#LAST_MODIFIED_END=utc.localize( -# datetime.strptime( -# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", -# "%Y-%m-%d" -# ) -#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-9-27", + f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", "%Y-%m-%d" ) ) @@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("hoststack", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit("9")) \ - .withColumn("day", lit("27")) \ + .withColumn("month", lit(datetime.now().month)) \ + .withColumn("day", lit(datetime.now().day)) \ .repartition(1) try: diff --git a/csit.infra.etl/coverage_mrr_rls2410.py b/csit.infra.etl/coverage_mrr_rls2410.py index e73cdbb603..9400d8d7fb 100644 --- a/csit.infra.etl/coverage_mrr_rls2410.py +++ b/csit.infra.etl/coverage_mrr_rls2410.py @@ -41,15 +41,9 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] -#LAST_MODIFIED_END=utc.localize( -# datetime.strptime( -# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", -# "%Y-%m-%d" -# ) -#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-9-27", + f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", "%Y-%m-%d" ) ) @@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("mrr", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit("9")) \ - .withColumn("day", lit("27")) \ + .withColumn("month", lit(datetime.now().month)) \ + .withColumn("day", lit(datetime.now().day)) \ .repartition(1) try: diff --git a/csit.infra.etl/coverage_ndrpdr_rls2410.py b/csit.infra.etl/coverage_ndrpdr_rls2410.py index d675ede96d..18b7627aa9 100644 --- a/csit.infra.etl/coverage_ndrpdr_rls2410.py +++ b/csit.infra.etl/coverage_ndrpdr_rls2410.py @@ -41,15 +41,9 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] -#LAST_MODIFIED_END=utc.localize( -# datetime.strptime( -# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", -# "%Y-%m-%d" -# ) -#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-9-27", + f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", "%Y-%m-%d" ) ) @@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit("9")) \ - .withColumn("day", lit("27")) \ + .withColumn("month", lit(datetime.now().month)) \ + .withColumn("day", lit(datetime.now().day)) \ .repartition(1) try: diff --git a/csit.infra.etl/coverage_reconf_rls2410.py b/csit.infra.etl/coverage_reconf_rls2410.py index 2f51eaebfb..e60a3d66ad 100644 --- a/csit.infra.etl/coverage_reconf_rls2410.py +++ b/csit.infra.etl/coverage_reconf_rls2410.py @@ -41,15 +41,9 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] -#LAST_MODIFIED_END=utc.localize( -# datetime.strptime( -# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", -# "%Y-%m-%d" -# ) -#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-9-27", + f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", "%Y-%m-%d" ) ) @@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("reconf", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit("9")) \ - .withColumn("day", lit("27")) \ + .withColumn("month", lit(datetime.now().month)) \ + .withColumn("day", lit(datetime.now().day)) \ .repartition(1) try: diff --git a/csit.infra.etl/coverage_soak_rls2410.py b/csit.infra.etl/coverage_soak_rls2410.py index d89b11b602..5dc9fb7ed0 100644 --- a/csit.infra.etl/coverage_soak_rls2410.py +++ b/csit.infra.etl/coverage_soak_rls2410.py @@ -41,15 +41,9 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] -#LAST_MODIFIED_END=utc.localize( -# datetime.strptime( -# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", -# "%Y-%m-%d" -# ) -#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-9-27", + f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", "%Y-%m-%d" ) ) @@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("soak", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit("9")) \ - .withColumn("day", lit("27")) \ + .withColumn("month", lit(datetime.now().month)) \ + .withColumn("day", lit(datetime.now().day)) \ .repartition(1) try: diff --git a/csit.infra.etl/iterative_hoststack_rls2410.py b/csit.infra.etl/iterative_hoststack_rls2410.py index 9489d3b321..6d9e3d633f 100644 --- a/csit.infra.etl/iterative_hoststack_rls2410.py +++ b/csit.infra.etl/iterative_hoststack_rls2410.py @@ -41,15 +41,9 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] -#LAST_MODIFIED_END=utc.localize( -# datetime.strptime( -# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", -# "%Y-%m-%d" -# ) -#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-9-27", + f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", "%Y-%m-%d" ) ) @@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("hoststack", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit("9")) \ - .withColumn("day", lit("27")) \ + .withColumn("month", lit(datetime.now().month)) \ + .withColumn("day", lit(datetime.now().day)) \ .repartition(1) try: diff --git a/csit.infra.etl/iterative_mrr_rls2410.py b/csit.infra.etl/iterative_mrr_rls2410.py index 50557fc4d8..d74e6a90f3 100644 --- a/csit.infra.etl/iterative_mrr_rls2410.py +++ b/csit.infra.etl/iterative_mrr_rls2410.py @@ -41,15 +41,9 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] -#LAST_MODIFIED_END=utc.localize( -# datetime.strptime( -# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", -# "%Y-%m-%d" -# ) -#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-9-27", + f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", "%Y-%m-%d" ) ) @@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("mrr", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit("9")) \ - .withColumn("day", lit("27")) \ + .withColumn("month", lit(datetime.now().month)) \ + .withColumn("day", lit(datetime.now().day)) \ .repartition(1) try: diff --git a/csit.infra.etl/iterative_ndrpdr_rls2410.py b/csit.infra.etl/iterative_ndrpdr_rls2410.py index 5835fa4c8c..0530dc8ef5 100644 --- a/csit.infra.etl/iterative_ndrpdr_rls2410.py +++ b/csit.infra.etl/iterative_ndrpdr_rls2410.py @@ -41,15 +41,9 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] -#LAST_MODIFIED_END=utc.localize( -# datetime.strptime( -# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", -# "%Y-%m-%d" -# ) -#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-9-27", + f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", "%Y-%m-%d" ) ) @@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit("9")) \ - .withColumn("day", lit("27")) \ + .withColumn("month", lit(datetime.now().month)) \ + .withColumn("day", lit(datetime.now().day)) \ .repartition(1) try: diff --git a/csit.infra.etl/iterative_reconf_rls2410.py b/csit.infra.etl/iterative_reconf_rls2410.py index 0781da21b9..41d2aedcc1 100644 --- a/csit.infra.etl/iterative_reconf_rls2410.py +++ b/csit.infra.etl/iterative_reconf_rls2410.py @@ -41,15 +41,9 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] -#LAST_MODIFIED_END=utc.localize( -# datetime.strptime( -# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", -# "%Y-%m-%d" -# ) -#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-9-27", + f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", "%Y-%m-%d" ) ) @@ -154,8 +148,8 @@ out_sdf.show(truncate=False) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit("9")) \ - .withColumn("day", lit("27")) \ + .withColumn("month", lit(datetime.now().month)) \ + .withColumn("day", lit(datetime.now().day)) \ .repartition(1) try: diff --git a/csit.infra.etl/iterative_soak_rls2410.py b/csit.infra.etl/iterative_soak_rls2410.py index 6388b8cdd0..f581c7c1e9 100644 --- a/csit.infra.etl/iterative_soak_rls2410.py +++ b/csit.infra.etl/iterative_soak_rls2410.py @@ -41,15 +41,9 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] -#LAST_MODIFIED_END=utc.localize( -# datetime.strptime( -# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", -# "%Y-%m-%d" -# ) -#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-9-27", + f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", "%Y-%m-%d" ) ) @@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("soak", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit("9")) \ - .withColumn("day", lit("27")) \ + .withColumn("month", lit(datetime.now().month)) \ + .withColumn("day", lit(datetime.now().day)) \ .repartition(1) try: -- cgit 1.2.3-korg