From 5c5678c335c59f2e4f65cf8e4f5ee8817fb3e897 Mon Sep 17 00:00:00 2001 From: Peter Mikus Date: Tue, 1 Oct 2024 07:31:51 +0200 Subject: test(etl): Manual run Signed-off-by: Peter Mikus Change-Id: Ibeeb187332e956d1f86267e90dc25a573d3999a1 --- csit.infra.etl/coverage_device_rls2410.py | 12 +++++++++--- csit.infra.etl/coverage_hoststack_rls2410.py | 12 +++++++++--- csit.infra.etl/coverage_mrr_rls2410.py | 12 +++++++++--- csit.infra.etl/coverage_ndrpdr_rls2410.py | 12 +++++++++--- csit.infra.etl/coverage_reconf_rls2410.py | 14 ++++++++++---- csit.infra.etl/coverage_soak_rls2410.py | 12 +++++++++--- csit.infra.etl/iterative_hoststack_rls2410.py | 12 +++++++++--- csit.infra.etl/iterative_mrr_rls2410.py | 12 +++++++++--- csit.infra.etl/iterative_ndrpdr_rls2410.py | 12 +++++++++--- csit.infra.etl/iterative_reconf_rls2410.py | 12 +++++++++--- csit.infra.etl/iterative_soak_rls2410.py | 12 +++++++++--- 11 files changed, 100 insertions(+), 34 deletions(-) diff --git a/csit.infra.etl/coverage_device_rls2410.py b/csit.infra.etl/coverage_device_rls2410.py index 48746daabb..1ffba9076f 100644 --- a/csit.infra.etl/coverage_device_rls2410.py +++ b/csit.infra.etl/coverage_device_rls2410.py @@ -41,9 +41,15 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] +#LAST_MODIFIED_END=utc.localize( +# datetime.strptime( +# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", +# "%Y-%m-%d" +# ) +#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", + f"{datetime.now().year}-9-29", "%Y-%m-%d" ) ) @@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("device", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit(datetime.now().month)) \ - .withColumn("day", lit(datetime.now().day)) \ + .withColumn("month", lit("9")) \ + .withColumn("day", lit("29")) \ .repartition(1) try: diff --git a/csit.infra.etl/coverage_hoststack_rls2410.py b/csit.infra.etl/coverage_hoststack_rls2410.py index 080e998b90..af12d83224 100644 --- a/csit.infra.etl/coverage_hoststack_rls2410.py +++ b/csit.infra.etl/coverage_hoststack_rls2410.py @@ -41,9 +41,15 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] +#LAST_MODIFIED_END=utc.localize( +# datetime.strptime( +# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", +# "%Y-%m-%d" +# ) +#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", + f"{datetime.now().year}-9-29", "%Y-%m-%d" ) ) @@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("hoststack", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit(datetime.now().month)) \ - .withColumn("day", lit(datetime.now().day)) \ + .withColumn("month", lit("9")) \ + .withColumn("day", lit("29")) \ .repartition(1) try: diff --git a/csit.infra.etl/coverage_mrr_rls2410.py b/csit.infra.etl/coverage_mrr_rls2410.py index 9400d8d7fb..c06b306e6f 100644 --- a/csit.infra.etl/coverage_mrr_rls2410.py +++ b/csit.infra.etl/coverage_mrr_rls2410.py @@ -41,9 +41,15 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] +#LAST_MODIFIED_END=utc.localize( +# datetime.strptime( +# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", +# "%Y-%m-%d" +# ) +#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", + f"{datetime.now().year}-9-29", "%Y-%m-%d" ) ) @@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("mrr", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit(datetime.now().month)) \ - .withColumn("day", lit(datetime.now().day)) \ + .withColumn("month", lit("9")) \ + .withColumn("day", lit("29")) \ .repartition(1) try: diff --git a/csit.infra.etl/coverage_ndrpdr_rls2410.py b/csit.infra.etl/coverage_ndrpdr_rls2410.py index 18b7627aa9..303a207f12 100644 --- a/csit.infra.etl/coverage_ndrpdr_rls2410.py +++ b/csit.infra.etl/coverage_ndrpdr_rls2410.py @@ -41,9 +41,15 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] +#LAST_MODIFIED_END=utc.localize( +# datetime.strptime( +# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", +# "%Y-%m-%d" +# ) +#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", + f"{datetime.now().year}-9-29", "%Y-%m-%d" ) ) @@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit(datetime.now().month)) \ - .withColumn("day", lit(datetime.now().day)) \ + .withColumn("month", lit("9")) \ + .withColumn("day", lit("29")) \ .repartition(1) try: diff --git a/csit.infra.etl/coverage_reconf_rls2410.py b/csit.infra.etl/coverage_reconf_rls2410.py index 4e596c24c6..4d45073b19 100644 --- a/csit.infra.etl/coverage_reconf_rls2410.py +++ b/csit.infra.etl/coverage_reconf_rls2410.py @@ -41,9 +41,15 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] +#LAST_MODIFIED_END=utc.localize( +# datetime.strptime( +# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", +# "%Y-%m-%d" +# ) +#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", + f"{datetime.now().year}-9-29", "%Y-%m-%d" ) ) @@ -139,6 +145,7 @@ paths = wr.s3.list_objects( last_modified_end=LAST_MODIFIED_END, ignore_suffix=IGNORE_SUFFIX, ignore_empty=True +) filtered_paths = [path for path in paths if "report-coverage-2410" in path] @@ -146,8 +153,8 @@ out_sdf = process_json_to_dataframe("reconf", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit(datetime.now().month)) \ - .withColumn("day", lit(datetime.now().day)) \ + .withColumn("month", lit("9")) \ + .withColumn("day", lit("29")) \ .repartition(1) try: @@ -158,7 +165,6 @@ try: ) except KeyError: boto3_session = session.Session() -) try: wr.s3.to_parquet( diff --git a/csit.infra.etl/coverage_soak_rls2410.py b/csit.infra.etl/coverage_soak_rls2410.py index 5dc9fb7ed0..79a46a531f 100644 --- a/csit.infra.etl/coverage_soak_rls2410.py +++ b/csit.infra.etl/coverage_soak_rls2410.py @@ -41,9 +41,15 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] +#LAST_MODIFIED_END=utc.localize( +# datetime.strptime( +# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", +# "%Y-%m-%d" +# ) +#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", + f"{datetime.now().year}-9-29", "%Y-%m-%d" ) ) @@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("soak", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit(datetime.now().month)) \ - .withColumn("day", lit(datetime.now().day)) \ + .withColumn("month", lit("9")) \ + .withColumn("day", lit("29")) \ .repartition(1) try: diff --git a/csit.infra.etl/iterative_hoststack_rls2410.py b/csit.infra.etl/iterative_hoststack_rls2410.py index 6d9e3d633f..743579fa24 100644 --- a/csit.infra.etl/iterative_hoststack_rls2410.py +++ b/csit.infra.etl/iterative_hoststack_rls2410.py @@ -41,9 +41,15 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] +#LAST_MODIFIED_END=utc.localize( +# datetime.strptime( +# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", +# "%Y-%m-%d" +# ) +#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", + f"{datetime.now().year}-9-29", "%Y-%m-%d" ) ) @@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("hoststack", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit(datetime.now().month)) \ - .withColumn("day", lit(datetime.now().day)) \ + .withColumn("month", lit("9")) \ + .withColumn("day", lit("29")) \ .repartition(1) try: diff --git a/csit.infra.etl/iterative_mrr_rls2410.py b/csit.infra.etl/iterative_mrr_rls2410.py index d74e6a90f3..431991428b 100644 --- a/csit.infra.etl/iterative_mrr_rls2410.py +++ b/csit.infra.etl/iterative_mrr_rls2410.py @@ -41,9 +41,15 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] +#LAST_MODIFIED_END=utc.localize( +# datetime.strptime( +# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", +# "%Y-%m-%d" +# ) +#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", + f"{datetime.now().year}-9-29", "%Y-%m-%d" ) ) @@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("mrr", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit(datetime.now().month)) \ - .withColumn("day", lit(datetime.now().day)) \ + .withColumn("month", lit("9")) \ + .withColumn("day", lit("29")) \ .repartition(1) try: diff --git a/csit.infra.etl/iterative_ndrpdr_rls2410.py b/csit.infra.etl/iterative_ndrpdr_rls2410.py index 0530dc8ef5..50f103ae60 100644 --- a/csit.infra.etl/iterative_ndrpdr_rls2410.py +++ b/csit.infra.etl/iterative_ndrpdr_rls2410.py @@ -41,9 +41,15 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] +#LAST_MODIFIED_END=utc.localize( +# datetime.strptime( +# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", +# "%Y-%m-%d" +# ) +#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", + f"{datetime.now().year}-9-29", "%Y-%m-%d" ) ) @@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit(datetime.now().month)) \ - .withColumn("day", lit(datetime.now().day)) \ + .withColumn("month", lit("9")) \ + .withColumn("day", lit("29")) \ .repartition(1) try: diff --git a/csit.infra.etl/iterative_reconf_rls2410.py b/csit.infra.etl/iterative_reconf_rls2410.py index 41d2aedcc1..6e4693d898 100644 --- a/csit.infra.etl/iterative_reconf_rls2410.py +++ b/csit.infra.etl/iterative_reconf_rls2410.py @@ -41,9 +41,15 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] +#LAST_MODIFIED_END=utc.localize( +# datetime.strptime( +# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", +# "%Y-%m-%d" +# ) +#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", + f"{datetime.now().year}-9-29", "%Y-%m-%d" ) ) @@ -148,8 +154,8 @@ out_sdf.show(truncate=False) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit(datetime.now().month)) \ - .withColumn("day", lit(datetime.now().day)) \ + .withColumn("month", lit("9")) \ + .withColumn("day", lit("29")) \ .repartition(1) try: diff --git a/csit.infra.etl/iterative_soak_rls2410.py b/csit.infra.etl/iterative_soak_rls2410.py index f581c7c1e9..b4123ba247 100644 --- a/csit.infra.etl/iterative_soak_rls2410.py +++ b/csit.infra.etl/iterative_soak_rls2410.py @@ -41,9 +41,15 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] +#LAST_MODIFIED_END=utc.localize( +# datetime.strptime( +# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", +# "%Y-%m-%d" +# ) +#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", + f"{datetime.now().year}-9-29", "%Y-%m-%d" ) ) @@ -147,8 +153,8 @@ out_sdf = process_json_to_dataframe("soak", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit(datetime.now().month)) \ - .withColumn("day", lit(datetime.now().day)) \ + .withColumn("month", lit("9")) \ + .withColumn("day", lit("29")) \ .repartition(1) try: -- cgit 1.2.3-korg