diff options
Diffstat (limited to 'csit.infra.etl/coverage_reconf_rls2410.py')
-rw-r--r-- | csit.infra.etl/coverage_reconf_rls2410.py | 14 |
1 files changed, 10 insertions, 4 deletions
diff --git a/csit.infra.etl/coverage_reconf_rls2410.py b/csit.infra.etl/coverage_reconf_rls2410.py index 4e596c24c6..4d45073b19 100644 --- a/csit.infra.etl/coverage_reconf_rls2410.py +++ b/csit.infra.etl/coverage_reconf_rls2410.py @@ -41,9 +41,15 @@ IGNORE_SUFFIX=[ "setup.output.info.json.gz", "teardown.output.info.json.gz" ] +#LAST_MODIFIED_END=utc.localize( +# datetime.strptime( +# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", +# "%Y-%m-%d" +# ) +#) LAST_MODIFIED_END=utc.localize( datetime.strptime( - f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}", + f"{datetime.now().year}-9-29", "%Y-%m-%d" ) ) @@ -139,6 +145,7 @@ paths = wr.s3.list_objects( last_modified_end=LAST_MODIFIED_END, ignore_suffix=IGNORE_SUFFIX, ignore_empty=True +) filtered_paths = [path for path in paths if "report-coverage-2410" in path] @@ -146,8 +153,8 @@ out_sdf = process_json_to_dataframe("reconf", filtered_paths) out_sdf.printSchema() out_sdf = out_sdf \ .withColumn("year", lit(datetime.now().year)) \ - .withColumn("month", lit(datetime.now().month)) \ - .withColumn("day", lit(datetime.now().day)) \ + .withColumn("month", lit("9")) \ + .withColumn("day", lit("29")) \ .repartition(1) try: @@ -158,7 +165,6 @@ try: ) except KeyError: boto3_session = session.Session() -) try: wr.s3.to_parquet( |