aboutsummaryrefslogtreecommitdiffstats
path: root/csit.infra.etl/iterative_mrr_rls2410.py
diff options
context:
space:
mode:
authorPeter Mikus <peter.mikus@protonmail.ch>2024-10-01 08:49:00 +0200
committerPeter Mikus <peter.mikus@protonmail.ch>2024-10-01 08:49:00 +0200
commit44ba652225bcf37dc3748ab62636a9a141421227 (patch)
tree0ea9499f5b72f1c0698173fe60654265942bc340 /csit.infra.etl/iterative_mrr_rls2410.py
parent0c08e569e2217acc520c50ff13eddf19d2d3fdc3 (diff)
revert(etl): Manual
Signed-off-by: Peter Mikus <peter.mikus@protonmail.ch> Change-Id: I42ab6a9db31dabcb4928798cb1ef77dd7f37deb5
Diffstat (limited to 'csit.infra.etl/iterative_mrr_rls2410.py')
-rw-r--r--csit.infra.etl/iterative_mrr_rls2410.py12
1 files changed, 3 insertions, 9 deletions
diff --git a/csit.infra.etl/iterative_mrr_rls2410.py b/csit.infra.etl/iterative_mrr_rls2410.py
index 50557fc4d8..d74e6a90f3 100644
--- a/csit.infra.etl/iterative_mrr_rls2410.py
+++ b/csit.infra.etl/iterative_mrr_rls2410.py
@@ -41,15 +41,9 @@ IGNORE_SUFFIX=[
"setup.output.info.json.gz",
"teardown.output.info.json.gz"
]
-#LAST_MODIFIED_END=utc.localize(
-# datetime.strptime(
-# f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
-# "%Y-%m-%d"
-# )
-#)
LAST_MODIFIED_END=utc.localize(
datetime.strptime(
- f"{datetime.now().year}-9-27",
+ f"{datetime.now().year}-{datetime.now().month}-{datetime.now().day}",
"%Y-%m-%d"
)
)
@@ -153,8 +147,8 @@ out_sdf = process_json_to_dataframe("mrr", filtered_paths)
out_sdf.printSchema()
out_sdf = out_sdf \
.withColumn("year", lit(datetime.now().year)) \
- .withColumn("month", lit("9")) \
- .withColumn("day", lit("27")) \
+ .withColumn("month", lit(datetime.now().month)) \
+ .withColumn("day", lit(datetime.now().day)) \
.repartition(1)
try: