diff options
author | pmikus <peter.mikus@protonmail.ch> | 2023-05-24 13:12:36 +0000 |
---|---|---|
committer | pmikus <peter.mikus@protonmail.ch> | 2023-05-24 13:22:15 +0000 |
commit | d164bef0373edfd2b6cc7d4aaa27b928065df3e5 (patch) | |
tree | dfbe8a84eade9d552b7865efe4eba113f2a5089d /csit.infra.etl | |
parent | 1fd3d96de221e7bf281f7ce96dc5cad0170a602d (diff) |
feat(etl): 2306
Signed-off-by: pmikus <peter.mikus@protonmail.ch>
Change-Id: I54e80bc78f6deb5d2774a2ddd7b3b4153ffb106b
Diffstat (limited to 'csit.infra.etl')
-rw-r--r-- | csit.infra.etl/coverage_device_rls2306.py (renamed from csit.infra.etl/coverage_device_rls2302.py) | 4 | ||||
-rw-r--r-- | csit.infra.etl/coverage_hoststack_rls2306.py (renamed from csit.infra.etl/coverage_hoststack_rls2302.py) | 4 | ||||
-rw-r--r-- | csit.infra.etl/coverage_mrr_rls2306.py (renamed from csit.infra.etl/coverage_mrr_rls2302.py) | 4 | ||||
-rw-r--r-- | csit.infra.etl/coverage_ndrpdr_rls2306.py (renamed from csit.infra.etl/coverage_ndrpdr_rls2302.py) | 4 | ||||
-rw-r--r-- | csit.infra.etl/coverage_reconf_rls2306.py (renamed from csit.infra.etl/coverage_reconf_rls2302.py) | 4 | ||||
-rw-r--r-- | csit.infra.etl/coverage_soak_rls2306.py (renamed from csit.infra.etl/coverage_soak_rls2302.py) | 4 | ||||
-rw-r--r-- | csit.infra.etl/iterative_hoststack_rls2306.py (renamed from csit.infra.etl/iterative_hoststack_rls2302.py) | 4 | ||||
-rw-r--r-- | csit.infra.etl/iterative_mrr_rls2306.py (renamed from csit.infra.etl/iterative_mrr_rls2302.py) | 4 | ||||
-rw-r--r-- | csit.infra.etl/iterative_ndrpdr_rls2306.py (renamed from csit.infra.etl/iterative_ndrpdr_rls2302.py) | 4 | ||||
-rw-r--r-- | csit.infra.etl/iterative_reconf_rls2306.py (renamed from csit.infra.etl/iterative_reconf_rls2302.py) | 4 | ||||
-rw-r--r-- | csit.infra.etl/iterative_soak_rls2306.py (renamed from csit.infra.etl/iterative_soak_rls2302.py) | 4 |
11 files changed, 22 insertions, 22 deletions
diff --git a/csit.infra.etl/coverage_device_rls2302.py b/csit.infra.etl/coverage_device_rls2306.py index 7152f95b0a..2d4f59b29c 100644 --- a/csit.infra.etl/coverage_device_rls2302.py +++ b/csit.infra.etl/coverage_device_rls2306.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2302" in path] +filtered_paths = [path for path in paths if "report-coverage-2306" in path] out_sdf = process_json_to_dataframe("device", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2302", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_hoststack_rls2302.py b/csit.infra.etl/coverage_hoststack_rls2306.py index 4ec286f679..9a5f7a5bc9 100644 --- a/csit.infra.etl/coverage_hoststack_rls2302.py +++ b/csit.infra.etl/coverage_hoststack_rls2306.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2302" in path] +filtered_paths = [path for path in paths if "report-coverage-2306" in path] out_sdf = process_json_to_dataframe("hoststack", filtered_paths) out_sdf.show(truncate=False) @@ -155,7 +155,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2302", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_mrr_rls2302.py b/csit.infra.etl/coverage_mrr_rls2306.py index 7ed4c93e79..02ea40ef2b 100644 --- a/csit.infra.etl/coverage_mrr_rls2302.py +++ b/csit.infra.etl/coverage_mrr_rls2306.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2302" in path] +filtered_paths = [path for path in paths if "report-coverage-2306" in path] out_sdf = process_json_to_dataframe("mrr", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2302", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_ndrpdr_rls2302.py b/csit.infra.etl/coverage_ndrpdr_rls2306.py index fd0f496b12..5987e3eaa1 100644 --- a/csit.infra.etl/coverage_ndrpdr_rls2302.py +++ b/csit.infra.etl/coverage_ndrpdr_rls2306.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2302" in path] +filtered_paths = [path for path in paths if "report-coverage-2306" in path] out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2302", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_reconf_rls2302.py b/csit.infra.etl/coverage_reconf_rls2306.py index 76e8c6f9ce..32885744d7 100644 --- a/csit.infra.etl/coverage_reconf_rls2302.py +++ b/csit.infra.etl/coverage_reconf_rls2306.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2302" in path] +filtered_paths = [path for path in paths if "report-coverage-2306" in path] out_sdf = process_json_to_dataframe("reconf", filtered_paths) out_sdf.show(truncate=False) @@ -155,7 +155,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2302", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/coverage_soak_rls2302.py b/csit.infra.etl/coverage_soak_rls2306.py index 7cd171470d..fd4b9fed35 100644 --- a/csit.infra.etl/coverage_soak_rls2302.py +++ b/csit.infra.etl/coverage_soak_rls2306.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-coverage-2302" in path] +filtered_paths = [path for path in paths if "report-coverage-2306" in path] out_sdf = process_json_to_dataframe("soak", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2302", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/coverage_rls2306", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_hoststack_rls2302.py b/csit.infra.etl/iterative_hoststack_rls2306.py index 6ecd723f34..0b179dea88 100644 --- a/csit.infra.etl/iterative_hoststack_rls2302.py +++ b/csit.infra.etl/iterative_hoststack_rls2306.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2302" in path] +filtered_paths = [path for path in paths if "report-iterative-2306" in path] out_sdf = process_json_to_dataframe("hoststack", filtered_paths) out_sdf.show(truncate=False) @@ -155,7 +155,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2302", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_mrr_rls2302.py b/csit.infra.etl/iterative_mrr_rls2306.py index 6ee7971aee..33ffff7ce4 100644 --- a/csit.infra.etl/iterative_mrr_rls2302.py +++ b/csit.infra.etl/iterative_mrr_rls2306.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2302" in path] +filtered_paths = [path for path in paths if "report-iterative-2306" in path] out_sdf = process_json_to_dataframe("mrr", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2302", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_ndrpdr_rls2302.py b/csit.infra.etl/iterative_ndrpdr_rls2306.py index 81e6f489c2..9b3a36be18 100644 --- a/csit.infra.etl/iterative_ndrpdr_rls2302.py +++ b/csit.infra.etl/iterative_ndrpdr_rls2306.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2302" in path] +filtered_paths = [path for path in paths if "report-iterative-2306" in path] out_sdf = process_json_to_dataframe("ndrpdr", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2302", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_reconf_rls2302.py b/csit.infra.etl/iterative_reconf_rls2306.py index 47f27a22a1..a45ac66a83 100644 --- a/csit.infra.etl/iterative_reconf_rls2302.py +++ b/csit.infra.etl/iterative_reconf_rls2306.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2302" in path] +filtered_paths = [path for path in paths if "report-iterative-2306" in path] out_sdf = process_json_to_dataframe("reconf", filtered_paths) out_sdf.show(truncate=False) @@ -155,7 +155,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2302", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", diff --git a/csit.infra.etl/iterative_soak_rls2302.py b/csit.infra.etl/iterative_soak_rls2306.py index ac24b6ba3a..ccc5012199 100644 --- a/csit.infra.etl/iterative_soak_rls2302.py +++ b/csit.infra.etl/iterative_soak_rls2306.py @@ -141,7 +141,7 @@ paths = wr.s3.list_objects( ignore_empty=True ) -filtered_paths = [path for path in paths if "report-iterative-2302" in path] +filtered_paths = [path for path in paths if "report-iterative-2306" in path] out_sdf = process_json_to_dataframe("soak", filtered_paths) out_sdf.printSchema() @@ -154,7 +154,7 @@ out_sdf = out_sdf \ try: wr.s3.to_parquet( df=out_sdf.toPandas(), - path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2302", + path=f"s3://{S3_DOCS_BUCKET}/csit/parquet/iterative_rls2306", dataset=True, partition_cols=["test_type", "year", "month", "day"], compression="snappy", |