aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTibor Frank <tifrank@cisco.com>2023-10-23 12:23:33 +0000
committerTibor Frank <tifrank@cisco.com>2023-10-23 12:23:33 +0000
commit8e577d49173221dec4c4ed9988ada844321cdffe (patch)
tree6fb8288f846454d6ed47590ddbb8951f8899f2fe
parentc2a1edf2e5dc9470cc5d0855257fe6d257cae6b5 (diff)
C-Dash: Improve writing pyarrow schemas
Change-Id: Icecbbbdd1c872fd6bfd2e0362941b047658c8f9b Signed-off-by: Tibor Frank <tifrank@cisco.com>
-rw-r--r--csit.infra.dash/app/cdash/data/data.py11
1 files changed, 7 insertions, 4 deletions
diff --git a/csit.infra.dash/app/cdash/data/data.py b/csit.infra.dash/app/cdash/data/data.py
index 783ebe25ff..1633a02062 100644
--- a/csit.infra.dash/app/cdash/data/data.py
+++ b/csit.infra.dash/app/cdash/data/data.py
@@ -222,12 +222,15 @@ class Data:
pd.api.types.is_string_dtype(itm["<column_name>"]),
pd.api.types.is_string_dtype(itm["telemetry"][0])
)):
- print(pa.Schema.from_pandas(itm))
+ schema = pa.Schema.from_pandas(itm)
pa.parquet.write_metadata(
- pa.Schema.from_pandas(itm),
- f"{C.PATH_TO_SCHEMAS}_tmp_schema"
+ schema, f"{C.PATH_TO_SCHEMAS}_tmp_schema"
)
- print(itm)
+ logging.info(schema.to_string(
+ truncate_metadata=False,
+ show_field_metadata=True,
+ show_schema_metadata=True
+ ))
break
except KeyError:
pass