diff options
| author |   Tibor Frank <tifrank@cisco.com> | 2023-10-23 12:23:33 +0000 |
|---|---|---|
| committer | Tibor Frank <tifrank@cisco.com> | 2023-10-23 12:23:33 +0000 |
| commit | 8e577d49173221dec4c4ed9988ada844321cdffe (patch) | |
| tree | 6fb8288f846454d6ed47590ddbb8951f8899f2fe | |
| parent | c2a1edf2e5dc9470cc5d0855257fe6d257cae6b5 (diff) | |
C-Dash: Improve writing pyarrow schemas
Change-Id: Icecbbbdd1c872fd6bfd2e0362941b047658c8f9b
Signed-off-by: Tibor Frank <tifrank@cisco.com>
| -rw-r--r-- | csit.infra.dash/app/cdash/data/data.py | 11 |
1 files changed, 7 insertions, 4 deletions
diff --git a/csit.infra.dash/app/cdash/data/data.py b/csit.infra.dash/app/cdash/data/data.py index 783ebe25ff..1633a02062 100644 --- a/csit.infra.dash/app/cdash/data/data.py +++ b/csit.infra.dash/app/cdash/data/data.py @@ -222,12 +222,15 @@ class Data: pd.api.types.is_string_dtype(itm["<column_name>"]), pd.api.types.is_string_dtype(itm["telemetry"][0]) )): - print(pa.Schema.from_pandas(itm)) + schema = pa.Schema.from_pandas(itm) pa.parquet.write_metadata( - pa.Schema.from_pandas(itm), - f"{C.PATH_TO_SCHEMAS}_tmp_schema" + schema, f"{C.PATH_TO_SCHEMAS}_tmp_schema" ) - print(itm) + logging.info(schema.to_string( + truncate_metadata=False, + show_field_metadata=True, + show_schema_metadata=True + )) break except KeyError: pass |