在 read_parquet(dtype_backend="pyarrow") 中设置 index 会引发 TypeError: 不了解嵌套 Arrow dtypes 的数据类型 '...'

import tempfile import pandas as pd import pyarrow as pa import dask.dataframe as dd member_type = pa.list_(pa.struct([("type", pa.string()), ("ref", pa.int64()), ("role", pa.string())])) pdf = pd.DataFrame({ "id": pd.array([1, 1, 2, 2, 2, 3] * 200, dtype="int64[pyarrow]"), "members": pd.array( [[{"type": "way", "ref": 10, "role": "outer"}], None, [{"type": "node", "ref": 12, "role": ""}]] * 400, dtype=pd.ArrowDtype(member_type)), })

tmp = tempfile.mkdtemp() dd.from_pandas(pdf, npartitions=4).to_parquet(tmp, write_metadata_file=False) ddf = dd.read_parquet(tmp, dtype_backend="pyarrow") ddf.compute() # OK ddf.map_partitions(len).compute() # OK ddf.set_index("id").compute() # TypeError: data type 'list<item: struct<...>>[pyarrow]' not understood