Incompatible dtypes between zarr 3 and zarr 2 #53
Description
I wrote a geff zarr with zarr 3.0.8 installed, but the zarr format set to 2. The node ids are strings so the dtype on that array is <U6. When trying to read that array with geff with zarr 2.18.7 installed, I get the following error, but only after loading data into memory, e.g. z_array[:]
I think it's likely that this is a zarr problem, not a geff problem, but I want to document it here just in case.
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
Cell In[3], line 1
----> 1 z[:]
File ~/Code/c-silicans-experiments/.pixi/envs/dev/lib/python3.13/site-packages/zarr/core.py:799, in Array.__getitem__(self, selection)
797 result = self.get_orthogonal_selection(pure_selection, fields=fields)
798 else:
--> 799 result = self.get_basic_selection(pure_selection, fields=fields)
800 return result
File ~/Code/c-silicans-experiments/.pixi/envs/dev/lib/python3.13/site-packages/zarr/core.py:925, in Array.get_basic_selection(self, selection, out, fields)
923 return self._get_basic_selection_zd(selection=selection, out=out, fields=fields)
924 else:
--> 925 return self._get_basic_selection_nd(selection=selection, out=out, fields=fields)
File ~/Code/c-silicans-experiments/.pixi/envs/dev/lib/python3.13/site-packages/zarr/core.py:967, in Array._get_basic_selection_nd(self, selection, out, fields)
961 def _get_basic_selection_nd(self, selection, out=None, fields=None):
962 # implementation of basic selection for array with at least one dimension
963
964 # setup indexer
965 indexer = BasicIndexer(selection, self)
--> 967 return self._get_selection(indexer=indexer, out=out, fields=fields)
File ~/Code/c-silicans-experiments/.pixi/envs/dev/lib/python3.13/site-packages/zarr/core.py:1342, in Array._get_selection(self, indexer, out, fields)
1339 if math.prod(out_shape) > 0:
1340 # allow storage to get multiple items at once
1341 lchunk_coords, lchunk_selection, lout_selection = zip(*indexer)
-> 1342 self._chunk_getitems(
1343 lchunk_coords,
1344 lchunk_selection,
1345 out,
1346 lout_selection,
1347 drop_axes=indexer.drop_axes,
1348 fields=fields,
1349 )
1350 if out.shape:
1351 return out
File ~/Code/c-silicans-experiments/.pixi/envs/dev/lib/python3.13/site-packages/zarr/core.py:2187, in Array._chunk_getitems(self, lchunk_coords, lchunk_selection, out, lout_selection, drop_axes, fields)
2185 for ckey, chunk_select, out_select in zip(ckeys, lchunk_selection, lout_selection):
2186 if ckey in cdatas:
-> 2187 self._process_chunk(
2188 out,
2189 cdatas[ckey],
2190 chunk_select,
2191 drop_axes,
2192 out_is_ndarray,
2193 fields,
2194 out_select,
2195 partial_read_decode=partial_read_decode,
2196 )
2197 else:
2198 # check exception type
2199 if self._fill_value is not None:
File ~/Code/c-silicans-experiments/.pixi/envs/dev/lib/python3.13/site-packages/zarr/core.py:2100, in Array._process_chunk(self, out, cdata, chunk_selection, drop_axes, out_is_ndarray, fields, out_selection, partial_read_decode)
2098 except ArrayIndexError:
2099 cdata = cdata.read_full()
-> 2100 chunk = self._decode_chunk(cdata)
2102 # select data from chunk
2103 if fields:
File ~/Code/c-silicans-experiments/.pixi/envs/dev/lib/python3.13/site-packages/zarr/core.py:2370, in Array._decode_chunk(self, cdata, start, nitems, expected_shape)
2367 # special case object dtype, because incorrect handling can lead to
2368 # segfaults and other bad things happening
2369 if self._dtype != object:
-> 2370 chunk = chunk.view(self._dtype)
2371 elif chunk.dtype != object:
2372 # If we end up here, someone must have hacked around with the filters.
2373 # We cannot deal with object arrays unless there is an object
2374 # codec in the filter chain, i.e., a filter that converts from object
2375 # array to something else during encoding, and converts back to object
2376 # array during decoding.
2377 raise RuntimeError("cannot read object array without object codec")
File ~/Code/c-silicans-experiments/.pixi/envs/dev/lib/python3.13/site-packages/numpy/_core/_internal.py:564, in _view_is_safe(oldtype, newtype)
561 return
563 if newtype.hasobject or oldtype.hasobject:
--> 564 raise TypeError("Cannot change data-type for array of references.")
565 return
TypeError: Cannot change data-type for array of references.