Refuse buffers of another typesize in Python prefilters and postfilters

src/blosc2/blosc2_ext.pyx

@@ -825,12 +825,14 @@ ctypedef struct user_filters_udata:
     int input_cdtype
     int output_cdtype
     int32_t chunkshape
+    int32_t itemsize  # of both dtypes
 
 ctypedef struct filler_udata:
     char* py_func
     uintptr_t inputs_id
     int output_cdtype
     int32_t chunkshape
+    int32_t itemsize  # of the output dtype
 
 ctypedef struct udf_udata:
     char* py_func
@@ -4793,6 +4795,7 @@ cdef class SChunk:
         postf_udata.input_cdtype = dtype_input.num
         postf_udata.output_cdtype = dtype_output.num
         postf_udata.chunkshape = self.schunk.chunksize // self.schunk.typesize
+        postf_udata.itemsize = dtype_input.itemsize
 
         postparams.user_data = postf_udata
         dparams.postparams = postparams
@@ -4847,6 +4850,7 @@ cdef class SChunk:
         fill_udata.inputs_id = inputs_id
         fill_udata.output_cdtype = np.dtype(dtype_output).num
         fill_udata.chunkshape = self.schunk.chunksize // self.schunk.typesize
+        fill_udata.itemsize = np.dtype(dtype_output).itemsize
 
         preparams.user_data = fill_udata
         cparams.preparams = preparams
@@ -4881,6 +4885,7 @@ cdef class SChunk:
         pref_udata.input_cdtype = dtype_input.num
         pref_udata.output_cdtype = dtype_output.num
         pref_udata.chunkshape = self.schunk.chunksize // self.schunk.typesize
+        pref_udata.itemsize = dtype_input.itemsize
 
         preparams.user_data = pref_udata
         cparams.preparams = preparams
@@ -5002,9 +5007,18 @@ cdef class SChunk:
             blosc2_schunk_free(schunk_ptr)
 
 
+# The arrays that general_postfilter, general_filler and general_prefilter hand to Python have
+# size // typesize items of the callback's dtype, so they only fit buffers whose typesize is the
+# dtype's itemsize.  Other buffers (C-Blosc2 before Blosc/c-blosc2#857 passes vlmetalayer content
+# to the postfilter, and a chunk can have another typesize than its SChunk) get a ValueError, raised
+# before anything is written and re-raised by the caller after the C call (_raise_callback_error()).
+
 # postfilter
 cdef int general_postfilter(blosc2_postfilter_params *params) with gil:
     cdef user_filters_udata *udata = <user_filters_udata *> params.user_data
+    if params.typesize != udata.itemsize:
+        raise ValueError(f"postfilter: data typesize {params.typesize} does not match "
+                         f"dtype itemsize {udata.itemsize}")
     cdef int nd = 1
     cdef np.npy_intp dims = params.size // params.typesize
     input = np.PyArray_SimpleNewFromData(nd, &dims, udata.input_cdtype, <void*>params.input)
@@ -5018,6 +5032,9 @@ cdef int general_postfilter(blosc2_postfilter_params *params) with gil:
 # filler
 cdef int general_filler(blosc2_prefilter_params *params):
     cdef filler_udata *udata = <filler_udata *> params.user_data
+    if params.output_typesize != udata.itemsize:
+        raise ValueError(f"filler: data typesize {params.output_typesize} does not match "
+                         f"dtype itemsize {udata.itemsize}")
     cdef int nd = 1
     cdef np.npy_intp dims = params.output_size // params.output_typesize
 
@@ -5666,6 +5683,10 @@ def nelem_from_inputs(inputs_tuple, nelem=None):
 # prefilter
 cdef int general_prefilter(blosc2_prefilter_params *params):
     cdef user_filters_udata *udata = <user_filters_udata *> params.user_data
+    # Typesize check: see general_postfilter
+    if params.output_typesize != udata.itemsize:
+        raise ValueError(f"prefilter: data typesize {params.output_typesize} does not match "
+                         f"dtype itemsize {udata.itemsize}")
     cdef int nd = 1
     cdef np.npy_intp dims = params.output_size // params.output_typesize
 

src/blosc2/schunk.py

@@ -1538,7 +1538,8 @@ def postfilter(self, input_dtype: np.dtype, output_dtype: np.dtype = None) -> No
         * `nthreads` must be 1 when decompressing.
 
         * The :paramref:`input_dtype` itemsize must be the same as the
-          :paramref:`output_dtype` itemsize.
+          :paramref:`output_dtype` itemsize, and the typesize of the chunks:
+          decompressing a chunk with another typesize fails.
 
         See Also
         --------
@@ -1640,6 +1641,7 @@ def filler(self, inputs_tuple: tuple[tuple], schunk_dtype: np.dtype, nelem: int
         Notes
         -----
         * Compression `nthreads` must be 1 when using this.
+        * The :paramref:`schunk_dtype` itemsize must be the typesize of the SChunk.
         * This does not need to be removed from the created `SChunk` instance.
 
         See Also
@@ -1674,16 +1676,19 @@ def initialize(func):
                 raise ValueError("Cannot apply a filler to a non empty SChunk")
             nelem_ = blosc2_ext.nelem_from_inputs(inputs_tuple, nelem)
             super(SChunk, self)._set_filler(func, id(inputs_tuple), schunk_dtype)
-            chunksize = self.chunksize
-            written_nbytes = 0
-            nbytes = nelem_ * self.typesize
-            while written_nbytes < nbytes:
-                chunk = np.zeros(chunksize // self.typesize, dtype=schunk_dtype)
-                self.append_data(chunk)
-                written_nbytes += chunksize
-                if (nbytes - written_nbytes) < self.chunksize:
-                    chunksize = nbytes - written_nbytes
-            self.remove_prefilter(func.__name__)
+            try:
+                chunksize = self.chunksize
+                written_nbytes = 0
+                nbytes = nelem_ * self.typesize
+                while written_nbytes < nbytes:
+                    chunk = np.zeros(chunksize // self.typesize, dtype=schunk_dtype)
+                    self.append_data(chunk)
+                    written_nbytes += chunksize
+                    if (nbytes - written_nbytes) < self.chunksize:
+                        chunksize = nbytes - written_nbytes
+            finally:
+                # Also after a failed append: the filler only keeps the id of inputs_tuple
+                self.remove_prefilter(func.__name__)
 
             def exec_func(*args):
                 func(*args)
@@ -1719,7 +1724,8 @@ def prefilter(self, input_dtype: np.dtype, output_dtype: np.dtype = None) -> Non
         * `nthreads` must be 1 when compressing.
 
         * The :paramref:`input_dtype` itemsize must be the same as the
-          :paramref:`output_dtype` itemsize.
+          :paramref:`output_dtype` itemsize, and the typesize of the SChunk:
+          compressing with another typesize fails.
 
         See Also
         --------

tests/test_postfilters.py

@@ -86,3 +86,82 @@ def postf3(input, output, offset):
         assert np.array_equal(data, res)
 
     blosc2.remove_urlpath(urlpath)
+
+
+# The postfilters below only record their calls and never touch their arrays, so a call with
+# arrays that do not fit the buffers cannot overrun them, with or without the typesize guard.
+# Each test runs with the GIL kept and released around the C calls.
+
+
+@pytest.fixture(params=[False, True], ids=["gil-held", "gil-released"])
+def releasegil(request):
+    old = blosc2.set_releasegil(request.param)
+    yield
+    blosc2.set_releasegil(old)
+
+
+@pytest.mark.parametrize("typesize", [1, 8])
+def test_postfilter_wrong_typesize(releasegil, typesize):
+    # int32 arrays of size // typesize items would overrun a typesize 1 buffer, and cover only
+    # half of a typesize 8 one: the postfilter refuses such chunks
+    data = np.arange(1000, dtype=np.int32)
+    schunk = blosc2.SChunk(
+        chunksize=data.nbytes,
+        data=data,
+        cparams={"typesize": typesize, "nthreads": 1},
+        dparams={"nthreads": 1},
+    )
+    calls = []
+
+    @schunk.postfilter(np.int32)
+    def postf_wrong_typesize(input, output, offset):
+        calls.append(input.size)
+
+    with pytest.raises(
+        ValueError, match=f"postfilter: data typesize {typesize} does not match dtype itemsize 4"
+    ):
+        schunk.decompress_chunk(0)
+    assert calls == []
+
+    schunk.remove_postfilter("postf_wrong_typesize")
+    assert np.array_equal(np.frombuffer(schunk.decompress_chunk(0), dtype=np.int32), data)
+
+
+def test_postfilter_vlmeta(releasegil):
+    # The postfilter is for chunk data, not for the msgpack content (typesize 1) of a vlmetalayer
+    data = np.arange(2000, dtype=np.int32)
+    schunk = blosc2.SChunk(
+        chunksize=4000,
+        data=data,
+        contiguous=False,
+        cparams={"typesize": 4, "nthreads": 1},
+        dparams={"nthreads": 1},
+    )
+    schunk.vlmeta["info"] = {"answer": 42}
+    calls = []
+
+    @schunk.postfilter(np.int32)
+    def postf_vlmeta(input, output, offset):
+        calls.append(input.size)
+
+    # C-Blosc2 before Blosc/c-blosc2#857 decompresses vlmetalayers with the postfilter, which then
+    # refuses them; with it the content is read without the postfilter.  Either way the callback
+    # never sees the content.
+    refused = "postfilter: data typesize 1 does not match dtype itemsize 4"
+    try:
+        info = schunk.vlmeta["info"]
+    except ValueError as e:
+        info = str(e)
+    assert info in ({"answer": 42}, refused)
+    assert calls == []
+
+    # Serializing an in-memory sparse super-chunk copies its vlmetalayers the same way
+    try:
+        copy = blosc2.schunk_from_cframe(schunk.to_cframe(), copy=True)
+        info = copy.vlmeta["info"]
+    except ValueError as e:
+        info = str(e)
+    assert info in ({"answer": 42}, refused)
+
+    schunk.remove_postfilter("postf_vlmeta")
+    assert schunk.vlmeta["info"] == {"answer": 42}

tests/test_prefilters.py

@@ -207,3 +207,61 @@ def pref3(input, output, offset):
         assert np.array_equal(new_data, res)
 
     blosc2.remove_urlpath(urlpath)
+
+
+# The prefilters below only record their calls and never touch their arrays, so a call with
+# arrays that do not fit the buffers cannot overrun them, with or without the typesize guard.
+# Each test runs with the GIL kept and released around the C calls.
+
+
+@pytest.fixture(params=[False, True], ids=["gil-held", "gil-released"])
+def releasegil(request):
+    old = blosc2.set_releasegil(request.param)
+    yield
+    blosc2.set_releasegil(old)
+
+
+@pytest.mark.parametrize("typesize", [1, 8])
+def test_prefilter_wrong_typesize(releasegil, typesize):
+    # int32 arrays of size // typesize items would overrun a typesize 1 buffer, and cover only
+    # half of a typesize 8 one: the prefilter refuses such chunks
+    data = np.arange(1000, dtype=np.int32)
+    schunk = blosc2.SChunk(chunksize=data.nbytes, cparams={"typesize": typesize, "nthreads": 1})
+    calls = []
+
+    @schunk.prefilter(np.int32)
+    def pref_wrong_typesize(input, output, offset):
+        calls.append(input.size)
+
+    with pytest.raises(
+        ValueError, match=f"prefilter: data typesize {typesize} does not match dtype itemsize 4"
+    ):
+        schunk.append_data(data)
+    assert calls == []
+    assert schunk.nchunks == 0
+
+    schunk.remove_prefilter("pref_wrong_typesize")
+    schunk.append_data(data)
+    assert np.array_equal(np.frombuffer(schunk.decompress_chunk(0), dtype=np.int32), data)
+
+
+@pytest.mark.parametrize("typesize", [1, 8])
+def test_filler_wrong_typesize(releasegil, typesize):
+    # Same for a filler, which then must not stay set: it keeps only the id of its inputs
+    data = np.arange(1000, dtype=np.int32)
+    schunk = blosc2.SChunk(chunksize=data.nbytes, cparams={"typesize": typesize, "nthreads": 1})
+    calls = []
+
+    with pytest.raises(
+        ValueError, match=f"filler: data typesize {typesize} does not match dtype itemsize 4"
+    ):
+
+        @schunk.filler(((data, np.int32),), np.int32)
+        def fill_wrong_typesize(inputs_tuple, output, offset):
+            calls.append(output.size)
+
+    assert calls == []
+    assert schunk.nchunks == 0
+    assert "fill_wrong_typesize" not in blosc2.prefilter_funcs
+    schunk.append_data(data)
+    assert schunk.nchunks == 1