From 8e8880fc2b4120ce261bd6067a813578b4ddd1ea Mon Sep 17 00:00:00 2001
From: Fabian Isensee <f.isensee@dkfz.de>
Date: Fri, 6 Nov 2026 22:43:51 +0100
Subject: [PATCH] Check the buffer size of orthogonal selections

get_oindex_numpy() and set_oindex_numpy() pass typesize * prod(selection
lengths) to the C orthogonal selection as the buffer size, but their
callers allocate the NumPy buffer for the array's dtype.  When the
dtype's items are smaller than the typesize, the C copy writes (get) or
reads (set) past the end of that buffer.  This happens with valid files:
c-blosc2 2.7 to 2.16 stored "|u1" as the dtype of arrays created without
one, whatever the typesize, e.g. c-blosc2's own
plugins/test_data/example_item_prices.b2nd (float32 items, typesize 4),
whose get_oselection_numpy() segfaults.  A damaged dtype string does the
same.  oindex, take(axis=...) and integer-array indexing of
multi-dimensional arrays go through the same two functions.

Check the length of the buffer against the selection at the start of
both functions, before the index arrays are allocated, and raise
ValueError when it is too small, as get_sparse_numpy() does (slices are
checked by c-blosc2, which gets their real buffer length).  This does
not depend on the buffer check of the C orthogonal selection, which
accepted too small buffers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DiPFzu8ziFQsxrJUkHCrzx
---
 src/blosc2/blosc2_ext.pyx                     |  26 ++++
 .../example_day_month_temp.b2nd               | Bin 0 -> 2633 bytes
 tests/ndarray/test_oindex_typesize.py         | 120 ++++++++++++++++++
 3 files changed, 146 insertions(+)
 create mode 100644 tests/data/legacy_u1_dtype/example_day_month_temp.b2nd
 create mode 100644 tests/ndarray/test_oindex_typesize.py

diff --git a/src/blosc2/blosc2_ext.pyx b/src/blosc2/blosc2_ext.pyx
index de90e254c..dd0e3af8f 100644
--- a/src/blosc2/blosc2_ext.pyx
+++ b/src/blosc2/blosc2_ext.pyx
@@ -6861,6 +6861,30 @@ cdef class NDArray:
         _check_rc(rc, "Error while getting the sparse selection")
         return arr
 
+    cdef _check_oindex_buffer(self, arr, key, str role):
+        """Raise ValueError if the buffer of `arr` (the `role` "destination" or "source" in the
+        message) cannot hold the orthogonal selection `key`.
+
+        The C copy moves typesize bytes per item, but an array's dtype can have smaller items:
+        arrays that c-blosc2 2.7-2.16 wrote without a dtype store "|u1" whatever the typesize,
+        or the dtype string is damaged.  A buffer allocated for the dtype is too small then.
+        """
+        cdef int64_t typesize = self.array.sc.typesize
+        nbytes = typesize  # a Python int: it cannot wrap around for huge selections
+        for k in key:
+            nbytes *= len(k)
+        cdef Py_buffer view
+        PyObject_GetBuffer(arr, &view, PyBUF_SIMPLE)
+        cdef Py_ssize_t buflen = view.len
+        PyBuffer_Release(&view)
+        if buflen < nbytes:
+            msg = (f"{role} buffer is smaller than the requested orthogonal selection "
+                   f"({buflen} < {nbytes} bytes)")
+            itemsize = getattr(arr, "itemsize", typesize)
+            if itemsize != typesize:
+                msg += f": its itemsize is {itemsize}, but the array's typesize is {typesize}"
+            raise ValueError(msg)
+
     @_handle_locked
     def get_oindex_numpy(self, arr, key):
         """
@@ -6868,6 +6892,7 @@ cdef class NDArray:
         """
         if len(key) != self.array.ndim:
             raise ValueError(f"Key must have {self.array.ndim} dimensions, got {len(key)}.")
+        self._check_oindex_buffer(arr, key, "destination")
         cdef int64_t[B2ND_MAX_DIM] buffershape_
         cdef int64_t** key_
         cdef int64_t buffersize_ = self.array.sc.typesize
@@ -6901,6 +6926,7 @@ cdef class NDArray:
         """
         if len(key) != self.array.ndim:
             raise ValueError(f"Key must have {self.array.ndim} dimensions, got {len(key)}.")
+        self._check_oindex_buffer(arr, key, "source")
         cdef int64_t[B2ND_MAX_DIM] buffershape_
         cdef int64_t** key_
         cdef int64_t buffersize_ = self.array.sc.typesize
diff --git a/tests/data/legacy_u1_dtype/example_day_month_temp.b2nd b/tests/data/legacy_u1_dtype/example_day_month_temp.b2nd
new file mode 100644
index 0000000000000000000000000000000000000000..3f2a09c244800b2db81022584399937b98e586a8
GIT binary patch
literal 2633
zcmb7GYiv|S6h3#$vZb{3+E7}dk(FYLlFC{z1f$+F{i7(v64Wf{k5XY%v^>(3SPjBe
zFp$Wjp=i6Un2WYjl7=({lSahe#X^XR7BHctJVrFJpwT8qD33(XnVGw(z#pgm=G^am
z=bV{2_mTa$v${M{w{+!lI0XP5KhQ)YYhHUERA!vkEWgTJj1wF*hNe&#P2u=Su;~BD
zz*rS%>2RK#PQbkvz-q59U$u;LUOxxcF|v;}LujX|W;L1%$k@p-t9a6vbJZmbvSZ!a
zN5fWROgs$G6#{4=tH!9+0Biw(c%|gJfGSp7h`UaGQ*BCz^%IYZ(2R{@no36Ewz$-B
zo-7Z}kBRx&<%T!1#fHlPcO8z(i(mHHax8#{tMi&un!ChZ=h9n8IFE)zo;UfY3?E<}
z8@_5Yoc9Cxer;9-#{mp87a#RaODPII@_w_<Ofq_viodabis3HlwrF3oko`1xKZx~B
z*(Se{xWo;w#SQN!?&HPhb4`AfcuaiOHrwb!!~xx=mZwxX;C$C9Jln~B8pp{>tgAU{
zxJ0gZVE3;+tDxcmI*)BOI{Tuccraqt<9c*gk}G_+T$}5SEGqUDRp<9{;@N{HpU)Q+
zFXbLJdcX45B({^EhfQ4Kg#Aq>eweK5<Zb-UaPGG)b`}1PDjblwhkbwZzIm{<MbEq^
zd}aXLUl(W3`Wp4~!~O9Pp84sp*1L&ItgPmJ@!)>=xV1TpEDv5UCN}zGE4(OkPw962
z9i#im=Q+M-8~#H>Kc5nxo%4`b^1_P?G)KI#zB;AzRp04+FQ>Q~0I%;l?QNS(K64OV
zMIWYgKA+E}=l?MKY1m{XURl`fYrS9bEzP+;Sb^?hc4(F;aCtv^y<7X<H2ECwI(^X*
zpQE`pqo@8=pBbN>_1Z7jn0W3-OuYA7UWJlrZ)EXpXGco&O5(PN&;5k`Jji-@b^0qQ
z-4Axf#N!9A8eOk1Z?5Ro`{#9R@x}e0`y2@{r1+$nc7@)5BXPuEIu7|tR`vOPFHLFA
z@ql$thS<{JdWb!Loj^s^^Wb4k;)c=r{vpN=l=)hB*)N9o))_rYR^sqp-*8?Z#L&$h
zhF6k}3jdpL4d*HzcDB_T&Uvo0`_BocUscTU*1l^-kCBavpF4u@i-(*)vS@RPD{(BZ
zfADzl{I#g~D<y*_pYNxKrxyg@3;jN*)c-Mst#JCgJ7Af_z!IHKSWUc=taF|wzSvJV
zL>MOU*W?W0d%~1BO?89@!Xtz#!a2fO!dgQ351NWVE`|_Y%qKYfvED{_nlO*Bj*vz8
zlyHquG?%6t!ezn%!ZN~j0x#I)&J4klpq|>}c<6~<RBs2`*<j0FkT3%p?FbCVCPTM9
zWh=D6N$7+hVMtDbet8qN$S!D!O@wa$T|htl0dz;Zf|}3KDozCm;nN2ur68~u1pf*4
zvkj#{c-CZr<X{nvh?GDO0)uk*QMqCQ$lMGIhP*-xEH6{mS=^6_X)z}w1nvfzQz&bd
zlAl%zD9w@u56H#02{|M*%A5kZQVFB)YkqJ)154HTzV0Cur7JJZ$ex-6n3`x!wvtWA
z2O}0F3P$OXB>NM|q<omA$)lw4$#Eo=CFhVGktGpp<Te>e#w7iL!?Kg$PL!tC+8!BC
zJUz6Sq`@1xn`oB0yJ!}=Lmb@{ek|3vtts{7yoXSlFq@Mt5i8MZTzX5&mDH}_%oJ5-
zT){cXg-O1trMjbuR+&$=K)#_Kf329F-l?coQcGDW`%jMQB6X7$da`u2uvL&5rL`DH
lP-E2D0$1?ksxqQI%QLmzWzi41u@}_)%CA=OK#-BP{{s8)4p;yH

literal 0
HcmV?d00001

diff --git a/tests/ndarray/test_oindex_typesize.py b/tests/ndarray/test_oindex_typesize.py
new file mode 100644
index 000000000..010c37816
--- /dev/null
+++ b/tests/ndarray/test_oindex_typesize.py
@@ -0,0 +1,120 @@
+#######################################################################
+# Copyright (c) 2019-present, Blosc Development Team <blosc@blosc.org>
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#######################################################################
+
+# Orthogonal selections copy typesize bytes per item in C into a NumPy buffer allocated for
+# the array's dtype.  Arrays whose dtype has smaller items than the typesize must raise
+# instead of overflowing that buffer.
+
+import shutil
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+import blosc2
+
+# Written by c-blosc2's examples/b2nd/example_frame_generator.c (c-blosc2 2.7-2.16 store
+# "|u1" in the b2nd metalayer of arrays created without a dtype): float32 items, typesize 4
+LEGACY_U1_FILE = Path(__file__).parent.parent / "data" / "legacy_u1_dtype" / "example_day_month_temp.b2nd"
+
+
+def with_dtype_string(arr, old, new):
+    """`arr` with dtype string `old` replaced by `new` (same length) in its b2nd metalayer"""
+    assert len(old) == len(new)
+    cframe = arr.to_cframe()
+    # The metalayers come before the chunks, so the first match is the dtype
+    return blosc2.ndarray_from_cframe(cframe.replace(old.encode(), new.encode(), 1), copy=True)
+
+
+def assert_oindex_raises(arr):
+    """The paths to the C orthogonal selection raise ValueError and leave the array unchanged"""
+    before = arr.tobytes()
+    full = [list(range(n)) for n in arr.shape]
+    match = f"typesize is {arr.schunk.typesize}"
+    with pytest.raises(ValueError, match=match):
+        arr.get_oselection_numpy(full)
+    with pytest.raises(ValueError, match=match):
+        arr.get_oselection_numpy([[0]] * arr.ndim)  # one item: a few bytes too many
+    with pytest.raises(ValueError, match=match):
+        arr.oindex[tuple(full)] = np.zeros(arr.shape, dtype=arr.dtype)
+    if arr.ndim > 1:
+        with pytest.raises(ValueError, match=match):
+            arr[[0, 1]]  # through take(axis=0)
+    assert arr.tobytes() == before
+
+
+def test_legacy_u1_file(tmp_path):
+    path = tmp_path / LEGACY_U1_FILE.name
+    shutil.copy(LEGACY_U1_FILE, path)
+    arr = blosc2.open(path, mode="a")
+    assert arr.dtype == np.uint8
+    assert arr.schunk.typesize == 4
+    assert arr.shape == (400, 3)
+    assert_oindex_raises(arr)
+    assert path.read_bytes() == LEGACY_U1_FILE.read_bytes()
+
+
+@pytest.mark.parametrize(
+    ("dtype", "shape", "chunks", "blocks", "stored"),
+    [
+        # As c-blosc2's plugins/test_data/example_item_prices.b2nd
+        (np.float32, (12, 25, 250), (8, 10, 50), (4, 5, 10), "|u1"),
+        (np.float64, (100,), (30,), (7,), "|u1"),
+        # A damaged dtype string
+        (np.float64, (6, 10), (4, 4), (2, 2), "<f2"),
+    ],
+)
+def test_dtype_smaller_than_typesize(dtype, shape, chunks, blocks, stored):
+    data = np.arange(np.prod(shape), dtype=dtype).reshape(shape)
+    arr = with_dtype_string(blosc2.asarray(data, chunks=chunks, blocks=blocks), data.dtype.str, stored)
+    assert arr.dtype == np.dtype(stored)
+    assert arr.schunk.typesize == data.itemsize
+    assert_oindex_raises(arr)
+
+
+def test_oindex_buffer_size():
+    shape = (6, 10)
+    data = np.arange(np.prod(shape), dtype=np.int32).reshape(shape)
+    arr = blosc2.asarray(data, chunks=(4, 4), blocks=(2, 2))
+    key = [[5, 0, 2], [9, 1]]
+    expected = data[np.ix_(*key)]
+    out = np.empty(expected.shape, dtype=arr.dtype)
+    np.testing.assert_array_equal(arr.get_oindex_numpy(out, key), expected)
+    with pytest.raises(ValueError, match="destination buffer is smaller"):
+        arr.get_oindex_numpy(np.empty(expected.size - 1, dtype=arr.dtype), key)
+    with pytest.raises(ValueError, match="source buffer is smaller"):
+        arr.set_oindex_numpy(key, np.empty(expected.size - 1, dtype=arr.dtype))
+    np.testing.assert_array_equal(arr[...], data)
+    # A larger buffer is fine, as for slices
+    out = np.zeros(expected.size + 3, dtype=arr.dtype)
+    arr.get_oindex_numpy(out, key)
+    np.testing.assert_array_equal(out[: expected.size].reshape(expected.shape), expected)
+    # Empty selections need no buffer
+    arr.get_oindex_numpy(np.empty(0, dtype=arr.dtype), [[], [1]])
+    arr.set_oindex_numpy([[2], []], np.empty(0, dtype=arr.dtype))
+    np.testing.assert_array_equal(arr[...], data)
+
+
+@pytest.mark.parametrize(
+    "dtype",
+    ["?", ">i4", "S7", "<U100", [("a", "<i4"), ("b", "<f8")]],
+)
+def test_oindex_valid_dtypes(dtype):
+    # The typesize of valid arrays is the dtype's itemsize, also above 255 bytes ("<U100")
+    dtype = np.dtype(dtype)
+    shape = (5, 7)
+    data = np.arange(np.prod(shape)).astype(dtype).reshape(shape)
+    arr = blosc2.asarray(data, chunks=(3, 4), blocks=(2, 2))
+    assert arr.schunk.typesize == dtype.itemsize
+    key = [[4, 0, 2], [6, 1, 1]]
+    np.testing.assert_array_equal(arr.oindex[tuple(key)], data[np.ix_(*key)])
+    key = [[4, 0, 2], [6, 1, 3]]
+    value = data[np.ix_([0, 1, 2], [0, 1, 2])]
+    arr.oindex[tuple(key)] = value
+    expected = data.copy()
+    expected[np.ix_(*key)] = value
+    np.testing.assert_array_equal(arr[...], expected)