From 9400032e2cc381ef1374171ffd5fd4ca7d556827 Mon Sep 17 00:00:00 2001 From: Prudhvi Boyapalli Date: Fri, 6 Nov 2026 16:39:06 -0500 Subject: [PATCH] Update nullable columns section of ctable basics tutorial --- doc/tutorials/13.ctable-basics.ipynb | 392 ++++++++++++++++++++++++++- 1 file changed, 377 insertions(+), 15 deletions(-) diff --git a/doc/tutorials/13.ctable-basics.ipynb b/doc/tutorials/13.ctable-basics.ipynb index d9f7c844b..b0bf58ee7 100644 --- a/doc/tutorials/13.ctable-basics.ipynb +++ b/doc/tutorials/13.ctable-basics.ipynb @@ -175,7 +175,7 @@ "Validation error: 1 validation error for _Validator_Sensor\n", "temperature\n", " Input should be less than or equal to 60 [type=less_than_equal, input_value=9999.0, input_type=float]\n", - " For further information visit https://errors.pydantic.dev/2.13/v/less_than_equal\n" + " For further information visit https://errors.pydantic.dev/2.14/v/less_than_equal\n" ] } ], @@ -249,7 +249,7 @@ "source": [ "### 1.4 Navigating the table\n", "\n", - "`head()`, `tail()`, and slicing give you quick views without materialising everything." + "`head()`, `tail()`, and `slice()` give you quick views without materialising everything." ] }, { @@ -282,6 +282,11 @@ " id location temperature active\n", "0 5 server 35.800000 True\n", "1 6 garden -1.000000 False\n", + "--- slice(1, 4) ---\n", + " id location temperature active\n", + "0 1 basement 18.100000 True\n", + "1 2 outdoor -3.200000 False\n", + "2 3 lab-A 20.000000 True\n", "\n", "Compression: 747 B compressed / 3,900 B uncompressed\n" ] @@ -294,6 +299,9 @@ "print(\"--- tail(2) ---\")\n", "print(t.tail(2))\n", "\n", + "print(\"--- slice(1, 4) ---\")\n", + "print(t.slice(1, 4))\n", + "\n", "print(f\"\\nCompression: {t.cbytes:,} B compressed / {t.nbytes:,} B uncompressed\")" ] }, @@ -1946,9 +1954,169 @@ "---\n", "## Part 9 — Nullable columns\n", "\n", - "Real-world data is often incomplete. CTable handles missing values through a **null sentinel** approach: you declare a specific value (e.g. `-1`, `\"\"`, or `float(\"nan\")`) that represents \"no data\" for a column. The library treats it transparently in aggregates, sorting, `unique()`, `value_counts()`, and Arrow export.\n", + "Real-world data is often incomplete. A nullable CTable column keeps track of missing values in one of two ways:\n", + "\n", + "- **Validity mask** (the default for `nullable=True` since 4.11.0): a separate boolean array records which rows are null, and you write nulls as `None`. Every value of the column's type stays usable as data.\n", + "- **Null sentinel**: a specific value (e.g. `-1`, `\"\"`, or `float(\"nan\")`) declared with `null_value=` represents \"no data\", stored in the column itself.\n", + "\n", + "Either way, nulls are handled transparently in aggregates, sorting, `unique()`, `value_counts()`, and Arrow export.\n", + "Sections 9.1–9.2 cover mask storage, 9.3–9.10 explicit sentinels, and 9.11–9.12 choosing and converting between the two with `NullPolicy` and `convert_nulls()`." + ] + }, + { + "cell_type": "markdown", + "id": "1167d23a", + "metadata": {}, + "source": [ + "### 9.1 Validity-mask storage" + ] + }, + { + "cell_type": "code", + "execution_count": 285, + "id": "c85df581", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " sensor_id temperature location\n", + "0 0 22.300000 roof\n", + "1 1 None cellar\n", + "2 2 18.700000 None\n", + "3 3 nan garage\n", + "4 4 None None\n", + "5 5 15.100000 roof\n", + "temperature null_storage: mask\n", + "temperature is_null : [False, True, False, False, True, False]\n", + "location null_count : 2\n" + ] + } + ], + "source": [ + "@dataclass\n", + "class SensorLogMask:\n", + " sensor_id: int = blosc2.field(blosc2.int32(ge=0))\n", + " # A bare nullable=True keeps nulls in a validity mask; write them as None\n", + " temperature: float = blosc2.field(blosc2.float64(nullable=True), default=None)\n", + " location: str = blosc2.field(blosc2.utf8(nullable=True), default=None)\n", + "\n", + "\n", + "mlog = blosc2.CTable(\n", + " SensorLogMask,\n", + " new_data=[\n", + " (0, 22.3, \"roof\"),\n", + " (1, None, \"cellar\"), # temperature: missing\n", + " (2, 18.7, None), # location: missing\n", + " (3, float(\"nan\"), \"garage\"), # temperature: a NaN reading, not a missing one\n", + " (4, None, None),\n", + " (5, 15.1, \"roof\"),\n", + " ],\n", + ")\n", + "print(mlog)\n", + "print(f\"temperature null_storage: {mlog.temperature.null_storage}\")\n", + "print(f\"temperature is_null : {mlog.temperature.is_null().tolist()}\")\n", + "print(f\"location null_count : {mlog['location'].null_count()}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 286, + "id": "95405553", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "masked view: [22.3 -- 18.7 nan -- 15.1]\n", + "nulls : 2\n", + "NaN values : 1\n", + "mean : nan\n" + ] + } + ], + "source": [ + "# Under mask storage NaN is a real value, distinct from null\n", + "mtemp = mlog[\"temperature\"]\n", + "print(\"masked view:\", mtemp.to_numpy(masked=True))\n", + "print(\"nulls :\", mtemp.null_count())\n", + "print(\"NaN values :\", int(np.isnan(mtemp.to_numpy(masked=True)).sum()))\n", + "print(\"mean :\", mtemp.mean())" + ] + }, + { + "cell_type": "markdown", + "id": "347c0bff", + "metadata": {}, + "source": [ + "### 9.2 Filtering nullable columns" + ] + }, + { + "cell_type": "code", + "execution_count": 287, + "id": "d6cb1d3c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Filtering for null temperatures:\n", + " sensor_id temperature location\n", + "0 1 None cellar\n", + "1 4 None None\n", + "\n", + "Filtering for non-null temperatures:\n", + " sensor_id temperature location\n", + "0 0 22.300000 roof\n", + "1 2 18.700000 None\n", + "2 3 nan garage\n", + "3 5 15.100000 roof\n", + "\n", + "Filtering with a predicate:\n", + "Predicate Sensor IDs Comment \n", + "------------------- ---------- ---------------------------------------------\n", + "temperature > 20 [0] Only 22.3 passes; nulls unknown, NaN is False\n", + "temperature <= 20 [2, 5] Nulls unknown, NaN is False \n", + "~(temperature > 20) [2, 3, 5] Nulls stay out; NaN flips to True \n", + "~(temperature < 20) [0, 3] Nulls stay out; NaN flips to True \n" + ] + } + ], + "source": [ + "print(\"Filtering for null temperatures:\")\n", + "print(mlog[mlog.temperature.is_null()])\n", + "print(\"\\nFiltering for non-null temperatures:\")\n", + "print(mlog.where(mlog.temperature.notnull()))\n", + "\n", + "# Nulls make a comparison \"unknown\", so those rows are never selected, even after ~.\n", + "# NaN is a real value: any comparison with it is False, so ~ turns it into True.\n", + "predicates = [\n", + " (\"temperature > 20\", mlog.temperature > 20, \"Only 22.3 passes; nulls unknown, NaN is False\"),\n", + " (\"temperature <= 20\", mlog.temperature <= 20, \"Nulls unknown, NaN is False\"),\n", + " (\"~(temperature > 20)\", ~(mlog.temperature > 20), \"Nulls stay out; NaN flips to True\"),\n", + " (\"~(temperature < 20)\", ~(mlog.temperature < 20), \"Nulls stay out; NaN flips to True\"),\n", + "]\n", + "\n", + "pred_header = (\"Predicate\", \"Sensor IDs\", \"Comment\")\n", + "pred_lines = [(f, str(mlog.where(expr).sensor_id[:].tolist()), d) for f, expr, d in predicates]\n", + "pred_widths = [max(len(r[i]) for r in [pred_header, *pred_lines]) for i in range(3)]\n", + "print(\"\\nFiltering with a predicate:\")\n", + "for r in [pred_header, (\"-\" * w for w in pred_widths), *pred_lines]:\n", + " print(\" \".join(cell.ljust(w) for cell, w in zip(r, pred_widths, strict=True)))" + ] + }, + { + "cell_type": "markdown", + "id": "aa61759f", + "metadata": {}, + "source": [ + "### 9.3 Null sentinels\n", "\n", - "This pattern is most useful for **integer and string** columns, which have no natural missing-value representation (unlike floats, which can use `NaN`)." + "Null sentinels can be used as an alternative to mask-based nullable columns by setting the `null_value` field in the spec." ] }, { @@ -2014,7 +2182,7 @@ "id": "b8eff116", "metadata": {}, "source": [ - "### 9.1 Detecting nulls" + "### 9.4 Detecting nulls" ] }, { @@ -2065,7 +2233,7 @@ "id": "e081d545", "metadata": {}, "source": [ - "### 9.2 Null-aware aggregates" + "### 9.5 Null-aware aggregates" ] }, { @@ -2113,7 +2281,7 @@ "id": "48f595ba", "metadata": {}, "source": [ - "### 9.3 Validation bypass" + "### 9.6 Validation bypass" ] }, { @@ -2160,7 +2328,7 @@ "id": "93c0bf84", "metadata": {}, "source": [ - "### 9.4 Sort: nulls always go last" + "### 9.7 Sort: nulls always go last" ] }, { @@ -2207,7 +2375,7 @@ "id": "fc3a11ed", "metadata": {}, "source": [ - "### 9.5 Arrow export: sentinels become Arrow nulls" + "### 9.8 Arrow export: sentinels become Arrow nulls" ] }, { @@ -2254,7 +2422,7 @@ "id": "77224bae", "metadata": {}, "source": [ - "### 9.6 CSV: empty cells become the null sentinel" + "### 9.9 CSV: empty cells become the null sentinel" ] }, { @@ -2313,9 +2481,9 @@ "id": "4736edd2", "metadata": {}, "source": [ - "### 9.7 `fillna()` and `dropna()` (new in 4.9.0)\n", + "### 9.10 `fillna()` and `dropna()` (new in 4.9.0)\n", "\n", - "Beyond detecting nulls, `Column.fillna(value)` replaces sentinel values with a given value (returning a plain array),\n", + "Beyond detecting nulls, `Column.fillna(value)` replaces nulls with a given value (returning a plain array),\n", "and `CTable.dropna(subset=None)` returns a **view** excluding any row where a nullable column (or a chosen subset) is null." ] }, @@ -2349,7 +2517,7 @@ } ], "source": [ - "# fillna(): replace sentinel values in one column\n", + "# fillna(): replace nulls in one column\n", "print(\"temperature, filled:\", log[\"temperature\"].fillna(0.0))\n", "print(\"location, filled :\", log[\"location\"].fillna(\"\"))\n", "\n", @@ -2361,6 +2529,200 @@ "print(f\"rows with known location : {len(only_missing_location)} / {len(log)}\")" ] }, + { + "cell_type": "markdown", + "id": "4963f2c9", + "metadata": {}, + "source": [ + "### 9.11 Choosing null storage with `NullPolicy`\n", + "\n", + "Sections 9.3–9.10 declared the sentinel in the schema, one column at a time, with `null_value=` on the spec. `NullPolicy` is an alternative to globally set the null storage from outside the schema. \n", + "\n", + "The default policy uses a validity mask; `blosc2.NullPolicy(null_storage=\"sentinel\")` switches to in-band sentinels using type-wide defaults. Or, `column_null_values={\"col\": value}` can be used to pick a sentinel for specific columns. \n", + "\n", + "Apply a policy with the `blosc2.null_policy(...)` context manager when creating a table. The result is saved in the table's schema, so it is never re-resolved on open." + ] + }, + { + "cell_type": "code", + "execution_count": 296, + "id": "7dd16096", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--- mask ---\n", + " sensor temp site\n", + "0 1 21.500000 roof\n", + "1 None 19.000000 None\n", + "2 3 None lab\n", + "3 4 nan yard\n", + "sensor storage=mask null_value=None nulls=1\n", + "temp storage=mask null_value=None nulls=1\n", + "site storage=mask null_value=None nulls=1\n", + "\n", + "--- sentinel ---\n", + " sensor temp site\n", + "0 1 21.500000 roof\n", + "1 -1 19.000000 ?\n", + "2 3 nan lab\n", + "3 4 nan yard\n", + "sensor storage=sentinel null_value=-1 nulls=1\n", + "temp storage=sentinel null_value=nan nulls=2\n", + "site storage=sentinel null_value='?' nulls=1\n", + "\n", + "Null Storage Mean Temperature Comment \n", + "------------ ---------------- ---------------------------------------------------------------------------\n", + "mask nan NaN is a real value, so it poisons the mean: (21.5 + 19.0 + NaN) / 3 = NaN \n", + "sentinel 20.25 NaN is the sentinel, so it is skipped and only 21.5 and 19.0 are considered\n" + ] + } + ], + "source": [ + "@dataclass\n", + "class Reading:\n", + " sensor: int = blosc2.field(blosc2.int32(nullable=True))\n", + " temp: float = blosc2.field(blosc2.float64(nullable=True))\n", + " site: str = blosc2.field(blosc2.string(max_length=8, nullable=True))\n", + "\n", + "\n", + "# Default policy: validity mask, nulls written as None\n", + "masked = CTable(\n", + " Reading, new_data=[(1, 21.5, \"roof\"), (None, 19.0, None), (3, None, \"lab\"), (4, np.nan, \"yard\")]\n", + ")\n", + "\n", + "# Sentinel policy: nulls stored in band, written as the sentinel itself\n", + "policy = blosc2.NullPolicy(null_storage=\"sentinel\", column_null_values={\"sensor\": -1, \"site\": \"?\"})\n", + "with blosc2.null_policy(policy):\n", + " sentinel = CTable(\n", + " Reading, new_data=[(1, 21.5, \"roof\"), (-1, 19.0, \"?\"), (3, np.nan, \"lab\"), (4, np.nan, \"yard\")]\n", + " )\n", + "\n", + "for label, ct in [(\"mask\", masked), (\"sentinel\", sentinel)]:\n", + " print(f\"--- {label} ---\")\n", + " print(ct)\n", + " for name in ct.col_names:\n", + " c = ct[name]\n", + " print(f\"{name:<7} storage={c.null_storage:<9} null_value={c.null_value!r:<6} nulls={c.null_count()}\")\n", + " print()\n", + "\n", + "\n", + "# temp is [21.5, 19.0, null, NaN] under mask and [21.5, 19.0, NaN, NaN] under sentinel\n", + "means = [\n", + " (\n", + " \"mask\",\n", + " masked.temp.mean(),\n", + " \"NaN is a real value, so it poisons the mean: (21.5 + 19.0 + NaN) / 3 = NaN\",\n", + " ),\n", + " (\n", + " \"sentinel\",\n", + " sentinel.temp.mean(),\n", + " \"NaN is the sentinel, so it is skipped and only 21.5 and 19.0 are considered\",\n", + " ),\n", + "]\n", + "mean_header = (\"Null Storage\", \"Mean Temperature\", \"Comment\")\n", + "mean_lines = [(a, f\"{b:.2f}\", d) for a, b, d in means]\n", + "mean_widths = [max(len(r[i]) for r in [mean_header, *mean_lines]) for i in range(3)]\n", + "for r in [mean_header, (\"-\" * w for w in mean_widths), *mean_lines]:\n", + " print(\" \".join(cell.ljust(w) for cell, w in zip(r, mean_widths, strict=True)))" + ] + }, + { + "cell_type": "markdown", + "id": "c0ac36e6", + "metadata": {}, + "source": [ + "### 9.12 Converting between null storage with `convert_nulls()`\n", + "\n", + "`convert_nulls()` returns a converted copy (pass `inplace=True` to rewrite the table itself). With no arguments it moves every sentinel column to mask storage; name a column and pass `to=\"sentinel\"` with a `null_value=` to go the other way.\n", + "\n", + "- **sentinel → mask** is lossless: every row holding the sentinel becomes a null and every other value is kept.\n", + "- **mask → sentinel** has to reserve one value to mean null, so it refuses a `null_value` that real data already holds (second cell below). A NaN sentinel is the one exception: NaN has always meant null under sentinel storage, so a genuine NaN is folded into the nulls instead of being refused." + ] + }, + { + "cell_type": "code", + "execution_count": 297, + "id": "637a67ff", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sentinel:\n", + " sensor temp site\n", + "0 1 21.500000 roof\n", + "1 -1 19.000000 ?\n", + "2 3 nan lab\n", + "3 4 nan yard\n", + "temperature null count: 2\n", + "\n", + "sentinel -> mask is lossless, null count is the same after conversion:\n", + " sensor temp site\n", + "0 1 21.500000 roof\n", + "1 None 19.000000 None\n", + "2 3 None lab\n", + "3 4 None yard\n", + "temperature null count: 2\n", + "\n", + "mask:\n", + " sensor temp site\n", + "0 1 21.500000 roof\n", + "1 None 19.000000 None\n", + "2 3 None lab\n", + "3 4 nan yard\n", + "temperature null count: 1\n", + "\n", + "mask -> sentinel, the real NaN at row 3 becomes a null:\n", + " sensor temp site\n", + "0 1 21.500000 roof\n", + "1 None 19.000000 None\n", + "2 3 nan lab\n", + "3 4 nan yard\n", + "temperature null count: 2\n", + "\n" + ] + } + ], + "source": [ + "to_mask = sentinel.convert_nulls()\n", + "to_sentinel = masked.convert_nulls(\"temp\", to=\"sentinel\", null_value=float(\"nan\"))\n", + "for label, ct in [\n", + " (\"sentinel\", sentinel),\n", + " (\"sentinel -> mask is lossless, null count is the same after conversion\", to_mask),\n", + " (\"mask\", masked),\n", + " (\"mask -> sentinel, the real NaN at row 3 becomes a null\", to_sentinel),\n", + "]:\n", + " print(f\"{label}:\")\n", + " print(ct)\n", + " print(f\"temperature null count: {ct['temp'].null_count()}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 298, + "id": "a72a1500", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "mask -> sentinel refuses a sentinel that collides with real data:\n", + "\tColumn 'sensor' already contains 3 at row 2 as a real value, so reserving it as the null sentinel would relabel that row as null. Pass a different null_value=, or keep the column on mask storage.\n" + ] + } + ], + "source": [ + "try:\n", + " masked.convert_nulls(\"sensor\", to=\"sentinel\", null_value=3)\n", + "except ValueError as e:\n", + " print(f\"mask -> sentinel refuses a sentinel that collides with real data:\\n\\t{e}\")" + ] + }, { "cell_type": "markdown", "id": "efd62639", @@ -2483,7 +2845,7 @@ } ], "source": [ - "# utf8 nulls are sentinel-based (like other scalar columns), so is_null()/fillna() work the same way\n", + "# utf8(nullable=True) uses mask storage like other scalar columns, so is_null()/fillna() work the same way\n", "print(\"note is_null:\", list(places[\"note\"].is_null()))\n", "print(\"note filled :\", list(places[\"note\"].fillna(\"\")))\n", "\n", @@ -2523,7 +2885,7 @@ "| Chaining | `CTable.assign(**exprs)` + `blosc2.col(name)` — pandas-3-style query chains |\n", "| Persist | `save(path)`, `to_b2z()`, `to_b2d()`, `CTable.open(path)`, `CTable.load(path)` |\n", "| Interop | `to_arrow()`, `from_arrow()`, `to_csv()`, `from_csv()`, zero-copy `__arrow_c_stream__` |\n", - "| Nullable | `null_value=` on spec, `is_null()`, `notnull()`, `null_count()`, `fillna()`, `dropna()` |\n", + "| Nullable | `nullable=True` (mask), `null_value=` (sentinel), `is_null()`, `notnull()`, `null_count()`, `fillna()`, `dropna()`, `NullPolicy`, `convert_nulls()` |\n", "| String columns | `string()` (fixed-width), `utf8()` (Arrow-style, recommended default), `dictionary()`, `vlstring()` |\n", "\n", "CTable is designed for **compressed analytical workloads** — large tables that need to stay small in RAM\n",