From aef90bd31b846a83273160bcb435348c0933a286 Mon Sep 17 00:00:00 2001 From: he_sk Date: Sun, 27 Sep 2026 00:28:20 +0800 Subject: [PATCH] fix: preserve GB18030 CLOB write boundaries --- .github/workflows/integration-tests.yml | 8 +++--- .../2026-09-26-type-connection-matrix.md | 4 +++ .../third_party/chunanyong_dm/PATCHES.md | 12 +++++++++ dpi_bridge/third_party/chunanyong_dm/a.go | 24 +++++++++++------ .../chunanyong_dm/clob_encoding.go | 27 +++++++++++++++++++ .../chunanyong_dm/clob_encoding_test.go | 16 +++++++++++ scripts/prepare_dm_ci.py | 8 ++++++ 7 files changed, 88 insertions(+), 11 deletions(-) create mode 100644 dpi_bridge/third_party/chunanyong_dm/clob_encoding.go create mode 100644 dpi_bridge/third_party/chunanyong_dm/clob_encoding_test.go diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 5004866..18678f4 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -11,13 +11,14 @@ permissions: jobs: real-dm-arm: - name: Real DM8 / ARM Linux / Python ${{ matrix.python-version }} + name: Real DM8 / ARM Linux / Python ${{ matrix.python-version }} / UNICODE ${{ matrix.charset }} runs-on: ubuntu-24.04-arm timeout-minutes: 35 strategy: fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + charset: [1, 0] steps: - uses: actions/checkout@v4 @@ -48,11 +49,12 @@ jobs: echo 'DM_TEST_HOST=127.0.0.1' echo 'DM_TEST_PORT=15236' echo 'DM_BFILE_TEST_CONTAINER=dmpython-ci-dm8' + echo 'DM_EXPECT_UNICODE=${{ matrix.charset }}' } >> "$GITHUB_ENV" docker run -d --user root --name dmpython-ci-dm8 \ -e SYSDBA_PWD="$admin_password" \ -e SYSAUDITOR_PWD="$admin_password" \ - -e CHARSET=1 -e DB_NAME=DMPYTEST -e INSTANCE_NAME=DMPYTEST \ + -e CHARSET=${{ matrix.charset }} -e DB_NAME=DMPYTEST -e INSTANCE_NAME=DMPYTEST \ -p 127.0.0.1:15236:5236 \ yhl452493373/dm8@sha256:5b9d23c04b148d5765d6077d95b64032be64daedabeef9cddc7a4216b9ad0a1a mkdir -p dpi_include @@ -86,7 +88,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: real-dm-arm-py${{ matrix.python-version }} + name: real-dm-arm-py${{ matrix.python-version }}-unicode${{ matrix.charset }} path: | pytest-dm.xml trend-dm.json diff --git a/docs/test-results/2026-09-26-type-connection-matrix.md b/docs/test-results/2026-09-26-type-connection-matrix.md index 88803f6..d071a5d 100644 --- a/docs/test-results/2026-09-26-type-connection-matrix.md +++ b/docs/test-results/2026-09-26-type-connection-matrix.md @@ -53,3 +53,7 @@ ## 后续回归：十进制与时区边界 `DECIMAL(30,8)` 的最大绝对值、正负最小非零值与零值单条写入通过；混合 `Decimal`、文本、科学计数法和 `NULL` 的 `executemany` 批量写入也保留精度。`TIME WITH TIME ZONE` 与 `TIMESTAMP WITH TIME ZONE` 在 `+14:00`、`-12:59` 偏移量和跨日附近的微秒值往返通过。本机官方 DM8 上 Python 3.9 至 3.13 各有 18 项针对性用例通过。 + +## 后续修复：GB18030 大字段边界 + +新增本机官方 ARM DM8 的 GB18030（`UNICODE=0`）隔离实例。修复前，包含中文和四字节 emoji 的 CLOB 跨越 16,000 字节写入边界时，文本读回虽完整，数据库的 `LENGTH(c)` 和 `DBMS_LOB.GETLENGTH(c)` 会少计 1 至 2 个字符。写入时现在让 GB18030 数据块结束在完整字符处。Python 3.10 在 UTF-8（`UNICODE=1`）和 GB18030 两套实例上的完整真实库回归各为 **166 passed、0 skipped**，另各有 2 个非真实库用例未选入。CI 增加 Python 3.9 至 3.13 × 两种编码的矩阵，并校验数据库实际 `UNICODE` 值；CI 结果待核实。 diff --git a/dpi_bridge/third_party/chunanyong_dm/PATCHES.md b/dpi_bridge/third_party/chunanyong_dm/PATCHES.md index 1fb0ace..5ef0ce9 100644 --- a/dpi_bridge/third_party/chunanyong_dm/PATCHES.md +++ b/dpi_bridge/third_party/chunanyong_dm/PATCHES.md @@ -31,6 +31,18 @@ - `test_clob_unicode_problem_patterns_length_contract` - `test_clob_unicode_problem_patterns_subprocess_no_crash` +## Patch: GB18030 CLOB write boundaries + +- Files: `a.go`, `clob_encoding.go` +- Problem: a 16,000-byte `PUT_DATA` chunk could split a GB18030 two- or + four-byte character. The CLOB content read back intact, but the database's + `LENGTH(c)` and `DBMS_LOB.GETLENGTH(c)` undercounted the characters. +- Fix: end each full GB18030 CLOB chunk at a complete character. Binary and + other parameter types retain their existing chunking. +- Regression: `test_clob_unicode_problem_patterns_length_contract` now runs + against both UTF-8 and GB18030 databases; `clob_encoding_test.go` covers + ASCII, two-byte, and four-byte boundaries. + ## Patch: initial connection timeout through endpoint groups - Files: `a.go`, `n.go`, `x.go`, `y.go`, `m.go` diff --git a/dpi_bridge/third_party/chunanyong_dm/a.go b/dpi_bridge/third_party/chunanyong_dm/a.go index 8a631f2..cd9913e 100644 --- a/dpi_bridge/third_party/chunanyong_dm/a.go +++ b/dpi_bridge/third_party/chunanyong_dm/a.go @@ -821,7 +821,10 @@ func (dm_build_666 *dm_build_414) dm_build_665(dm_build_667 *DmStatement, dm_bui var dm_build_671 = Dm_build_4() dm_build_670.read(dm_build_671) var dm_build_672 = 0 - var utf8ClobParam = dm_build_668.colType == CLOB && dm_build_666.dm_build_418.getServerEncoding() == "UTF-8" + clobEncoding := "" + if dm_build_668.colType == CLOB { + clobEncoding = dm_build_666.dm_build_418.getServerEncoding() + } for !dm_build_670.isReadOver() || dm_build_671.Dm_build_5() > 0 { if !dm_build_670.isReadOver() && dm_build_671.Dm_build_5() < Dm_build_820 { dm_build_670.read(dm_build_671) @@ -831,13 +834,18 @@ func (dm_build_666 *dm_build_414) dm_build_665(dm_build_667 *DmStatement, dm_bui } else { dm_build_672 = dm_build_671.Dm_build_5() } - if utf8ClobParam && dm_build_672 == Dm_build_820 && dm_build_671.Dm_build_5() > dm_build_672 { - safeLen := dm_build_672 - for safeLen > 0 && (dm_build_671.dm_build_32(safeLen)&0xC0) == 0x80 { - safeLen-- - } - if safeLen > 0 { - dm_build_672 = safeLen + if dm_build_672 == Dm_build_820 && dm_build_671.Dm_build_5() > dm_build_672 { + switch clobEncoding { + case ENCODING_UTF8: + safeLen := dm_build_672 + for safeLen > 0 && (dm_build_671.dm_build_32(safeLen)&0xC0) == 0x80 { + safeLen-- + } + if safeLen > 0 { + dm_build_672 = safeLen + } + case ENCODING_GB18030: + dm_build_672 = gb18030SafePrefix(dm_build_671, dm_build_672) } } diff --git a/dpi_bridge/third_party/chunanyong_dm/clob_encoding.go b/dpi_bridge/third_party/chunanyong_dm/clob_encoding.go new file mode 100644 index 0000000..86d9874 --- /dev/null +++ b/dpi_bridge/third_party/chunanyong_dm/clob_encoding.go @@ -0,0 +1,27 @@ +package dm + +// gb18030SafePrefix returns the longest whole-character prefix within limit. +// The CLOB PUT_DATA protocol must not split a two- or four-byte character. +func gb18030SafePrefix(buffer *Dm_build_0, limit int) int { + position := 0 + for position < limit { + width := 1 + first := buffer.dm_build_32(position) + if first >= 0x81 && first <= 0xFE { + if position+1 >= limit { + break + } + second := buffer.dm_build_32(position + 1) + if second >= 0x30 && second <= 0x39 { + width = 4 + } else { + width = 2 + } + } + if position+width > limit { + break + } + position += width + } + return position +} diff --git a/dpi_bridge/third_party/chunanyong_dm/clob_encoding_test.go b/dpi_bridge/third_party/chunanyong_dm/clob_encoding_test.go new file mode 100644 index 0000000..a230a66 --- /dev/null +++ b/dpi_bridge/third_party/chunanyong_dm/clob_encoding_test.go @@ -0,0 +1,16 @@ +package dm + +import "testing" + +func TestGB18030SafePrefix(t *testing.T) { + // ASCII, two-byte Chinese, four-byte supplementary character, ASCII. + encoded := []byte{'A', 0xD6, 0xD0, 0x95, 0x30, 0x8B, 0x34, 'B'} + buffer := Dm_build_4() + buffer.Dm_build_26(encoded, 0, len(encoded)) + want := []int{0, 1, 1, 3, 3, 3, 3, 7, 8} + for limit, expected := range want { + if got := gb18030SafePrefix(buffer, limit); got != expected { + t.Errorf("limit %d: got %d, want %d", limit, got, expected) + } + } +} diff --git a/scripts/prepare_dm_ci.py b/scripts/prepare_dm_ci.py index 382bb60..6472525 100644 --- a/scripts/prepare_dm_ci.py +++ b/scripts/prepare_dm_ci.py @@ -46,6 +46,13 @@ assert cur.fetchone() == (1,) cur.execute("SELECT ID_CODE FROM V$INSTANCE") server_id_code = cur.fetchone()[0] + cur.execute("SELECT UNICODE") + server_unicode = int(cur.fetchone()[0]) + expected_unicode = int(os.environ["DM_EXPECT_UNICODE"]) + if server_unicode != expected_unicode: + raise RuntimeError( + f"DM8 UNICODE={server_unicode}, expected {expected_unicode}" + ) finally: test_conn.close() Path("dm-ci-environment.json").write_text( @@ -55,6 +62,7 @@ "machine": platform.machine(), "driver": dmPython.version, "server_id_code": server_id_code, + "server_unicode": server_unicode, }, indent=2, ) + "\n",