From c43a9d5dd2647bd2dadc1222c7e5168c43d28325 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:34:15 +0000 Subject: [PATCH 1/2] test(benchmarks): add DuckShop edge cases for text, number, array, clock and XML functions Cover text and encoding functions, flooring and bit masks, null-safe comparisons, a comma join after an explicit join, order-free HASH_AGG, array edits, clock text conversions with clock-free CURRENT_DATE and SYSDATE facts, and XML parsing, with expectations captured on Snowflake. --- .../edge_cases/int_edge_amount_bit_reads.sql | 121 ++++++++++++++++++ .../int_edge_basket_array_edits.sql | 57 +++++++++ .../edge_cases/int_edge_basket_xml_reads.sql | 55 ++++++++ .../edge_cases/int_edge_clock_text_reads.sql | 44 +++++++ .../edge_cases/int_edge_sku_text_reads.sql | 61 +++++++++ .../duckshop/project/sqlbuild_project.toml | 16 +++ .../test_int_edge_amount_bit_reads.sql | 92 +++++++++++++ .../test_int_edge_basket_array_edits.sql | 52 ++++++++ .../test_int_edge_basket_xml_reads.sql | 40 ++++++ .../test_int_edge_clock_text_reads.sql | 59 +++++++++ .../test_int_edge_sku_text_reads.sql | 103 +++++++++++++++ 11 files changed, 700 insertions(+) create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_amount_bit_reads.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_basket_array_edits.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_basket_xml_reads.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_clock_text_reads.sql create mode 100644 benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_sku_text_reads.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_amount_bit_reads.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_basket_array_edits.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_basket_xml_reads.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_clock_text_reads.sql create mode 100644 benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_sku_text_reads.sql diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_amount_bit_reads.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_amount_bit_reads.sql new file mode 100644 index 000000000..3afd7d338 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_amount_bit_reads.sql @@ -0,0 +1,121 @@ +MODEL ( + description """ + One row per reconciliation check with numeric reads at their edges: FLOAT ratios and amounts + floored with and without a scale, BITAND of keys, ZEROIFNULL, ::BIGINT and ::DOUBLE casts, keys + compared with IS NOT DISTINCT FROM and EQUAL_NULL, a comma join to the group list written after + an explicit join, and HASH_AGG of a group read in two row orders, which Snowflake does not let + change the hash + """, + tags [edge_case], + columns ( + check_id (type VARCHAR, nullable false, audits [not_null, unique]), + ratio_floor (type FLOAT), + amount_floor_tenths (type NUMBER(38, 1)), + narrow_key_bits (type NUMBER(38, 0)), + amount_or_zero (type NUMBER(18, 2)), + narrow_key_bigint (type NUMBER(38, 0)), + amount_double (type DOUBLE), + keys_not_distinct (type BOOLEAN), + keys_equal_null (type BOOLEAN), + group_name (type VARCHAR), + group_check_count (type NUMBER(38, 0)), + group_hash_is_order_free (type BOOLEAN), + ), +); + +WITH checks AS ( + SELECT + c.check_id, + c.group_code, + c.narrow_key, + c.wide_key, + c.amount, + c.ratio + FROM __ref("stg_edge_amount_checks") AS c +), + +group_names AS ( + SELECT + v.group_code, + v.group_name + FROM VALUES + ('G-1', 'Mallard desk'), + ('G-2', 'Teal desk'), + ('G-3', 'Eider desk') AS v(group_code, group_name) +), + +group_sizes AS ( + SELECT + c.group_code, + COUNT(*) AS group_check_count + FROM checks AS c + GROUP BY + c.group_code +), + +checks_forward AS ( + SELECT + k.group_code, + k.check_id, + k.amount + FROM checks AS k + ORDER BY + k.check_id +), + +checks_backward AS ( + SELECT + k.group_code, + k.check_id, + k.amount + FROM checks AS k + ORDER BY + k.check_id DESC +), + +hash_forward AS ( + SELECT + c.group_code, + HASH_AGG(c.check_id, c.amount) AS group_hash + FROM checks_forward AS c + GROUP BY + c.group_code +), + +hash_backward AS ( + SELECT + c.group_code, + HASH_AGG(c.check_id, c.amount) AS group_hash + FROM checks_backward AS c + GROUP BY + c.group_code +), + +final AS ( + SELECT + c.check_id, + FLOOR(c.ratio)::FLOAT AS ratio_floor, + FLOOR(c.amount, 1)::NUMBER(38, 1) AS amount_floor_tenths, + BITAND(c.narrow_key, 6)::NUMBER(38, 0) AS narrow_key_bits, + ZEROIFNULL(c.amount)::NUMBER(18, 2) AS amount_or_zero, + c.narrow_key::BIGINT AS narrow_key_bigint, + c.amount::DOUBLE AS amount_double, + (c.narrow_key IS NOT DISTINCT FROM c.wide_key)::BOOLEAN AS keys_not_distinct, + EQUAL_NULL(c.narrow_key, c.wide_key)::BOOLEAN AS keys_equal_null, + n.group_name::VARCHAR AS group_name, + s.group_check_count::NUMBER(38, 0) AS group_check_count, + (f.group_hash = b.group_hash)::BOOLEAN AS group_hash_is_order_free + FROM checks AS c + INNER JOIN group_sizes AS s + ON c.group_code = s.group_code, + group_names AS n, + hash_forward AS f, + hash_backward AS b + WHERE + n.group_code = c.group_code + AND f.group_code = c.group_code + AND b.group_code = c.group_code +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_basket_array_edits.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_basket_array_edits.sql new file mode 100644 index 000000000..57b53fdc9 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_basket_array_edits.sql @@ -0,0 +1,57 @@ +MODEL ( + description """ + One row per basket with its ordered SKU list edited by array functions: ARRAY_APPEND, + ARRAY_INTERSECTION with a fixed list, ARRAY_SLICE with positive and negative bounds, TO_ARRAY of + a single tag and of a NULL, and an array written with brackets + """, + tags [edge_case], + columns ( + basket_id (type VARCHAR, nullable false, audits [not_null, unique]), + skus (type ARRAY), + skus_with_bag (type ARRAY), + common_skus (type ARRAY), + first_two_skus (type ARRAY), + all_but_last_sku (type ARRAY), + first_tag_array (type ARRAY), + missing_tag_array (type ARRAY), + bracket_pair (type ARRAY), + ), +); + +WITH lines AS ( + SELECT + l.basket_id, + l.line_no, + l.sku, + l.tag + FROM __ref("stg_edge_basket_lines") AS l +), + +baskets AS ( + SELECT + l.basket_id, + ARRAY_AGG(l.sku) WITHIN GROUP (ORDER BY l.line_no, l.sku) AS skus, + MIN_BY(l.tag, l.line_no) AS first_tag + FROM lines AS l + GROUP BY + l.basket_id +), + +final AS ( + SELECT + b.basket_id, + b.skus::ARRAY AS skus, + ARRAY_APPEND(b.skus, 'BAG-00')::ARRAY AS skus_with_bag, + ARRAY_SORT( + ARRAY_INTERSECTION(b.skus, ARRAY_CONSTRUCT('MALLARD-01', 'EIDER-03', 'SWAN-09')) + )::ARRAY AS common_skus, + ARRAY_SLICE(b.skus, 0, 2)::ARRAY AS first_two_skus, + ARRAY_SLICE(b.skus, 0, -1)::ARRAY AS all_but_last_sku, + TO_ARRAY(b.first_tag)::ARRAY AS first_tag_array, + TO_ARRAY(NULL)::ARRAY AS missing_tag_array, + [b.basket_id, ARRAY_SIZE(b.skus)]::ARRAY AS bracket_pair + FROM baskets AS b +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_basket_xml_reads.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_basket_xml_reads.sql new file mode 100644 index 000000000..3aa8cfdf2 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_basket_xml_reads.sql @@ -0,0 +1,55 @@ +MODEL ( + description """ + One row per basket line written as a small XML document and read back: CHECK_XML of the text and + of a broken copy, PARSE_XML, and XMLGET of the line's tag and SKU elements, including a line + whose tag element is empty + """, + tags [edge_case], + columns ( + line_id (type VARCHAR, nullable false, audits [not_null, unique]), + document_is_valid (type BOOLEAN), + broken_copy_is_valid (type BOOLEAN), + root_name (type VARCHAR), + tag_text (type VARCHAR), + sku_text (type VARCHAR), + ), +); + +WITH lines AS ( + SELECT + l.line_id, + l.sku, + l.tag + FROM __ref("stg_edge_basket_lines") AS l +), + +documents AS ( + SELECT + l.line_id, + ( + '' || l.sku || '' || COALESCE(l.tag, '') || '' + )::VARCHAR AS document_text + FROM lines AS l +), + +parsed AS ( + SELECT + d.line_id, + d.document_text, + PARSE_XML(d.document_text) AS document + FROM documents AS d +), + +final AS ( + SELECT + p.line_id, + (CHECK_XML(p.document_text) IS NULL)::BOOLEAN AS document_is_valid, + (CHECK_XML(REPLACE(p.document_text, '', '')) IS NULL)::BOOLEAN AS broken_copy_is_valid, + GET(p.document, '@')::VARCHAR AS root_name, + XMLGET(p.document, 'tag'):"$"::VARCHAR AS tag_text, + XMLGET(p.document, 'sku'):"$"::VARCHAR AS sku_text + FROM parsed AS p +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_clock_text_reads.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_clock_text_reads.sql new file mode 100644 index 000000000..4081e4bf8 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_clock_text_reads.sql @@ -0,0 +1,44 @@ +MODEL ( + description """ + One row per partner clock text read by conversion functions: DATE and TO_TIME of the text's date + and time parts, TRY_TO_TIMESTAMP of the whole text, and TO_TIMESTAMP_LTZ of zoned text read back + as the instant at UTC, so the session time zone does not change it. CURRENT_DATE and SYSDATE are + read only for facts that hold on any day: the date is after 2020, and the date differs from + itself by zero days + """, + tags [edge_case], + columns ( + text_id (type VARCHAR, nullable false, audits [not_null, unique]), + clock_date (type DATE), + clock_time (type TIME), + clock_timestamp (type TIMESTAMP_NTZ), + zoned_instant_utc (type TIMESTAMP_NTZ), + today_is_after_2020 (type BOOLEAN), + today_minus_today (type NUMBER(38, 0)), + system_clock_is_after_2020 (type BOOLEAN), + ), +); + +WITH texts AS ( + SELECT + t.text_id, + t.zoned_text, + t.wall_clock_text + FROM __ref("stg_edge_date_texts") AS t +), + +final AS ( + SELECT + t.text_id, + DATE(SPLIT_PART(t.wall_clock_text, ' ', 1))::DATE AS clock_date, + TO_TIME(SPLIT_PART(t.wall_clock_text, ' ', 2))::TIME AS clock_time, + TRY_TO_TIMESTAMP(t.wall_clock_text)::TIMESTAMP_NTZ AS clock_timestamp, + CONVERT_TIMEZONE('UTC', TO_TIMESTAMP_LTZ(t.zoned_text))::TIMESTAMP_NTZ AS zoned_instant_utc, + (CURRENT_DATE >= '2020-01-01'::DATE)::BOOLEAN AS today_is_after_2020, + DATEDIFF('day', CURRENT_DATE, CURRENT_DATE)::NUMBER(38, 0) AS today_minus_today, + (SYSDATE() >= '2020-01-01 00:00:00'::TIMESTAMP_NTZ)::BOOLEAN AS system_clock_is_after_2020 + FROM texts AS t +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_sku_text_reads.sql b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_sku_text_reads.sql new file mode 100644 index 000000000..10a63f577 --- /dev/null +++ b/benchmarks/duckshop/project/models/intermediate/edge_cases/int_edge_sku_text_reads.sql @@ -0,0 +1,61 @@ +MODEL ( + description """ + One row per basket line with its SKU read by text functions: STARTSWITH and ENDSWITH, SUBSTRING, + POSITION written with IN and with a comma, CHARINDEX, CHR, REGEXP_SUBSTR_ALL, ::STRING and CAST + ... AS STRING, hexadecimal and Base64 round trips with HEX_ENCODE, HEX_DECODE_STRING, + TRY_HEX_DECODE_STRING and TRY_BASE64_DECODE_BINARY, and a filter on a text literal that carries + a trailing comment + """, + tags [edge_case], + columns ( + line_id (type VARCHAR, nullable false, audits [not_null, unique]), + is_mallard (type BOOLEAN), + ends_in_one (type BOOLEAN), + sku_family (type VARCHAR), + dash_position (type NUMBER(38, 0)), + dash_position_comma (type NUMBER(38, 0)), + second_dash_position (type NUMBER(38, 0)), + sku_with_marker (type VARCHAR), + sku_digits (type ARRAY), + line_no_text (type VARCHAR), + sku_hex (type VARCHAR), + sku_from_hex (type VARCHAR), + odd_hex_text (type VARCHAR), + tag_base64_hex (type VARCHAR), + is_gift_line (type BOOLEAN), + ), +); + +WITH lines AS ( + SELECT + l.line_id, + l.line_no, + l.sku, + l.tag + FROM __ref("stg_edge_basket_lines") AS l +), + +final AS ( + SELECT + l.line_id, + STARTSWITH(l.sku, 'MALLARD')::BOOLEAN AS is_mallard, + ENDSWITH(l.sku, '1')::BOOLEAN AS ends_in_one, + SUBSTRING(l.sku, 1, POSITION('-' IN l.sku) - 1)::STRING AS sku_family, + POSITION('-' IN l.sku)::NUMBER(38, 0) AS dash_position, + POSITION('-', l.sku)::NUMBER(38, 0) AS dash_position_comma, + CHARINDEX('-', l.sku || '-x', 3)::NUMBER(38, 0) AS second_dash_position, + (l.sku || CHR(35))::STRING AS sku_with_marker, + REGEXP_SUBSTR_ALL(l.sku, '[0-9]')::ARRAY AS sku_digits, + CAST(l.line_no AS STRING) AS line_no_text, + HEX_ENCODE(l.sku)::STRING AS sku_hex, + HEX_DECODE_STRING(HEX_ENCODE(l.sku))::STRING AS sku_from_hex, + TRY_HEX_DECODE_STRING(l.tag)::STRING AS odd_hex_text, + HEX_ENCODE(TRY_BASE64_DECODE_BINARY(l.tag))::STRING AS tag_base64_hex, + ( + l.tag = 'gift' /* the tag gift baskets carry */ + )::BOOLEAN AS is_gift_line + FROM lines AS l +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/sqlbuild_project.toml b/benchmarks/duckshop/project/sqlbuild_project.toml index 103f00c94..80bed24c3 100644 --- a/benchmarks/duckshop/project/sqlbuild_project.toml +++ b/benchmarks/duckshop/project/sqlbuild_project.toml @@ -315,6 +315,22 @@ The edge case leaves two calculated columns unaliased on purpose, and reads them Snowflake gives them. """ +[[rules.rule_ignores]] +rules = ["SQLCLEAN040"] +paths = ["models/intermediate/edge_cases/int_edge_amount_bit_reads.sql"] +reason = """ +The edge case joins the group list with a comma after an explicit join on purpose, and Rules read +the comma-joined relation as part of the preceding ON condition. +""" + +[[rules.rule_ignores]] +rules = ["SQLCLEAN033"] +paths = ["models/intermediate/edge_cases/int_edge_sku_text_reads.sql"] +reason = """ +The edge case attaches a comment to a text literal on purpose, the shape that parsers keep as an +annotated expression. +""" + [[rules.rule_ignores]] rules = ["SQLCLEAN027", "SQLCLEAN028"] paths = ["models/intermediate/edge_cases/int_edge_category_walk.sql"] diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_amount_bit_reads.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_amount_bit_reads.sql new file mode 100644 index 000000000..bdae23e66 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_amount_bit_reads.sql @@ -0,0 +1,92 @@ +TEST (name "int_edge_amount_bit_reads__floors_masks_and_compares_amounts_and_keys"); + +WITH __ref__stg_edge_amount_checks AS ( + SELECT + v.check_id, + v.group_code, + v.narrow_key::NUMBER(18, 0) AS narrow_key, + v.wide_key::NUMBER(38, 0) AS wide_key, + NULL::VARCHAR AS key_text, + v.amount::NUMBER(18, 2) AS amount, + NULL::NUMBER(20, 15) AS fine_amount, + v.ratio::FLOAT AS ratio, + NULL::VARCHAR AS json_text + FROM VALUES + ('AC-01', 'G-1', 7, 7, 10.05, -2.5), + ('AC-02', 'G-1', 12, NULL, -0.55, 2.5), + ('AC-03', 'G-2', NULL, NULL, NULL, 0.29), + ('AC-04', 'G-2', 5, 6, 0.00, NULL), + ('AC-05', 'G-3', 13, 13, 999.99, -0.0) AS v(check_id, group_code, narrow_key, wide_key, amount, ratio) +), + +__expected__int_edge_amount_bit_reads AS ( + SELECT + 'AC-01' AS check_id, + '-3.0'::FLOAT AS ratio_floor, + 10.0::NUMBER(38, 1) AS amount_floor_tenths, + 6 AS narrow_key_bits, + 10.05::NUMBER(18, 2) AS amount_or_zero, + 7 AS narrow_key_bigint, + '10.05'::FLOAT AS amount_double, + TRUE AS keys_not_distinct, + TRUE AS keys_equal_null, + 'Mallard desk' AS group_name, + 2 AS group_check_count, + TRUE AS group_hash_is_order_free + UNION ALL + SELECT + 'AC-02' AS check_id, + '2.0'::FLOAT AS ratio_floor, + -0.6::NUMBER(38, 1) AS amount_floor_tenths, + 4 AS narrow_key_bits, + -0.55::NUMBER(18, 2) AS amount_or_zero, + 12 AS narrow_key_bigint, + '-0.55'::FLOAT AS amount_double, + FALSE AS keys_not_distinct, + FALSE AS keys_equal_null, + 'Mallard desk' AS group_name, + 2 AS group_check_count, + TRUE AS group_hash_is_order_free + UNION ALL + SELECT + 'AC-03' AS check_id, + '0.0'::FLOAT AS ratio_floor, + CAST(NULL AS NUMBER(38, 1)) AS amount_floor_tenths, + CAST(NULL AS NUMBER(38, 0)) AS narrow_key_bits, + 0.00::NUMBER(18, 2) AS amount_or_zero, + CAST(NULL AS NUMBER(38, 0)) AS narrow_key_bigint, + CAST(NULL AS FLOAT) AS amount_double, + TRUE AS keys_not_distinct, + TRUE AS keys_equal_null, + 'Teal desk' AS group_name, + 2 AS group_check_count, + TRUE AS group_hash_is_order_free + UNION ALL + SELECT + 'AC-04' AS check_id, + CAST(NULL AS FLOAT) AS ratio_floor, + 0.0::NUMBER(38, 1) AS amount_floor_tenths, + 4 AS narrow_key_bits, + 0.00::NUMBER(18, 2) AS amount_or_zero, + 5 AS narrow_key_bigint, + '0.0'::FLOAT AS amount_double, + FALSE AS keys_not_distinct, + FALSE AS keys_equal_null, + 'Teal desk' AS group_name, + 2 AS group_check_count, + TRUE AS group_hash_is_order_free + UNION ALL + SELECT + 'AC-05' AS check_id, + '0.0'::FLOAT AS ratio_floor, + 999.9::NUMBER(38, 1) AS amount_floor_tenths, + 4 AS narrow_key_bits, + 999.99::NUMBER(18, 2) AS amount_or_zero, + 13 AS narrow_key_bigint, + '999.99'::FLOAT AS amount_double, + TRUE AS keys_not_distinct, + TRUE AS keys_equal_null, + 'Eider desk' AS group_name, + 1 AS group_check_count, + TRUE AS group_hash_is_order_free +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_basket_array_edits.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_basket_array_edits.sql new file mode 100644 index 000000000..8afb8d019 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_basket_array_edits.sql @@ -0,0 +1,52 @@ +TEST (name "int_edge_basket_array_edits__appends_intersects_and_slices_sku_lists"); + +WITH __ref__stg_edge_basket_lines AS ( + SELECT + v.line_id, + v.basket_id, + v.line_no::NUMBER(38, 0) AS line_no, + v.sku, + v.tag + FROM VALUES + ('BL-01', 'B-1', 1, 'MALLARD-01', 'bath'), + ('BL-02', 'B-1', 2, 'TEAL-02', 'gift'), + ('BL-03', 'B-1', 3, 'EIDER-03', 'bath'), + ('BL-04', 'B-2', 1, 'PINTAIL-04', 'garden'), + ('BL-05', 'B-3', 1, 'MERGANSER-05', NULL), + ('BL-06', 'B-3', 2, 'EIDER-03', 'gift') AS v(line_id, basket_id, line_no, sku, tag) +), + +__expected__int_edge_basket_array_edits AS ( + SELECT + 'B-1' AS basket_id, + PARSE_JSON('["MALLARD-01","TEAL-02","EIDER-03"]')::ARRAY AS skus, + PARSE_JSON('["MALLARD-01","TEAL-02","EIDER-03","BAG-00"]')::ARRAY AS skus_with_bag, + PARSE_JSON('["EIDER-03","MALLARD-01"]')::ARRAY AS common_skus, + PARSE_JSON('["MALLARD-01","TEAL-02"]')::ARRAY AS first_two_skus, + PARSE_JSON('["MALLARD-01","TEAL-02"]')::ARRAY AS all_but_last_sku, + PARSE_JSON('["bath"]')::ARRAY AS first_tag_array, + CAST(NULL AS ARRAY) AS missing_tag_array, + PARSE_JSON('["B-1",3]')::ARRAY AS bracket_pair + UNION ALL + SELECT + 'B-2' AS basket_id, + PARSE_JSON('["PINTAIL-04"]')::ARRAY AS skus, + PARSE_JSON('["PINTAIL-04","BAG-00"]')::ARRAY AS skus_with_bag, + PARSE_JSON('[]')::ARRAY AS common_skus, + PARSE_JSON('["PINTAIL-04"]')::ARRAY AS first_two_skus, + PARSE_JSON('[]')::ARRAY AS all_but_last_sku, + PARSE_JSON('["garden"]')::ARRAY AS first_tag_array, + CAST(NULL AS ARRAY) AS missing_tag_array, + PARSE_JSON('["B-2",1]')::ARRAY AS bracket_pair + UNION ALL + SELECT + 'B-3' AS basket_id, + PARSE_JSON('["MERGANSER-05","EIDER-03"]')::ARRAY AS skus, + PARSE_JSON('["MERGANSER-05","EIDER-03","BAG-00"]')::ARRAY AS skus_with_bag, + PARSE_JSON('["EIDER-03"]')::ARRAY AS common_skus, + PARSE_JSON('["MERGANSER-05","EIDER-03"]')::ARRAY AS first_two_skus, + PARSE_JSON('["MERGANSER-05"]')::ARRAY AS all_but_last_sku, + CAST(NULL AS ARRAY) AS first_tag_array, + CAST(NULL AS ARRAY) AS missing_tag_array, + PARSE_JSON('["B-3",2]')::ARRAY AS bracket_pair +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_basket_xml_reads.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_basket_xml_reads.sql new file mode 100644 index 000000000..2baf81b20 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_basket_xml_reads.sql @@ -0,0 +1,40 @@ +TEST (name "int_edge_basket_xml_reads__reads_elements_back_from_line_documents"); + +WITH __ref__stg_edge_basket_lines AS ( + SELECT + v.line_id, + v.basket_id, + v.line_no::NUMBER(38, 0) AS line_no, + v.sku, + v.tag + FROM VALUES + ('BL-01', 'B-1', 1, 'MALLARD-01', 'bath'), + ('BL-02', 'B-1', 2, 'TEAL-02', NULL), + ('BL-03', 'B-2', 1, 'EIDER-03', 'gift wrap') AS v(line_id, basket_id, line_no, sku, tag) +), + +__expected__int_edge_basket_xml_reads AS ( + SELECT + 'BL-01' AS line_id, + TRUE AS document_is_valid, + FALSE AS broken_copy_is_valid, + 'line' AS root_name, + 'bath' AS tag_text, + 'MALLARD-01' AS sku_text + UNION ALL + SELECT + 'BL-02' AS line_id, + TRUE AS document_is_valid, + FALSE AS broken_copy_is_valid, + 'line' AS root_name, + '' AS tag_text, + 'TEAL-02' AS sku_text + UNION ALL + SELECT + 'BL-03' AS line_id, + TRUE AS document_is_valid, + FALSE AS broken_copy_is_valid, + 'line' AS root_name, + 'gift wrap' AS tag_text, + 'EIDER-03' AS sku_text +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_clock_text_reads.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_clock_text_reads.sql new file mode 100644 index 000000000..5b070d885 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_clock_text_reads.sql @@ -0,0 +1,59 @@ +TEST (name "int_edge_clock_text_reads__reads_clock_text_parts_and_clock_free_facts"); + +WITH __ref__stg_edge_date_texts AS ( + SELECT + v.text_id, + NULL::VARCHAR AS date_text, + v.zoned_text, + NULL::TIMESTAMP_NTZ(3) AS wall_clock, + v.wall_clock_text, + NULL::TIME(3) AS local_time, + NULL::TIME(9) AS fine_time + FROM VALUES + ('CT-01', '2026-03-06T08:30:00+09:00', '2026-03-05 23:30:00.1239'), + ('CT-02', '2026-03-05T23:30:00Z', '1969-12-31 23:59:59.999'), + ('CT-03', '2026-12-31T23:30:00-05:30', '2026-12-31 00:00:00'), + ('CT-04', NULL, NULL) AS v(text_id, zoned_text, wall_clock_text) +), + +__expected__int_edge_clock_text_reads AS ( + SELECT + 'CT-01' AS text_id, + '2026-03-05'::DATE AS clock_date, + '23:30:00.1239'::TIME(9) AS clock_time, + '2026-03-05 23:30:00.1239'::TIMESTAMP_NTZ(9) AS clock_timestamp, + '2026-03-05 23:30:00'::TIMESTAMP_NTZ(9) AS zoned_instant_utc, + TRUE AS today_is_after_2020, + 0 AS today_minus_today, + TRUE AS system_clock_is_after_2020 + UNION ALL + SELECT + 'CT-02' AS text_id, + '1969-12-31'::DATE AS clock_date, + '23:59:59.999'::TIME(9) AS clock_time, + '1969-12-31 23:59:59.999'::TIMESTAMP_NTZ(9) AS clock_timestamp, + '2026-03-05 23:30:00'::TIMESTAMP_NTZ(9) AS zoned_instant_utc, + TRUE AS today_is_after_2020, + 0 AS today_minus_today, + TRUE AS system_clock_is_after_2020 + UNION ALL + SELECT + 'CT-03' AS text_id, + '2026-12-31'::DATE AS clock_date, + '00:00:00'::TIME(9) AS clock_time, + '2026-12-31 00:00:00'::TIMESTAMP_NTZ(9) AS clock_timestamp, + '2027-01-01 05:00:00'::TIMESTAMP_NTZ(9) AS zoned_instant_utc, + TRUE AS today_is_after_2020, + 0 AS today_minus_today, + TRUE AS system_clock_is_after_2020 + UNION ALL + SELECT + 'CT-04' AS text_id, + CAST(NULL AS DATE) AS clock_date, + CAST(NULL AS TIME(9)) AS clock_time, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS clock_timestamp, + CAST(NULL AS TIMESTAMP_NTZ(9)) AS zoned_instant_utc, + TRUE AS today_is_after_2020, + 0 AS today_minus_today, + TRUE AS system_clock_is_after_2020 +) diff --git a/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_sku_text_reads.sql b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_sku_text_reads.sql new file mode 100644 index 000000000..49a8c88b7 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/intermediate/edge_cases/test_int_edge_sku_text_reads.sql @@ -0,0 +1,103 @@ +TEST (name "int_edge_sku_text_reads__reads_skus_with_text_and_encoding_functions"); + +WITH __ref__stg_edge_basket_lines AS ( + SELECT + v.line_id, + v.basket_id, + v.line_no::NUMBER(38, 0) AS line_no, + v.sku, + v.tag + FROM VALUES + ('BL-01', 'B-1', 1, 'MALLARD-01', 'gift'), + ('BL-02', 'B-1', 2, 'TEAL-02', '6869'), + ('BL-03', 'B-2', 1, 'EIDER-1-B', 'Z2lmdA=='), + ('BL-04', 'B-2', 2, 'SWAN', 'abc'), + ('BL-05', 'B-3', 1, 'MALLARD-11', NULL) AS v(line_id, basket_id, line_no, sku, tag) +), + +__expected__int_edge_sku_text_reads AS ( + SELECT + 'BL-01' AS line_id, + TRUE AS is_mallard, + TRUE AS ends_in_one, + 'MALLARD' AS sku_family, + 8 AS dash_position, + 8 AS dash_position_comma, + 8 AS second_dash_position, + 'MALLARD-01#' AS sku_with_marker, + PARSE_JSON('["0","1"]')::ARRAY AS sku_digits, + '1' AS line_no_text, + '4D414C4C4152442D3031' AS sku_hex, + 'MALLARD-01' AS sku_from_hex, + CAST(NULL AS VARCHAR) AS odd_hex_text, + '8227ED' AS tag_base64_hex, + TRUE AS is_gift_line + UNION ALL + SELECT + 'BL-02' AS line_id, + FALSE AS is_mallard, + FALSE AS ends_in_one, + 'TEAL' AS sku_family, + 5 AS dash_position, + 5 AS dash_position_comma, + 5 AS second_dash_position, + 'TEAL-02#' AS sku_with_marker, + PARSE_JSON('["0","2"]')::ARRAY AS sku_digits, + '2' AS line_no_text, + '5445414C2D3032' AS sku_hex, + 'TEAL-02' AS sku_from_hex, + 'hi' AS odd_hex_text, + 'EBCEBD' AS tag_base64_hex, + FALSE AS is_gift_line + UNION ALL + SELECT + 'BL-03' AS line_id, + FALSE AS is_mallard, + FALSE AS ends_in_one, + 'EIDER' AS sku_family, + 6 AS dash_position, + 6 AS dash_position_comma, + 6 AS second_dash_position, + 'EIDER-1-B#' AS sku_with_marker, + PARSE_JSON('["1"]')::ARRAY AS sku_digits, + '1' AS line_no_text, + '45494445522D312D42' AS sku_hex, + 'EIDER-1-B' AS sku_from_hex, + CAST(NULL AS VARCHAR) AS odd_hex_text, + '67696674' AS tag_base64_hex, + FALSE AS is_gift_line + UNION ALL + SELECT + 'BL-04' AS line_id, + FALSE AS is_mallard, + FALSE AS ends_in_one, + '' AS sku_family, + 0 AS dash_position, + 0 AS dash_position_comma, + 5 AS second_dash_position, + 'SWAN#' AS sku_with_marker, + PARSE_JSON('[]')::ARRAY AS sku_digits, + '2' AS line_no_text, + '5357414E' AS sku_hex, + 'SWAN' AS sku_from_hex, + CAST(NULL AS VARCHAR) AS odd_hex_text, + CAST(NULL AS VARCHAR) AS tag_base64_hex, + FALSE AS is_gift_line + UNION ALL + SELECT + 'BL-05' AS line_id, + TRUE AS is_mallard, + TRUE AS ends_in_one, + 'MALLARD' AS sku_family, + 8 AS dash_position, + 8 AS dash_position_comma, + 8 AS second_dash_position, + 'MALLARD-11#' AS sku_with_marker, + PARSE_JSON('["1","1"]')::ARRAY AS sku_digits, + '1' AS line_no_text, + '4D414C4C4152442D3131' AS sku_hex, + 'MALLARD-11' AS sku_from_hex, + CAST(NULL AS VARCHAR) AS odd_hex_text, + CAST(NULL AS VARCHAR) AS tag_base64_hex, + CAST(NULL AS BOOLEAN) AS is_gift_line +) From 59b03d179fb3a443385ebb7ba861295c8976095b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:36:02 +0000 Subject: [PATCH 2/2] docs(benchmarks): list the DuckShop construct-gap edge cases --- benchmarks/duckshop/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index 04cc4d03a..00872da01 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -121,16 +121,16 @@ project and does not change it: - Python functions in `functions/python/edge_*.py`, tagged `edge_case`, and their tests in `tests/unit/functions/python/test_edge_*.sql`. -It has 67 models (12 staging, 45 intermediate, 10 marts), 5 Python functions, 92 unit tests and 4 +It has 72 models (12 staging, 50 intermediate, 10 marts), 5 Python functions, 97 unit tests and 4 scenarios, in eight groups: | Group | Covers | |-------|--------| -| Semi-structured | JSON null against SQL NULL and missing keys; document text with sorted, case-differing and escaped keys; JSON number fidelity and casts out of `VARIANT`; path-like keys (`''`, `*`, `$`-prefixed, dotted) with `GET` and `GET_IGNORE_CASE`; `OBJECT_CONSTRUCT_KEEP_NULL`, `OBJECT_INSERT`, `OBJECT_DELETE`, `OBJECT_PICK`; array builders, `ARRAY_EXCEPT` and number spellings in `ARRAY_POSITION`; `FLATTEN` outer; lenient and malformed `TRY_PARSE_JSON`; typed lambdas; whole numbers written with zero fractions, numbers with two decimal points and numbers past 38 digits; `FLATTEN` `RECURSIVE` paths for keys holding `$`, `-`, `@`, `[`, spaces and non-ASCII letters | -| Ordering through arrays | an unordered `ARRAY_AGG` ranked under `LIMIT` by position, by name, through `IFF` and over `UNION ALL`, with fixtures whose pick does not depend on element order; order-free reads (`ARRAY_SIZE`, `ARRAY_CONTAINS`, `ARRAYS_OVERLAP`, `ARRAY_DISTINCT` of `ARRAY_CAT`, `FLATTEN` values through a CTE); `SELECT DISTINCT` over arrays repeated in different orders | +| Semi-structured | JSON null against SQL NULL and missing keys; document text with sorted, case-differing and escaped keys; JSON number fidelity and casts out of `VARIANT`; path-like keys (`''`, `*`, `$`-prefixed, dotted) with `GET` and `GET_IGNORE_CASE`; `OBJECT_CONSTRUCT_KEEP_NULL`, `OBJECT_INSERT`, `OBJECT_DELETE`, `OBJECT_PICK`; array builders, `ARRAY_EXCEPT` and number spellings in `ARRAY_POSITION`; `FLATTEN` outer; lenient and malformed `TRY_PARSE_JSON`; typed lambdas; whole numbers written with zero fractions, numbers with two decimal points and numbers past 38 digits; `FLATTEN` `RECURSIVE` paths for keys holding `$`, `-`, `@`, `[`, spaces and non-ASCII letters; `PARSE_XML`, `XMLGET` and `CHECK_XML` over small documents, including an empty element | +| Ordering through arrays | an unordered `ARRAY_AGG` ranked under `LIMIT` by position, by name, through `IFF` and over `UNION ALL`, with fixtures whose pick does not depend on element order; order-free reads (`ARRAY_SIZE`, `ARRAY_CONTAINS`, `ARRAYS_OVERLAP`, `ARRAY_DISTINCT` of `ARRAY_CAT`, `FLATTEN` values through a CTE); `SELECT DISTINCT` over arrays repeated in different orders; `ARRAY_APPEND`, `ARRAY_INTERSECTION`, `ARRAY_SLICE` with negative bounds, `TO_ARRAY` and bracket array literals | | VARIANT comparisons | `ARRAY_MAX`, `ARRAY_MIN` and `ARRAY_REVERSE` over arrays mixing numbers, strings, booleans, JSON nulls and integers past 2^53; `MAX` across kinds and non-ASCII strings; `=` and `EQUAL_NULL` for `1`, `"1"` and `1.0`, reordered keys, JSON null and DATE or TIMESTAMP values held in a `VARIANT` against their text; `MIN_BY` and `MAX_BY` of `VARIANT` values; `PARTITION BY`, `SELECT DISTINCT` and `ARRAY_AGG(DISTINCT)` over `VARIANT`; JSON nulls in `ARRAY_REMOVE`, `ARRAY_DISTINCT` and `ARRAYS_OVERLAP`; decimals with trailing zeros in `ARRAY_CONTAINS` and `ARRAY_POSITION`; `OBJECT_CONSTRUCT(* EXCLUDE ("note"))` | -| Dates and times | `TRY_TO_TIMESTAMP_NTZ` and its siblings over free text; epoch milliseconds as numbers and text; daylight-saving gaps and overlaps; fractional offsets; nanosecond truncation and epoch parts after 2242; pre-1970 text with sub-microsecond digits; date-part normalisation; month ends and leap days; week numbering across a year; sentinel timestamps; `TRY_TO_DATE` with numeric and month-name formats over unpadded fields, two-digit years, ISO `T` and `Z`, trailing letters, a long s, mixed separators and runs of spaces; text read as `TIMESTAMP_TZ` and `TIMESTAMP_LTZ` with `T`, `Z`, `+00:00`, `+0000`, `+09:00` and `-05:30`, back as the UTC instant, date, year and day start, and compared with text, `TIMESTAMP_NTZ` and `DATE`; `FIRST_VALUE` of a zoned value as a `DATE`; `DATEADD` and `TIMEADD` on `TIME(3)` and the type they return; `DATEDIFF` across units; `TIME(9)`, `FF3` and bare `FF` formats truncating | -| Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds | +| Dates and times | `TRY_TO_TIMESTAMP_NTZ` and its siblings over free text; epoch milliseconds as numbers and text; daylight-saving gaps and overlaps; fractional offsets; nanosecond truncation and epoch parts after 2242; pre-1970 text with sub-microsecond digits; date-part normalisation; month ends and leap days; week numbering across a year; sentinel timestamps; `TRY_TO_DATE` with numeric and month-name formats over unpadded fields, two-digit years, ISO `T` and `Z`, trailing letters, a long s, mixed separators and runs of spaces; text read as `TIMESTAMP_TZ` and `TIMESTAMP_LTZ` with `T`, `Z`, `+00:00`, `+0000`, `+09:00` and `-05:30`, back as the UTC instant, date, year and day start, and compared with text, `TIMESTAMP_NTZ` and `DATE`; `FIRST_VALUE` of a zoned value as a `DATE`; `DATEADD` and `TIMEADD` on `TIME(3)` and the type they return; `DATEDIFF` across units; `TIME(9)`, `FF3` and bare `FF` formats truncating; `DATE`, `TO_TIME`, `TRY_TO_TIMESTAMP` and `TO_TIMESTAMP_LTZ` of clock text, and `CURRENT_DATE` and `SYSDATE` read only for facts that hold on any day | +| Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts; `JOIN ... USING` between `NUMBER(18,0)`, `NUMBER(38,0)` and text keys, with the types of the merged and qualified keys; NULL arithmetic and the types of NULL, decimal and exponent literals, `COUNT`, `ROW_NUMBER`, `%` and `NULLIF`; `MEDIAN` at scales 0 to 15; `DIV0` by literal zeros; `MODE`, `MEDIAN` and `COUNT_IF` over no rows; aggregates inside window functions and `QUALIFY`; `ROLLUP` totals; `TRUNC` of `FLOAT`; infinity words in other scripts; `CASE` without `ELSE` and `DECODE` over text; regular expressions with a `?` after a lazy quantifier, classes opening with `]`, empty matches and `.` against a carriage return; case-insensitive matches outside ASCII; `ILIKE` with a long s; `JAROWINKLER_SIMILARITY` of accented names in both cases; Base64 text with line feeds; `STARTSWITH`, `ENDSWITH`, `SUBSTRING`, `POSITION` with `IN` and with a comma, `CHARINDEX`, `CHR`, `REGEXP_SUBSTR_ALL`, `::STRING` and `CAST ... AS STRING`; `HEX_ENCODE`, `HEX_DECODE_STRING`, `TRY_HEX_DECODE_STRING` and `TRY_BASE64_DECODE_BINARY`; a comment attached to a text literal; `FLOOR` with and without a scale, `BITAND`, `ZEROIFNULL`, `::BIGINT` and `::DOUBLE`; `IS NOT DISTINCT FROM` and `EQUAL_NULL`; a comma join after an explicit join; `HASH_AGG` read in two row orders | | Lineage and analysis | clauses whose columns decide rows without being output (`WHERE`, `QUALIFY`, `GROUP BY` by position and `ALL`, `ORDER BY` under `LIMIT`, `EXISTS`, a `FLATTEN` input, `ROLLUP` by position, `EXCEPT` under `LIMIT`); `RIGHT JOIN ... USING`, `NATURAL JOIN`, a `USING` key under a later outer join, and unmatched `ASOF JOIN` rows; `CONNECT BY` beside a column named `level`; a recursive CTE without `RECURSIVE` whose columns shift per iteration; `LAST_DAY` with a date part; positional columns and the names Snowflake gives unaliased expressions; `TEXT(5)` and widened `COUNT` contracts | | Python functions | volatile tokens and numbers from a column, a constant and no argument, and `DIV0` of a volatile call beside look-alike names, checked by assertions that hold for any values; a handler answering NULL input without `STRICT`; Python `None` as JSON null, and datetimes, Decimals and integers past 2^53 handed to `VARIANT` and `FLOAT` results | | Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a snapshot keyed and compared on quoted camelCase columns; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild |