Fix Unicode titlecase, folding and encoded case mappings

corpus/506_unicode_case_mapping.rb

@@ -0,0 +1,43 @@
+# Case mapping is context-independent in Ruby. Titlecase and case folding
+# need their own mappings, including characters that map to themselves.
+samples = ["ΣΣ Σ", "ßß", "ʼnʼn", "DžLjNjDz", "ᾈᾀ", "և", "აᲐ", "Ꭰꭰ", "Iİıi",
+           "漢字𠀀🙂aZ"]
+%w[UTF-8 UTF-16LE UTF-16BE UTF-32LE UTF-32BE].each do |encoding|
+  samples.each do |sample|
+    source = sample.encode(encoding)
+    results = [:upcase, :downcase, :capitalize, :swapcase].map do |method|
+      result = source.public_send(method)
+      [result.encode('UTF-8'), result.encoding.name, result.valid_encoding?]
+    end
+    p results
+    p source.downcase(:fold).encode('UTF-8')
+  end
+  source = "Iİıi".encode(encoding)
+  p [:upcase, :downcase, :capitalize, :swapcase].map { |m| source.public_send(m, :turkic).encode('UTF-8') }
+  p "aΣß".encode(encoding).capitalize(:ascii).encode('UTF-8')
+  # Compare bytes here: noncharacter rendering by String#inspect is separate
+  # from the case mapping and the mask's upper bound.
+  high = "\u{10FFFF}Aß".encode(encoding)
+  p [:upcase, :downcase, :capitalize, :swapcase].map { |m| high.public_send(m).bytes }
+  p high.downcase(:fold).bytes
+end
+
+# Even an all-ASCII source can map to a non-ASCII byte in a single-byte
+# encoding. Its bytes must be encoded back rather than copied from UTF-8.
+%w[ISO-8859-1 ISO-8859-3 ISO-8859-9 Windows-1254 Windows-1257].each do |encoding|
+  p "ßß".encode(encoding).capitalize.bytes
+  source = 'Ii'.encode(encoding)
+  p [:upcase, :downcase, :capitalize, :swapcase].map { |m| source.public_send(m, :turkic).bytes }
+end
+
+value = "ßΣ"
+p [value.capitalize!.equal?(value), value, value.capitalize!]
+p "Ꭰ".casecmp?("ꭰ")
+%w[GBK CP936 CP949].each do |encoding|
+  value = "\x80aZ".b.force_encoding(encoding)
+  p [value.valid_encoding?, value.tr('a-z', 'A-Z').bytes,
+     value.upcase.bytes, value.downcase.bytes, value.capitalize.bytes, value.swapcase.bytes]
+end
+value = "\x81A".b.force_encoding('GBK')
+p [value.valid_encoding?, value.downcase.bytes]
+p "\xff".b.force_encoding('GBK').valid_encoding?

m_state.mere

@@ -5022,13 +5022,13 @@ let rec enc_valid_gb18030 = fn (s: str) -> fn (i: int) ->
            && (let b3 = ord (char_at s (i + 3)) in b3 >= 48 && b3 <= 57) && enc_valid_gb18030 s (i + 4))
         else if (b1 >= 64 && b1 <= 126) || (b1 >= 128 && b1 <= 254) then enc_valid_gb18030 s (i + 2)
         else false));
-let enc_valid_db = fn (s: str) -> fn (l_lo: int) -> fn (l_hi: int) -> fn (t1_lo: int) -> fn (t1_hi: int) -> fn (t2_lo: int) -> fn (t2_hi: int) ->
+let enc_valid_db = fn (s: str) -> fn (l_lo: int) -> fn (l_hi: int) -> fn (t1_lo: int) -> fn (t1_hi: int) -> fn (t2_lo: int) -> fn (t2_hi: int) -> fn (single_80: bool) ->
   let n = str_len s in
   let rec go = fn (i: int) ->
     if i >= n then true
     else
       let b0 = ord (char_at s i) in
-      if b0 < 128 then go (i + 1)
+      if b0 < 128 || (single_80 && b0 == 128) then go (i + 1)
       else if b0 >= l_lo && b0 <= l_hi then
         (if i + 1 < n then
            (let b1 = ord (char_at s (i + 1)) in
@@ -5073,14 +5073,14 @@ let enc_valid = fn (s: str) -> fn (e: str) ->
      go 0)
   else if str_eq e "EUC-JP" || str_eq e "eucJP-ms" || str_eq e "CP51932" || str_eq e "EUC-JIS-2004"
           || str_eq e "stateless-ISO-2022-JP-KDDI" then enc_valid_eucjp s
-  else if str_eq e "EUC-KR" || str_eq e "GB2312" || str_eq e "EUC-TW" || str_eq e "GB12345" then enc_valid_db s 161 254 161 254 (0 - 1) (0 - 1)
+  else if str_eq e "EUC-KR" || str_eq e "GB2312" || str_eq e "EUC-TW" || str_eq e "GB12345" then enc_valid_db s 161 254 161 254 (0 - 1) (0 - 1) false
   else if str_eq e "Shift_JIS" || str_eq e "Windows-31J" || str_eq e "MacJapanese"
           || str_eq e "SJIS-DoCoMo" || str_eq e "SJIS-KDDI" || str_eq e "SJIS-SoftBank" then
     enc_valid_sjis s
   else if str_eq e "Big5" || str_eq e "CP950" || str_eq e "CP951"
-          || str_eq e "Big5-HKSCS" || str_eq e "Big5-UAO" then enc_valid_db s 129 254 64 126 161 254
-  else if str_eq e "CP949" then enc_valid_db s 129 254 65 90 97 254
-  else if str_eq e "GBK" || str_eq e "CP936" then enc_valid_db s 129 254 64 126 128 254
+          || str_eq e "Big5-HKSCS" || str_eq e "Big5-UAO" then enc_valid_db s 129 254 64 126 161 254 false
+  else if str_eq e "CP949" then enc_valid_db s 129 254 65 90 97 254 true
+  else if str_eq e "GBK" || str_eq e "CP936" then enc_valid_db s 129 254 64 126 128 254 true
   else if str_eq e "GB18030" then enc_valid_gb18030 s 0
   else if str_eq e "Emacs-Mule" then enc_valid_emacs s
   else true;

m_unicode.mere

@@ -480,16 +480,23 @@ let sbcs_cased = fn (e: str) ->
   || str_eq e "Windows-1250" || str_eq e "Windows-1251" || str_eq e "Windows-1252" || str_eq e "Windows-1253"
   || str_eq e "Windows-1254" || str_eq e "Windows-1257";
 let sbcs_case = fn (s: str) -> fn (e: str) -> fn (kind: str) -> fn (mode: str) ->
+  // Only the single-byte encodings with Turkish case tables honor :turkic.
+  // Windows-1257 also has byte-level overrides in Ruby's encoding table.
+  let effective_mode =
+    (if str_eq mode "t" && (str_eq e "ISO-8859-3" || str_eq e "ISO-8859-9" || str_eq e "Windows-1254") then mode
+     else if str_eq mode "t" then "" else mode) in
   let b9 = strbuf_new () in
   let rec go = fn (i: int) -> fn (first: bool) ->
     if i >= str_len s then ()
     else
       (let c = char_at s i in
-       let k2 = (if str_eq kind "c" then (if first then "u" else "d") else kind) in
+       let k2 = (if str_eq kind "c" && (if first then false else true) then "d" else kind) in
        let out =
-         (match decode_cps c e 0 with
+         (if str_eq mode "t" && str_eq e "Windows-1257" && ord c == 73 && (str_eq k2 "d" || str_eq k2 "s") then chr 185
+          else if str_eq mode "t" && str_eq e "Windows-1257" && ord c == 105 && (str_eq k2 "u" || str_eq k2 "c" || str_eq k2 "s") then chr 169
+          else match decode_cps c e 0 with
           | Cons (cp, Nil) ->
-            (let m9 = rb_case_map1 k2 mode cp in
+            (let m9 = rb_case_map1 k2 effective_mode cp in
              // (Turkish İ lowers to a plain i in these encodings)
              let m2 = (match m9 with Cons (105, Cons (775, Nil)) -> Cons (105, (Nil : int list)) | _ -> m9) in
              match encode_cps m2 e "" with
@@ -508,15 +515,15 @@ let rb_case_apply = fn (sh: int) -> fn (kind: str) -> fn (mode: str) ->
            else (let _ = prim_raise "ArgumentError" "input string invalid" in ())) in
   let body =
     (if enc_ascii_compat e && enc_multibyte e && (if enc_utf8_family e then false else true) then es_case_ascii (sget sh) e kind
-     else if sbcs_cased e && (if str_is_ascii sh then false else true) then sbcs_case (sget sh) e kind mode
+     else if sbcs_cased e && ((if str_is_ascii sh then false else true) || str_eq mode "t") then sbcs_case (sget sh) e kind mode
      else if enc_ascii_compat e then rb_case_str kind mode (sget sh)
      else
        (let cps = decode_cps (sget sh) e 0 in
         let rec tr = fn (xs: int list) -> fn (first: bool) ->
           (match xs with
            | Nil -> (Nil : int list)
            | Cons (cp, t) ->
-             let k2 = (if str_eq kind "c" then (if first then "u" else "d") else kind) in
+             let k2 = (if str_eq kind "c" && (if first then false else true) then "d" else kind) in
              rb_iseq_app (rb_case_map1 k2 mode cp) (tr t false)) in
         (match encode_cps (tr cps true) e "" with (bytes, _) -> bytes))) in
   let c = mkstr body in

tools/gen_case_tables.rb

@@ -1,8 +1,8 @@
-# Generates the Unicode case-mapping tables embedded in main.mere between the
-# ENC_CASE_TABLES markers, asking ruby's own String#upcase/#downcase (and
-# downcase(:fold) for casecmp?'s folding). Only mappings the arithmetic core
-# (ASCII, Latin-1, Greek, Cyrillic offsets) does NOT already cover are listed.
-# Entries are "CP:CP[ CP...]" hex, comma-separated; three tables: UP, DOWN, FOLD.
+# Generates the Unicode case-mapping tables embedded in m_codec.mere using
+# the reference Ruby. Entries are "CP:CP[ CP...]" hex, comma-separated.
+# PAGES is an ASCII 0/1 mask of the pages containing any table entry.
+# FOLD, TITLE and SWAP store exceptions to downcase, upcase and the usual
+# lower/upper swap respectively, including identity mappings when necessary.
 # only ASCII is excluded: the arithmetic core answers Latin-1/Greek/Cyrillic
 # too, but claiming a range here silently drops its irregular members (the
 # first version dropped 0xDF, and eszett stopped upcasing).
@@ -18,16 +18,27 @@ def table(kind)
           when :up then ch.upcase
           when :down then ch.downcase
           when :fold then ch.downcase(:fold)
+          when :title then ch.capitalize
+          when :swap then ch.swapcase
           end
-    next if out == ch
-    next if kind != :fold && covered_arith(cp)
-    # fold table only needs entries that differ from plain downcase
-    next if kind == :fold && out == ch.downcase && covered_arith(cp)
-    next if kind == :fold && out == ch.downcase && !covered_arith(cp)
+    fallback = case kind
+               when :fold then ch.downcase
+               when :title then ch.upcase
+               when :swap then ch.downcase == ch ? ch.upcase : ch.downcase
+               else ch
+               end
+    next if out == fallback
+    next if covered_arith(cp)
     rows << format("%X:%s", cp, out.codepoints.map { |c| format("%X", c) }.join(" "))
   end
   rows.join(",")
 end
-puts "UP"; puts table(:up)
-puts "DOWN"; puts table(:down)
-puts "FOLD"; puts table(:fold)
+tables = %i[up down fold title swap].to_h { |kind| [kind, table(kind)] }
+tables.each { |kind, rows| puts kind.to_s.upcase; puts rows }
+# Most Unicode codepoints have no case mapping. One byte per 256-codepoint
+# page lets the walker avoid hexadecimal key construction and table lookups
+# there. A marked page may contain unchanged characters; an unmarked one
+# must not contain any entry, including an identity exception.
+pages = tables.values.flat_map { |rows| rows.split(',').map { |r| r.to_i(16) / 256 } }.uniq
+puts "PAGES"
+puts (0..pages.max).map { |page| pages.include?(page) ? '1' : '0' }.join