From 102b9c3eabb5a98660ba4a7ab2a0aa90160f1fa0 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 25 Aug 2026 12:51:43 +0200 Subject: [PATCH] Write a string in whichever form is shorter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A PDF string may be written between parentheses or in hex, and the writer only knew the first. A literal string spells every unprintable byte as a four-character octal escape, and text that is not Latin-1 is written in UTF-16 — so a string of ordinary prose in any other script came out four times its own length. The hex form costs two characters a byte whatever the byte is, so the writer now measures both and takes the shorter. The corpus makes the case: one file carries a diagram's whole source in its information dictionary, and rewriting it produced 3.7 MB from 1.2 MB, almost all of it octal escapes. Across the whole corpus the rewrite drops from 98.9% of the input to 98.8%, which is what a change that only bites on non-Latin text looks like in an English-language corpus; on the file that motivated it, the string itself halves. All 118 833 files still rewrite to an identical fingerprint. Exact 100% statement coverage, go vet clean. Co-authored-by: Claude Opus 5 (1M context) --- writer.go | 42 ++++++++++++++++++++++++++++++++++++++++-- writer_test.go | 6 +++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/writer.go b/writer.go index d5bc4d1..85d2258 100644 --- a/writer.go +++ b/writer.go @@ -72,9 +72,47 @@ func appendReal(dst []byte, f float64) []byte { return strconv.AppendFloat(dst, f, 'f', -1, 64) } -// appendString writes a literal string, escaping only what has to be escaped -// and rendering anything unprintable as an octal escape. +// appendString writes a string in whichever of the two forms a PDF allows is +// shorter. A literal string spells an unprintable byte as a four-character +// octal escape, so text in UTF-16 — which is how a PDF says anything that is +// not Latin-1 — comes out four times its length; the hex form costs two +// characters a byte whatever the byte is. func appendString(dst []byte, s []byte) []byte { + if literalStringCost(s) > 2*len(s)+2 { + return appendHexString(dst, s) + } + return appendLiteralString(dst, s) +} + +// literalStringCost is how many characters the literal form would take. +func literalStringCost(s []byte) int { + n := 2 + for _, c := range s { + switch { + case c == '(' || c == ')' || c == '\\' || c == '\n' || c == '\r' || c == '\t': + n += 2 + case c < 32 || c > 126: + n += 4 + default: + n++ + } + } + return n +} + +// appendHexString writes the form. +func appendHexString(dst []byte, s []byte) []byte { + const hex = "0123456789ABCDEF" + dst = append(dst, '<') + for _, c := range s { + dst = append(dst, hex[c>>4], hex[c&15]) + } + return append(dst, '>') +} + +// appendLiteralString writes the (parenthesised) form, escaping only what has +// to be escaped and rendering anything unprintable as an octal escape. +func appendLiteralString(dst []byte, s []byte) []byte { dst = append(dst, '(') for _, c := range s { switch { diff --git a/writer_test.go b/writer_test.go index ea7c0b0..e0c1aaa 100644 --- a/writer_test.go +++ b/writer_test.go @@ -32,7 +32,11 @@ func TestAppendObject(t *testing.T) { {String("plain"), "(plain)"}, {String("a(b)c\\"), `(a\(b\)c\\)`}, {String("\n\r\t"), `(\n\r\t)`}, - {String{0x00, 0xFF}, `(\000\377)`}, + {String{0x00, 0xFF}, "<00FF>"}, + // One awkward byte in a long readable string still comes out readable. + {String("a readable line with one \x00 in it"), `(a readable line with one \000 in it)`}, + // Text in UTF-16 goes out as hex, which is half the size. + {String{0xFE, 0xFF, 0x00, 0x41}, ""}, {Name("Simple"), "/Simple"}, {Name("With Space"), "/With#20Space"}, {Name("h#sh"), "/h#23sh"},