From e8e5b02b08a6e454e73c02a345529bb76caaaf65 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 25 Aug 2026 13:12:50 +0200 Subject: [PATCH] Add object streams and cross-reference streams to the writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A PDF is mostly small dictionaries, and a dictionary written on its own cannot be compressed at all — there is nowhere to say how. Object streams are the format's answer: pack many objects into one stream and compress that. This adds them, and the cross-reference stream that is the only form able to say where a packed object lives. NewPackedWriter is the same Writer with that turned on. Objects of generation zero that are not themselves streams wait to be gathered up; streams are written where they stand, and one arriving with no filter of its own is deflated, since nothing this package generates comes compressed and a packed file is being written to be small. Two hundred objects go in each stream — one has to be inflated whole to read any object in it, so packing everything into one would make opening a file read all of it. The header is raised to 1.5, the version that introduced this, which is 2003. Measured on the corpus: all 118 833 files still rewrite to an identical fingerprint, and the output falls from 98.8% of the input to 94.6% — about 1.9 GB across the corpus. It also settles the 355 files that were coming out larger than they went in. The reader has been able to read both forms since its second wave, so this is the writer catching up with it — and the two now check each other on every file in the corpus. Exact 100% statement coverage, go vet clean, nine cross-compilation targets. Co-authored-by: Claude Opus 5 (1M context) --- README.md | 14 ++- objstm.go | 123 ++++++++++++++++++++++++ objstm_test.go | 252 +++++++++++++++++++++++++++++++++++++++++++++++++ writer.go | 58 +++++++++++- 4 files changed, 439 insertions(+), 8 deletions(-) create mode 100644 objstm.go create mode 100644 objstm_test.go diff --git a/README.md b/README.md index 615b173..4d3c0fb 100644 --- a/README.md +++ b/README.md @@ -55,10 +55,12 @@ the **document structure**: otherwise every candidate EI is tried until the data before one actually decodes. - **Writing** — the exact inverse of the parser: objects rendered in PDF - syntax with stable dictionary ordering, and a `Writer` that copies whole - object graphs out of one or more documents, renumbering as it goes, then - lays down a cross-reference table and a trailer. - + syntax with stable dictionary ordering and strings in whichever of the two + forms is shorter, and a `Writer` that copies whole object graphs out of one + or more documents, renumbering as it goes, then lays down a cross-reference + table and a trailer. `NewPackedWriter` instead puts what it can into + compressed **object streams** and ends in a **cross-reference stream**, + which is what makes a file small. Measured against a corpus of **118 863 real PDFs** — Matplotlib, cairo, pdfTeX, Ghostscript, Adobe, R, Apache FOP, PDF 1.3 through 1.7 — `Open` succeeds on **118 833** of them and finds 138 337 pages, in 8 seconds, with no @@ -79,7 +81,9 @@ decode, and no operator outside the seventy the format defines — beyond four Rewriting is checked the same way: every one of the **118 833** files that open is copied object by object into a new file, which is then re-read and compared on page count, media boxes and the bytes of every content stream. -**All 118 833 match**, 35.5 GB in and 35.2 GB out, in twenty-eight seconds. +**All 118 833 match**, 35.5 GB in and 35.2 GB out, in twenty-eight seconds — +and **all 118 833 match again** when the packed writer is used instead, which +brings the output down to **94.6%** of the input. Next wave: the operations built on all of this. diff --git a/objstm.go b/objstm.go new file mode 100644 index 0000000..78b060f --- /dev/null +++ b/objstm.go @@ -0,0 +1,123 @@ +package reader + +import ( + "bytes" + "compress/zlib" + "fmt" +) + +// objectsPerStream bounds how many objects go into one object stream. A +// stream has to be inflated whole to read any object in it, so packing +// everything into one would make opening a file read all of it. +const objectsPerStream = 200 + +// packedAt says which object stream an object was packed into, and where. +type packedAt struct { + stream int + index int +} + +// pendingObject is an object waiting to be packed. +type pendingObject struct { + ref Ref + obj Object +} + +// packObjects writes every waiting object into object streams. It is called +// once, from Finish. +func (w *Writer) packObjects() { + for start := 0; start < len(w.pending); start += objectsPerStream { + end := start + objectsPerStream + if end > len(w.pending) { + end = len(w.pending) + } + w.packGroup(w.pending[start:end]) + } + w.pending = nil +} + +// packGroup writes one object stream holding the given objects. +func (w *Writer) packGroup(group []pendingObject) { + var head, payload bytes.Buffer + for _, p := range group { + fmt.Fprintf(&head, "%d %d ", p.ref.Num, payload.Len()) + payload.Write(AppendObject(nil, p.obj)) + payload.WriteByte('\n') + } + body := append(head.Bytes(), payload.Bytes()...) + stream := w.Reserve() + for i, p := range group { + w.packed[p.ref.Num] = packedAt{stream: stream.Num, index: i} + } + w.writeInline(stream, &Stream{ + Dict: Dict{ + "Type": Name("ObjStm"), + "N": Integer(len(group)), + "First": Integer(head.Len()), + }, + Raw: body, + }) +} + +// flateCompress compresses a stream's data, which is what makes packing worth doing. +func flateCompress(data []byte) []byte { + var buf bytes.Buffer + zw := zlib.NewWriter(&buf) + // A bytes.Buffer never refuses bytes, and Close only flushes. + zw.Write(data) + zw.Close() + return buf.Bytes() +} + +// finishWithXrefStream writes the cross-reference information as a stream +// rather than a table, which is the only form that can name an object inside +// an object stream. +func (w *Writer) finishWithXrefStream(trailer Dict) ([]byte, error) { + w.packObjects() + xref := w.Reserve() + // The cross-reference stream is reserved after everything else — the + // object streams included — so it always carries the highest number. + high := xref.Num + start := w.buf.Len() + w.offsets[xref.Num] = start + + rows := make([]byte, 0, (high+1)*7) + for num := 0; num <= high; num++ { + switch { + case num == 0: + rows = append(rows, 0, 0, 0, 0, 0, 0xFF, 0xFF) + case w.offsets[num] != 0 || num == xref.Num: + rows = append(rows, xrefRow(1, int64(w.offsets[num]), 0)...) + default: + if at, ok := w.packed[num]; ok { + rows = append(rows, xrefRow(2, int64(at.stream), int64(at.index))...) + continue + } + rows = append(rows, 0, 0, 0, 0, 0, 0xFF, 0xFF) + } + } + + dict := Dict{ + "Type": Name("XRef"), + "Size": Integer(high + 1), + "W": Array{Integer(1), Integer(4), Integer(2)}, + } + for k, v := range trailer { + dict[k] = v + } + w.writeInline(xref, &Stream{Dict: dict, Raw: rows}) + fmt.Fprintf(&w.buf, "startxref\n%d\n%%%%EOF\n", start) + if w.err != nil { + return nil, w.err + } + return w.buf.Bytes(), nil +} + +// xrefRow renders one three-field row of a cross-reference stream. +func xrefRow(kind byte, f2, f3 int64) []byte { + return []byte{ + kind, + byte(f2 >> 24), byte(f2 >> 16), byte(f2 >> 8), byte(f2), + byte(f3 >> 8), byte(f3), + } +} diff --git a/objstm_test.go b/objstm_test.go new file mode 100644 index 0000000..ae6efd7 --- /dev/null +++ b/objstm_test.go @@ -0,0 +1,252 @@ +package reader + +import ( + "bytes" + "testing" +) + +func TestPackedWriterBuildsAReadableFile(t *testing.T) { + w := NewPackedWriter("") + page := w.Reserve() + contents := w.Add(&Stream{Dict: Dict{}, Raw: []byte("BT ET")}) + pages := w.Add(Dict{"Type": Name("Pages"), "Kids": Array{page}, "Count": Integer(1), + "MediaBox": Array{Integer(0), Integer(0), Integer(300), Integer(400)}}) + w.Put(page, Dict{"Type": Name("Page"), "Parent": pages, "Contents": contents}) + root := w.Add(Dict{"Type": Name("Catalog"), "Pages": pages}) + out, err := w.Finish(Dict{"Root": root}) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(out, []byte("%PDF-1.7")) { + t.Errorf("header = %q", out[:9]) + } + // The pages are inside an object stream, and the file ends in a + // cross-reference stream rather than a table. + if !bytes.Contains(out, []byte("/ObjStm")) { + t.Error("nothing was packed") + } + if !bytes.Contains(out, []byte("/XRef")) { + t.Error("no cross-reference stream was written") + } + if bytes.Contains(out, []byte("\ntrailer\n")) { + t.Error("a trailer was written as well") + } + d, err := Open(out) + if err != nil { + t.Fatal(err) + } + if d.Repaired() { + t.Error("the file it wrote had to be repaired") + } + if got := d.PageCount(); got != 1 { + t.Fatalf("PageCount() = %d", got) + } + data, err := d.PageContent(1) + if err != nil || string(data) != "BT ET" { + t.Errorf("content = %q, %v", data, err) + } +} + +func TestPackedWriterRaisesTheVersion(t *testing.T) { + out, err := NewPackedWriter("1.3").Finish(Dict{}) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(out, []byte("%PDF-1.5")) { + t.Errorf("header = %q", out[:9]) + } + // A version that already allows it is left alone. + out, err = NewPackedWriter("2.0").Finish(Dict{}) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(out, []byte("%PDF-2.0")) { + t.Errorf("header = %q", out[:9]) + } +} + +func TestPackedWriterCompressesStreamsItIsGiven(t *testing.T) { + long := bytes.Repeat([]byte("compress me, please. "), 500) + w := NewPackedWriter("") + plain := w.Add(&Stream{Dict: Dict{}, Raw: long}) + // One that already says how it is encoded is left exactly as it is. + already := w.Add(&Stream{Dict: Dict{"Filter": Name("ASCIIHexDecode")}, Raw: []byte("4142>")}) + out, err := w.Finish(Dict{"Root": plain}) + if err != nil { + t.Fatal(err) + } + if len(out) > len(long)/2 { + t.Errorf("the stream was not compressed: %d bytes for %d", len(out), len(long)) + } + d := &Document{buf: out, xref: map[int]xrefEntry{}, cache: map[int]Object{}, + loading: map[int]bool{}, objStms: map[int]map[int]Object{}} + if err := d.loadXref(); err != nil { + t.Fatal(err) + } + o, err := d.Get(plain) + if err != nil { + t.Fatal(err) + } + s, ok := ToStream(o) + if !ok { + t.Fatalf("the stream came back as a %s", o.Kind()) + } + data, _, err := d.DecodeStream(s) + if err != nil || !bytes.Equal(data, long) { + t.Errorf("round trip: %d bytes, %v", len(data), err) + } + o, _ = d.Get(already) + s, _ = ToStream(o) + if f, _ := ToName(s.Dict.Get("Filter")); f != "ASCIIHexDecode" { + t.Errorf("an encoded stream was re-encoded: %v", s.Dict) + } +} + +func TestPackedWriterSplitsIntoSeveralStreams(t *testing.T) { + w := NewPackedWriter("") + const n = objectsPerStream*2 + 5 + refs := make([]Ref, n) + for i := range refs { + refs[i] = w.Add(Dict{"Index": Integer(i)}) + } + pagesRef := w.Reserve() + page := w.Add(Dict{"Type": Name("Page"), "Parent": pagesRef, + "MediaBox": Array{Integer(0), Integer(0), Integer(1), Integer(1)}}) + w.Put(pagesRef, Dict{"Type": Name("Pages"), "Kids": Array{page}, "Count": Integer(1)}) + root := w.Add(Dict{"Type": Name("Catalog"), "Pages": pagesRef}) + out, err := w.Finish(Dict{"Root": root}) + if err != nil { + t.Fatal(err) + } + if got := bytes.Count(out, []byte("/ObjStm")); got != 3 { + t.Errorf("%d object streams, want three", got) + } + d, err := Open(out) + if err != nil { + t.Fatal(err) + } + for i, ref := range refs { + o, err := d.Get(ref) + if err != nil { + t.Fatalf("object %d: %v", i, err) + } + dict, ok := ToDict(o) + if !ok { + t.Fatalf("object %d came back as a %s", i, o.Kind()) + } + if v, _ := ToInt(dict.Get("Index")); int(v) != i { + t.Errorf("object %d holds %v", i, dict.Get("Index")) + } + } +} + +func TestPackedWriterKeepsGapsFree(t *testing.T) { + w := NewPackedWriter("") + w.Put(Ref{Num: 10}, Integer(1)) + out, err := w.Finish(Dict{}) + if err != nil { + t.Fatal(err) + } + d := &Document{buf: out, xref: map[int]xrefEntry{}, cache: map[int]Object{}, + loading: map[int]bool{}, objStms: map[int]map[int]Object{}} + if err := d.loadXref(); err != nil { + t.Fatal(err) + } + if e := d.xref[5]; e.kind != 'f' { + t.Errorf("object 5 = %+v, want a free entry", e) + } + if e := d.xref[10]; e.kind != 'o' { + t.Errorf("object 10 = %+v, want one held in a stream", e) + } +} + +func TestPackedWriterLeavesOtherGenerationsWhereTheyAre(t *testing.T) { + // Only generation zero may be packed; anything else is written in place. + w := NewPackedWriter("") + w.Put(Ref{Num: 1, Gen: 3}, Integer(7)) + out, err := w.Finish(Dict{}) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(out, []byte("1 3 obj")) { + t.Error("the object was packed although its generation is not zero") + } +} + +func TestPackedWriterRefusesToWriteAnObjectTwice(t *testing.T) { + w := NewPackedWriter("") + ref := w.Add(Integer(1)) + w.Put(ref, Integer(2)) + if w.Err() == nil { + t.Fatal("want an error") + } + if _, err := w.Finish(Dict{}); err == nil { + t.Error("Finish should report it too") + } +} + +func TestPackedAndPlainAgree(t *testing.T) { + // The two forms must describe the same document. + build := func(packed bool) []byte { + var w *Writer + if packed { + w = NewPackedWriter("1.7") + } else { + w = NewWriter("1.7") + } + src, err := Open(onePage()) + if err != nil { + t.Fatal(err) + } + root := w.Copy(src, src.Trailer().Get("Root")) + out, err := w.Finish(Dict{"Root": root}) + if err != nil { + t.Fatal(err) + } + return out + } + for _, packed := range []bool{false, true} { + d, err := Open(build(packed)) + if err != nil { + t.Fatalf("packed=%v: %v", packed, err) + } + if got := d.PageCount(); got != 1 { + t.Errorf("packed=%v: PageCount() = %d", packed, got) + } + data, err := d.PageContent(1) + if err != nil || string(data) != "BT ET" { + t.Errorf("packed=%v: content = %q, %v", packed, data, err) + } + } +} + +func TestXrefRow(t *testing.T) { + got := xrefRow(1, 0x01020304, 0x0506) + want := []byte{1, 1, 2, 3, 4, 5, 6} + if !bytes.Equal(got, want) { + t.Errorf("xrefRow = % x, want % x", got, want) + } +} + +func TestCloneDict(t *testing.T) { + in := Dict{"A": Integer(1)} + out := cloneDict(in) + out["B"] = Integer(2) + if _, ok := in["B"]; ok { + t.Error("the original was changed") + } + if v, _ := ToInt(out.Get("A")); v != 1 { + t.Errorf("the copy lost an entry: %v", out) + } +} + +func TestFlateCompress(t *testing.T) { + data := bytes.Repeat([]byte("x"), 1000) + got, err := flateDecode(flateCompress(data)) + if err != nil || !bytes.Equal(got, data) { + t.Errorf("round trip: %d bytes, %v", len(got), err) + } + if got := flateCompress(nil); len(got) == 0 { + t.Error("compressing nothing produced nothing at all") + } +} diff --git a/writer.go b/writer.go index 85d2258..665e7a8 100644 --- a/writer.go +++ b/writer.go @@ -189,21 +189,42 @@ func appendStream(dst []byte, s *Stream) []byte { type Writer struct { buf bytes.Buffer offsets map[int]int + packed map[int]packedAt + pending []pendingObject + seen map[int]bool next int copied map[*Document]map[int]Ref err error + pack bool } // NewWriter starts a file with the given version in its header, "1.7" when the -// version is empty. -func NewWriter(version string) *Writer { +// version is empty. Objects are written one after another and the file ends in +// a cross-reference table, which every reader that has ever existed can read. +func NewWriter(version string) *Writer { return newWriter(version, false) } + +// NewPackedWriter starts a file that puts what it can into compressed object +// streams and ends in a cross-reference stream. That is smaller — usually by +// a lot, since a file is mostly small dictionaries — and has been readable +// since PDF 1.5, which is 2003; the version in the header is raised to 1.5 if +// it is lower. +func NewPackedWriter(version string) *Writer { return newWriter(version, true) } + +// newWriter starts a file of either kind. +func newWriter(version string, pack bool) *Writer { if version == "" { version = "1.7" } + if pack && version < "1.5" { + version = "1.5" + } w := &Writer{ offsets: map[int]int{}, + packed: map[int]packedAt{}, + seen: map[int]bool{}, next: 1, copied: map[*Document]map[int]Ref{}, + pack: pack, } fmt.Fprintf(&w.buf, "%%PDF-%s\n", version) // The four bytes above 127 tell every tool downstream that this file is @@ -221,19 +242,47 @@ func (w *Writer) Reserve() Ref { // Put writes an object under a reserved number. func (w *Writer) Put(ref Ref, o Object) { - if _, seen := w.offsets[ref.Num]; seen { + if w.seen[ref.Num] { w.note(fmt.Errorf("reader: object %d written twice", ref.Num)) return } + w.seen[ref.Num] = true if ref.Num >= w.next { w.next = ref.Num + 1 } + // A stream cannot go inside another stream, and only generation zero may + // be packed; everything else can wait to be gathered up at the end. + if _, isStream := o.(*Stream); w.pack && !isStream && ref.Gen == 0 { + w.pending = append(w.pending, pendingObject{ref: ref, obj: o}) + return + } + w.writeInline(ref, o) +} + +// writeInline writes an object where it stands in the file. +func (w *Writer) writeInline(ref Ref, o Object) { + if s, ok := o.(*Stream); ok && w.pack && s.Dict.Get("Filter").Kind() == KindNull { + // Nothing this package generated arrives compressed, and a packed + // file is being written to be small. + s = &Stream{Dict: cloneDict(s.Dict), Raw: flateCompress(s.Raw)} + s.Dict["Filter"] = Name("FlateDecode") + o = s + } w.offsets[ref.Num] = w.buf.Len() fmt.Fprintf(&w.buf, "%d %d obj\n", ref.Num, ref.Gen) w.buf.Write(AppendObject(nil, o)) w.buf.WriteString("\nendobj\n") } +// cloneDict copies a dictionary one level deep. +func cloneDict(d Dict) Dict { + out := Dict{} + for k, v := range d { + out[k] = v + } + return out +} + // Add reserves a number, writes the object under it and returns the reference. func (w *Writer) Add(o Object) Ref { ref := w.Reserve() @@ -255,6 +304,9 @@ func (w *Writer) note(err error) { // file. /Size is filled in; the caller supplies /Root and whatever else the // trailer needs. func (w *Writer) Finish(trailer Dict) ([]byte, error) { + if w.pack { + return w.finishWithXrefStream(trailer) + } high := 0 for num := range w.offsets { if num > high {