diff --git a/README.md b/README.md index 615b173..4d3c0fb 100644 --- a/README.md +++ b/README.md @@ -55,10 +55,12 @@ the **document structure**: otherwise every candidate EI is tried until the data before one actually decodes. - **Writing** — the exact inverse of the parser: objects rendered in PDF - syntax with stable dictionary ordering, and a `Writer` that copies whole - object graphs out of one or more documents, renumbering as it goes, then - lays down a cross-reference table and a trailer. - + syntax with stable dictionary ordering and strings in whichever of the two + forms is shorter, and a `Writer` that copies whole object graphs out of one + or more documents, renumbering as it goes, then lays down a cross-reference + table and a trailer. `NewPackedWriter` instead puts what it can into + compressed **object streams** and ends in a **cross-reference stream**, + which is what makes a file small. Measured against a corpus of **118 863 real PDFs** — Matplotlib, cairo, pdfTeX, Ghostscript, Adobe, R, Apache FOP, PDF 1.3 through 1.7 — `Open` succeeds on **118 833** of them and finds 138 337 pages, in 8 seconds, with no @@ -79,7 +81,9 @@ decode, and no operator outside the seventy the format defines — beyond four Rewriting is checked the same way: every one of the **118 833** files that open is copied object by object into a new file, which is then re-read and compared on page count, media boxes and the bytes of every content stream. -**All 118 833 match**, 35.5 GB in and 35.2 GB out, in twenty-eight seconds. +**All 118 833 match**, 35.5 GB in and 35.2 GB out, in twenty-eight seconds — +and **all 118 833 match again** when the packed writer is used instead, which +brings the output down to **94.6%** of the input. Next wave: the operations built on all of this. diff --git a/objstm.go b/objstm.go new file mode 100644 index 0000000..78b060f --- /dev/null +++ b/objstm.go @@ -0,0 +1,123 @@ +package reader + +import ( + "bytes" + "compress/zlib" + "fmt" +) + +// objectsPerStream bounds how many objects go into one object stream. A +// stream has to be inflated whole to read any object in it, so packing +// everything into one would make opening a file read all of it. +const objectsPerStream = 200 + +// packedAt says which object stream an object was packed into, and where. +type packedAt struct { + stream int + index int +} + +// pendingObject is an object waiting to be packed. +type pendingObject struct { + ref Ref + obj Object +} + +// packObjects writes every waiting object into object streams. It is called +// once, from Finish. +func (w *Writer) packObjects() { + for start := 0; start < len(w.pending); start += objectsPerStream { + end := start + objectsPerStream + if end > len(w.pending) { + end = len(w.pending) + } + w.packGroup(w.pending[start:end]) + } + w.pending = nil +} + +// packGroup writes one object stream holding the given objects. +func (w *Writer) packGroup(group []pendingObject) { + var head, payload bytes.Buffer + for _, p := range group { + fmt.Fprintf(&head, "%d %d ", p.ref.Num, payload.Len()) + payload.Write(AppendObject(nil, p.obj)) + payload.WriteByte('\n') + } + body := append(head.Bytes(), payload.Bytes()...) + stream := w.Reserve() + for i, p := range group { + w.packed[p.ref.Num] = packedAt{stream: stream.Num, index: i} + } + w.writeInline(stream, &Stream{ + Dict: Dict{ + "Type": Name("ObjStm"), + "N": Integer(len(group)), + "First": Integer(head.Len()), + }, + Raw: body, + }) +} + +// flateCompress compresses a stream's data, which is what makes packing worth doing. +func flateCompress(data []byte) []byte { + var buf bytes.Buffer + zw := zlib.NewWriter(&buf) + // A bytes.Buffer never refuses bytes, and Close only flushes. + zw.Write(data) + zw.Close() + return buf.Bytes() +} + +// finishWithXrefStream writes the cross-reference information as a stream +// rather than a table, which is the only form that can name an object inside +// an object stream. +func (w *Writer) finishWithXrefStream(trailer Dict) ([]byte, error) { + w.packObjects() + xref := w.Reserve() + // The cross-reference stream is reserved after everything else — the + // object streams included — so it always carries the highest number. + high := xref.Num + start := w.buf.Len() + w.offsets[xref.Num] = start + + rows := make([]byte, 0, (high+1)*7) + for num := 0; num <= high; num++ { + switch { + case num == 0: + rows = append(rows, 0, 0, 0, 0, 0, 0xFF, 0xFF) + case w.offsets[num] != 0 || num == xref.Num: + rows = append(rows, xrefRow(1, int64(w.offsets[num]), 0)...) + default: + if at, ok := w.packed[num]; ok { + rows = append(rows, xrefRow(2, int64(at.stream), int64(at.index))...) + continue + } + rows = append(rows, 0, 0, 0, 0, 0, 0xFF, 0xFF) + } + } + + dict := Dict{ + "Type": Name("XRef"), + "Size": Integer(high + 1), + "W": Array{Integer(1), Integer(4), Integer(2)}, + } + for k, v := range trailer { + dict[k] = v + } + w.writeInline(xref, &Stream{Dict: dict, Raw: rows}) + fmt.Fprintf(&w.buf, "startxref\n%d\n%%%%EOF\n", start) + if w.err != nil { + return nil, w.err + } + return w.buf.Bytes(), nil +} + +// xrefRow renders one three-field row of a cross-reference stream. +func xrefRow(kind byte, f2, f3 int64) []byte { + return []byte{ + kind, + byte(f2 >> 24), byte(f2 >> 16), byte(f2 >> 8), byte(f2), + byte(f3 >> 8), byte(f3), + } +} diff --git a/objstm_test.go b/objstm_test.go new file mode 100644 index 0000000..ae6efd7 --- /dev/null +++ b/objstm_test.go @@ -0,0 +1,252 @@ +package reader + +import ( + "bytes" + "testing" +) + +func TestPackedWriterBuildsAReadableFile(t *testing.T) { + w := NewPackedWriter("") + page := w.Reserve() + contents := w.Add(&Stream{Dict: Dict{}, Raw: []byte("BT ET")}) + pages := w.Add(Dict{"Type": Name("Pages"), "Kids": Array{page}, "Count": Integer(1), + "MediaBox": Array{Integer(0), Integer(0), Integer(300), Integer(400)}}) + w.Put(page, Dict{"Type": Name("Page"), "Parent": pages, "Contents": contents}) + root := w.Add(Dict{"Type": Name("Catalog"), "Pages": pages}) + out, err := w.Finish(Dict{"Root": root}) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(out, []byte("%PDF-1.7")) { + t.Errorf("header = %q", out[:9]) + } + // The pages are inside an object stream, and the file ends in a + // cross-reference stream rather than a table. + if !bytes.Contains(out, []byte("/ObjStm")) { + t.Error("nothing was packed") + } + if !bytes.Contains(out, []byte("/XRef")) { + t.Error("no cross-reference stream was written") + } + if bytes.Contains(out, []byte("\ntrailer\n")) { + t.Error("a trailer was written as well") + } + d, err := Open(out) + if err != nil { + t.Fatal(err) + } + if d.Repaired() { + t.Error("the file it wrote had to be repaired") + } + if got := d.PageCount(); got != 1 { + t.Fatalf("PageCount() = %d", got) + } + data, err := d.PageContent(1) + if err != nil || string(data) != "BT ET" { + t.Errorf("content = %q, %v", data, err) + } +} + +func TestPackedWriterRaisesTheVersion(t *testing.T) { + out, err := NewPackedWriter("1.3").Finish(Dict{}) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(out, []byte("%PDF-1.5")) { + t.Errorf("header = %q", out[:9]) + } + // A version that already allows it is left alone. + out, err = NewPackedWriter("2.0").Finish(Dict{}) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(out, []byte("%PDF-2.0")) { + t.Errorf("header = %q", out[:9]) + } +} + +func TestPackedWriterCompressesStreamsItIsGiven(t *testing.T) { + long := bytes.Repeat([]byte("compress me, please. "), 500) + w := NewPackedWriter("") + plain := w.Add(&Stream{Dict: Dict{}, Raw: long}) + // One that already says how it is encoded is left exactly as it is. + already := w.Add(&Stream{Dict: Dict{"Filter": Name("ASCIIHexDecode")}, Raw: []byte("4142>")}) + out, err := w.Finish(Dict{"Root": plain}) + if err != nil { + t.Fatal(err) + } + if len(out) > len(long)/2 { + t.Errorf("the stream was not compressed: %d bytes for %d", len(out), len(long)) + } + d := &Document{buf: out, xref: map[int]xrefEntry{}, cache: map[int]Object{}, + loading: map[int]bool{}, objStms: map[int]map[int]Object{}} + if err := d.loadXref(); err != nil { + t.Fatal(err) + } + o, err := d.Get(plain) + if err != nil { + t.Fatal(err) + } + s, ok := ToStream(o) + if !ok { + t.Fatalf("the stream came back as a %s", o.Kind()) + } + data, _, err := d.DecodeStream(s) + if err != nil || !bytes.Equal(data, long) { + t.Errorf("round trip: %d bytes, %v", len(data), err) + } + o, _ = d.Get(already) + s, _ = ToStream(o) + if f, _ := ToName(s.Dict.Get("Filter")); f != "ASCIIHexDecode" { + t.Errorf("an encoded stream was re-encoded: %v", s.Dict) + } +} + +func TestPackedWriterSplitsIntoSeveralStreams(t *testing.T) { + w := NewPackedWriter("") + const n = objectsPerStream*2 + 5 + refs := make([]Ref, n) + for i := range refs { + refs[i] = w.Add(Dict{"Index": Integer(i)}) + } + pagesRef := w.Reserve() + page := w.Add(Dict{"Type": Name("Page"), "Parent": pagesRef, + "MediaBox": Array{Integer(0), Integer(0), Integer(1), Integer(1)}}) + w.Put(pagesRef, Dict{"Type": Name("Pages"), "Kids": Array{page}, "Count": Integer(1)}) + root := w.Add(Dict{"Type": Name("Catalog"), "Pages": pagesRef}) + out, err := w.Finish(Dict{"Root": root}) + if err != nil { + t.Fatal(err) + } + if got := bytes.Count(out, []byte("/ObjStm")); got != 3 { + t.Errorf("%d object streams, want three", got) + } + d, err := Open(out) + if err != nil { + t.Fatal(err) + } + for i, ref := range refs { + o, err := d.Get(ref) + if err != nil { + t.Fatalf("object %d: %v", i, err) + } + dict, ok := ToDict(o) + if !ok { + t.Fatalf("object %d came back as a %s", i, o.Kind()) + } + if v, _ := ToInt(dict.Get("Index")); int(v) != i { + t.Errorf("object %d holds %v", i, dict.Get("Index")) + } + } +} + +func TestPackedWriterKeepsGapsFree(t *testing.T) { + w := NewPackedWriter("") + w.Put(Ref{Num: 10}, Integer(1)) + out, err := w.Finish(Dict{}) + if err != nil { + t.Fatal(err) + } + d := &Document{buf: out, xref: map[int]xrefEntry{}, cache: map[int]Object{}, + loading: map[int]bool{}, objStms: map[int]map[int]Object{}} + if err := d.loadXref(); err != nil { + t.Fatal(err) + } + if e := d.xref[5]; e.kind != 'f' { + t.Errorf("object 5 = %+v, want a free entry", e) + } + if e := d.xref[10]; e.kind != 'o' { + t.Errorf("object 10 = %+v, want one held in a stream", e) + } +} + +func TestPackedWriterLeavesOtherGenerationsWhereTheyAre(t *testing.T) { + // Only generation zero may be packed; anything else is written in place. + w := NewPackedWriter("") + w.Put(Ref{Num: 1, Gen: 3}, Integer(7)) + out, err := w.Finish(Dict{}) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(out, []byte("1 3 obj")) { + t.Error("the object was packed although its generation is not zero") + } +} + +func TestPackedWriterRefusesToWriteAnObjectTwice(t *testing.T) { + w := NewPackedWriter("") + ref := w.Add(Integer(1)) + w.Put(ref, Integer(2)) + if w.Err() == nil { + t.Fatal("want an error") + } + if _, err := w.Finish(Dict{}); err == nil { + t.Error("Finish should report it too") + } +} + +func TestPackedAndPlainAgree(t *testing.T) { + // The two forms must describe the same document. + build := func(packed bool) []byte { + var w *Writer + if packed { + w = NewPackedWriter("1.7") + } else { + w = NewWriter("1.7") + } + src, err := Open(onePage()) + if err != nil { + t.Fatal(err) + } + root := w.Copy(src, src.Trailer().Get("Root")) + out, err := w.Finish(Dict{"Root": root}) + if err != nil { + t.Fatal(err) + } + return out + } + for _, packed := range []bool{false, true} { + d, err := Open(build(packed)) + if err != nil { + t.Fatalf("packed=%v: %v", packed, err) + } + if got := d.PageCount(); got != 1 { + t.Errorf("packed=%v: PageCount() = %d", packed, got) + } + data, err := d.PageContent(1) + if err != nil || string(data) != "BT ET" { + t.Errorf("packed=%v: content = %q, %v", packed, data, err) + } + } +} + +func TestXrefRow(t *testing.T) { + got := xrefRow(1, 0x01020304, 0x0506) + want := []byte{1, 1, 2, 3, 4, 5, 6} + if !bytes.Equal(got, want) { + t.Errorf("xrefRow = % x, want % x", got, want) + } +} + +func TestCloneDict(t *testing.T) { + in := Dict{"A": Integer(1)} + out := cloneDict(in) + out["B"] = Integer(2) + if _, ok := in["B"]; ok { + t.Error("the original was changed") + } + if v, _ := ToInt(out.Get("A")); v != 1 { + t.Errorf("the copy lost an entry: %v", out) + } +} + +func TestFlateCompress(t *testing.T) { + data := bytes.Repeat([]byte("x"), 1000) + got, err := flateDecode(flateCompress(data)) + if err != nil || !bytes.Equal(got, data) { + t.Errorf("round trip: %d bytes, %v", len(got), err) + } + if got := flateCompress(nil); len(got) == 0 { + t.Error("compressing nothing produced nothing at all") + } +} diff --git a/writer.go b/writer.go index 85d2258..665e7a8 100644 --- a/writer.go +++ b/writer.go @@ -189,21 +189,42 @@ func appendStream(dst []byte, s *Stream) []byte { type Writer struct { buf bytes.Buffer offsets map[int]int + packed map[int]packedAt + pending []pendingObject + seen map[int]bool next int copied map[*Document]map[int]Ref err error + pack bool } // NewWriter starts a file with the given version in its header, "1.7" when the -// version is empty. -func NewWriter(version string) *Writer { +// version is empty. Objects are written one after another and the file ends in +// a cross-reference table, which every reader that has ever existed can read. +func NewWriter(version string) *Writer { return newWriter(version, false) } + +// NewPackedWriter starts a file that puts what it can into compressed object +// streams and ends in a cross-reference stream. That is smaller — usually by +// a lot, since a file is mostly small dictionaries — and has been readable +// since PDF 1.5, which is 2003; the version in the header is raised to 1.5 if +// it is lower. +func NewPackedWriter(version string) *Writer { return newWriter(version, true) } + +// newWriter starts a file of either kind. +func newWriter(version string, pack bool) *Writer { if version == "" { version = "1.7" } + if pack && version < "1.5" { + version = "1.5" + } w := &Writer{ offsets: map[int]int{}, + packed: map[int]packedAt{}, + seen: map[int]bool{}, next: 1, copied: map[*Document]map[int]Ref{}, + pack: pack, } fmt.Fprintf(&w.buf, "%%PDF-%s\n", version) // The four bytes above 127 tell every tool downstream that this file is @@ -221,19 +242,47 @@ func (w *Writer) Reserve() Ref { // Put writes an object under a reserved number. func (w *Writer) Put(ref Ref, o Object) { - if _, seen := w.offsets[ref.Num]; seen { + if w.seen[ref.Num] { w.note(fmt.Errorf("reader: object %d written twice", ref.Num)) return } + w.seen[ref.Num] = true if ref.Num >= w.next { w.next = ref.Num + 1 } + // A stream cannot go inside another stream, and only generation zero may + // be packed; everything else can wait to be gathered up at the end. + if _, isStream := o.(*Stream); w.pack && !isStream && ref.Gen == 0 { + w.pending = append(w.pending, pendingObject{ref: ref, obj: o}) + return + } + w.writeInline(ref, o) +} + +// writeInline writes an object where it stands in the file. +func (w *Writer) writeInline(ref Ref, o Object) { + if s, ok := o.(*Stream); ok && w.pack && s.Dict.Get("Filter").Kind() == KindNull { + // Nothing this package generated arrives compressed, and a packed + // file is being written to be small. + s = &Stream{Dict: cloneDict(s.Dict), Raw: flateCompress(s.Raw)} + s.Dict["Filter"] = Name("FlateDecode") + o = s + } w.offsets[ref.Num] = w.buf.Len() fmt.Fprintf(&w.buf, "%d %d obj\n", ref.Num, ref.Gen) w.buf.Write(AppendObject(nil, o)) w.buf.WriteString("\nendobj\n") } +// cloneDict copies a dictionary one level deep. +func cloneDict(d Dict) Dict { + out := Dict{} + for k, v := range d { + out[k] = v + } + return out +} + // Add reserves a number, writes the object under it and returns the reference. func (w *Writer) Add(o Object) Ref { ref := w.Reserve() @@ -255,6 +304,9 @@ func (w *Writer) note(err error) { // file. /Size is filled in; the caller supplies /Root and whatever else the // trailer needs. func (w *Writer) Finish(trailer Dict) ([]byte, error) { + if w.pack { + return w.finishWithXrefStream(trailer) + } high := 0 for num := range w.offsets { if num > high {