diff --git a/RELEASENOTES.md b/RELEASENOTES.md index a9d1263..648d43c 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -6,6 +6,7 @@ - Fixed `ua-1` (PDF/UA-1, the accessibility standard) being rejected by `pdfStandards`. The accepted names are now taken from `typst_pdf::PdfStandard` itself, so every standard Typst supports is accepted, including ones added by later Typst releases. The documented `v-` prefix on plain PDF versions still works, and applies only to them: `v-a-2b` is not a spelling of `a-2b`. ### Changed +- `TypstCompiler.FromSource(...)` and the static one-shot helpers no longer copy the source onto the managed heap before handing it to the compiler. It is encoded straight into the native buffer, so a source over roughly 42,500 characters no longer puts an array on the large object heap for the duration of one call: a 200 KB source measured 200,024 bytes of managed heap before and none after. Native memory allocated while building a compiler is also released now when construction fails part way through, rather than leaking. - **Breaking:** an invalid combination of PDF standards now fails the compilation instead of silently producing an ordinary PDF. The validation error from `PdfStandards::new` was discarded and export fell back to the default, so a pipeline could believe it was writing PDF/A while it was not. Combinations such as two PDF/A levels, or a PDF/A level that contradicts the requested PDF version, now throw with the message and hints from Typst. Callers passing a contradictory combination today receive a document and will receive an exception after this change. - Note for PDF/A and PDF/UA: the exporter deliberately writes no timestamp, so the document has to carry its own date (`#set document(date: ...)`) and, for PDF/UA, a title and language. - `compiler.CompilePdf(Stream)`, `compiler.CompilePdfAsync(Stream)`, `compiler.CompilePdf(string outputFile)` and `compiler.CompilePdfAsync(string outputFile)` now stream the document straight from native memory to the destination and return the compiler warnings, rather than returning a `PdfResult` that had to be materialised on the managed heap first. Use `compiler.CompilePdf()` when you want the bytes. diff --git a/src/typstsharp.tests/Tests.cs b/src/typstsharp.tests/Tests.cs index 4a5b967..05e9249 100644 --- a/src/typstsharp.tests/Tests.cs +++ b/src/typstsharp.tests/Tests.cs @@ -343,6 +343,42 @@ public async Task ErrorWithNullByteIsHandledCorrectly() await Assert.That(ex!.Message).Contains("foo\0bar"); } + /// + /// A null pointer tells the native side there is no in-memory source at all, so an empty + /// document has to arrive as a real pointer with length 0 rather than as nothing. + /// + [Test] + public async Task EmptySourceIsDistinguishedFromNoSourceAtAll() + { + using var compiler = TypstCompiler.FromSource(""); + using var document = compiler.CompileToDocument(); + + await Assert.That(document.GetOutputLength()).IsGreaterThan(0); + } + + /// + /// The source crosses the boundary as UTF-8 bytes with an explicit length. A source whose UTF-8 + /// byte count differs from its char count, and one past the 85,000-byte threshold where the + /// array this replaces would have gone to the large object heap, are where a mistake in that + /// encoding would surface. + /// + [Test] + public async Task LargeSourceWithMultiByteCharactersIsCompiledInFull() + { + var builder = new StringBuilder("= Grüezi mitenand\n\n"); + for (int i = 0; i < 4000; i++) + { + builder.Append("Paragraph ").Append(i).Append(" über Zürich.\n\n"); + } + builder.Append("= Schluss\n"); + + using var compiler = TypstCompiler.FromSource(builder.ToString()); + var plainText = GetPlainText(compiler.CompilePdf()); + + await Assert.That(plainText).Contains("Grüezi mitenand"); + await Assert.That(plainText).Contains("Schluss"); + } + [Test] public async Task SourceAfterNullByteIsNotTruncated() { diff --git a/src/typstsharp/TypstCompiler.cs b/src/typstsharp/TypstCompiler.cs index bbf1b22..c750427 100644 --- a/src/typstsharp/TypstCompiler.cs +++ b/src/typstsharp/TypstCompiler.cs @@ -235,54 +235,82 @@ private unsafe TypstCompiler(string? inputPath, string? inputSource, Fonts? font root = Path.GetDirectoryName(inputPath); } - var inputPathPtr = inputPath != null ? Marshal.StringToCoTaskMemUTF8(inputPath) : IntPtr.Zero; - - // The source goes over as raw UTF-8 bytes with an explicit length. A Typst - // document may contain NUL bytes, and a NUL-terminated string would be - // silently truncated at the first one. - byte[]? inputSourceBytes = null; + // These pointers are native memory that the finally block below releases, so they are + // declared out here and allocated inside the try. Allocating them before it would leak + // whatever had been allocated already if a later step threw, and several steps can: + // fontPaths may be a lazy sequence supplied by the caller, and sysInputs is serialized. + IntPtr inputPathPtr = IntPtr.Zero; + IntPtr inputSourcePtr = IntPtr.Zero; + IntPtr rootPtr = IntPtr.Zero; + IntPtr[] fontPathPtrs = []; + IntPtr packagePathPtr = IntPtr.Zero; + IntPtr sysInputsPtr = IntPtr.Zero; nuint inputSourceLen = 0; - if (inputSource != null) - { - var encoded = Encoding.UTF8.GetBytes(inputSource); - inputSourceLen = (nuint)encoded.Length; - // `fixed` over an empty array yields a null pointer, which the native - // side reads as "no source at all". A one-byte placeholder keeps an - // empty document distinguishable; the length passed stays 0. - inputSourceBytes = encoded.Length == 0 ? new byte[1] : encoded; - } - IntPtr rootPtr = IntPtr.Zero; - if (!string.IsNullOrWhiteSpace(root)) + try { - rootPtr = Marshal.StringToCoTaskMemUTF8(root); - } + if (inputPath != null) + { + inputPathPtr = Marshal.StringToCoTaskMemUTF8(inputPath); + } - var fontPathsList = fontPaths.ToList(); - var fontPathPtrs = new IntPtr[fontPathsList.Count]; - for (int i = 0; i < fontPathsList.Count; i++) - { - fontPathPtrs[i] = Marshal.StringToCoTaskMemUTF8(fontPathsList[i]); - } + // The source goes over as raw UTF-8 bytes with an explicit length. A Typst + // document may contain NUL bytes, and a NUL-terminated string would be + // silently truncated at the first one. + if (inputSource != null) + { + // Encoding straight into native memory keeps a document-sized array off the managed + // heap; a source of any size would otherwise be copied there, and a large one would + // land on the large object heap, only to be garbage as soon as the call returns. + int byteCount = Encoding.UTF8.GetByteCount(inputSource); + + // A null pointer reads as "no source at all" on the native side, so an empty + // document still needs one real byte behind the pointer; the length stays 0. + inputSourcePtr = Marshal.AllocCoTaskMem(byteCount == 0 ? 1 : byteCount); + int written = 0; + if (byteCount > 0) + { + fixed (char* chars = inputSource) + { + written = Encoding.UTF8.GetBytes(chars, inputSource.Length, (byte*)inputSourcePtr, byteCount); + } + } - var packagePathPtr = packagePath != null ? Marshal.StringToCoTaskMemUTF8(packagePath) : IntPtr.Zero; + // The length comes from the encode rather than the count, so the native side can + // never be handed a length that reaches past what was written. + inputSourceLen = (nuint)written; + } - var sysInputsJson = sysInputs == null ? "{}" : JsonSerializer.Serialize>(sysInputs, sourceGenOptions); - var sysInputsPtr = Marshal.StringToCoTaskMemUTF8(sysInputsJson); + if (!string.IsNullOrWhiteSpace(root)) + { + rootPtr = Marshal.StringToCoTaskMemUTF8(root); + } + + var fontPathsList = fontPaths.ToList(); + fontPathPtrs = new IntPtr[fontPathsList.Count]; + for (int i = 0; i < fontPathPtrs.Length; i++) + { + fontPathPtrs[i] = Marshal.StringToCoTaskMemUTF8(fontPathsList[i]); + } + + if (packagePath != null) + { + packagePathPtr = Marshal.StringToCoTaskMemUTF8(packagePath); + } + + var sysInputsJson = sysInputs == null ? "{}" : JsonSerializer.Serialize>(sysInputs, sourceGenOptions); + sysInputsPtr = Marshal.StringToCoTaskMemUTF8(sysInputsJson); - try - { fixed (IntPtr* fontPathsRawPtr = fontPathPtrs) - fixed (byte* inputSourcePtr = inputSourceBytes) { - IntPtr* fontPathsPtr = fontPathsList.Count == 0 ? null : fontPathsRawPtr; + IntPtr* fontPathsPtr = fontPathPtrs.Length == 0 ? null : fontPathsRawPtr; _compiler = CsBindgen.NativeMethods.create_compiler( (byte*)rootPtr, (byte*)inputPathPtr, - inputSourcePtr, + (byte*)inputSourcePtr, inputSourceLen, (byte**)fontPathsPtr, - (nuint)fontPathsList.Count, + (nuint)fontPathPtrs.Length, (byte*)packagePathPtr, (byte*)sysInputsPtr, ignoreSystemFonts, @@ -296,11 +324,14 @@ private unsafe TypstCompiler(string? inputPath, string? inputSource, Fonts? font } finally { + // A throw part way through leaves some of these null, and the font path array only + // partly filled. FreeCoTaskMem ignores a null pointer, so the loop needs no guard. if (rootPtr != IntPtr.Zero) Marshal.FreeCoTaskMem(rootPtr); if (inputPathPtr != IntPtr.Zero) Marshal.FreeCoTaskMem(inputPathPtr); + if (inputSourcePtr != IntPtr.Zero) Marshal.FreeCoTaskMem(inputSourcePtr); foreach (var ptr in fontPathPtrs) Marshal.FreeCoTaskMem(ptr); if (packagePathPtr != IntPtr.Zero) Marshal.FreeCoTaskMem(packagePathPtr); - Marshal.FreeCoTaskMem(sysInputsPtr); + if (sysInputsPtr != IntPtr.Zero) Marshal.FreeCoTaskMem(sysInputsPtr); } }