Cap malloc arenas so extraction jobs fit their memory limit

efile_app/efile/management/commands/process_document_extractions.py

@@ -1,6 +1,7 @@
 import io
 import logging
 import multiprocessing
+import os
 import time
 
 from django.conf import settings
@@ -54,6 +55,13 @@ def handle(self, *args, **options):
         )
 
         next_rollup = 0.0
+        # The child's memory cap is on address space, and glibc reserves a
+        # malloc arena of up to 64 MB per thread, up to eight per CPU core. A
+        # job that uses about 250 MB reached 800 MB of address space on an
+        # eight-core machine. Two arenas keep each child's address space close
+        # to what it uses, whatever the host's core count. Spawned children
+        # read this at startup; an operator's own value is kept.
+        os.environ.setdefault("MALLOC_ARENA_MAX", "2")
 
         def rollup_if_due():
             nonlocal next_rollup

efile_app/efile/tests/test_document_extractions.py

@@ -1,3 +1,4 @@
+import os
 import re
 import shutil
 from pathlib import Path
@@ -453,3 +454,26 @@ def test_management_command_processes_and_retries_failures(extraction_draft):
         assert job.status == DocumentExtraction.Status.FAILED
         assert job.attempts == 2
         assert "Document analysis failed" in job.error
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize(("configured", "expected"), [(None, "2"), ("4", "4")])
+def test_worker_children_start_with_few_malloc_arenas(extraction_draft, monkeypatch, configured, expected):
+    from django.core.management import call_command
+
+    if configured is None:
+        monkeypatch.delenv("MALLOC_ARENA_MAX", raising=False)
+    else:
+        monkeypatch.setenv("MALLOC_ARENA_MAX", configured)
+    document = reviewed_document(draft=extraction_draft, role=FilingDocument.Role.LEAD, name="petition.pdf")
+    queue_document_extraction(document)
+    seen = []
+    child = MagicMock()
+    child.is_alive.return_value = False
+    child.start.side_effect = lambda: seen.append(os.environ.get("MALLOC_ARENA_MAX"))
+    with patch(
+        "efile.management.commands.process_document_extractions.multiprocessing.get_context",
+        return_value=MagicMock(Process=MagicMock(return_value=child)),
+    ):
+        call_command("process_document_extractions", once=True)
+    assert seen == [expected]