From 096479488d57cb20b565450ab57777eacc629fad Mon Sep 17 00:00:00 2001 From: Quinten Steenhuis Date: Thu, 8 Oct 2026 08:58:26 -0400 Subject: [PATCH] Cap malloc arenas so extraction jobs fit their memory limit Each extraction child runs under RLIMIT_AS, which counts address space rather than memory in use. glibc reserves a malloc arena of up to 64 MB per thread, up to eight per CPU core, so the same job's address space grows with the host's core count. On an eight-core workstation a job that uses about 250 MB reached about 820 MB of address space, and every job failed with MemoryError under the 768 MB default; on Fly's one-CPU worker the ceiling is much lower. Set MALLOC_ARENA_MAX=2 for the spawned children (an operator's own value is kept). The same job then peaks at about 440 MB of address space and completes under the default limit. Co-Authored-By: Claude Opus 5.5 --- .../commands/process_document_extractions.py | 8 +++++++ .../efile/tests/test_document_extractions.py | 24 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/efile_app/efile/management/commands/process_document_extractions.py b/efile_app/efile/management/commands/process_document_extractions.py index 6a3cbc1c..c6f84b4f 100644 --- a/efile_app/efile/management/commands/process_document_extractions.py +++ b/efile_app/efile/management/commands/process_document_extractions.py @@ -1,6 +1,7 @@ import io import logging import multiprocessing +import os import time from django.conf import settings @@ -54,6 +55,13 @@ def handle(self, *args, **options): ) next_rollup = 0.0 + # The child's memory cap is on address space, and glibc reserves a + # malloc arena of up to 64 MB per thread, up to eight per CPU core. A + # job that uses about 250 MB reached 800 MB of address space on an + # eight-core machine. Two arenas keep each child's address space close + # to what it uses, whatever the host's core count. Spawned children + # read this at startup; an operator's own value is kept. + os.environ.setdefault("MALLOC_ARENA_MAX", "2") def rollup_if_due(): nonlocal next_rollup diff --git a/efile_app/efile/tests/test_document_extractions.py b/efile_app/efile/tests/test_document_extractions.py index c94c3a39..16a177b4 100644 --- a/efile_app/efile/tests/test_document_extractions.py +++ b/efile_app/efile/tests/test_document_extractions.py @@ -1,3 +1,4 @@ +import os import re import shutil from pathlib import Path @@ -453,3 +454,26 @@ def test_management_command_processes_and_retries_failures(extraction_draft): assert job.status == DocumentExtraction.Status.FAILED assert job.attempts == 2 assert "Document analysis failed" in job.error + + +@pytest.mark.django_db +@pytest.mark.parametrize(("configured", "expected"), [(None, "2"), ("4", "4")]) +def test_worker_children_start_with_few_malloc_arenas(extraction_draft, monkeypatch, configured, expected): + from django.core.management import call_command + + if configured is None: + monkeypatch.delenv("MALLOC_ARENA_MAX", raising=False) + else: + monkeypatch.setenv("MALLOC_ARENA_MAX", configured) + document = reviewed_document(draft=extraction_draft, role=FilingDocument.Role.LEAD, name="petition.pdf") + queue_document_extraction(document) + seen = [] + child = MagicMock() + child.is_alive.return_value = False + child.start.side_effect = lambda: seen.append(os.environ.get("MALLOC_ARENA_MAX")) + with patch( + "efile.management.commands.process_document_extractions.multiprocessing.get_context", + return_value=MagicMock(Process=MagicMock(return_value=child)), + ): + call_command("process_document_extractions", once=True) + assert seen == [expected]