From fd9dc65777da425ef26584c2fd7bd817a5109a1f Mon Sep 17 00:00:00 2001 From: GeneralJerel <85066839+GeneralJerel@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:03:31 -0700 Subject: [PATCH] fix(agent): set the step budget from what a turn actually costs A turn died in production with "Recursion limit of 25 reached without hitting a stop condition". The reflex is to read that as the agent over-recursing. It was not: 25 super-steps is four tool calls. The limit counts super-steps, and the middleware chain makes one tool call cost four of them (model -> TodoListMiddleware.after_model -> CopilotKitMiddleware.after_model -> tools) on top of a six-step baseline. Measured against the real graph rather than reasoned about, driving it with a scripted model that makes a known number of tool calls: tool calls 0 1 2 5 super-steps 6 10 14 26 recursion_limit=25 -> completes at most 4 tool calls recursion_limit=60 -> completes at most 13 Four was the right size when the limit was lowered from 100 to 25 in 4ac1a7a for a triage-only bot. The agent now loads 85 tools and gets asked research questions: "find how many PRs shipped in CPK related to channels, categorize them, chart it" answered from 212 PRs and must have sat right at the cap, and the next comparable question went over. A search, a page of results, and a chart is already four calls. Raise the default to 60 and read it from AGENT_RECURSION_LIMIT so it can be tuned without a deploy. Reject a value too low to finish a single tool call, since that breaks every turn rather than tightening anything. The tests pin the conversion to the graph, so if a middleware is added or removed the arithmetic fails rather than silently misreporting -- and what startup prints is checked against what the graph actually completes at four different limits. --- agent/agent.py | 54 +++++++- agent/tests/test_agent_configuration.py | 175 +++++++++++++++++++++++- 2 files changed, 227 insertions(+), 2 deletions(-) diff --git a/agent/agent.py b/agent/agent.py index 9f5f8df7..07d5ecf6 100644 --- a/agent/agent.py +++ b/agent/agent.py @@ -1,6 +1,7 @@ """OpenTag's triage-first Deep Agent.""" import os +from collections.abc import Mapping from pathlib import Path from copilotkit import CopilotKitMiddleware @@ -30,6 +31,21 @@ ) VALID_VERBOSITY_LEVELS = frozenset({"low", "medium", "high"}) +# Super-steps the graph may take in one turn -- NOT a tool-call budget. The +# middleware chain makes one tool call cost four super-steps (model -> +# TodoListMiddleware.after_model -> CopilotKitMiddleware.after_model -> tools) +# on top of a six-step baseline, so the usable budget is about (limit - 6) / 4. +# +# Measured, not estimated: see test_one_tool_call_costs_four_super_steps and +# test_the_old_limit_allowed_only_four_tool_calls. The previous value of 25 +# completed at most FOUR tool calls in a turn, which is why research questions +# died mid-answer. 60 completes thirteen. +DEFAULT_RECURSION_LIMIT = 60 + +# Below this the agent cannot complete a single tool call and every research +# turn dies; treat it as a misconfiguration rather than a tuning choice. +MIN_RECURSION_LIMIT = 10 + # Deep Agents adds shell execution and a general-purpose delegation tool by # default. OpenTag has no sandbox for execute, and delegating routine turns to # another agent adds latency without improving triage. @@ -56,6 +72,36 @@ def _validated_openai_setting( return value +# A turn with no tool calls costs six super-steps, and each tool call four. +# Both measured; see test_one_tool_call_costs_four_super_steps. +_TURN_BASELINE_STEPS = 6 +_STEPS_PER_TOOL_CALL = 4 + + +def tool_call_budget(limit: int) -> int: + """Tool calls a super-step limit actually buys, which is what people mean.""" + return max(0, (limit - _TURN_BASELINE_STEPS) // _STEPS_PER_TOOL_CALL) + + +def recursion_limit(env: Mapping[str, str] = os.environ) -> int: + """Read AGENT_RECURSION_LIMIT, or fall back to the tuned default.""" + raw = env.get("AGENT_RECURSION_LIMIT") + if raw is None or not raw.strip(): + return DEFAULT_RECURSION_LIMIT + try: + value = int(raw) + except ValueError as error: + raise RuntimeError( + f'Invalid AGENT_RECURSION_LIMIT: "{raw}" — must be an integer' + ) from error + if value < MIN_RECURSION_LIMIT: + raise RuntimeError( + f"Invalid AGENT_RECURSION_LIMIT: {value} — must be at least " + f"{MIN_RECURSION_LIMIT}, or the agent cannot finish a tool call" + ) + return value + + def build_agent(): """Build the OpenTag triage graph.""" api_key = os.environ.get("OPENAI_API_KEY") @@ -111,4 +157,10 @@ def build_agent(): print(f"[AGENT] internal-source tools: {len(internal_tools)}") print(f"[AGENT] Main tools: {[t.name for t in main_tools]}") - return agent_graph.with_config({"recursion_limit": 25}) + limit = recursion_limit() + print( + f"[AGENT] recursion limit: {limit} " + f"({tool_call_budget(limit)} tool calls per turn)" + ) + + return agent_graph.with_config({"recursion_limit": limit}) diff --git a/agent/tests/test_agent_configuration.py b/agent/tests/test_agent_configuration.py index 94c2eee9..2ab49ba8 100644 --- a/agent/tests/test_agent_configuration.py +++ b/agent/tests/test_agent_configuration.py @@ -7,7 +7,9 @@ from langchain_core.language_models.chat_models import BaseChatModel from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage from langchain_core.outputs import ChatGeneration, ChatResult +from langchain_core.tools import tool from langchain_openai import ChatOpenAI as RealChatOpenAI +from langgraph.errors import GraphRecursionError from pydantic import Field @@ -97,9 +99,180 @@ def test_build_agent_rejects_invalid_openai_tuning(monkeypatch, name, value): def test_build_agent_bounds_graph_recursion(monkeypatch): + monkeypatch.delenv("AGENT_RECURSION_LIMIT", raising=False) _, captured = build_with_captured_configuration(monkeypatch) - assert captured["config"] == {"recursion_limit": 25} + assert captured["config"] == { + "recursion_limit": agent_mod.DEFAULT_RECURSION_LIMIT + } + + +class ScriptedModel(BaseChatModel): + """Makes exactly `budget` tool calls, one per turn, then answers.""" + + model_name: str = "gpt-5.5" + budget: int = 0 + made: int = Field(default=0) + + @property + def _llm_type(self): + return "scripted" + + def _get_ls_params(self, **_kwargs): + return { + "ls_provider": "openai", + "ls_model_name": self.model_name, + "ls_model_type": "chat", + } + + def bind_tools(self, tools, **_kwargs): + del tools + return self + + def _generate(self, messages, stop=None, run_manager=None, **_kwargs): + del messages, stop, run_manager + if self.made >= self.budget: + return ChatResult( + generations=[ChatGeneration(message=AIMessage(content="done"))] + ) + self.made += 1 + return ChatResult( + generations=[ + ChatGeneration( + message=AIMessage( + content="", + tool_calls=[ + { + "name": "lookup", + "args": {"query": f"q{self.made}"}, + "id": f"call-{self.made}", + } + ], + ) + ) + ] + ) + + +def graph_making_tool_calls(monkeypatch, count, limit=None): + """The real agent graph, driven by a model that makes `count` tool calls.""" + + @tool + def lookup(query: str) -> str: + """Look something up.""" + return f"result for {query}" + + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + for name in ( + "TAVILY_API_KEY", + "GITHUB_PERSONAL_ACCESS_TOKEN", + "POSTHOG_PERSONAL_API_KEY", + "LINEAR_API_KEY", + "NOTION_MCP_AUTH_TOKEN", + ): + monkeypatch.delenv(name, raising=False) + if limit is None: + monkeypatch.delenv("AGENT_RECURSION_LIMIT", raising=False) + else: + monkeypatch.setenv("AGENT_RECURSION_LIMIT", str(limit)) + + monkeypatch.setattr( + agent_mod, "ChatOpenAI", lambda **_kwargs: ScriptedModel(budget=count) + ) + monkeypatch.setattr(agent_mod, "internal_source_tools", lambda: [lookup]) + return agent_mod.build_agent() + + +def super_steps_for(monkeypatch, count): + graph = graph_making_tool_calls(monkeypatch, count, limit=1000) + return sum( + 1 + for _ in graph.stream( + {"messages": [{"role": "user", "content": "go"}]}, + config={"configurable": {"thread_id": f"steps-{count}"}}, + stream_mode="updates", + ) + ) + + +def completes(monkeypatch, count, limit): + graph = graph_making_tool_calls(monkeypatch, count, limit=limit) + try: + graph.invoke( + {"messages": [{"role": "user", "content": "go"}]}, + config={"configurable": {"thread_id": f"survive-{count}-{limit}"}}, + ) + return True + except GraphRecursionError: + return False + + +def test_one_tool_call_costs_four_super_steps(monkeypatch): + """The limit counts super-steps, and the middleware chain multiplies them. + + model -> TodoListMiddleware.after_model -> CopilotKitMiddleware.after_model + -> tools. Measuring it here keeps the budget arithmetic honest: if a + middleware is added or removed, the cost per tool call changes and the + limit has to move with it. + """ + baseline = super_steps_for(monkeypatch, 0) + one = super_steps_for(monkeypatch, 1) + five = super_steps_for(monkeypatch, 5) + + assert one - baseline == 4 + assert five - baseline == 20 + + +def test_the_old_limit_allowed_only_four_tool_calls(monkeypatch): + """Why 25 was failing real turns -- it was never a 25-tool-call budget.""" + assert completes(monkeypatch, 4, limit=25) is True + assert completes(monkeypatch, 5, limit=25) is False + + +@pytest.mark.parametrize("limit", [25, 40, 60, 100]) +def test_the_reported_tool_call_budget_is_the_real_one(monkeypatch, limit): + """What startup prints has to match what the graph actually completes. + + The startup line is the only place an operator learns what a limit buys, + so it must not drift from the graph -- including if a middleware changes + the per-tool-call cost. + """ + budget = agent_mod.tool_call_budget(limit) + + assert completes(monkeypatch, budget, limit=limit) is True + assert completes(monkeypatch, budget + 1, limit=limit) is False + + +def test_the_default_limit_covers_a_research_turn(monkeypatch): + """A GitHub search, pagination, and a chart must fit in one turn.""" + assert completes( + monkeypatch, 12, limit=agent_mod.DEFAULT_RECURSION_LIMIT + ) is True + + +def test_recursion_limit_is_configurable(monkeypatch): + monkeypatch.setenv("AGENT_RECURSION_LIMIT", "120") + _, captured = build_with_captured_configuration(monkeypatch) + + assert captured["config"] == {"recursion_limit": 120} + + +def test_a_blank_recursion_limit_falls_back_to_the_default(): + assert ( + agent_mod.recursion_limit({"AGENT_RECURSION_LIMIT": " "}) + == agent_mod.DEFAULT_RECURSION_LIMIT + ) + assert agent_mod.recursion_limit({}) == agent_mod.DEFAULT_RECURSION_LIMIT + + +def test_a_nonsense_recursion_limit_is_rejected(): + with pytest.raises(RuntimeError, match="AGENT_RECURSION_LIMIT"): + agent_mod.recursion_limit({"AGENT_RECURSION_LIMIT": "lots"}) + + +def test_a_recursion_limit_too_low_to_finish_a_tool_call_is_rejected(): + with pytest.raises(RuntimeError, match="at least"): + agent_mod.recursion_limit({"AGENT_RECURSION_LIMIT": "4"}) def test_build_agent_refreshes_current_date_before_each_model_call(monkeypatch):