From 89056a7817f1a5d5d390679906ef26601a6d1fe4 Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Mon, 17 Aug 2026 11:56:26 +0800 Subject: [PATCH 1/4] feat: agent test service layer and endpoints Add the agent-test-service.js REST client, its api-endpoints.js URL group, and agentTestTypes.js JSDoc typedefs for suites/cases/runs/results, following the existing agent-service.js conventions. Wires the $agentTestTypes alias in svelte.config.js. No page consumes this yet (Tasks 3-6). --- src/lib/helpers/types/agentTestTypes.js | 195 +++++++++++++++++++++++ src/lib/services/agent-test-service.js | 200 ++++++++++++++++++++++++ src/lib/services/api-endpoints.js | 14 +- svelte.config.js | 3 +- 4 files changed, 410 insertions(+), 2 deletions(-) create mode 100644 src/lib/helpers/types/agentTestTypes.js create mode 100644 src/lib/services/agent-test-service.js diff --git a/src/lib/helpers/types/agentTestTypes.js b/src/lib/helpers/types/agentTestTypes.js new file mode 100644 index 00000000..0d482657 --- /dev/null +++ b/src/lib/helpers/types/agentTestTypes.js @@ -0,0 +1,195 @@ +/** + * @typedef {Object} AgentTestSuite + * @property {string} id + * @property {string} agentId + * @property {string} name + * @property {string?} description + * @property {boolean} enabled + * @property {string?} judgeProvider - llmJudge 断言用的 provider。P1 中 llmJudge 恒定判失败，与是否配置无关。 + * @property {string?} judgeModel + * @property {string[]} extraAllowedFunctions - 在默认控制流白名单之外额外放行的函数名。 + * @property {string[]} forceBlockedFunctions - 强制阻断的函数名，优先级高于白名单。 + * @property {number} caseTimeoutSeconds + * @property {string} createDate - ISO date string. + * @property {string} updateDate - ISO date string. + */ + +/** + * POST /agent-test/suites 与 PUT /agent-test/suites/{id} 的请求体。 + * + * **PUT 是整份替换，不是 PATCH。** 提交前必须先 GET 出完整 suite，只改用户动过的字段， + * 其余原样带回去 —— agentId/name 漏传时后端会兜底保留原值，但 caseTimeoutSeconds/ + * extraAllowedFunctions/forceBlockedFunctions 漏传会被后端重置为默认值（120 / [] / []）。 + * + * @typedef {Object} AgentTestSuiteUpsertRequest + * @property {string} agentId + * @property {string} name + * @property {string?} [description] + * @property {boolean?} [enabled] - 省略或 null = 保持不变（新建时视为 true）；显式传 true/false 才会改变启用状态。 + * @property {string?} [judgeProvider] + * @property {string?} [judgeModel] + * @property {string[]} extraAllowedFunctions + * @property {string[]} forceBlockedFunctions + * @property {number} caseTimeoutSeconds - 默认 120。 + */ + +/** + * @typedef {Object} TestTurn + * @property {number} index + * @property {string} userMessage + * @property {TestAssertion[]} assertions - 本轮级断言（区别于用例整案级的 AgentTestCase.assertions）。 + */ + +/** + * @typedef {Object} TestState + * @property {string} key + * @property {string} value + * @property {number} [activeRounds] - 默认 -1（永久有效）。 + * @property {boolean} [global] + */ + +/** + * @typedef {Object} TestToolMock + * @property {string} functionName + * @property {string?} [argsMatchJson] - 可选：入参子集匹配（JSON 字符串），用于同名工具多次调用给不同返回。 + * @property {number?} [callIndex] - 可选：只命中第 N 次调用（0 基）。 + * @property {string} [resultContent] - 假返回，写入 message.Content，通常是 JSON 文本。 + * @property {boolean} [stopCompletion] - 模拟"中止本轮 LLM 续写"的真实行为。 + * @property {TestState[]?} [stateWrites] - mock 命中时一并写入的 state；很多 IFunctionCallback 完全靠 state 跨轮传数据，只 mock 返回值会让后续函数读不到 state。 + */ + +/** + * 一条断言。 + * + * 后端保存时校验的必填字段（不满足会 400，但错误信息不指明是哪一条 —— 表单应在提交前自行挡住）： + * - outputContains / outputNotContains / outputRegex / routedToAgent / llmJudge：`expected` 必填。 + * - toolCalled / toolNotCalled / stateEquals：`target` 必填。 + * + * 注意一个后端校验没覆盖、但求值时会咬人的点：`stateEquals` 后端保存时只强制 `target`， + * `expected` 留空并不会 400 —— 但求值时会拿 state 的实际值去比较 null/空串，稳定判不过 + * （不是报错，是永远失败）。表单应比后端的 400 校验更严格，把 `stateEquals` 的 `expected` + * 也当必填处理。 + * + * `llmJudge` 在 P1 永远判失败（后端返回 `"llmJudge is not available in P1"`），与 minScore + * 或 suite 是否配置了 judgeProvider/judgeModel 无关；表单可以让用户选它，但要在旁边标注 + * "P1 不可用，会判失败"。 + * + * @typedef {Object} TestAssertion + * @property {string} type - outputContains | outputNotContains | outputRegex | toolCalled | toolNotCalled | stateEquals | routedToAgent | llmJudge + * @property {string?} [target] - 函数名 / state key / agent 名。 + * @property {string?} [expected] - 期望值 / 正则 / 判官标准。 + * @property {string?} [argsMatchJson] - toolCalled 的入参子集匹配（JSON 字符串）。 + * @property {number?} [minScore] - llmJudge 通过阈值。 + * @property {boolean} [fatal] - 失败则中止该用例后续轮。 + */ + +/** + * @typedef {Object} AgentTestCase + * @property {string} id + * @property {string} suiteId + * @property {string} name + * @property {boolean} enabled - 录制生成的草稿用例落库为 false，需人工审阅后手动启用才会加入正式跑批。 + * @property {TestTurn[]} turns - 长度 1 即单轮用例。 + * @property {TestAssertion[]} assertions - 整案级断言：全部轮跑完后求值。 + * @property {TestState[]} initialStates - 会话开始前注入，映射 BotSharp 的 MessageState。 + * @property {TestToolMock[]} mocks + * @property {string} unmockedToolPolicy - P1 只接受 "Block"；提交 "Passthrough" 会 400（"Passthrough is not supported in P1"）。表单不要提供这个选项。 + * @property {string?} sourceConversationId - 录制来源会话，便于回溯；手写用例为 null。 + * @property {string} createDate - ISO date string. + * @property {string} updateDate - ISO date string. + */ + +/** + * POST /agent-test/cases 与 PUT /agent-test/cases/{id} 的请求体。 + * + * **PUT 是整份替换，不是 PATCH。** 提交前必须先 GET 出完整 case，只改用户动过的部分， + * 其余字段（尤其是编辑器未必给控件的 `initialStates`、`unmockedToolPolicy`、 + * `sourceConversationId`）原样带回去 —— 任何字段在 UI 上没有对应控件也必须原样带回， + * 否则保存一次就把它清空了。`suiteId` 漏传时后端会兜底保留原值（除非目标 suite 真的存在 + * 且不同才会重新校验）。 + * + * @typedef {Object} AgentTestCaseUpsertRequest + * @property {string} suiteId + * @property {string} name + * @property {boolean} [enabled] - 默认 true。 + * @property {TestTurn[]} turns + * @property {TestAssertion[]} assertions + * @property {TestState[]} initialStates + * @property {TestToolMock[]} mocks + * @property {string} [unmockedToolPolicy] - 固定传 "Block"；默认值本身就是 "Block"。 + * @property {string?} [sourceConversationId] + */ + +/** + * @typedef {Object} AgentTestRun + * @property {string} id + * @property {string} suiteId + * @property {string} status - Pending | Running | Passed | Failed | Error | Cancelled。 + * `Failed` = 跑了但断言没过；`Error` = 没跑成（超时、canary 失败、用例没有 turns、caseIds + * 匹配不到任何用例）。UI 必须把这两者显示成不同的东西 —— 混在一起会让"平台坏了"看起来 + * 像"agent 回归了"。 + * @property {string?} triggeredBy - 触发者的用户 id。 + * @property {string[]?} caseIds - 本次运行只跑这些 case；null/空 = 跑 suite 下全部启用的 case。 + * @property {number} totalCount + * @property {number} passedCount + * @property {number} failedCount + * @property {number} errorCount + * @property {boolean} cancelRequested + * @property {string?} startedAt - ISO date string；未开始为 null。 + * @property {string?} completedAt - ISO date string；未结束为 null。 + * @property {string} createDate - ISO date string。注意 AgentTestRun 没有 updateDate 字段。 + */ + +/** + * @typedef {Object} AssertionResult + * @property {string} type + * @property {string?} target + * @property {string?} expected + * @property {string?} actual + * @property {boolean} passed + * @property {string?} message + */ + +/** + * @typedef {Object} TurnResult + * @property {number} index + * @property {string} userMessage + * @property {string?} output + * @property {AssertionResult[]} assertions + */ + +/** + * @typedef {Object} ObservedToolCall + * @property {number} turnIndex + * @property {string} functionName + * @property {string?} argsJson + * @property {string} outcome - Mocked | Blocked。`Blocked` 要显眼 —— 意味着 agent 试图调用一个 + * 用例没有 mock 的工具，通常正是失败的根因。 + * @property {string?} resultContent + */ + +/** + * @typedef {Object} AgentTestCaseResult + * @property {string} id + * @property {string} runId + * @property {string} caseId + * @property {string} caseName + * @property {string} status - Passed | Failed | Error | Cancelled（不会是 Pending/Running）。 + * @property {string?} conversationId - 本次执行新建的会话 id，不复用线上会话。 + * @property {number} durationMs + * @property {string?} error - 基础设施层面的失败原因（超时、canary 未生效、用例没有 turns 等）， + * 与断言失败区分开；`status` 为 `Error` 时应把这段文本显示出来。 + * @property {TurnResult[]} turns + * @property {AssertionResult[]} assertions - 整案级断言结果。 + * @property {ObservedToolCall[]} observedToolCalls + * @property {string} createDate - ISO date string. + */ + +/** + * GET /agent-test/runs/{id} 的响应体（后端 AgentTestRunDetailDto 的 camelCase 投影）。 + * @typedef {Object} AgentTestRunDetail + * @property {AgentTestRun} run + * @property {AgentTestCaseResult[]} results + */ + +export default {}; diff --git a/src/lib/services/agent-test-service.js b/src/lib/services/agent-test-service.js new file mode 100644 index 00000000..9f0d78cb --- /dev/null +++ b/src/lib/services/agent-test-service.js @@ -0,0 +1,200 @@ +import { endpoints } from '$lib/services/api-endpoints.js'; +import axios from 'axios'; + +/** + * Get agent test suites, optionally filtered by agent + * @param {string?} [agentId] + * @returns {Promise} + */ +export async function getSuites(agentId = null) { + const url = endpoints.agentTestSuiteListUrl; + const response = await axios.get(url, { + params: { agentId: agentId } + }); + return response.data; +} + +/** + * Get one agent test suite + * @param {string} id + * @returns {Promise} + */ +export async function getSuite(id) { + const url = endpoints.agentTestSuiteDetailUrl.replace("{id}", id); + const response = await axios.get(url); + return response.data; +} + +/** + * Create an agent test suite + * @param {import('$agentTestTypes').AgentTestSuiteUpsertRequest} body + * @returns {Promise} + */ +export async function createSuite(body) { + const url = endpoints.agentTestSuiteListUrl; + const response = await axios.post(url, body); + return response.data; +} + +/** + * Replace an agent test suite. This is a FULL replace, not a patch -- GET the + * complete suite first, change only what the user touched, and send every + * field back. caseTimeoutSeconds/extraAllowedFunctions/forceBlockedFunctions + * are reset to their defaults by the backend if omitted (agentId/name fall + * back to the existing value instead). + * @param {string} id + * @param {import('$agentTestTypes').AgentTestSuiteUpsertRequest} body + * @returns {Promise} + */ +export async function updateSuite(id, body) { + const url = endpoints.agentTestSuiteDetailUrl.replace("{id}", id); + const response = await axios.put(url, body); + return response.data; +} + +/** + * Delete an agent test suite + * @param {string} id + */ +export async function deleteSuite(id) { + const url = endpoints.agentTestSuiteDetailUrl.replace("{id}", id); + await axios.delete(url); +} + +/** + * Get the test cases in a suite (newest first) + * @param {string} suiteId + * @returns {Promise} + */ +export async function getCases(suiteId) { + const url = endpoints.agentTestCaseListUrl; + const response = await axios.get(url, { + params: { suiteId: suiteId } + }); + return response.data; +} + +/** + * Get one test case + * @param {string} id + * @returns {Promise} + */ +export async function getCase(id) { + const url = endpoints.agentTestCaseDetailUrl.replace("{id}", id); + const response = await axios.get(url); + return response.data; +} + +/** + * Create a test case + * @param {import('$agentTestTypes').AgentTestCaseUpsertRequest} body + * @returns {Promise} + */ +export async function createCase(body) { + const url = endpoints.agentTestCaseListUrl; + const response = await axios.post(url, body); + return response.data; +} + +/** + * Replace a test case. This is a FULL replace, not a patch -- GET the complete + * case first, change only what the user touched, and send every field back. + * Any field with no editor control (e.g. initialStates/sourceConversationId on + * a recorded case) must still be carried through untouched, or saving once + * silently clears it. + * @param {string} id + * @param {import('$agentTestTypes').AgentTestCaseUpsertRequest} body + * @returns {Promise} + */ +export async function updateCase(id, body) { + const url = endpoints.agentTestCaseDetailUrl.replace("{id}", id); + const response = await axios.put(url, body); + return response.data; +} + +/** + * Delete a test case + * @param {string} id + */ +export async function deleteCase(id) { + const url = endpoints.agentTestCaseDetailUrl.replace("{id}", id); + await axios.delete(url); +} + +/** + * Trigger a run of a suite. This is fire-and-forget: the run is created + * Pending and queued, and this resolves immediately with the run row -- it + * does not wait for the run to finish. + * @param {string} suiteId + * @param {string[]?} [caseIds] - Only run these cases; omit/empty to run every enabled case in the suite. + * @returns {Promise} + */ +export async function triggerRun(suiteId, caseIds = null) { + const url = endpoints.agentTestSuiteRunUrl.replace("{id}", suiteId); + const response = await axios.post(url, { caseIds: caseIds }); + return response.data; +} + +/** + * Get the run history for a suite + * @param {string} suiteId + * @returns {Promise} + */ +export async function getRuns(suiteId) { + const url = endpoints.agentTestRunListUrl; + const response = await axios.get(url, { + params: { suiteId: suiteId } + }); + return response.data; +} + +/** + * Get a run together with every one of its case results + * @param {string} id + * @returns {Promise} + */ +export async function getRun(id) { + const url = endpoints.agentTestRunDetailUrl.replace("{id}", id); + const response = await axios.get(url); + return response.data; +} + +/** + * Cancel a pending or running run. The backend returns 409 if the run has + * already reached a terminal status (Passed/Failed/Error/Cancelled). + * @param {string} id + */ +export async function cancelRun(id) { + const url = endpoints.agentTestRunCancelUrl.replace("{id}", id); + await axios.post(url); +} + +/** + * Record a draft test case from a real conversation. The returned case comes + * back with enabled=false -- it must be reviewed and explicitly enabled + * before it joins a normal run. + * @param {string} conversationId + * @param {string} suiteId + * @returns {Promise} + */ +export async function recordCase(conversationId, suiteId) { + const url = endpoints.agentTestRecordUrl; + const response = await axios.post(url, { + conversationId: conversationId, + suiteId: suiteId + }); + return response.data; +} + +/** + * Get the mock-target candidates (function names) for an agent's mock editor + * @param {string} agentId + * @returns {Promise} + */ +export async function getMockTargets(agentId) { + const url = endpoints.agentTestMockTargetsUrl; + const response = await axios.get(url, { + params: { agentId: agentId } + }); + return response.data; +} diff --git a/src/lib/services/api-endpoints.js b/src/lib/services/api-endpoints.js index ff75973d..8984b2b8 100644 --- a/src/lib/services/api-endpoints.js +++ b/src/lib/services/api-endpoints.js @@ -135,6 +135,18 @@ export const endpoints = { // Google geocode api addressUrl: `${host}/address/options`, - mcpServerConfigsUrl: `${host}/mcp/server-configs` + mcpServerConfigsUrl: `${host}/mcp/server-configs`, + + // agent test + agentTestSuiteListUrl: `${host}/agent-test/suites`, + agentTestSuiteDetailUrl: `${host}/agent-test/suites/{id}`, + agentTestSuiteRunUrl: `${host}/agent-test/suites/{id}/run`, + agentTestCaseListUrl: `${host}/agent-test/cases`, + agentTestCaseDetailUrl: `${host}/agent-test/cases/{id}`, + agentTestRunListUrl: `${host}/agent-test/runs`, + agentTestRunDetailUrl: `${host}/agent-test/runs/{id}`, + agentTestRunCancelUrl: `${host}/agent-test/runs/{id}/cancel`, + agentTestRecordUrl: `${host}/agent-test/record`, + agentTestMockTargetsUrl: `${host}/agent-test/mock-targets` } diff --git a/svelte.config.js b/svelte.config.js index bf5de71d..1022c8d1 100644 --- a/svelte.config.js +++ b/svelte.config.js @@ -18,7 +18,8 @@ const config = { $pluginTypes: './src/lib/helpers/types/pluginTypes.js', $realtimeTypes: './src/lib/helpers/types/realtimeTypes.js', $instructTypes: './src/lib/helpers/types/instructTypes.js', - $mcpTypes: './src/lib/helpers/types/mcpTypes.js' + $mcpTypes: './src/lib/helpers/types/mcpTypes.js', + $agentTestTypes: './src/lib/helpers/types/agentTestTypes.js' }, // for static deployment From 1e4dcdfb3dc2532ba251b62cd3da792d373e67cc Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Mon, 17 Aug 2026 12:32:01 +0800 Subject: [PATCH 2/4] feat: agent test suite list page Adds the suite list page (page/agent-test): agent filter with Filter/Reset, table of suites (name/agent/enabled), create modal, Swal delete confirmation, and loading/empty/error states. Registers the new route in svelte.config.js's prerender entries, required for adapter-static's crawl:false + strict build to succeed. --- src/routes/page/agent-test/+page.svelte | 418 ++++++++++++++++++++++++ svelte.config.js | 3 +- 2 files changed, 420 insertions(+), 1 deletion(-) create mode 100644 src/routes/page/agent-test/+page.svelte diff --git a/src/routes/page/agent-test/+page.svelte b/src/routes/page/agent-test/+page.svelte new file mode 100644 index 00000000..f4a598ed --- /dev/null +++ b/src/routes/page/agent-test/+page.svelte @@ -0,0 +1,418 @@ + + + + + + + +
+
+
+
+
+
{$_('Test Suites')}
+ +
+
+
+
+
+ changeNewSuiteAgent(e)} + /> +
+
+ + +
+
+ + +
+ +
+
+ + +
+
+
+
+
+{/if} diff --git a/svelte.config.js b/svelte.config.js index 1022c8d1..220e2ac9 100644 --- a/svelte.config.js +++ b/svelte.config.js @@ -76,7 +76,8 @@ const config = { "/page/knowledge-base/relationships", "/page/knowledge-base/documents", "/page/knowledge-base/dictionary", - "/page/knowledge-base/[embed]/[embedType]" + "/page/knowledge-base/[embed]/[embedType]", + "/page/agent-test" ] } }, From df7f49f8862c8993d8de56972ed32800680335a9 Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Tue, 18 Aug 2026 14:07:56 +0800 Subject: [PATCH 3/4] feat: agent test pages, model comparison, AI extraction, i18n Completes the four agent-test pages on top of the existing suite list, then adds the UI for multi-model runs and AI-assisted case extraction. New routes: the suite detail page (settings, case list with multi-select, run history), the case editor (turns, per-turn and case-level assertions, initial states, tool mocks with state writes; caseId=new creates), and the run detail page (case -> turn -> assertion, observed tool calls, 2s polling while non-terminal, cancel, re-run failures). Shared status/validation helpers live in $lib/helpers/utils/agent-test.js. Failed and Errored are deliberately styled and worded differently throughout. Collapsing them is the one mistake that makes "the harness broke" read as "the agent regressed", and the run page exists to tell those apart. Multi-model runs. The run button opens a modal that picks any number of chat-capable models and shows the resulting execution count up front, since each model multiplies the token cost. The run page grows a comparison grid: one row per case, one column per model, status and duration in each cell, per-model totals in the footer. Durations are summed rather than averaged -- "how long does the whole suite take on this model" is what decides whether an upgrade is affordable, and an average hides one pathological case. AI extraction. The record modal takes an optional extraction model, defaulting to off so the deterministic recorder (which never leaves the system) stays the default. Picking one warns which vendor the user messages and tool names go to, and states that tool arguments and results do not. recordCase became recordCases and returns a list. Judge provider/model became cascading dropdowns fed by /llm-configs and filtered to chat-capable models. Unlike the pattern they copy, a stored value the catalogue no longer offers stays selectable -- suite PUT is a full replace, so silently dropping it would blank the field on any unrelated save. i18n: the pages already called $_(), but no key existed in en.json or zh.json, so svelte-i18n fell back to the key and Chinese rendered as English. All 248 keys now resolve in both locales, including the strings that were previously hardcoded outside the markup (confirm dialogs, toasts, validation text, and the accessible names of icon-only buttons). Also fixes a relative goto in the suite list that resolved against the current URL. Co-Authored-By: Claude Opus 5 --- src/lib/helpers/types/agentTestTypes.js | 15 + src/lib/helpers/utils/agent-test.js | 173 +++ src/lib/langs/en.json | 482 +++++-- src/lib/langs/zh.json | 513 ++++++-- src/lib/services/agent-test-service.js | 26 +- src/routes/page/agent-test/+page.svelte | 33 +- .../page/agent-test/[suiteId]/+page.svelte | 1124 +++++++++++++++++ .../[suiteId]/case/[caseId]/+page.svelte | 775 ++++++++++++ .../page/agent-test/run/[runId]/+page.svelte | 530 ++++++++ svelte.config.js | 5 +- 10 files changed, 3389 insertions(+), 287 deletions(-) create mode 100644 src/lib/helpers/utils/agent-test.js create mode 100644 src/routes/page/agent-test/[suiteId]/+page.svelte create mode 100644 src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte create mode 100644 src/routes/page/agent-test/run/[runId]/+page.svelte diff --git a/src/lib/helpers/types/agentTestTypes.js b/src/lib/helpers/types/agentTestTypes.js index 0d482657..e6785ebb 100644 --- a/src/lib/helpers/types/agentTestTypes.js +++ b/src/lib/helpers/types/agentTestTypes.js @@ -120,10 +120,21 @@ * @property {string?} [sourceConversationId] */ +/** + * One model to run against. + * @typedef {Object} TestModel + * @property {string} provider + * @property {string} model + */ + /** * @typedef {Object} AgentTestRun * @property {string} id * @property {string} suiteId + * @property {TestModel[]?} models - Models this run swept; null/empty = a single pass on the + * agent's own LlmConfig. When set, the executor runs the cartesian product of cases x models, + * so `totalCount` is "cases x models" and does NOT equal the case count -- showing it as a case + * count in the UI will not add up. * @property {string} status - Pending | Running | Passed | Failed | Error | Cancelled。 * `Failed` = 跑了但断言没过；`Error` = 没跑成（超时、canary 失败、用例没有 turns、caseIds * 匹配不到任何用例）。UI 必须把这两者显示成不同的东西 —— 混在一起会让"平台坏了"看起来 @@ -176,6 +187,10 @@ * @property {string} caseName * @property {string} status - Passed | Failed | Error | Cancelled（不会是 Pending/Running）。 * @property {string?} conversationId - 本次执行新建的会话 id，不复用线上会话。 + * @property {string?} provider - Which model produced this result; null = the agent's own + * LlmConfig was used. + * @property {string?} model - As above. In a multi-model run the same `caseId` yields several + * results, told apart by these two fields. * @property {number} durationMs * @property {string?} error - 基础设施层面的失败原因（超时、canary 未生效、用例没有 turns 等）， * 与断言失败区分开；`status` 为 `Error` 时应把这段文本显示出来。 diff --git a/src/lib/helpers/utils/agent-test.js b/src/lib/helpers/utils/agent-test.js new file mode 100644 index 00000000..41c06c92 --- /dev/null +++ b/src/lib/helpers/utils/agent-test.js @@ -0,0 +1,173 @@ +import { get } from 'svelte/store'; +import { _ } from 'svelte-i18n'; + +/** + * Shared helpers for the agent test pages. Kept out of the page components + * because all four of them have to agree on what a status means -- showing + * `Failed` and `Error` the same way is the one mistake that makes "the harness + * broke" look like "the agent regressed". + */ + +/** + * Translate from plain JS, where the `$_` auto-subscription is not available. + * Read at call time, never at module load -- the dictionary is fetched + * asynchronously by setupI18n and is empty while the module is first evaluated. + * + * This is a one-shot read, not a subscription, so the result does not + * re-translate if the user switches language afterwards. That is fine for what + * uses it (toasts, confirm dialogs, validation text -- all short-lived or + * recomputed on the next keystroke). Anything that has to survive a language + * switch on screen belongs in the markup as `$_(...)` instead. + * @param {string} id + * @param {Record} [values] + * @returns {string} + */ +export function t(id, values = undefined) { + return get(_)(id, values ? { values } : undefined); +} + +/** Assertion types the backend can actually evaluate (AssertionTypes in AssertionEvaluator.cs). */ +export const ASSERTION_TYPES = [ + 'outputContains', + 'outputNotContains', + 'outputRegex', + 'toolCalled', + 'toolNotCalled', + 'stateEquals', + 'routedToAgent', + 'llmJudge' +]; + +/** Types whose `expected` the backend rejects as empty. */ +const EXPECTED_REQUIRED = ['outputContains', 'outputNotContains', 'outputRegex', 'routedToAgent', 'llmJudge']; + +/** Types whose `target` the backend rejects as empty. */ +const TARGET_REQUIRED = ['toolCalled', 'toolNotCalled', 'stateEquals']; + +/** + * Bootstrap contextual class for a run/case status. + * @param {string?} status + * @returns {string} + */ +export function statusColor(status) { + switch (status) { + case 'Passed': return 'success'; + // Failed = it ran and an assertion did not hold. Error = it never got to + // assert (timeout, canary, no turns). Different colours on purpose. + case 'Failed': return 'danger'; + case 'Error': return 'warning'; + case 'Running': return 'info'; + case 'Pending': return 'secondary'; + case 'Cancelled': return 'dark'; + default: return 'secondary'; + } +} + +/** + * A run in Pending/Running is still moving, so the detail page keeps polling + * and the cancel button stays live. + * @param {string?} status + */ +export function isTerminalStatus(status) { + return status === 'Passed' || status === 'Failed' || status === 'Error' || status === 'Cancelled'; +} + +/** + * Pull the backend's own message out of an axios error. These endpoints return + * genuinely actionable 400s ("Passthrough is not supported in P1", "assertion + * 'outputRegex' requires a non-empty 'expected' value", "suite is disabled"), + * so swallowing them behind a generic string throws away the only clue there + * is. Those messages are server-side English and are shown as-is; only the + * strings this file owns are translated. + * @param {any} err + * @param {string} fallback - already-translated text + * @returns {string} + */ +export function errorMessage(err, fallback) { + const data = err?.response?.data; + if (typeof data === 'string' && data.trim()) { + return data; + } + if (typeof data?.message === 'string' && data.message.trim()) { + return data.message; + } + if (err?.response?.status === 401 || err?.response?.status === 403) { + return t('You do not have permission for this. Running and recording require an admin or root account.'); + } + return fallback; +} + +/** + * Validate one assertion the way the backend does, plus the two cases it lets + * through but that can never pass at evaluation time. + * @param {import('$agentTestTypes').TestAssertion} assertion + * @returns {string?} null when well-formed, otherwise a message for the author + */ +export function validateAssertion(assertion) { + const type = assertion?.type; + if (!type) { + return t('Assertion type is required.'); + } + + if (EXPECTED_REQUIRED.includes(type) && !assertion.expected?.trim()) { + return t('Assertion "{type}" needs an expected value.', { type }); + } + + if (TARGET_REQUIRED.includes(type) && !assertion.target?.trim()) { + return t('Assertion "{type}" needs a target.', { type }); + } + + // Stricter than the backend on purpose: stateEquals saves fine with an empty + // `expected`, then compares the real state value against null and fails on + // every single run. A silent always-red assertion is worse than a save error. + if (type === 'stateEquals' && !assertion.expected?.trim()) { + return t('Assertion "stateEquals" needs an expected value, otherwise it can never pass.'); + } + + if (type === 'outputRegex' && assertion.expected?.trim()) { + try { + new RegExp(assertion.expected); + } catch { + return t('"{pattern}" is not a valid regular expression.', { pattern: assertion.expected }); + } + } + + if (assertion.argsMatchJson?.trim() && !isParsableJson(assertion.argsMatchJson)) { + return t('The args match on an assertion is not valid JSON.'); + } + + return null; +} + +/** + * @param {string} text + * @returns {boolean} + */ +export function isParsableJson(text) { + try { + JSON.parse(text); + return true; + } catch { + return false; + } +} + +/** + * @param {number?} ms + * @returns {string} + */ +export function formatDuration(ms) { + if (ms == null) return '--'; + if (ms < 1000) return t('{ms} ms', { ms }); + return t('{seconds} s', { seconds: (ms / 1000).toFixed(1) }); +} + +/** + * @param {string?} iso + * @returns {string} + */ +export function formatDateTime(iso) { + if (!iso) return '--'; + const date = new Date(iso); + return Number.isNaN(date.getTime()) ? '--' : date.toLocaleString(); +} diff --git a/src/lib/langs/en.json b/src/lib/langs/en.json index 7f905c0b..7fb5bf67 100644 --- a/src/lib/langs/en.json +++ b/src/lib/langs/en.json @@ -168,138 +168,372 @@ "Your item is shipped": "Your item is shipped", "As a skeptical Cambridge friend of mine occidental": "As a skeptical Cambridge friend of mine occidental", "1 hours ago": "1 hours ago", - - "Agents":"Agents", - "Evaluator":"Evaluator", - "Router":"Router", - "Home":"Home", - "Dashboard":"Dashboard", - "Agent":"Agent", - "Knowledge Base":"Knowledge Base", - "Conversation":"Conversation", - "Task":"Task", - "MongoDB":"MongoDB", - "Plugins":"Plugins", + "Agents": "Agents", + "Evaluator": "Evaluator", + "Router": "Router", + "Home": "Home", + "Dashboard": "Dashboard", + "Agent": "Agent", + "Knowledge Base": "Knowledge Base", + "Conversation": "Conversation", + "Task": "Task", + "MongoDB": "MongoDB", + "Plugins": "Plugins", "Settings": "Settings", - "Search":"Search", - "Welcome Back !":"Welcome Back !", - "Admin":"Admin", - "Agent Manager":"Agent Manager", - "Conversations":"Conversations", - "Token Cost":"Token Cost", - "View Profile":"View Profile", - "Monthly Cost":"Monthly Cost", - "This month":"This month", + "Search": "Search", + "Welcome Back !": "Welcome Back !", + "Admin": "Admin", + "Agent Manager": "Agent Manager", + "Conversations": "Conversations", + "Token Cost": "Token Cost", + "View Profile": "View Profile", + "Monthly Cost": "Monthly Cost", + "This month": "This month", "View More": "View More", - "Total Cost":"Total Cost", - "Average Cost":"Average Cost", - "Week":"Week", - "Month":"Month", - "Year":"Year", - "Token Spent":"Token Spent", - "Channel Stats":"Channel Stats", - "Learn more":"Learn more", - "sales":"sales", - "Activity":"Activity", - "Read more":"Read more", - "Top Client Usage":"Top Client Usage", + "Total Cost": "Total Cost", + "Average Cost": "Average Cost", + "Week": "Week", + "Month": "Month", + "Year": "Year", + "Token Spent": "Token Spent", + "Channel Stats": "Channel Stats", + "Learn more": "Learn more", + "sales": "sales", + "Activity": "Activity", + "Read more": "Read more", + "Top Client Usage": "Top Client Usage", "Logout": "Logout", "Profile": "Profile", "Notifications": "Notifications", - "View All":"View All", + "View All": "View All", "Your order is placed": "Your order is placed", - "min ago":"min ago", + "min ago": "min ago", "List": "List", - "Communication":"Communication", + "Communication": "Communication", "Conversation List": "Conversation List", "Task List": "Task List", - "Knowledge Manager":"Knowledge Manager", - "Migrate agents from file repository to MongoDB":"Migrate agents from file repository to MongoDB", - "Setting":"Setting", - "Start Migration":"Start Migration", - "Plugin":"Plugin", - "My Profile":"My Profile", - "Detail":"Detail", - "System & Plugin Settings":"System & Plugin Settings", - "Disabled":"Disabled", - "Enabled":"Enabled", - "Public":"Public", - "Private":"Private", - "Build":"Build", - "Train":"Train", - "Test":"Test", - "My Files":"My Files", - "Upload":"Upload", - "Recent Files":"Recent Files", - "Storage":"Storage", - "0 GB (0%) of 1 GB used":"0 GB (0%) of 1 GB used", - "Provided by BotSharp":"Provided by BotSharp", - "Provided by Pizza AI Assistant":"Provided by Pizza AI Assistant", - "Upgrade Features":"Upgrade Features", - "Upgrade":"Upgrade", - "Files":"Files", - "Design":"Design", - "Google Drive":"Google Drive", - "Starred":"Starred", - "Trash":"Trash", - "Dropbox":"Dropbox", - "Open":"Open", - "Edit":"Edit" , - "Rename":"Rename", - "Remove":"Remove", - "Share Files":"Share Files", - "Share with me":"Share with me", - "Other Actions":"Other Actions", - "Name":"Name", - "Date modified":"Date modified", - "Size":"Size", + "Knowledge Manager": "Knowledge Manager", + "Migrate agents from file repository to MongoDB": "Migrate agents from file repository to MongoDB", + "Setting": "Setting", + "Start Migration": "Start Migration", + "Plugin": "Plugin", + "My Profile": "My Profile", + "Detail": "Detail", + "System & Plugin Settings": "System & Plugin Settings", + "Disabled": "Disabled", + "Enabled": "Enabled", + "Public": "Public", + "Private": "Private", + "Build": "Build", + "Train": "Train", + "Test": "Test", + "My Files": "My Files", + "Upload": "Upload", + "Recent Files": "Recent Files", + "Storage": "Storage", + "0 GB (0%) of 1 GB used": "0 GB (0%) of 1 GB used", + "Provided by BotSharp": "Provided by BotSharp", + "Provided by Pizza AI Assistant": "Provided by Pizza AI Assistant", + "Upgrade Features": "Upgrade Features", + "Upgrade": "Upgrade", + "Files": "Files", + "Design": "Design", + "Google Drive": "Google Drive", + "Starred": "Starred", + "Trash": "Trash", + "Dropbox": "Dropbox", + "Open": "Open", + "Edit": "Edit", + "Rename": "Rename", + "Remove": "Remove", + "Share Files": "Share Files", + "Share with me": "Share with me", + "Other Actions": "Other Actions", + "Name": "Name", + "Date modified": "Date modified", + "Size": "Size", "Images": "Images", "Video": "Video", - "Music":"Music", - "Document":"Document", - "Others":"Others", - "Another action":"Another action", - "Something else here":"Something else here", - "Search for ...":"Search for ...", - "Status":"Status", - "Completed":"Completed", - "Select Channel":"Select Channel", - "Live Chat":"Live Chat", - "Phone":"Phone", - "Email":"Email", - "Filter":"Filter", - "Title":"Title", - "User Name":"User Name", - "Channel":"Channel", - "Posted Date":"Posted Date", - "Last Date":"Last Date", - "Action":"Action", - "Active":"Active", - "Description":"Description", - "Details":"Details", - "Updated Date":"Updated Date", - "Showing":"Showing", - "of":"of", - "to":"to", - "entries":"entries", - "View":"View", - "Install":"Install", - "Dialogs":"Dialogs", - "Agent Overview":"Agent Overview", - "Save Agent":"Save Agent", - "Conversation Detail":"Conversation Detail", - "closed":"closed", - "webchat":"webchat", - "User":"User", - "Location":"Location", - "Delete Conversation":" Delete Conversation", - "Design & Developed by open source community":"Design & Developed by open source community", - "Personal Information":"Personal Information", - "First Name":"First Name", - "Last Name":"Last Name", - "Account Origin":"Account Origin", - "Update Date":"Update Date", - "Create Date":"Create Date", - "Active now":"Active now", - "Reset":"Reset" -} \ No newline at end of file + "Music": "Music", + "Document": "Document", + "Others": "Others", + "Another action": "Another action", + "Something else here": "Something else here", + "Search for ...": "Search for ...", + "Status": "Status", + "Completed": "Completed", + "Select Channel": "Select Channel", + "Live Chat": "Live Chat", + "Phone": "Phone", + "Email": "Email", + "Filter": "Filter", + "Title": "Title", + "User Name": "User Name", + "Channel": "Channel", + "Posted Date": "Posted Date", + "Last Date": "Last Date", + "Action": "Action", + "Active": "Active", + "Description": "Description", + "Details": "Details", + "Updated Date": "Updated Date", + "Showing": "Showing", + "of": "of", + "to": "to", + "entries": "entries", + "View": "View", + "Install": "Install", + "Dialogs": "Dialogs", + "Agent Overview": "Agent Overview", + "Save Agent": "Save Agent", + "Conversation Detail": "Conversation Detail", + "closed": "closed", + "webchat": "webchat", + "User": "User", + "Location": "Location", + "Delete Conversation": " Delete Conversation", + "Design & Developed by open source community": "Design & Developed by open source community", + "Personal Information": "Personal Information", + "First Name": "First Name", + "Last Name": "Last Name", + "Account Origin": "Account Origin", + "Update Date": "Update Date", + "Create Date": "Create Date", + "Active now": "Active now", + "Reset": "Reset", + "Agent Testing": "Agent Testing", + "Test Suites": "Test Suites", + "Test Suite": "Test Suite", + "Test Cases": "Test Cases", + "Test Case": "Test Case", + "Test Run": "Test Run", + "Run History": "Run History", + "Case Results": "Case Results", + "Suite Settings": "Suite Settings", + "New Test Suite": "New Test Suite", + "New Test Case": "New Test Case", + "Edit Test Case": "Edit Test Case", + "Record a Draft Case": "Record a Draft Case", + "Turns": "Turns", + "Turn": "Turn", + "Case Assertions": "Case Assertions", + "Assertions": "Assertions", + "Initial States": "Initial States", + "Tool Mocks": "Tool Mocks", + "Observed Tool Calls": "Observed Tool Calls", + "New Suite": "New Suite", + "New Case": "New Case", + "Add Turn": "Add Turn", + "Add Assertion": "Add Assertion", + "Add State": "Add State", + "Add State Write": "Add State Write", + "Add Mock": "Add Mock", + "Run All Enabled": "Run All Enabled", + "Run Selected": "Run Selected", + "Cancel Run": "Cancel Run", + "Re-run Failures": "Re-run Failures", + "Record from Conversation": "Record from Conversation", + "Record": "Record", + "Refresh": "Refresh", + "Retry": "Retry", + "Save": "Save", + "Create": "Create", + "Cancel": "Cancel", + "Close": "Close", + "Delete": "Delete", + "Enable": "Enable", + "Disable": "Disable", + "Back": "Back", + "Back to Suite": "Back to Suite", + "Back to Suites": "Back to Suites", + "Show details": "Show details", + "Hide details": "Hide details", + "Clear selection": "Clear selection", + "All Agents": "All Agents", + "Select Agent": "Select Agent", + "Select all cases": "Select all cases", + "Select case {name}": "Select case {name}", + "Edit case": "Edit case", + "Delete case": "Delete case", + "Enable case": "Enable case", + "Disable case": "Disable case", + "View run": "View run", + "Remove turn": "Remove turn", + "Move turn up": "Move turn up", + "Move turn down": "Move turn down", + "Remove assertion": "Remove assertion", + "Remove state": "Remove state", + "Remove mock": "Remove mock", + "Source": "Source", + "Scope": "Scope", + "Started": "Started", + "Duration": "Duration", + "Result": "Result", + "Type": "Type", + "Target": "Target", + "Expected": "Expected", + "Actual": "Actual", + "Message": "Message", + "Function": "Function", + "Outcome": "Outcome", + "Args": "Args", + "Key": "Key", + "Value": "Value", + "Total": "Total", + "Mocks": "Mocks", + "Mock": "Mock", + "Run": "Run", + "Fatal": "Fatal", + "Global": "Global", + "Active rounds": "Active rounds", + "Function name": "Function name", + "Result content": "Result content", + "Stop completion": "Stop completion", + "User message": "User message", + "Agent output": "Agent output", + "Conversation id": "Conversation id", + "Judge provider": "Judge provider", + "Judge model": "Judge model", + "Case timeout": "Case timeout", + "Case timeout (seconds)": "Case timeout (seconds)", + "Extra allowed functions": "Extra allowed functions", + "Force blocked functions": "Force blocked functions", + "Unmocked tool policy": "Unmocked tool policy", + "Enabled (included in runs)": "Enabled (included in runs)", + "Triggered by": "Triggered by", + "Args match (JSON subset, optional)": "Args match (JSON subset, optional)", + "Call index (0-based, optional)": "Call index (0-based, optional)", + "Enter suite name": "Enter suite name", + "Optional description": "Optional description", + "function name": "function name", + "state key": "state key", + "An existing conversation with at least one tool call": "An existing conversation with at least one tool call", + "Pending": "Pending", + "Running": "Running", + "Passed": "Passed", + "Failed": "Failed", + "Error": "Error", + "Cancelled": "Cancelled", + "Cancelling": "Cancelling", + "Mocked": "Mocked", + "Blocked": "Blocked", + "Pass": "Pass", + "Fail": "Fail", + "Draft": "Draft", + "Recorded": "Recorded", + "Manual": "Manual", + "Subset": "Subset", + "All enabled": "All enabled", + "passed": "passed", + "failed": "failed", + "errored": "errored", + "Failed (assertion)": "Failed (assertion)", + "Errored (never ran)": "Errored (never ran)", + "Harness error": "Harness error", + "Partial run of": "Partial run of", + "selected case(s).": "selected case(s).", + "No test suites yet.": "No test suites yet.", + "No test suites for this agent yet.": "No test suites for this agent yet.", + "No test cases in this suite yet.": "No test cases in this suite yet.", + "This suite has never been run.": "This suite has never been run.", + "No case results yet.": "No case results yet.", + "This run produced no case results.": "This run produced no case results.", + "No turns yet. A case needs at least one.": "No turns yet. A case needs at least one.", + "No case-level assertions.": "No case-level assertions.", + "No initial states.": "No initial states.", + "No assertions on this turn.": "No assertions on this turn.", + "The agent called no tools during this case.": "The agent called no tools during this case.", + "No mocks. Every tool call this case makes will be blocked.": "No mocks. Every tool call this case makes will be blocked.", + "A disabled suite cannot be run at all.": "A disabled suite cannot be run at all.", + "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.": "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.", + "llmJudge assertions always fail in P1 regardless of these two fields.": "llmJudge assertions always fail in P1 regardless of these two fields.", + "Always fails in P1.": "Always fails in P1.", + "One function name per line. These run for real during a test -- only list functions with no side effects.": "One function name per line. These run for real during a test -- only list functions with no side effects.", + "One per line. Blocked even if allow-listed elsewhere.": "One per line. Blocked even if allow-listed elsewhere.", + "Assertions checked right after this turn": "Assertions checked right after this turn", + "Evaluated once, after every turn has run.": "Evaluated once, after every turn has run.", + "Injected before the conversation starts. Active rounds -1 means it never expires.": "Injected before the conversation starts. Active rounds -1 means it never expires.", + "Every tool this case does not mock is blocked. If the agent needs a tool to move forward, mock it here.": "Every tool this case does not mock is blocked. If the agent needs a tool to move forward, mock it here.", + "State written when this mock is hit": "State written when this mock is hit", + "Many tools pass data to later turns through state, not through their return value.": "Many tools pass data to later turns through state, not through their return value.", + "Any tool this case does not mock is blocked instead of executed. P1 has no other option.": "Any tool this case does not mock is blocked instead of executed. P1 has no other option.", + "Recorded from conversation": "Recorded from conversation", + "It may contain live customer data -- review before enabling.": "It may contain live customer data -- review before enabling.", + "Recorded from conversation {id}. May contain live customer data.": "Recorded from conversation {id}. May contain live customer data.", + "Recording copies the raw conversation into the test store, including any phone numbers, addresses and tenant names it contains. The draft lands disabled -- review it before enabling.": "Recording copies the raw conversation into the test store, including any phone numbers, addresses and tenant names it contains. The draft lands disabled -- review it before enabling.", + "Errored cases never reached their assertions -- a timeout, a mock-seam failure, or a case with no turns. That is a harness problem, not an agent regression.": "Errored cases never reached their assertions -- a timeout, a mock-seam failure, or a case with no turns. That is a harness problem, not an agent regression.", + "This run is still going. Refreshing every couple of seconds.": "This run is still going. Refreshing every couple of seconds.", + "The agent tried to call a tool this case does not mock, so it was blocked. This is often the root cause of the failure above.": "The agent tried to call a tool this case does not mock, so it was blocked. This is often the root cause of the failure above.", + "Are you sure?": "Are you sure?", + "Yes, delete it!": "Yes, delete it!", + "Yes, run it": "Yes, run it", + "Run this suite?": "Run this suite?", + "Delete test suite \"{name}\"? You won't be able to revert this!": "Delete test suite \"{name}\"? You won't be able to revert this!", + "Delete test case \"{name}\"? You won't be able to revert this!": "Delete test case \"{name}\"? You won't be able to revert this!", + "{count} case(s) will run against the live model. This costs real tokens and is not rate limited.": "{count} case(s) will run against the live model. This costs real tokens and is not rate limited.", + "Test suite created!": "Test suite created!", + "Test suite deleted!": "Test suite deleted!", + "Test case created!": "Test case created!", + "Test case saved!": "Test case saved!", + "Test case deleted!": "Test case deleted!", + "Suite settings saved!": "Suite settings saved!", + "Case enabled.": "Case enabled.", + "Case disabled.": "Case disabled.", + "Run queued!": "Run queued!", + "Re-run queued!": "Re-run queued!", + "Cancellation requested.": "Cancellation requested.", + "Draft case recorded. Review it, then enable it.": "Draft case recorded. Review it, then enable it.", + "Failed to load test suites. Please try again.": "Failed to load test suites. Please try again.", + "Failed to load this test suite.": "Failed to load this test suite.", + "Failed to load this test case.": "Failed to load this test case.", + "Failed to load the suite this case belongs to.": "Failed to load the suite this case belongs to.", + "Failed to load this run.": "Failed to load this run.", + "Failed to create test suite.": "Failed to create test suite.", + "Failed to delete test suite.": "Failed to delete test suite.", + "Failed to save suite settings.": "Failed to save suite settings.", + "Failed to save the test case.": "Failed to save the test case.", + "Failed to update the case.": "Failed to update the case.", + "Failed to delete the test case.": "Failed to delete the test case.", + "Failed to start the run.": "Failed to start the run.", + "Failed to start the re-run.": "Failed to start the re-run.", + "Failed to cancel the run.": "Failed to cancel the run.", + "Failed to record a draft from that conversation.": "Failed to record a draft from that conversation.", + "You do not have permission for this. Running and recording require an admin or root account.": "You do not have permission for this. Running and recording require an admin or root account.", + "Fix these before saving:": "Fix these before saving:", + "Name is required.": "Name is required.", + "Add at least one turn -- a case with no turns errors when it runs.": "Add at least one turn -- a case with no turns errors when it runs.", + "Turn {n}: the user message is empty.": "Turn {n}: the user message is empty.", + "Turn {n}, assertion {m}: {error}": "Turn {n}, assertion {m}: {error}", + "Case assertion {n}: {error}": "Case assertion {n}: {error}", + "Initial state {n}: the key is empty.": "Initial state {n}: the key is empty.", + "Mock {n}: the function name is empty.": "Mock {n}: the function name is empty.", + "Mock {n}: the args match is not valid JSON.": "Mock {n}: the args match is not valid JSON.", + "Mock {n}: the call index is 0-based and cannot be negative.": "Mock {n}: the call index is 0-based and cannot be negative.", + "Mock {n}, state write {m}: the key is empty.": "Mock {n}, state write {m}: the key is empty.", + "Assertion type is required.": "Assertion type is required.", + "Assertion \"{type}\" needs an expected value.": "Assertion \"{type}\" needs an expected value.", + "Assertion \"{type}\" needs a target.": "Assertion \"{type}\" needs a target.", + "Assertion \"stateEquals\" needs an expected value, otherwise it can never pass.": "Assertion \"stateEquals\" needs an expected value, otherwise it can never pass.", + "\"{pattern}\" is not a valid regular expression.": "\"{pattern}\" is not a valid regular expression.", + "The args match on an assertion is not valid JSON.": "The args match on an assertion is not valid JSON.", + "{ms} ms": "{ms} ms", + "{seconds} s": "{seconds} s", + "Not configured": "Not configured", + "This provider has no chat-capable model registered.": "This provider has no chat-capable model registered.", + "Case": "Case", + "Models": "Models", + "models": "models", + "Model Comparison": "Model Comparison", + "Running every enabled case": "Running every enabled case", + "Running the selected cases": "Running the selected cases", + "Use the agent's own model": "Use the agent's own model", + "Pick nothing to run once on whatever model each agent is configured with. Pick two or more to run the whole set once per model and compare them side by side.": "Pick nothing to run once on whatever model each agent is configured with. Pick two or more to run the whole set once per model and compare them side by side.", + "No chat-capable model is registered, so this run will use the agent's own configuration.": "No chat-capable model is registered, so this run will use the agent's own configuration.", + "{count} execution(s) will run against live models. This costs real tokens and is not rate limited.": "{count} execution(s) will run against live models. This costs real tokens and is not rate limited.", + "Durations are wall-clock for the whole case, including mocked tool calls, so they are comparable between models but are not a pure model-latency measurement.": "Durations are wall-clock for the whole case, including mocked tool calls, so they are comparable between models but are not a pure model-latency measurement.", + "AI extraction model": "AI extraction model", + "Do not use AI (one case for the whole conversation)": "Do not use AI (one case for the whole conversation)", + "With a model picked, the conversation is split into one case per scenario it covers. The model only decides where to split and what to name each case -- mocks, assertions and state still come verbatim from the conversation.": "With a model picked, the conversation is split into one case per scenario it covers. The model only decides where to split and what to name each case -- mocks, assertions and state still come verbatim from the conversation.", + "AI extraction additionally sends the user messages and tool names to {provider}. Tool arguments and results are not sent.": "AI extraction additionally sends the user messages and tool names to {provider}. Tool arguments and results are not sent.", + "{count} draft cases recorded. Review them, then enable them.": "{count} draft cases recorded. Review them, then enable them." +} diff --git a/src/lib/langs/zh.json b/src/lib/langs/zh.json index c2c06b19..ce11d2fc 100644 --- a/src/lib/langs/zh.json +++ b/src/lib/langs/zh.json @@ -102,7 +102,7 @@ "saas": "萨斯", "crypto": "加密货币", "blog": "博客", - "jobs":"工作" + "jobs": "工作" } }, "layouts": { @@ -214,21 +214,20 @@ "detail": "博客详细信息" } }, - "jobs" : { + "jobs": { "text": "工作", "list": { - "joblist" : "工作清單", - "jobgrid" : "工作網格", - "applyjob" : "申請工作", - "jobdetails" : "工作詳情", - "jobcategories" : "工作類別", - "candidate" : { + "joblist": "工作清單", + "jobgrid": "工作網格", + "applyjob": "申請工作", + "jobdetails": "工作詳情", + "jobcategories": "工作類別", + "candidate": { "text": "候選人", "list": { - "list" : "列表", - "overview" : "概述" + "list": "列表", + "overview": "概述" } - } } }, @@ -296,8 +295,8 @@ "lightbox": "灯箱", "cropper": "圖像裁剪器", "drawer": "抽屉", - "toast":"吐司", - "utility":"效用" + "toast": "吐司", + "utility": "效用" } }, "forms": { @@ -362,139 +361,373 @@ } } }, - - "Agents":"智能代理", - "Evaluator":"评估者", - "Evaluating":"评估器", - "Router":"路由器", + "Agents": "智能代理", + "Evaluator": "评估者", + "Evaluating": "评估器", + "Router": "路由器", "Routing": "路由图", "Plugins": "插件", "Settings": "设置", - "Dashboard":"指示板", - "Agent":"代理", - "Knowledge Base":"知识库", - "Conversation":"对话", - "Task":"任务", - "MongoDB":"MongoDB", - "Home":"主页", - "Search":"搜索", - "Welcome Back !":"欢迎回来 !", - "Admin":"管理", - "Agent Manager":"代理管理器", - "Conversations":"对话", - "Token Cost":"令牌成本", - "View Profile":"查看配置文件", - "Monthly Cost":"每月的成本", - "This month":"这个月", - "View More":"查看更多", - "Total Cost":"总成本", - "Average Cost":"平均成本", - "Week":"周", - "Month":"月", - "Year":"年", - "Token Spent":"令牌用量统计", - "Channel Stats":"通道数据", - "Learn more":"了解更多", - "sales":"销售额", - "Activity":"活动", - "Top Client Usage":"客户端用量", + "Dashboard": "指示板", + "Agent": "代理", + "Knowledge Base": "知识库", + "Conversation": "对话", + "Task": "任务", + "MongoDB": "MongoDB", + "Home": "主页", + "Search": "搜索", + "Welcome Back !": "欢迎回来 !", + "Admin": "管理", + "Agent Manager": "代理管理器", + "Conversations": "对话", + "Token Cost": "令牌成本", + "View Profile": "查看配置文件", + "Monthly Cost": "每月的成本", + "This month": "这个月", + "View More": "查看更多", + "Total Cost": "总成本", + "Average Cost": "平均成本", + "Week": "周", + "Month": "月", + "Year": "年", + "Token Spent": "令牌用量统计", + "Channel Stats": "通道数据", + "Learn more": "了解更多", + "sales": "销售额", + "Activity": "活动", + "Top Client Usage": "客户端用量", "Logout": "登出", "Profile": "配置文件", "Notifications": "通知", - "View All":"查看全部", - "Your order is placed":"您的订单已经下达", - "min ago":"分钟前", - "List":"列表", - "Communication":"沟通", + "View All": "查看全部", + "Your order is placed": "您的订单已经下达", + "min ago": "分钟前", + "List": "列表", + "Communication": "沟通", "Conversation List": "对话列表", "Task List": "任务列表", - "Knowledge Manager":"知识管理", - "Migrate agents from file repository to MongoDB":"将代理从文件存储库迁移到MongoDB", - "Setting":"设置", - "Start Migration":"开始迁移", - "Plugin":"插件", - "My Profile":"我的资料", - "Detail":"细节", - "System & Plugin Settings":"系统和插件设置", - "Disabled":"禁用", - "Enabled":"启用", - "Public":"公共", - "Private":"私人", - "Build":"构建", - "Train":"训练", - "Test":"测试", - "My Files":"我的文件", - "Upload":"上传", - "Recent Files":"最近的文件", - "Storage":"存储", - "0 GB (0%) of 1 GB used":"已使用1GB的0 GB (0%)", - "Provided by BotSharp":"由BotSharp提供", - "Provided by Pizza AI Assistant":"由Pizza AI Assistant提供", - "Upgrade Features":"升级功能", - "Upgrade":"升级", - "Files":"文件", - "Design":"设计", - "Google Drive":"谷歌驱动", - "Starred":"收藏", - "Trash":"回收站", - "Dropbox":"云文件管理", - "Open":"打开", - "Edit":"编辑", - "Rename":"重命名", - "Remove":"移除", - "Share Files":"共享文件", - "Share with me":"与我分享", - "Other Actions":"其他操作", - "Name":"名字", - "Date modified":"修改日期", - "Size":"大小", + "Knowledge Manager": "知识管理", + "Migrate agents from file repository to MongoDB": "将代理从文件存储库迁移到MongoDB", + "Setting": "设置", + "Start Migration": "开始迁移", + "Plugin": "插件", + "My Profile": "我的资料", + "Detail": "细节", + "System & Plugin Settings": "系统和插件设置", + "Disabled": "禁用", + "Enabled": "启用", + "Public": "公共", + "Private": "私人", + "Build": "构建", + "Train": "训练", + "Test": "测试", + "My Files": "我的文件", + "Upload": "上传", + "Recent Files": "最近的文件", + "Storage": "存储", + "0 GB (0%) of 1 GB used": "已使用1GB的0 GB (0%)", + "Provided by BotSharp": "由BotSharp提供", + "Provided by Pizza AI Assistant": "由Pizza AI Assistant提供", + "Upgrade Features": "升级功能", + "Upgrade": "升级", + "Files": "文件", + "Design": "设计", + "Google Drive": "谷歌驱动", + "Starred": "收藏", + "Trash": "回收站", + "Dropbox": "云文件管理", + "Open": "打开", + "Edit": "编辑", + "Rename": "重命名", + "Remove": "移除", + "Share Files": "共享文件", + "Share with me": "与我分享", + "Other Actions": "其他操作", + "Name": "名字", + "Date modified": "修改日期", + "Size": "大小", "Images": "图片", - "Video":"视频", - "Music":"音乐", - "Document":"文档", - "Others":"其他", - "Another action":"其他操作", - "Something else here":"一些别的东西在这里", - "Search for ...":"搜索 ...", - "Status":"状态", - "Completed":"完成", - "Select Channel":"选择通道", - "Live Chat":"即时聊天", - "Phone":"电话", - "Email":"邮箱", - "Filter":"过滤器", - "Title":"标题", - "User Name":"用户名", - "Channel":"通道", - "Posted Date":"发布日期", - "Last Date":"最后日期", - "Action":"操作", - "Active":"活跃中", - "Description":"描述", - "Details":"细节", - "Updated Date":"更新日期", - "Showing":"显示", - "of":"的", - "to":"到", - "entries":"条目", - "View":"查看", - "Install":"安装", - "Dialogs":"对话框", - "Agent Overview":"代理概述", - "Save Agent":"保存代理", - "Conversation Detail":"谈话细节", - "closed":"关闭", - "webchat":"聊天", - "User":"用户", - "Location":"位置", - "Delete Conversation":"删除对话", - "Design & Developed by open source community":"由开源社区设计和开发", - "Personal Information":"个人信息", - "First Name":"名", - "Last Name":"姓", - "Account Origin":"账户起源", - "Update Date":"更新日期", - "Create Date":"创建日期", - "Active now":"正在工作", - "Reset":"重置" -} \ No newline at end of file + "Video": "视频", + "Music": "音乐", + "Document": "文档", + "Others": "其他", + "Another action": "其他操作", + "Something else here": "一些别的东西在这里", + "Search for ...": "搜索 ...", + "Status": "状态", + "Completed": "完成", + "Select Channel": "选择通道", + "Live Chat": "即时聊天", + "Phone": "电话", + "Email": "邮箱", + "Filter": "过滤器", + "Title": "标题", + "User Name": "用户名", + "Channel": "通道", + "Posted Date": "发布日期", + "Last Date": "最后日期", + "Action": "操作", + "Active": "活跃中", + "Description": "描述", + "Details": "细节", + "Updated Date": "更新日期", + "Showing": "显示", + "of": "的", + "to": "到", + "entries": "条目", + "View": "查看", + "Install": "安装", + "Dialogs": "对话框", + "Agent Overview": "代理概述", + "Save Agent": "保存代理", + "Conversation Detail": "谈话细节", + "closed": "关闭", + "webchat": "聊天", + "User": "用户", + "Location": "位置", + "Delete Conversation": "删除对话", + "Design & Developed by open source community": "由开源社区设计和开发", + "Personal Information": "个人信息", + "First Name": "名", + "Last Name": "姓", + "Account Origin": "账户起源", + "Update Date": "更新日期", + "Create Date": "创建日期", + "Active now": "正在工作", + "Reset": "重置", + "Agent Testing": "Agent 测试", + "Test Suites": "测试套件", + "Test Suite": "测试套件", + "Test Cases": "测试用例", + "Test Case": "测试用例", + "Test Run": "测试运行", + "Run History": "运行历史", + "Case Results": "用例结果", + "Suite Settings": "套件设置", + "New Test Suite": "新建测试套件", + "New Test Case": "新建测试用例", + "Edit Test Case": "编辑测试用例", + "Record a Draft Case": "录制草稿用例", + "Turns": "对话轮次", + "Turn": "第几轮", + "Case Assertions": "整案断言", + "Assertions": "断言", + "Initial States": "初始 state", + "Tool Mocks": "工具 mock", + "Observed Tool Calls": "实际工具调用", + "New Suite": "新建套件", + "New Case": "新建用例", + "Add Turn": "添加轮次", + "Add Assertion": "添加断言", + "Add State": "添加 state", + "Add State Write": "添加 state 写入", + "Add Mock": "添加 mock", + "Run All Enabled": "运行全部启用用例", + "Run Selected": "运行选中用例", + "Cancel Run": "取消运行", + "Re-run Failures": "重跑失败用例", + "Record from Conversation": "从会话录制", + "Record": "录制", + "Refresh": "刷新", + "Retry": "重试", + "Save": "保存", + "Create": "创建", + "Cancel": "取消", + "Close": "关闭", + "Delete": "删除", + "Enable": "启用", + "Disable": "停用", + "Back": "返回", + "Back to Suite": "返回套件", + "Back to Suites": "返回套件列表", + "Show details": "展开详情", + "Hide details": "收起详情", + "Clear selection": "清除选择", + "All Agents": "全部 Agent", + "Select Agent": "选择 Agent", + "Select all cases": "全选用例", + "Select case {name}": "选择用例 {name}", + "Edit case": "编辑用例", + "Delete case": "删除用例", + "Enable case": "启用用例", + "Disable case": "停用用例", + "View run": "查看运行", + "Remove turn": "删除该轮", + "Move turn up": "上移该轮", + "Move turn down": "下移该轮", + "Remove assertion": "删除该断言", + "Remove state": "删除该 state", + "Remove mock": "删除该 mock", + "Source": "来源", + "Scope": "范围", + "Started": "开始时间", + "Duration": "耗时", + "Result": "结果", + "Type": "类型", + "Target": "目标", + "Expected": "期望值", + "Actual": "实际值", + "Message": "说明", + "Function": "函数", + "Outcome": "处理结果", + "Args": "入参", + "Key": "键", + "Value": "值", + "Total": "总数", + "Mocks": "mock 数", + "Mock": "mock", + "Run": "运行", + "Fatal": "致命", + "Global": "全局", + "Active rounds": "有效轮数", + "Function name": "函数名", + "Result content": "返回内容", + "Stop completion": "中止本轮续写", + "User message": "用户消息", + "Agent output": "Agent 输出", + "Conversation id": "会话 id", + "Judge provider": "判官 provider", + "Judge model": "判官 model", + "Case timeout": "用例超时", + "Case timeout (seconds)": "用例超时（秒）", + "Extra allowed functions": "额外放行的函数", + "Force blocked functions": "强制阻断的函数", + "Unmocked tool policy": "未 mock 工具策略", + "Enabled (included in runs)": "启用（纳入运行）", + "Triggered by": "触发者", + "Args match (JSON subset, optional)": "入参匹配（JSON 子集，可选）", + "Call index (0-based, optional)": "调用序号（0 基，可选）", + "Enter suite name": "输入套件名称", + "Optional description": "可选描述", + "function name": "函数名", + "state key": "state 键名", + "An existing conversation with at least one tool call": "一个至少包含一次工具调用的已有会话", + "Pending": "等待中", + "Running": "运行中", + "Passed": "通过", + "Failed": "失败", + "Error": "异常", + "Cancelled": "已取消", + "Cancelling": "取消中", + "Mocked": "已 mock", + "Blocked": "已阻断", + "Pass": "通过", + "Fail": "不通过", + "Draft": "草稿", + "Recorded": "录制", + "Manual": "手写", + "Subset": "子集", + "All enabled": "全部启用", + "passed": "通过", + "failed": "失败", + "errored": "异常", + "Failed (assertion)": "失败（断言不通过）", + "Errored (never ran)": "异常（未跑完）", + "Harness error": "测试台错误", + "Partial run of": "本次只跑了", + "selected case(s).": "个选中的用例。", + "No test suites yet.": "还没有测试套件。", + "No test suites for this agent yet.": "这个 Agent 还没有测试套件。", + "No test cases in this suite yet.": "这个套件里还没有测试用例。", + "This suite has never been run.": "这个套件从未运行过。", + "No case results yet.": "还没有用例结果。", + "This run produced no case results.": "这次运行没有产生任何用例结果。", + "No turns yet. A case needs at least one.": "还没有轮次。一条用例至少要有一轮。", + "No case-level assertions.": "没有整案级断言。", + "No initial states.": "没有初始 state。", + "No assertions on this turn.": "这一轮没有断言。", + "The agent called no tools during this case.": "这条用例执行期间 Agent 没有调用任何工具。", + "No mocks. Every tool call this case makes will be blocked.": "没有 mock。这条用例发起的每一次工具调用都会被阻断。", + "A disabled suite cannot be run at all.": "停用的套件完全无法运行。", + "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.": "这个套件已停用。在设置里重新启用之前，服务端会拒绝任何运行请求。", + "llmJudge assertions always fail in P1 regardless of these two fields.": "P1 阶段 llmJudge 断言恒定判失败，与这两个字段是否配置无关。", + "Always fails in P1.": "P1 阶段恒定判失败。", + "One function name per line. These run for real during a test -- only list functions with no side effects.": "每行一个函数名。这些函数在测试期间会真实执行——只填没有副作用的函数。", + "One per line. Blocked even if allow-listed elsewhere.": "每行一个。即使在别处被放行也照样阻断。", + "Assertions checked right after this turn": "本轮结束后立即求值的断言", + "Evaluated once, after every turn has run.": "全部轮次跑完后求值一次。", + "Injected before the conversation starts. Active rounds -1 means it never expires.": "会话开始前注入。有效轮数为 -1 表示永不过期。", + "Every tool this case does not mock is blocked. If the agent needs a tool to move forward, mock it here.": "这条用例没有 mock 的工具一律被阻断。Agent 需要哪个工具才能往下走，就在这里给它加 mock。", + "State written when this mock is hit": "命中这个 mock 时写入的 state", + "Many tools pass data to later turns through state, not through their return value.": "很多工具是靠 state 而不是靠返回值把数据传给后续轮次的。", + "Any tool this case does not mock is blocked instead of executed. P1 has no other option.": "这条用例没有 mock 的工具会被阻断而不是执行。P1 阶段没有别的选项。", + "Recorded from conversation": "录制自会话", + "It may contain live customer data -- review before enabling.": "可能包含线上客户数据——启用前请先审阅。", + "Recorded from conversation {id}. May contain live customer data.": "录制自会话 {id}，可能包含线上客户数据。", + "Recording copies the raw conversation into the test store, including any phone numbers, addresses and tenant names it contains. The draft lands disabled -- review it before enabling.": "录制会把原始会话内容复制进测试库，其中包含的电话号码、地址、租户姓名也一并复制。生成的草稿默认停用——请审阅后再启用。", + "Errored cases never reached their assertions -- a timeout, a mock-seam failure, or a case with no turns. That is a harness problem, not an agent regression.": "异常的用例根本没走到断言——可能是超时、mock 接缝失效，或者用例没有轮次。这是测试台的问题，不是 Agent 回归。", + "This run is still going. Refreshing every couple of seconds.": "这次运行还在进行中，每隔几秒自动刷新。", + "The agent tried to call a tool this case does not mock, so it was blocked. This is often the root cause of the failure above.": "Agent 试图调用一个这条用例没有 mock 的工具，因此被阻断。这通常正是上面那个失败的根因。", + "Are you sure?": "确定吗？", + "Yes, delete it!": "确定删除", + "Yes, run it": "确定运行", + "Run this suite?": "运行这个套件？", + "Delete test suite \"{name}\"? You won't be able to revert this!": "删除测试套件“{name}”？此操作不可撤销！", + "Delete test case \"{name}\"? You won't be able to revert this!": "删除测试用例“{name}”？此操作不可撤销！", + "{count} case(s) will run against the live model. This costs real tokens and is not rate limited.": "将有 {count} 条用例真实调用模型运行。这会消耗真实 token，且没有任何限流。", + "Test suite created!": "测试套件已创建！", + "Test suite deleted!": "测试套件已删除！", + "Test case created!": "测试用例已创建！", + "Test case saved!": "测试用例已保存！", + "Test case deleted!": "测试用例已删除！", + "Suite settings saved!": "套件设置已保存！", + "Case enabled.": "用例已启用。", + "Case disabled.": "用例已停用。", + "Run queued!": "运行已入队！", + "Re-run queued!": "重跑已入队！", + "Cancellation requested.": "已请求取消。", + "Draft case recorded. Review it, then enable it.": "草稿用例已录制。请审阅后再启用。", + "Failed to load test suites. Please try again.": "加载测试套件失败，请重试。", + "Failed to load this test suite.": "加载这个测试套件失败。", + "Failed to load this test case.": "加载这条测试用例失败。", + "Failed to load the suite this case belongs to.": "加载这条用例所属的套件失败。", + "Failed to load this run.": "加载这次运行失败。", + "Failed to create test suite.": "创建测试套件失败。", + "Failed to delete test suite.": "删除测试套件失败。", + "Failed to save suite settings.": "保存套件设置失败。", + "Failed to save the test case.": "保存测试用例失败。", + "Failed to update the case.": "更新测试用例失败。", + "Failed to delete the test case.": "删除测试用例失败。", + "Failed to start the run.": "启动运行失败。", + "Failed to start the re-run.": "启动重跑失败。", + "Failed to cancel the run.": "取消运行失败。", + "Failed to record a draft from that conversation.": "从该会话录制草稿失败。", + "You do not have permission for this. Running and recording require an admin or root account.": "你没有权限执行此操作。运行和录制需要 admin 或 root 账号。", + "Fix these before saving:": "保存前请先修复：", + "Name is required.": "名称必填。", + "Add at least one turn -- a case with no turns errors when it runs.": "至少添加一轮——没有轮次的用例运行时会直接异常。", + "Turn {n}: the user message is empty.": "第 {n} 轮：用户消息为空。", + "Turn {n}, assertion {m}: {error}": "第 {n} 轮第 {m} 条断言：{error}", + "Case assertion {n}: {error}": "第 {n} 条整案断言：{error}", + "Initial state {n}: the key is empty.": "第 {n} 条初始 state：键名为空。", + "Mock {n}: the function name is empty.": "第 {n} 个 mock：函数名为空。", + "Mock {n}: the args match is not valid JSON.": "第 {n} 个 mock：入参匹配不是合法 JSON。", + "Mock {n}: the call index is 0-based and cannot be negative.": "第 {n} 个 mock：调用序号从 0 开始，不能为负。", + "Mock {n}, state write {m}: the key is empty.": "第 {n} 个 mock 的第 {m} 条 state 写入：键名为空。", + "Assertion type is required.": "断言类型必填。", + "Assertion \"{type}\" needs an expected value.": "断言“{type}”需要填期望值。", + "Assertion \"{type}\" needs a target.": "断言“{type}”需要填目标。", + "Assertion \"stateEquals\" needs an expected value, otherwise it can never pass.": "断言“stateEquals”必须填期望值，否则它永远不可能通过。", + "\"{pattern}\" is not a valid regular expression.": "“{pattern}”不是合法的正则表达式。", + "The args match on an assertion is not valid JSON.": "某条断言的入参匹配不是合法 JSON。", + "{ms} ms": "{ms} 毫秒", + "{seconds} s": "{seconds} 秒", + "Not configured": "未配置", + "This provider has no chat-capable model registered.": "该 provider 下没有注册任何支持 chat 的模型。", + "Case": "用例", + "Models": "模型", + "models": "个模型", + "Model Comparison": "模型对比", + "Running every enabled case": "本次运行全部启用用例", + "Running the selected cases": "本次运行选中的用例", + "Use the agent's own model": "改用 agent 自己的模型", + "Pick nothing to run once on whatever model each agent is configured with. Pick two or more to run the whole set once per model and compare them side by side.": "不选就按各 agent 自己配置的模型跑一遍。选两个及以上，整套用例会在每个模型上各跑一遍，结果并排对比。", + "No chat-capable model is registered, so this run will use the agent's own configuration.": "没有注册任何支持 chat 的模型，本次运行会使用 agent 自己的配置。", + "{count} execution(s) will run against live models. This costs real tokens and is not rate limited.": "将真实调用模型执行 {count} 次。这会消耗真实 token，且没有任何限流。", + "Durations are wall-clock for the whole case, including mocked tool calls, so they are comparable between models but are not a pure model-latency measurement.": "耗时是整条用例的墙钟时间，包含 mock 工具调用在内，所以模型之间可比，但不是纯粹的模型延迟。", + "AI extraction model": "AI 抽取模型", + "Do not use AI (one case for the whole conversation)": "不使用 AI（整段会话生成一条用例）", + "With a model picked, the conversation is split into one case per scenario it covers. The model only decides where to split and what to name each case -- mocks, assertions and state still come verbatim from the conversation.": "选了模型后，会话会按覆盖的场景切成多条用例。模型只决定切在哪、每条叫什么——mock、断言和 state 仍旧逐字来自真实会话。", + "AI extraction additionally sends the user messages and tool names to {provider}. Tool arguments and results are not sent.": "AI 抽取会额外把用户消息和函数名发给 {provider}。函数入参与返回内容不外发。", + "{count} draft cases recorded. Review them, then enable them.": "已录制 {count} 条草稿用例。请逐条审阅后再启用。" +} diff --git a/src/lib/services/agent-test-service.js b/src/lib/services/agent-test-service.js index 9f0d78cb..cc41d747 100644 --- a/src/lib/services/agent-test-service.js +++ b/src/lib/services/agent-test-service.js @@ -127,11 +127,14 @@ export async function deleteCase(id) { * does not wait for the run to finish. * @param {string} suiteId * @param {string[]?} [caseIds] - Only run these cases; omit/empty to run every enabled case in the suite. + * @param {import('$agentTestTypes').TestModel[]?} [models] - Run every case once per model listed here; + * omit/empty to run a single pass using each agent's own LlmConfig. N models means N times the + * executions and N times the token cost. * @returns {Promise} */ -export async function triggerRun(suiteId, caseIds = null) { +export async function triggerRun(suiteId, caseIds = null, models = null) { const url = endpoints.agentTestSuiteRunUrl.replace("{id}", suiteId); - const response = await axios.post(url, { caseIds: caseIds }); + const response = await axios.post(url, { caseIds: caseIds, models: models }); return response.data; } @@ -170,18 +173,25 @@ export async function cancelRun(id) { } /** - * Record a draft test case from a real conversation. The returned case comes - * back with enabled=false -- it must be reviewed and explicitly enabled - * before it joins a normal run. + * Record one or more draft test cases from a real conversation. Every returned + * case comes back with enabled=false -- each must be reviewed and explicitly + * enabled before it joins a normal run. * @param {string} conversationId * @param {string} suiteId - * @returns {Promise} + * @param {import('$agentTestTypes').TestModel?} [model] - Use this model to split the + * conversation into one or more scenarios, each becoming its own case. Omit to use the + * deterministic recorder, which always produces exactly one case and calls no model. + * The model only decides where to cut and what to name each case; mocks, assertions and + * state still come verbatim from the conversation. Passing a model sends the conversation's + * user messages and tool NAMES to that vendor (tool arguments and results are withheld). + * @returns {Promise} */ -export async function recordCase(conversationId, suiteId) { +export async function recordCases(conversationId, suiteId, model = null) { const url = endpoints.agentTestRecordUrl; const response = await axios.post(url, { conversationId: conversationId, - suiteId: suiteId + suiteId: suiteId, + model: model }); return response.data; } diff --git a/src/routes/page/agent-test/+page.svelte b/src/routes/page/agent-test/+page.svelte index f4a598ed..9e3427be 100644 --- a/src/routes/page/agent-test/+page.svelte +++ b/src/routes/page/agent-test/+page.svelte @@ -10,6 +10,7 @@ import Select from '$lib/common/dropdowns/Select.svelte'; import { getAgentOptions } from '$lib/services/agent-service.js'; import { getSuites, createSuite, deleteSuite } from '$lib/services/agent-test-service.js'; + import { t } from '$lib/helpers/utils/agent-test.js'; const duration = 3000; const nameMaxLength = 200; @@ -73,7 +74,7 @@ suites = res || []; }).catch(() => { suites = []; - loadErrorText = 'Failed to load test suites. Please try again.'; + loadErrorText = t('Failed to load test suites. Please try again.'); }); } @@ -106,7 +107,10 @@ /** @param {string} suiteId */ function goToSuite(suiteId) { - goto(`page/agent-test/${suiteId}`); + // Absolute: a relative path resolves against the current URL, so the same + // call from /page/agent-test/ (with the trailing slash) would land on + // /page/agent-test/page/agent-test/. + goto(`/page/agent-test/${suiteId}`); } function openCreateModal() { @@ -138,7 +142,7 @@ }).then(() => { isCreateModalOpen = false; isComplete = true; - successText = 'Test suite created!'; + successText = t('Test suite created!'); setTimeout(() => { isComplete = false; successText = ''; @@ -146,7 +150,7 @@ refreshSuites(); }).catch(() => { isError = true; - errorText = 'Failed to create test suite.'; + errorText = t('Failed to create test suite.'); setTimeout(() => { isError = false; errorText = ''; @@ -174,12 +178,13 @@ function openDeleteModal(suite) { // @ts-ignore Swal.fire({ - title: 'Are you sure?', - text: `Delete test suite "${suite.name}"? You won't be able to revert this!`, + title: t('Are you sure?'), + text: t('Delete test suite "{name}"? You won\'t be able to revert this!', { name: suite.name }), icon: 'warning', customClass: 'custom-modal', showCancelButton: true, - confirmButtonText: 'Yes, delete it!' + cancelButtonText: t('Cancel'), + confirmButtonText: t('Yes, delete it!') }).then((result) => { if (result.value) { handleDeleteSuite(suite.id); @@ -192,7 +197,7 @@ isLoading = true; deleteSuite(suiteId).then(() => { isComplete = true; - successText = 'Test suite deleted!'; + successText = t('Test suite deleted!'); setTimeout(() => { isComplete = false; successText = ''; @@ -200,7 +205,7 @@ return loadSuites(); }).catch(() => { isError = true; - errorText = 'Failed to delete test suite.'; + errorText = t('Failed to delete test suite.'); setTimeout(() => { isError = false; errorText = ''; @@ -251,7 +256,7 @@ class="btn btn-soft-secondary w-100" data-bs-toggle="tooltip" data-bs-placement="bottom" - title="Filter" + title={$_('Filter')} onclick={() => applyAgentFilter()} > @@ -264,7 +269,7 @@ class="btn btn-warning w-100" data-bs-toggle="tooltip" data-bs-placement="bottom" - title="Reset" + title={$_('Reset')} onclick={() => resetAgentFilter()} > @@ -321,11 +326,11 @@
    -
  • +
  • +
submitCreateSuite(e)}> diff --git a/src/routes/page/agent-test/[suiteId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/+page.svelte new file mode 100644 index 00000000..fe4ac4b4 --- /dev/null +++ b/src/routes/page/agent-test/[suiteId]/+page.svelte @@ -0,0 +1,1124 @@ + + + + + + + +{#if loadErrorText} +
+
+ +
+
+{:else if suite} +
+
+
+
+
+
+
+ {suite.name} + {#if suite.enabled} + {$_('Enabled')} + {:else} + {$_('Disabled')} + {/if} +
+

+ {$_('Agent')}: {agentName(suite.agentId)} + | + {$_('Case timeout')}: {suite.caseTimeoutSeconds}s +

+ {#if suite.description} +

{suite.description}

+ {/if} +
+
+ + + + + {#if selectedCaseIds.length > 0} + + {/if} + +
+
+ {#if !suite.enabled} + + {/if} +
+
+
+
+ +
+
+
+
+
+
{$_('Test Cases')} ({cases.length})
+ {#if selectedCaseIds.length > 0} + + {/if} +
+
+
+ {#if cases.length === 0} +
+

{$_('No test cases in this suite yet.')}

+
+ + +
+
+ {:else} +
+ + + + + + + + + + + + + + + {#each cases as testCase (testCase.id)} + {@const assertionCount = (testCase.assertions?.length || 0) + + (testCase.turns || []).reduce((sum, t) => sum + (t.assertions?.length || 0), 0)} + + + + + + + + + + + {/each} + +
+ toggleAllCases()} + /> + {$_('Name')}{$_('Turns')}{$_('Mocks')}{$_('Assertions')}{$_('Source')}{$_('Enabled')}{$_('Action')}
+ toggleCase(testCase.id)} + /> + + + {testCase.turns?.length || 0}{testCase.mocks?.length || 0}{assertionCount} + {#if testCase.sourceConversationId} + + {$_('Recorded')} + + {:else} + {$_('Manual')} + {/if} + + {#if testCase.enabled} + {$_('Enabled')} + {:else} + {$_('Draft')} + {/if} + +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+
+
+ {/if} +
+
+
+
+ +
+
+
+
+
+
{$_('Run History')}
+ +
+
+
+ {#if runs.length === 0} +

{$_('This suite has never been run.')}

+ {:else} +
+ + + + + + + + + + + + + {#each runs as run (run.id)} + + + + + + + + + {/each} + +
{$_('Status')}{$_('Result')}{$_('Scope')}{$_('Started')}{$_('Duration')}{$_('Action')}
+ {$_(run.status)} + {#if run.cancelRequested && !isTerminalStatus(run.status)} + {$_('Cancelling')} + {/if} + + {run.passedCount} {$_('passed')}, + {run.failedCount} {$_('failed')}, + {run.errorCount} {$_('errored')} + / {run.totalCount} + + {#if run.caseIds && run.caseIds.length > 0} + {$_('Subset')} ({run.caseIds.length}) + {:else} + {$_('All enabled')} + {/if} + {#if run.models && run.models.length > 0} + `${m.provider}/${m.model}`).join('\n')} + > + {run.models.length} {$_('models')} + + {/if} + {formatDateTime(run.startedAt)}{runDuration(run)} +
    +
  • + +
    • + {#if !isTerminalStatus(run.status)} +
  • + +
    • + {/if} +
+
+
+ {/if} +
+
+
+
+{/if} + +{#if isSettingsOpen} + +
+{/if} + +{#if isRunModalOpen} + +
+{/if} + +{#if isRecordOpen} + +
+{/if} diff --git a/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte new file mode 100644 index 00000000..a1fd9b04 --- /dev/null +++ b/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte @@ -0,0 +1,775 @@ + + +{#snippet assertionRows(list, idPrefix)} + {#each list as assertion, i} +
+
+
+ + + {#if assertion.type === 'llmJudge'} +
{$_('Always fails in P1.')}
+ {/if} +
+
+ + +
+
+ + +
+
+
+ + +
+ +
+ {#if assertion.type === 'toolCalled'} +
+ + +
+ {/if} +
+
+ {/each} +{/snippet} + +{#snippet stateRows(list, idPrefix)} + {#each list as state, i} +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+
+ +
+
+ {/each} +{/snippet} + + + + + + + + {#each mockTargets as target} + + {/each} + + +{#if loadErrorText} +
+
+ +
+
+{:else} +
+
+
+
+
+
{isNew ? $_('New Test Case') : $_('Edit Test Case')}
+
+ + +
+
+
+ + {#if validationErrors.length > 0} +
+ +
+ {/if} + +
+
+
+ + +
+
+
+ + +
+
+ {#if form.sourceConversationId} +
+ +
+ {/if} +
+
+ {$_('Unmocked tool policy')}: {form.unmockedToolPolicy}. + {$_('Any tool this case does not mock is blocked instead of executed. P1 has no other option.')} +
+
+
+
+
+
+
+ +
+
+
+
+
+
{$_('Turns')} ({form.turns.length})
+ +
+
+
+ {#each form.turns as turn, i (i)} +
+
+
{$_('Turn')} {i + 1}
+
+ + + +
+
+
+ + +
+
+ {$_('Assertions checked right after this turn')} ({turn.assertions.length}) + +
+ {@render assertionRows(turn.assertions, `turn-${i}-assertion`)} +
+ {/each} + {#if form.turns.length === 0} +

{$_('No turns yet. A case needs at least one.')}

+ {/if} +
+
+
+
+ +
+
+
+
+
+
{$_('Case Assertions')} ({form.assertions.length})
+ +
+
+
+

{$_('Evaluated once, after every turn has run.')}

+ {@render assertionRows(form.assertions, 'case-assertion')} + {#if form.assertions.length === 0} +

{$_('No case-level assertions.')}

+ {/if} +
+
+
+
+ +
+
+
+
+
+
{$_('Initial States')} ({form.initialStates.length})
+ +
+
+
+

{$_('Injected before the conversation starts. Active rounds -1 means it never expires.')}

+ {@render stateRows(form.initialStates, 'initial-state')} + {#if form.initialStates.length === 0} +

{$_('No initial states.')}

+ {/if} +
+
+
+
+ +
+
+
+
+
+
{$_('Tool Mocks')} ({form.mocks.length})
+ +
+
+
+

+ {$_('Every tool this case does not mock is blocked. If the agent needs a tool to move forward, mock it here.')} +

+ {#each form.mocks as mock, i (i)} +
+
+
{$_('Mock')} {i + 1}
+ +
+
+
+ + +
+
+ + +
+
+
+ + +
+
+
+ + +
+
+ + +
+
+
+ + {$_('State written when this mock is hit')} ({mock.stateWrites?.length || 0}). + {$_('Many tools pass data to later turns through state, not through their return value.')} + + +
+ {@render stateRows(mock.stateWrites || [], `mock-${i}-state`)} +
+
+
+ {/each} + {#if form.mocks.length === 0} +

{$_('No mocks. Every tool call this case makes will be blocked.')}

+ {/if} +
+
+
+
+ +
+
+
+
+ + +
+
+
+
+{/if} diff --git a/src/routes/page/agent-test/run/[runId]/+page.svelte b/src/routes/page/agent-test/run/[runId]/+page.svelte new file mode 100644 index 00000000..464b90cb --- /dev/null +++ b/src/routes/page/agent-test/run/[runId]/+page.svelte @@ -0,0 +1,530 @@ + + +{#snippet assertionTable(list)} +
+ + + + + + + + + + + + + {#each list as assertion, i (i)} + + + + + + + + + {/each} + +
{$_('Result')}{$_('Type')}{$_('Target')}{$_('Expected')}{$_('Actual')}{$_('Message')}
+ {#if assertion.passed} + {$_('Pass')} + {:else} + {$_('Fail')} + {/if} + {assertion.type}{assertion.target || '--'}{assertion.expected || '--'}{assertion.actual || '--'}{assertion.message || '--'}
+
+{/snippet} + + + + + + +{#if loadErrorText} +
+
+ +
+
+{:else if run} +
+
+
+
+
+
+
+ {$_('Run')} {run.id} + {$_(run.status)} + {#if run.cancelRequested && !isFinished} + {$_('Cancelling')} + {/if} +
+

+ {$_('Started')}: {formatDateTime(run.startedAt)} + | + {$_('Completed')}: {formatDateTime(run.completedAt)} + {#if run.triggeredBy} + | + {$_('Triggered by')}: {run.triggeredBy} + {/if} +

+ {#if run.caseIds && run.caseIds.length > 0} +

+ {$_('Partial run of')} {run.caseIds.length} {$_('selected case(s).')} +

+ {/if} +
+
+ + + {#if !isFinished} + + {:else if failedCaseIds.length > 0} + + {/if} +
+
+
+
+
+
+
+

{run.totalCount}

+ {$_('Total')} +
+
+
+
+

{run.passedCount}

+ {$_('Passed')} +
+
+
+
+

{run.failedCount}

+ {$_('Failed (assertion)')} +
+
+
+
+

{run.errorCount}

+ {$_('Errored (never ran)')} +
+
+
+ {#if run.errorCount > 0} + + {/if} + {#if !isFinished} +

+ {$_('This run is still going. Refreshing every couple of seconds.')} +

+ {/if} +
+
+
+
+ + {#if isComparison} +
+
+
+
+
{$_('Model Comparison')}
+
+
+
+ + + + + {#each modelColumns as col (col.key)} + + {/each} + + + + {#each comparisonRows as row (row.caseId)} + + + {#each modelColumns as col (col.key)} + {@const cell = row.cells[col.key]} + + {/each} + + {/each} + + + + + {#each modelSummaries as summary (summary.key)} + + {/each} + + +
{$_('Case')} +
{col.model}
+
{col.provider}
+
{row.caseName} + {#if cell} + {$_(cell.status)} + {formatDuration(cell.durationMs)} + {:else} + -- + {/if} +
{$_('Total')} +
{summary.passed}/{summary.total} {$_('passed')}
+
{formatDuration(summary.totalMs)}
+
+
+

+ {$_('Durations are wall-clock for the whole case, including mocked tool calls, so they are comparable between models but are not a pure model-latency measurement.')} +

+
+
+
+
+ {/if} + +
+
+
+
+
{$_('Case Results')} ({results.length})
+
+
+ {#if results.length === 0} +

+ {isFinished ? $_('This run produced no case results.') : $_('No case results yet.')} +

+ {:else} + {#each results as result (result.id)} + {@const isExpanded = expandedIds.includes(result.id)} +
+
+
+ {$_(result.status)} + {result.caseName} + {#if result.model} + + {result.model} + {/if} + {formatDuration(result.durationMs)} + {#if result.conversationId} + {result.conversationId} + {/if} +
+ +
+ + {#if result.error} +
+ +
+ {/if} + + {#if isExpanded} +
+ {#each result.turns as turn (turn.index)} +
+
{$_('Turn')} {turn.index + 1}
+
+
{$_('User')}
+
{turn.userMessage}
+
+
+
{$_('Agent output')}
+
{turn.output || '--'}
+
+ {#if turn.assertions?.length > 0} + {@render assertionTable(turn.assertions)} + {:else} +

{$_('No assertions on this turn.')}

+ {/if} +
+ {/each} + + {#if result.assertions?.length > 0} +
+
{$_('Case Assertions')}
+ {@render assertionTable(result.assertions)} +
+ {/if} + +
+
{$_('Observed Tool Calls')} ({result.observedToolCalls?.length || 0})
+ {#if result.observedToolCalls?.length > 0} +
+ + + + + + + + + + + + {#each result.observedToolCalls as call, i (i)} + + + + + + + + {/each} + +
{$_('Turn')}{$_('Function')}{$_('Outcome')}{$_('Args')}{$_('Result')}
{call.turnIndex + 1}{call.functionName} + {#if call.outcome === 'Blocked'} + + {$_('Blocked')} + + {:else} + {$_(call.outcome)} + {/if} + {call.argsJson || '--'}{call.resultContent || '--'}
+
+ {:else} +

{$_('The agent called no tools during this case.')}

+ {/if} +
+
+ {/if} +
+ {/each} + {/if} +
+
+
+
+{/if} diff --git a/svelte.config.js b/svelte.config.js index 220e2ac9..130d3715 100644 --- a/svelte.config.js +++ b/svelte.config.js @@ -77,7 +77,10 @@ const config = { "/page/knowledge-base/documents", "/page/knowledge-base/dictionary", "/page/knowledge-base/[embed]/[embedType]", - "/page/agent-test" + "/page/agent-test", + "/page/agent-test/[suiteId]", + "/page/agent-test/[suiteId]/case/[caseId]", + "/page/agent-test/run/[runId]" ] } }, From 0048702955260f61a4ce8d2d0cdb07248a4b9f57 Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Tue, 18 Aug 2026 15:07:54 +0800 Subject: [PATCH 4/4] fix: show why a run failed, and stop runs that can only fail A run that stops before executing anything produces no case results, so the per-case error alert had nothing to render and the page showed status Error, 0/0/0/0, and "This run produced no case results." -- with the actual reason nowhere on screen. It now renders run.error above the (empty) results section, replaces that misleading empty-state line with a pointer to it, and repeats the reason inline in the suite's run history so it is visible without opening the run at all. The commonest way to reach that state was a trap this UI created: disabled cases were selectable and "Run Selected" happily queued them, which the executor then rejected wholesale. Selection counts now consider only enabled cases, a mixed selection warns how many will be skipped, an all-disabled selection is blocked with the reason, and only enabled ids are sent. Also translates the remaining Chinese in agentTestTypes.js to English. Only comments and JSDoc -- zh.json holds translations, which are data and untouched. Co-Authored-By: Claude Opus 5 --- src/lib/helpers/types/agentTestTypes.js | 147 ++++++++++-------- src/lib/langs/en.json | 6 +- src/lib/langs/zh.json | 6 +- .../page/agent-test/[suiteId]/+page.svelte | 38 ++++- .../page/agent-test/run/[runId]/+page.svelte | 14 +- 5 files changed, 139 insertions(+), 72 deletions(-) diff --git a/src/lib/helpers/types/agentTestTypes.js b/src/lib/helpers/types/agentTestTypes.js index e6785ebb..9c9cf950 100644 --- a/src/lib/helpers/types/agentTestTypes.js +++ b/src/lib/helpers/types/agentTestTypes.js @@ -5,82 +5,91 @@ * @property {string} name * @property {string?} description * @property {boolean} enabled - * @property {string?} judgeProvider - llmJudge 断言用的 provider。P1 中 llmJudge 恒定判失败，与是否配置无关。 + * @property {string?} judgeProvider - Provider for llmJudge assertions. In P1 llmJudge always fails, + * whether or not this is configured. * @property {string?} judgeModel - * @property {string[]} extraAllowedFunctions - 在默认控制流白名单之外额外放行的函数名。 - * @property {string[]} forceBlockedFunctions - 强制阻断的函数名，优先级高于白名单。 + * @property {string[]} extraAllowedFunctions - Functions let through on top of the default + * control-flow allow list. + * @property {string[]} forceBlockedFunctions - Functions blocked outright; wins over the allow list. * @property {number} caseTimeoutSeconds * @property {string} createDate - ISO date string. * @property {string} updateDate - ISO date string. */ /** - * POST /agent-test/suites 与 PUT /agent-test/suites/{id} 的请求体。 + * Body for POST /agent-test/suites and PUT /agent-test/suites/{id}. * - * **PUT 是整份替换，不是 PATCH。** 提交前必须先 GET 出完整 suite，只改用户动过的字段， - * 其余原样带回去 —— agentId/name 漏传时后端会兜底保留原值，但 caseTimeoutSeconds/ - * extraAllowedFunctions/forceBlockedFunctions 漏传会被后端重置为默认值（120 / [] / []）。 + * **PUT is a full replace, not a PATCH.** GET the complete suite first, change only what the user + * touched, and send everything else back unchanged -- a missing agentId/name falls back to the + * stored value, but a missing caseTimeoutSeconds/extraAllowedFunctions/forceBlockedFunctions is + * reset to the backend default (120 / [] / []). * * @typedef {Object} AgentTestSuiteUpsertRequest * @property {string} agentId * @property {string} name * @property {string?} [description] - * @property {boolean?} [enabled] - 省略或 null = 保持不变（新建时视为 true）；显式传 true/false 才会改变启用状态。 + * @property {boolean?} [enabled] - Omitted or null = leave unchanged (treated as true on create); + * only an explicit true/false flips the enabled state. * @property {string?} [judgeProvider] * @property {string?} [judgeModel] * @property {string[]} extraAllowedFunctions * @property {string[]} forceBlockedFunctions - * @property {number} caseTimeoutSeconds - 默认 120。 + * @property {number} caseTimeoutSeconds - Defaults to 120. */ /** * @typedef {Object} TestTurn * @property {number} index * @property {string} userMessage - * @property {TestAssertion[]} assertions - 本轮级断言（区别于用例整案级的 AgentTestCase.assertions）。 + * @property {TestAssertion[]} assertions - Turn-level assertions, as opposed to the case-level + * AgentTestCase.assertions. */ /** * @typedef {Object} TestState * @property {string} key * @property {string} value - * @property {number} [activeRounds] - 默认 -1（永久有效）。 + * @property {number} [activeRounds] - Defaults to -1 (never expires). * @property {boolean} [global] */ /** * @typedef {Object} TestToolMock * @property {string} functionName - * @property {string?} [argsMatchJson] - 可选：入参子集匹配（JSON 字符串），用于同名工具多次调用给不同返回。 - * @property {number?} [callIndex] - 可选：只命中第 N 次调用（0 基）。 - * @property {string} [resultContent] - 假返回，写入 message.Content，通常是 JSON 文本。 - * @property {boolean} [stopCompletion] - 模拟"中止本轮 LLM 续写"的真实行为。 - * @property {TestState[]?} [stateWrites] - mock 命中时一并写入的 state；很多 IFunctionCallback 完全靠 state 跨轮传数据，只 mock 返回值会让后续函数读不到 state。 + * @property {string?} [argsMatchJson] - Optional argument-subset match (a JSON string), for giving + * different returns to repeated calls of the same tool. + * @property {number?} [callIndex] - Optional: match only the Nth call (0-based). + * @property {string} [resultContent] - The faked return, written to message.Content; usually JSON text. + * @property {boolean} [stopCompletion] - Reproduces a real tool's "stop this turn's LLM completion". + * @property {TestState[]?} [stateWrites] - State written when this mock is hit. Many + * IFunctionCallback implementations pass data across turns purely through state, so mocking only + * the return value leaves later functions unable to read what they expect. */ /** - * 一条断言。 + * One assertion. * - * 后端保存时校验的必填字段（不满足会 400，但错误信息不指明是哪一条 —— 表单应在提交前自行挡住）： - * - outputContains / outputNotContains / outputRegex / routedToAgent / llmJudge：`expected` 必填。 - * - toolCalled / toolNotCalled / stateEquals：`target` 必填。 + * Fields the backend enforces on save (a violation is a 400, but the message does not say which + * assertion -- the form should block it first): + * - outputContains / outputNotContains / outputRegex / routedToAgent / llmJudge: `expected` required. + * - toolCalled / toolNotCalled / stateEquals: `target` required. * - * 注意一个后端校验没覆盖、但求值时会咬人的点：`stateEquals` 后端保存时只强制 `target`， - * `expected` 留空并不会 400 —— 但求值时会拿 state 的实际值去比较 null/空串，稳定判不过 - * （不是报错，是永远失败）。表单应比后端的 400 校验更严格，把 `stateEquals` 的 `expected` - * 也当必填处理。 + * One trap the backend does NOT cover but evaluation does: `stateEquals` only requires `target` on + * save, so an empty `expected` saves fine -- and then compares the real state value against + * null/empty on every run and can never pass. It fails rather than errors, so nothing points at it. + * The form is deliberately stricter and treats `expected` as required there too. * - * `llmJudge` 在 P1 永远判失败（后端返回 `"llmJudge is not available in P1"`），与 minScore - * 或 suite 是否配置了 judgeProvider/judgeModel 无关；表单可以让用户选它，但要在旁边标注 - * "P1 不可用，会判失败"。 + * `llmJudge` always fails in P1 (the backend returns "llmJudge is not available in P1"), regardless + * of minScore or whether the suite configured judgeProvider/judgeModel. The form may offer it, but + * says so next to it. * * @typedef {Object} TestAssertion * @property {string} type - outputContains | outputNotContains | outputRegex | toolCalled | toolNotCalled | stateEquals | routedToAgent | llmJudge - * @property {string?} [target] - 函数名 / state key / agent 名。 - * @property {string?} [expected] - 期望值 / 正则 / 判官标准。 - * @property {string?} [argsMatchJson] - toolCalled 的入参子集匹配（JSON 字符串）。 - * @property {number?} [minScore] - llmJudge 通过阈值。 - * @property {boolean} [fatal] - 失败则中止该用例后续轮。 + * @property {string?} [target] - Function name / state key / agent name. + * @property {string?} [expected] - Expected value / regex / judging criteria. + * @property {string?} [argsMatchJson] - Argument-subset match for toolCalled (a JSON string). + * @property {number?} [minScore] - Pass threshold for llmJudge. + * @property {boolean} [fatal] - On failure, abort the remaining turns of this case. */ /** @@ -88,35 +97,39 @@ * @property {string} id * @property {string} suiteId * @property {string} name - * @property {boolean} enabled - 录制生成的草稿用例落库为 false，需人工审阅后手动启用才会加入正式跑批。 - * @property {TestTurn[]} turns - 长度 1 即单轮用例。 - * @property {TestAssertion[]} assertions - 整案级断言：全部轮跑完后求值。 - * @property {TestState[]} initialStates - 会话开始前注入，映射 BotSharp 的 MessageState。 + * @property {boolean} enabled - Recorded drafts land as false; a human has to review and enable them + * before they join a normal run. + * @property {TestTurn[]} turns - Length 1 is a single-turn case. + * @property {TestAssertion[]} assertions - Case-level assertions, evaluated after every turn has run. + * @property {TestState[]} initialStates - Injected before the conversation starts; maps to BotSharp's + * MessageState. * @property {TestToolMock[]} mocks - * @property {string} unmockedToolPolicy - P1 只接受 "Block"；提交 "Passthrough" 会 400（"Passthrough is not supported in P1"）。表单不要提供这个选项。 - * @property {string?} sourceConversationId - 录制来源会话，便于回溯；手写用例为 null。 + * @property {string} unmockedToolPolicy - P1 accepts only "Block"; sending "Passthrough" is a 400 + * ("Passthrough is not supported in P1"). The form should not offer it. + * @property {string?} sourceConversationId - The conversation this was recorded from, for + * traceability; null for a hand-written case. * @property {string} createDate - ISO date string. * @property {string} updateDate - ISO date string. */ /** - * POST /agent-test/cases 与 PUT /agent-test/cases/{id} 的请求体。 + * Body for POST /agent-test/cases and PUT /agent-test/cases/{id}. * - * **PUT 是整份替换，不是 PATCH。** 提交前必须先 GET 出完整 case，只改用户动过的部分， - * 其余字段（尤其是编辑器未必给控件的 `initialStates`、`unmockedToolPolicy`、 - * `sourceConversationId`）原样带回去 —— 任何字段在 UI 上没有对应控件也必须原样带回， - * 否则保存一次就把它清空了。`suiteId` 漏传时后端会兜底保留原值（除非目标 suite 真的存在 - * 且不同才会重新校验）。 + * **PUT is a full replace, not a PATCH.** GET the complete case first, change only what the user + * touched, and send every other field back unchanged -- especially the ones the editor may not give + * a control for (`initialStates`, `unmockedToolPolicy`, `sourceConversationId`). Any field without a + * control still has to travel, or saving once clears it. A missing `suiteId` falls back to the + * stored value (only a genuinely different, existing suite is re-validated). * * @typedef {Object} AgentTestCaseUpsertRequest * @property {string} suiteId * @property {string} name - * @property {boolean} [enabled] - 默认 true。 + * @property {boolean} [enabled] - Defaults to true. * @property {TestTurn[]} turns * @property {TestAssertion[]} assertions * @property {TestState[]} initialStates * @property {TestToolMock[]} mocks - * @property {string} [unmockedToolPolicy] - 固定传 "Block"；默认值本身就是 "Block"。 + * @property {string} [unmockedToolPolicy] - Always send "Block"; that is also the default. * @property {string?} [sourceConversationId] */ @@ -135,20 +148,25 @@ * agent's own LlmConfig. When set, the executor runs the cartesian product of cases x models, * so `totalCount` is "cases x models" and does NOT equal the case count -- showing it as a case * count in the UI will not add up. - * @property {string} status - Pending | Running | Passed | Failed | Error | Cancelled。 - * `Failed` = 跑了但断言没过；`Error` = 没跑成（超时、canary 失败、用例没有 turns、caseIds - * 匹配不到任何用例）。UI 必须把这两者显示成不同的东西 —— 混在一起会让"平台坏了"看起来 - * 像"agent 回归了"。 - * @property {string?} triggeredBy - 触发者的用户 id。 - * @property {string[]?} caseIds - 本次运行只跑这些 case；null/空 = 跑 suite 下全部启用的 case。 + * @property {string} status - Pending | Running | Passed | Failed | Error | Cancelled. + * `Failed` = it ran and an assertion did not hold. `Error` = it never got that far (timeout, + * canary failure, a case with no turns, caseIds matching nothing). The UI must show these as + * different things -- collapsing them makes "the harness broke" read as "the agent regressed". + * @property {string?} error - Why the run ended as `Error`: an infrastructure stop that happened + * before or instead of executing cases (suite gone or disabled, the selected cases were all + * disabled, the host restarted mid-run, an unhandled crash). Distinct from + * AgentTestCaseResult.error, which explains one case -- a run can fail with ZERO case results, + * and then this is the only place the reason exists. + * @property {string?} triggeredBy - User id of whoever triggered it. + * @property {string[]?} caseIds - Run only these cases; null/empty = every enabled case in the suite. * @property {number} totalCount * @property {number} passedCount * @property {number} failedCount * @property {number} errorCount * @property {boolean} cancelRequested - * @property {string?} startedAt - ISO date string；未开始为 null。 - * @property {string?} completedAt - ISO date string；未结束为 null。 - * @property {string} createDate - ISO date string。注意 AgentTestRun 没有 updateDate 字段。 + * @property {string?} startedAt - ISO date string; null until it starts. + * @property {string?} completedAt - ISO date string; null until it finishes. + * @property {string} createDate - ISO date string. Note AgentTestRun has no updateDate field. */ /** @@ -174,8 +192,8 @@ * @property {number} turnIndex * @property {string} functionName * @property {string?} argsJson - * @property {string} outcome - Mocked | Blocked。`Blocked` 要显眼 —— 意味着 agent 试图调用一个 - * 用例没有 mock 的工具，通常正是失败的根因。 + * @property {string} outcome - Mocked | Blocked. `Blocked` needs to stand out: it means the agent + * tried to call a tool this case does not mock, which is usually the root cause of the failure. * @property {string?} resultContent */ @@ -185,23 +203,26 @@ * @property {string} runId * @property {string} caseId * @property {string} caseName - * @property {string} status - Passed | Failed | Error | Cancelled（不会是 Pending/Running）。 - * @property {string?} conversationId - 本次执行新建的会话 id，不复用线上会话。 + * @property {string} status - Passed | Failed | Error | Cancelled (never Pending/Running). + * @property {string?} conversationId - The conversation this execution created; live conversations + * are never reused. * @property {string?} provider - Which model produced this result; null = the agent's own * LlmConfig was used. * @property {string?} model - As above. In a multi-model run the same `caseId` yields several * results, told apart by these two fields. * @property {number} durationMs - * @property {string?} error - 基础设施层面的失败原因（超时、canary 未生效、用例没有 turns 等）， - * 与断言失败区分开；`status` 为 `Error` 时应把这段文本显示出来。 + * @property {string?} error - Infrastructure-level reason for failure (timeout, a dead mock seam, a + * case with no turns), as distinct from an assertion failure. Show this text whenever `status` is + * `Error`. * @property {TurnResult[]} turns - * @property {AssertionResult[]} assertions - 整案级断言结果。 + * @property {AssertionResult[]} assertions - Case-level assertion results. * @property {ObservedToolCall[]} observedToolCalls * @property {string} createDate - ISO date string. */ /** - * GET /agent-test/runs/{id} 的响应体（后端 AgentTestRunDetailDto 的 camelCase 投影）。 + * Body of GET /agent-test/runs/{id} (the camelCase projection of the backend's + * AgentTestRunDetailDto). * @typedef {Object} AgentTestRunDetail * @property {AgentTestRun} run * @property {AgentTestCaseResult[]} results diff --git a/src/lib/langs/en.json b/src/lib/langs/en.json index 7fb5bf67..1c0f2772 100644 --- a/src/lib/langs/en.json +++ b/src/lib/langs/en.json @@ -535,5 +535,9 @@ "Do not use AI (one case for the whole conversation)": "Do not use AI (one case for the whole conversation)", "With a model picked, the conversation is split into one case per scenario it covers. The model only decides where to split and what to name each case -- mocks, assertions and state still come verbatim from the conversation.": "With a model picked, the conversation is split into one case per scenario it covers. The model only decides where to split and what to name each case -- mocks, assertions and state still come verbatim from the conversation.", "AI extraction additionally sends the user messages and tool names to {provider}. Tool arguments and results are not sent.": "AI extraction additionally sends the user messages and tool names to {provider}. Tool arguments and results are not sent.", - "{count} draft cases recorded. Review them, then enable them.": "{count} draft cases recorded. Review them, then enable them." + "{count} draft cases recorded. Review them, then enable them.": "{count} draft cases recorded. Review them, then enable them.", + "This run could not complete": "This run could not complete", + "Nothing ran -- see the reason above.": "Nothing ran -- see the reason above.", + "Nothing here can run -- every case you picked is disabled. Enable at least one first.": "Nothing here can run -- every case you picked is disabled. Enable at least one first.", + "{count} of the selected case(s) are disabled and will be skipped.": "{count} of the selected case(s) are disabled and will be skipped." } diff --git a/src/lib/langs/zh.json b/src/lib/langs/zh.json index ce11d2fc..078783bf 100644 --- a/src/lib/langs/zh.json +++ b/src/lib/langs/zh.json @@ -729,5 +729,9 @@ "Do not use AI (one case for the whole conversation)": "不使用 AI（整段会话生成一条用例）", "With a model picked, the conversation is split into one case per scenario it covers. The model only decides where to split and what to name each case -- mocks, assertions and state still come verbatim from the conversation.": "选了模型后，会话会按覆盖的场景切成多条用例。模型只决定切在哪、每条叫什么——mock、断言和 state 仍旧逐字来自真实会话。", "AI extraction additionally sends the user messages and tool names to {provider}. Tool arguments and results are not sent.": "AI 抽取会额外把用户消息和函数名发给 {provider}。函数入参与返回内容不外发。", - "{count} draft cases recorded. Review them, then enable them.": "已录制 {count} 条草稿用例。请逐条审阅后再启用。" + "{count} draft cases recorded. Review them, then enable them.": "已录制 {count} 条草稿用例。请逐条审阅后再启用。", + "This run could not complete": "这次运行没能执行", + "Nothing ran -- see the reason above.": "什么都没跑，原因见上方。", + "Nothing here can run -- every case you picked is disabled. Enable at least one first.": "这样跑不起来——你勾选的用例全部处于停用状态。请先至少启用一条。", + "{count} of the selected case(s) are disabled and will be skipped.": "勾选的用例里有 {count} 条处于停用状态，将被跳过。" } diff --git a/src/routes/page/agent-test/[suiteId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/+page.svelte index fe4ac4b4..b700f156 100644 --- a/src/routes/page/agent-test/[suiteId]/+page.svelte +++ b/src/routes/page/agent-test/[suiteId]/+page.svelte @@ -117,8 +117,18 @@ .filter(x => x.models.length > 0) ); + /** + * Of the ticked cases, the ones that can actually run. The executor drops disabled cases and + * then fails the entire run with "none of them matched an enabled case" -- observed as a run + * with status Error, zero results and no visible reason. Counting them here stops it earlier. + */ + let selectedEnabledCount = $derived( + cases.filter(c => c.enabled && selectedCaseIds.includes(c.id)).length + ); + let selectedDisabledCount = $derived(selectedCaseIds.length - selectedEnabledCount); + /** Cases x models -- what the run will actually execute, and what it will cost. */ - let plannedCaseCount = $derived(runPartial ? selectedCaseIds.length : enabledCaseCount); + let plannedCaseCount = $derived(runPartial ? selectedEnabledCount : enabledCaseCount); let plannedExecutionCount = $derived(plannedCaseCount * Math.max(1, selectedRunModels.length)); let enabledCaseCount = $derived(cases.filter(x => x.enabled).length); @@ -429,7 +439,8 @@ isTriggering = true; triggerRun( suiteId, - runPartial ? selectedCaseIds : null, + // Enabled only: a disabled id contributes nothing but noise to the run's CaseIds record. + runPartial ? cases.filter(c => c.enabled && selectedCaseIds.includes(c.id)).map(c => c.id) : null, selectedRunModels.length > 0 ? selectedRunModels : null ).then(run => { isRunModalOpen = false; @@ -595,7 +606,7 @@ {:else} {/if} - {$_('Run Selected')} ({selectedCaseIds.length}) + {$_('Run Selected')} ({selectedEnabledCount}) {/if}
{/if} - + {#if runPartial && selectedDisabledCount > 0} + + {/if} + + {#if plannedCaseCount === 0} + + {:else} + + {/if}
+ {#if run.error} + + + {/if} {#if run.errorCount > 0}