diff --git a/src/lib/helpers/types/agentTestTypes.js b/src/lib/helpers/types/agentTestTypes.js new file mode 100644 index 00000000..9c9cf950 --- /dev/null +++ b/src/lib/helpers/types/agentTestTypes.js @@ -0,0 +1,231 @@ +/** + * @typedef {Object} AgentTestSuite + * @property {string} id + * @property {string} agentId + * @property {string} name + * @property {string?} description + * @property {boolean} enabled + * @property {string?} judgeProvider - Provider for llmJudge assertions. In P1 llmJudge always fails, + * whether or not this is configured. + * @property {string?} judgeModel + * @property {string[]} extraAllowedFunctions - Functions let through on top of the default + * control-flow allow list. + * @property {string[]} forceBlockedFunctions - Functions blocked outright; wins over the allow list. + * @property {number} caseTimeoutSeconds + * @property {string} createDate - ISO date string. + * @property {string} updateDate - ISO date string. + */ + +/** + * Body for POST /agent-test/suites and PUT /agent-test/suites/{id}. + * + * **PUT is a full replace, not a PATCH.** GET the complete suite first, change only what the user + * touched, and send everything else back unchanged -- a missing agentId/name falls back to the + * stored value, but a missing caseTimeoutSeconds/extraAllowedFunctions/forceBlockedFunctions is + * reset to the backend default (120 / [] / []). + * + * @typedef {Object} AgentTestSuiteUpsertRequest + * @property {string} agentId + * @property {string} name + * @property {string?} [description] + * @property {boolean?} [enabled] - Omitted or null = leave unchanged (treated as true on create); + * only an explicit true/false flips the enabled state. + * @property {string?} [judgeProvider] + * @property {string?} [judgeModel] + * @property {string[]} extraAllowedFunctions + * @property {string[]} forceBlockedFunctions + * @property {number} caseTimeoutSeconds - Defaults to 120. + */ + +/** + * @typedef {Object} TestTurn + * @property {number} index + * @property {string} userMessage + * @property {TestAssertion[]} assertions - Turn-level assertions, as opposed to the case-level + * AgentTestCase.assertions. + */ + +/** + * @typedef {Object} TestState + * @property {string} key + * @property {string} value + * @property {number} [activeRounds] - Defaults to -1 (never expires). + * @property {boolean} [global] + */ + +/** + * @typedef {Object} TestToolMock + * @property {string} functionName + * @property {string?} [argsMatchJson] - Optional argument-subset match (a JSON string), for giving + * different returns to repeated calls of the same tool. + * @property {number?} [callIndex] - Optional: match only the Nth call (0-based). + * @property {string} [resultContent] - The faked return, written to message.Content; usually JSON text. + * @property {boolean} [stopCompletion] - Reproduces a real tool's "stop this turn's LLM completion". + * @property {TestState[]?} [stateWrites] - State written when this mock is hit. Many + * IFunctionCallback implementations pass data across turns purely through state, so mocking only + * the return value leaves later functions unable to read what they expect. + */ + +/** + * One assertion. + * + * Fields the backend enforces on save (a violation is a 400, but the message does not say which + * assertion -- the form should block it first): + * - outputContains / outputNotContains / outputRegex / routedToAgent / llmJudge: `expected` required. + * - toolCalled / toolNotCalled / stateEquals: `target` required. + * + * One trap the backend does NOT cover but evaluation does: `stateEquals` only requires `target` on + * save, so an empty `expected` saves fine -- and then compares the real state value against + * null/empty on every run and can never pass. It fails rather than errors, so nothing points at it. + * The form is deliberately stricter and treats `expected` as required there too. + * + * `llmJudge` always fails in P1 (the backend returns "llmJudge is not available in P1"), regardless + * of minScore or whether the suite configured judgeProvider/judgeModel. The form may offer it, but + * says so next to it. + * + * @typedef {Object} TestAssertion + * @property {string} type - outputContains | outputNotContains | outputRegex | toolCalled | toolNotCalled | stateEquals | routedToAgent | llmJudge + * @property {string?} [target] - Function name / state key / agent name. + * @property {string?} [expected] - Expected value / regex / judging criteria. + * @property {string?} [argsMatchJson] - Argument-subset match for toolCalled (a JSON string). + * @property {number?} [minScore] - Pass threshold for llmJudge. + * @property {boolean} [fatal] - On failure, abort the remaining turns of this case. + */ + +/** + * @typedef {Object} AgentTestCase + * @property {string} id + * @property {string} suiteId + * @property {string} name + * @property {boolean} enabled - Recorded drafts land as false; a human has to review and enable them + * before they join a normal run. + * @property {TestTurn[]} turns - Length 1 is a single-turn case. + * @property {TestAssertion[]} assertions - Case-level assertions, evaluated after every turn has run. + * @property {TestState[]} initialStates - Injected before the conversation starts; maps to BotSharp's + * MessageState. + * @property {TestToolMock[]} mocks + * @property {string} unmockedToolPolicy - P1 accepts only "Block"; sending "Passthrough" is a 400 + * ("Passthrough is not supported in P1"). The form should not offer it. + * @property {string?} sourceConversationId - The conversation this was recorded from, for + * traceability; null for a hand-written case. + * @property {string} createDate - ISO date string. + * @property {string} updateDate - ISO date string. + */ + +/** + * Body for POST /agent-test/cases and PUT /agent-test/cases/{id}. + * + * **PUT is a full replace, not a PATCH.** GET the complete case first, change only what the user + * touched, and send every other field back unchanged -- especially the ones the editor may not give + * a control for (`initialStates`, `unmockedToolPolicy`, `sourceConversationId`). Any field without a + * control still has to travel, or saving once clears it. A missing `suiteId` falls back to the + * stored value (only a genuinely different, existing suite is re-validated). + * + * @typedef {Object} AgentTestCaseUpsertRequest + * @property {string} suiteId + * @property {string} name + * @property {boolean} [enabled] - Defaults to true. + * @property {TestTurn[]} turns + * @property {TestAssertion[]} assertions + * @property {TestState[]} initialStates + * @property {TestToolMock[]} mocks + * @property {string} [unmockedToolPolicy] - Always send "Block"; that is also the default. + * @property {string?} [sourceConversationId] + */ + +/** + * One model to run against. + * @typedef {Object} TestModel + * @property {string} provider + * @property {string} model + */ + +/** + * @typedef {Object} AgentTestRun + * @property {string} id + * @property {string} suiteId + * @property {TestModel[]?} models - Models this run swept; null/empty = a single pass on the + * agent's own LlmConfig. When set, the executor runs the cartesian product of cases x models, + * so `totalCount` is "cases x models" and does NOT equal the case count -- showing it as a case + * count in the UI will not add up. + * @property {string} status - Pending | Running | Passed | Failed | Error | Cancelled. + * `Failed` = it ran and an assertion did not hold. `Error` = it never got that far (timeout, + * canary failure, a case with no turns, caseIds matching nothing). The UI must show these as + * different things -- collapsing them makes "the harness broke" read as "the agent regressed". + * @property {string?} error - Why the run ended as `Error`: an infrastructure stop that happened + * before or instead of executing cases (suite gone or disabled, the selected cases were all + * disabled, the host restarted mid-run, an unhandled crash). Distinct from + * AgentTestCaseResult.error, which explains one case -- a run can fail with ZERO case results, + * and then this is the only place the reason exists. + * @property {string?} triggeredBy - User id of whoever triggered it. + * @property {string[]?} caseIds - Run only these cases; null/empty = every enabled case in the suite. + * @property {number} totalCount + * @property {number} passedCount + * @property {number} failedCount + * @property {number} errorCount + * @property {boolean} cancelRequested + * @property {string?} startedAt - ISO date string; null until it starts. + * @property {string?} completedAt - ISO date string; null until it finishes. + * @property {string} createDate - ISO date string. Note AgentTestRun has no updateDate field. + */ + +/** + * @typedef {Object} AssertionResult + * @property {string} type + * @property {string?} target + * @property {string?} expected + * @property {string?} actual + * @property {boolean} passed + * @property {string?} message + */ + +/** + * @typedef {Object} TurnResult + * @property {number} index + * @property {string} userMessage + * @property {string?} output + * @property {AssertionResult[]} assertions + */ + +/** + * @typedef {Object} ObservedToolCall + * @property {number} turnIndex + * @property {string} functionName + * @property {string?} argsJson + * @property {string} outcome - Mocked | Blocked. `Blocked` needs to stand out: it means the agent + * tried to call a tool this case does not mock, which is usually the root cause of the failure. + * @property {string?} resultContent + */ + +/** + * @typedef {Object} AgentTestCaseResult + * @property {string} id + * @property {string} runId + * @property {string} caseId + * @property {string} caseName + * @property {string} status - Passed | Failed | Error | Cancelled (never Pending/Running). + * @property {string?} conversationId - The conversation this execution created; live conversations + * are never reused. + * @property {string?} provider - Which model produced this result; null = the agent's own + * LlmConfig was used. + * @property {string?} model - As above. In a multi-model run the same `caseId` yields several + * results, told apart by these two fields. + * @property {number} durationMs + * @property {string?} error - Infrastructure-level reason for failure (timeout, a dead mock seam, a + * case with no turns), as distinct from an assertion failure. Show this text whenever `status` is + * `Error`. + * @property {TurnResult[]} turns + * @property {AssertionResult[]} assertions - Case-level assertion results. + * @property {ObservedToolCall[]} observedToolCalls + * @property {string} createDate - ISO date string. + */ + +/** + * Body of GET /agent-test/runs/{id} (the camelCase projection of the backend's + * AgentTestRunDetailDto). + * @typedef {Object} AgentTestRunDetail + * @property {AgentTestRun} run + * @property {AgentTestCaseResult[]} results + */ + +export default {}; diff --git a/src/lib/helpers/utils/agent-test.js b/src/lib/helpers/utils/agent-test.js new file mode 100644 index 00000000..41c06c92 --- /dev/null +++ b/src/lib/helpers/utils/agent-test.js @@ -0,0 +1,173 @@ +import { get } from 'svelte/store'; +import { _ } from 'svelte-i18n'; + +/** + * Shared helpers for the agent test pages. Kept out of the page components + * because all four of them have to agree on what a status means -- showing + * `Failed` and `Error` the same way is the one mistake that makes "the harness + * broke" look like "the agent regressed". + */ + +/** + * Translate from plain JS, where the `$_` auto-subscription is not available. + * Read at call time, never at module load -- the dictionary is fetched + * asynchronously by setupI18n and is empty while the module is first evaluated. + * + * This is a one-shot read, not a subscription, so the result does not + * re-translate if the user switches language afterwards. That is fine for what + * uses it (toasts, confirm dialogs, validation text -- all short-lived or + * recomputed on the next keystroke). Anything that has to survive a language + * switch on screen belongs in the markup as `$_(...)` instead. + * @param {string} id + * @param {Record} [values] + * @returns {string} + */ +export function t(id, values = undefined) { + return get(_)(id, values ? { values } : undefined); +} + +/** Assertion types the backend can actually evaluate (AssertionTypes in AssertionEvaluator.cs). */ +export const ASSERTION_TYPES = [ + 'outputContains', + 'outputNotContains', + 'outputRegex', + 'toolCalled', + 'toolNotCalled', + 'stateEquals', + 'routedToAgent', + 'llmJudge' +]; + +/** Types whose `expected` the backend rejects as empty. */ +const EXPECTED_REQUIRED = ['outputContains', 'outputNotContains', 'outputRegex', 'routedToAgent', 'llmJudge']; + +/** Types whose `target` the backend rejects as empty. */ +const TARGET_REQUIRED = ['toolCalled', 'toolNotCalled', 'stateEquals']; + +/** + * Bootstrap contextual class for a run/case status. + * @param {string?} status + * @returns {string} + */ +export function statusColor(status) { + switch (status) { + case 'Passed': return 'success'; + // Failed = it ran and an assertion did not hold. Error = it never got to + // assert (timeout, canary, no turns). Different colours on purpose. + case 'Failed': return 'danger'; + case 'Error': return 'warning'; + case 'Running': return 'info'; + case 'Pending': return 'secondary'; + case 'Cancelled': return 'dark'; + default: return 'secondary'; + } +} + +/** + * A run in Pending/Running is still moving, so the detail page keeps polling + * and the cancel button stays live. + * @param {string?} status + */ +export function isTerminalStatus(status) { + return status === 'Passed' || status === 'Failed' || status === 'Error' || status === 'Cancelled'; +} + +/** + * Pull the backend's own message out of an axios error. These endpoints return + * genuinely actionable 400s ("Passthrough is not supported in P1", "assertion + * 'outputRegex' requires a non-empty 'expected' value", "suite is disabled"), + * so swallowing them behind a generic string throws away the only clue there + * is. Those messages are server-side English and are shown as-is; only the + * strings this file owns are translated. + * @param {any} err + * @param {string} fallback - already-translated text + * @returns {string} + */ +export function errorMessage(err, fallback) { + const data = err?.response?.data; + if (typeof data === 'string' && data.trim()) { + return data; + } + if (typeof data?.message === 'string' && data.message.trim()) { + return data.message; + } + if (err?.response?.status === 401 || err?.response?.status === 403) { + return t('You do not have permission for this. Running and recording require an admin or root account.'); + } + return fallback; +} + +/** + * Validate one assertion the way the backend does, plus the two cases it lets + * through but that can never pass at evaluation time. + * @param {import('$agentTestTypes').TestAssertion} assertion + * @returns {string?} null when well-formed, otherwise a message for the author + */ +export function validateAssertion(assertion) { + const type = assertion?.type; + if (!type) { + return t('Assertion type is required.'); + } + + if (EXPECTED_REQUIRED.includes(type) && !assertion.expected?.trim()) { + return t('Assertion "{type}" needs an expected value.', { type }); + } + + if (TARGET_REQUIRED.includes(type) && !assertion.target?.trim()) { + return t('Assertion "{type}" needs a target.', { type }); + } + + // Stricter than the backend on purpose: stateEquals saves fine with an empty + // `expected`, then compares the real state value against null and fails on + // every single run. A silent always-red assertion is worse than a save error. + if (type === 'stateEquals' && !assertion.expected?.trim()) { + return t('Assertion "stateEquals" needs an expected value, otherwise it can never pass.'); + } + + if (type === 'outputRegex' && assertion.expected?.trim()) { + try { + new RegExp(assertion.expected); + } catch { + return t('"{pattern}" is not a valid regular expression.', { pattern: assertion.expected }); + } + } + + if (assertion.argsMatchJson?.trim() && !isParsableJson(assertion.argsMatchJson)) { + return t('The args match on an assertion is not valid JSON.'); + } + + return null; +} + +/** + * @param {string} text + * @returns {boolean} + */ +export function isParsableJson(text) { + try { + JSON.parse(text); + return true; + } catch { + return false; + } +} + +/** + * @param {number?} ms + * @returns {string} + */ +export function formatDuration(ms) { + if (ms == null) return '--'; + if (ms < 1000) return t('{ms} ms', { ms }); + return t('{seconds} s', { seconds: (ms / 1000).toFixed(1) }); +} + +/** + * @param {string?} iso + * @returns {string} + */ +export function formatDateTime(iso) { + if (!iso) return '--'; + const date = new Date(iso); + return Number.isNaN(date.getTime()) ? '--' : date.toLocaleString(); +} diff --git a/src/lib/langs/en.json b/src/lib/langs/en.json index 7f905c0b..1c0f2772 100644 --- a/src/lib/langs/en.json +++ b/src/lib/langs/en.json @@ -168,138 +168,376 @@ "Your item is shipped": "Your item is shipped", "As a skeptical Cambridge friend of mine occidental": "As a skeptical Cambridge friend of mine occidental", "1 hours ago": "1 hours ago", - - "Agents":"Agents", - "Evaluator":"Evaluator", - "Router":"Router", - "Home":"Home", - "Dashboard":"Dashboard", - "Agent":"Agent", - "Knowledge Base":"Knowledge Base", - "Conversation":"Conversation", - "Task":"Task", - "MongoDB":"MongoDB", - "Plugins":"Plugins", + "Agents": "Agents", + "Evaluator": "Evaluator", + "Router": "Router", + "Home": "Home", + "Dashboard": "Dashboard", + "Agent": "Agent", + "Knowledge Base": "Knowledge Base", + "Conversation": "Conversation", + "Task": "Task", + "MongoDB": "MongoDB", + "Plugins": "Plugins", "Settings": "Settings", - "Search":"Search", - "Welcome Back !":"Welcome Back !", - "Admin":"Admin", - "Agent Manager":"Agent Manager", - "Conversations":"Conversations", - "Token Cost":"Token Cost", - "View Profile":"View Profile", - "Monthly Cost":"Monthly Cost", - "This month":"This month", + "Search": "Search", + "Welcome Back !": "Welcome Back !", + "Admin": "Admin", + "Agent Manager": "Agent Manager", + "Conversations": "Conversations", + "Token Cost": "Token Cost", + "View Profile": "View Profile", + "Monthly Cost": "Monthly Cost", + "This month": "This month", "View More": "View More", - "Total Cost":"Total Cost", - "Average Cost":"Average Cost", - "Week":"Week", - "Month":"Month", - "Year":"Year", - "Token Spent":"Token Spent", - "Channel Stats":"Channel Stats", - "Learn more":"Learn more", - "sales":"sales", - "Activity":"Activity", - "Read more":"Read more", - "Top Client Usage":"Top Client Usage", + "Total Cost": "Total Cost", + "Average Cost": "Average Cost", + "Week": "Week", + "Month": "Month", + "Year": "Year", + "Token Spent": "Token Spent", + "Channel Stats": "Channel Stats", + "Learn more": "Learn more", + "sales": "sales", + "Activity": "Activity", + "Read more": "Read more", + "Top Client Usage": "Top Client Usage", "Logout": "Logout", "Profile": "Profile", "Notifications": "Notifications", - "View All":"View All", + "View All": "View All", "Your order is placed": "Your order is placed", - "min ago":"min ago", + "min ago": "min ago", "List": "List", - "Communication":"Communication", + "Communication": "Communication", "Conversation List": "Conversation List", "Task List": "Task List", - "Knowledge Manager":"Knowledge Manager", - "Migrate agents from file repository to MongoDB":"Migrate agents from file repository to MongoDB", - "Setting":"Setting", - "Start Migration":"Start Migration", - "Plugin":"Plugin", - "My Profile":"My Profile", - "Detail":"Detail", - "System & Plugin Settings":"System & Plugin Settings", - "Disabled":"Disabled", - "Enabled":"Enabled", - "Public":"Public", - "Private":"Private", - "Build":"Build", - "Train":"Train", - "Test":"Test", - "My Files":"My Files", - "Upload":"Upload", - "Recent Files":"Recent Files", - "Storage":"Storage", - "0 GB (0%) of 1 GB used":"0 GB (0%) of 1 GB used", - "Provided by BotSharp":"Provided by BotSharp", - "Provided by Pizza AI Assistant":"Provided by Pizza AI Assistant", - "Upgrade Features":"Upgrade Features", - "Upgrade":"Upgrade", - "Files":"Files", - "Design":"Design", - "Google Drive":"Google Drive", - "Starred":"Starred", - "Trash":"Trash", - "Dropbox":"Dropbox", - "Open":"Open", - "Edit":"Edit" , - "Rename":"Rename", - "Remove":"Remove", - "Share Files":"Share Files", - "Share with me":"Share with me", - "Other Actions":"Other Actions", - "Name":"Name", - "Date modified":"Date modified", - "Size":"Size", + "Knowledge Manager": "Knowledge Manager", + "Migrate agents from file repository to MongoDB": "Migrate agents from file repository to MongoDB", + "Setting": "Setting", + "Start Migration": "Start Migration", + "Plugin": "Plugin", + "My Profile": "My Profile", + "Detail": "Detail", + "System & Plugin Settings": "System & Plugin Settings", + "Disabled": "Disabled", + "Enabled": "Enabled", + "Public": "Public", + "Private": "Private", + "Build": "Build", + "Train": "Train", + "Test": "Test", + "My Files": "My Files", + "Upload": "Upload", + "Recent Files": "Recent Files", + "Storage": "Storage", + "0 GB (0%) of 1 GB used": "0 GB (0%) of 1 GB used", + "Provided by BotSharp": "Provided by BotSharp", + "Provided by Pizza AI Assistant": "Provided by Pizza AI Assistant", + "Upgrade Features": "Upgrade Features", + "Upgrade": "Upgrade", + "Files": "Files", + "Design": "Design", + "Google Drive": "Google Drive", + "Starred": "Starred", + "Trash": "Trash", + "Dropbox": "Dropbox", + "Open": "Open", + "Edit": "Edit", + "Rename": "Rename", + "Remove": "Remove", + "Share Files": "Share Files", + "Share with me": "Share with me", + "Other Actions": "Other Actions", + "Name": "Name", + "Date modified": "Date modified", + "Size": "Size", "Images": "Images", "Video": "Video", - "Music":"Music", - "Document":"Document", - "Others":"Others", - "Another action":"Another action", - "Something else here":"Something else here", - "Search for ...":"Search for ...", - "Status":"Status", - "Completed":"Completed", - "Select Channel":"Select Channel", - "Live Chat":"Live Chat", - "Phone":"Phone", - "Email":"Email", - "Filter":"Filter", - "Title":"Title", - "User Name":"User Name", - "Channel":"Channel", - "Posted Date":"Posted Date", - "Last Date":"Last Date", - "Action":"Action", - "Active":"Active", - "Description":"Description", - "Details":"Details", - "Updated Date":"Updated Date", - "Showing":"Showing", - "of":"of", - "to":"to", - "entries":"entries", - "View":"View", - "Install":"Install", - "Dialogs":"Dialogs", - "Agent Overview":"Agent Overview", - "Save Agent":"Save Agent", - "Conversation Detail":"Conversation Detail", - "closed":"closed", - "webchat":"webchat", - "User":"User", - "Location":"Location", - "Delete Conversation":" Delete Conversation", - "Design & Developed by open source community":"Design & Developed by open source community", - "Personal Information":"Personal Information", - "First Name":"First Name", - "Last Name":"Last Name", - "Account Origin":"Account Origin", - "Update Date":"Update Date", - "Create Date":"Create Date", - "Active now":"Active now", - "Reset":"Reset" -} \ No newline at end of file + "Music": "Music", + "Document": "Document", + "Others": "Others", + "Another action": "Another action", + "Something else here": "Something else here", + "Search for ...": "Search for ...", + "Status": "Status", + "Completed": "Completed", + "Select Channel": "Select Channel", + "Live Chat": "Live Chat", + "Phone": "Phone", + "Email": "Email", + "Filter": "Filter", + "Title": "Title", + "User Name": "User Name", + "Channel": "Channel", + "Posted Date": "Posted Date", + "Last Date": "Last Date", + "Action": "Action", + "Active": "Active", + "Description": "Description", + "Details": "Details", + "Updated Date": "Updated Date", + "Showing": "Showing", + "of": "of", + "to": "to", + "entries": "entries", + "View": "View", + "Install": "Install", + "Dialogs": "Dialogs", + "Agent Overview": "Agent Overview", + "Save Agent": "Save Agent", + "Conversation Detail": "Conversation Detail", + "closed": "closed", + "webchat": "webchat", + "User": "User", + "Location": "Location", + "Delete Conversation": " Delete Conversation", + "Design & Developed by open source community": "Design & Developed by open source community", + "Personal Information": "Personal Information", + "First Name": "First Name", + "Last Name": "Last Name", + "Account Origin": "Account Origin", + "Update Date": "Update Date", + "Create Date": "Create Date", + "Active now": "Active now", + "Reset": "Reset", + "Agent Testing": "Agent Testing", + "Test Suites": "Test Suites", + "Test Suite": "Test Suite", + "Test Cases": "Test Cases", + "Test Case": "Test Case", + "Test Run": "Test Run", + "Run History": "Run History", + "Case Results": "Case Results", + "Suite Settings": "Suite Settings", + "New Test Suite": "New Test Suite", + "New Test Case": "New Test Case", + "Edit Test Case": "Edit Test Case", + "Record a Draft Case": "Record a Draft Case", + "Turns": "Turns", + "Turn": "Turn", + "Case Assertions": "Case Assertions", + "Assertions": "Assertions", + "Initial States": "Initial States", + "Tool Mocks": "Tool Mocks", + "Observed Tool Calls": "Observed Tool Calls", + "New Suite": "New Suite", + "New Case": "New Case", + "Add Turn": "Add Turn", + "Add Assertion": "Add Assertion", + "Add State": "Add State", + "Add State Write": "Add State Write", + "Add Mock": "Add Mock", + "Run All Enabled": "Run All Enabled", + "Run Selected": "Run Selected", + "Cancel Run": "Cancel Run", + "Re-run Failures": "Re-run Failures", + "Record from Conversation": "Record from Conversation", + "Record": "Record", + "Refresh": "Refresh", + "Retry": "Retry", + "Save": "Save", + "Create": "Create", + "Cancel": "Cancel", + "Close": "Close", + "Delete": "Delete", + "Enable": "Enable", + "Disable": "Disable", + "Back": "Back", + "Back to Suite": "Back to Suite", + "Back to Suites": "Back to Suites", + "Show details": "Show details", + "Hide details": "Hide details", + "Clear selection": "Clear selection", + "All Agents": "All Agents", + "Select Agent": "Select Agent", + "Select all cases": "Select all cases", + "Select case {name}": "Select case {name}", + "Edit case": "Edit case", + "Delete case": "Delete case", + "Enable case": "Enable case", + "Disable case": "Disable case", + "View run": "View run", + "Remove turn": "Remove turn", + "Move turn up": "Move turn up", + "Move turn down": "Move turn down", + "Remove assertion": "Remove assertion", + "Remove state": "Remove state", + "Remove mock": "Remove mock", + "Source": "Source", + "Scope": "Scope", + "Started": "Started", + "Duration": "Duration", + "Result": "Result", + "Type": "Type", + "Target": "Target", + "Expected": "Expected", + "Actual": "Actual", + "Message": "Message", + "Function": "Function", + "Outcome": "Outcome", + "Args": "Args", + "Key": "Key", + "Value": "Value", + "Total": "Total", + "Mocks": "Mocks", + "Mock": "Mock", + "Run": "Run", + "Fatal": "Fatal", + "Global": "Global", + "Active rounds": "Active rounds", + "Function name": "Function name", + "Result content": "Result content", + "Stop completion": "Stop completion", + "User message": "User message", + "Agent output": "Agent output", + "Conversation id": "Conversation id", + "Judge provider": "Judge provider", + "Judge model": "Judge model", + "Case timeout": "Case timeout", + "Case timeout (seconds)": "Case timeout (seconds)", + "Extra allowed functions": "Extra allowed functions", + "Force blocked functions": "Force blocked functions", + "Unmocked tool policy": "Unmocked tool policy", + "Enabled (included in runs)": "Enabled (included in runs)", + "Triggered by": "Triggered by", + "Args match (JSON subset, optional)": "Args match (JSON subset, optional)", + "Call index (0-based, optional)": "Call index (0-based, optional)", + "Enter suite name": "Enter suite name", + "Optional description": "Optional description", + "function name": "function name", + "state key": "state key", + "An existing conversation with at least one tool call": "An existing conversation with at least one tool call", + "Pending": "Pending", + "Running": "Running", + "Passed": "Passed", + "Failed": "Failed", + "Error": "Error", + "Cancelled": "Cancelled", + "Cancelling": "Cancelling", + "Mocked": "Mocked", + "Blocked": "Blocked", + "Pass": "Pass", + "Fail": "Fail", + "Draft": "Draft", + "Recorded": "Recorded", + "Manual": "Manual", + "Subset": "Subset", + "All enabled": "All enabled", + "passed": "passed", + "failed": "failed", + "errored": "errored", + "Failed (assertion)": "Failed (assertion)", + "Errored (never ran)": "Errored (never ran)", + "Harness error": "Harness error", + "Partial run of": "Partial run of", + "selected case(s).": "selected case(s).", + "No test suites yet.": "No test suites yet.", + "No test suites for this agent yet.": "No test suites for this agent yet.", + "No test cases in this suite yet.": "No test cases in this suite yet.", + "This suite has never been run.": "This suite has never been run.", + "No case results yet.": "No case results yet.", + "This run produced no case results.": "This run produced no case results.", + "No turns yet. A case needs at least one.": "No turns yet. A case needs at least one.", + "No case-level assertions.": "No case-level assertions.", + "No initial states.": "No initial states.", + "No assertions on this turn.": "No assertions on this turn.", + "The agent called no tools during this case.": "The agent called no tools during this case.", + "No mocks. Every tool call this case makes will be blocked.": "No mocks. Every tool call this case makes will be blocked.", + "A disabled suite cannot be run at all.": "A disabled suite cannot be run at all.", + "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.": "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.", + "llmJudge assertions always fail in P1 regardless of these two fields.": "llmJudge assertions always fail in P1 regardless of these two fields.", + "Always fails in P1.": "Always fails in P1.", + "One function name per line. These run for real during a test -- only list functions with no side effects.": "One function name per line. These run for real during a test -- only list functions with no side effects.", + "One per line. Blocked even if allow-listed elsewhere.": "One per line. Blocked even if allow-listed elsewhere.", + "Assertions checked right after this turn": "Assertions checked right after this turn", + "Evaluated once, after every turn has run.": "Evaluated once, after every turn has run.", + "Injected before the conversation starts. Active rounds -1 means it never expires.": "Injected before the conversation starts. Active rounds -1 means it never expires.", + "Every tool this case does not mock is blocked. If the agent needs a tool to move forward, mock it here.": "Every tool this case does not mock is blocked. If the agent needs a tool to move forward, mock it here.", + "State written when this mock is hit": "State written when this mock is hit", + "Many tools pass data to later turns through state, not through their return value.": "Many tools pass data to later turns through state, not through their return value.", + "Any tool this case does not mock is blocked instead of executed. P1 has no other option.": "Any tool this case does not mock is blocked instead of executed. P1 has no other option.", + "Recorded from conversation": "Recorded from conversation", + "It may contain live customer data -- review before enabling.": "It may contain live customer data -- review before enabling.", + "Recorded from conversation {id}. May contain live customer data.": "Recorded from conversation {id}. May contain live customer data.", + "Recording copies the raw conversation into the test store, including any phone numbers, addresses and tenant names it contains. The draft lands disabled -- review it before enabling.": "Recording copies the raw conversation into the test store, including any phone numbers, addresses and tenant names it contains. The draft lands disabled -- review it before enabling.", + "Errored cases never reached their assertions -- a timeout, a mock-seam failure, or a case with no turns. That is a harness problem, not an agent regression.": "Errored cases never reached their assertions -- a timeout, a mock-seam failure, or a case with no turns. That is a harness problem, not an agent regression.", + "This run is still going. Refreshing every couple of seconds.": "This run is still going. Refreshing every couple of seconds.", + "The agent tried to call a tool this case does not mock, so it was blocked. This is often the root cause of the failure above.": "The agent tried to call a tool this case does not mock, so it was blocked. This is often the root cause of the failure above.", + "Are you sure?": "Are you sure?", + "Yes, delete it!": "Yes, delete it!", + "Yes, run it": "Yes, run it", + "Run this suite?": "Run this suite?", + "Delete test suite \"{name}\"? You won't be able to revert this!": "Delete test suite \"{name}\"? You won't be able to revert this!", + "Delete test case \"{name}\"? You won't be able to revert this!": "Delete test case \"{name}\"? You won't be able to revert this!", + "{count} case(s) will run against the live model. This costs real tokens and is not rate limited.": "{count} case(s) will run against the live model. This costs real tokens and is not rate limited.", + "Test suite created!": "Test suite created!", + "Test suite deleted!": "Test suite deleted!", + "Test case created!": "Test case created!", + "Test case saved!": "Test case saved!", + "Test case deleted!": "Test case deleted!", + "Suite settings saved!": "Suite settings saved!", + "Case enabled.": "Case enabled.", + "Case disabled.": "Case disabled.", + "Run queued!": "Run queued!", + "Re-run queued!": "Re-run queued!", + "Cancellation requested.": "Cancellation requested.", + "Draft case recorded. Review it, then enable it.": "Draft case recorded. Review it, then enable it.", + "Failed to load test suites. Please try again.": "Failed to load test suites. Please try again.", + "Failed to load this test suite.": "Failed to load this test suite.", + "Failed to load this test case.": "Failed to load this test case.", + "Failed to load the suite this case belongs to.": "Failed to load the suite this case belongs to.", + "Failed to load this run.": "Failed to load this run.", + "Failed to create test suite.": "Failed to create test suite.", + "Failed to delete test suite.": "Failed to delete test suite.", + "Failed to save suite settings.": "Failed to save suite settings.", + "Failed to save the test case.": "Failed to save the test case.", + "Failed to update the case.": "Failed to update the case.", + "Failed to delete the test case.": "Failed to delete the test case.", + "Failed to start the run.": "Failed to start the run.", + "Failed to start the re-run.": "Failed to start the re-run.", + "Failed to cancel the run.": "Failed to cancel the run.", + "Failed to record a draft from that conversation.": "Failed to record a draft from that conversation.", + "You do not have permission for this. Running and recording require an admin or root account.": "You do not have permission for this. Running and recording require an admin or root account.", + "Fix these before saving:": "Fix these before saving:", + "Name is required.": "Name is required.", + "Add at least one turn -- a case with no turns errors when it runs.": "Add at least one turn -- a case with no turns errors when it runs.", + "Turn {n}: the user message is empty.": "Turn {n}: the user message is empty.", + "Turn {n}, assertion {m}: {error}": "Turn {n}, assertion {m}: {error}", + "Case assertion {n}: {error}": "Case assertion {n}: {error}", + "Initial state {n}: the key is empty.": "Initial state {n}: the key is empty.", + "Mock {n}: the function name is empty.": "Mock {n}: the function name is empty.", + "Mock {n}: the args match is not valid JSON.": "Mock {n}: the args match is not valid JSON.", + "Mock {n}: the call index is 0-based and cannot be negative.": "Mock {n}: the call index is 0-based and cannot be negative.", + "Mock {n}, state write {m}: the key is empty.": "Mock {n}, state write {m}: the key is empty.", + "Assertion type is required.": "Assertion type is required.", + "Assertion \"{type}\" needs an expected value.": "Assertion \"{type}\" needs an expected value.", + "Assertion \"{type}\" needs a target.": "Assertion \"{type}\" needs a target.", + "Assertion \"stateEquals\" needs an expected value, otherwise it can never pass.": "Assertion \"stateEquals\" needs an expected value, otherwise it can never pass.", + "\"{pattern}\" is not a valid regular expression.": "\"{pattern}\" is not a valid regular expression.", + "The args match on an assertion is not valid JSON.": "The args match on an assertion is not valid JSON.", + "{ms} ms": "{ms} ms", + "{seconds} s": "{seconds} s", + "Not configured": "Not configured", + "This provider has no chat-capable model registered.": "This provider has no chat-capable model registered.", + "Case": "Case", + "Models": "Models", + "models": "models", + "Model Comparison": "Model Comparison", + "Running every enabled case": "Running every enabled case", + "Running the selected cases": "Running the selected cases", + "Use the agent's own model": "Use the agent's own model", + "Pick nothing to run once on whatever model each agent is configured with. Pick two or more to run the whole set once per model and compare them side by side.": "Pick nothing to run once on whatever model each agent is configured with. Pick two or more to run the whole set once per model and compare them side by side.", + "No chat-capable model is registered, so this run will use the agent's own configuration.": "No chat-capable model is registered, so this run will use the agent's own configuration.", + "{count} execution(s) will run against live models. This costs real tokens and is not rate limited.": "{count} execution(s) will run against live models. This costs real tokens and is not rate limited.", + "Durations are wall-clock for the whole case, including mocked tool calls, so they are comparable between models but are not a pure model-latency measurement.": "Durations are wall-clock for the whole case, including mocked tool calls, so they are comparable between models but are not a pure model-latency measurement.", + "AI extraction model": "AI extraction model", + "Do not use AI (one case for the whole conversation)": "Do not use AI (one case for the whole conversation)", + "With a model picked, the conversation is split into one case per scenario it covers. The model only decides where to split and what to name each case -- mocks, assertions and state still come verbatim from the conversation.": "With a model picked, the conversation is split into one case per scenario it covers. The model only decides where to split and what to name each case -- mocks, assertions and state still come verbatim from the conversation.", + "AI extraction additionally sends the user messages and tool names to {provider}. Tool arguments and results are not sent.": "AI extraction additionally sends the user messages and tool names to {provider}. Tool arguments and results are not sent.", + "{count} draft cases recorded. Review them, then enable them.": "{count} draft cases recorded. Review them, then enable them.", + "This run could not complete": "This run could not complete", + "Nothing ran -- see the reason above.": "Nothing ran -- see the reason above.", + "Nothing here can run -- every case you picked is disabled. Enable at least one first.": "Nothing here can run -- every case you picked is disabled. Enable at least one first.", + "{count} of the selected case(s) are disabled and will be skipped.": "{count} of the selected case(s) are disabled and will be skipped." +} diff --git a/src/lib/langs/zh.json b/src/lib/langs/zh.json index c2c06b19..078783bf 100644 --- a/src/lib/langs/zh.json +++ b/src/lib/langs/zh.json @@ -102,7 +102,7 @@ "saas": "萨斯", "crypto": "加密货币", "blog": "博客", - "jobs":"工作" + "jobs": "工作" } }, "layouts": { @@ -214,21 +214,20 @@ "detail": "博客详细信息" } }, - "jobs" : { + "jobs": { "text": "工作", "list": { - "joblist" : "工作清單", - "jobgrid" : "工作網格", - "applyjob" : "申請工作", - "jobdetails" : "工作詳情", - "jobcategories" : "工作類別", - "candidate" : { + "joblist": "工作清單", + "jobgrid": "工作網格", + "applyjob": "申請工作", + "jobdetails": "工作詳情", + "jobcategories": "工作類別", + "candidate": { "text": "候選人", "list": { - "list" : "列表", - "overview" : "概述" + "list": "列表", + "overview": "概述" } - } } }, @@ -296,8 +295,8 @@ "lightbox": "灯箱", "cropper": "圖像裁剪器", "drawer": "抽屉", - "toast":"吐司", - "utility":"效用" + "toast": "吐司", + "utility": "效用" } }, "forms": { @@ -362,139 +361,377 @@ } } }, - - "Agents":"智能代理", - "Evaluator":"评估者", - "Evaluating":"评估器", - "Router":"路由器", + "Agents": "智能代理", + "Evaluator": "评估者", + "Evaluating": "评估器", + "Router": "路由器", "Routing": "路由图", "Plugins": "插件", "Settings": "设置", - "Dashboard":"指示板", - "Agent":"代理", - "Knowledge Base":"知识库", - "Conversation":"对话", - "Task":"任务", - "MongoDB":"MongoDB", - "Home":"主页", - "Search":"搜索", - "Welcome Back !":"欢迎回来 !", - "Admin":"管理", - "Agent Manager":"代理管理器", - "Conversations":"对话", - "Token Cost":"令牌成本", - "View Profile":"查看配置文件", - "Monthly Cost":"每月的成本", - "This month":"这个月", - "View More":"查看更多", - "Total Cost":"总成本", - "Average Cost":"平均成本", - "Week":"周", - "Month":"月", - "Year":"年", - "Token Spent":"令牌用量统计", - "Channel Stats":"通道数据", - "Learn more":"了解更多", - "sales":"销售额", - "Activity":"活动", - "Top Client Usage":"客户端用量", + "Dashboard": "指示板", + "Agent": "代理", + "Knowledge Base": "知识库", + "Conversation": "对话", + "Task": "任务", + "MongoDB": "MongoDB", + "Home": "主页", + "Search": "搜索", + "Welcome Back !": "欢迎回来 !", + "Admin": "管理", + "Agent Manager": "代理管理器", + "Conversations": "对话", + "Token Cost": "令牌成本", + "View Profile": "查看配置文件", + "Monthly Cost": "每月的成本", + "This month": "这个月", + "View More": "查看更多", + "Total Cost": "总成本", + "Average Cost": "平均成本", + "Week": "周", + "Month": "月", + "Year": "年", + "Token Spent": "令牌用量统计", + "Channel Stats": "通道数据", + "Learn more": "了解更多", + "sales": "销售额", + "Activity": "活动", + "Top Client Usage": "客户端用量", "Logout": "登出", "Profile": "配置文件", "Notifications": "通知", - "View All":"查看全部", - "Your order is placed":"您的订单已经下达", - "min ago":"分钟前", - "List":"列表", - "Communication":"沟通", + "View All": "查看全部", + "Your order is placed": "您的订单已经下达", + "min ago": "分钟前", + "List": "列表", + "Communication": "沟通", "Conversation List": "对话列表", "Task List": "任务列表", - "Knowledge Manager":"知识管理", - "Migrate agents from file repository to MongoDB":"将代理从文件存储库迁移到MongoDB", - "Setting":"设置", - "Start Migration":"开始迁移", - "Plugin":"插件", - "My Profile":"我的资料", - "Detail":"细节", - "System & Plugin Settings":"系统和插件设置", - "Disabled":"禁用", - "Enabled":"启用", - "Public":"公共", - "Private":"私人", - "Build":"构建", - "Train":"训练", - "Test":"测试", - "My Files":"我的文件", - "Upload":"上传", - "Recent Files":"最近的文件", - "Storage":"存储", - "0 GB (0%) of 1 GB used":"已使用1GB的0 GB (0%)", - "Provided by BotSharp":"由BotSharp提供", - "Provided by Pizza AI Assistant":"由Pizza AI Assistant提供", - "Upgrade Features":"升级功能", - "Upgrade":"升级", - "Files":"文件", - "Design":"设计", - "Google Drive":"谷歌驱动", - "Starred":"收藏", - "Trash":"回收站", - "Dropbox":"云文件管理", - "Open":"打开", - "Edit":"编辑", - "Rename":"重命名", - "Remove":"移除", - "Share Files":"共享文件", - "Share with me":"与我分享", - "Other Actions":"其他操作", - "Name":"名字", - "Date modified":"修改日期", - "Size":"大小", + "Knowledge Manager": "知识管理", + "Migrate agents from file repository to MongoDB": "将代理从文件存储库迁移到MongoDB", + "Setting": "设置", + "Start Migration": "开始迁移", + "Plugin": "插件", + "My Profile": "我的资料", + "Detail": "细节", + "System & Plugin Settings": "系统和插件设置", + "Disabled": "禁用", + "Enabled": "启用", + "Public": "公共", + "Private": "私人", + "Build": "构建", + "Train": "训练", + "Test": "测试", + "My Files": "我的文件", + "Upload": "上传", + "Recent Files": "最近的文件", + "Storage": "存储", + "0 GB (0%) of 1 GB used": "已使用1GB的0 GB (0%)", + "Provided by BotSharp": "由BotSharp提供", + "Provided by Pizza AI Assistant": "由Pizza AI Assistant提供", + "Upgrade Features": "升级功能", + "Upgrade": "升级", + "Files": "文件", + "Design": "设计", + "Google Drive": "谷歌驱动", + "Starred": "收藏", + "Trash": "回收站", + "Dropbox": "云文件管理", + "Open": "打开", + "Edit": "编辑", + "Rename": "重命名", + "Remove": "移除", + "Share Files": "共享文件", + "Share with me": "与我分享", + "Other Actions": "其他操作", + "Name": "名字", + "Date modified": "修改日期", + "Size": "大小", "Images": "图片", - "Video":"视频", - "Music":"音乐", - "Document":"文档", - "Others":"其他", - "Another action":"其他操作", - "Something else here":"一些别的东西在这里", - "Search for ...":"搜索 ...", - "Status":"状态", - "Completed":"完成", - "Select Channel":"选择通道", - "Live Chat":"即时聊天", - "Phone":"电话", - "Email":"邮箱", - "Filter":"过滤器", - "Title":"标题", - "User Name":"用户名", - "Channel":"通道", - "Posted Date":"发布日期", - "Last Date":"最后日期", - "Action":"操作", - "Active":"活跃中", - "Description":"描述", - "Details":"细节", - "Updated Date":"更新日期", - "Showing":"显示", - "of":"的", - "to":"到", - "entries":"条目", - "View":"查看", - "Install":"安装", - "Dialogs":"对话框", - "Agent Overview":"代理概述", - "Save Agent":"保存代理", - "Conversation Detail":"谈话细节", - "closed":"关闭", - "webchat":"聊天", - "User":"用户", - "Location":"位置", - "Delete Conversation":"删除对话", - "Design & Developed by open source community":"由开源社区设计和开发", - "Personal Information":"个人信息", - "First Name":"名", - "Last Name":"姓", - "Account Origin":"账户起源", - "Update Date":"更新日期", - "Create Date":"创建日期", - "Active now":"正在工作", - "Reset":"重置" -} \ No newline at end of file + "Video": "视频", + "Music": "音乐", + "Document": "文档", + "Others": "其他", + "Another action": "其他操作", + "Something else here": "一些别的东西在这里", + "Search for ...": "搜索 ...", + "Status": "状态", + "Completed": "完成", + "Select Channel": "选择通道", + "Live Chat": "即时聊天", + "Phone": "电话", + "Email": "邮箱", + "Filter": "过滤器", + "Title": "标题", + "User Name": "用户名", + "Channel": "通道", + "Posted Date": "发布日期", + "Last Date": "最后日期", + "Action": "操作", + "Active": "活跃中", + "Description": "描述", + "Details": "细节", + "Updated Date": "更新日期", + "Showing": "显示", + "of": "的", + "to": "到", + "entries": "条目", + "View": "查看", + "Install": "安装", + "Dialogs": "对话框", + "Agent Overview": "代理概述", + "Save Agent": "保存代理", + "Conversation Detail": "谈话细节", + "closed": "关闭", + "webchat": "聊天", + "User": "用户", + "Location": "位置", + "Delete Conversation": "删除对话", + "Design & Developed by open source community": "由开源社区设计和开发", + "Personal Information": "个人信息", + "First Name": "名", + "Last Name": "姓", + "Account Origin": "账户起源", + "Update Date": "更新日期", + "Create Date": "创建日期", + "Active now": "正在工作", + "Reset": "重置", + "Agent Testing": "Agent 测试", + "Test Suites": "测试套件", + "Test Suite": "测试套件", + "Test Cases": "测试用例", + "Test Case": "测试用例", + "Test Run": "测试运行", + "Run History": "运行历史", + "Case Results": "用例结果", + "Suite Settings": "套件设置", + "New Test Suite": "新建测试套件", + "New Test Case": "新建测试用例", + "Edit Test Case": "编辑测试用例", + "Record a Draft Case": "录制草稿用例", + "Turns": "对话轮次", + "Turn": "第几轮", + "Case Assertions": "整案断言", + "Assertions": "断言", + "Initial States": "初始 state", + "Tool Mocks": "工具 mock", + "Observed Tool Calls": "实际工具调用", + "New Suite": "新建套件", + "New Case": "新建用例", + "Add Turn": "添加轮次", + "Add Assertion": "添加断言", + "Add State": "添加 state", + "Add State Write": "添加 state 写入", + "Add Mock": "添加 mock", + "Run All Enabled": "运行全部启用用例", + "Run Selected": "运行选中用例", + "Cancel Run": "取消运行", + "Re-run Failures": "重跑失败用例", + "Record from Conversation": "从会话录制", + "Record": "录制", + "Refresh": "刷新", + "Retry": "重试", + "Save": "保存", + "Create": "创建", + "Cancel": "取消", + "Close": "关闭", + "Delete": "删除", + "Enable": "启用", + "Disable": "停用", + "Back": "返回", + "Back to Suite": "返回套件", + "Back to Suites": "返回套件列表", + "Show details": "展开详情", + "Hide details": "收起详情", + "Clear selection": "清除选择", + "All Agents": "全部 Agent", + "Select Agent": "选择 Agent", + "Select all cases": "全选用例", + "Select case {name}": "选择用例 {name}", + "Edit case": "编辑用例", + "Delete case": "删除用例", + "Enable case": "启用用例", + "Disable case": "停用用例", + "View run": "查看运行", + "Remove turn": "删除该轮", + "Move turn up": "上移该轮", + "Move turn down": "下移该轮", + "Remove assertion": "删除该断言", + "Remove state": "删除该 state", + "Remove mock": "删除该 mock", + "Source": "来源", + "Scope": "范围", + "Started": "开始时间", + "Duration": "耗时", + "Result": "结果", + "Type": "类型", + "Target": "目标", + "Expected": "期望值", + "Actual": "实际值", + "Message": "说明", + "Function": "函数", + "Outcome": "处理结果", + "Args": "入参", + "Key": "键", + "Value": "值", + "Total": "总数", + "Mocks": "mock 数", + "Mock": "mock", + "Run": "运行", + "Fatal": "致命", + "Global": "全局", + "Active rounds": "有效轮数", + "Function name": "函数名", + "Result content": "返回内容", + "Stop completion": "中止本轮续写", + "User message": "用户消息", + "Agent output": "Agent 输出", + "Conversation id": "会话 id", + "Judge provider": "判官 provider", + "Judge model": "判官 model", + "Case timeout": "用例超时", + "Case timeout (seconds)": "用例超时（秒）", + "Extra allowed functions": "额外放行的函数", + "Force blocked functions": "强制阻断的函数", + "Unmocked tool policy": "未 mock 工具策略", + "Enabled (included in runs)": "启用（纳入运行）", + "Triggered by": "触发者", + "Args match (JSON subset, optional)": "入参匹配（JSON 子集，可选）", + "Call index (0-based, optional)": "调用序号（0 基，可选）", + "Enter suite name": "输入套件名称", + "Optional description": "可选描述", + "function name": "函数名", + "state key": "state 键名", + "An existing conversation with at least one tool call": "一个至少包含一次工具调用的已有会话", + "Pending": "等待中", + "Running": "运行中", + "Passed": "通过", + "Failed": "失败", + "Error": "异常", + "Cancelled": "已取消", + "Cancelling": "取消中", + "Mocked": "已 mock", + "Blocked": "已阻断", + "Pass": "通过", + "Fail": "不通过", + "Draft": "草稿", + "Recorded": "录制", + "Manual": "手写", + "Subset": "子集", + "All enabled": "全部启用", + "passed": "通过", + "failed": "失败", + "errored": "异常", + "Failed (assertion)": "失败（断言不通过）", + "Errored (never ran)": "异常（未跑完）", + "Harness error": "测试台错误", + "Partial run of": "本次只跑了", + "selected case(s).": "个选中的用例。", + "No test suites yet.": "还没有测试套件。", + "No test suites for this agent yet.": "这个 Agent 还没有测试套件。", + "No test cases in this suite yet.": "这个套件里还没有测试用例。", + "This suite has never been run.": "这个套件从未运行过。", + "No case results yet.": "还没有用例结果。", + "This run produced no case results.": "这次运行没有产生任何用例结果。", + "No turns yet. A case needs at least one.": "还没有轮次。一条用例至少要有一轮。", + "No case-level assertions.": "没有整案级断言。", + "No initial states.": "没有初始 state。", + "No assertions on this turn.": "这一轮没有断言。", + "The agent called no tools during this case.": "这条用例执行期间 Agent 没有调用任何工具。", + "No mocks. Every tool call this case makes will be blocked.": "没有 mock。这条用例发起的每一次工具调用都会被阻断。", + "A disabled suite cannot be run at all.": "停用的套件完全无法运行。", + "This suite is disabled. Triggering a run is rejected by the server until you enable it in Settings.": "这个套件已停用。在设置里重新启用之前，服务端会拒绝任何运行请求。", + "llmJudge assertions always fail in P1 regardless of these two fields.": "P1 阶段 llmJudge 断言恒定判失败，与这两个字段是否配置无关。", + "Always fails in P1.": "P1 阶段恒定判失败。", + "One function name per line. These run for real during a test -- only list functions with no side effects.": "每行一个函数名。这些函数在测试期间会真实执行——只填没有副作用的函数。", + "One per line. Blocked even if allow-listed elsewhere.": "每行一个。即使在别处被放行也照样阻断。", + "Assertions checked right after this turn": "本轮结束后立即求值的断言", + "Evaluated once, after every turn has run.": "全部轮次跑完后求值一次。", + "Injected before the conversation starts. Active rounds -1 means it never expires.": "会话开始前注入。有效轮数为 -1 表示永不过期。", + "Every tool this case does not mock is blocked. If the agent needs a tool to move forward, mock it here.": "这条用例没有 mock 的工具一律被阻断。Agent 需要哪个工具才能往下走，就在这里给它加 mock。", + "State written when this mock is hit": "命中这个 mock 时写入的 state", + "Many tools pass data to later turns through state, not through their return value.": "很多工具是靠 state 而不是靠返回值把数据传给后续轮次的。", + "Any tool this case does not mock is blocked instead of executed. P1 has no other option.": "这条用例没有 mock 的工具会被阻断而不是执行。P1 阶段没有别的选项。", + "Recorded from conversation": "录制自会话", + "It may contain live customer data -- review before enabling.": "可能包含线上客户数据——启用前请先审阅。", + "Recorded from conversation {id}. May contain live customer data.": "录制自会话 {id}，可能包含线上客户数据。", + "Recording copies the raw conversation into the test store, including any phone numbers, addresses and tenant names it contains. The draft lands disabled -- review it before enabling.": "录制会把原始会话内容复制进测试库，其中包含的电话号码、地址、租户姓名也一并复制。生成的草稿默认停用——请审阅后再启用。", + "Errored cases never reached their assertions -- a timeout, a mock-seam failure, or a case with no turns. That is a harness problem, not an agent regression.": "异常的用例根本没走到断言——可能是超时、mock 接缝失效，或者用例没有轮次。这是测试台的问题，不是 Agent 回归。", + "This run is still going. Refreshing every couple of seconds.": "这次运行还在进行中，每隔几秒自动刷新。", + "The agent tried to call a tool this case does not mock, so it was blocked. This is often the root cause of the failure above.": "Agent 试图调用一个这条用例没有 mock 的工具，因此被阻断。这通常正是上面那个失败的根因。", + "Are you sure?": "确定吗？", + "Yes, delete it!": "确定删除", + "Yes, run it": "确定运行", + "Run this suite?": "运行这个套件？", + "Delete test suite \"{name}\"? You won't be able to revert this!": "删除测试套件“{name}”？此操作不可撤销！", + "Delete test case \"{name}\"? You won't be able to revert this!": "删除测试用例“{name}”？此操作不可撤销！", + "{count} case(s) will run against the live model. This costs real tokens and is not rate limited.": "将有 {count} 条用例真实调用模型运行。这会消耗真实 token，且没有任何限流。", + "Test suite created!": "测试套件已创建！", + "Test suite deleted!": "测试套件已删除！", + "Test case created!": "测试用例已创建！", + "Test case saved!": "测试用例已保存！", + "Test case deleted!": "测试用例已删除！", + "Suite settings saved!": "套件设置已保存！", + "Case enabled.": "用例已启用。", + "Case disabled.": "用例已停用。", + "Run queued!": "运行已入队！", + "Re-run queued!": "重跑已入队！", + "Cancellation requested.": "已请求取消。", + "Draft case recorded. Review it, then enable it.": "草稿用例已录制。请审阅后再启用。", + "Failed to load test suites. Please try again.": "加载测试套件失败，请重试。", + "Failed to load this test suite.": "加载这个测试套件失败。", + "Failed to load this test case.": "加载这条测试用例失败。", + "Failed to load the suite this case belongs to.": "加载这条用例所属的套件失败。", + "Failed to load this run.": "加载这次运行失败。", + "Failed to create test suite.": "创建测试套件失败。", + "Failed to delete test suite.": "删除测试套件失败。", + "Failed to save suite settings.": "保存套件设置失败。", + "Failed to save the test case.": "保存测试用例失败。", + "Failed to update the case.": "更新测试用例失败。", + "Failed to delete the test case.": "删除测试用例失败。", + "Failed to start the run.": "启动运行失败。", + "Failed to start the re-run.": "启动重跑失败。", + "Failed to cancel the run.": "取消运行失败。", + "Failed to record a draft from that conversation.": "从该会话录制草稿失败。", + "You do not have permission for this. Running and recording require an admin or root account.": "你没有权限执行此操作。运行和录制需要 admin 或 root 账号。", + "Fix these before saving:": "保存前请先修复：", + "Name is required.": "名称必填。", + "Add at least one turn -- a case with no turns errors when it runs.": "至少添加一轮——没有轮次的用例运行时会直接异常。", + "Turn {n}: the user message is empty.": "第 {n} 轮：用户消息为空。", + "Turn {n}, assertion {m}: {error}": "第 {n} 轮第 {m} 条断言：{error}", + "Case assertion {n}: {error}": "第 {n} 条整案断言：{error}", + "Initial state {n}: the key is empty.": "第 {n} 条初始 state：键名为空。", + "Mock {n}: the function name is empty.": "第 {n} 个 mock：函数名为空。", + "Mock {n}: the args match is not valid JSON.": "第 {n} 个 mock：入参匹配不是合法 JSON。", + "Mock {n}: the call index is 0-based and cannot be negative.": "第 {n} 个 mock：调用序号从 0 开始，不能为负。", + "Mock {n}, state write {m}: the key is empty.": "第 {n} 个 mock 的第 {m} 条 state 写入：键名为空。", + "Assertion type is required.": "断言类型必填。", + "Assertion \"{type}\" needs an expected value.": "断言“{type}”需要填期望值。", + "Assertion \"{type}\" needs a target.": "断言“{type}”需要填目标。", + "Assertion \"stateEquals\" needs an expected value, otherwise it can never pass.": "断言“stateEquals”必须填期望值，否则它永远不可能通过。", + "\"{pattern}\" is not a valid regular expression.": "“{pattern}”不是合法的正则表达式。", + "The args match on an assertion is not valid JSON.": "某条断言的入参匹配不是合法 JSON。", + "{ms} ms": "{ms} 毫秒", + "{seconds} s": "{seconds} 秒", + "Not configured": "未配置", + "This provider has no chat-capable model registered.": "该 provider 下没有注册任何支持 chat 的模型。", + "Case": "用例", + "Models": "模型", + "models": "个模型", + "Model Comparison": "模型对比", + "Running every enabled case": "本次运行全部启用用例", + "Running the selected cases": "本次运行选中的用例", + "Use the agent's own model": "改用 agent 自己的模型", + "Pick nothing to run once on whatever model each agent is configured with. Pick two or more to run the whole set once per model and compare them side by side.": "不选就按各 agent 自己配置的模型跑一遍。选两个及以上，整套用例会在每个模型上各跑一遍，结果并排对比。", + "No chat-capable model is registered, so this run will use the agent's own configuration.": "没有注册任何支持 chat 的模型，本次运行会使用 agent 自己的配置。", + "{count} execution(s) will run against live models. This costs real tokens and is not rate limited.": "将真实调用模型执行 {count} 次。这会消耗真实 token，且没有任何限流。", + "Durations are wall-clock for the whole case, including mocked tool calls, so they are comparable between models but are not a pure model-latency measurement.": "耗时是整条用例的墙钟时间，包含 mock 工具调用在内，所以模型之间可比，但不是纯粹的模型延迟。", + "AI extraction model": "AI 抽取模型", + "Do not use AI (one case for the whole conversation)": "不使用 AI（整段会话生成一条用例）", + "With a model picked, the conversation is split into one case per scenario it covers. The model only decides where to split and what to name each case -- mocks, assertions and state still come verbatim from the conversation.": "选了模型后，会话会按覆盖的场景切成多条用例。模型只决定切在哪、每条叫什么——mock、断言和 state 仍旧逐字来自真实会话。", + "AI extraction additionally sends the user messages and tool names to {provider}. Tool arguments and results are not sent.": "AI 抽取会额外把用户消息和函数名发给 {provider}。函数入参与返回内容不外发。", + "{count} draft cases recorded. Review them, then enable them.": "已录制 {count} 条草稿用例。请逐条审阅后再启用。", + "This run could not complete": "这次运行没能执行", + "Nothing ran -- see the reason above.": "什么都没跑，原因见上方。", + "Nothing here can run -- every case you picked is disabled. Enable at least one first.": "这样跑不起来——你勾选的用例全部处于停用状态。请先至少启用一条。", + "{count} of the selected case(s) are disabled and will be skipped.": "勾选的用例里有 {count} 条处于停用状态，将被跳过。" +} diff --git a/src/lib/services/agent-test-service.js b/src/lib/services/agent-test-service.js new file mode 100644 index 00000000..cc41d747 --- /dev/null +++ b/src/lib/services/agent-test-service.js @@ -0,0 +1,210 @@ +import { endpoints } from '$lib/services/api-endpoints.js'; +import axios from 'axios'; + +/** + * Get agent test suites, optionally filtered by agent + * @param {string?} [agentId] + * @returns {Promise} + */ +export async function getSuites(agentId = null) { + const url = endpoints.agentTestSuiteListUrl; + const response = await axios.get(url, { + params: { agentId: agentId } + }); + return response.data; +} + +/** + * Get one agent test suite + * @param {string} id + * @returns {Promise} + */ +export async function getSuite(id) { + const url = endpoints.agentTestSuiteDetailUrl.replace("{id}", id); + const response = await axios.get(url); + return response.data; +} + +/** + * Create an agent test suite + * @param {import('$agentTestTypes').AgentTestSuiteUpsertRequest} body + * @returns {Promise} + */ +export async function createSuite(body) { + const url = endpoints.agentTestSuiteListUrl; + const response = await axios.post(url, body); + return response.data; +} + +/** + * Replace an agent test suite. This is a FULL replace, not a patch -- GET the + * complete suite first, change only what the user touched, and send every + * field back. caseTimeoutSeconds/extraAllowedFunctions/forceBlockedFunctions + * are reset to their defaults by the backend if omitted (agentId/name fall + * back to the existing value instead). + * @param {string} id + * @param {import('$agentTestTypes').AgentTestSuiteUpsertRequest} body + * @returns {Promise} + */ +export async function updateSuite(id, body) { + const url = endpoints.agentTestSuiteDetailUrl.replace("{id}", id); + const response = await axios.put(url, body); + return response.data; +} + +/** + * Delete an agent test suite + * @param {string} id + */ +export async function deleteSuite(id) { + const url = endpoints.agentTestSuiteDetailUrl.replace("{id}", id); + await axios.delete(url); +} + +/** + * Get the test cases in a suite (newest first) + * @param {string} suiteId + * @returns {Promise} + */ +export async function getCases(suiteId) { + const url = endpoints.agentTestCaseListUrl; + const response = await axios.get(url, { + params: { suiteId: suiteId } + }); + return response.data; +} + +/** + * Get one test case + * @param {string} id + * @returns {Promise} + */ +export async function getCase(id) { + const url = endpoints.agentTestCaseDetailUrl.replace("{id}", id); + const response = await axios.get(url); + return response.data; +} + +/** + * Create a test case + * @param {import('$agentTestTypes').AgentTestCaseUpsertRequest} body + * @returns {Promise} + */ +export async function createCase(body) { + const url = endpoints.agentTestCaseListUrl; + const response = await axios.post(url, body); + return response.data; +} + +/** + * Replace a test case. This is a FULL replace, not a patch -- GET the complete + * case first, change only what the user touched, and send every field back. + * Any field with no editor control (e.g. initialStates/sourceConversationId on + * a recorded case) must still be carried through untouched, or saving once + * silently clears it. + * @param {string} id + * @param {import('$agentTestTypes').AgentTestCaseUpsertRequest} body + * @returns {Promise} + */ +export async function updateCase(id, body) { + const url = endpoints.agentTestCaseDetailUrl.replace("{id}", id); + const response = await axios.put(url, body); + return response.data; +} + +/** + * Delete a test case + * @param {string} id + */ +export async function deleteCase(id) { + const url = endpoints.agentTestCaseDetailUrl.replace("{id}", id); + await axios.delete(url); +} + +/** + * Trigger a run of a suite. This is fire-and-forget: the run is created + * Pending and queued, and this resolves immediately with the run row -- it + * does not wait for the run to finish. + * @param {string} suiteId + * @param {string[]?} [caseIds] - Only run these cases; omit/empty to run every enabled case in the suite. + * @param {import('$agentTestTypes').TestModel[]?} [models] - Run every case once per model listed here; + * omit/empty to run a single pass using each agent's own LlmConfig. N models means N times the + * executions and N times the token cost. + * @returns {Promise} + */ +export async function triggerRun(suiteId, caseIds = null, models = null) { + const url = endpoints.agentTestSuiteRunUrl.replace("{id}", suiteId); + const response = await axios.post(url, { caseIds: caseIds, models: models }); + return response.data; +} + +/** + * Get the run history for a suite + * @param {string} suiteId + * @returns {Promise} + */ +export async function getRuns(suiteId) { + const url = endpoints.agentTestRunListUrl; + const response = await axios.get(url, { + params: { suiteId: suiteId } + }); + return response.data; +} + +/** + * Get a run together with every one of its case results + * @param {string} id + * @returns {Promise} + */ +export async function getRun(id) { + const url = endpoints.agentTestRunDetailUrl.replace("{id}", id); + const response = await axios.get(url); + return response.data; +} + +/** + * Cancel a pending or running run. The backend returns 409 if the run has + * already reached a terminal status (Passed/Failed/Error/Cancelled). + * @param {string} id + */ +export async function cancelRun(id) { + const url = endpoints.agentTestRunCancelUrl.replace("{id}", id); + await axios.post(url); +} + +/** + * Record one or more draft test cases from a real conversation. Every returned + * case comes back with enabled=false -- each must be reviewed and explicitly + * enabled before it joins a normal run. + * @param {string} conversationId + * @param {string} suiteId + * @param {import('$agentTestTypes').TestModel?} [model] - Use this model to split the + * conversation into one or more scenarios, each becoming its own case. Omit to use the + * deterministic recorder, which always produces exactly one case and calls no model. + * The model only decides where to cut and what to name each case; mocks, assertions and + * state still come verbatim from the conversation. Passing a model sends the conversation's + * user messages and tool NAMES to that vendor (tool arguments and results are withheld). + * @returns {Promise} + */ +export async function recordCases(conversationId, suiteId, model = null) { + const url = endpoints.agentTestRecordUrl; + const response = await axios.post(url, { + conversationId: conversationId, + suiteId: suiteId, + model: model + }); + return response.data; +} + +/** + * Get the mock-target candidates (function names) for an agent's mock editor + * @param {string} agentId + * @returns {Promise} + */ +export async function getMockTargets(agentId) { + const url = endpoints.agentTestMockTargetsUrl; + const response = await axios.get(url, { + params: { agentId: agentId } + }); + return response.data; +} diff --git a/src/lib/services/api-endpoints.js b/src/lib/services/api-endpoints.js index ff75973d..8984b2b8 100644 --- a/src/lib/services/api-endpoints.js +++ b/src/lib/services/api-endpoints.js @@ -135,6 +135,18 @@ export const endpoints = { // Google geocode api addressUrl: `${host}/address/options`, - mcpServerConfigsUrl: `${host}/mcp/server-configs` + mcpServerConfigsUrl: `${host}/mcp/server-configs`, + + // agent test + agentTestSuiteListUrl: `${host}/agent-test/suites`, + agentTestSuiteDetailUrl: `${host}/agent-test/suites/{id}`, + agentTestSuiteRunUrl: `${host}/agent-test/suites/{id}/run`, + agentTestCaseListUrl: `${host}/agent-test/cases`, + agentTestCaseDetailUrl: `${host}/agent-test/cases/{id}`, + agentTestRunListUrl: `${host}/agent-test/runs`, + agentTestRunDetailUrl: `${host}/agent-test/runs/{id}`, + agentTestRunCancelUrl: `${host}/agent-test/runs/{id}/cancel`, + agentTestRecordUrl: `${host}/agent-test/record`, + agentTestMockTargetsUrl: `${host}/agent-test/mock-targets` } diff --git a/src/routes/page/agent-test/+page.svelte b/src/routes/page/agent-test/+page.svelte new file mode 100644 index 00000000..9e3427be --- /dev/null +++ b/src/routes/page/agent-test/+page.svelte @@ -0,0 +1,423 @@ + + + + + + + +
+
+
+
+
+
{$_('Test Suites')}
+ +
+
+
+
+
+ changeNewSuiteAgent(e)} + /> +
+
+ + +
+
+ + +
+ +
+
+ + +
+
+
+
+
+{/if} diff --git a/src/routes/page/agent-test/[suiteId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/+page.svelte new file mode 100644 index 00000000..b700f156 --- /dev/null +++ b/src/routes/page/agent-test/[suiteId]/+page.svelte @@ -0,0 +1,1150 @@ + + + + + + + +{#if loadErrorText} +
+
+ +
+
+{:else if suite} +
+
+
+
+
+
+
+ {suite.name} + {#if suite.enabled} + {$_('Enabled')} + {:else} + {$_('Disabled')} + {/if} +
+

+ {$_('Agent')}: {agentName(suite.agentId)} + | + {$_('Case timeout')}: {suite.caseTimeoutSeconds}s +

+ {#if suite.description} +

{suite.description}

+ {/if} +
+
+ + + + + {#if selectedCaseIds.length > 0} + + {/if} + +
+
+ {#if !suite.enabled} + + {/if} +
+
+
+
+ +
+
+
+
+
+
{$_('Test Cases')} ({cases.length})
+ {#if selectedCaseIds.length > 0} + + {/if} +
+
+
+ {#if cases.length === 0} +
+

{$_('No test cases in this suite yet.')}

+
+ + +
+
+ {:else} +
+ + + + + + + + + + + + + + + {#each cases as testCase (testCase.id)} + {@const assertionCount = (testCase.assertions?.length || 0) + + (testCase.turns || []).reduce((sum, t) => sum + (t.assertions?.length || 0), 0)} + + + + + + + + + + + {/each} + +
+ toggleAllCases()} + /> + {$_('Name')}{$_('Turns')}{$_('Mocks')}{$_('Assertions')}{$_('Source')}{$_('Enabled')}{$_('Action')}
+ toggleCase(testCase.id)} + /> + + + {testCase.turns?.length || 0}{testCase.mocks?.length || 0}{assertionCount} + {#if testCase.sourceConversationId} + + {$_('Recorded')} + + {:else} + {$_('Manual')} + {/if} + + {#if testCase.enabled} + {$_('Enabled')} + {:else} + {$_('Draft')} + {/if} + +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+
+
+ {/if} +
+
+
+
+ +
+
+
+
+
+
{$_('Run History')}
+ +
+
+
+ {#if runs.length === 0} +

{$_('This suite has never been run.')}

+ {:else} +
+ + + + + + + + + + + + + {#each runs as run (run.id)} + + + + + + + + + {/each} + +
{$_('Status')}{$_('Result')}{$_('Scope')}{$_('Started')}{$_('Duration')}{$_('Action')}
+ {$_(run.status)} + {#if run.cancelRequested && !isTerminalStatus(run.status)} + {$_('Cancelling')} + {/if} + {#if run.error} +
{run.error}
+ {/if} + 		+ {run.passedCount} {$_('passed')}, + {run.failedCount} {$_('failed')}, + {run.errorCount} {$_('errored')} + / {run.totalCount} + + {#if run.caseIds && run.caseIds.length > 0} + {$_('Subset')} ({run.caseIds.length}) + {:else} + {$_('All enabled')} + {/if} + {#if run.models && run.models.length > 0} + `${m.provider}/${m.model}`).join('\n')} + > + {run.models.length} {$_('models')} + + {/if} + {formatDateTime(run.startedAt)}{runDuration(run)} +
    +
  • + +
    • + {#if !isTerminalStatus(run.status)} +
  • + +
    • + {/if} +
+
+
+ {/if} +
+
+
+
+{/if} + +{#if isSettingsOpen} + +
+{/if} + +{#if isRunModalOpen} + +
+{/if} + +{#if isRecordOpen} + +
+{/if} diff --git a/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte b/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte new file mode 100644 index 00000000..a1fd9b04 --- /dev/null +++ b/src/routes/page/agent-test/[suiteId]/case/[caseId]/+page.svelte @@ -0,0 +1,775 @@ + + +{#snippet assertionRows(list, idPrefix)} + {#each list as assertion, i} +
+
+
+ + + {#if assertion.type === 'llmJudge'} +
{$_('Always fails in P1.')}
+ {/if} +
+
+ + +
+
+ + +
+
+
+ + +
+ +
+ {#if assertion.type === 'toolCalled'} +
+ + +
+ {/if} +
+
+ {/each} +{/snippet} + +{#snippet stateRows(list, idPrefix)} + {#each list as state, i} +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+
+ +
+
+ {/each} +{/snippet} + + + + + + + + {#each mockTargets as target} + + {/each} + + +{#if loadErrorText} +
+
+ +
+
+{:else} +
+
+
+
+
+
{isNew ? $_('New Test Case') : $_('Edit Test Case')}
+
+ + +
+
+
+ + {#if validationErrors.length > 0} +
+ +
+ {/if} + +
+
+
+ + +
+
+
+ + +
+
+ {#if form.sourceConversationId} +
+ +
+ {/if} +
+
+ {$_('Unmocked tool policy')}: {form.unmockedToolPolicy}. + {$_('Any tool this case does not mock is blocked instead of executed. P1 has no other option.')} +
+
+
+
+
+
+
+ +
+
+
+
+
+
{$_('Turns')} ({form.turns.length})
+ +
+
+
+ {#each form.turns as turn, i (i)} +
+
+
{$_('Turn')} {i + 1}
+
+ + + +
+
+
+ + +
+
+ {$_('Assertions checked right after this turn')} ({turn.assertions.length}) + +
+ {@render assertionRows(turn.assertions, `turn-${i}-assertion`)} +
+ {/each} + {#if form.turns.length === 0} +

{$_('No turns yet. A case needs at least one.')}

+ {/if} +
+
+
+
+ +
+
+
+
+
+
{$_('Case Assertions')} ({form.assertions.length})
+ +
+
+
+

{$_('Evaluated once, after every turn has run.')}

+ {@render assertionRows(form.assertions, 'case-assertion')} + {#if form.assertions.length === 0} +

{$_('No case-level assertions.')}

+ {/if} +
+
+
+
+ +
+
+
+
+
+
{$_('Initial States')} ({form.initialStates.length})
+ +
+
+
+

{$_('Injected before the conversation starts. Active rounds -1 means it never expires.')}

+ {@render stateRows(form.initialStates, 'initial-state')} + {#if form.initialStates.length === 0} +

{$_('No initial states.')}

+ {/if} +
+
+
+
+ +
+
+
+
+
+
{$_('Tool Mocks')} ({form.mocks.length})
+ +
+
+
+

+ {$_('Every tool this case does not mock is blocked. If the agent needs a tool to move forward, mock it here.')} +

+ {#each form.mocks as mock, i (i)} +
+
+
{$_('Mock')} {i + 1}
+ +
+
+
+ + +
+
+ + +
+
+
+ + +
+
+
+ + +
+
+ + +
+
+
+ + {$_('State written when this mock is hit')} ({mock.stateWrites?.length || 0}). + {$_('Many tools pass data to later turns through state, not through their return value.')} + + +
+ {@render stateRows(mock.stateWrites || [], `mock-${i}-state`)} +
+
+
+ {/each} + {#if form.mocks.length === 0} +

{$_('No mocks. Every tool call this case makes will be blocked.')}

+ {/if} +
+
+
+
+ +
+
+
+
+ + +
+
+
+
+{/if} diff --git a/src/routes/page/agent-test/run/[runId]/+page.svelte b/src/routes/page/agent-test/run/[runId]/+page.svelte new file mode 100644 index 00000000..26c09148 --- /dev/null +++ b/src/routes/page/agent-test/run/[runId]/+page.svelte @@ -0,0 +1,542 @@ + + +{#snippet assertionTable(list)} +
+ + + + + + + + + + + + + {#each list as assertion, i (i)} + + + + + + + + + {/each} + +
{$_('Result')}{$_('Type')}{$_('Target')}{$_('Expected')}{$_('Actual')}{$_('Message')}
+ {#if assertion.passed} + {$_('Pass')} + {:else} + {$_('Fail')} + {/if} + {assertion.type}{assertion.target || '--'}{assertion.expected || '--'}{assertion.actual || '--'}{assertion.message || '--'}
+
+{/snippet} + + + + + + +{#if loadErrorText} +
+
+ +
+
+{:else if run} +
+
+
+
+
+
+
+ {$_('Run')} {run.id} + {$_(run.status)} + {#if run.cancelRequested && !isFinished} + {$_('Cancelling')} + {/if} +
+

+ {$_('Started')}: {formatDateTime(run.startedAt)} + | + {$_('Completed')}: {formatDateTime(run.completedAt)} + {#if run.triggeredBy} + | + {$_('Triggered by')}: {run.triggeredBy} + {/if} +

+ {#if run.caseIds && run.caseIds.length > 0} +

+ {$_('Partial run of')} {run.caseIds.length} {$_('selected case(s).')} +

+ {/if} +
+
+ + + {#if !isFinished} + + {:else if failedCaseIds.length > 0} + + {/if} +
+
+
+
+
+
+
+

{run.totalCount}

+ {$_('Total')} +
+
+
+
+

{run.passedCount}

+ {$_('Passed')} +
+
+
+
+

{run.failedCount}

+ {$_('Failed (assertion)')} +
+
+
+
+

{run.errorCount}

+ {$_('Errored (never ran)')} +
+
+
+ {#if run.error} + + + {/if} + {#if run.errorCount > 0} + + {/if} + {#if !isFinished} +

+ {$_('This run is still going. Refreshing every couple of seconds.')} +

+ {/if} +
+
+
+
+ + {#if isComparison} +
+
+
+
+
{$_('Model Comparison')}
+
+
+
+ + + + + {#each modelColumns as col (col.key)} + + {/each} + + + + {#each comparisonRows as row (row.caseId)} + + + {#each modelColumns as col (col.key)} + {@const cell = row.cells[col.key]} + + {/each} + + {/each} + + + + + {#each modelSummaries as summary (summary.key)} + + {/each} + + +
{$_('Case')} +
{col.model}
+
{col.provider}
+
{row.caseName} + {#if cell} + {$_(cell.status)} + {formatDuration(cell.durationMs)} + {:else} + -- + {/if} +
{$_('Total')} +
{summary.passed}/{summary.total} {$_('passed')}
+
{formatDuration(summary.totalMs)}
+
+
+

+ {$_('Durations are wall-clock for the whole case, including mocked tool calls, so they are comparable between models but are not a pure model-latency measurement.')} +

+
+
+
+
+ {/if} + +
+
+
+
+
{$_('Case Results')} ({results.length})
+
+
+ {#if results.length === 0} +

+ {#if run.error} + {$_('Nothing ran -- see the reason above.')} + {:else} + {isFinished ? $_('This run produced no case results.') : $_('No case results yet.')} + {/if} +

+ {:else} + {#each results as result (result.id)} + {@const isExpanded = expandedIds.includes(result.id)} +
+
+
+ {$_(result.status)} + {result.caseName} + {#if result.model} + + {result.model} + {/if} + {formatDuration(result.durationMs)} + {#if result.conversationId} + {result.conversationId} + {/if} +
+ +
+ + {#if result.error} +
+ +
+ {/if} + + {#if isExpanded} +
+ {#each result.turns as turn (turn.index)} +
+
{$_('Turn')} {turn.index + 1}
+
+
{$_('User')}
+
{turn.userMessage}
+
+
+
{$_('Agent output')}
+
{turn.output || '--'}
+
+ {#if turn.assertions?.length > 0} + {@render assertionTable(turn.assertions)} + {:else} +

{$_('No assertions on this turn.')}

+ {/if} +
+ {/each} + + {#if result.assertions?.length > 0} +
+
{$_('Case Assertions')}
+ {@render assertionTable(result.assertions)} +
+ {/if} + +
+
{$_('Observed Tool Calls')} ({result.observedToolCalls?.length || 0})
+ {#if result.observedToolCalls?.length > 0} +
+ + + + + + + + + + + + {#each result.observedToolCalls as call, i (i)} + + + + + + + + {/each} + +
{$_('Turn')}{$_('Function')}{$_('Outcome')}{$_('Args')}{$_('Result')}
{call.turnIndex + 1}{call.functionName} + {#if call.outcome === 'Blocked'} + + {$_('Blocked')} + + {:else} + {$_(call.outcome)} + {/if} + {call.argsJson || '--'}{call.resultContent || '--'}
+
+ {:else} +

{$_('The agent called no tools during this case.')}

+ {/if} +
+
+ {/if} +
+ {/each} + {/if} +
+
+
+
+{/if} diff --git a/svelte.config.js b/svelte.config.js index bf5de71d..130d3715 100644 --- a/svelte.config.js +++ b/svelte.config.js @@ -18,7 +18,8 @@ const config = { $pluginTypes: './src/lib/helpers/types/pluginTypes.js', $realtimeTypes: './src/lib/helpers/types/realtimeTypes.js', $instructTypes: './src/lib/helpers/types/instructTypes.js', - $mcpTypes: './src/lib/helpers/types/mcpTypes.js' + $mcpTypes: './src/lib/helpers/types/mcpTypes.js', + $agentTestTypes: './src/lib/helpers/types/agentTestTypes.js' }, // for static deployment @@ -75,7 +76,11 @@ const config = { "/page/knowledge-base/relationships", "/page/knowledge-base/documents", "/page/knowledge-base/dictionary", - "/page/knowledge-base/[embed]/[embedType]" + "/page/knowledge-base/[embed]/[embedType]", + "/page/agent-test", + "/page/agent-test/[suiteId]", + "/page/agent-test/[suiteId]/case/[caseId]", + "/page/agent-test/run/[runId]" ] } },