Feature/agent test set p1

BotSharp.sln

@@ -165,6 +165,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.MicrosoftTe
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Core.UnitTests", "tests\BotSharp.Core.UnitTests\BotSharp.Core.UnitTests.csproj", "{3585AE68-43CE-B724-72C1-579D9FFE1D6E}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.AgentTesting", "src\Plugins\BotSharp.Plugin.AgentTesting\BotSharp.Plugin.AgentTesting.csproj", "{57CE6EAE-8957-435A-8A14-5A693EA8B520}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -955,6 +957,18 @@ Global
 		{3585AE68-43CE-B724-72C1-579D9FFE1D6E}.Release|x64.Build.0 = Release|Any CPU
 		{3585AE68-43CE-B724-72C1-579D9FFE1D6E}.Release|x86.ActiveCfg = Release|Any CPU
 		{3585AE68-43CE-B724-72C1-579D9FFE1D6E}.Release|x86.Build.0 = Release|Any CPU
+		{57CE6EAE-8957-435A-8A14-5A693EA8B520}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{57CE6EAE-8957-435A-8A14-5A693EA8B520}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{57CE6EAE-8957-435A-8A14-5A693EA8B520}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{57CE6EAE-8957-435A-8A14-5A693EA8B520}.Debug|x64.Build.0 = Debug|Any CPU
+		{57CE6EAE-8957-435A-8A14-5A693EA8B520}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{57CE6EAE-8957-435A-8A14-5A693EA8B520}.Debug|x86.Build.0 = Debug|Any CPU
+		{57CE6EAE-8957-435A-8A14-5A693EA8B520}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{57CE6EAE-8957-435A-8A14-5A693EA8B520}.Release|Any CPU.Build.0 = Release|Any CPU
+		{57CE6EAE-8957-435A-8A14-5A693EA8B520}.Release|x64.ActiveCfg = Release|Any CPU
+		{57CE6EAE-8957-435A-8A14-5A693EA8B520}.Release|x64.Build.0 = Release|Any CPU
+		{57CE6EAE-8957-435A-8A14-5A693EA8B520}.Release|x86.ActiveCfg = Release|Any CPU
+		{57CE6EAE-8957-435A-8A14-5A693EA8B520}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1034,6 +1048,7 @@ Global
 		{58D3A2C3-F96F-5E57-2C6B-ECE59D6A18FC} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B}
 		{19FA8311-69D1-4729-AE9B-D10979C099EC} = {58D3A2C3-F96F-5E57-2C6B-ECE59D6A18FC}
 		{3585AE68-43CE-B724-72C1-579D9FFE1D6E} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
+		{57CE6EAE-8957-435A-8A14-5A693EA8B520} = {58D3A2C3-F96F-5E57-2C6B-ECE59D6A18FC}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

src/Infrastructure/BotSharp.Abstraction/Routing/Executor/IFunctionExecutorFactory.cs

@@ -0,0 +1,11 @@
+namespace BotSharp.Abstraction.Routing.Executor;
+
+/// <summary>
+/// Decides who executes a given function name. Every function-call path has to go through here,
+/// or IFunctionExecutorProvider gets bypassed -- BotSharp.Core.Rules' ToolCallAction used to be
+/// exactly such a bypass.
+/// </summary>
+public interface IFunctionExecutorFactory
+{
+    IFunctionExecutor? Create(string functionName, Agent agent);
+}

src/Infrastructure/BotSharp.Abstraction/Routing/Executor/IFunctionExecutorProvider.cs

@@ -0,0 +1,14 @@
+namespace BotSharp.Abstraction.Routing.Executor;
+
+/// <summary>
+/// Lets an external component take over the execution of a function. Returning null means "not
+/// mine" and hands off to the next provider, or to the built-in resolution chain. The typical use
+/// is swapping a real tool for a fake during a test, or blocking a function by policy.
+/// </summary>
+public interface IFunctionExecutorProvider
+{
+    /// <summary>Lower is asked first.</summary>
+    int Order => 0;
+
+    IFunctionExecutor? TryResolve(string functionName, Agent agent);
+}

src/Infrastructure/BotSharp.Core.Rules/Actions/ToolCallAction.cs

@@ -1,5 +1,6 @@
 using BotSharp.Abstraction.Functions;
 using BotSharp.Abstraction.Graph.Models;
+using BotSharp.Abstraction.Routing.Executor;
 
 namespace BotSharp.Core.Rules.Actions;
 
@@ -43,9 +44,27 @@ public async Task<RuleNodeResult> ExecuteAsync(
         RuleFlowContext context)
     {
         var funcName = context.Parameters.TryGetValue("function_name", out var fName) ? fName : null;
-        var func = _services.GetServices<IFunctionCallback>().FirstOrDefault(x => x.Name.IsEqualTo(funcName));
 
-        if (func == null)
+        // A missing/blank function_name has to fail gracefully. The old IsEqualTo lookup was
+        // null-safe -- it simply matched no callback. Going through the factory instead means null
+        // reaches IFunctionExecutorProvider.TryResolve, whose contract declares a non-null string,
+        // and some implementations (a mock/blocking provider doing a Dictionary lookup by name, for
+        // instance) would throw ArgumentNullException rather than returning Success = false the way
+        // this used to. So it is rejected before the factory is touched at all.
+        string? canonicalName = null;
+        IFunctionExecutor? executor = null;
+        if (!string.IsNullOrWhiteSpace(funcName))
+        {
+            // Registered callbacks are used only to resolve the CANONICAL name, preserving the
+            // original case-insensitive semantics; the execution itself must go through the factory,
+            // or IFunctionExecutorProvider is bypassed on the rule path.
+            canonicalName = _services.GetServices<IFunctionCallback>()
+                .FirstOrDefault(x => x.Name.IsEqualTo(funcName))?.Name ?? funcName;
+            executor = _services.GetRequiredService<IFunctionExecutorFactory>()
+                .Create(canonicalName, agent);
+        }
+
+        if (executor == null || canonicalName == null)
         {
             var errorMsg = $"Unable to find function '{funcName}' when running action {agent.Name}-{trigger.Name}";
             _logger.LogWarning(errorMsg);
@@ -57,15 +76,15 @@ public async Task<RuleNodeResult> ExecuteAsync(
         }
 
         var funcArg = context.Parameters.TryGetObjectValueOrDefault<RoleDialogModel>("function_argument", new()) ?? new();
-        await func.Execute(funcArg);
+        await executor.ExecuteAsync(funcArg);
 
         return new RuleNodeResult
         {
             Success = true,
             Response = funcArg?.RichContent?.Message?.Text ?? funcArg?.Content,
             Data = new()
             {
-                ["function_name"] = func.Name!,
+                ["function_name"] = canonicalName,
                 ["function_argument"] = funcArg?.ConvertToString() ?? "{}",
                 ["function_call_result"] = funcArg?.RichContent?.Message?.Text ?? funcArg?.Content ?? string.Empty
             }

src/Infrastructure/BotSharp.Core/Routing/Executor/FunctionExecutorFactory.cs

@@ -3,11 +3,45 @@
 
 namespace BotSharp.Core.Routing.Executor;
 
-internal class FunctionExecutorFactory
+public class FunctionExecutorFactory : IFunctionExecutorFactory
 {
-    public static IFunctionExecutor? Create(IServiceProvider services, string functionName, Agent agent)
+    private readonly IServiceProvider _services;
+
+    public FunctionExecutorFactory(IServiceProvider services)
+    {
+        _services = services;
+    }
+
+    public IFunctionExecutor? Create(string functionName, Agent agent)
     {
-        var functionCall = services.GetServices<IFunctionCallback>().FirstOrDefault(x => x.Name == functionName);
+        // This is the one place in the repo that decides who executes a given function name (see
+        // the interface's own doc comment), so it cannot rely on every caller validating first --
+        // RoutingService.InvokeFunction has no guard at all and passes `name` straight through. A
+        // null/blank function name reaching one of the registered IFunctionExecutorProvider
+        // implementations below would make some of them throw ArgumentNullException instead of
+        // returning null the way "no such function" does: a mock/blocking provider doing a
+        // Dictionary lookup by name, for one, and ToolCallActionTests' NullIntolerantProvider
+        // already proves that shape exists. Guarding once here saves every caller from repeating
+        // the same check.
+        if (string.IsNullOrWhiteSpace(functionName))
+        {
+            return null;
+        }
+
+        // Give external providers first refusal. Order is stable (ascending Order), never the DI
+        // registration order.
+        var providers = _services.GetServices<IFunctionExecutorProvider>().OrderBy(x => x.Order);
+        foreach (var provider in providers)
+        {
+            var claimed = provider.TryResolve(functionName, agent);
+            if (claimed != null)
+            {
+                return claimed;
+            }
+        }
+
+        // The three stages below are the pre-existing logic verbatim -- same order, same semantics.
+        var functionCall = _services.GetServices<IFunctionCallback>().FirstOrDefault(x => x.Name == functionName);
         if (functionCall != null)
         {
             return new FunctionCallbackExecutor(functionCall);
@@ -17,15 +51,15 @@ internal class FunctionExecutorFactory
         var funcDef = functions.FirstOrDefault(x => x.Name == functionName);
         if (!string.IsNullOrWhiteSpace(funcDef?.Output))
         {
-            return new DummyFunctionExecutor(services, funcDef);
+            return new DummyFunctionExecutor(_services, funcDef);
         }
 
         var mcpServerId = agent?.McpTools?.Where(x => x.Functions.Any(y => y.Name == funcDef?.Name))?.FirstOrDefault()?.ServerId;
         if (!string.IsNullOrWhiteSpace(mcpServerId))
         {
-            return new McpToolExecutor(services, mcpServerId, functionName);
+            return new McpToolExecutor(_services, mcpServerId, functionName);
         }
-        
+
         return null;
     }
 }

src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs

@@ -1,6 +1,8 @@
+using BotSharp.Abstraction.Routing.Executor;
 using BotSharp.Abstraction.Routing.Reasoning;
 using BotSharp.Abstraction.Routing.Settings;
 using BotSharp.Abstraction.Settings;
+using BotSharp.Core.Routing.Executor;
 using BotSharp.Core.Routing.Hooks;
 using BotSharp.Core.Routing.Reasoning;
 using Microsoft.Extensions.Configuration;
@@ -31,6 +33,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         });
 
         services.AddScoped<IRoutingService, RoutingService>();
+        services.AddScoped<IFunctionExecutorFactory, FunctionExecutorFactory>();
         services.AddScoped<IAgentHook, RoutingAgentHook>();
 
         services.AddScoped<IRoutingReasoner, NaiveReasoner>();

src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeFunction.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Routing.Executor;
 using BotSharp.Abstraction.Routing.Models;
 using BotSharp.Core.MessageHub;
 using BotSharp.Core.Routing.Executor;
@@ -13,7 +14,7 @@ public async Task<bool> InvokeFunction(string name, RoleDialogModel message, Inv
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.GetAgent(currentAgentId);
 
-        var funcExecutor = FunctionExecutorFactory.Create(_services, name, agent);
+        var funcExecutor = _services.GetRequiredService<IFunctionExecutorFactory>().Create(name, agent);
         if (funcExecutor == null)
         {
             message.StopCompletion = true;

src/Plugins/BotSharp.Plugin.AgentTesting/AgentTestingPlugin.cs

@@ -0,0 +1,89 @@
+using BotSharp.Abstraction.Agents;
+using BotSharp.Abstraction.Repositories.Settings;
+using BotSharp.Abstraction.Users.Enums;
+using BotSharp.Plugin.AgentTesting.Repositories;
+using BotSharp.Plugin.AgentTesting.Runtime;
+using BotSharp.Plugin.AgentTesting.Services;
+
+namespace BotSharp.Plugin.AgentTesting;
+
+public class AgentTestingPlugin : IBotSharpPlugin
+{
+    public string Id => "5c1f4d38-9a2e-4b7c-8f61-2d0e7a9c4b13";
+    public string Name => "Agent Testing";
+    public string Description => "Per-agent regression test sets: scripted multi-turn cases, mocked tools, deterministic assertions.";
+
+    /// <summary>
+    /// Without this the four pages exist but nothing links to them -- the suite list is only
+    /// reachable by typing the URL. Same shape as QaAutomationPlugin: sit under the "One Brain"
+    /// header when it is there, fall back to "Apps" (seeded by BotSharp.OpenAPI's
+    /// PluginController), and position with the section's own weight.
+    ///
+    /// Roles matches the gate that actually matters here: TriggerRun and RecordCase carry
+    /// [BotSharpAuth], which is Root/Admin only. The read endpoints are plain [Authorize], but a
+    /// menu entry leading to a page whose primary buttons 401 is worse than no entry.
+    /// </summary>
+    public bool AttachMenu(List<PluginMenuDef> menu)
+    {
+        var section = menu.FirstOrDefault(x => x.Label == "One Brain")
+            ?? menu.FirstOrDefault(x => x.Label == "Apps");
+
+        if (section != null)
+        {
+            menu.Add(new PluginMenuDef("Agent Testing", icon: "bx bx-test-tube", link: "page/agent-test", weight: section.Weight + 2)
+            {
+                Roles = new List<string> { UserRole.Root, UserRole.Admin }
+            });
+        }
+
+        return true;
+    }
+
+    public void RegisterDI(IServiceCollection services, IConfiguration config)
+    {
+        // Singleton: the test context has to be visible across requests and across threads.
+        services.AddSingleton<IAgentTestRunRegistry, AgentTestRunRegistry>();
+
+        // The seam that takes over function execution. Lose this line and mocking silently stops
+        // working -- the runner's canary self-check is what catches that.
+        services.AddScoped<IFunctionExecutorProvider, TestMockExecutorProvider>();
+
+        // The model-override seam, which lets one run sweep the same agent across several models.
+        // Losing this line does not raise an error: every model would run on the agent's own
+        // LlmConfig and the comparison grid would show them all behaving identically.
+        services.AddScoped<IAgentHook, AgentTestModelOverrideHook>();
+
+        services.AddScoped<IAgentConversationDriver, BotSharpAgentConversationDriver>();
+
+        // Registered by its ICaseRunner interface rather than its concrete type, so that
+        // AgentTestRunQueue can substitute a wrapper that opens a fresh DI scope per call in
+        // production (see AgentTestRunQueue.ScopedCaseRunner). The behaviour itself -- multi-turn,
+        // canary, timeout -- is unchanged by that substitution.
+        services.AddScoped<ICaseRunner, AgentTestCaseRunner>();
+
+        // The harness owns its four collections, so it also owns the Mongo connection to them.
+        // Singleton because MongoClient is thread-safe and holds a connection pool -- one per DI
+        // scope would open a fresh pool for every case a run executes. Bound here rather than taken
+        // from the container because BotSharpDatabaseSettings is not registered as a service on
+        // every host configuration, while `config` always is.
+        var dbSettings = new BotSharpDatabaseSettings();
+        config.Bind("Database", dbSettings);
+        services.AddSingleton(new AgentTestMongoDbContext(dbSettings));
+
+        services.AddScoped<IAgentTestRepository, AgentTestRepository>();
+
+        // Task 9: no custom interface -- injected by its own concrete type into
+        // AgentTestController, so it needs an explicit registration the same way every other
+        // service in this method does (there is no auto-discovery mechanism in this plugin).
+        services.AddScoped<ICaseSegmenter, LlmCaseSegmenter>();
+        services.AddScoped<AgentTestRecorder>();
+
+        // AgentTestRunQueue is both a singleton and a BackgroundService: all three lines point at
+        // the same instance, sharing one Channel. Same shape as WeChatBackgroundService elsewhere in
+        // this repo -- AddSingleton<T>(), AddHostedService forwarding to that same instance, then
+        // the interface type forwarding to it as well.
+        services.AddSingleton<AgentTestRunQueue>();
+        services.AddHostedService(s => s.GetRequiredService<AgentTestRunQueue>());
+        services.AddSingleton<IAgentTestRunQueue>(s => s.GetRequiredService<AgentTestRunQueue>());
+    }
+}

src/Plugins/BotSharp.Plugin.AgentTesting/BotSharp.Plugin.AgentTesting.csproj

@@ -0,0 +1,27 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>$(TargetFramework)</TargetFramework>
+    <LangVersion>$(LangVersion)</LangVersion>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <VersionPrefix>$(BotSharpVersion)</VersionPrefix>
+    <GeneratePackageOnBuild>$(GeneratePackageOnBuild)</GeneratePackageOnBuild>
+    <GenerateDocumentationFile>$(GenerateDocumentationFile)</GenerateDocumentationFile>
+    <OutputPath>$(SolutionDir)packages</OutputPath>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="MongoDB.Driver" />
+  </ItemGroup>
+
+  <!-- The plugin exposes a REST controller (AgentTestController). -->
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Infrastructure\BotSharp.Abstraction\BotSharp.Abstraction.csproj" />
+  </ItemGroup>
+
+</Project>