Merge releases/r10.0 into develop (Hamza's r10.0 changes, ahead of the production release)

SW.Bitween.Api/Resources/Xchanges/XchangeFilters.cs

@@ -253,6 +253,24 @@ internal static IQueryable<XchangeRow> ApplySpecialFilters(this IQueryable<Xchan
             condition.Filters.Remove(errorFilter);
         }
 
+        // The one promoted property the value below has to be found in. Sent apart from the value
+        // because the client used to send them together as "key:value", which only matched a value
+        // that starts with what was typed: "SO-1001" was found by "SO", never by "1001".
+        var keyFilters = condition.Filters.Where(f => f.Field == "PromotedPropertyKey").ToList();
+        var key = keyFilters.Select(f => f.Value?.ToString()).FirstOrDefault(v => !string.IsNullOrEmpty(v))?.ToLower();
+        foreach (var keyFilter in keyFilters)
+            condition.Filters.Remove(keyFilter);
+
+        // Stored as "key:value,key:value": the key comes first or after a comma, which keeps "id"
+        // from matching "orderId:". Asked of lower(properties_raw) as it is, not with a comma put in
+        // front: Postgres keeps statistics for that expression (its trigram index), and without them
+        // it guessed almost nothing matched, so a common key fetched and sorted every match rather
+        // than reading the newest 25 - 6s against 1M exchanges.
+        var keyTag = "," + key + ":";
+        if (key != null)
+            query = query.Where(i => i.PromotedPropertiesRaw.ToLower().StartsWith(key + ":")
+                                     || i.PromotedPropertiesRaw.ToLower().Contains(keyTag));
+
         var propertiesFilters = condition.Filters
             .Where(f => f.Field == "PromotedPropertiesRaw").ToList();
         foreach (var propertyFilter in propertiesFilters)
@@ -264,6 +282,19 @@ internal static IQueryable<XchangeRow> ApplySpecialFilters(this IQueryable<Xchan
             // the search to stay case-insensitive. No index is lost: a Contains is a
             // leading-wildcard LIKE, which the b-tree on this column could never serve.
             query = query.Where(i => i.PromotedPropertiesRaw.ToLower().Contains(value));
+
+            // With a key, only inside that key's value: what follows ",key:" up to the next comma.
+            // A value that itself holds a comma is only searched up to it, since the stored text
+            // cannot tell that comma from the one before the next key. IndexOf rather than Contains
+            // for the same estimate reason as the key above: Postgres guesses a third of rows pass
+            // an unknown comparison but under 1% pass an unknown LIKE.
+            if (key != null)
+                query = from i in query
+                        let rest = ("," + i.PromotedPropertiesRaw.ToLower())
+                            .Substring(("," + i.PromotedPropertiesRaw.ToLower()).IndexOf(keyTag) + keyTag.Length) + ","
+                        where rest.Substring(0, rest.IndexOf(",")).IndexOf(value) >= 0
+                        select i;
+
             condition.Filters.Remove(propertyFilter);
         }
 

SW.Bitween.Api/SW.Bitween.Api.csproj

@@ -38,8 +38,8 @@
          them to ask the broker what exists; without this it would have to rebuild each of them
          from configuration and get them subtly wrong. Already in the app through SW.Bitween.Web -
          keep the version in step with it. -->
-    <PackageReference Include="SimplyWorks.Bus" Version="10.0.1" />
-    <PackageReference Include="SimplyWorks.Bus.RabbitMqExtensions" Version="10.0.1" />
+    <PackageReference Include="SimplyWorks.Bus" Version="10.0.3" />
+    <PackageReference Include="SimplyWorks.Bus.RabbitMqExtensions" Version="10.0.3" />
     <!-- 8.1.9 is the first version carrying AuditOptions, which AuditPolicy needs. -->
     <PackageReference Include="SimplyWorks.EfCoreExtensions" Version="10.0.0" />
     <PackageReference Include="SimplyWorks.Scheduler.Sdk" Version="10.0.0" />

SW.Bitween.IntegrationTests/SW.Bitween.IntegrationTests.csproj

@@ -55,7 +55,7 @@
     <PackageReference Include="SimplyWorks.Serverless.Tooling" Version="10.2.2" />
 
     <!-- Bus (for IPublish + AddBus registration) -->
-    <PackageReference Include="SimplyWorks.Bus" Version="10.0.1" />
+    <PackageReference Include="SimplyWorks.Bus" Version="10.0.3" />
 
     <!-- EF Core naming conventions (used by PgSql context) -->
     <PackageReference Include="EFCore.NamingConventions" Version="9.0.0" />

SW.Bitween.IntegrationTests/Tests/PromotedPropertySearchTests.cs

@@ -0,0 +1,73 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using SW.Bitween.Domain;
+using SW.Bitween.IntegrationTests.Fixtures;
+using SW.Bitween.Model;
+using SW.PrimitiveTypes;
+using Xunit;
+
+namespace SW.Bitween.IntegrationTests.Tests;
+
+/// <summary>
+/// The Exchanges page's promoted property filter: a key picked from the list and text that has to
+/// be found anywhere inside that key's value.
+/// </summary>
+[Collection("Bitween")]
+public class PromotedPropertySearchTests(BitweenFixture fixture)
+{
+    [Fact]
+    public async Task A_picked_key_finds_part_of_its_own_value_and_nothing_in_other_keys()
+    {
+        await using var scope = fixture.CreateScope();
+        scope.Superuser();
+        var db = scope.ServiceProvider.GetRequiredService<BitweenDbContext>();
+
+        var document = new Document(null, $"Prop search {Guid.NewGuid():N}"[..24], DocumentFormat.Json);
+        document.SetDictionaries(new Dictionary<string, string>
+            { ["orderNo"] = "$.o", ["customer"] = "$.c", ["refOrderNo"] = "$.r" });
+        db.Add(document);
+        await db.SaveChangesAsync();
+
+        var service = scope.ServiceProvider.GetRequiredService<XchangeService>();
+        async Task<string> Exchange(params (string Key, string Value)[] properties)
+        {
+            var xchange = await service.CreateXchange(document, null, new XchangeFile("{}"));
+            var filtered = new FilterResult();
+            foreach (var (key, value) in properties)
+                filtered.Properties[key] = value;
+            db.Add(new XchangePromotedProperties(xchange.Id, filtered));
+            await db.SaveChangesAsync();
+            return xchange.Id;
+        }
+
+        var a = await Exchange(("orderNo", "SO-1001-X"), ("customer", "Acme"));
+        // "1001" sits after orderNo here, in the next key's value.
+        var b = await Exchange(("orderNo", "PO-7"), ("customer", "Zed-1001"));
+        // A key whose name ends in the picked one.
+        var c = await Exchange(("refOrderNo", "R-1001"));
+
+        await using var readScope = fixture.CreateScope();
+        readScope.Superuser();
+        async Task<string[]> Found(params string[] filters)
+        {
+            var page = (SearchyResponse<XchangeRow>)await ActivatorUtilities
+                .CreateInstance<Resources.Xchanges.Search>(readScope.ServiceProvider)
+                .Handle(new SearchyRequest(filters.Append($"DocumentId:1:{document.Id}").ToArray()), false, null);
+            return page.Result.Select(r => r.Id).Order().ToArray();
+        }
+
+        string[] Ids(params string[] ids) => ids.Order().ToArray();
+
+        // The middle of the value, which a "orderNo:1001" term never found.
+        Assert.Equal(Ids(a), await Found("PromotedPropertyKey:1:orderNo", "PromotedPropertiesRaw:4:1001"));
+        Assert.Equal(Ids(a), await Found("PromotedPropertyKey:1:ORDERNO", "PromotedPropertiesRaw:4:01-x"));
+        Assert.Equal(Ids(b), await Found("PromotedPropertyKey:1:orderNo", "PromotedPropertiesRaw:4:7"));
+        // A key on its own: "carries this property at all".
+        Assert.Equal(Ids(a, b), await Found("PromotedPropertyKey:1:orderNo"));
+        // No key: anywhere, as before.
+        Assert.Equal(Ids(a, b, c), await Found("PromotedPropertiesRaw:4:1001"));
+    }
+}

SW.Bitween.UnitTests/SW.Bitween.UnitTests.csproj

@@ -17,7 +17,7 @@
     <PackageReference Include="MSTest.TestAdapter" Version="3.7.1" />
     <PackageReference Include="MSTest.TestFramework" Version="3.7.1" />
     <PackageReference Include="SimplyWorks.CqApi" Version="8.2.6" />
-    <PackageReference Include="SimplyWorks.Bus" Version="10.0.1" />
+    <PackageReference Include="SimplyWorks.Bus" Version="10.0.3" />
   </ItemGroup>
 
   <ItemGroup>

SW.Bitween.Web/ClientApp/src/api/http/exchanges.ts

@@ -193,13 +193,11 @@ function buildExchangeFilters(query: ExchangeQuery): URLSearchParams {
     params.append("filter", `ReceiveAttemptId:1:${query.receiveAttemptId}`);
   if (query.correlationId?.trim()) params.append("filter", `CorrelationId:1:${query.correlationId.trim()}`);
   if (query.latest) params.append("filter", "LatestOnly:1:true");
-  // PromotedPropertiesRaw is stored as "key:value,key:value", so prefixing the key turns
-  // the same substring match into a scoped one - no schema or endpoint change needed.
-  // Typing "merchant:Acme" into the value box has therefore always worked; the picker
-  // just makes it something you can find.
-  const propertyValue = query.property?.trim() ?? "";
-  const propertyTerm = query.propertyKey ? `${query.propertyKey}:${propertyValue}` : propertyValue;
-  if (propertyTerm) params.append("filter", `PromotedPropertiesRaw:4:${propertyTerm}`);
+  // The key goes apart from the value so the server can look anywhere inside that key's value.
+  // Sent as one "key:value" term it only matched values starting with what was typed.
+  if (query.propertyKey) params.append("filter", `PromotedPropertyKey:1:${query.propertyKey}`);
+  const propertyValue = query.property?.trim();
+  if (propertyValue) params.append("filter", `PromotedPropertiesRaw:4:${propertyValue}`);
   if (query.errorContains?.trim()) params.append("filter", `ErrorContains:4:${query.errorContains.trim()}`);
   if (query.isRetry) params.append("filter", "IsRetry:1:true");
   if (query.scheduledRetry) params.append("filter", "HasScheduledRetry:1:true");

SW.Bitween.Web/ClientApp/src/pages/exchanges/ExchangesPage.tsx

@@ -1,7 +1,7 @@
 import { useCallback, useEffect, useMemo, useState } from "react";
 import { Link, useSearchParams } from "react-router";
 import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
-import { ArrowUp, Download, Pause, Play, Plus, RotateCcw, X } from "lucide-react";
+import { ArrowUp, Download, Pause, Play, Plus, RefreshCw, RotateCcw, X } from "lucide-react";
 import {
   api,
   type BulkRetrySelection,
@@ -94,12 +94,16 @@ export function ExchangesPage() {
 
   // A relative range ("last hour") is worked out each time the list is asked for, so a live list
   // keeps showing the last hour rather than the hour before the page was opened.
-  const { data, isLoading, dataUpdatedAt } = useQuery({
+  const { data, isLoading, isFetching, isPlaceholderData, dataUpdatedAt, refetch } = useQuery({
     queryKey: keys.exchanges.search(viewSearch(searchParams) + `&offset=${query.offset}`),
     queryFn: () => api.searchExchanges(readQuery(searchParams)),
     refetchInterval: refreshMs || false,
     placeholderData: keepPreviousData,
   });
+  // Loads someone is waiting on: a changed filter or page (the old rows stay up meanwhile), or the
+  // Refresh button. Not the background ticks of live refresh, which would dim the list every 15s.
+  const [refreshing, setRefreshing] = useState(false);
+  const reloading = isPlaceholderData || refreshing;
 
   // --- counts and volume ---
   // With a time filter, one request answers both. Without one the list covers all time, which
@@ -124,6 +128,16 @@ export function ExchangesPage() {
     retry: false,
   });
   const volume = timed ? breakdown.data : chartBreakdown.data;
+
+  // Now, rather than at the next tick - or at all, paused. The counts and the chart come along so
+  // they never disagree with the rows. The chart's query is off when the list is timed, and a
+  // refetch would run it anyway, against no parameters.
+  const refresh = () => {
+    setRefreshing(true);
+    void Promise.all([refetch(), breakdown.refetch(), chartParams ? chartBreakdown.refetch() : null]).finally(() =>
+      setRefreshing(false),
+    );
+  };
   const rangeEndsNow = !searchParams.get("to") && searchParams.get("range") !== "yesterday";
 
   const subscriptions = useSubscriptionsCache().data ?? [];
@@ -347,7 +361,13 @@ export function ExchangesPage() {
         description="Every message that flows through Bitween. Open one to follow its journey and read its documents."
         actions={
           <>
-            <LiveControl refreshMs={refreshMs} onChange={setRefreshMs} updatedAt={dataUpdatedAt} />
+            <LiveControl
+              refreshMs={refreshMs}
+              onChange={setRefreshMs}
+              updatedAt={dataUpdatedAt}
+              onRefresh={refresh}
+              fetching={isFetching}
+            />
             <Can permission="exchanges.operate">
               <Link to="/exchanges/new">
                 <Button variant="primary">
@@ -442,29 +462,34 @@ export function ExchangesPage() {
                     : "Traffic will show up here as soon as a subscription processes something."}
               </EmptyState>
             ) : (
-              <div className="rounded-xl border border-ink-200 bg-white">
-                <ExchangeList
-                  rows={rows}
-                  openId={openId}
-                  onOpen={toggleOpen}
-                  compact={!!openRow}
-                  selection={{
-                    isSelected,
-                    toggle: toggleSelected,
-                    allOnPage: allOnPageSelected,
-                    toggleAllOnPage: () => {
-                      // In "all matching" mode the box stands for the whole filter, so
-                      // clearing it drops the whole selection rather than excluding 25 rows.
-                      if (allMatching) return clearSelection();
-                      setSelected((prev) => {
-                        const next = new Set(prev);
-                        if (allOnPageSelected) rows.forEach((r) => next.delete(r.id));
-                        else rows.forEach((r) => next.add(r.id));
-                        return next;
-                      });
-                    },
-                  }}
-                />
+              <div className="relative rounded-xl border border-ink-200 bg-white" aria-busy={reloading}>
+                {reloading && (
+                  <div className="absolute inset-x-0 top-0 z-10 h-0.5 animate-pulse bg-crimson-500" title="Loading…" />
+                )}
+                <div className={`transition-opacity ${reloading ? "opacity-50" : ""}`}>
+                  <ExchangeList
+                    rows={rows}
+                    openId={openId}
+                    onOpen={toggleOpen}
+                    compact={!!openRow}
+                    selection={{
+                      isSelected,
+                      toggle: toggleSelected,
+                      allOnPage: allOnPageSelected,
+                      toggleAllOnPage: () => {
+                        // In "all matching" mode the box stands for the whole filter, so
+                        // clearing it drops the whole selection rather than excluding 25 rows.
+                        if (allMatching) return clearSelection();
+                        setSelected((prev) => {
+                          const next = new Set(prev);
+                          if (allOnPageSelected) rows.forEach((r) => next.delete(r.id));
+                          else rows.forEach((r) => next.add(r.id));
+                          return next;
+                        });
+                      },
+                    }}
+                  />
+                </div>
 
                 {/* - paging - */}
                 <div className="flex flex-wrap items-center justify-between gap-2 border-t border-ink-100 px-4 py-2.5 text-[13px] text-ink-500">
@@ -630,10 +655,14 @@ function LiveControl({
   refreshMs,
   onChange,
   updatedAt,
+  onRefresh,
+  fetching,
 }: {
   refreshMs: number;
   onChange: (ms: number) => void;
   updatedAt: number;
+  onRefresh: () => void;
+  fetching: boolean;
 }) {
   const [lastLive, setLastLive] = useState(15_000);
   const live = refreshMs > 0;
@@ -671,6 +700,15 @@ function LiveControl({
       >
         {live ? <Pause className="size-3.5" aria-hidden /> : <Play className="size-3.5" aria-hidden />}
       </button>
+      <button
+        type="button"
+        onClick={onRefresh}
+        aria-label="Refresh"
+        title="Load the latest exchanges now"
+        className="flex w-9 items-center justify-center border-l border-ink-100 text-ink-600 hover:bg-ink-50"
+      >
+        <RefreshCw className={`size-3.5 ${fetching ? "animate-spin" : ""}`} aria-hidden />
+      </button>
     </span>
   );
 }

SW.Bitween.Web/ClientApp/src/pages/exchanges/__tests__/ExchangesQueryBar.test.tsx

@@ -305,3 +305,31 @@ describe("pausing", () => {
     expect(filters(asked)).toContain("StartedOn:6:2026-09-20T10:00:00.001Z");
   });
 });
+
+describe("refresh", () => {
+  it("asks for the list and the counts again at once, even paused", async () => {
+    const api = backend();
+    const { user } = renderApp("/exchanges", { handlers: api.handlers });
+
+    // Paused, nothing else asks: every request after the click is the button's.
+    await user.click(await screen.findByRole("button", { name: "Pause live updates" }));
+    await screen.findByText("SO-0");
+    const before = { listed: api.listed.length, summarised: api.summarised.length };
+
+    await user.click(screen.getByRole("button", { name: "Refresh" }));
+    await waitFor(() => expect(api.listed.length).toBeGreaterThan(before.listed));
+    expect(api.summarised.length).toBeGreaterThan(before.summarised);
+  });
+});
+
+describe("promoted property", () => {
+  it("sends the picked key apart from the value, so any part of that key's value matches", async () => {
+    const api = backend();
+    renderApp("/exchanges?propertyKey=orderNo&property=1001", { handlers: api.handlers });
+
+    await waitFor(() => expect(api.listed.length).toBeGreaterThan(0));
+    const sent = filters(api.listed.at(-1)!);
+    expect(sent).toContain("PromotedPropertyKey:1:orderNo");
+    expect(sent).toContain("PromotedPropertiesRaw:4:1001");
+  });
+});

SW.Bitween.Web/SW.Bitween.Web.csproj

@@ -46,7 +46,7 @@
     <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.19.0" />
     <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.19.0" />
     <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.19.0" />
-    <PackageReference Include="SimplyWorks.Bus" Version="10.0.1" />
+    <PackageReference Include="SimplyWorks.Bus" Version="10.0.3" />
     <PackageReference Include="SimplyWorks.CloudFiles.AS.Extensions" Version="10.0.1" />
     <PackageReference Include="SimplyWorks.CloudFiles.OC.Extensions" Version="10.0.1" />
     <PackageReference Include="SimplyWorks.CloudFiles.S3.Extensions" Version="10.0.1" />

docs/configuration.md

@@ -45,6 +45,20 @@ published in this repository. On startup, while that account still has the defau
 
 Once the password is anything else the value is ignored, so it can stay in place or be removed.
 
+### Bus messages from other applications
+
+A message another application publishes to Bitween's bus can carry its user's token. Bitween checks
+it with its own `Token` values, so a publisher signing with a different key fails every message it
+sends: it goes straight to the queue's `.bad` queue and no exchange is created. Give both
+applications the same `Token` values. Until they match, `Bus:OnInvalidUserToken`
+(env: `Bus__OnInvalidUserToken`) can change what happens:
+
+| Value | Effect |
+|---|---|
+| `Fail` *(default)* | The message goes to the `.bad` queue without retrying. |
+| `ContinueAnonymous` | The message is processed without a user, and a warning is logged. |
+| `ContinueAnonymousSilently` | The same, without the warning. |
+
 ## `Bitween` section
 
 | Key | Default | Description |