Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions apps/desktop/e2e/fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,24 @@ async function seedRailRenderSessions(userDataDir: string): Promise<void> {
}
}

async function seedOnboardingPerfSessions(userDataDir: string, count: number): Promise<void> {
const store = createSessionStore(path.join(userDataDir, 'workspaces', 'default'));
try {
for (let index = 0; index < count; index += 1) {
await store.create({
cwd: path.join(userDataDir, 'project'),
llmConnectionSlug: 'e2e',
model: 'claude-sonnet-4-5-20250929',
permissionMode: 'ask',
name: `Onboarding perf row ${index}`,
labels: [],
});
}
} finally {
await store.close?.();
}
}

async function seedParentRemovalSessions(userDataDir: string): Promise<void> {
const workspaceRoot = path.join(userDataDir, 'workspaces', 'default');
const store = createSessionStore(workspaceRoot);
Expand Down Expand Up @@ -403,6 +421,7 @@ export async function withE2eWindow(
gitReviewExtraFiles,
parentRemovalSessions,
railRenderSessions,
onboardingPerfSessions,
newTaskProject,
tracePath,
testInfo,
Expand All @@ -421,6 +440,7 @@ export async function withE2eWindow(
gitReviewExtraFiles?: number;
parentRemovalSessions?: boolean;
railRenderSessions?: boolean;
onboardingPerfSessions?: number;
newTaskProject?: boolean;
tracePath?: string;
/** Attaches captured main/renderer console output when the test fails. */
Expand All @@ -441,6 +461,9 @@ export async function withE2eWindow(
if (seed) await seedE2eConnection(userDataDir);
if (parentRemovalSessions) await seedParentRemovalSessions(userDataDir);
if (railRenderSessions) await seedRailRenderSessions(userDataDir);
if (onboardingPerfSessions !== undefined) {
await seedOnboardingPerfSessions(userDataDir, onboardingPerfSessions);
}
if (invocableSkills) await seedE2eInvocableSkills(userDataDir);
if (gitReviewExtraFiles !== undefined) {
await seedE2eGitReviewProject(userDataDir, gitReviewExtraFiles);
Expand Down
1 change: 1 addition & 0 deletions apps/desktop/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
"test:dist": "node --test --test-force-exit \"dist/main/**/*.test.js\" scripts/dev-app-runtime.test.mjs scripts/vite-workspace-packages.test.mjs scripts/vite-client-plugin-slots.test.mjs scripts/build-executor-plugins.test.mjs",
"e2e": "npm run build:with-deps && playwright test --config e2e/playwright.config.ts",
"measure": "npm run build:with-deps && playwright test --config perf/playwright.config.ts",
"measure:onboarding": "node scripts/measure-onboarding-session-update.mjs",
"build:with-deps": "npm run build:workspace-deps && npm run build",
"smoke:real-window": "npm run build:with-deps && node ../../scripts/desktop-real-window-smoke.mjs",
"smoke:programmatic-window": "npm run build:with-deps && node ../../scripts/desktop-real-window-smoke.mjs --programmatic-only",
Expand Down
62 changes: 62 additions & 0 deletions apps/desktop/perf/onboarding-session-update.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import { test, withE2eWindow } from '../e2e/fixtures';

// Run only when measuring a PR. The complete read is the pre-change path;
// the targeted read is the new path. This records actual Electron bridge and
// Host time, but has no injected network latency. The synthetic companion
// script covers 30 ms transport latency and Host request/byte counts.
for (const count of [100, 1_000, 5_000]) {
test(`onboarding bridge read cost with ${count} Sessions`, async () => {
await withE2eWindow({
seed: true,
onboardingPerfSessions: count,
readinessSelector: '.maka-composer',
readinessTimeoutMs: 300_000,
}, async (page) => {
const samples = await page.evaluate(async () => {
const initial = await window.maka.onboarding.getSnapshot();
const sessionId = initial.sessions[Math.floor(initial.sessions.length / 2)]?.id;
if (!sessionId) throw new Error('Onboarding performance fixture has no Session');
const sample = async (read: () => Promise<unknown>) => {
const durations: number[] = [];
let bytes = 0;
for (let index = 0; index < 5; index += 1) {
const start = performance.now();
const result = await read();
durations.push(performance.now() - start);
bytes += new TextEncoder().encode(JSON.stringify(result)).byteLength;
}
durations.sort((a, b) => a - b);
return { medianMs: durations[2], p95Ms: durations[4], bytesPerRead: bytes / 5 };
};
return {
sessionCount: initial.sessions.length,
full: await sample(() => window.maka.onboarding.getSnapshot()),
targeted: await sample(() => window.maka.onboarding.getSessionUpdate(sessionId)),
};
});
if (samples.sessionCount < count) {
throw new Error(`Only ${samples.sessionCount} of ${count} seeded Sessions appeared`);
}
console.log(JSON.stringify({ kind: 'electron-onboarding', ...samples }));
});
});
}
10 changes: 2 additions & 8 deletions apps/desktop/renderer-architecture.json
Original file line number Diff line number Diff line change
Expand Up @@ -3796,11 +3796,7 @@
}
},
"src/renderer/use-onboarding-snapshot.ts": {
"bridgePaths": {
"window.maka.connections.subscribeEvents": 1,
"window.maka.onboarding.getSnapshot": 1,
"window.maka.sessions.subscribeChanges": 1
},
"bridgePaths": {},
"environmentCapabilities": {},
"hookCalls": {
"useEffect": 1,
Expand All @@ -3812,9 +3808,7 @@
"unresolvedDependencies": 0,
"actionFactories": [],
"dependencyPaths": {
"./locales/onboarding-copy.js": 1,
"@maka/core/onboarding-milestone": 1,
"@maka/core/redaction": 1,
"./platform/desktop/onboarding-snapshot-bridge.js": 2,
"@maka/ui": 1,
"react": 1
}
Expand Down
204 changes: 204 additions & 0 deletions apps/desktop/scripts/measure-onboarding-session-update.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

// Run after `npm run build`. The "before" path invokes the same complete
// snapshot used by the old Renderer invalidation; the "after" path invokes
// one targeted change. This is a synthetic Host transport, not an Electron UI
// trace. It deliberately does not include the initial bootstrap cost.

import { performance } from 'node:perf_hooks';
import { createOnboardingService } from '../dist/main/onboarding-service.js';
import {
applyOnboardingSessionUpdate,
createOnboardingSnapshotPoller,
} from '../dist/renderer/use-onboarding-snapshot.js';
import { readRuntimeHostSessions } from '../../../packages/runtime-host/dist/client/catalog-reader.js';

const counts = [100, 1_000, 5_000];
const latencies = [0, 30];
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

function percentile(samples, fraction) {
const sorted = [...samples].sort((a, b) => a - b);
return Number(sorted[Math.ceil(fraction * sorted.length) - 1].toFixed(2));
}

async function measure(count, roundTripMs) {
const rows = Array.from({ length: count }, (_, index) => ({
id: `session-${index}`, backend: 'plugin-executor', llmConnectionSlug: '',
model: '', connectionLocked: false,
}));
const rowsById = new Map(rows.map((row) => [row.id, row]));
let requests = [];
const connection = {
async request(_operation, input) {
if (roundTripMs) await delay(roundTripMs);
const result = input.kind === 'get'
? { kind: 'session', session: rowsById.get(input.sessionId) ?? null }
: (() => {
const offset = input.kind === 'list_start' ? 0 : Number(input.cursor);
const next = Math.min(offset + 32, rows.length);
return {
kind: 'page', revision: 1, sessions: rows.slice(offset, next),
nextCursor: next < rows.length ? String(next) : null,
};
})();
requests.push({
kind: input.kind,
bytes: Buffer.byteLength(JSON.stringify(result)),
rows: result.kind === 'page' ? result.sessions.length : Number(result.session !== null),
});
return result;
},
};
const service = createOnboardingService({
listConnections: async () => [],
getDefaultSlug: async () => null,
listSessions: () => readRuntimeHostSessions(connection),
getSession: async (id) => (await connection.request('session.catalog.query', {
kind: 'get', sessionId: id,
})).session,
getMilestones: async () => [{ id: 'initial_onboarding', completedAt: 1 }],
upsertMilestone: async () => [{ id: 'initial_onboarding', completedAt: 1 }],
hasCredential: async () => true,
});
await service.getSnapshot(); // Both paths start after identical coverage.
if (process.argv.includes('--burst')) {
const results = {};
for (const name of ['before_full_snapshot', 'after_targeted_update']) {
let ipcBytes = 0;
const poller = createOnboardingSnapshotPoller({
getSnapshot: async () => {
const result = await service.getSnapshot();
ipcBytes += Buffer.byteLength(JSON.stringify(result));
return result;
},
...(name === 'after_targeted_update' ? {
getSessionUpdate: async (id) => {
const result = await service.getSessionUpdate(id);
ipcBytes += Buffer.byteLength(JSON.stringify(result));
return { ...result, sessionId: id };
},
} : {}),
}, { onSnapshot() {}, onSessionUpdate() {}, onError(error) { throw new Error(error); } }, () => 'en');
await poller.pull();
const durations = [];
let listRequests = 0;
let getRequests = 0;
let hostBytes = 0;
let transferredBytes = 0;
const sampleCount = roundTripMs ? 5 : 10;
const sessionId = `session-${Math.floor(count / 2)}`;
for (let sample = 0; sample < sampleCount; sample += 1) {
requests = [];
ipcBytes = 0;
const start = performance.now();
const first = name === 'after_targeted_update'
? poller.pullSession(sessionId)
: poller.pull();
for (let event = 0; event < 10; event += 1) {
if (name === 'after_targeted_update') void poller.pullSession(sessionId);
else void poller.pull();
}
await first;
durations.push(performance.now() - start);
listRequests += requests.filter(({ kind }) => kind.startsWith('list_')).length;
getRequests += requests.filter(({ kind }) => kind === 'get').length;
hostBytes += requests.reduce((sum, request) => sum + request.bytes, 0);
transferredBytes += ipcBytes;
}
poller.dispose();
results[name] = {
medianMs: percentile(durations, 0.5), p95Ms: percentile(durations, 0.95),
listRequestsPerBurst: listRequests / sampleCount,
getRequestsPerBurst: getRequests / sampleCount,
hostBytesPerBurst: Math.round(hostBytes / sampleCount),
ipcBytesPerBurst: Math.round(transferredBytes / sampleCount),
};
}
return { count, roundTripMs, burstNotifications: 11, ...results };
}
const sampleCount = roundTripMs ? 7 : 30;
const results = {};
for (const [name, run] of [
['before_full_snapshot', () => service.getSnapshot()],
['after_targeted_update', () => service.getSessionUpdate(`session-${Math.floor(count / 2)}`)],
]) {
const durations = [];
let listRequests = 0;
let getRequests = 0;
let hostBytes = 0;
let ipcBytes = 0;
let materializedRows = 0;
for (let index = 0; index < sampleCount; index++) {
requests = [];
const start = performance.now();
const response = await run();
durations.push(performance.now() - start);
listRequests += requests.filter(({ kind }) => kind.startsWith('list_')).length;
getRequests += requests.filter(({ kind }) => kind === 'get').length;
hostBytes += requests.reduce((sum, request) => sum + request.bytes, 0);
materializedRows += requests.reduce((sum, request) => sum + request.rows, 0);
ipcBytes += Buffer.byteLength(JSON.stringify(response));
}
results[name] = {
medianMs: percentile(durations, 0.5), p95Ms: percentile(durations, 0.95),
listRequestsPerChange: listRequests / sampleCount,
getRequestsPerChange: getRequests / sampleCount,
hostBytesPerChange: Math.round(hostBytes / sampleCount),
ipcBytesPerChange: Math.round(ipcBytes / sampleCount),
materializedRowsPerChange: materializedRows / sampleCount,
};
}
return { count, roundTripMs, sampleCount, ...results };
}

if (process.argv.includes('--renderer-copy')) {
for (const count of counts) {
let snapshot = {
state: { kind: 'needs_connection' }, milestones: [], sessions: [], connections: [],
defaultSlug: null, chatModelChoices: [],
sessionSendOutcomes: Object.fromEntries(
Array.from({ length: count }, (_, index) => [`session-${index}`, { kind: 'ready' }]),
),
};
const durations = [];
for (let index = 0; index < 230; index += 1) {
const update = {
kind: 'delta', sessionId: `session-${Math.floor(count / 2)}`,
outcome: index % 2 === 0
? { kind: 'blocked', reason: 'fake_backend', connectionLocked: false }
: { kind: 'ready' },
};
const start = performance.now();
snapshot = applyOnboardingSessionUpdate(snapshot, update);
if (index >= 30) durations.push(performance.now() - start);
}
console.log(JSON.stringify({
kind: 'renderer-record-copy', count,
medianMs: percentile(durations, 0.5), p95Ms: percentile(durations, 0.95),
}));
}
} else {
for (const count of counts) {
for (const roundTripMs of latencies) {
console.log(JSON.stringify(await measure(count, roundTripMs)));
}
}
}
Loading