From 04a3799f4afb454dbf1d7897bcb9768e301a134c Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Wed, 4 Nov 2026 04:32:05 +0800 Subject: [PATCH 1/2] feat(vieval): rank the core scores of a task with `rank` on defineScore `defineScore(name, { rank })` replaces `primary`: rank 1 is the headline score, 2 the next core score, and so on; scores without a rank are supporting scores. Two scores of one task cannot share a rank, and a rank is a whole number from 1. The `score.defined` event adds `rank` and keeps `primary` (written for rank 1), so reports written before ranks still open and readers keep their declaration order. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- .../learn/assertions-scores-and-metrics.md | 12 ++--- .../learn/assertions-scores-and-metrics.md | 10 ++--- .../projects/showcase/evals/cases.eval.ts | 2 +- packages/vieval/README.md | 2 +- packages/vieval/src/dsl/scores.ts | 40 ++++++++++------- packages/vieval/src/dsl/task.test.ts | 45 ++++++++++++++----- packages/vieval/src/events/index.ts | 7 ++- 8 files changed, 79 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index b7cbba8..1a8c5a4 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ const natural = defineScore('formula.natural', { grader: { id: 'judge', kind: 'm const pass = defineScore('formula.pass', { combine: { all: [correct, { atLeast: 0.5, score: natural }] }, label: 'Pass', - primary: true, + rank: 1, }) describeTask('read-formula', () => { diff --git a/docs/content/en/guide/learn/assertions-scores-and-metrics.md b/docs/content/en/guide/learn/assertions-scores-and-metrics.md index aff07ac..2175ac6 100644 --- a/docs/content/en/guide/learn/assertions-scores-and-metrics.md +++ b/docs/content/en/guide/learn/assertions-scores-and-metrics.md @@ -88,13 +88,13 @@ describeTask('retrieval', ({ caseOf }) => { - `recordScore` also accepts a bare name, for example `recordScore('retrieval.recall', 0.5)`. The score then has no label or description. - A definition is announced to the report the first time a case records it. List the task's scores in the `scores` option of `describeTask` to announce them when the task starts. A listed score is known before the first case, and it counts as missing for a passed case that never records it. -## Choose the primary score +## Choose the primary score and rank the core scores Pages, leaderboards, comparisons, and the CLI summary line show one score per task first, with its label next to its value. That is the **primary** score. -- Mark it with `primary: true`. At most one score of a task is primary; a second one throws when it is announced. -- A task with exactly one score uses that score as primary without the flag. -- A task with several scores and none marked primary has no primary score. Its headline falls back to the share of passed cases. +- Give it `rank: 1`. Rank the other core scores `2`, `3`, and so on, in order of importance: viewers list scores by rank and put the first ranks on the axes of comparison charts. Two scores of a task cannot share a rank; the second one throws when it is announced. A score without a rank is a supporting score, listed after the ranked ones. +- A task with exactly one score uses that score as primary without a rank. +- A task with several scores and none of rank 1 has no primary score. Its headline falls back to the share of passed cases. - When the tasks of a run have different primary scores, the run has no run-level headline score. Each task's primary score is listed with its task name, and the completion rate stays as the run headline. ## Combine scores @@ -115,7 +115,7 @@ const pass = defineScore('tts.pass', { // A bare input means `atLeast: 1`, so a mixed reading (0.5) does not pass. combine: { all: [correct, natural] }, label: { 'en': 'Pass', 'zh-Hans': '通过' }, - primary: true, + rank: 1, }) describeTask('read aloud', ({ caseOf }) => { @@ -155,7 +155,7 @@ One rule decides how each case counts in each score. The task result, `run-summa Set `errors` per score. `exclude` follows the view that a provider outage is not a result of the test object; the completion rate then shows how many cases were left out. `zero` matches benchmarks that count every error as a wrong answer, for example Harbor and Terminal-Bench: ```ts -const solved = defineScore('bench.solved', { errors: 'zero', primary: true }) +const solved = defineScore('bench.solved', { errors: 'zero', rank: 1 }) ``` Each score summary reports `mean` (over the counted cases, or `null`), `n` (cases that counted), and `total` (settled cases that were not skipped). An empty case selection, for example a `--case` filter that matches nothing, returns no scores. diff --git a/docs/content/zh-hans/guide/learn/assertions-scores-and-metrics.md b/docs/content/zh-hans/guide/learn/assertions-scores-and-metrics.md index fd55157..614040c 100644 --- a/docs/content/zh-hans/guide/learn/assertions-scores-and-metrics.md +++ b/docs/content/zh-hans/guide/learn/assertions-scores-and-metrics.md @@ -92,13 +92,13 @@ describeTask('retrieval', ({ caseOf }) => { - `recordScore` 也可以直接传分数名，例如 `recordScore('retrieval.recall', 0.5)`，这时分数没有名称和说明。 - 用例第一次记录某个定义时，Vieval 会把它写入报告。也可以在 `describeTask` 的 `scores` 选项中列出任务的分数，让它们在任务开始时就写入报告。列出的分数在第一个用例之前就是已知的；通过的用例如果没有记录它，就算作缺失。 -## 选择主分数 +## 选择主分数并给核心分数排序 页面、排行榜、对比和 CLI 汇总行会优先显示每个任务的一个分数，并在数值旁边显示它的名称。这个分数就是**主分数**（primary）。 -- 用 `primary: true` 标记主分数。一个任务最多只有一个主分数，声明第二个时会抛出错误。 +- 用 `rank: 1` 标记主分数，其他核心分数按重要程度依次标 `2`、`3`……查看器按 rank 排列分数，并把排在前面的分数放到对比图的坐标轴上。同一个任务里两个分数不能用同一个 rank，声明第二个时会抛出错误。没有 rank 的分数是辅助分数，排在有 rank 的分数之后。 - 只有一个分数的任务不需要标记，这个分数就是主分数。 -- 有多个分数但都没有标记的任务没有主分数，标题数值会回退为通过用例的比例。 +- 有多个分数但没有 `rank: 1` 的任务没有主分数，标题数值会回退为通过用例的比例。 - 如果一次运行中各任务的主分数不同，运行级别就不显示标题分数，而是按任务名分别列出各自的主分数，运行的标题数值只保留执行成功率。 ## 组合分数 @@ -120,7 +120,7 @@ const pass = defineScore('tts.pass', { // 不带阈值的输入相当于 `atLeast: 1`，所以 mixed（0.5）不算通过。 combine: { all: [correct, natural] }, label: { 'en': 'Pass', 'zh-Hans': '通过' }, - primary: true, + rank: 1, }) describeTask('read aloud', ({ caseOf }) => { @@ -161,7 +161,7 @@ describeTask('read aloud', ({ caseOf }) => { `errors` 按分数分别设置。`exclude` 的出发点是：服务故障不是被测对象的结果，被排除的用例数量由执行成功率体现。`zero` 适合把每个错误都算作答错的基准测试，例如 Harbor 和 Terminal-Bench： ```ts -const solved = defineScore('bench.solved', { errors: 'zero', primary: true }) +const solved = defineScore('bench.solved', { errors: 'zero', rank: 1 }) ``` 每个分数的汇总包含 `mean`（计入用例的平均值，没有用例计入时为 `null`）、`n`（计入的用例数）和 `total`（已结束且未跳过的用例数）。如果没有选中任何用例，例如 `--case` 过滤后为空，任务不会返回分数。 diff --git a/packages/ui/test/projects/showcase/evals/cases.eval.ts b/packages/ui/test/projects/showcase/evals/cases.eval.ts index 296b334..213f3ca 100644 --- a/packages/ui/test/projects/showcase/evals/cases.eval.ts +++ b/packages/ui/test/projects/showcase/evals/cases.eval.ts @@ -26,7 +26,7 @@ const pass = defineScore('showcase.pass', { combine: { all: [{ atLeast: 0.7, score: correct }, { atLeast: 0.6, score: quality }] }, description: { 'en': 'The answer is at least 70% correct and the judge rates it at least 0.6.', 'zh-Hans': '答案至少 70% 正确，且评委评分不低于 0.6。' }, label: { 'en': 'Pass', 'zh-Hans': '通过' }, - primary: true, + rank: 1, }) // 64 cheap cases: a long case list with metrics and scores that differ by model. diff --git a/packages/vieval/README.md b/packages/vieval/README.md index 7e7c1bc..1f5a99a 100644 --- a/packages/vieval/README.md +++ b/packages/vieval/README.md @@ -151,7 +151,7 @@ const natural = defineScore('formula.natural', { grader: { id: 'judge', kind: 'm const pass = defineScore('formula.pass', { combine: { all: [correct, { atLeast: 0.5, score: natural }] }, label: 'Pass', - primary: true, + rank: 1, }) describeTask('read-formula', () => { diff --git a/packages/vieval/src/dsl/scores.ts b/packages/vieval/src/dsl/scores.ts index a979063..9e633b4 100644 --- a/packages/vieval/src/dsl/scores.ts +++ b/packages/vieval/src/dsl/scores.ts @@ -57,12 +57,15 @@ export interface ScoreMetadata { */ label?: LocalizedText /** - * Whether this is the headline score of its task: pages, leaderboards, and comparisons show it first. At most one - * score of a task is primary. A task with exactly one score uses it without this flag. + * Importance of this score among the scores of its task: `1` is the headline score (pages, leaderboards, and + * comparisons show it first), `2` the next core score, and so on. Viewers list scores by rank and put the first ranks + * on the axes of comparison charts. Two scores of one task cannot share a rank. A score without a rank is a + * supporting score, listed after the ranked ones. A task with exactly one score uses it as the headline without a + * rank. * - * @default false + * @default undefined */ - primary?: boolean + rank?: number } /** @@ -89,7 +92,7 @@ export interface ScoreRecordOptions { export type ScoreRef = ScoreDefinition | string /** - * Announces score definitions for one task and enforces one primary score per task. + * Announces score definitions for one task and enforces one score per rank in each task. * * Use when: * - a task starts (its `scores` option) or a case records a definition for the first time @@ -101,7 +104,7 @@ export type ScoreRef = ScoreDefinition | string * - the `score.defined` payloads to emit, in order: inputs of a derived score come before it * * Throws: - * - when a second score of the task is primary + * - when a second score of the task has the rank of an announced one */ export function announceScore(announcements: Announcements, score: ScoreDefinition): ScoreDefinedData[] { const emitted: ScoreDefinedData[] = [] @@ -112,11 +115,11 @@ export function announceScore(announcements: Announcements, sc } } const defined = toScoreDefinedData(definition) - const otherPrimary = defined.primary === true - ? announcements.latest().find(existing => existing.primary === true && existing.name !== defined.name) - : undefined - if (otherPrimary != null) { - throw new Error(`Score "${defined.name}" is primary, but score "${otherPrimary.name}" of the same task already is.`) + const sameRank = defined.rank == null + ? undefined + : announcements.latest().find(existing => existing.rank === defined.rank && existing.name !== defined.name) + if (sameRank != null) { + throw new Error(`Score "${defined.name}" has rank ${String(defined.rank)}, but score "${sameRank.name}" of the same task already has it.`) } if (announcements.announce(defined)) { emitted.push(defined) @@ -144,10 +147,13 @@ export function announceScore(announcements: Announcements, sc * @example * const correct = defineScore('formula.correct', { label: { 'en': 'Correct', 'zh-Hans': '正确' } }) * const natural = defineScore('formula.natural', { grader: { kind: 'model', id: 'judge' } }) - * const pass = defineScore('formula.pass', { combine: { all: [correct, natural] }, primary: true }) + * const pass = defineScore('formula.pass', { combine: { all: [correct, natural] }, rank: 1 }) * context.recordScore(correct, 1) */ export function defineScore(name: string, metadata: ScoreMetadata = {}): ScoreDefinition { + if (metadata.rank != null && (!Number.isInteger(metadata.rank) || metadata.rank < 1)) { + throw new Error(`Score "${name}" has rank ${String(metadata.rank)}; a rank is a whole number from 1.`) + } const combine = metadata.combine if (combine != null) { const inputs = 'all' in combine ? combine.all : 'mean' in combine ? combine.mean : combine.weighted @@ -189,16 +195,18 @@ function combinationInputs(combine: ScoreCombination | undefined): Array<{ atLea } /** - * Normalizes a score definition into its `score.defined` payload. + * Normalizes a score definition into its `score.defined` payload. Rank 1 is also written as `primary`, the headline + * flag that reports and readers have always used, so a report says which score leads without knowing about ranks. * * Before: - * - `{ name: 'pass', combine: { all: [correct, { score: natural, atLeast: 0.5 }] }, primary: true }` + * - `{ name: 'pass', combine: { all: [correct, { score: natural, atLeast: 0.5 }] }, rank: 1 }` * * After: - * - `{ name: 'pass', combine: { kind: 'all', inputs: [{ name: 'correct' }, { name: 'natural', atLeast: 0.5 }] }, primary: true }` + * - `{ name: 'pass', combine: { kind: 'all', inputs: [{ name: 'correct' }, { name: 'natural', atLeast: 0.5 }] }, primary: true, rank: 1 }` */ function toScoreDefinedData(definition: ScoreDefinition): ScoreDefinedData { - const { combine, ...rest } = definition + const { combine, ...metadata } = definition + const rest = metadata.rank === 1 ? { ...metadata, primary: true } : metadata if (combine == null) { return { ...rest } } diff --git a/packages/vieval/src/dsl/task.test.ts b/packages/vieval/src/dsl/task.test.ts index ba58b30..adfcf1f 100644 --- a/packages/vieval/src/dsl/task.test.ts +++ b/packages/vieval/src/dsl/task.test.ts @@ -366,7 +366,7 @@ describe('describeTask DSL', { timeout: 10000 }, () => { it('summarizes named scores and derives a combined primary score when each case try settles', async () => { const correct = defineScore('correct') const natural = defineScore('natural') - const pass = defineScore('pass', { combine: { all: [correct, { atLeast: 0.5, score: natural }] }, primary: true }) + const pass = defineScore('pass', { combine: { all: [correct, { atLeast: 0.5, score: natural }] }, rank: 1 }) const taskDefinition = describeTask('dsl-custom-scores', () => { caseOf('case-1', (context) => { context.recordScore(correct, 1) @@ -2204,7 +2204,7 @@ describe('describeTask named scores', () => { it('announces the scores option before the first case, inputs of a derived score first', async () => { const correct = defineScore('correct', { label: { 'en': 'Correct', 'zh-Hans': '正确' } }) const natural = defineScore('natural', { grader: { id: 'judge', kind: 'model' } }) - const pass = defineScore('pass', { combine: { all: [correct, natural] }, primary: true }) + const pass = defineScore('pass', { combine: { all: [correct, natural] }, rank: 1 }) const emitted: Array<{ caseId?: string, data: unknown, event: string }> = [] await runTask(describeTask('scores-announced', () => { @@ -2217,7 +2217,7 @@ describe('describeTask named scores', () => { const defined = emitted.filter(event => event.event === 'score.defined') expect(defined.map(event => (event.data as { name: string }).name)).toEqual(['correct', 'natural', 'pass']) expect(defined.every(event => event.caseId == null)).toBe(true) - expect(defined[2]!.data).toEqual({ combine: { inputs: [{ name: 'correct' }, { name: 'natural' }], kind: 'all' }, name: 'pass', primary: true }) + expect(defined[2]!.data).toEqual({ combine: { inputs: [{ name: 'correct' }, { name: 'natural' }], kind: 'all' }, name: 'pass', primary: true, rank: 1 }) expect(emitted.findIndex(event => event.event === 'score.defined')).toBeLessThan(emitted.findIndex(event => event.event === 'case.started')) }) @@ -2227,7 +2227,7 @@ describe('describeTask named scores', () => { */ it('emits derived scores when the try settles, before case.ended', async () => { const correct = defineScore('correct') - const pass = defineScore('pass', { combine: { all: [correct] }, primary: true }) + const pass = defineScore('pass', { combine: { all: [correct] }, rank: 1 }) const emitted: Array<{ caseId?: string, data: unknown, event: string }> = [] const runResult = await runTask(describeTask('scores-derived', () => { @@ -2248,7 +2248,7 @@ describe('describeTask named scores', () => { it('records unscored values without counting them and marks derived scores unscored', async () => { const correct = defineScore('correct') const natural = defineScore('natural') - const pass = defineScore('pass', { combine: { all: [correct, natural] }, primary: true }) + const pass = defineScore('pass', { combine: { all: [correct, natural] }, rank: 1 }) const emitted: Array<{ caseId?: string, data: unknown, event: string }> = [] const runResult = await runTask(describeTask('scores-unscored', () => { @@ -2293,15 +2293,40 @@ describe('describeTask named scores', () => { /** * @example - * two primary scores in one task -> the task run throws when the second is announced. + * two scores of rank 1 in one task -> the task run throws when the second is announced. */ - it('rejects a second primary score in one task', async () => { - const first = defineScore('first', { primary: true }) - const second = defineScore('second', { primary: true }) + it('rejects a second score of the same rank in one task', async () => { + const first = defineScore('first', { rank: 1 }) + const second = defineScore('second', { rank: 1 }) await expect(runTask(describeTask('scores-two-primaries', () => { caseOf('case-1', () => {}) - }, { scores: [first, second] }), [])).rejects.toThrow('Score "second" is primary, but score "first" of the same task already is.') + }, { scores: [first, second] }), [])).rejects.toThrow('Score "second" has rank 1, but score "first" of the same task already has it.') + }) + + /** + * @example + * expect(defined.map(data => [data.name, data.rank, data.primary])).toEqual([['overall', 1, true], ['intent', 2, undefined]]) + */ + it('writes rank 1 as the primary score and keeps the other ranks', async () => { + const overall = defineScore('overall', { rank: 1 }) + const intent = defineScore('intent', { rank: 2 }) + const emitted: Array<{ caseId?: string, data: unknown, event: string }> = [] + + await runTask(describeTask('scores-ranks', () => { + caseOf('case-1', () => {}) + }, { scores: [overall, intent] }), emitted) + const defined = emitted.filter(event => event.event === 'score.defined').map(event => event.data as { name: string, primary?: boolean, rank?: number }) + expect(defined.map(data => [data.name, data.rank, data.primary])).toEqual([['overall', 1, true], ['intent', 2, undefined]]) + }) + + /** + * @example + * expect(() => defineScore('a', { rank: 0 })).toThrow() + */ + it('rejects a rank that is not a whole number from 1', () => { + expect(() => defineScore('a', { rank: 0 })).toThrow('Score "a" has rank 0; a rank is a whole number from 1.') + expect(() => defineScore('a', { rank: 1.5 })).toThrow('Score "a" has rank 1.5; a rank is a whole number from 1.') }) /** diff --git a/packages/vieval/src/events/index.ts b/packages/vieval/src/events/index.ts index fc1805e..94414b7 100644 --- a/packages/vieval/src/events/index.ts +++ b/packages/vieval/src/events/index.ts @@ -318,8 +318,13 @@ export const scoreDefinedDataSchema = v.object({ label: v.optional(localizedTextSchema), /** Score name, the same as in `score.recorded`. */ name: v.string(), - /** Whether this is the headline score of its task. At most one score of a task is primary. */ + /** Whether this is the headline score of its task (`rank` 1). At most one score of a task is primary. */ primary: v.optional(v.boolean()), + /** + * Importance among the scores of its task: 1 is the headline, 2 the next core score, and so on; absent for a + * supporting score. Reports written before ranks existed have none, and readers then keep the declaration order. + */ + rank: v.optional(v.number()), }) const scoreRecordedEntries = { From 16d45902b33444420f4ad7f903005afef43cd356 Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Wed, 4 Nov 2026 04:32:40 +0800 Subject: [PATCH 2/2] feat(ui): compare runs across workspaces on two axes, by case, and score by score - Comparisons list and home: experiments run in several workspaces as leaderboards; only comparisons of two or more things are listed. - Compare: Y / X / baseline pickers with an axis preview beside the menu (a drawer on phones); a two-axis distribution with 95% intervals, the Pareto front, a crosshair, linear or log scales, and a click-pinned column card; differences from the baseline paired by case (an estimation plot); every score and metric as compact heat cells shared with the matrix (`HeatCell`), with the best per axis outlined and a beyond-noise tally; columns can be hidden from the charts; wide screens lay the charts out in two or three columns. - Server: comparison rows split by the matrix axes every column has (an axis only one column runs is averaged), and cells carry counted scores, numeric metrics, and the case duration. - Viewer orders scores by rank; losses take the theme hue turned half a circle (`--series-1-opposite`); heat fills stay within 50% so the page ink keeps its contrast. Co-Authored-By: Claude Opus 5.5 --- packages/ui/API.md | 15 +- packages/ui/README.md | 6 +- packages/ui/client/src/caseMetrics.ts | 4 +- packages/ui/client/src/comparison.test.ts | 155 ++++++ packages/ui/client/src/comparison.ts | 262 +++++++++++ .../ui/client/src/components/CaseCard.vue | 8 +- .../ui/client/src/components/ChoiceMenu.vue | 181 +++++++ .../src/components/CompareAxisPreview.vue | 96 ++++ .../ui/client/src/components/CompareDelta.vue | 165 +++++++ .../ui/client/src/components/ComparePlane.vue | 444 ++++++++++++++++++ .../src/components/ComparePointCard.vue | 121 +++++ .../src/components/CompareScoreTable.vue | 195 ++++++++ .../src/components/ComparisonPreview.vue | 81 ++++ .../ui/client/src/components/DotStrip.vue | 31 ++ .../ui/client/src/components/HeatCell.vue | 95 ++++ .../ui/client/src/components/MarginBar.vue | 4 +- .../client/src/components/MatrixCellCard.vue | 2 +- .../src/components/MatrixCellDetail.vue | 2 +- .../client/src/components/MatrixHeatCell.vue | 63 +-- .../ui/client/src/components/MatrixView.vue | 17 +- .../src/components/MetricBesideCard.vue | 26 +- .../ui/client/src/components/PairsHeatmap.vue | 4 +- .../ui/client/src/components/ScoresPanel.vue | 5 +- .../ui/client/src/composables/useCatalog.ts | 48 ++ .../ui/client/src/i18n/locales/en/common.ts | 5 + .../ui/client/src/i18n/locales/en/compare.ts | 85 +++- .../client/src/i18n/locales/zh-Hans/common.ts | 5 + .../src/i18n/locales/zh-Hans/compare.ts | 85 +++- packages/ui/client/src/pages/CasePage.vue | 7 +- packages/ui/client/src/pages/ComparePage.vue | 403 +++++++++++++--- packages/ui/client/src/pages/HomePage.vue | 15 +- packages/ui/client/src/scores.test.ts | 20 +- packages/ui/client/src/scores.ts | 20 + packages/ui/client/src/styles.css | 11 +- packages/ui/src/server/app.test.ts | 2 +- packages/ui/src/server/compare.test.ts | 119 ++++- packages/ui/src/server/compare.ts | 146 +++++- packages/ui/src/shared/contract.ts | 29 +- packages/ui/uno.config.ts | 3 + 39 files changed, 2790 insertions(+), 195 deletions(-) create mode 100644 packages/ui/client/src/comparison.test.ts create mode 100644 packages/ui/client/src/comparison.ts create mode 100644 packages/ui/client/src/components/ChoiceMenu.vue create mode 100644 packages/ui/client/src/components/CompareAxisPreview.vue create mode 100644 packages/ui/client/src/components/CompareDelta.vue create mode 100644 packages/ui/client/src/components/ComparePlane.vue create mode 100644 packages/ui/client/src/components/ComparePointCard.vue create mode 100644 packages/ui/client/src/components/CompareScoreTable.vue create mode 100644 packages/ui/client/src/components/ComparisonPreview.vue create mode 100644 packages/ui/client/src/components/DotStrip.vue create mode 100644 packages/ui/client/src/components/HeatCell.vue diff --git a/packages/ui/API.md b/packages/ui/API.md index 13e9aa9..8ade99b 100644 --- a/packages/ui/API.md +++ b/packages/ui/API.md @@ -160,10 +160,17 @@ storage. Spans of the live OpenTelemetry runtime are not collected. ### Comparisons -All return `ComparisonResponse`: `columns` (each with the runs it covers), `rows` (one per case name; `cells[column]` -= `{ state, scores, roundCount }`, `scores` being `{ values, unscored }`; `changed` when the state or the value of -`primary` differs between columns), `primary` (the primary score the rows compare, when the columns share one; without -it, `changed` uses the state only), `scoreDefinitions`, `summary`. Each column's runs carry `headline`. +All return `ComparisonResponse`: `columns` (each with the runs it covers), `rows` (one per case name and `variant`; +`cells[column]` = `{ state, scores, counted, metrics, durationMs, roundCount, pooled }`, `scores` being +`{ values, unscored }`, `counted` the value the case counts with in each score (a failed case counts 0), `metrics` the +numeric metrics of the case; `changed` when the state or the value of `primary` differs between columns), `primary` +(the primary score the rows compare, when the columns share one; without it, `changed` uses the state only), +`scoreDefinitions`, `metricDefinitions`, `summary`. Each column's runs carry `headline`, `workspaceId`, and `project`. + +When a column ran a case of one name in several tasks, rows are split by the matrix axes that tell those tasks apart +and that every column has (`variant`, such as `context=none`), so the columns stay paired case by case. Tasks that +differ only in an axis some column lacks are averaged into one cell (`pooled` = how many; scores, metrics, and duration +are means, the state is the worst). | Route | Columns | |---|---| diff --git a/packages/ui/README.md b/packages/ui/README.md index 23b2a6a..e835204 100644 --- a/packages/ui/README.md +++ b/packages/ui/README.md @@ -76,7 +76,7 @@ every number on screen comes from a run's events, report files, or evidence dire | Run | `#/runs/` | Exact score with its coverage (share of settled cases that produced a result) and the case bar in the header; a **No result** card that groups the cases without a result by error name and message (long numbers and ids masked), with one example case per group; progress bar and progress chart (solid: completed cases over time; dashed: projection at the current ETA to the estimated total, fainter when it is a lower bound; "now" marker; recent rate in cases/h), cases done / estimated, running cases, tasks done, elapsed time, ETA and expected finish, current position (running cases with round, retry, elapsed). Tabs: **Execution** (virtualized live tree `project → task → round → case → try`, state filters, search, *Current* jumps to the first running case), **Scores & metrics** (scores, **Case metrics** over every case of the run with the eval entry and the matrix axes as categories, opening on the parallel sets, matrix rows table and chart, scores per round, numeric metrics as line charts, other metrics collapsed to one line each, expandable to every value per case), **Console logs** (all `console.*` lines of the run, linked to their case), **OpenTelemetry** (trace waterfall, log records, gauges, raw `otlp/*.json`). | | Task | `#/runs//task?taskId=` | Matrix values, the exact score with how the cases' scores spread, rounds (auto attempts) with pass rates, **Case metrics**, the virtualized case list (sortable by score, duration, or a numeric column; metric columns, string metrics such as a case id included, scrolling sideways when they do not fit and shown under the name on phones; the charts' selection as a chip that clears it). Case metrics draws every case against its metrics, and picks the chart by the number of dimensions (`state` is the color, not a dimension): one dimension is a strip of dots, two a plane (metric × metric scatter, metric × category bands, category × category grid), three or more parallel coordinates with a scatterplot matrix of the first five picked metrics. A metric recorded as a list of numbers per case (the grade of each vote, every sample's reward) is drawn as two numeric axes, its mean and its length. The chart opens on the five metrics that best tell failed from passed cases (AUC, scores left out), or, when every case passed (a scored eval), cases below the mean exact score from the rest; the *Axes* button opens a searchable list of the axes on the chart, the others ranked by AUC, metrics that move with another one (\|r\| ≥ 0.95 on log values), and those hidden as constant or repeated. *Color by* recolors by state, a category, or a metric; it opens on the exact score when every case ended the same way. Scores are colored low to high with the low end taking the most ink, so the cases that lose the score stand out. A metric is colored in five equal steps; its bands sit in the toolbar with their case counts, *+ cut* splits the widest band with a cut line drawn across the metric's axis (drag it to move, drag it past an end of the axis to remove it, *steps* goes back to five steps). Clicking a legend entry or a band selects its cases. While the color or axes menu is open, the metric under the pointer or the keyboard is described beside the menu (on phones: one line of description per metric, details in a second drawer). Besides the chart, **Sets** draws the category axes as parallel sets (ribbons as wide as their cases, each value with the mean exact score of the selection, best on top; titles drag to reorder and show the axis card on hover, like the parallel coordinates), and **Groups** lists the values of one category as bars with their mean score, lowest first. Dragging on an axis or in a scatterplot cell, or clicking a category value, selects cases; the case list below shows the selection. Hovering a line or dot previews its case; clicking pins it (its line and its value on every axis stay highlighted) and opens a card with the case's categories on one line and its most unusual metrics (furthest from the median) with a small histogram each, the rest folded, and a link to the case. On phones a tap opens that card in a bottom drawer (vaul). | | Case | `#/runs//case?taskId=&caseId=` | **Overview**: error, timeline of rounds (auto attempts) and tries (retries), input and output as object trees, metrics. **Console log**, **Artifacts**, **Evidence**, **Inference**: calls, input and output tokens (per model when a case used several, plus their sum), model time, and one row per model call with the tool calls it asked for. | -| Compare | `#/compare?view=experiments\|attempts\|runs\|matrix&…` | Per column: newest run, state, exact score, case counts; a bar chart of exact score and pass rate; a case × column table (state, exact score, rounds) with an "only differing" filter. `matrix` compares the matrix rows (for example `model` × `repeat`) of one eval entry inside one run. | +| Compare | `#/compare?view=experiments\|attempts\|runs\|matrix&…` | Without a `view`: the experiments run in several workspaces as leaderboards (also on the home page), then only the comparisons that have two or more things to compare. With one: per column, newest run, state, exact score, case counts; Y / X / baseline pickers that preview each axis beside the menu; a **two-axis distribution** (one dot per column with 95% intervals, the Pareto front, a crosshair, linear or log scales, a click-pinned card to make a column the baseline or open its run); the **differences from the baseline** paired by case; **every score and metric** as compact heat cells (best per axis outlined, a beyond-noise tally per column); columns can be hidden from the charts; a case × column table (state, exact score, rounds) with an "only differing" filter. Rows split by the matrix axes every column has; an axis only one column runs is averaged. `matrix` compares the matrix rows (for example `model` × `repeat`) of one eval entry inside one run. | Ids are query parameters because task and case ids contain `/` and `%`. Breadcrumbs show the path `projects → workspace → project → experiment → attempt → run → task → case` up to the page shown. Ancestors are links; @@ -152,7 +152,9 @@ A report path is `/////`. The four | try | `autoRetry` of a case | vieval retried one case at once inside a round. | — | *Compare → experiments* puts the newest run of each experiment of a project side by side. *Compare → attempts* puts -the runs of one experiment side by side. Neither changes the data; both read the same report directories. +the runs of one experiment side by side. *Same experiment across workspaces* puts the newest run of each workspace that +ran one experiment side by side (the same eval against several products). None of them changes the data; all read the +same report directories. ## Security of artifacts and evidence diff --git a/packages/ui/client/src/caseMetrics.ts b/packages/ui/client/src/caseMetrics.ts index e152536..9053704 100644 --- a/packages/ui/client/src/caseMetrics.ts +++ b/packages/ui/client/src/caseMetrics.ts @@ -9,7 +9,7 @@ import { parseTaskId } from '../../src/shared/reduce' import { formatCompact, formatScore } from './format' import { i18n } from './i18n' import { localize } from './localize' -import { primaryScoreName, scoreColumn } from './scores' +import { orderScores, primaryScoreName, scoreColumn } from './scores' /** One case of a task, flattened for the multi-metric charts (parallel coordinates, scatterplot matrix). */ export interface CaseRow { @@ -165,7 +165,7 @@ export function buildCaseTable(cases: readonly CaseNode[], metrics: MetricsRespo } } const primary = primaryScoreName(metrics?.scoreDefinitions ?? [], scoreNames) - const scores = primary == null ? scoreNames : [primary, ...scoreNames.filter(name => name !== primary)] + const scores = orderScores(primary == null ? scoreNames : [primary, ...scoreNames.filter(name => name !== primary)], metrics?.scoreDefinitions ?? []) const derivedNames: string[] = [] for (const series of metrics?.nonNumeric ?? []) { for (const point of series.values) { diff --git a/packages/ui/client/src/comparison.test.ts b/packages/ui/client/src/comparison.test.ts new file mode 100644 index 0000000..2e706df --- /dev/null +++ b/packages/ui/client/src/comparison.test.ts @@ -0,0 +1,155 @@ +import type { ComparisonCell, ComparisonResponse } from '../../src/shared/contract' +import type { CompareAxis } from './comparison' + +import { describe, expect, it } from 'vitest' + +import { columnIntervals, compareAxes, distinctLabels, pairedDeltas, paretoFront } from './comparison' + +const OVERALL: CompareAxis = { better: 'higher', kind: 'score', label: 'Overall', name: 'overall' } +const LATENCY: CompareAxis = { better: 'lower', kind: 'metric', label: 'Latency', name: 'latency_ms' } + +function cell(overall: number, latency?: number): ComparisonCell { + return { counted: { overall }, metrics: latency == null ? {} : { latency_ms: latency }, roundCount: 1, scores: { unscored: {}, values: { overall } }, state: 'passed' } +} + +/** A comparison of the given columns; `values[column][row]` is the overall score of that case (`null`: not run). */ +function comparison(values: Record): Pick { + const ids = Object.keys(values) + const length = Math.max(...Object.values(values).map(list => list.length)) + return { + columns: ids.map(id => ({ id, label: id, runs: [], scoreGraders: {} })), + metricDefinitions: [], + primary: 'overall', + rows: Array.from({ length }, (_, index) => ({ + caseName: `case-${index}`, + cells: Object.fromEntries(ids.flatMap(id => (values[id]![index] == null ? [] : [[id, cell(values[id]![index]!, 100 * (index + 1))]]))), + changed: false, + })), + scoreDefinitions: [{ name: 'overall', primary: true }], + } +} + +describe('columnIntervals', () => { + /** + * @example + * expect(interval.mean).toBe(200); expect(interval.high - interval.mean).toBeCloseTo(4.303 * 100 / Math.sqrt(3)) + */ + it('puts a Student-t 95% interval around each column mean', () => { + // Latencies of 100, 200, and 300 ms (the helper records 100 ms per case index). + const interval = columnIntervals(comparison({ a: [0, 0.5, 1] }), LATENCY).get('a')! + + expect(interval.mean).toBe(200) + expect(interval.n).toBe(3) + // sd = 100, n = 3, t(2) = 4.303 + expect(interval.high - interval.mean).toBeCloseTo(4.303 * 100 / Math.sqrt(3), 6) + }) +}) + +describe('columnIntervals of a score', () => { + /** + * @example + * expect(interval.high).toBe(1) + */ + it('keeps the interval of a score inside 0–100%', () => { + const interval = columnIntervals(comparison({ a: [1, 1, 0.5] }), OVERALL).get('a')! + + expect(interval.high).toBe(1) + expect(interval.low).toBeGreaterThanOrEqual(0) + }) +}) + +describe('pairedDeltas', () => { + /** + * @example + * expect(delta.interval).toMatchObject({ high: 0.1, low: 0.1, mean: 0.1 }) + */ + it('pairs the cases, so a column better by the same amount on every case has an interval of zero width', () => { + const [delta] = pairedDeltas(comparison({ base: [0.2, 0.5, 0.9], better: [0.3, 0.6, 1] }), OVERALL, 'base') + + expect(delta?.columnId).toBe('better') + expect(delta?.interval.mean).toBeCloseTo(0.1, 9) + expect(delta?.interval.high - delta!.interval.low).toBeCloseTo(0, 9) + expect([delta?.wins, delta?.losses, delta?.ties]).toEqual([3, 0, 0]) + }) + + /** + * @example + * expect(delta.differences.map(difference => difference.key)).toEqual(['case-0']) + */ + it('leaves out the cases one of the two did not run', () => { + const [delta] = pairedDeltas(comparison({ base: [0.5, null], other: [0.5, 1] }), OVERALL, 'base') + + expect(delta?.differences.map(difference => difference.key)).toEqual(['case-0']) + expect([delta?.wins, delta?.losses, delta?.ties]).toEqual([0, 0, 1]) + }) + + /** + * @example + * expect([delta.wins, delta.losses]).toEqual([2, 0]) + */ + it('counts a lower value as a win on an axis that is better lower', () => { + const result = comparison({ base: [0.5, 0.5], fast: [0.5, 0.5] }) + result.rows[0]!.cells.fast = cell(0.5, 50) + result.rows[1]!.cells.fast = cell(0.5, 150) + const [delta] = pairedDeltas(result, LATENCY, 'base') + + expect([delta?.wins, delta?.losses]).toEqual([2, 0]) + }) +}) + +describe('compareAxes', () => { + /** + * @example + * expect(compareAxes(result).map(axis => axis.name)).toEqual(['overall', 'latency_ms', 'duration']) + */ + it('lists the scores, then the metrics that vary between cases, then the duration', () => { + const result = comparison({ a: [0.5, 1] }) + result.rows[0]!.cells.a = { ...cell(0.5, 100), durationMs: 1000, metrics: { latency_ms: 100, repeat: 1 } } + result.rows[1]!.cells.a = { ...cell(1, 200), metrics: { latency_ms: 200, repeat: 1 } } + + expect(compareAxes(result).map(axis => axis.name)).toEqual(['overall', 'latency_ms', 'duration']) + }) +}) + +describe('paretoFront', () => { + /** + * @example + * expect(paretoFront(points, LATENCY, OVERALL)).toEqual(['cheap', 'best']) + */ + it('keeps the points no other point beats on both axes, ordered along x', () => { + const points = [ + { id: 'best', x: 300, y: 0.9 }, + { id: 'cheap', x: 100, y: 0.6 }, + { id: 'beaten', x: 300, y: 0.7 }, + ] + + expect(paretoFront(points, LATENCY, OVERALL)).toEqual(['cheap', 'best']) + }) + + /** + * @example + * expect(paretoFront(points, { ...LATENCY, better: undefined }, OVERALL)).toEqual([]) + */ + it('has no front when an axis does not say which way is better', () => { + expect(paretoFront([{ id: 'a', x: 1, y: 1 }], { ...LATENCY, better: undefined }, OVERALL)).toEqual([]) + }) +}) + +describe('distinctLabels', () => { + /** + * @example + * expect(distinctLabels([['arion', 'p-arion', 'exp', 'a1'], ['raw', 'p-raw', 'exp', 'a2']])).toEqual(['arion', 'raw']) + */ + it('names runs of several workspaces by the workspace alone', () => { + expect(distinctLabels([['arion', 'p-arion', 'exp', 'a1'], ['raw', 'p-raw', 'exp', 'a2']])).toEqual(['arion', 'raw']) + }) + + /** + * @example + * expect(labels).toEqual(['ws · exp-a', 'ws · exp-b'].map(...)) // the experiment, since the workspace is shared + */ + it('skips the levels the runs share and adds levels until every label is unique', () => { + expect(distinctLabels([['ws', 'p', 'exp-a', 'a1'], ['ws', 'p', 'exp-b', 'a2']])).toEqual(['exp-a', 'exp-b']) + expect(distinctLabels([['ws', 'p', 'exp-a', 'a1'], ['ws', 'p', 'exp-a', 'a2'], ['ws', 'p', 'exp-b', 'a3']])).toEqual(['exp-a · a1', 'exp-a · a2', 'exp-b · a3']) + }) +}) diff --git a/packages/ui/client/src/comparison.ts b/packages/ui/client/src/comparison.ts new file mode 100644 index 0000000..e4d0c7d --- /dev/null +++ b/packages/ui/client/src/comparison.ts @@ -0,0 +1,262 @@ +import type { ComparisonCell, ComparisonResponse } from '../../src/shared/contract' +import type { ColumnDefinition } from './scores' + +import { columnDefinitions, formatColumnValue, metricLabel } from './caseMetrics' +import { formatDuration, formatScore } from './format' +import { i18n } from './i18n' +import { orderScores, scoreLabel } from './scores' + +/** 97.5% quantiles of Student's t for 1–30 degrees of freedom (see {@link meanInterval}). */ +const T_975 = [12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, 2.045, 2.042] + +/** A quantity the comparison charts can put on an axis: a score, a numeric metric, or the case duration. */ +export interface CompareAxis { + /** Which way is better; `undefined` when the metric does not say (no Pareto front then). */ + better?: 'higher' | 'lower' + kind: 'duration' | 'metric' | 'score' + label: string + /** Score or metric name; `duration` for the case duration. */ + name: string +} + +/** Mean of a quantity over cases with its 95% confidence interval. */ +export interface Interval { + high: number + low: number + mean: number + /** Cases the mean is over. */ + n: number +} + +/** One column against the baseline over the cases both ran (paired by row). */ +export interface PairedDelta { + columnId: string + /** Difference of each shared case (column − baseline), by row key. */ + differences: { key: string, value: number }[] + /** Mean difference and its 95% confidence interval. */ + interval: Interval + /** Cases where the column is better, worse, or equal, by the axis's direction (higher is better without one). */ + losses: number + ties: number + wins: number +} + +/** Value a cell has on an axis: the counted score (a failed case counts 0), the metric, or the duration. */ +export function axisValue(cell: ComparisonCell | undefined, axis: Pick): number | undefined { + if (cell == null) + return undefined + if (axis.kind === 'score') + return cell.counted[axis.name] + if (axis.kind === 'metric') + return cell.metrics[axis.name] + return cell.durationMs +} + +/** + * Whether an axis can be drawn on a log scale: every column mean is above 0. Scores qualify too (a log scale spreads + * columns that cluster near the top less than near the bottom, which can separate low scorers); an interval end at or + * below 0 is cut at the axis edge. + */ +export function canLogScale(intervals: Iterable): boolean { + const means = [...intervals].map(interval => interval.mean) + return means.length > 0 && means.every(mean => mean > 0) +} + +/** + * Mean of each column on an axis over its cases, with a 95% confidence interval. + * + * Use when: + * - placing the columns on the score plane, where the interval tells a real gap from noise + * + * Returns: + * - by column id; a column without a value on the axis is absent + */ +export function columnIntervals(result: Pick, axis: CompareAxis): Map { + const intervals = new Map() + for (const column of result.columns) { + const values = result.rows.map(row => axisValue(row.cells[column.id], axis)).filter(value => value != null) + const interval = meanInterval(values) + if (interval == null) + continue + // A score is a share of 0–1; the t-interval of a few cases near either end reaches past it ("103.7%"). + intervals.set(column.id, axis.kind === 'score' ? { ...interval, high: Math.min(1, interval.high), low: Math.max(0, interval.low) } : interval) + } + return intervals +} + +/** + * Axes the comparison can be read by: every score the cells count (by rank, see {@link orderScores}), then every numeric metric that + * varies between cases, then the case duration. + * + * Use when: + * - filling the axis pickers of the comparison charts + * + * Returns: + * - scores better when higher unless their definition says otherwise; metrics with the direction their definition + * declares; duration better when lower + */ +export function compareAxes(result: Pick): CompareAxis[] { + const cells = result.rows.flatMap(row => Object.values(row.cells)) + const scores = new Set(cells.flatMap(cell => Object.keys(cell.counted))) + const metrics = new Map>() + for (const cell of cells) { + for (const [name, value] of Object.entries(cell.metrics)) { + const set = metrics.get(name) ?? new Set() + set.add(value) + metrics.set(name, set) + } + } + const definitions = columnDefinitions(result.metricDefinitions, result.scoreDefinitions) + // Ranked scores first (the headline leads), then the supporting scores: the primary first, then by definition order. + const definitionOrder = result.scoreDefinitions.map(definition => definition.name) + const orderOf = (name: string) => (name === result.primary ? -1 : definitionOrder.includes(name) ? definitionOrder.indexOf(name) : definitionOrder.length) + const byDefinition = [...scores].sort((left, right) => orderOf(left) - orderOf(right) || left.localeCompare(right)) + const scoreAxes = orderScores(byDefinition, result.scoreDefinitions).map((name): CompareAxis => ({ + better: definitions.get(name)?.better ?? 'higher', + kind: 'score', + label: scoreLabel(name, result.scoreDefinitions), + name, + })) + // A metric with one value across every case (an index, a repeat number) cannot separate the columns. + const metricAxes = [...metrics].filter(([name, values]) => values.size > 1 && !scores.has(name)).map(([name]) => name).sort().map((name): CompareAxis => ({ + better: definitions.get(name)?.better, + kind: 'metric', + label: metricLabel(name, definitions), + name, + })) + const durations = cells.some(cell => cell.durationMs != null) + return [...scoreAxes, ...metricAxes, ...(durations ? [{ better: 'lower' as const, kind: 'duration' as const, label: i18n.global.t('compare.chart.duration'), name: 'duration' }] : [])] +} + +/** + * Picks the column labels: the hierarchy levels (workspace, project, experiment, attempt) that tell the columns apart, + * from the most general one that differs, adding levels until every label is unique. + * + * Before: + * - runs of `arion/rephrase-arion/asr` and `raw/rephrase-raw/asr`, attempts `a-1`, `a-2` + * + * After: + * - "arion", "raw" + */ +export function distinctLabels(levels: readonly (readonly string[])[]): string[] { + const depth = levels[0]?.length ?? 0 + const varying = Array.from({ length: depth }, (_, level) => level).filter(level => new Set(levels.map(parts => parts[level])).size > 1) + const picked: number[] = [] + for (const level of varying) { + picked.push(level) + if (new Set(levels.map(parts => picked.map(index => parts[index]).join(' · '))).size === levels.length) + break + } + return levels.map(parts => (picked.length === 0 ? parts.at(-1) ?? '' : picked.map(index => parts[index]).join(' · '))) +} + +/** + * Formats a difference on an axis with its sign: score points (`+3.2 pt`), durations, or metric values. + * + * Before: + * - 0.032 on a score; -1200 on the duration + * + * After: + * - "+3.2 pt"; "−1.2s" + */ +export function formatAxisDelta(value: number, axis: CompareAxis, definitions: ReadonlyMap): string { + const sign = value > 0 ? '+' : value < 0 ? '−' : '±' + if (axis.kind === 'score') + return `${sign}${i18n.global.t('compare.chart.points', { n: (Math.abs(value) * 100).toFixed(1) })}` + return `${sign}${formatAxisValue(Math.abs(value), axis, definitions)}` +} + +/** Formats a value of an axis: scores as percentages, durations as durations, metrics by their unit (3 significant digits). */ +export function formatAxisValue(value: number, axis: CompareAxis, definitions: ReadonlyMap): string { + if (axis.kind === 'score') + return formatScore(value) + if (axis.kind === 'duration') + return formatDuration(value) + // Means and differences of a metric carry the noise of averaging ("765.83 ms"); three significant digits are plenty + // next to intervals of ±100 ms. + return formatColumnValue(value === 0 ? 0 : Number(value.toPrecision(3)), definitions.get(axis.name)) +} + +/** + * Each column against the baseline column, paired by case: the mean of the per-case differences with its 95% + * confidence interval, and how many cases got better or worse. + * + * Use when: + * - reading whether a column really beats the baseline: pairing by case removes the spread between easy and hard + * cases, so the interval is far narrower than comparing two means + * + * Expects: + * - `baseline` to be one of the column ids + * + * Returns: + * - one entry per other column with at least one case shared with the baseline, in column order + */ +export function pairedDeltas(result: Pick, axis: CompareAxis, baseline: string): PairedDelta[] { + const lowerIsBetter = axis.better === 'lower' + return result.columns.filter(column => column.id !== baseline).flatMap((column) => { + const differences = result.rows.flatMap((row) => { + const base = axisValue(row.cells[baseline], axis) + const value = axisValue(row.cells[column.id], axis) + return base == null || value == null ? [] : [{ key: rowKey(row), value: value - base }] + }) + const interval = meanInterval(differences.map(difference => difference.value)) + if (interval == null) + return [] + const better = (value: number) => (lowerIsBetter ? value < 0 : value > 0) + const worse = (value: number) => (lowerIsBetter ? value > 0 : value < 0) + return [{ + columnId: column.id, + differences, + interval, + losses: differences.filter(difference => worse(difference.value)).length, + ties: differences.filter(difference => difference.value === 0).length, + wins: differences.filter(difference => better(difference.value)).length, + }] + }) +} + +/** + * Columns no other column beats on both axes (the Pareto front), by each axis's direction. + * + * Use when: + * - the plane puts a score against a cost (latency, tokens): the front is every column worth picking, the rest are + * beaten by one of them on both + * + * Returns: + * - the ids on the front, ordered along the x axis; empty when an axis has no direction + */ +export function paretoFront(points: readonly { id: string, x: number, y: number }[], x: CompareAxis, y: CompareAxis): string[] { + if (x.better == null || y.better == null) + return [] + // Flip a lower-is-better axis so that larger is better on both. + const sx = x.better === 'higher' ? 1 : -1 + const sy = y.better === 'higher' ? 1 : -1 + const dominated = (point: { x: number, y: number }) => points.some(other => + other !== point && sx * other.x >= sx * point.x && sy * other.y >= sy * point.y && (other.x !== point.x || other.y !== point.y)) + return points.filter(point => !dominated(point)).toSorted((left, right) => left.x - right.x).map(point => point.id) +} + +/** Rows ordered as the comparison table lists them; `key` is unique across rows. */ +export function rowKey(row: Pick): string { + return row.variant == null ? row.caseName : `${row.caseName}\u0000${row.variant}` +} + +/** + * Mean with a two-sided 95% Student-t confidence interval; one value has an interval of zero width. + * + * NOTICE: + * The comparisons are over a handful of cases (often 3–20), where the normal 1.96 understates the interval: with 6 + * cases the t quantile is 2.57. Values past 30 degrees of freedom use the normal quantile; the table holds the + * 97.5% quantiles of Student's t for 1–30 degrees of freedom. + */ +function meanInterval(values: readonly number[]): Interval | undefined { + const n = values.length + if (n === 0) + return undefined + const mean = values.reduce((sum, value) => sum + value, 0) / n + if (n === 1) + return { high: mean, low: mean, mean, n } + const variance = values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (n - 1) + const half = (T_975[n - 2] ?? 1.96) * Math.sqrt(variance / n) + return { high: mean + half, low: mean - half, mean, n } +} diff --git a/packages/ui/client/src/components/CaseCard.vue b/packages/ui/client/src/components/CaseCard.vue index 525fb60..8116ded 100644 --- a/packages/ui/client/src/components/CaseCard.vue +++ b/packages/ui/client/src/components/CaseCard.vue @@ -88,8 +88,8 @@ const lines = computed(() => props.axes.map((axis) => { const notability = Math.abs(percentile - 50) / 50 return { bin, detail: `p${percentile}`, heights, kind: 'numeric' as const, name: axis.name, notability, position, text: props.formatOf(axis.name, value) } })) -/** Small round icon action beside the case name; finger-sized on phones (the card opens in a drawer there). */ -const action = 'grid place-items-center size-9 md:size-6 shrink-0 rounded-full border border-neutral-200 dark:border-neutral-700/60 text-neutral-600 dark:text-neutral-300 transition-colors hover:(text-primary-700 dark:text-primary-200 border-primary-500/50 bg-primary-500/8) focus-visible:(outline-2 outline-offset-1 outline-primary-500)' +// The small round actions beside the case name previously lived here as a local class string; they are the +// `card-action` shortcut now, shared with the comparison plane's column card so both draw their icons at one size. const active = computed(() => lines.value.find(line => line.name === props.activeAxis)) /** Numeric rows shown before "more": the values that sit furthest from the middle of all cases. */ @@ -147,7 +147,7 @@ const rows = computed(() => (showTypical.value ? [...notable.value, ...typical.v