Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 19 additions & 6 deletions server/services/artificialAnalysis.js
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
import { getSettings, updateSettingsWith } from './settings.js';
import { ServerError } from '../lib/errorHandler.js';
import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';
import { modelComparisonImportSchema } from '../lib/validation.js';
import { importModelComparison } from './modelComparison.js';
import { canonicalCatalogModelSlug } from '../lib/comparisonModelScope.js';

export const KNOWN_EFFORTS = ['non-reasoning', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'ultracode'];

const ARTIFICIAL_ANALYSIS_REQUEST_TIMEOUT_MS = 15_000;
const ARTIFICIAL_ANALYSIS_MAX_PAGES = 50;

export function slugify(text) {
return String(text || '')
.toLowerCase()
Expand Down Expand Up @@ -188,13 +192,20 @@ export function transformAAModelsToObservations(models, options = {}) {

export async function fetchAllArtificialAnalysisModels(apiKey) {
if (!apiKey) throw new ServerError('Artificial Analysis API key is required', { status: 400 });
let page = 1;
const allModels = [];
let intelligenceIndexVersion;
while (true) {
const res = await fetch(`https://artificialanalysis.ai/api/v2/language/models/free?page=${page}`, {
headers: { 'x-api-key': apiKey },
});
for (let page = 1; page <= ARTIFICIAL_ANALYSIS_MAX_PAGES; page++) {
let res;
try {
res = await fetchWithTimeout(`https://artificialanalysis.ai/api/v2/language/models/free?page=${page}`, {
headers: { 'x-api-key': apiKey },
}, ARTIFICIAL_ANALYSIS_REQUEST_TIMEOUT_MS);
} catch (error) {
if (error?.name === 'AbortError') {
throw new ServerError('Artificial Analysis request timed out; retry sync', { status: 502 });
}
throw error;
}
if (!res.ok) {
throw new ServerError(`Artificial Analysis API failed (${res.status}): ${res.statusText || 'request rejected'}`, { status: res.status === 401 || res.status === 403 ? 401 : 502 });
}
Expand All @@ -209,7 +220,9 @@ export async function fetchAllArtificialAnalysisModels(apiKey) {
intelligenceIndexVersion = version;
allModels.push(...json.data);
if (!json.pagination?.has_more) break;
page++;
if (page === ARTIFICIAL_ANALYSIS_MAX_PAGES) {
throw new ServerError('Artificial Analysis pagination exceeded 50 pages; retry sync', { status: 502 });
}
}
return { models: allModels, intelligenceIndexVersion };
}
Expand Down
56 changes: 56 additions & 0 deletions server/services/artificialAnalysis.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,62 @@ describe('artificialAnalysis service', () => {
globalThis.fetch = originalFetch;
}
});

it('maps a hung page request to a redacted 502 timeout', async () => {
vi.useFakeTimers();
const apiKey = 'secret-test-key';
vi.stubGlobal('fetch', vi.fn((_url, { signal }) => new Promise((_resolve, reject) => {
signal.addEventListener('abort', () => reject(new DOMException('aborted', 'AbortError')));
})));

try {
const pending = fetchAllArtificialAnalysisModels(apiKey);
vi.advanceTimersByTime(15_000);

const error = await pending.catch(rejection => rejection);
expect(error).toMatchObject({
name: 'ServerError',
status: 502,
message: 'Artificial Analysis request timed out; retry sync',
});
expect(fetch).toHaveBeenCalledTimes(1);
expect(fetch).toHaveBeenCalledWith(
expect.stringContaining('page=1'),
expect.objectContaining({ headers: { 'x-api-key': apiKey } }),
);
expect(error.message).not.toContain(apiKey);
} finally {
vi.useRealTimers();
vi.unstubAllGlobals();
}
});

it('stops after 50 pages when pagination never ends', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({
intelligence_index_version: 4.3,
data: [{ id: 'model', name: 'Page Model', slug: 'page-model' }],
pagination: { has_more: true },
}),
});
vi.stubGlobal('fetch', fetchMock);

try {
await expect(fetchAllArtificialAnalysisModels('test-key')).rejects.toMatchObject({
name: 'ServerError',
status: 502,
message: 'Artificial Analysis pagination exceeded 50 pages; retry sync',
});
expect(fetchMock).toHaveBeenCalledTimes(50);
expect(fetchMock).toHaveBeenLastCalledWith(
'https://artificialanalysis.ai/api/v2/language/models/free?page=50',
expect.objectContaining({ headers: { 'x-api-key': 'test-key' } }),
);
} finally {
vi.unstubAllGlobals();
}
});
});

describe('syncArtificialAnalysisCatalog', () => {
Expand Down