mirror of
https://github.com/Routstr/routstr-core.git
synced 2026-10-05 12:28:22 +00:00
feat: add multi-provider certification page
This commit is contained in:
9qeklajc
@@ -0,0 +1,568 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useMemo, useState } from 'react';
|
||||
import Link from 'next/link';
|
||||
import { useQueries, useQuery } from '@tanstack/react-query';
|
||||
import {
|
||||
ArrowLeft,
|
||||
CheckCircle2,
|
||||
ChevronDown,
|
||||
Loader2,
|
||||
Play,
|
||||
RotateCcw,
|
||||
Server,
|
||||
} from 'lucide-react';
|
||||
|
||||
import { AppPageShell } from '@/components/app-page-shell';
|
||||
import { PageHeader } from '@/components/page-header';
|
||||
import {
|
||||
ProviderCertificationResults,
|
||||
summarizeCertificationResults,
|
||||
} from '@/components/provider-certification-results';
|
||||
import {
|
||||
getCertificationModelNames,
|
||||
ProviderCertificationSetupPanel,
|
||||
} from '@/components/provider-certification-setup';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Card, CardContent } from '@/components/ui/card';
|
||||
import { Checkbox } from '@/components/ui/checkbox';
|
||||
import {
|
||||
Command,
|
||||
CommandEmpty,
|
||||
CommandInput,
|
||||
CommandItem,
|
||||
CommandList,
|
||||
} from '@/components/ui/command';
|
||||
import {
|
||||
Popover,
|
||||
PopoverContent,
|
||||
PopoverTrigger,
|
||||
} from '@/components/ui/popover';
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from '@/components/ui/select';
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
|
||||
import { runProviderCertification } from '@/hooks/use-provider-certification-runner';
|
||||
import { AdminService } from '@/lib/api/services/admin';
|
||||
import type {
|
||||
ProviderModels,
|
||||
UpstreamProvider,
|
||||
} from '@/lib/api/services/admin';
|
||||
import {
|
||||
buildModelRuns,
|
||||
countCertificationTargets,
|
||||
emptyCertificationSetup,
|
||||
getModelsNeedingPath,
|
||||
} from '@/lib/provider-certification';
|
||||
import type {
|
||||
CertificationProgress,
|
||||
ModelCertificationResult,
|
||||
ProviderCertificationSetup,
|
||||
} from '@/lib/provider-certification';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
function providerName(provider: UpstreamProvider): string {
|
||||
return provider.slug || provider.provider_type;
|
||||
}
|
||||
|
||||
export default function MultiProviderCertificationPage() {
|
||||
const [selectedProviderIds, setSelectedProviderIds] = useState<number[]>([]);
|
||||
const [activeProviderId, setActiveProviderId] = useState<number | null>(null);
|
||||
const [setups, setSetups] = useState<
|
||||
Record<number, ProviderCertificationSetup>
|
||||
>({});
|
||||
const [workspaceTab, setWorkspaceTab] = useState<'setup' | 'results'>(
|
||||
'setup'
|
||||
);
|
||||
const [resultsByProvider, setResultsByProvider] = useState<
|
||||
Record<number, ModelCertificationResult[]>
|
||||
>({});
|
||||
const [progressByProvider, setProgressByProvider] = useState<
|
||||
Record<number, CertificationProgress | null>
|
||||
>({});
|
||||
const [runningProviderIds, setRunningProviderIds] = useState<number[]>([]);
|
||||
|
||||
const providersQuery = useQuery({
|
||||
queryKey: ['upstream-providers'],
|
||||
queryFn: () => AdminService.getUpstreamProviders(),
|
||||
refetchOnWindowFocus: false,
|
||||
});
|
||||
const providers = useMemo(
|
||||
() => providersQuery.data ?? [],
|
||||
[providersQuery.data]
|
||||
);
|
||||
const providersById = useMemo(
|
||||
() => new Map(providers.map((provider) => [provider.id, provider])),
|
||||
[providers]
|
||||
);
|
||||
|
||||
const modelQueries = useQueries({
|
||||
queries: selectedProviderIds.map((providerId) => ({
|
||||
queryKey: ['provider-models', providerId],
|
||||
queryFn: () => AdminService.getProviderModels(providerId),
|
||||
refetchOnWindowFocus: false,
|
||||
})),
|
||||
});
|
||||
const modelQueryByProvider = new Map(
|
||||
selectedProviderIds.map((providerId, index) => [
|
||||
providerId,
|
||||
modelQueries[index],
|
||||
])
|
||||
);
|
||||
|
||||
const selectedProviders = selectedProviderIds
|
||||
.map((providerId) => providersById.get(providerId))
|
||||
.filter((provider): provider is UpstreamProvider => Boolean(provider));
|
||||
const activeProvider = activeProviderId
|
||||
? providersById.get(activeProviderId)
|
||||
: undefined;
|
||||
const activeSetup = activeProviderId
|
||||
? (setups[activeProviderId] ?? emptyCertificationSetup())
|
||||
: undefined;
|
||||
const activeModelsQuery = activeProviderId
|
||||
? modelQueryByProvider.get(activeProviderId)
|
||||
: undefined;
|
||||
|
||||
useEffect(() => {
|
||||
if (runningProviderIds.length === 0) return;
|
||||
const warnBeforeUnload = (event: BeforeUnloadEvent) => {
|
||||
event.preventDefault();
|
||||
};
|
||||
window.addEventListener('beforeunload', warnBeforeUnload);
|
||||
return () => window.removeEventListener('beforeunload', warnBeforeUnload);
|
||||
}, [runningProviderIds.length]);
|
||||
|
||||
const toggleProvider = (providerId: number) => {
|
||||
if (runningProviderIds.includes(providerId)) return;
|
||||
const selected = selectedProviderIds.includes(providerId);
|
||||
if (selected) {
|
||||
const remaining = selectedProviderIds.filter((id) => id !== providerId);
|
||||
setSelectedProviderIds(remaining);
|
||||
if (activeProviderId === providerId) {
|
||||
setActiveProviderId(remaining[0] ?? null);
|
||||
}
|
||||
return;
|
||||
}
|
||||
setSelectedProviderIds((current) => [...current, providerId]);
|
||||
setSetups((current) => ({
|
||||
...current,
|
||||
[providerId]: current[providerId] ?? emptyCertificationSetup(),
|
||||
}));
|
||||
setActiveProviderId(providerId);
|
||||
};
|
||||
|
||||
const updateProviderSetup = (
|
||||
providerId: number,
|
||||
setup: ProviderCertificationSetup
|
||||
) => {
|
||||
setSetups((current) => ({ ...current, [providerId]: setup }));
|
||||
setResultsByProvider((current) => ({ ...current, [providerId]: [] }));
|
||||
};
|
||||
|
||||
const providerRuns = (providerId: number) =>
|
||||
buildModelRuns(
|
||||
setups[providerId] ?? emptyCertificationSetup(),
|
||||
modelQueryByProvider.get(providerId)?.data as ProviderModels | undefined
|
||||
);
|
||||
|
||||
const isProviderReady = (providerId: number): boolean => {
|
||||
const setup = setups[providerId] ?? emptyCertificationSetup();
|
||||
return (
|
||||
setup.selectedModelIds.length > 0 &&
|
||||
getModelsNeedingPath(setup).length === 0 &&
|
||||
Boolean(modelQueryByProvider.get(providerId)?.data)
|
||||
);
|
||||
};
|
||||
|
||||
const runOneProvider = async (providerId: number) => {
|
||||
const setup = setups[providerId] ?? emptyCertificationSetup();
|
||||
const modelRuns = providerRuns(providerId);
|
||||
setResultsByProvider((current) => ({ ...current, [providerId]: [] }));
|
||||
setProgressByProvider((current) => ({ ...current, [providerId]: null }));
|
||||
setRunningProviderIds((current) =>
|
||||
current.includes(providerId) ? current : [...current, providerId]
|
||||
);
|
||||
try {
|
||||
await runProviderCertification({
|
||||
providerId,
|
||||
modelRuns,
|
||||
includeCache: setup.checkCache,
|
||||
onProgress: (progress) =>
|
||||
setProgressByProvider((current) => ({
|
||||
...current,
|
||||
[providerId]: progress,
|
||||
})),
|
||||
onResults: (results) =>
|
||||
setResultsByProvider((current) => ({
|
||||
...current,
|
||||
[providerId]: results,
|
||||
})),
|
||||
});
|
||||
} finally {
|
||||
setProgressByProvider((current) => ({
|
||||
...current,
|
||||
[providerId]: null,
|
||||
}));
|
||||
setRunningProviderIds((current) =>
|
||||
current.filter((id) => id !== providerId)
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const runAllProviders = () => {
|
||||
setWorkspaceTab('results');
|
||||
void Promise.all(selectedProviderIds.map(runOneProvider));
|
||||
};
|
||||
|
||||
const activeResults = activeProviderId
|
||||
? (resultsByProvider[activeProviderId] ?? [])
|
||||
: [];
|
||||
const activeNames = getCertificationModelNames(
|
||||
activeModelsQuery?.data as ProviderModels | undefined
|
||||
);
|
||||
|
||||
const totalModels = selectedProviderIds.reduce(
|
||||
(total, providerId) =>
|
||||
total + (setups[providerId]?.selectedModelIds.length ?? 0),
|
||||
0
|
||||
);
|
||||
const totalRoutes = selectedProviderIds.reduce(
|
||||
(total, providerId) =>
|
||||
total + countCertificationTargets(providerRuns(providerId)),
|
||||
0
|
||||
);
|
||||
const incompleteProviders = selectedProviderIds.filter(
|
||||
(providerId) => !isProviderReady(providerId)
|
||||
);
|
||||
const allReady =
|
||||
selectedProviderIds.length > 0 && incompleteProviders.length === 0;
|
||||
const allResults = Object.values(resultsByProvider).flat();
|
||||
const aggregateSummary = summarizeCertificationResults(allResults);
|
||||
const pendingRoutes = Math.max(totalRoutes - allResults.length, 0);
|
||||
|
||||
return (
|
||||
<AppPageShell contentClassName='mx-auto flex w-full max-w-7xl flex-col'>
|
||||
<div className='flex min-h-0 flex-1 flex-col gap-4'>
|
||||
<PageHeader
|
||||
title='Provider Certification'
|
||||
description='Configure and certify models across multiple upstream providers.'
|
||||
actions={
|
||||
<div className='flex gap-2'>
|
||||
<Button asChild variant='outline'>
|
||||
<Link href='/providers'>
|
||||
<ArrowLeft className='h-4 w-4' />
|
||||
Providers
|
||||
</Link>
|
||||
</Button>
|
||||
<Popover>
|
||||
<PopoverTrigger asChild>
|
||||
<Button variant='outline'>
|
||||
<Server className='h-4 w-4' />
|
||||
Select providers
|
||||
<Badge variant='secondary'>
|
||||
{selectedProviderIds.length}
|
||||
</Badge>
|
||||
<ChevronDown className='h-4 w-4' />
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent align='end' className='w-80 p-0'>
|
||||
<Command className='h-auto'>
|
||||
<CommandInput placeholder='Search providers…' />
|
||||
<CommandList className='max-h-72'>
|
||||
<CommandEmpty>
|
||||
{providersQuery.isLoading
|
||||
? 'Loading providers…'
|
||||
: 'No providers found'}
|
||||
</CommandEmpty>
|
||||
{providers.map((provider) => (
|
||||
<CommandItem
|
||||
key={provider.id}
|
||||
value={`${providerName(provider)} ${provider.base_url}`}
|
||||
onSelect={() => toggleProvider(provider.id)}
|
||||
disabled={runningProviderIds.includes(provider.id)}
|
||||
>
|
||||
<Checkbox
|
||||
checked={selectedProviderIds.includes(provider.id)}
|
||||
tabIndex={-1}
|
||||
aria-hidden='true'
|
||||
className='pointer-events-none'
|
||||
/>
|
||||
<div className='min-w-0 flex-1'>
|
||||
<div className='truncate'>
|
||||
{providerName(provider)}
|
||||
</div>
|
||||
<div className='text-muted-foreground truncate text-xs'>
|
||||
{provider.base_url}
|
||||
</div>
|
||||
</div>
|
||||
<Badge
|
||||
variant={provider.enabled ? 'secondary' : 'outline'}
|
||||
className='text-[10px]'
|
||||
>
|
||||
{provider.enabled ? 'Enabled' : 'Disabled'}
|
||||
</Badge>
|
||||
</CommandItem>
|
||||
))}
|
||||
</CommandList>
|
||||
</Command>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
</div>
|
||||
}
|
||||
/>
|
||||
|
||||
{selectedProviders.length === 0 ? (
|
||||
<Card className='flex min-h-72 items-center justify-center'>
|
||||
<CardContent className='space-y-3 text-center'>
|
||||
<Server className='text-muted-foreground mx-auto h-8 w-8' />
|
||||
<div>
|
||||
<div className='font-medium'>Select providers to certify</div>
|
||||
<p className='text-muted-foreground mt-1 text-sm'>
|
||||
Choose two or more providers to configure independent model
|
||||
and path runs.
|
||||
</p>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
) : (
|
||||
<div className='grid min-h-0 flex-1 gap-4 md:grid-cols-[240px_minmax(0,1fr)]'>
|
||||
<aside className='hidden min-h-0 space-y-2 overflow-y-auto rounded-lg border p-2 md:block'>
|
||||
{selectedProviders.map((provider) => {
|
||||
const setup = setups[provider.id] ?? emptyCertificationSetup();
|
||||
const runs = providerRuns(provider.id);
|
||||
const results = resultsByProvider[provider.id] ?? [];
|
||||
const summary = summarizeCertificationResults(results);
|
||||
const running = runningProviderIds.includes(provider.id);
|
||||
return (
|
||||
<button
|
||||
key={provider.id}
|
||||
type='button'
|
||||
onClick={() => setActiveProviderId(provider.id)}
|
||||
className={cn(
|
||||
'hover:bg-muted w-full space-y-2 rounded-md border p-3 text-left transition-colors',
|
||||
activeProviderId === provider.id &&
|
||||
'border-primary bg-muted/60'
|
||||
)}
|
||||
>
|
||||
<div className='flex items-start justify-between gap-2'>
|
||||
<div className='min-w-0'>
|
||||
<div className='truncate text-sm font-medium'>
|
||||
{providerName(provider)}
|
||||
</div>
|
||||
<div className='text-muted-foreground truncate text-xs'>
|
||||
{provider.base_url}
|
||||
</div>
|
||||
</div>
|
||||
{running ? (
|
||||
<Loader2 className='h-4 w-4 shrink-0 animate-spin' />
|
||||
) : results.length > 0 ? (
|
||||
<CheckCircle2 className='h-4 w-4 shrink-0 text-emerald-600' />
|
||||
) : null}
|
||||
</div>
|
||||
<div className='text-muted-foreground flex flex-wrap gap-1 text-[11px]'>
|
||||
<span>{setup.selectedModelIds.length} models</span>
|
||||
<span>·</span>
|
||||
<span>{countCertificationTargets(runs)} routes</span>
|
||||
</div>
|
||||
{results.length > 0 && (
|
||||
<div className='flex flex-wrap gap-1'>
|
||||
<Badge variant='secondary' className='text-[10px]'>
|
||||
{summary.ok} ok
|
||||
</Badge>
|
||||
{(summary.warn > 0 ||
|
||||
summary.fail > 0 ||
|
||||
summary.error > 0) && (
|
||||
<Badge variant='outline' className='text-[10px]'>
|
||||
{summary.warn + summary.fail + summary.error} issues
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</aside>
|
||||
|
||||
<div className='min-h-0 space-y-3 md:hidden'>
|
||||
<Select
|
||||
value={activeProviderId?.toString()}
|
||||
onValueChange={(value) => setActiveProviderId(Number(value))}
|
||||
>
|
||||
<SelectTrigger className='w-full'>
|
||||
<SelectValue placeholder='Choose active provider' />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{selectedProviders.map((provider) => (
|
||||
<SelectItem
|
||||
key={provider.id}
|
||||
value={provider.id.toString()}
|
||||
>
|
||||
{providerName(provider)}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
|
||||
{activeProvider && activeSetup ? (
|
||||
<Card className='flex min-h-[36rem] min-w-0 flex-col overflow-hidden p-0 md:min-h-0'>
|
||||
<div className='shrink-0 border-b px-4 py-3'>
|
||||
<div className='flex flex-wrap items-start justify-between gap-2'>
|
||||
<div className='min-w-0'>
|
||||
<div className='font-medium'>
|
||||
{providerName(activeProvider)}
|
||||
</div>
|
||||
<div className='text-muted-foreground truncate text-xs'>
|
||||
{activeProvider.base_url}
|
||||
</div>
|
||||
</div>
|
||||
<Badge
|
||||
variant={activeProvider.enabled ? 'secondary' : 'outline'}
|
||||
>
|
||||
{activeProvider.enabled ? 'Enabled' : 'Disabled'}
|
||||
</Badge>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<Tabs
|
||||
value={workspaceTab}
|
||||
onValueChange={(value) =>
|
||||
setWorkspaceTab(value as 'setup' | 'results')
|
||||
}
|
||||
className='min-h-0 flex-1 overflow-hidden px-4 pb-4'
|
||||
>
|
||||
<TabsList className='grid w-full shrink-0 grid-cols-2'>
|
||||
<TabsTrigger value='setup'>Setup</TabsTrigger>
|
||||
<TabsTrigger value='results'>
|
||||
Results
|
||||
{activeResults.length > 0 && (
|
||||
<Badge
|
||||
variant='secondary'
|
||||
className='ml-1 px-1.5 py-0 text-xs'
|
||||
>
|
||||
{activeResults.length}
|
||||
</Badge>
|
||||
)}
|
||||
</TabsTrigger>
|
||||
</TabsList>
|
||||
|
||||
<TabsContent
|
||||
value='setup'
|
||||
className='mt-0 min-h-0 overflow-hidden data-[state=active]:flex data-[state=active]:flex-col'
|
||||
>
|
||||
<div className='min-h-0 flex-1 overflow-y-auto pr-1'>
|
||||
<ProviderCertificationSetupPanel
|
||||
models={
|
||||
activeModelsQuery?.data as ProviderModels | undefined
|
||||
}
|
||||
isLoading={activeModelsQuery?.isLoading}
|
||||
error={activeModelsQuery?.error}
|
||||
setup={activeSetup}
|
||||
onChange={(next) =>
|
||||
updateProviderSetup(activeProvider.id, next)
|
||||
}
|
||||
disabled={runningProviderIds.includes(
|
||||
activeProvider.id
|
||||
)}
|
||||
idPrefix={`multi-certify-${activeProvider.id}`}
|
||||
/>
|
||||
</div>
|
||||
<div className='mt-3 flex shrink-0 justify-end border-t pt-3'>
|
||||
<Button
|
||||
variant='outline'
|
||||
size='sm'
|
||||
onClick={() => {
|
||||
setWorkspaceTab('results');
|
||||
void runOneProvider(activeProvider.id);
|
||||
}}
|
||||
disabled={
|
||||
runningProviderIds.includes(activeProvider.id) ||
|
||||
!isProviderReady(activeProvider.id)
|
||||
}
|
||||
>
|
||||
{runningProviderIds.includes(activeProvider.id) ? (
|
||||
<Loader2 className='h-4 w-4 animate-spin' />
|
||||
) : (
|
||||
<RotateCcw className='h-4 w-4' />
|
||||
)}
|
||||
Run this provider
|
||||
</Button>
|
||||
</div>
|
||||
</TabsContent>
|
||||
|
||||
<TabsContent
|
||||
value='results'
|
||||
className='mt-0 min-h-0 overflow-hidden data-[state=active]:flex data-[state=active]:flex-col'
|
||||
>
|
||||
<div className='mb-2 flex shrink-0 justify-end'>
|
||||
<Button
|
||||
variant='outline'
|
||||
size='sm'
|
||||
onClick={() => void runOneProvider(activeProvider.id)}
|
||||
disabled={
|
||||
runningProviderIds.includes(activeProvider.id) ||
|
||||
!isProviderReady(activeProvider.id)
|
||||
}
|
||||
>
|
||||
{runningProviderIds.includes(activeProvider.id) ? (
|
||||
<Loader2 className='h-4 w-4 animate-spin' />
|
||||
) : (
|
||||
<RotateCcw className='h-4 w-4' />
|
||||
)}
|
||||
Run this provider again
|
||||
</Button>
|
||||
</div>
|
||||
<ProviderCertificationResults
|
||||
results={activeResults}
|
||||
progress={progressByProvider[activeProvider.id]}
|
||||
namesById={activeNames}
|
||||
emptyMessage='Run this provider or the full batch to see results.'
|
||||
/>
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
</Card>
|
||||
) : null}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{selectedProviders.length > 0 && (
|
||||
<div className='bg-background sticky bottom-0 flex shrink-0 flex-col gap-3 rounded-lg border p-3 shadow-sm sm:flex-row sm:items-center sm:justify-between'>
|
||||
<div className='text-sm'>
|
||||
<div className='font-medium'>
|
||||
{selectedProviderIds.length} providers · {totalModels} models ·{' '}
|
||||
{totalRoutes} routes
|
||||
</div>
|
||||
<div className='text-muted-foreground text-xs'>
|
||||
{incompleteProviders.length > 0
|
||||
? `${incompleteProviders.length} provider${incompleteProviders.length === 1 ? '' : 's'} need a model or path selection. `
|
||||
: 'Providers run concurrently; models are sequential and paths run in parallel. '}
|
||||
Status: {aggregateSummary.ok} ok, {aggregateSummary.warn}{' '}
|
||||
warnings, {aggregateSummary.fail} failed,{' '}
|
||||
{aggregateSummary.error} errors, {runningProviderIds.length}{' '}
|
||||
running, {pendingRoutes} pending.
|
||||
</div>
|
||||
</div>
|
||||
<Button
|
||||
onClick={runAllProviders}
|
||||
disabled={!allReady || runningProviderIds.length > 0}
|
||||
>
|
||||
{runningProviderIds.length > 0 ? (
|
||||
<Loader2 className='h-4 w-4 animate-spin' />
|
||||
) : (
|
||||
<Play className='h-4 w-4' />
|
||||
)}
|
||||
{runningProviderIds.length > 0
|
||||
? `Running ${runningProviderIds.length} provider${runningProviderIds.length === 1 ? '' : 's'}`
|
||||
: 'Run all providers'}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</AppPageShell>
|
||||
);
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
'use client';
|
||||
|
||||
import Link from 'next/link';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Card, CardContent } from '@/components/ui/card';
|
||||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
@@ -18,7 +19,7 @@ import { BatchOverrideDialog } from '@/components/batch-override-dialog';
|
||||
import { ProviderCard } from '@/components/provider-card';
|
||||
import { ProviderFormDialogContent } from '@/components/provider-form-dialog-content';
|
||||
import { Skeleton } from '@/components/ui/skeleton';
|
||||
import { AlertCircle, Plus, Server } from 'lucide-react';
|
||||
import { AlertCircle, BadgeCheck, Plus, Server } from 'lucide-react';
|
||||
import { Alert, AlertDescription } from '@/components/ui/alert';
|
||||
import { Dialog, DialogTrigger } from '@/components/ui/dialog';
|
||||
import {
|
||||
@@ -348,12 +349,20 @@ export default function ProvidersPage() {
|
||||
title='Upstream Providers'
|
||||
description='Manage your AI provider connections and credentials.'
|
||||
actions={
|
||||
<>
|
||||
<Button asChild variant='outline'>
|
||||
<Link href='/providers/certification'>
|
||||
<BadgeCheck className='h-4 w-4' />
|
||||
Certify Providers
|
||||
</Link>
|
||||
</Button>
|
||||
<DialogTrigger asChild>
|
||||
<Button>
|
||||
<Plus className='h-4 w-4' />
|
||||
Add Provider
|
||||
</Button>
|
||||
</DialogTrigger>
|
||||
</>
|
||||
}
|
||||
/>
|
||||
<ProviderFormDialogContent
|
||||
|
||||
@@ -1,37 +1,16 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useMutation, useQuery } from '@tanstack/react-query';
|
||||
import {
|
||||
AlertTriangle,
|
||||
CheckCircle2,
|
||||
ChevronDown,
|
||||
ChevronRight,
|
||||
Loader2,
|
||||
RotateCcw,
|
||||
XCircle,
|
||||
} from 'lucide-react';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { Loader2, RotateCcw } from 'lucide-react';
|
||||
|
||||
import { AdminService } from '@/lib/api/services/admin';
|
||||
import type {
|
||||
AdminModel,
|
||||
CertificationPath,
|
||||
CertificationRow,
|
||||
CertificationStatus,
|
||||
ProviderCertification,
|
||||
UpstreamProvider,
|
||||
} from '@/lib/api/services/admin';
|
||||
import { ProviderCertificationResults } from '@/components/provider-certification-results';
|
||||
import {
|
||||
getCertificationModelNames,
|
||||
ProviderCertificationSetupPanel,
|
||||
} from '@/components/provider-certification-setup';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Checkbox } from '@/components/ui/checkbox';
|
||||
import {
|
||||
Command,
|
||||
CommandEmpty,
|
||||
CommandGroup,
|
||||
CommandInput,
|
||||
CommandItem,
|
||||
CommandList,
|
||||
} from '@/components/ui/command';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
@@ -39,15 +18,17 @@ import {
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { Label } from '@/components/ui/label';
|
||||
import {
|
||||
Popover,
|
||||
PopoverContent,
|
||||
PopoverTrigger,
|
||||
} from '@/components/ui/popover';
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
|
||||
import { ToggleGroup, ToggleGroupItem } from '@/components/ui/toggle-group';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { useProviderCertificationRunner } from '@/hooks/use-provider-certification-runner';
|
||||
import { AdminService } from '@/lib/api/services/admin';
|
||||
import type { UpstreamProvider } from '@/lib/api/services/admin';
|
||||
import {
|
||||
buildModelRuns,
|
||||
countCertificationTargets,
|
||||
emptyCertificationSetup,
|
||||
getModelsNeedingPath,
|
||||
} from '@/lib/provider-certification';
|
||||
import type { ProviderCertificationSetup } from '@/lib/provider-certification';
|
||||
|
||||
interface ProviderCertificationDialogProps {
|
||||
provider: UpstreamProvider;
|
||||
@@ -55,185 +36,19 @@ interface ProviderCertificationDialogProps {
|
||||
onOpenChange: (open: boolean) => void;
|
||||
}
|
||||
|
||||
interface ModelOption {
|
||||
model: AdminModel;
|
||||
source: 'configured' | 'discovered';
|
||||
}
|
||||
|
||||
type ModelPathMode = 'default' | 'selected' | 'all';
|
||||
|
||||
interface ModelPathTarget {
|
||||
path?: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
interface ModelRun {
|
||||
modelId: string;
|
||||
targets: ModelPathTarget[];
|
||||
}
|
||||
|
||||
interface ModelCertificationResult {
|
||||
resultKey: string;
|
||||
modelId: string;
|
||||
pathLabel: string;
|
||||
report?: ProviderCertification;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
const STATUS_STYLES: Record<
|
||||
CertificationStatus,
|
||||
{ label: string; icon: typeof CheckCircle2; className: string }
|
||||
> = {
|
||||
ok: {
|
||||
label: 'OK',
|
||||
icon: CheckCircle2,
|
||||
className:
|
||||
'border-emerald-500/40 bg-emerald-500/10 text-emerald-700 dark:text-emerald-400',
|
||||
},
|
||||
warn: {
|
||||
label: 'Warn',
|
||||
icon: AlertTriangle,
|
||||
className:
|
||||
'border-amber-500/40 bg-amber-500/10 text-amber-700 dark:text-amber-400',
|
||||
},
|
||||
fail: {
|
||||
label: 'Fail',
|
||||
icon: XCircle,
|
||||
className: 'border-red-500/40 bg-red-500/10 text-red-700 dark:text-red-400',
|
||||
},
|
||||
};
|
||||
|
||||
function StatusBadge({ status }: { status: CertificationStatus }) {
|
||||
const style = STATUS_STYLES[status];
|
||||
const Icon = style.icon;
|
||||
return (
|
||||
<Badge variant='outline' className={cn('gap-1', style.className)}>
|
||||
<Icon className='h-3 w-3' />
|
||||
{style.label}
|
||||
</Badge>
|
||||
);
|
||||
}
|
||||
|
||||
function RowItem({ row }: { row: CertificationRow }) {
|
||||
const [showEvidence, setShowEvidence] = useState(false);
|
||||
const hasEvidence = Object.keys(row.evidence).length > 0;
|
||||
return (
|
||||
<li className='rounded-md border p-3 text-sm'>
|
||||
<div className='flex items-start justify-between gap-3'>
|
||||
<div className='min-w-0'>
|
||||
<div className='font-medium'>{row.title}</div>
|
||||
<div className='text-muted-foreground mt-0.5 break-words'>
|
||||
{row.detail}
|
||||
</div>
|
||||
<div className='text-muted-foreground mt-1 font-mono text-xs'>
|
||||
{row.id}
|
||||
</div>
|
||||
</div>
|
||||
<StatusBadge status={row.status} />
|
||||
</div>
|
||||
{hasEvidence && (
|
||||
<div className='mt-2'>
|
||||
<button
|
||||
type='button'
|
||||
onClick={() => setShowEvidence((value) => !value)}
|
||||
className='text-muted-foreground hover:text-foreground inline-flex items-center gap-1 text-xs'
|
||||
>
|
||||
{showEvidence ? (
|
||||
<ChevronDown className='h-3 w-3' />
|
||||
) : (
|
||||
<ChevronRight className='h-3 w-3' />
|
||||
)}
|
||||
Evidence
|
||||
</button>
|
||||
{showEvidence && (
|
||||
<pre className='bg-muted mt-2 max-h-64 overflow-auto rounded p-2 font-mono text-xs'>
|
||||
{JSON.stringify(row.evidence, null, 2)}
|
||||
</pre>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</li>
|
||||
);
|
||||
}
|
||||
|
||||
function ChecklistSummary({ report }: { report: ProviderCertification }) {
|
||||
return (
|
||||
<ul className='grid gap-2 sm:grid-cols-2'>
|
||||
{report.checklist.map((goal) => {
|
||||
const style = STATUS_STYLES[goal.status];
|
||||
const Icon = style.icon;
|
||||
return (
|
||||
<li
|
||||
key={goal.goal}
|
||||
className={cn(
|
||||
'flex items-start gap-2 rounded-md border p-2 text-sm',
|
||||
style.className
|
||||
)}
|
||||
>
|
||||
<Icon className='mt-0.5 h-4 w-4 shrink-0' />
|
||||
<span>{goal.label}</span>
|
||||
</li>
|
||||
);
|
||||
})}
|
||||
</ul>
|
||||
);
|
||||
}
|
||||
|
||||
function CertificationReport({ report }: { report: ProviderCertification }) {
|
||||
const failing = report.rows.filter((row) => row.status === 'fail').length;
|
||||
const warning = report.rows.filter((row) => row.status === 'warn').length;
|
||||
|
||||
return (
|
||||
<div className='space-y-4'>
|
||||
<ChecklistSummary report={report} />
|
||||
<div className='text-muted-foreground flex flex-wrap items-center gap-x-3 gap-y-1 text-xs'>
|
||||
<span>
|
||||
{report.rows.length} checks · {failing} failed · {warning} warnings
|
||||
</span>
|
||||
<span>Generated {new Date(report.generated_at).toLocaleString()}</span>
|
||||
</div>
|
||||
<ul className='space-y-2'>
|
||||
{report.rows.map((row) => (
|
||||
<RowItem key={row.id} row={row} />
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function getErrorMessage(error: unknown): string {
|
||||
if (error instanceof Error) return error.message;
|
||||
return 'Certification request failed';
|
||||
}
|
||||
|
||||
function resultStatus(
|
||||
result: ModelCertificationResult
|
||||
): CertificationStatus | 'error' {
|
||||
if (result.error || !result.report) return 'error';
|
||||
if (result.report.rows.some((row) => row.status === 'fail')) return 'fail';
|
||||
if (result.report.rows.some((row) => row.status === 'warn')) return 'warn';
|
||||
return 'ok';
|
||||
}
|
||||
|
||||
export function ProviderCertificationDialog({
|
||||
provider,
|
||||
open,
|
||||
onOpenChange,
|
||||
}: ProviderCertificationDialogProps) {
|
||||
const [checkCache, setCheckCache] = useState(true);
|
||||
const [workspaceTab, setWorkspaceTab] = useState<'setup' | 'results'>('setup');
|
||||
const [selectedModelIds, setSelectedModelIds] = useState<string[]>([]);
|
||||
const [pathModes, setPathModes] = useState<Record<string, ModelPathMode>>({});
|
||||
const [selectedModelPaths, setSelectedModelPaths] = useState<
|
||||
Record<string, string[]>
|
||||
>({});
|
||||
const [results, setResults] = useState<ModelCertificationResult[]>([]);
|
||||
const [currentModel, setCurrentModel] = useState<{
|
||||
id: string;
|
||||
index: number;
|
||||
total: number;
|
||||
pathCount: number;
|
||||
} | null>(null);
|
||||
const [workspaceTab, setWorkspaceTab] = useState<'setup' | 'results'>(
|
||||
'setup'
|
||||
);
|
||||
const [setup, setSetup] = useState<ProviderCertificationSetup>(
|
||||
emptyCertificationSetup
|
||||
);
|
||||
const { results, progress, isPending, run, reset } =
|
||||
useProviderCertificationRunner(provider.id);
|
||||
|
||||
const models = useQuery({
|
||||
queryKey: ['provider-models', provider.id],
|
||||
@@ -241,183 +56,27 @@ export function ProviderCertificationDialog({
|
||||
enabled: open,
|
||||
});
|
||||
|
||||
const certify = useMutation({
|
||||
mutationFn: async ({
|
||||
modelRuns,
|
||||
includeCache,
|
||||
}: {
|
||||
modelRuns: ModelRun[];
|
||||
includeCache: boolean;
|
||||
}) => {
|
||||
const completed: ModelCertificationResult[] = [];
|
||||
setResults([]);
|
||||
setWorkspaceTab('results');
|
||||
|
||||
for (const [index, run] of modelRuns.entries()) {
|
||||
setCurrentModel({
|
||||
id: run.modelId,
|
||||
index: index + 1,
|
||||
total: modelRuns.length,
|
||||
pathCount: run.targets.length,
|
||||
});
|
||||
const batch = await Promise.all(
|
||||
run.targets.map(async (target): Promise<ModelCertificationResult> => {
|
||||
const resultKey = `${run.modelId}::${target.path ?? 'default'}`;
|
||||
try {
|
||||
const report = await AdminService.certifyProvider(provider.id, {
|
||||
model_id: run.modelId,
|
||||
model_path: target.path,
|
||||
check_cache: includeCache,
|
||||
});
|
||||
return {
|
||||
resultKey,
|
||||
modelId: run.modelId,
|
||||
pathLabel: target.label,
|
||||
report,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
resultKey,
|
||||
modelId: run.modelId,
|
||||
pathLabel: target.label,
|
||||
error: getErrorMessage(error),
|
||||
};
|
||||
}
|
||||
})
|
||||
);
|
||||
completed.push(...batch);
|
||||
setResults([...completed]);
|
||||
}
|
||||
|
||||
return completed;
|
||||
},
|
||||
onSettled: () => setCurrentModel(null),
|
||||
});
|
||||
const { reset: resetCertification } = certify;
|
||||
|
||||
useEffect(() => {
|
||||
if (!open) {
|
||||
resetCertification();
|
||||
reset();
|
||||
setWorkspaceTab('setup');
|
||||
setSelectedModelIds([]);
|
||||
setPathModes({});
|
||||
setSelectedModelPaths({});
|
||||
setResults([]);
|
||||
setCurrentModel(null);
|
||||
setSetup(emptyCertificationSetup());
|
||||
}
|
||||
}, [open, resetCertification]);
|
||||
}, [open, reset]);
|
||||
|
||||
const configuredOptions: ModelOption[] =
|
||||
models.data?.db_models.map((model) => ({
|
||||
model,
|
||||
source: 'configured',
|
||||
})) ?? [];
|
||||
const discoveredOptions: ModelOption[] =
|
||||
models.data?.remote_models.map((model) => ({
|
||||
model,
|
||||
source: 'discovered',
|
||||
})) ?? [];
|
||||
const allOptions = [...configuredOptions, ...discoveredOptions];
|
||||
const namesById = new Map(
|
||||
allOptions.map(({ model }) => [model.id, model.name || model.id])
|
||||
);
|
||||
const modelRuns = buildModelRuns(setup, models.data);
|
||||
const targetCount = countCertificationTargets(modelRuns);
|
||||
const modelsNeedingPath = getModelsNeedingPath(setup);
|
||||
const namesById = getCertificationModelNames(models.data);
|
||||
|
||||
const pathsForModel = (modelId: string): CertificationPath[] =>
|
||||
(models.data?.certification_paths[modelId] ?? []).filter(
|
||||
(path) => path.endpoint_tag
|
||||
);
|
||||
|
||||
const pathLabel = (path: CertificationPath): string =>
|
||||
path.endpoint_name && path.endpoint_name !== path.endpoint_tag
|
||||
? `${path.endpoint_name} (${path.endpoint_tag})`
|
||||
: path.endpoint_tag || 'Provider default';
|
||||
|
||||
const toggleModel = (modelId: string) => {
|
||||
const isSelected = selectedModelIds.includes(modelId);
|
||||
setSelectedModelIds((current) =>
|
||||
isSelected
|
||||
? current.filter((id) => id !== modelId)
|
||||
: [...current, modelId]
|
||||
);
|
||||
setPathModes((current) => {
|
||||
const next = { ...current };
|
||||
if (isSelected) delete next[modelId];
|
||||
else next[modelId] = 'default';
|
||||
return next;
|
||||
});
|
||||
setSelectedModelPaths((current) => {
|
||||
const next = { ...current };
|
||||
if (isSelected) delete next[modelId];
|
||||
return next;
|
||||
});
|
||||
resetCertification();
|
||||
setResults([]);
|
||||
const updateSetup = (nextSetup: ProviderCertificationSetup) => {
|
||||
setSetup(nextSetup);
|
||||
reset();
|
||||
};
|
||||
|
||||
const modelsNeedingPath = selectedModelIds.filter(
|
||||
(modelId) =>
|
||||
pathModes[modelId] === 'selected' &&
|
||||
(selectedModelPaths[modelId]?.length ?? 0) === 0
|
||||
);
|
||||
|
||||
const buildModelRuns = (): ModelRun[] =>
|
||||
selectedModelIds.map((modelId) => {
|
||||
const paths = pathsForModel(modelId);
|
||||
const mode = pathModes[modelId] ?? 'default';
|
||||
if (mode === 'all') {
|
||||
return {
|
||||
modelId,
|
||||
targets: paths.map((path) => ({
|
||||
path: path.path,
|
||||
label: pathLabel(path),
|
||||
})),
|
||||
};
|
||||
}
|
||||
if (mode === 'selected') {
|
||||
const selected = new Set(selectedModelPaths[modelId] ?? []);
|
||||
return {
|
||||
modelId,
|
||||
targets: paths
|
||||
.filter((path) => selected.has(path.path))
|
||||
.map((path) => ({ path: path.path, label: pathLabel(path) })),
|
||||
};
|
||||
}
|
||||
return { modelId, targets: [{ label: 'Provider default' }] };
|
||||
});
|
||||
|
||||
const modelRuns = buildModelRuns();
|
||||
const targetCount = modelRuns.reduce(
|
||||
(total, run) => total + run.targets.length,
|
||||
0
|
||||
);
|
||||
|
||||
const renderModelGroup = (label: string, options: ModelOption[]) => {
|
||||
if (options.length === 0) return null;
|
||||
return (
|
||||
<CommandGroup heading={`${label} (${options.length})`}>
|
||||
{options.map(({ model }) => (
|
||||
<CommandItem
|
||||
key={`${label}-${model.id}`}
|
||||
value={`${model.name} ${model.id}`}
|
||||
onSelect={() => toggleModel(model.id)}
|
||||
disabled={certify.isPending}
|
||||
>
|
||||
<Checkbox
|
||||
checked={selectedModelIds.includes(model.id)}
|
||||
tabIndex={-1}
|
||||
aria-hidden='true'
|
||||
className='pointer-events-none'
|
||||
/>
|
||||
<span className='min-w-0 flex-1 truncate'>
|
||||
{model.name || model.id}
|
||||
</span>
|
||||
<span className='text-muted-foreground max-w-56 truncate font-mono text-xs'>
|
||||
{model.id}
|
||||
</span>
|
||||
</CommandItem>
|
||||
))}
|
||||
</CommandGroup>
|
||||
);
|
||||
const runCertification = () => {
|
||||
setWorkspaceTab('results');
|
||||
void run(modelRuns, setup.checkCache);
|
||||
};
|
||||
|
||||
return (
|
||||
@@ -426,9 +85,9 @@ export function ProviderCertificationDialog({
|
||||
<DialogHeader className='shrink-0'>
|
||||
<DialogTitle>Certify upstream models</DialogTitle>
|
||||
<DialogDescription>
|
||||
Select models, then use the provider default, choose specific
|
||||
paths, or test every path. Models run one at a time; paths for the
|
||||
same model run in parallel against {provider.base_url}.
|
||||
Select models, then use the provider default, choose specific paths,
|
||||
or test every path. Models run one at a time; paths for the same
|
||||
model run in parallel against {provider.base_url}.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
@@ -443,10 +102,10 @@ export function ProviderCertificationDialog({
|
||||
<TabsTrigger value='setup'>Setup</TabsTrigger>
|
||||
<TabsTrigger
|
||||
value='results'
|
||||
disabled={!certify.isPending && results.length === 0}
|
||||
disabled={!isPending && results.length === 0}
|
||||
>
|
||||
Results
|
||||
{(certify.isPending || results.length > 0) && (
|
||||
{(isPending || results.length > 0) && (
|
||||
<Badge variant='secondary' className='ml-1 px-1.5 py-0 text-xs'>
|
||||
{results.length}/{targetCount}
|
||||
</Badge>
|
||||
@@ -458,214 +117,37 @@ export function ProviderCertificationDialog({
|
||||
value='setup'
|
||||
className='mt-0 min-h-0 overflow-hidden data-[state=active]:flex data-[state=active]:flex-col'
|
||||
>
|
||||
<div className='min-h-0 flex-1 space-y-4 overflow-y-auto pr-1'>
|
||||
<div className='space-y-2'>
|
||||
<div className='flex items-center justify-between gap-3'>
|
||||
<Label>Models</Label>
|
||||
<div className='flex items-center gap-2'>
|
||||
<span className='text-muted-foreground text-xs'>
|
||||
{selectedModelIds.length} selected
|
||||
</span>
|
||||
{selectedModelIds.length > 0 && !certify.isPending && (
|
||||
<Button
|
||||
type='button'
|
||||
variant='ghost'
|
||||
size='sm'
|
||||
onClick={() => {
|
||||
setSelectedModelIds([]);
|
||||
setPathModes({});
|
||||
setSelectedModelPaths({});
|
||||
setResults([]);
|
||||
resetCertification();
|
||||
}}
|
||||
>
|
||||
Clear
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<Command className='h-auto rounded-md border'>
|
||||
<CommandInput
|
||||
placeholder='Search models by name or ID…'
|
||||
disabled={models.isLoading || certify.isPending}
|
||||
<div className='min-h-0 flex-1 overflow-y-auto pr-1'>
|
||||
<ProviderCertificationSetupPanel
|
||||
models={models.data}
|
||||
isLoading={models.isLoading}
|
||||
error={models.error}
|
||||
setup={setup}
|
||||
onChange={updateSetup}
|
||||
disabled={isPending}
|
||||
idPrefix={`certify-${provider.id}`}
|
||||
/>
|
||||
<CommandList className='max-h-64'>
|
||||
<CommandEmpty>
|
||||
{models.isLoading ? 'Loading models…' : 'No models found'}
|
||||
</CommandEmpty>
|
||||
{renderModelGroup('Configured models', configuredOptions)}
|
||||
{renderModelGroup('Discovered models', discoveredOptions)}
|
||||
</CommandList>
|
||||
</Command>
|
||||
{models.isError && (
|
||||
<p className='text-destructive text-sm'>
|
||||
{getErrorMessage(models.error)}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{selectedModelIds.map((modelId) => {
|
||||
const paths = pathsForModel(modelId);
|
||||
const mode = pathModes[modelId] ?? 'default';
|
||||
const selectedPaths = selectedModelPaths[modelId] ?? [];
|
||||
return (
|
||||
<div key={modelId} className='space-y-3 rounded-md border p-3'>
|
||||
<div className='min-w-0'>
|
||||
<div className='truncate text-sm font-medium'>
|
||||
{namesById.get(modelId) ?? modelId}
|
||||
</div>
|
||||
<div className='text-muted-foreground truncate font-mono text-xs'>
|
||||
{modelId}
|
||||
</div>
|
||||
</div>
|
||||
{paths.length > 0 ? (
|
||||
<>
|
||||
<ToggleGroup
|
||||
type='single'
|
||||
variant='outline'
|
||||
size='sm'
|
||||
value={mode}
|
||||
onValueChange={(value) => {
|
||||
if (!value) return;
|
||||
setPathModes((current) => ({
|
||||
...current,
|
||||
[modelId]: value as ModelPathMode,
|
||||
}));
|
||||
}}
|
||||
disabled={certify.isPending}
|
||||
className='w-full justify-start'
|
||||
>
|
||||
<ToggleGroupItem value='default'>Default</ToggleGroupItem>
|
||||
<ToggleGroupItem value='selected'>
|
||||
Choose paths
|
||||
</ToggleGroupItem>
|
||||
<ToggleGroupItem value='all'>All paths</ToggleGroupItem>
|
||||
</ToggleGroup>
|
||||
{mode === 'default' && (
|
||||
<p className='text-muted-foreground text-xs'>
|
||||
Uses the upstream provider's normal model routing.
|
||||
</p>
|
||||
)}
|
||||
{mode === 'selected' && (
|
||||
<Popover>
|
||||
<PopoverTrigger asChild>
|
||||
<Button
|
||||
type='button'
|
||||
variant='outline'
|
||||
size='sm'
|
||||
className='w-full justify-between'
|
||||
disabled={certify.isPending}
|
||||
>
|
||||
<span className='truncate'>
|
||||
{selectedPaths.length === 0
|
||||
? 'Select paths'
|
||||
: `${selectedPaths.length} path${selectedPaths.length === 1 ? '' : 's'} selected`}
|
||||
</span>
|
||||
<ChevronDown className='h-4 w-4' />
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent
|
||||
align='start'
|
||||
className='w-80 max-w-[calc(100vw-2rem)] p-2'
|
||||
>
|
||||
<div className='max-h-64 space-y-1 overflow-y-auto overscroll-contain'>
|
||||
{paths.map((path) => {
|
||||
const checked = selectedPaths.includes(
|
||||
path.path
|
||||
);
|
||||
return (
|
||||
<label
|
||||
key={path.path}
|
||||
className='hover:bg-muted flex cursor-pointer items-center gap-2 rounded-sm px-2 py-1.5 text-sm'
|
||||
>
|
||||
<Checkbox
|
||||
checked={checked}
|
||||
onCheckedChange={(value) =>
|
||||
setSelectedModelPaths((current) => {
|
||||
const previous =
|
||||
current[modelId] ?? [];
|
||||
return {
|
||||
...current,
|
||||
[modelId]:
|
||||
value === true
|
||||
? [...previous, path.path]
|
||||
: previous.filter(
|
||||
(item) =>
|
||||
item !== path.path
|
||||
),
|
||||
};
|
||||
})
|
||||
}
|
||||
/>
|
||||
<span className='min-w-0 truncate'>
|
||||
{pathLabel(path)}
|
||||
</span>
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
)}
|
||||
{mode === 'all' && (
|
||||
<p className='text-muted-foreground text-xs'>
|
||||
All {paths.length} paths will run in parallel.
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
<p className='text-muted-foreground text-xs'>
|
||||
Only the provider default route is available.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
{modelsNeedingPath.length > 0 && (
|
||||
<p className='text-muted-foreground text-xs'>
|
||||
Choose at least one path for each model using “Choose paths”.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className='mt-3 flex shrink-0 flex-wrap items-center justify-between gap-3 border-t pt-3'>
|
||||
<div className='flex items-center gap-2'>
|
||||
<Checkbox
|
||||
id={`certify-cache-${provider.id}`}
|
||||
checked={checkCache}
|
||||
onCheckedChange={(value) => setCheckCache(value === true)}
|
||||
disabled={certify.isPending}
|
||||
/>
|
||||
<Label
|
||||
htmlFor={`certify-cache-${provider.id}`}
|
||||
className='text-sm'
|
||||
>
|
||||
Probe prompt caching and margin
|
||||
</Label>
|
||||
</div>
|
||||
<div className='mt-3 flex shrink-0 justify-end border-t pt-3'>
|
||||
<Button
|
||||
variant='outline'
|
||||
size='sm'
|
||||
onClick={() =>
|
||||
certify.mutate({
|
||||
modelRuns,
|
||||
includeCache: checkCache,
|
||||
})
|
||||
}
|
||||
onClick={runCertification}
|
||||
disabled={
|
||||
certify.isPending ||
|
||||
selectedModelIds.length === 0 ||
|
||||
isPending ||
|
||||
setup.selectedModelIds.length === 0 ||
|
||||
modelsNeedingPath.length > 0
|
||||
}
|
||||
className='gap-1.5'
|
||||
>
|
||||
{certify.isPending ? (
|
||||
{isPending ? (
|
||||
<Loader2 className='h-4 w-4 animate-spin' />
|
||||
) : (
|
||||
<RotateCcw className='h-4 w-4' />
|
||||
)}
|
||||
{certify.isPending
|
||||
? `Running ${currentModel?.index ?? 1} of ${currentModel?.total ?? selectedModelIds.length}`
|
||||
{isPending
|
||||
? `Running ${progress?.modelIndex ?? 1} of ${progress?.modelTotal ?? setup.selectedModelIds.length}`
|
||||
: results.length > 0
|
||||
? `Run ${targetCount} route${targetCount === 1 ? '' : 's'} again`
|
||||
: `Certify ${targetCount || ''} route${targetCount === 1 ? '' : 's'}`}
|
||||
@@ -677,119 +159,11 @@ export function ProviderCertificationDialog({
|
||||
value='results'
|
||||
className='mt-0 min-h-0 overflow-hidden data-[state=active]:flex data-[state=active]:flex-col'
|
||||
>
|
||||
{currentModel && (
|
||||
<div className='text-muted-foreground flex shrink-0 items-center gap-2 rounded-md border p-3 text-sm'>
|
||||
<Loader2 className='h-4 w-4 animate-spin' />
|
||||
<span className='min-w-0 truncate'>
|
||||
Probing {namesById.get(currentModel.id) ?? currentModel.id}
|
||||
{currentModel.pathCount > 1
|
||||
? ` across ${currentModel.pathCount} paths in parallel`
|
||||
: ''}{' '}
|
||||
— model {currentModel.index} of {currentModel.total}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{results.length === 0 ? (
|
||||
<div className='text-muted-foreground flex min-h-0 flex-1 items-center justify-center text-center text-sm'>
|
||||
Results will appear here as certification completes.
|
||||
</div>
|
||||
) : (
|
||||
<Tabs
|
||||
key={results.map((result) => result.resultKey).join('|')}
|
||||
defaultValue={results[0].resultKey}
|
||||
className='min-h-0 flex-1 overflow-hidden'
|
||||
>
|
||||
<div className='max-h-28 shrink-0 overflow-y-auto rounded-md border p-2'>
|
||||
<TabsList className='flex h-auto w-full flex-wrap justify-start gap-1 bg-transparent p-0'>
|
||||
{results.map((result, index) => {
|
||||
const status = resultStatus(result);
|
||||
const Icon =
|
||||
status === 'error'
|
||||
? XCircle
|
||||
: STATUS_STYLES[status].icon;
|
||||
const modelRouteNumber = results
|
||||
.slice(0, index + 1)
|
||||
.filter((item) => item.modelId === result.modelId).length;
|
||||
const modelRouteCount = results.filter(
|
||||
(item) => item.modelId === result.modelId
|
||||
).length;
|
||||
return (
|
||||
<TabsTrigger
|
||||
key={result.resultKey}
|
||||
value={result.resultKey}
|
||||
title={namesById.get(result.modelId) ?? result.modelId}
|
||||
className='h-7 min-w-0 max-w-44 gap-1 px-2 text-xs'
|
||||
>
|
||||
<Icon
|
||||
className={cn(
|
||||
status === 'ok' && 'text-emerald-600',
|
||||
status === 'warn' && 'text-amber-600',
|
||||
(status === 'fail' || status === 'error') &&
|
||||
'text-red-600'
|
||||
)}
|
||||
<ProviderCertificationResults
|
||||
results={results}
|
||||
progress={progress}
|
||||
namesById={namesById}
|
||||
/>
|
||||
<span className='truncate'>
|
||||
{namesById.get(result.modelId) ?? result.modelId}
|
||||
</span>
|
||||
{modelRouteCount > 1 && (
|
||||
<span className='text-muted-foreground'>
|
||||
{modelRouteNumber}
|
||||
</span>
|
||||
)}
|
||||
</TabsTrigger>
|
||||
);
|
||||
})}
|
||||
</TabsList>
|
||||
</div>
|
||||
|
||||
<div className='min-h-0 flex-1 overflow-y-auto pr-1'>
|
||||
{results.map((result) => {
|
||||
const status = resultStatus(result);
|
||||
return (
|
||||
<TabsContent
|
||||
key={result.resultKey}
|
||||
value={result.resultKey}
|
||||
className='space-y-3'
|
||||
>
|
||||
<div className='bg-muted/30 space-y-2 rounded-md border p-3'>
|
||||
<div className='flex flex-wrap items-center justify-between gap-2'>
|
||||
<div className='font-medium'>
|
||||
{namesById.get(result.modelId) ?? result.modelId}
|
||||
</div>
|
||||
{status === 'error' ? (
|
||||
<Badge
|
||||
variant='outline'
|
||||
className='border-red-500/40 bg-red-500/10 text-red-700 dark:text-red-400'
|
||||
>
|
||||
Error
|
||||
</Badge>
|
||||
) : (
|
||||
<StatusBadge status={status} />
|
||||
)}
|
||||
</div>
|
||||
<div className='grid gap-1 text-xs'>
|
||||
<span className='text-muted-foreground'>
|
||||
Model path
|
||||
</span>
|
||||
<div className='rounded bg-background px-2 py-1.5 font-medium'>
|
||||
{result.pathLabel}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{result.report ? (
|
||||
<CertificationReport report={result.report} />
|
||||
) : (
|
||||
<div className='rounded-md border border-red-500/40 bg-red-500/10 p-3 text-sm text-red-700 dark:text-red-400'>
|
||||
{result.error ?? 'Certification failed'}
|
||||
</div>
|
||||
)}
|
||||
</TabsContent>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</Tabs>
|
||||
)}
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
</DialogContent>
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
'use client';
|
||||
|
||||
import { useEffect, useState } from 'react';
|
||||
import {
|
||||
AlertTriangle,
|
||||
CheckCircle2,
|
||||
ChevronDown,
|
||||
ChevronRight,
|
||||
Loader2,
|
||||
XCircle,
|
||||
} from 'lucide-react';
|
||||
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
|
||||
import type {
|
||||
CertificationRow,
|
||||
CertificationStatus,
|
||||
ProviderCertification,
|
||||
} from '@/lib/api/services/admin';
|
||||
import type {
|
||||
CertificationProgress,
|
||||
ModelCertificationResult,
|
||||
} from '@/lib/provider-certification';
|
||||
import { getCertificationResultStatus } from '@/lib/provider-certification';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
const STATUS_STYLES: Record<
|
||||
CertificationStatus,
|
||||
{ label: string; icon: typeof CheckCircle2; className: string }
|
||||
> = {
|
||||
ok: {
|
||||
label: 'OK',
|
||||
icon: CheckCircle2,
|
||||
className:
|
||||
'border-emerald-500/40 bg-emerald-500/10 text-emerald-700 dark:text-emerald-400',
|
||||
},
|
||||
warn: {
|
||||
label: 'Warn',
|
||||
icon: AlertTriangle,
|
||||
className:
|
||||
'border-amber-500/40 bg-amber-500/10 text-amber-700 dark:text-amber-400',
|
||||
},
|
||||
fail: {
|
||||
label: 'Fail',
|
||||
icon: XCircle,
|
||||
className: 'border-red-500/40 bg-red-500/10 text-red-700 dark:text-red-400',
|
||||
},
|
||||
};
|
||||
|
||||
export interface CertificationResultSummary {
|
||||
ok: number;
|
||||
warn: number;
|
||||
fail: number;
|
||||
error: number;
|
||||
}
|
||||
|
||||
export function summarizeCertificationResults(
|
||||
results: ModelCertificationResult[]
|
||||
): CertificationResultSummary {
|
||||
const summary = { ok: 0, warn: 0, fail: 0, error: 0 };
|
||||
for (const result of results) {
|
||||
summary[getCertificationResultStatus(result)] += 1;
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
function StatusBadge({ status }: { status: CertificationStatus }) {
|
||||
const style = STATUS_STYLES[status];
|
||||
const Icon = style.icon;
|
||||
return (
|
||||
<Badge variant='outline' className={cn('gap-1', style.className)}>
|
||||
<Icon className='h-3 w-3' />
|
||||
{style.label}
|
||||
</Badge>
|
||||
);
|
||||
}
|
||||
|
||||
function ResultStatusBadge({
|
||||
status,
|
||||
}: {
|
||||
status: CertificationStatus | 'error';
|
||||
}) {
|
||||
if (status === 'error') {
|
||||
return (
|
||||
<Badge
|
||||
variant='outline'
|
||||
className='border-red-500/40 bg-red-500/10 text-red-700 dark:text-red-400'
|
||||
>
|
||||
Error
|
||||
</Badge>
|
||||
);
|
||||
}
|
||||
return <StatusBadge status={status} />;
|
||||
}
|
||||
|
||||
function RowItem({ row }: { row: CertificationRow }) {
|
||||
const [showEvidence, setShowEvidence] = useState(false);
|
||||
const hasEvidence = Object.keys(row.evidence).length > 0;
|
||||
return (
|
||||
<li className='rounded-md border p-3 text-sm'>
|
||||
<div className='flex items-start justify-between gap-3'>
|
||||
<div className='min-w-0'>
|
||||
<div className='font-medium'>{row.title}</div>
|
||||
<div className='text-muted-foreground mt-0.5 break-words'>
|
||||
{row.detail}
|
||||
</div>
|
||||
<div className='text-muted-foreground mt-1 font-mono text-xs'>
|
||||
{row.id}
|
||||
</div>
|
||||
</div>
|
||||
<StatusBadge status={row.status} />
|
||||
</div>
|
||||
{hasEvidence && (
|
||||
<div className='mt-2'>
|
||||
<button
|
||||
type='button'
|
||||
onClick={() => setShowEvidence((value) => !value)}
|
||||
className='text-muted-foreground hover:text-foreground inline-flex items-center gap-1 text-xs'
|
||||
>
|
||||
{showEvidence ? (
|
||||
<ChevronDown className='h-3 w-3' />
|
||||
) : (
|
||||
<ChevronRight className='h-3 w-3' />
|
||||
)}
|
||||
Evidence
|
||||
</button>
|
||||
{showEvidence && (
|
||||
<pre className='bg-muted mt-2 max-h-64 overflow-auto rounded p-2 font-mono text-xs'>
|
||||
{JSON.stringify(row.evidence, null, 2)}
|
||||
</pre>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</li>
|
||||
);
|
||||
}
|
||||
|
||||
function CertificationReport({ report }: { report: ProviderCertification }) {
|
||||
const failing = report.rows.filter((row) => row.status === 'fail').length;
|
||||
const warning = report.rows.filter((row) => row.status === 'warn').length;
|
||||
|
||||
return (
|
||||
<div className='space-y-4'>
|
||||
<ul className='grid gap-2 sm:grid-cols-2'>
|
||||
{report.checklist.map((goal) => {
|
||||
const style = STATUS_STYLES[goal.status];
|
||||
const Icon = style.icon;
|
||||
return (
|
||||
<li
|
||||
key={goal.goal}
|
||||
className={cn(
|
||||
'flex items-start gap-2 rounded-md border p-2 text-sm',
|
||||
style.className
|
||||
)}
|
||||
>
|
||||
<Icon className='mt-0.5 h-4 w-4 shrink-0' />
|
||||
<span>{goal.label}</span>
|
||||
</li>
|
||||
);
|
||||
})}
|
||||
</ul>
|
||||
<div className='text-muted-foreground flex flex-wrap items-center gap-x-3 gap-y-1 text-xs'>
|
||||
<span>
|
||||
{report.rows.length} checks · {failing} failed · {warning} warnings
|
||||
</span>
|
||||
<span>Generated {new Date(report.generated_at).toLocaleString()}</span>
|
||||
</div>
|
||||
<ul className='space-y-2'>
|
||||
{report.rows.map((row) => (
|
||||
<RowItem key={row.id} row={row} />
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
interface ProviderCertificationResultsProps {
|
||||
results: ModelCertificationResult[];
|
||||
progress?: CertificationProgress | null;
|
||||
namesById: Map<string, string>;
|
||||
emptyMessage?: string;
|
||||
}
|
||||
|
||||
export function ProviderCertificationResults({
|
||||
results,
|
||||
progress,
|
||||
namesById,
|
||||
emptyMessage = 'Results will appear here as certification completes.',
|
||||
}: ProviderCertificationResultsProps) {
|
||||
const [activeResult, setActiveResult] = useState<string>('');
|
||||
|
||||
useEffect(() => {
|
||||
if (
|
||||
results.length > 0 &&
|
||||
!results.some((result) => result.resultKey === activeResult)
|
||||
) {
|
||||
setActiveResult(results[0].resultKey);
|
||||
}
|
||||
}, [activeResult, results]);
|
||||
|
||||
return (
|
||||
<div className='flex min-h-0 flex-1 flex-col gap-2 overflow-hidden'>
|
||||
{progress && (
|
||||
<div className='text-muted-foreground flex shrink-0 items-center gap-2 rounded-md border p-3 text-sm'>
|
||||
<Loader2 className='h-4 w-4 animate-spin' />
|
||||
<span className='min-w-0 truncate'>
|
||||
Probing {namesById.get(progress.modelId) ?? progress.modelId}
|
||||
{progress.pathCount > 1
|
||||
? ` across ${progress.pathCount} paths in parallel`
|
||||
: ''}{' '}
|
||||
— model {progress.modelIndex} of {progress.modelTotal}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{results.length === 0 ? (
|
||||
<div className='text-muted-foreground flex min-h-40 flex-1 items-center justify-center text-center text-sm'>
|
||||
{emptyMessage}
|
||||
</div>
|
||||
) : (
|
||||
<Tabs
|
||||
value={activeResult}
|
||||
onValueChange={setActiveResult}
|
||||
className='min-h-0 flex-1 overflow-hidden'
|
||||
>
|
||||
<div className='max-h-28 shrink-0 overflow-y-auto rounded-md border p-2'>
|
||||
<TabsList className='flex h-auto w-full flex-wrap justify-start gap-1 bg-transparent p-0'>
|
||||
{results.map((result, index) => {
|
||||
const status = getCertificationResultStatus(result);
|
||||
const Icon =
|
||||
status === 'error' ? XCircle : STATUS_STYLES[status].icon;
|
||||
const modelRouteNumber = results
|
||||
.slice(0, index + 1)
|
||||
.filter((item) => item.modelId === result.modelId).length;
|
||||
const modelRouteCount = results.filter(
|
||||
(item) => item.modelId === result.modelId
|
||||
).length;
|
||||
return (
|
||||
<TabsTrigger
|
||||
key={result.resultKey}
|
||||
value={result.resultKey}
|
||||
title={namesById.get(result.modelId) ?? result.modelId}
|
||||
className='h-7 max-w-44 min-w-0 gap-1 px-2 text-xs'
|
||||
>
|
||||
<Icon
|
||||
className={cn(
|
||||
status === 'ok' && 'text-emerald-600',
|
||||
status === 'warn' && 'text-amber-600',
|
||||
(status === 'fail' || status === 'error') &&
|
||||
'text-red-600'
|
||||
)}
|
||||
/>
|
||||
<span className='truncate'>
|
||||
{namesById.get(result.modelId) ?? result.modelId}
|
||||
</span>
|
||||
{modelRouteCount > 1 && (
|
||||
<span className='text-muted-foreground'>
|
||||
{modelRouteNumber}
|
||||
</span>
|
||||
)}
|
||||
</TabsTrigger>
|
||||
);
|
||||
})}
|
||||
</TabsList>
|
||||
</div>
|
||||
|
||||
<div className='min-h-0 flex-1 overflow-y-auto pr-1'>
|
||||
{results.map((result) => {
|
||||
const status = getCertificationResultStatus(result);
|
||||
return (
|
||||
<TabsContent
|
||||
key={result.resultKey}
|
||||
value={result.resultKey}
|
||||
className='space-y-3'
|
||||
>
|
||||
<div className='bg-muted/30 space-y-2 rounded-md border p-3'>
|
||||
<div className='flex flex-wrap items-center justify-between gap-2'>
|
||||
<div className='font-medium'>
|
||||
{namesById.get(result.modelId) ?? result.modelId}
|
||||
</div>
|
||||
<ResultStatusBadge status={status} />
|
||||
</div>
|
||||
<div className='grid gap-1 text-xs'>
|
||||
<span className='text-muted-foreground'>Model path</span>
|
||||
<div className='bg-background rounded px-2 py-1.5 font-medium break-words'>
|
||||
{result.pathLabel}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{result.report ? (
|
||||
<CertificationReport report={result.report} />
|
||||
) : (
|
||||
<div className='rounded-md border border-red-500/40 bg-red-500/10 p-3 text-sm text-red-700 dark:text-red-400'>
|
||||
{result.error ?? 'Certification failed'}
|
||||
</div>
|
||||
)}
|
||||
</TabsContent>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</Tabs>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
'use client';
|
||||
|
||||
import { ChevronDown } from 'lucide-react';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Checkbox } from '@/components/ui/checkbox';
|
||||
import {
|
||||
Command,
|
||||
CommandEmpty,
|
||||
CommandGroup,
|
||||
CommandInput,
|
||||
CommandItem,
|
||||
CommandList,
|
||||
} from '@/components/ui/command';
|
||||
import { Label } from '@/components/ui/label';
|
||||
import {
|
||||
Popover,
|
||||
PopoverContent,
|
||||
PopoverTrigger,
|
||||
} from '@/components/ui/popover';
|
||||
import { ToggleGroup, ToggleGroupItem } from '@/components/ui/toggle-group';
|
||||
import type { AdminModel, ProviderModels } from '@/lib/api/services/admin';
|
||||
import type {
|
||||
ModelPathMode,
|
||||
ProviderCertificationSetup,
|
||||
} from '@/lib/provider-certification';
|
||||
import {
|
||||
emptyCertificationSetup,
|
||||
getCertificationPathLabel,
|
||||
getErrorMessage,
|
||||
getExactCertificationPaths,
|
||||
getModelsNeedingPath,
|
||||
} from '@/lib/provider-certification';
|
||||
|
||||
interface ModelOption {
|
||||
model: AdminModel;
|
||||
source: 'configured' | 'discovered';
|
||||
}
|
||||
|
||||
interface ProviderCertificationSetupProps {
|
||||
models?: ProviderModels;
|
||||
isLoading?: boolean;
|
||||
error?: unknown;
|
||||
setup: ProviderCertificationSetup;
|
||||
onChange: (setup: ProviderCertificationSetup) => void;
|
||||
disabled?: boolean;
|
||||
idPrefix: string;
|
||||
}
|
||||
|
||||
export function getCertificationModelNames(
|
||||
models: ProviderModels | undefined
|
||||
): Map<string, string> {
|
||||
return new Map(
|
||||
[...(models?.db_models ?? []), ...(models?.remote_models ?? [])].map(
|
||||
(model) => [model.id, model.name || model.id]
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
export function ProviderCertificationSetupPanel({
|
||||
models,
|
||||
isLoading = false,
|
||||
error,
|
||||
setup,
|
||||
onChange,
|
||||
disabled = false,
|
||||
idPrefix,
|
||||
}: ProviderCertificationSetupProps) {
|
||||
const configuredOptions: ModelOption[] = (models?.db_models ?? []).map(
|
||||
(model) => ({ model, source: 'configured' })
|
||||
);
|
||||
const discoveredOptions: ModelOption[] = (models?.remote_models ?? []).map(
|
||||
(model) => ({ model, source: 'discovered' })
|
||||
);
|
||||
const namesById = getCertificationModelNames(models);
|
||||
const modelsNeedingPath = getModelsNeedingPath(setup);
|
||||
|
||||
const toggleModel = (modelId: string) => {
|
||||
const isSelected = setup.selectedModelIds.includes(modelId);
|
||||
const pathModes = { ...setup.pathModes };
|
||||
const selectedModelPaths = { ...setup.selectedModelPaths };
|
||||
if (isSelected) {
|
||||
delete pathModes[modelId];
|
||||
delete selectedModelPaths[modelId];
|
||||
} else {
|
||||
pathModes[modelId] = 'default';
|
||||
}
|
||||
onChange({
|
||||
...setup,
|
||||
selectedModelIds: isSelected
|
||||
? setup.selectedModelIds.filter((id) => id !== modelId)
|
||||
: [...setup.selectedModelIds, modelId],
|
||||
pathModes,
|
||||
selectedModelPaths,
|
||||
});
|
||||
};
|
||||
|
||||
const renderModelGroup = (label: string, options: ModelOption[]) => {
|
||||
if (options.length === 0) return null;
|
||||
return (
|
||||
<CommandGroup heading={`${label} (${options.length})`}>
|
||||
{options.map(({ model }) => (
|
||||
<CommandItem
|
||||
key={`${label}-${model.id}`}
|
||||
value={`${model.name} ${model.id}`}
|
||||
onSelect={() => toggleModel(model.id)}
|
||||
disabled={disabled}
|
||||
>
|
||||
<Checkbox
|
||||
checked={setup.selectedModelIds.includes(model.id)}
|
||||
tabIndex={-1}
|
||||
aria-hidden='true'
|
||||
className='pointer-events-none'
|
||||
/>
|
||||
<span className='min-w-0 flex-1 truncate'>
|
||||
{model.name || model.id}
|
||||
</span>
|
||||
<span className='text-muted-foreground max-w-56 truncate font-mono text-xs'>
|
||||
{model.id}
|
||||
</span>
|
||||
</CommandItem>
|
||||
))}
|
||||
</CommandGroup>
|
||||
);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className='space-y-4'>
|
||||
<div className='space-y-2'>
|
||||
<div className='flex items-center justify-between gap-3'>
|
||||
<Label>Models</Label>
|
||||
<div className='flex items-center gap-2'>
|
||||
<span className='text-muted-foreground text-xs'>
|
||||
{setup.selectedModelIds.length} selected
|
||||
</span>
|
||||
{setup.selectedModelIds.length > 0 && !disabled && (
|
||||
<Button
|
||||
type='button'
|
||||
variant='ghost'
|
||||
size='sm'
|
||||
onClick={() =>
|
||||
onChange({
|
||||
...emptyCertificationSetup(),
|
||||
checkCache: setup.checkCache,
|
||||
})
|
||||
}
|
||||
>
|
||||
Clear
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<Command className='h-auto rounded-md border'>
|
||||
<CommandInput
|
||||
placeholder='Search models by name or ID…'
|
||||
disabled={isLoading || disabled}
|
||||
/>
|
||||
<CommandList className='max-h-64'>
|
||||
<CommandEmpty>
|
||||
{isLoading ? 'Loading models…' : 'No models found'}
|
||||
</CommandEmpty>
|
||||
{renderModelGroup('Configured models', configuredOptions)}
|
||||
{renderModelGroup('Discovered models', discoveredOptions)}
|
||||
</CommandList>
|
||||
</Command>
|
||||
{Boolean(error) && (
|
||||
<p className='text-destructive text-sm'>{getErrorMessage(error)}</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{setup.selectedModelIds.map((modelId) => {
|
||||
const paths = getExactCertificationPaths(models, modelId);
|
||||
const mode = setup.pathModes[modelId] ?? 'default';
|
||||
const selectedPaths = setup.selectedModelPaths[modelId] ?? [];
|
||||
return (
|
||||
<div key={modelId} className='space-y-3 rounded-md border p-3'>
|
||||
<div className='min-w-0'>
|
||||
<div className='truncate text-sm font-medium'>
|
||||
{namesById.get(modelId) ?? modelId}
|
||||
</div>
|
||||
<div className='text-muted-foreground truncate font-mono text-xs'>
|
||||
{modelId}
|
||||
</div>
|
||||
</div>
|
||||
{paths.length > 0 ? (
|
||||
<>
|
||||
<ToggleGroup
|
||||
type='single'
|
||||
variant='outline'
|
||||
size='sm'
|
||||
value={mode}
|
||||
onValueChange={(value) => {
|
||||
if (!value) return;
|
||||
onChange({
|
||||
...setup,
|
||||
pathModes: {
|
||||
...setup.pathModes,
|
||||
[modelId]: value as ModelPathMode,
|
||||
},
|
||||
});
|
||||
}}
|
||||
disabled={disabled}
|
||||
className='w-full justify-start'
|
||||
>
|
||||
<ToggleGroupItem value='default'>Default</ToggleGroupItem>
|
||||
<ToggleGroupItem value='selected'>
|
||||
Choose paths
|
||||
</ToggleGroupItem>
|
||||
<ToggleGroupItem value='all'>All paths</ToggleGroupItem>
|
||||
</ToggleGroup>
|
||||
{mode === 'default' && (
|
||||
<p className='text-muted-foreground text-xs'>
|
||||
Uses the upstream provider's normal model routing.
|
||||
</p>
|
||||
)}
|
||||
{mode === 'selected' && (
|
||||
<Popover>
|
||||
<PopoverTrigger asChild>
|
||||
<Button
|
||||
type='button'
|
||||
variant='outline'
|
||||
size='sm'
|
||||
className='w-full justify-between'
|
||||
disabled={disabled}
|
||||
>
|
||||
<span className='truncate'>
|
||||
{selectedPaths.length === 0
|
||||
? 'Select paths'
|
||||
: `${selectedPaths.length} path${selectedPaths.length === 1 ? '' : 's'} selected`}
|
||||
</span>
|
||||
<ChevronDown className='h-4 w-4' />
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent
|
||||
align='start'
|
||||
className='w-80 max-w-[calc(100vw-2rem)] p-2'
|
||||
>
|
||||
<div className='max-h-64 space-y-1 overflow-y-auto overscroll-contain'>
|
||||
{paths.map((path) => {
|
||||
const checked = selectedPaths.includes(path.path);
|
||||
return (
|
||||
<label
|
||||
key={path.path}
|
||||
className='hover:bg-muted flex cursor-pointer items-center gap-2 rounded-sm px-2 py-1.5 text-sm'
|
||||
>
|
||||
<Checkbox
|
||||
checked={checked}
|
||||
onCheckedChange={(value) => {
|
||||
onChange({
|
||||
...setup,
|
||||
selectedModelPaths: {
|
||||
...setup.selectedModelPaths,
|
||||
[modelId]:
|
||||
value === true
|
||||
? [...selectedPaths, path.path]
|
||||
: selectedPaths.filter(
|
||||
(item) => item !== path.path
|
||||
),
|
||||
},
|
||||
});
|
||||
}}
|
||||
/>
|
||||
<span className='min-w-0 truncate'>
|
||||
{getCertificationPathLabel(path)}
|
||||
</span>
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
)}
|
||||
{mode === 'all' && (
|
||||
<p className='text-muted-foreground text-xs'>
|
||||
All {paths.length} paths will run in parallel.
|
||||
</p>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
<p className='text-muted-foreground text-xs'>
|
||||
Only the provider default route is available.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
|
||||
{modelsNeedingPath.length > 0 && (
|
||||
<p className='text-muted-foreground text-xs'>
|
||||
Choose at least one path for each model using “Choose paths”.
|
||||
</p>
|
||||
)}
|
||||
|
||||
<div className='flex items-center gap-2'>
|
||||
<Checkbox
|
||||
id={`${idPrefix}-cache`}
|
||||
checked={setup.checkCache}
|
||||
onCheckedChange={(value) =>
|
||||
onChange({ ...setup, checkCache: value === true })
|
||||
}
|
||||
disabled={disabled}
|
||||
/>
|
||||
<Label htmlFor={`${idPrefix}-cache`} className='text-sm'>
|
||||
Probe prompt caching and margin
|
||||
</Label>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
'use client';
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
|
||||
import { AdminService } from '@/lib/api/services/admin';
|
||||
import type {
|
||||
CertificationProgress,
|
||||
ModelCertificationResult,
|
||||
ModelRun,
|
||||
} from '@/lib/provider-certification';
|
||||
import { getErrorMessage } from '@/lib/provider-certification';
|
||||
|
||||
interface RunProviderCertificationOptions {
|
||||
providerId: number;
|
||||
modelRuns: ModelRun[];
|
||||
includeCache: boolean;
|
||||
onProgress?: (progress: CertificationProgress | null) => void;
|
||||
onResults?: (results: ModelCertificationResult[]) => void;
|
||||
}
|
||||
|
||||
export async function runProviderCertification({
|
||||
providerId,
|
||||
modelRuns,
|
||||
includeCache,
|
||||
onProgress,
|
||||
onResults,
|
||||
}: RunProviderCertificationOptions): Promise<ModelCertificationResult[]> {
|
||||
const completed: ModelCertificationResult[] = [];
|
||||
|
||||
for (const [index, run] of modelRuns.entries()) {
|
||||
onProgress?.({
|
||||
modelId: run.modelId,
|
||||
modelIndex: index + 1,
|
||||
modelTotal: modelRuns.length,
|
||||
pathCount: run.targets.length,
|
||||
});
|
||||
const batch = await Promise.all(
|
||||
run.targets.map(async (target): Promise<ModelCertificationResult> => {
|
||||
const resultKey = `${providerId}::${run.modelId}::${target.path ?? 'default'}`;
|
||||
try {
|
||||
const report = await AdminService.certifyProvider(providerId, {
|
||||
model_id: run.modelId,
|
||||
model_path: target.path,
|
||||
check_cache: includeCache,
|
||||
});
|
||||
return {
|
||||
resultKey,
|
||||
providerId,
|
||||
modelId: run.modelId,
|
||||
pathLabel: target.label,
|
||||
report,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
resultKey,
|
||||
providerId,
|
||||
modelId: run.modelId,
|
||||
pathLabel: target.label,
|
||||
error: getErrorMessage(error),
|
||||
};
|
||||
}
|
||||
})
|
||||
);
|
||||
completed.push(...batch);
|
||||
onResults?.([...completed]);
|
||||
}
|
||||
|
||||
onProgress?.(null);
|
||||
return completed;
|
||||
}
|
||||
|
||||
export function useProviderCertificationRunner(providerId: number) {
|
||||
const [results, setResults] = useState<ModelCertificationResult[]>([]);
|
||||
const [progress, setProgress] = useState<CertificationProgress | null>(null);
|
||||
const [isPending, setIsPending] = useState(false);
|
||||
const generation = useRef(0);
|
||||
const mounted = useRef(true);
|
||||
|
||||
useEffect(() => {
|
||||
mounted.current = true;
|
||||
return () => {
|
||||
mounted.current = false;
|
||||
generation.current += 1;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const reset = useCallback(() => {
|
||||
generation.current += 1;
|
||||
setResults([]);
|
||||
setProgress(null);
|
||||
setIsPending(false);
|
||||
}, []);
|
||||
|
||||
const run = useCallback(
|
||||
async (modelRuns: ModelRun[], includeCache: boolean) => {
|
||||
const runGeneration = generation.current + 1;
|
||||
generation.current = runGeneration;
|
||||
setResults([]);
|
||||
setProgress(null);
|
||||
setIsPending(true);
|
||||
try {
|
||||
return await runProviderCertification({
|
||||
providerId,
|
||||
modelRuns,
|
||||
includeCache,
|
||||
onProgress: (nextProgress) => {
|
||||
if (mounted.current && generation.current === runGeneration) {
|
||||
setProgress(nextProgress);
|
||||
}
|
||||
},
|
||||
onResults: (nextResults) => {
|
||||
if (mounted.current && generation.current === runGeneration) {
|
||||
setResults(nextResults);
|
||||
}
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
if (mounted.current && generation.current === runGeneration) {
|
||||
setProgress(null);
|
||||
setIsPending(false);
|
||||
}
|
||||
}
|
||||
},
|
||||
[providerId]
|
||||
);
|
||||
|
||||
return { results, progress, isPending, run, reset };
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import type {
|
||||
CertificationPath,
|
||||
CertificationStatus,
|
||||
ProviderCertification,
|
||||
ProviderModels,
|
||||
} from '@/lib/api/services/admin';
|
||||
|
||||
export type ModelPathMode = 'default' | 'selected' | 'all';
|
||||
|
||||
export interface ProviderCertificationSetup {
|
||||
selectedModelIds: string[];
|
||||
pathModes: Record<string, ModelPathMode>;
|
||||
selectedModelPaths: Record<string, string[]>;
|
||||
checkCache: boolean;
|
||||
}
|
||||
|
||||
export interface ModelPathTarget {
|
||||
path?: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export interface ModelRun {
|
||||
modelId: string;
|
||||
targets: ModelPathTarget[];
|
||||
}
|
||||
|
||||
export interface CertificationProgress {
|
||||
modelId: string;
|
||||
modelIndex: number;
|
||||
modelTotal: number;
|
||||
pathCount: number;
|
||||
}
|
||||
|
||||
export interface ModelCertificationResult {
|
||||
resultKey: string;
|
||||
providerId: number;
|
||||
modelId: string;
|
||||
pathLabel: string;
|
||||
report?: ProviderCertification;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export const emptyCertificationSetup = (): ProviderCertificationSetup => ({
|
||||
selectedModelIds: [],
|
||||
pathModes: {},
|
||||
selectedModelPaths: {},
|
||||
checkCache: true,
|
||||
});
|
||||
|
||||
export const getExactCertificationPaths = (
|
||||
models: ProviderModels | undefined,
|
||||
modelId: string
|
||||
): CertificationPath[] =>
|
||||
(models?.certification_paths[modelId] ?? []).filter(
|
||||
(path) => path.endpoint_tag
|
||||
);
|
||||
|
||||
export const getCertificationPathLabel = (path: CertificationPath): string =>
|
||||
path.endpoint_name && path.endpoint_name !== path.endpoint_tag
|
||||
? `${path.endpoint_name} (${path.endpoint_tag})`
|
||||
: path.endpoint_tag || 'Provider default';
|
||||
|
||||
export const getModelsNeedingPath = (
|
||||
setup: ProviderCertificationSetup
|
||||
): string[] =>
|
||||
setup.selectedModelIds.filter(
|
||||
(modelId) =>
|
||||
setup.pathModes[modelId] === 'selected' &&
|
||||
(setup.selectedModelPaths[modelId]?.length ?? 0) === 0
|
||||
);
|
||||
|
||||
export const buildModelRuns = (
|
||||
setup: ProviderCertificationSetup,
|
||||
models: ProviderModels | undefined
|
||||
): ModelRun[] =>
|
||||
setup.selectedModelIds.map((modelId) => {
|
||||
const paths = getExactCertificationPaths(models, modelId);
|
||||
const mode = setup.pathModes[modelId] ?? 'default';
|
||||
if (mode === 'all') {
|
||||
return {
|
||||
modelId,
|
||||
targets: paths.map((path) => ({
|
||||
path: path.path,
|
||||
label: getCertificationPathLabel(path),
|
||||
})),
|
||||
};
|
||||
}
|
||||
if (mode === 'selected') {
|
||||
const selected = new Set(setup.selectedModelPaths[modelId] ?? []);
|
||||
return {
|
||||
modelId,
|
||||
targets: paths
|
||||
.filter((path) => selected.has(path.path))
|
||||
.map((path) => ({
|
||||
path: path.path,
|
||||
label: getCertificationPathLabel(path),
|
||||
})),
|
||||
};
|
||||
}
|
||||
return { modelId, targets: [{ label: 'Provider default' }] };
|
||||
});
|
||||
|
||||
export const countCertificationTargets = (runs: ModelRun[]): number =>
|
||||
runs.reduce((total, run) => total + run.targets.length, 0);
|
||||
|
||||
export const getCertificationResultStatus = (
|
||||
result: ModelCertificationResult
|
||||
): CertificationStatus | 'error' => {
|
||||
if (result.error || !result.report) return 'error';
|
||||
if (result.report.rows.some((row) => row.status === 'fail')) return 'fail';
|
||||
if (result.report.rows.some((row) => row.status === 'warn')) return 'warn';
|
||||
return 'ok';
|
||||
};
|
||||
|
||||
export const getErrorMessage = (error: unknown): string =>
|
||||
error instanceof Error ? error.message : 'Certification request failed';
|
||||
Reference in New Issue
Block a user