From 98bb904525d88751a2e0edb74cc66613404485ea Mon Sep 17 00:00:00 2001 From: 9qeklajc Date: Wed, 23 Sep 2026 22:10:33 +0200 Subject: [PATCH] feat: add multi-provider certification page --- ui/app/providers/certification/page.tsx | 568 +++++++++++++ ui/app/providers/page.tsx | 21 +- .../provider-certification-dialog.tsx | 756 ++---------------- .../provider-certification-results.tsx | 305 +++++++ .../provider-certification-setup.tsx | 309 +++++++ ui/hooks/use-provider-certification-runner.ts | 128 +++ ui/lib/provider-certification.ts | 116 +++ 7 files changed, 1506 insertions(+), 697 deletions(-) create mode 100644 ui/app/providers/certification/page.tsx create mode 100644 ui/components/provider-certification-results.tsx create mode 100644 ui/components/provider-certification-setup.tsx create mode 100644 ui/hooks/use-provider-certification-runner.ts create mode 100644 ui/lib/provider-certification.ts diff --git a/ui/app/providers/certification/page.tsx b/ui/app/providers/certification/page.tsx new file mode 100644 index 00000000..ef1b0ef4 --- /dev/null +++ b/ui/app/providers/certification/page.tsx @@ -0,0 +1,568 @@ +'use client'; + +import { useEffect, useMemo, useState } from 'react'; +import Link from 'next/link'; +import { useQueries, useQuery } from '@tanstack/react-query'; +import { + ArrowLeft, + CheckCircle2, + ChevronDown, + Loader2, + Play, + RotateCcw, + Server, +} from 'lucide-react'; + +import { AppPageShell } from '@/components/app-page-shell'; +import { PageHeader } from '@/components/page-header'; +import { + ProviderCertificationResults, + summarizeCertificationResults, +} from '@/components/provider-certification-results'; +import { + getCertificationModelNames, + ProviderCertificationSetupPanel, +} from '@/components/provider-certification-setup'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { Card, CardContent } from '@/components/ui/card'; +import { Checkbox } from '@/components/ui/checkbox'; +import { + Command, + CommandEmpty, + CommandInput, + CommandItem, + CommandList, +} from '@/components/ui/command'; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from '@/components/ui/popover'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; +import { runProviderCertification } from '@/hooks/use-provider-certification-runner'; +import { AdminService } from '@/lib/api/services/admin'; +import type { + ProviderModels, + UpstreamProvider, +} from '@/lib/api/services/admin'; +import { + buildModelRuns, + countCertificationTargets, + emptyCertificationSetup, + getModelsNeedingPath, +} from '@/lib/provider-certification'; +import type { + CertificationProgress, + ModelCertificationResult, + ProviderCertificationSetup, +} from '@/lib/provider-certification'; +import { cn } from '@/lib/utils'; + +function providerName(provider: UpstreamProvider): string { + return provider.slug || provider.provider_type; +} + +export default function MultiProviderCertificationPage() { + const [selectedProviderIds, setSelectedProviderIds] = useState([]); + const [activeProviderId, setActiveProviderId] = useState(null); + const [setups, setSetups] = useState< + Record + >({}); + const [workspaceTab, setWorkspaceTab] = useState<'setup' | 'results'>( + 'setup' + ); + const [resultsByProvider, setResultsByProvider] = useState< + Record + >({}); + const [progressByProvider, setProgressByProvider] = useState< + Record + >({}); + const [runningProviderIds, setRunningProviderIds] = useState([]); + + const providersQuery = useQuery({ + queryKey: ['upstream-providers'], + queryFn: () => AdminService.getUpstreamProviders(), + refetchOnWindowFocus: false, + }); + const providers = useMemo( + () => providersQuery.data ?? [], + [providersQuery.data] + ); + const providersById = useMemo( + () => new Map(providers.map((provider) => [provider.id, provider])), + [providers] + ); + + const modelQueries = useQueries({ + queries: selectedProviderIds.map((providerId) => ({ + queryKey: ['provider-models', providerId], + queryFn: () => AdminService.getProviderModels(providerId), + refetchOnWindowFocus: false, + })), + }); + const modelQueryByProvider = new Map( + selectedProviderIds.map((providerId, index) => [ + providerId, + modelQueries[index], + ]) + ); + + const selectedProviders = selectedProviderIds + .map((providerId) => providersById.get(providerId)) + .filter((provider): provider is UpstreamProvider => Boolean(provider)); + const activeProvider = activeProviderId + ? providersById.get(activeProviderId) + : undefined; + const activeSetup = activeProviderId + ? (setups[activeProviderId] ?? emptyCertificationSetup()) + : undefined; + const activeModelsQuery = activeProviderId + ? modelQueryByProvider.get(activeProviderId) + : undefined; + + useEffect(() => { + if (runningProviderIds.length === 0) return; + const warnBeforeUnload = (event: BeforeUnloadEvent) => { + event.preventDefault(); + }; + window.addEventListener('beforeunload', warnBeforeUnload); + return () => window.removeEventListener('beforeunload', warnBeforeUnload); + }, [runningProviderIds.length]); + + const toggleProvider = (providerId: number) => { + if (runningProviderIds.includes(providerId)) return; + const selected = selectedProviderIds.includes(providerId); + if (selected) { + const remaining = selectedProviderIds.filter((id) => id !== providerId); + setSelectedProviderIds(remaining); + if (activeProviderId === providerId) { + setActiveProviderId(remaining[0] ?? null); + } + return; + } + setSelectedProviderIds((current) => [...current, providerId]); + setSetups((current) => ({ + ...current, + [providerId]: current[providerId] ?? emptyCertificationSetup(), + })); + setActiveProviderId(providerId); + }; + + const updateProviderSetup = ( + providerId: number, + setup: ProviderCertificationSetup + ) => { + setSetups((current) => ({ ...current, [providerId]: setup })); + setResultsByProvider((current) => ({ ...current, [providerId]: [] })); + }; + + const providerRuns = (providerId: number) => + buildModelRuns( + setups[providerId] ?? emptyCertificationSetup(), + modelQueryByProvider.get(providerId)?.data as ProviderModels | undefined + ); + + const isProviderReady = (providerId: number): boolean => { + const setup = setups[providerId] ?? emptyCertificationSetup(); + return ( + setup.selectedModelIds.length > 0 && + getModelsNeedingPath(setup).length === 0 && + Boolean(modelQueryByProvider.get(providerId)?.data) + ); + }; + + const runOneProvider = async (providerId: number) => { + const setup = setups[providerId] ?? emptyCertificationSetup(); + const modelRuns = providerRuns(providerId); + setResultsByProvider((current) => ({ ...current, [providerId]: [] })); + setProgressByProvider((current) => ({ ...current, [providerId]: null })); + setRunningProviderIds((current) => + current.includes(providerId) ? current : [...current, providerId] + ); + try { + await runProviderCertification({ + providerId, + modelRuns, + includeCache: setup.checkCache, + onProgress: (progress) => + setProgressByProvider((current) => ({ + ...current, + [providerId]: progress, + })), + onResults: (results) => + setResultsByProvider((current) => ({ + ...current, + [providerId]: results, + })), + }); + } finally { + setProgressByProvider((current) => ({ + ...current, + [providerId]: null, + })); + setRunningProviderIds((current) => + current.filter((id) => id !== providerId) + ); + } + }; + + const runAllProviders = () => { + setWorkspaceTab('results'); + void Promise.all(selectedProviderIds.map(runOneProvider)); + }; + + const activeResults = activeProviderId + ? (resultsByProvider[activeProviderId] ?? []) + : []; + const activeNames = getCertificationModelNames( + activeModelsQuery?.data as ProviderModels | undefined + ); + + const totalModels = selectedProviderIds.reduce( + (total, providerId) => + total + (setups[providerId]?.selectedModelIds.length ?? 0), + 0 + ); + const totalRoutes = selectedProviderIds.reduce( + (total, providerId) => + total + countCertificationTargets(providerRuns(providerId)), + 0 + ); + const incompleteProviders = selectedProviderIds.filter( + (providerId) => !isProviderReady(providerId) + ); + const allReady = + selectedProviderIds.length > 0 && incompleteProviders.length === 0; + const allResults = Object.values(resultsByProvider).flat(); + const aggregateSummary = summarizeCertificationResults(allResults); + const pendingRoutes = Math.max(totalRoutes - allResults.length, 0); + + return ( + +
+ + + + + + + + + + + + {providersQuery.isLoading + ? 'Loading providers…' + : 'No providers found'} + + {providers.map((provider) => ( + toggleProvider(provider.id)} + disabled={runningProviderIds.includes(provider.id)} + > + + ))} + + + + +
+ } + /> + + {selectedProviders.length === 0 ? ( + + + +
+
Select providers to certify
+

+ Choose two or more providers to configure independent model + and path runs. +

+
+ + + ) : ( +
+ + +
+ +
+ + {activeProvider && activeSetup ? ( + +
+
+
+
+ {providerName(activeProvider)} +
+
+ {activeProvider.base_url} +
+
+ + {activeProvider.enabled ? 'Enabled' : 'Disabled'} + +
+
+ + + setWorkspaceTab(value as 'setup' | 'results') + } + className='min-h-0 flex-1 overflow-hidden px-4 pb-4' + > + + Setup + + Results + {activeResults.length > 0 && ( + + {activeResults.length} + + )} + + + + +
+ + updateProviderSetup(activeProvider.id, next) + } + disabled={runningProviderIds.includes( + activeProvider.id + )} + idPrefix={`multi-certify-${activeProvider.id}`} + /> +
+
+ +
+ + + +
+ +
+ + + + + ) : null} +
+ )} + + {selectedProviders.length > 0 && ( +
+
+
+ {selectedProviderIds.length} providers · {totalModels} models ·{' '} + {totalRoutes} routes +
+
+ {incompleteProviders.length > 0 + ? `${incompleteProviders.length} provider${incompleteProviders.length === 1 ? '' : 's'} need a model or path selection. ` + : 'Providers run concurrently; models are sequential and paths run in parallel. '} + Status: {aggregateSummary.ok} ok, {aggregateSummary.warn}{' '} + warnings, {aggregateSummary.fail} failed,{' '} + {aggregateSummary.error} errors, {runningProviderIds.length}{' '} + running, {pendingRoutes} pending. +
+
+ +
+ )} + + + ); +} diff --git a/ui/app/providers/page.tsx b/ui/app/providers/page.tsx index 669ac088..250fcafa 100644 --- a/ui/app/providers/page.tsx +++ b/ui/app/providers/page.tsx @@ -1,5 +1,6 @@ 'use client'; +import Link from 'next/link'; import { Button } from '@/components/ui/button'; import { Card, CardContent } from '@/components/ui/card'; import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; @@ -18,7 +19,7 @@ import { BatchOverrideDialog } from '@/components/batch-override-dialog'; import { ProviderCard } from '@/components/provider-card'; import { ProviderFormDialogContent } from '@/components/provider-form-dialog-content'; import { Skeleton } from '@/components/ui/skeleton'; -import { AlertCircle, Plus, Server } from 'lucide-react'; +import { AlertCircle, BadgeCheck, Plus, Server } from 'lucide-react'; import { Alert, AlertDescription } from '@/components/ui/alert'; import { Dialog, DialogTrigger } from '@/components/ui/dialog'; import { @@ -348,12 +349,20 @@ export default function ProvidersPage() { title='Upstream Providers' description='Manage your AI provider connections and credentials.' actions={ - - - + + + + } /> void; } -interface ModelOption { - model: AdminModel; - source: 'configured' | 'discovered'; -} - -type ModelPathMode = 'default' | 'selected' | 'all'; - -interface ModelPathTarget { - path?: string; - label: string; -} - -interface ModelRun { - modelId: string; - targets: ModelPathTarget[]; -} - -interface ModelCertificationResult { - resultKey: string; - modelId: string; - pathLabel: string; - report?: ProviderCertification; - error?: string; -} - -const STATUS_STYLES: Record< - CertificationStatus, - { label: string; icon: typeof CheckCircle2; className: string } -> = { - ok: { - label: 'OK', - icon: CheckCircle2, - className: - 'border-emerald-500/40 bg-emerald-500/10 text-emerald-700 dark:text-emerald-400', - }, - warn: { - label: 'Warn', - icon: AlertTriangle, - className: - 'border-amber-500/40 bg-amber-500/10 text-amber-700 dark:text-amber-400', - }, - fail: { - label: 'Fail', - icon: XCircle, - className: 'border-red-500/40 bg-red-500/10 text-red-700 dark:text-red-400', - }, -}; - -function StatusBadge({ status }: { status: CertificationStatus }) { - const style = STATUS_STYLES[status]; - const Icon = style.icon; - return ( - - - {style.label} - - ); -} - -function RowItem({ row }: { row: CertificationRow }) { - const [showEvidence, setShowEvidence] = useState(false); - const hasEvidence = Object.keys(row.evidence).length > 0; - return ( -
  • -
    -
    -
    {row.title}
    -
    - {row.detail} -
    -
    - {row.id} -
    -
    - -
    - {hasEvidence && ( -
    - - {showEvidence && ( - 
    -              {JSON.stringify(row.evidence, null, 2)}
-
    - )} -
    - )} -
    • - ); -} - -function ChecklistSummary({ report }: { report: ProviderCertification }) { - return ( -
      - {report.checklist.map((goal) => { - const style = STATUS_STYLES[goal.status]; - const Icon = style.icon; - return ( -
    • - - {goal.label} -
      • - ); - })} -
    - ); -} - -function CertificationReport({ report }: { report: ProviderCertification }) { - const failing = report.rows.filter((row) => row.status === 'fail').length; - const warning = report.rows.filter((row) => row.status === 'warn').length; - - return ( -
    - -
    - - {report.rows.length} checks · {failing} failed · {warning} warnings - - Generated {new Date(report.generated_at).toLocaleString()} -
    -
      - {report.rows.map((row) => ( - - ))} -
    -
    - ); -} - -function getErrorMessage(error: unknown): string { - if (error instanceof Error) return error.message; - return 'Certification request failed'; -} - -function resultStatus( - result: ModelCertificationResult -): CertificationStatus | 'error' { - if (result.error || !result.report) return 'error'; - if (result.report.rows.some((row) => row.status === 'fail')) return 'fail'; - if (result.report.rows.some((row) => row.status === 'warn')) return 'warn'; - return 'ok'; -} - export function ProviderCertificationDialog({ provider, open, onOpenChange, }: ProviderCertificationDialogProps) { - const [checkCache, setCheckCache] = useState(true); - const [workspaceTab, setWorkspaceTab] = useState<'setup' | 'results'>('setup'); - const [selectedModelIds, setSelectedModelIds] = useState([]); - const [pathModes, setPathModes] = useState>({}); - const [selectedModelPaths, setSelectedModelPaths] = useState< - Record - >({}); - const [results, setResults] = useState([]); - const [currentModel, setCurrentModel] = useState<{ - id: string; - index: number; - total: number; - pathCount: number; - } | null>(null); + const [workspaceTab, setWorkspaceTab] = useState<'setup' | 'results'>( + 'setup' + ); + const [setup, setSetup] = useState( + emptyCertificationSetup + ); + const { results, progress, isPending, run, reset } = + useProviderCertificationRunner(provider.id); const models = useQuery({ queryKey: ['provider-models', provider.id], @@ -241,183 +56,27 @@ export function ProviderCertificationDialog({ enabled: open, }); - const certify = useMutation({ - mutationFn: async ({ - modelRuns, - includeCache, - }: { - modelRuns: ModelRun[]; - includeCache: boolean; - }) => { - const completed: ModelCertificationResult[] = []; - setResults([]); - setWorkspaceTab('results'); - - for (const [index, run] of modelRuns.entries()) { - setCurrentModel({ - id: run.modelId, - index: index + 1, - total: modelRuns.length, - pathCount: run.targets.length, - }); - const batch = await Promise.all( - run.targets.map(async (target): Promise => { - const resultKey = `${run.modelId}::${target.path ?? 'default'}`; - try { - const report = await AdminService.certifyProvider(provider.id, { - model_id: run.modelId, - model_path: target.path, - check_cache: includeCache, - }); - return { - resultKey, - modelId: run.modelId, - pathLabel: target.label, - report, - }; - } catch (error) { - return { - resultKey, - modelId: run.modelId, - pathLabel: target.label, - error: getErrorMessage(error), - }; - } - }) - ); - completed.push(...batch); - setResults([...completed]); - } - - return completed; - }, - onSettled: () => setCurrentModel(null), - }); - const { reset: resetCertification } = certify; - useEffect(() => { if (!open) { - resetCertification(); + reset(); setWorkspaceTab('setup'); - setSelectedModelIds([]); - setPathModes({}); - setSelectedModelPaths({}); - setResults([]); - setCurrentModel(null); + setSetup(emptyCertificationSetup()); } - }, [open, resetCertification]); + }, [open, reset]); - const configuredOptions: ModelOption[] = - models.data?.db_models.map((model) => ({ - model, - source: 'configured', - })) ?? []; - const discoveredOptions: ModelOption[] = - models.data?.remote_models.map((model) => ({ - model, - source: 'discovered', - })) ?? []; - const allOptions = [...configuredOptions, ...discoveredOptions]; - const namesById = new Map( - allOptions.map(({ model }) => [model.id, model.name || model.id]) - ); + const modelRuns = buildModelRuns(setup, models.data); + const targetCount = countCertificationTargets(modelRuns); + const modelsNeedingPath = getModelsNeedingPath(setup); + const namesById = getCertificationModelNames(models.data); - const pathsForModel = (modelId: string): CertificationPath[] => - (models.data?.certification_paths[modelId] ?? []).filter( - (path) => path.endpoint_tag - ); - - const pathLabel = (path: CertificationPath): string => - path.endpoint_name && path.endpoint_name !== path.endpoint_tag - ? `${path.endpoint_name} (${path.endpoint_tag})` - : path.endpoint_tag || 'Provider default'; - - const toggleModel = (modelId: string) => { - const isSelected = selectedModelIds.includes(modelId); - setSelectedModelIds((current) => - isSelected - ? current.filter((id) => id !== modelId) - : [...current, modelId] - ); - setPathModes((current) => { - const next = { ...current }; - if (isSelected) delete next[modelId]; - else next[modelId] = 'default'; - return next; - }); - setSelectedModelPaths((current) => { - const next = { ...current }; - if (isSelected) delete next[modelId]; - return next; - }); - resetCertification(); - setResults([]); + const updateSetup = (nextSetup: ProviderCertificationSetup) => { + setSetup(nextSetup); + reset(); }; - const modelsNeedingPath = selectedModelIds.filter( - (modelId) => - pathModes[modelId] === 'selected' && - (selectedModelPaths[modelId]?.length ?? 0) === 0 - ); - - const buildModelRuns = (): ModelRun[] => - selectedModelIds.map((modelId) => { - const paths = pathsForModel(modelId); - const mode = pathModes[modelId] ?? 'default'; - if (mode === 'all') { - return { - modelId, - targets: paths.map((path) => ({ - path: path.path, - label: pathLabel(path), - })), - }; - } - if (mode === 'selected') { - const selected = new Set(selectedModelPaths[modelId] ?? []); - return { - modelId, - targets: paths - .filter((path) => selected.has(path.path)) - .map((path) => ({ path: path.path, label: pathLabel(path) })), - }; - } - return { modelId, targets: [{ label: 'Provider default' }] }; - }); - - const modelRuns = buildModelRuns(); - const targetCount = modelRuns.reduce( - (total, run) => total + run.targets.length, - 0 - ); - - const renderModelGroup = (label: string, options: ModelOption[]) => { - if (options.length === 0) return null; - return ( - - {options.map(({ model }) => ( - toggleModel(model.id)} - disabled={certify.isPending} - > - - ))} - - ); + const runCertification = () => { + setWorkspaceTab('results'); + void run(modelRuns, setup.checkCache); }; return ( @@ -426,9 +85,9 @@ export function ProviderCertificationDialog({ Certify upstream models - Select models, then use the provider default, choose specific - paths, or test every path. Models run one at a time; paths for the - same model run in parallel against {provider.base_url}. + Select models, then use the provider default, choose specific paths, + or test every path. Models run one at a time; paths for the same + model run in parallel against {provider.base_url}. @@ -443,10 +102,10 @@ export function ProviderCertificationDialog({ Setup Results - {(certify.isPending || results.length > 0) && ( + {(isPending || results.length > 0) && ( {results.length}/{targetCount} @@ -458,214 +117,37 @@ export function ProviderCertificationDialog({ value='setup' className='mt-0 min-h-0 overflow-hidden data-[state=active]:flex data-[state=active]:flex-col' > -
    -
    -
    - -
    - - {selectedModelIds.length} selected - - {selectedModelIds.length > 0 && !certify.isPending && ( - - )} -
    -
    - - - - - {models.isLoading ? 'Loading models…' : 'No models found'} - - {renderModelGroup('Configured models', configuredOptions)} - {renderModelGroup('Discovered models', discoveredOptions)} - - - {models.isError && ( -

    - {getErrorMessage(models.error)} -

    - )} -
    - - {selectedModelIds.map((modelId) => { - const paths = pathsForModel(modelId); - const mode = pathModes[modelId] ?? 'default'; - const selectedPaths = selectedModelPaths[modelId] ?? []; - return ( -
    -
    -
    - {namesById.get(modelId) ?? modelId} -
    -
    - {modelId} -
    -
    - {paths.length > 0 ? ( - <> - { - if (!value) return; - setPathModes((current) => ({ - ...current, - [modelId]: value as ModelPathMode, - })); - }} - disabled={certify.isPending} - className='w-full justify-start' - > - Default - - Choose paths - - All paths - - {mode === 'default' && ( -

    - Uses the upstream provider's normal model routing. -

    - )} - {mode === 'selected' && ( - - - - - -
    - {paths.map((path) => { - const checked = selectedPaths.includes( - path.path - ); - return ( - - ); - })} -
    -     -     - )} - {mode === 'all' && ( -

    - All {paths.length} paths will run in parallel. -

    - )} - - ) : ( -

    - Only the provider default route is available. -

    - )} -
    - ); - })} - {modelsNeedingPath.length > 0 && ( -

    - Choose at least one path for each model using “Choose paths”. -

    - )} +
    +
    -
    -
    - setCheckCache(value === true)} - disabled={certify.isPending} - /> - -
    +
    + {showEvidence && ( + 
    +              {JSON.stringify(row.evidence, null, 2)}
+
    + )} +
    + )} + + ); +} + +function CertificationReport({ report }: { report: ProviderCertification }) { + const failing = report.rows.filter((row) => row.status === 'fail').length; + const warning = report.rows.filter((row) => row.status === 'warn').length; + + return ( +
    +
      + {report.checklist.map((goal) => { + const style = STATUS_STYLES[goal.status]; + const Icon = style.icon; + return ( +
    • + + {goal.label} +
      • + ); + })} +
    +
    + + {report.rows.length} checks · {failing} failed · {warning} warnings + + Generated {new Date(report.generated_at).toLocaleString()} +
    +
      + {report.rows.map((row) => ( + + ))} +
    +
    + ); +} + +interface ProviderCertificationResultsProps { + results: ModelCertificationResult[]; + progress?: CertificationProgress | null; + namesById: Map; + emptyMessage?: string; +} + +export function ProviderCertificationResults({ + results, + progress, + namesById, + emptyMessage = 'Results will appear here as certification completes.', +}: ProviderCertificationResultsProps) { + const [activeResult, setActiveResult] = useState(''); + + useEffect(() => { + if ( + results.length > 0 && + !results.some((result) => result.resultKey === activeResult) + ) { + setActiveResult(results[0].resultKey); + } + }, [activeResult, results]); + + return ( +
    + {progress && ( +
    + + + Probing {namesById.get(progress.modelId) ?? progress.modelId} + {progress.pathCount > 1 + ? ` across ${progress.pathCount} paths in parallel` + : ''}{' '} + — model {progress.modelIndex} of {progress.modelTotal} + +
    + )} + + {results.length === 0 ? ( +
    + {emptyMessage} +
    + ) : ( + +
    + + {results.map((result, index) => { + const status = getCertificationResultStatus(result); + const Icon = + status === 'error' ? XCircle : STATUS_STYLES[status].icon; + const modelRouteNumber = results + .slice(0, index + 1) + .filter((item) => item.modelId === result.modelId).length; + const modelRouteCount = results.filter( + (item) => item.modelId === result.modelId + ).length; + return ( + + + + {namesById.get(result.modelId) ?? result.modelId} + + {modelRouteCount > 1 && ( + + {modelRouteNumber} + + )} + + ); + })} + +
    + +
    + {results.map((result) => { + const status = getCertificationResultStatus(result); + return ( + +
    +
    +
    + {namesById.get(result.modelId) ?? result.modelId} +
    + +
    +
    + Model path +
    + {result.pathLabel} +
    +
    +
    + {result.report ? ( + + ) : ( +
    + {result.error ?? 'Certification failed'} +
    + )} +     + ); + })} +
    +     + )} +
    + ); +} diff --git a/ui/components/provider-certification-setup.tsx b/ui/components/provider-certification-setup.tsx new file mode 100644 index 00000000..2baa5415 --- /dev/null +++ b/ui/components/provider-certification-setup.tsx @@ -0,0 +1,309 @@ +'use client'; + +import { ChevronDown } from 'lucide-react'; + +import { Button } from '@/components/ui/button'; +import { Checkbox } from '@/components/ui/checkbox'; +import { + Command, + CommandEmpty, + CommandGroup, + CommandInput, + CommandItem, + CommandList, +} from '@/components/ui/command'; +import { Label } from '@/components/ui/label'; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from '@/components/ui/popover'; +import { ToggleGroup, ToggleGroupItem } from '@/components/ui/toggle-group'; +import type { AdminModel, ProviderModels } from '@/lib/api/services/admin'; +import type { + ModelPathMode, + ProviderCertificationSetup, +} from '@/lib/provider-certification'; +import { + emptyCertificationSetup, + getCertificationPathLabel, + getErrorMessage, + getExactCertificationPaths, + getModelsNeedingPath, +} from '@/lib/provider-certification'; + +interface ModelOption { + model: AdminModel; + source: 'configured' | 'discovered'; +} + +interface ProviderCertificationSetupProps { + models?: ProviderModels; + isLoading?: boolean; + error?: unknown; + setup: ProviderCertificationSetup; + onChange: (setup: ProviderCertificationSetup) => void; + disabled?: boolean; + idPrefix: string; +} + +export function getCertificationModelNames( + models: ProviderModels | undefined +): Map { + return new Map( + [...(models?.db_models ?? []), ...(models?.remote_models ?? [])].map( + (model) => [model.id, model.name || model.id] + ) + ); +} + +export function ProviderCertificationSetupPanel({ + models, + isLoading = false, + error, + setup, + onChange, + disabled = false, + idPrefix, +}: ProviderCertificationSetupProps) { + const configuredOptions: ModelOption[] = (models?.db_models ?? []).map( + (model) => ({ model, source: 'configured' }) + ); + const discoveredOptions: ModelOption[] = (models?.remote_models ?? []).map( + (model) => ({ model, source: 'discovered' }) + ); + const namesById = getCertificationModelNames(models); + const modelsNeedingPath = getModelsNeedingPath(setup); + + const toggleModel = (modelId: string) => { + const isSelected = setup.selectedModelIds.includes(modelId); + const pathModes = { ...setup.pathModes }; + const selectedModelPaths = { ...setup.selectedModelPaths }; + if (isSelected) { + delete pathModes[modelId]; + delete selectedModelPaths[modelId]; + } else { + pathModes[modelId] = 'default'; + } + onChange({ + ...setup, + selectedModelIds: isSelected + ? setup.selectedModelIds.filter((id) => id !== modelId) + : [...setup.selectedModelIds, modelId], + pathModes, + selectedModelPaths, + }); + }; + + const renderModelGroup = (label: string, options: ModelOption[]) => { + if (options.length === 0) return null; + return ( + + {options.map(({ model }) => ( + toggleModel(model.id)} + disabled={disabled} + > + + ))} + + ); + }; + + return ( +
    +
    +
    + +
    + + {setup.selectedModelIds.length} selected + + {setup.selectedModelIds.length > 0 && !disabled && ( + + )} +
    +
    + + + + + {isLoading ? 'Loading models…' : 'No models found'} + + {renderModelGroup('Configured models', configuredOptions)} + {renderModelGroup('Discovered models', discoveredOptions)} + + + {Boolean(error) && ( +

    {getErrorMessage(error)}

    + )} +
    + + {setup.selectedModelIds.map((modelId) => { + const paths = getExactCertificationPaths(models, modelId); + const mode = setup.pathModes[modelId] ?? 'default'; + const selectedPaths = setup.selectedModelPaths[modelId] ?? []; + return ( +
    +
    +
    + {namesById.get(modelId) ?? modelId} +
    +
    + {modelId} +
    +
    + {paths.length > 0 ? ( + <> + { + if (!value) return; + onChange({ + ...setup, + pathModes: { + ...setup.pathModes, + [modelId]: value as ModelPathMode, + }, + }); + }} + disabled={disabled} + className='w-full justify-start' + > + Default + + Choose paths + + All paths + + {mode === 'default' && ( +

    + Uses the upstream provider's normal model routing. +

    + )} + {mode === 'selected' && ( + + + + + +
    + {paths.map((path) => { + const checked = selectedPaths.includes(path.path); + return ( + + ); + })} +
    +     +     + )} + {mode === 'all' && ( +

    + All {paths.length} paths will run in parallel. +

    + )} + + ) : ( +

    + Only the provider default route is available. +

    + )} +
    + ); + })} + + {modelsNeedingPath.length > 0 && ( +

    + Choose at least one path for each model using “Choose paths”. +

    + )} + +
    + + onChange({ ...setup, checkCache: value === true }) + } + disabled={disabled} + /> + +
    +
    + ); +} diff --git a/ui/hooks/use-provider-certification-runner.ts b/ui/hooks/use-provider-certification-runner.ts new file mode 100644 index 00000000..aff3b914 --- /dev/null +++ b/ui/hooks/use-provider-certification-runner.ts @@ -0,0 +1,128 @@ +'use client'; + +import { useCallback, useEffect, useRef, useState } from 'react'; + +import { AdminService } from '@/lib/api/services/admin'; +import type { + CertificationProgress, + ModelCertificationResult, + ModelRun, +} from '@/lib/provider-certification'; +import { getErrorMessage } from '@/lib/provider-certification'; + +interface RunProviderCertificationOptions { + providerId: number; + modelRuns: ModelRun[]; + includeCache: boolean; + onProgress?: (progress: CertificationProgress | null) => void; + onResults?: (results: ModelCertificationResult[]) => void; +} + +export async function runProviderCertification({ + providerId, + modelRuns, + includeCache, + onProgress, + onResults, +}: RunProviderCertificationOptions): Promise { + const completed: ModelCertificationResult[] = []; + + for (const [index, run] of modelRuns.entries()) { + onProgress?.({ + modelId: run.modelId, + modelIndex: index + 1, + modelTotal: modelRuns.length, + pathCount: run.targets.length, + }); + const batch = await Promise.all( + run.targets.map(async (target): Promise => { + const resultKey = `${providerId}::${run.modelId}::${target.path ?? 'default'}`; + try { + const report = await AdminService.certifyProvider(providerId, { + model_id: run.modelId, + model_path: target.path, + check_cache: includeCache, + }); + return { + resultKey, + providerId, + modelId: run.modelId, + pathLabel: target.label, + report, + }; + } catch (error) { + return { + resultKey, + providerId, + modelId: run.modelId, + pathLabel: target.label, + error: getErrorMessage(error), + }; + } + }) + ); + completed.push(...batch); + onResults?.([...completed]); + } + + onProgress?.(null); + return completed; +} + +export function useProviderCertificationRunner(providerId: number) { + const [results, setResults] = useState([]); + const [progress, setProgress] = useState(null); + const [isPending, setIsPending] = useState(false); + const generation = useRef(0); + const mounted = useRef(true); + + useEffect(() => { + mounted.current = true; + return () => { + mounted.current = false; + generation.current += 1; + }; + }, []); + + const reset = useCallback(() => { + generation.current += 1; + setResults([]); + setProgress(null); + setIsPending(false); + }, []); + + const run = useCallback( + async (modelRuns: ModelRun[], includeCache: boolean) => { + const runGeneration = generation.current + 1; + generation.current = runGeneration; + setResults([]); + setProgress(null); + setIsPending(true); + try { + return await runProviderCertification({ + providerId, + modelRuns, + includeCache, + onProgress: (nextProgress) => { + if (mounted.current && generation.current === runGeneration) { + setProgress(nextProgress); + } + }, + onResults: (nextResults) => { + if (mounted.current && generation.current === runGeneration) { + setResults(nextResults); + } + }, + }); + } finally { + if (mounted.current && generation.current === runGeneration) { + setProgress(null); + setIsPending(false); + } + } + }, + [providerId] + ); + + return { results, progress, isPending, run, reset }; +} diff --git a/ui/lib/provider-certification.ts b/ui/lib/provider-certification.ts new file mode 100644 index 00000000..8cdcebc1 --- /dev/null +++ b/ui/lib/provider-certification.ts @@ -0,0 +1,116 @@ +import type { + CertificationPath, + CertificationStatus, + ProviderCertification, + ProviderModels, +} from '@/lib/api/services/admin'; + +export type ModelPathMode = 'default' | 'selected' | 'all'; + +export interface ProviderCertificationSetup { + selectedModelIds: string[]; + pathModes: Record; + selectedModelPaths: Record; + checkCache: boolean; +} + +export interface ModelPathTarget { + path?: string; + label: string; +} + +export interface ModelRun { + modelId: string; + targets: ModelPathTarget[]; +} + +export interface CertificationProgress { + modelId: string; + modelIndex: number; + modelTotal: number; + pathCount: number; +} + +export interface ModelCertificationResult { + resultKey: string; + providerId: number; + modelId: string; + pathLabel: string; + report?: ProviderCertification; + error?: string; +} + +export const emptyCertificationSetup = (): ProviderCertificationSetup => ({ + selectedModelIds: [], + pathModes: {}, + selectedModelPaths: {}, + checkCache: true, +}); + +export const getExactCertificationPaths = ( + models: ProviderModels | undefined, + modelId: string +): CertificationPath[] => + (models?.certification_paths[modelId] ?? []).filter( + (path) => path.endpoint_tag + ); + +export const getCertificationPathLabel = (path: CertificationPath): string => + path.endpoint_name && path.endpoint_name !== path.endpoint_tag + ? `${path.endpoint_name} (${path.endpoint_tag})` + : path.endpoint_tag || 'Provider default'; + +export const getModelsNeedingPath = ( + setup: ProviderCertificationSetup +): string[] => + setup.selectedModelIds.filter( + (modelId) => + setup.pathModes[modelId] === 'selected' && + (setup.selectedModelPaths[modelId]?.length ?? 0) === 0 + ); + +export const buildModelRuns = ( + setup: ProviderCertificationSetup, + models: ProviderModels | undefined +): ModelRun[] => + setup.selectedModelIds.map((modelId) => { + const paths = getExactCertificationPaths(models, modelId); + const mode = setup.pathModes[modelId] ?? 'default'; + if (mode === 'all') { + return { + modelId, + targets: paths.map((path) => ({ + path: path.path, + label: getCertificationPathLabel(path), + })), + }; + } + if (mode === 'selected') { + const selected = new Set(setup.selectedModelPaths[modelId] ?? []); + return { + modelId, + targets: paths + .filter((path) => selected.has(path.path)) + .map((path) => ({ + path: path.path, + label: getCertificationPathLabel(path), + })), + }; + } + return { modelId, targets: [{ label: 'Provider default' }] }; + }); + +export const countCertificationTargets = (runs: ModelRun[]): number => + runs.reduce((total, run) => total + run.targets.length, 0); + +export const getCertificationResultStatus = ( + result: ModelCertificationResult +): CertificationStatus | 'error' => { + if (result.error || !result.report) return 'error'; + if (result.report.rows.some((row) => row.status === 'fail')) return 'fail'; + if (result.report.rows.some((row) => row.status === 'warn')) return 'warn'; + return 'ok'; +}; + +export const getErrorMessage = (error: unknown): string => + error instanceof Error ? error.message : 'Certification request failed';