feat: cap proxied completion budget via configurable maxTokens

Inject a default max_tokens (chat/completions) or max_output_tokens
(responses) when a client omits one, so the SDK prices against
completion x maxTokens instead of the provider's worst-case
max_completion_cost. Default 64000; set 0 to disable.
This commit is contained in:
redshift
2026-08-16 14:09:50 +01:00
parent 3ce091283b
commit 530db89677
3 changed files with 28 additions and 0 deletions
+18
View File
@@ -325,6 +325,8 @@ export function createDaemonRequestHandler(deps: {
getModelProviders: (modelId: string) => Promise<any>;
refreshProvidersAndModels: () => Promise<void>;
mode?: "xcashu" | "apikeys";
/** Default max_tokens/max_output_tokens to inject when the client omits one. */
maxTokens: number;
/** Nostr hex pubkey for routstr review/model events (kind 38425/38423). */
routstrPubkey?: string;
usageTrackingDriver: UsageTrackingDriver;
@@ -1486,6 +1488,22 @@ export function createDaemonRequestHandler(deps: {
return;
}
// Cap the completion budget when the client does not set an output token
// limit. Without this, the SDK prices at the provider's worst-case
// max_completion_cost, which varies widely across providers (2.3× for
// kimi-k3) and balloons during provider failover. Chat/completions use
// max_tokens; the OpenAI Responses API uses max_output_tokens.
if (deps.maxTokens > 0) {
const isResponsesPath = url.pathname.includes("/responses");
if (isResponsesPath) {
if (typeof bodyObj.max_output_tokens !== "number") {
bodyObj.max_output_tokens = deps.maxTokens;
}
} else if (typeof bodyObj.max_tokens !== "number") {
bodyObj.max_tokens = deps.maxTokens;
}
}
const forcedProvider: string | undefined =
url.searchParams.get("provider") ||
(req.headers["x-routstr-provider"] as string | undefined) ||
+1
View File
@@ -217,6 +217,7 @@ async function main(): Promise<void> {
getModelProviders,
refreshProvidersAndModels,
mode: config.mode || "apikeys",
maxTokens: config.maxTokens ?? 64000,
routstrPubkey: config.routstrPubkey,
usageTrackingDriver,
providerManager,
+9
View File
@@ -55,6 +55,14 @@ export interface RoutstrdConfig {
relays?: string[];
/** NWC integration configuration */
nwc?: NwcConfig;
/**
* Default max_tokens (chat/completions) and max_output_tokens (responses)
* injected into proxied requests when the client does not supply one.
* Caps the completion budget the SDK prices against (completion × maxTokens)
* instead of the provider's worst-case max_completion_cost. Set to 0 to
* disable injection and always forward the client's value as-is.
*/
maxTokens?: number;
}
export const DEFAULT_CONFIG: RoutstrdConfig = {
@@ -63,4 +71,5 @@ export const DEFAULT_CONFIG: RoutstrdConfig = {
provider: null,
cocodPath: null,
mode: "apikeys",
maxTokens: 64000,
};