From a64dd0df2cecb3b4aaf82e0983c70d4dae41ed8c Mon Sep 17 00:00:00 2001 From: 9qeklajc <9qeklajc> Date: Sat, 28 Jun 2025 22:33:55 +0200 Subject: [PATCH 1/6] get models should passthrough --- router/proxy.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/router/proxy.py b/router/proxy.py index bf1644ed..12bf9e0b 100644 --- a/router/proxy.py +++ b/router/proxy.py @@ -168,7 +168,7 @@ async def forward_to_upstream( path: str, headers: dict, request_body: bytes | None, - key: ApiKey, + key: ApiKey | None, session: AsyncSession, ) -> Response | StreamingResponse: """Forward request to upstream and handle the response.""" @@ -313,14 +313,16 @@ async def proxy( key = await get_bearer_token_key(headers, path, session, auth) else: - return Response( - content=json.dumps({"detail": "Unauthorized"}), - status_code=401, - media_type="application/json", - ) + key = None + if request.method not in ["GET"]: + return Response( + content=json.dumps({"detail": "Unauthorized"}), + status_code=401, + media_type="application/json", + ) # Only pay for request if we have request body data (for completions endpoints) - if request_body_dict: + if request_body_dict and key is not None: await pay_for_request(key, session, request_body_dict) # Prepare headers for upstream @@ -331,7 +333,7 @@ async def proxy( request, path, headers, request_body, key, session ) - if response.status_code != 200 and key.refund_address == "X-CASHU": + if response.status_code != 200 and (key is not None and key.refund_address == "X-CASHU"): refund_token = await x_cashu_refund(key, session) response = Response( content=json.dumps( @@ -350,7 +352,7 @@ async def proxy( response.headers["X-Cashu"] = refund_token return response - if key.refund_address == "X-CASHU": + if key is not None and key.refund_address == "X-CASHU": refund_token = await x_cashu_refund(key, session) response.headers["X-Cashu"] = refund_token From 708b79d979b17c1bb57055ddcbbdfcfad88903e1 Mon Sep 17 00:00:00 2001 From: 9qeklajc <9qeklajc> Date: Sat, 28 Jun 2025 22:38:06 +0200 Subject: [PATCH 2/6] fmt --- router/proxy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/router/proxy.py b/router/proxy.py index 12bf9e0b..50292d81 100644 --- a/router/proxy.py +++ b/router/proxy.py @@ -333,7 +333,9 @@ async def proxy( request, path, headers, request_body, key, session ) - if response.status_code != 200 and (key is not None and key.refund_address == "X-CASHU"): + if response.status_code != 200 and ( + key is not None and key.refund_address == "X-CASHU" + ): refund_token = await x_cashu_refund(key, session) response = Response( content=json.dumps( From 71623b121f729ba83ee2d2b5ee6b76d5ee45e561 Mon Sep 17 00:00:00 2001 From: 9qeklajc <9qeklajc> Date: Sun, 29 Jun 2025 12:17:41 +0200 Subject: [PATCH 3/6] add simple forward func --- router/proxy.py | 55 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/router/proxy.py b/router/proxy.py index 50292d81..5de67a94 100644 --- a/router/proxy.py +++ b/router/proxy.py @@ -1,6 +1,7 @@ import json import os import re +import traceback from typing import AsyncGenerator import httpx @@ -168,7 +169,7 @@ async def forward_to_upstream( path: str, headers: dict, request_body: bytes | None, - key: ApiKey | None, + key: ApiKey, session: AsyncSession, ) -> Response | StreamingResponse: """Forward request to upstream and handle the response.""" @@ -302,6 +303,9 @@ async def proxy( media_type="application/json", ) + # Prepare headers for upstream + headers = prepare_upstream_headers(dict(request.headers)) + # Handle authentication if x_cashu := headers.get("x-cashu", None): # Check token balance before authentication for cashu tokens @@ -313,21 +317,18 @@ async def proxy( key = await get_bearer_token_key(headers, path, session, auth) else: - key = None if request.method not in ["GET"]: return Response( content=json.dumps({"detail": "Unauthorized"}), status_code=401, media_type="application/json", ) + return await forward_get_to_upstream(request, path, headers) # Only pay for request if we have request body data (for completions endpoints) if request_body_dict and key is not None: await pay_for_request(key, session, request_body_dict) - # Prepare headers for upstream - headers = prepare_upstream_headers(dict(request.headers)) - # Forward to upstream and handle response response = await forward_to_upstream( request, path, headers, request_body, key, session @@ -392,3 +393,47 @@ async def get_bearer_token_key( refund_address, key_expiry_time, # type: ignore ) + + +async def forward_get_to_upstream( + request: Request, + path: str, + headers: dict, +) -> Response | StreamingResponse: + """Forward request to upstream and handle the response.""" + if path.startswith("v1/"): + path = path.replace("v1/", "") + + url = f"{UPSTREAM_BASE_URL}/{path}" + + async with httpx.AsyncClient( + transport=httpx.AsyncHTTPTransport(retries=1), + timeout=None, + ) as client: + try: + response = await client.send( + client.build_request( + request.method, + url, + headers=headers, + content=request.stream(), + params=request.query_params, + ), + ) + + return StreamingResponse( + response.aiter_bytes(), + status_code=response.status_code, + headers=dict(response.headers), + ) + except Exception as exc: + tb = traceback.format_exc() + print( + f"Unexpected error: {exc}\n" + f"Request details: method={request.method}, url={url}, headers={headers}, " + f"path={path}, query_params={dict(request.query_params)}\n" + f"Traceback:\n{tb}" + ) + return create_error_response( + "internal_error", "An unexpected server error occurred", 500 + ) From b97ec5effdac0543217a7e1713798b0650400a54 Mon Sep 17 00:00:00 2001 From: 9qeklajc <9qeklajc> Date: Sun, 29 Jun 2025 12:25:24 +0200 Subject: [PATCH 4/6] revert changes --- router/proxy.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/router/proxy.py b/router/proxy.py index 5de67a94..5aa9a087 100644 --- a/router/proxy.py +++ b/router/proxy.py @@ -303,9 +303,6 @@ async def proxy( media_type="application/json", ) - # Prepare headers for upstream - headers = prepare_upstream_headers(dict(request.headers)) - # Handle authentication if x_cashu := headers.get("x-cashu", None): # Check token balance before authentication for cashu tokens @@ -323,12 +320,18 @@ async def proxy( status_code=401, media_type="application/json", ) + + # Prepare headers for upstream + headers = prepare_upstream_headers(dict(request.headers)) return await forward_get_to_upstream(request, path, headers) # Only pay for request if we have request body data (for completions endpoints) if request_body_dict and key is not None: await pay_for_request(key, session, request_body_dict) + # Prepare headers for upstream + headers = prepare_upstream_headers(dict(request.headers)) + # Forward to upstream and handle response response = await forward_to_upstream( request, path, headers, request_body, key, session From fb9474858fbbbfb5be0b04a214084998d633d55b Mon Sep 17 00:00:00 2001 From: 9qeklajc <9qeklajc> Date: Sun, 29 Jun 2025 12:34:54 +0200 Subject: [PATCH 5/6] remove redundant checks --- router/proxy.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/router/proxy.py b/router/proxy.py index 5aa9a087..b9336229 100644 --- a/router/proxy.py +++ b/router/proxy.py @@ -326,7 +326,7 @@ async def proxy( return await forward_get_to_upstream(request, path, headers) # Only pay for request if we have request body data (for completions endpoints) - if request_body_dict and key is not None: + if request_body_dict: await pay_for_request(key, session, request_body_dict) # Prepare headers for upstream @@ -337,9 +337,8 @@ async def proxy( request, path, headers, request_body, key, session ) - if response.status_code != 200 and ( - key is not None and key.refund_address == "X-CASHU" - ): + if response.status_code != 200 and key.refund_address == "X-CASHU": + print(key) refund_token = await x_cashu_refund(key, session) response = Response( content=json.dumps( From 0818b29fc9e74c275a96ae4994214ea82130d285 Mon Sep 17 00:00:00 2001 From: 9qeklajc <9qeklajc> Date: Sun, 29 Jun 2025 12:42:17 +0200 Subject: [PATCH 6/6] clean up --- router/proxy.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/router/proxy.py b/router/proxy.py index b9336229..2e1017cc 100644 --- a/router/proxy.py +++ b/router/proxy.py @@ -338,7 +338,6 @@ async def proxy( ) if response.status_code != 200 and key.refund_address == "X-CASHU": - print(key) refund_token = await x_cashu_refund(key, session) response = Response( content=json.dumps( @@ -357,7 +356,7 @@ async def proxy( response.headers["X-Cashu"] = refund_token return response - if key is not None and key.refund_address == "X-CASHU": + if key.refund_address == "X-CASHU": refund_token = await x_cashu_refund(key, session) response.headers["X-Cashu"] = refund_token