Files
routstr-core/routstr/core/exceptions.py
T
redshift 3643a27a4f fix(ehbp): classify upstream TLS/connect failures as upstream-scoped
A TLS handshake to a Tinfoil EHBP enclave that outlives the stdlib's
60s handshake_timeout raises ConnectionAbortedError. That is neither an
httpx.RequestError nor an asyncio.TimeoutError, so it escaped
forward_with_trailer's connect handler and fell through to
forward_ehbp_request's bare `except Exception`, which raised a
node-scoped UpstreamError(status_code=500). Bearer clients then saw a
misleading 500 ("An unexpected server error occurred") even though the
fault was on the provider hop.

Classify connect-phase transport failures at the source:

- ConnectionAbortedError (stdlib TLS handshake timer) -> EhbpTimeoutError
  (UPSTREAM_TIMEOUT, reported as 424)
- other ssl.SSLError/ConnectionError/OSError -> new EhbpConnectionError
  (UPSTREAM_UNAVAILABLE, reported as 424)

Both are UpstreamError subclasses carrying ERROR_SCOPE_UPSTREAM, so
create_upstream_error_response answers 424 with the
X-Routstr-Error-Scope: upstream header instead of implying the node is
down, and provider failover still applies.

Tests cover the handshake-timeout branch, the connection-failure branch,
the new error's metadata, and bearer-path propagation.
2026-09-27 17:44:38 +05:30

233 lines
8.2 KiB
Python

import math
from fastapi import Request
from fastapi.encoders import jsonable_encoder
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from .error_scope import (
ERROR_SCOPE_UPSTREAM,
UPSTREAM_ERROR_STATUS,
UPSTREAM_UNAVAILABLE,
)
from .logging import get_logger
logger = get_logger(__name__)
class UpstreamError(Exception):
"""Exception raised when an upstream provider fails.
``code`` carries a stable, machine-readable classification (e.g.
``UPSTREAM_RATE_LIMIT``) so callers can distinguish failure kinds without
string-matching the message. ``details`` holds optional structured,
redaction-safe context. Both default to ``None`` for backwards
compatibility.
``from_upstream_response`` is True only when ``status_code`` is the status
the upstream itself answered with, as opposed to a status this proxy chose
for a transport failure, timeout or internal fault. Callers use it to
decide whether a status is safe to retry.
``scope`` is ``"upstream"`` for provider failures, reported to the caller
as ``424`` (see :mod:`routstr.core.error_scope`), or ``"node"`` for local
faults, which keep their status. ``status_code`` stays the provider's own
status whenever ``from_upstream_response`` is True; the caller-visible
mapping happens at response construction. Proxy-chosen statuses (transport
failure, timeout) may already be the caller-visible one — only read
``status_code`` as a provider status behind ``from_upstream_response``.
"""
def __init__(
self,
message: str,
status_code: int = 502,
code: str | None = None,
details: dict[str, object] | None = None,
from_upstream_response: bool = False,
scope: str = ERROR_SCOPE_UPSTREAM,
):
self.message = message
self.status_code = status_code
self.code = code
self.details = details
self.from_upstream_response = from_upstream_response
self.scope = scope
super().__init__(message)
class EhbpTimeoutError(UpstreamError):
"""Raised when an EHBP upstream times out waiting for a response.
Distinct from a generic :class:`UpstreamError` so callers can map the
failure to a stable ``UPSTREAM_TIMEOUT`` code instead of a misleading
``500`` internal server error. Reported as ``424``: the timeout happened on
the provider hop, not this node.
``details`` carries optional structured, redaction-safe context and is
forwarded to the client by ``create_upstream_error_response``.
"""
def __init__(self, message: str, details: dict[str, object] | None = None):
super().__init__(
message,
status_code=UPSTREAM_ERROR_STATUS,
code="UPSTREAM_TIMEOUT",
details=details,
)
class EhbpConnectionError(UpstreamError):
"""Raised when an EHBP upstream cannot be reached.
Covers transport failures while establishing the provider connection: DNS
resolution, TCP refused/reset, or a TLS error that is not a handshake
timeout. Distinct from a generic :class:`UpstreamError` so the failure is
attributed to the provider hop (``UPSTREAM_UNAVAILABLE``, reported as
``424``) instead of being flattened into a misleading node-scoped ``500``.
``details`` carries optional structured, redaction-safe context and is
forwarded to the client by ``create_upstream_error_response``.
"""
def __init__(self, message: str, details: dict[str, object] | None = None):
super().__init__(
message,
status_code=UPSTREAM_ERROR_STATUS,
code=UPSTREAM_UNAVAILABLE,
details=details,
)
def _error_message_from_detail(detail: object) -> str | None:
"""Extract a message from an HTTPException ``detail``, capped at 200 chars."""
if isinstance(detail, dict):
error = detail.get("error")
if isinstance(error, dict):
msg = error.get("message")
return str(msg)[:200] if isinstance(msg, str) else None
if isinstance(error, str):
return error[:200]
return None
if isinstance(detail, str):
return detail[:200]
return None
async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
"""Handle HTTP exceptions and include request ID in response."""
request_id = getattr(request.state, "request_id", "unknown")
# Get status code and detail - works for both FastAPI and Starlette HTTPException
status_code = getattr(exc, "status_code", 500)
detail = getattr(exc, "detail", str(exc))
path = request.url.path
error_type: str | None = None
error_code: str | None = None
if isinstance(detail, dict):
error = detail.get("error")
if isinstance(error, dict):
error_type = error.get("type")
error_code = error.get("code")
# 5xx logs as error/warning, 4xx at INFO.
if status_code >= 500:
log_fn = (
logger.warning
if error_type in {"mint_unreachable", "mint_rate_limited"}
else logger.error
)
else:
log_fn = logger.info
log_fn(
f"HTTP {status_code} on {path}: {detail}",
extra={
"request_id": request_id,
"status_code": status_code,
"detail": detail,
"path": path,
"error_type": error_type,
"error_code": error_code,
"level": "http" if status_code < 500 else "server",
},
)
# Stash for LoggingMiddleware's completion log.
request.state.error_detail = {
"error_type": error_type,
"error_code": error_code,
"error_message": _error_message_from_detail(detail),
}
if isinstance(detail, dict) and "error" in detail:
content = {"detail": detail, **detail}
else:
content = {"detail": detail}
content["request_id"] = request_id
headers = getattr(exc, "headers", None)
return JSONResponse(status_code=status_code, content=content, headers=headers)
def json_compliant(value: object) -> object:
"""Render non-finite floats as text so a reply carrying them can serialize.
``json`` parses the bare ``NaN``/``Infinity``/``-Infinity`` literals into
real floats, so a request body — and a stored row written from one — may
hold one anywhere. ``JSONResponse`` encodes with ``allow_nan=False`` and
raises on them, which would turn a reply that merely *quotes* the offending
value into a 500.
"""
if isinstance(value, float) and not math.isfinite(value):
return repr(value)
if isinstance(value, dict):
return {key: json_compliant(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [json_compliant(item) for item in value]
return value
async def validation_exception_handler(
request: Request, exc: Exception
) -> JSONResponse:
"""Answer a request-validation failure with a 422 that always serializes.
Pydantic echoes the rejected value back in each error's ``input`` field. A
non-finite float there breaks the encoder, so the 422 escapes as a 500 and
reports a client's bad rate as a server fault.
"""
request_id = getattr(request.state, "request_id", "unknown")
errors = exc.errors() if isinstance(exc, RequestValidationError) else []
return JSONResponse(
status_code=422,
content={
"detail": json_compliant(jsonable_encoder(errors)),
"request_id": request_id,
},
)
async def general_exception_handler(request: Request, exc: Exception) -> JSONResponse:
"""Handle general exceptions and include request ID in response."""
request_id = getattr(request.state, "request_id", "unknown")
logger.error(
"Unhandled exception",
extra={
"request_id": request_id,
"error": str(exc),
"error_type": type(exc).__name__,
"path": request.url.path,
},
exc_info=True,
)
return JSONResponse(
status_code=500,
content={
"detail": "Internal server error, please contact support with the request ID.",
"request_id": request_id,
},
)