mirror of
https://github.com/Routstr/routstr-core.git
synced 2026-07-31 07:46:15 +00:00
Compare commits
119
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bb2a05b67c | ||
|
|
e2f89a2645 | ||
|
|
16fc548b48 | ||
|
|
c5da73f1e9 | ||
|
|
06dba681c5 | ||
|
|
f96acbb99c | ||
|
|
0a00527626 | ||
|
|
73a3f12469 | ||
|
|
f15eab9f10 | ||
|
|
344c3c5f21 | ||
|
|
1d4b8d7cb2 | ||
|
|
b0c70ecddc | ||
|
|
4c292580e8 | ||
|
|
04879cad56 | ||
|
|
ab80657507 | ||
|
|
b94d95fc2b | ||
|
|
27f81dbf42 | ||
|
|
66ba31d0df | ||
|
|
16679c1f4e | ||
|
|
a9593ab416 | ||
|
|
6221ee8152 | ||
|
|
65ea28cb85 | ||
|
|
2ed20b1b85 | ||
|
|
88fe9758a3 | ||
|
|
03b00f3eb6 | ||
|
|
7e8c2033ae | ||
|
|
52742a6a04 | ||
|
|
030d8b61ce | ||
|
|
f47e16aa61 | ||
|
|
eb6a612e5a | ||
|
|
c2a2d76eae | ||
|
|
d4657dca5a | ||
|
|
7106dfe330 | ||
|
|
1045f1061b | ||
|
|
0415806c5a | ||
|
|
5d5c849180 | ||
|
|
b8700dde40 | ||
|
|
4872c318d5 | ||
|
|
56a67c0a86 | ||
|
|
afcb3f7cda | ||
|
|
14748c28fb | ||
|
|
8aa2fa5c4a | ||
|
|
a024d5be5e | ||
|
|
c40723c1e2 | ||
|
|
6c5c3f0b5f | ||
|
|
86adee9b1b | ||
|
|
dbe7a53afd | ||
|
|
e0c74e3a46 | ||
|
|
18965b4ea4 | ||
|
|
97dc10a8ad | ||
|
|
87850c97b9 | ||
|
|
fcc87718ff | ||
|
|
50437a1cc6 | ||
|
|
7f5a0cf1ae | ||
|
|
3f850c54f4 | ||
|
|
df4d4c44e6 | ||
|
|
02cd2cfeec | ||
|
|
0aebfc6dbe | ||
|
|
b76fa17f81 | ||
|
|
ed8ac914f9 | ||
|
|
6c762f0c3d | ||
|
|
2b4f70442a | ||
|
|
99724cc5f5 | ||
|
|
e19f679609 | ||
|
|
bd1edcef26 | ||
|
|
22a94a68a4 | ||
|
|
4defe4f227 | ||
|
|
f8125a8a2d | ||
|
|
999a5634fa | ||
|
|
2c218cce49 | ||
|
|
fa0b366f9a | ||
|
|
a3a4d69ed3 | ||
|
|
c9533c872a | ||
|
|
90da3803c6 | ||
|
|
8b3b59e176 | ||
|
|
be5e323c68 | ||
|
|
f6d1a41728 | ||
|
|
b3bf1f0e90 | ||
|
|
3035292d2a | ||
|
|
ac436d0862 | ||
|
|
19082231f9 | ||
|
|
281607108c | ||
|
|
8314f3c1b0 | ||
|
|
1a9041766b | ||
|
|
01c01fe8ad | ||
|
|
892aed61cc | ||
|
|
c6733dbb62 | ||
|
|
57fef44ce7 | ||
|
|
c671d277d3 | ||
|
|
9460e24f21 | ||
|
|
f31929b538 | ||
|
|
129ea7bb76 | ||
|
|
b88c6d84fa | ||
|
|
238beb3e52 | ||
|
|
dc833d9f5f | ||
|
|
31e7ff8fbd | ||
|
|
a1850dfa55 | ||
|
|
9eacabde0e | ||
|
|
73f52ef1fd | ||
|
|
110ec5da5d | ||
|
|
a5cee796c9 | ||
|
|
aabf08a930 | ||
|
|
07c15de106 | ||
|
|
aea24d23da | ||
|
|
e100c77288 | ||
|
|
057a752b1e | ||
|
|
5daa2602f5 | ||
|
|
d80912b10e | ||
|
|
8f087bf07a | ||
|
|
744321153f | ||
|
|
b81c5add6a | ||
|
|
a1118a510d | ||
|
|
7c2e2ce512 | ||
|
|
89b93fde98 | ||
|
|
9b4b4d734f | ||
|
|
edfb0f127c | ||
|
|
dc25659cff | ||
|
|
349d8dd009 | ||
|
|
6097193442 |
+14
-2
@@ -5,7 +5,20 @@ UPSTREAM_API_KEY=your-upstream-api-key
|
||||
# Tinfoil (confidential inference enclaves, EHBP)
|
||||
# TINFOIL_API_KEY=your-tinfoil-api-key
|
||||
|
||||
# ADMIN_PASSWORD=secure-admin-password
|
||||
# Secret key used to encrypt node secrets at rest (optional). If unset, the node
|
||||
# generates one on first start, writes it to routstr_secret.key (override the path
|
||||
# with ROUTSTR_SECRET_KEY_FILE), and prints it once — back that file up, because
|
||||
# losing the key makes previously encrypted secrets unreadable. Set it explicitly
|
||||
# to manage the key yourself (recommended in production). Generate one with:
|
||||
# uv run python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
|
||||
ROUTSTR_SECRET_KEY=
|
||||
|
||||
# The admin password and the Nostr identity (nsec) are NOT set here. The admin
|
||||
# password is generated and logged once on first start (read it from the logs to
|
||||
# sign in); both are managed afterwards from the admin UI and stored encrypted in
|
||||
# the database. ADMIN_PASSWORD / NSEC are still read once as a legacy seed for
|
||||
# existing deployments, but new nodes should set them in the UI — a value left in
|
||||
# .env is ignored once the node has been configured.
|
||||
|
||||
# Database
|
||||
# DATABASE_URL=sqlite+aiosqlite:///keys.db
|
||||
@@ -13,7 +26,6 @@ UPSTREAM_API_KEY=your-upstream-api-key
|
||||
# Node Information
|
||||
# NAME=My Routstr Node
|
||||
# DESCRIPTION=Fast AI API access with Bitcoin payments
|
||||
# NSEC=nsec1...
|
||||
# HTTP_URL=https://api.mynode.com
|
||||
# ONION_URL=http://mynode.onion (auto fetched from compose)
|
||||
# RELAYS="wss://relay.damus.io,wss://relay.nostr.band,wss://eden.nostr.land,wss://relay.routstr.com"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
__pycache__
|
||||
.env
|
||||
keys.db
|
||||
routstr_secret.key
|
||||
wallet.sqlite3
|
||||
|
||||
# Python build artifacts
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ WORKDIR /app/ui
|
||||
RUN corepack enable pnpm && corepack prepare pnpm@10.15.0 --activate
|
||||
|
||||
# Copy UI source
|
||||
COPY ui/package.json ui/pnpm-lock.yaml* ./
|
||||
COPY ui/package.json ui/pnpm-lock.yaml* ui/pnpm-workspace.yaml* ./
|
||||
RUN pnpm install --frozen-lockfile
|
||||
|
||||
COPY ui/ ./
|
||||
|
||||
@@ -55,19 +55,41 @@ If you are a node runner, start a Routstr Core instance using Docker Compose:
|
||||
|
||||
1. **Prepare your `.env`**:
|
||||
```bash
|
||||
ADMIN_PASSWORD=mysecretpassword
|
||||
# Optional: encrypts node secrets at rest. If unset, the node generates a key
|
||||
# on first start, writes it to routstr_secret.key, and prints it once — back
|
||||
# up that file. Set it explicitly to manage the key yourself (recommended in
|
||||
# production).
|
||||
ROUTSTR_SECRET_KEY=<generated-key>
|
||||
NAME="My AI Node"
|
||||
DESCRIPTION="Fast access to models"
|
||||
NSEC=yournsec
|
||||
RECEIVE_LN_ADDRESS=yourname@wallet.com
|
||||
```
|
||||
|
||||
Your Nostr identity (`nsec`) is not set in `.env` — configure it from the admin
|
||||
UI after first start, where it's stored encrypted in the database. (`NSEC` in
|
||||
`.env` is still read once as a legacy seed for existing deployments.)
|
||||
|
||||
If you don't set one, a key is generated and printed on first start — save it
|
||||
somewhere safe (losing it makes previously encrypted secrets unreadable). To
|
||||
supply your own, generate it once and keep it stable:
|
||||
```bash
|
||||
uv run python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
|
||||
```
|
||||
|
||||
2. **Start the services**:
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
3. **Configure**:
|
||||
3. **Get your admin password**:
|
||||
On first start the node generates an admin password and logs it once with the
|
||||
`/admin` URL. Read it from the logs:
|
||||
```bash
|
||||
docker compose logs routstr | grep -i admin
|
||||
```
|
||||
(Lost it? Reset with `docker compose exec routstr /.venv/bin/python scripts/reset_admin_password.py --regenerate`.)
|
||||
|
||||
4. **Configure**:
|
||||
Open [http://localhost:8000/admin/](http://localhost:8000/admin/) to connect your AI providers and set pricing.
|
||||
|
||||
For full instructions, see the **[Provider Quick Start Guide](https://docs.routstr.com/provider/quickstart/)**.
|
||||
|
||||
@@ -13,6 +13,7 @@ services:
|
||||
- ./ui_out:/output:z
|
||||
command:
|
||||
["sh", "-c", "mkdir -p /output && cp -r /app/built/. /output/ && echo 'UI build copied to mounted volume' && ls -la /output/ && echo 'UI built and ready' && tail -f /dev/null"]
|
||||
restart: unless-stopped
|
||||
|
||||
routstr:
|
||||
build: .
|
||||
@@ -31,6 +32,7 @@ services:
|
||||
- 8000:8000
|
||||
extra_hosts: # Needed to access locally running models
|
||||
- "host.docker.internal:host-gateway"
|
||||
restart: unless-stopped
|
||||
|
||||
tor:
|
||||
image: ghcr.io/hundehausen/tor-hidden-service:latest
|
||||
@@ -41,6 +43,7 @@ services:
|
||||
- HS_ROUTER=routstr:8000:80
|
||||
depends_on:
|
||||
- routstr
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
tor-data:
|
||||
|
||||
@@ -327,6 +327,62 @@ GET /v1/models
|
||||
}
|
||||
```
|
||||
|
||||
### List Model Paths
|
||||
|
||||
Get the selectable upstream routes for each advertised model. This endpoint is
|
||||
discovery-only; request-side selection will be added separately.
|
||||
|
||||
```http
|
||||
GET /v1/models/paths
|
||||
```
|
||||
|
||||
**Response:**
|
||||
|
||||
```json
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"id": "anthropic/claude-sonnet-4",
|
||||
"paths": [
|
||||
{
|
||||
"path": "url=https%3A%2F%2Fapi.anthropic.com%2Fv1&provider-id=12&model-id=anthropic%2Fclaude-sonnet-4",
|
||||
"provider": {"id": 12, "slug": "anthropic-primary", "type": "anthropic"},
|
||||
"endpoint": null
|
||||
},
|
||||
{
|
||||
"path": "url=https%3A%2F%2Fopenrouter.ai%2Fapi%2Fv1&provider-id=42&model-id=anthropic%2Fclaude-sonnet-4&endpoint=google-vertex%2Fus",
|
||||
"provider": {"id": 42, "slug": "openrouter-main", "type": "openrouter"},
|
||||
"endpoint": {"tag": "google-vertex/us", "name": "Google"}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"updated_at": 1753500000
|
||||
}
|
||||
```
|
||||
|
||||
`path` is an opaque, percent-encoded selector. Clients must store and return it
|
||||
unchanged rather than parsing or reconstructing it. It identifies the exact
|
||||
configured route with `url`, `provider-id`, and `model-id`. To avoid exposing
|
||||
private network details, a configured private IP address or any URL with an
|
||||
explicit port is advertised as `http://localhost`. OpenRouter routes additionally
|
||||
preserve the exact machine-readable endpoint `tag`. Provider slugs/types and
|
||||
endpoint names remain display data. When request-side selection is implemented,
|
||||
an endpoint tag must not silently fall back to another backend.
|
||||
|
||||
### List Paths for One Model
|
||||
|
||||
Use the exact model ID advertised by `/v1/models`. The query parameter safely
|
||||
supports IDs containing `/`.
|
||||
|
||||
```http
|
||||
GET /v1/models/paths/model?model_id=anthropic/claude-sonnet-4
|
||||
```
|
||||
|
||||
The response uses the same path objects and `updated_at` field as the collection
|
||||
endpoint. An unknown model returns `404 Model not found`. A known model whose
|
||||
paths have not been discovered yet returns `200` with an empty `data` array.
|
||||
|
||||
## Wallet Management
|
||||
|
||||
### Create Wallet (Coming Soon)
|
||||
|
||||
@@ -100,6 +100,7 @@ All errors follow a consistent format:
|
||||
Standard OpenAI-compatible endpoints:
|
||||
|
||||
- **Models**: `/v1/models`
|
||||
- **Model paths**: `/v1/models/paths`, `/v1/models/paths/model?model_id=...`
|
||||
- **Responses**: `/v1/responses`
|
||||
- **Chat Completions**: `/v1/chat/completions`
|
||||
- **Embeddings**: `/v1/embeddings`
|
||||
@@ -302,7 +303,7 @@ Get node metadata:
|
||||
GET /v1/info
|
||||
```
|
||||
|
||||
Supported models and pricing are available at `/v1/models`.
|
||||
Supported models and pricing are available at `/v1/models`. Upstream provider path discovery is available at `/v1/models/paths` and `/v1/models/paths/model?model_id=...`.
|
||||
|
||||
## Next Steps
|
||||
|
||||
|
||||
+28
-36
@@ -53,25 +53,22 @@ The actual EHBP forwarding logic does **not** live in `base.py`.
|
||||
|
||||
### `routstr/upstream/ehbp.py`
|
||||
|
||||
Contains the shared opaque EHBP transport helpers:
|
||||
Contains the shared opaque EHBP transport and billing helpers:
|
||||
|
||||
- `EHBPForwardingTarget` — provider-specific target URL plus extra headers
|
||||
- `forward_ehbp_request()` — forwards the raw encrypted body to an EHBP-capable
|
||||
provider, streams the encrypted response back untouched, and finalizes bearer
|
||||
billing at max cost because usage is encrypted
|
||||
- `forward_ehbp_x_cashu_request()` — redeems the Cashu token, forwards raw,
|
||||
refunds the full token on upstream failure, and refunds any value above
|
||||
`max_cost_for_model` on success
|
||||
- `forward_ehbp_request()` — forwards the encrypted body, captures Tinfoil
|
||||
usage from a response header or streaming HTTP trailer, and finalizes bearer
|
||||
billing at actual cost (falling back to max cost when usage is unavailable)
|
||||
- `forward_ehbp_x_cashu_request()` — redeems the Cashu token, refunds the full
|
||||
token on upstream failure, and refunds the difference between the redeemed
|
||||
amount and actual cost (or max cost when usage is unavailable)
|
||||
|
||||
### `routstr/upstream/ppqai.py`
|
||||
### Provider support
|
||||
|
||||
- Sets `supports_ehbp = True`.
|
||||
- Implements `get_ehbp_forwarding_target()` to forward to
|
||||
`https://api.ppq.ai/private/v1/...` — the PPQ.AI enclave endpoint that
|
||||
understands EHBP and returns the `Ehbp-Response-Nonce` header.
|
||||
- Adds `X-Private-Model` with the model's `forwarded_model_id` (e.g.
|
||||
`private/kimi-k2-6`). PPQ.AI's billing layer needs this since it can't
|
||||
decrypt the body.
|
||||
EHBP is currently enabled only for `TinfoilUpstreamProvider`. It forwards to
|
||||
Tinfoil's attested enclave and requests `X-Tinfoil-Usage-Metrics` for billing.
|
||||
PPQ.AI retains its private-target implementation, but `supports_ehbp = False`
|
||||
until it has a provider-specific trusted usage/model-binding strategy.
|
||||
|
||||
## Why it's done this way
|
||||
|
||||
@@ -84,10 +81,10 @@ The proxy is a **blind relay** for EHBP requests. It cannot decrypt the body
|
||||
4. Pass through EHBP protocol headers (`Ehbp-Encapsulated-Key` on request,
|
||||
`Ehbp-Response-Nonce` on response)
|
||||
|
||||
Cost tracking happens at the proxy level using `max_cost_for_model` from the
|
||||
model registry. Because EHBP responses are encrypted, Routstr cannot reconcile
|
||||
against token usage. Bearer requests reserve and then finalize max-cost billing;
|
||||
X-Cashu requests redeem the token and refund any amount above max cost.
|
||||
Cost tracking happens at the proxy level. Routstr reserves or redeems up to
|
||||
`max_cost_for_model`, then Tinfoil's out-of-band usage header/trailer allows it
|
||||
to finalize at actual token cost. If trusted usage is missing or invalid, the
|
||||
proxy safely falls back to max-cost billing.
|
||||
|
||||
## End-to-end flow
|
||||
|
||||
@@ -125,7 +122,7 @@ Three parties see three different model IDs:
|
||||
| Party | Header/Body | Value | Source |
|
||||
|---|---|---|---|
|
||||
| Routstr proxy | `X-Routstr-Model` header | `tinfoil-kimi-k2-6` | SDK sends full caller-facing id |
|
||||
| PPQ.AI billing | `X-Private-Model` header | `private/kimi-k2-6` | Proxy sends `forwarded_model_id` |
|
||||
| Tinfoil usage metrics | `model` field | `kimi-k2-6` | Enclave reports the model actually served |
|
||||
| Tinfoil enclave | `body.model` (encrypted) | `kimi-k2-6` | SDK strips `tinfoil-` prefix before encryption |
|
||||
|
||||
## Implementation status
|
||||
@@ -135,24 +132,19 @@ implements the direct blind-upstream pattern described above. The shared EHBP
|
||||
helpers in `routstr/upstream/ehbp.py` were extended to:
|
||||
|
||||
- Request usage metrics via `X-Tinfoil-Request-Usage-Metrics: true`.
|
||||
- Parse `X-Tinfoil-Usage-Metrics` from the response header (non-streaming).
|
||||
- Override the forwarding URL with `X-Tinfoil-Enclave-Url` when the SDK sends it.
|
||||
- Finalize bearer billing with actual token cost via `adjust_payment_for_tokens`.
|
||||
- Parse `X-Tinfoil-Usage-Metrics` from the response header (non-streaming) or
|
||||
HTTP trailer (streaming).
|
||||
- Override the forwarding URL with a validated `X-Tinfoil-Enclave-Url` when the
|
||||
SDK sends it.
|
||||
- Finalize bearer billing with the dedicated EHBP actual-cost finalizer.
|
||||
- Compute X-Cashu refunds from actual cost instead of max cost.
|
||||
|
||||
See `docs/tinfoil-direct-integration.md` for the full implementation notes.
|
||||
|
||||
## Not yet tested
|
||||
## Verification status
|
||||
|
||||
These changes were written without integration testing due to the complexity
|
||||
of the full stack (SDK + proxy + PPQ.AI enclave + Cashu mint). Needs end-to-end
|
||||
verification with a real `tinfoil-*` model request.
|
||||
|
||||
Important assumptions to verify:
|
||||
|
||||
- PPQ.AI accepts `/private/v1/...` with `X-Private-Model`.
|
||||
- PPQ.AI enforces consistency between `X-Private-Model` and the encrypted
|
||||
`body.model`, otherwise a malicious client could understate
|
||||
`X-Routstr-Model` for billing.
|
||||
- SDK behavior on non-2xx proxy-generated errors that do not carry
|
||||
`Ehbp-Response-Nonce`.
|
||||
Unit coverage includes usage parsing, target validation, HTTP trailer capture,
|
||||
response-size limits, and bearer payment finalization. End-to-end requests have
|
||||
verified both non-streaming usage headers and streaming usage trailers against
|
||||
Tinfoil. SDK behavior on proxy-generated non-2xx responses without an
|
||||
`Ehbp-Response-Nonce` still merits explicit end-to-end coverage.
|
||||
|
||||
@@ -13,7 +13,10 @@ Before running your node, you should create a `.env` file in the project root. T
|
||||
### Example .env
|
||||
|
||||
```bash
|
||||
ADMIN_PASSWORD=your-secure-password
|
||||
# Encrypts node secrets at rest. Optional — if unset, the node generates a key on
|
||||
# first start and prints it once (back it up). Set it to manage the key yourself
|
||||
# (recommended in production). See "Secrets at Rest" below.
|
||||
ROUTSTR_SECRET_KEY=
|
||||
|
||||
# Node Identity
|
||||
NAME="My AI Node"
|
||||
@@ -25,10 +28,10 @@ RECEIVE_LN_ADDRESS=yourname@wallet.com
|
||||
|
||||
### Setting the UI Password
|
||||
|
||||
There are two ways to set or change your Admin Dashboard password:
|
||||
On first start the node generates an admin password and logs it once — read it from the container logs to sign in. You can then change it two ways:
|
||||
|
||||
1. **Via Environment Variable**: Set `ADMIN_PASSWORD` in your `.env` file before starting the container. This will be the password used for the first login.
|
||||
2. **Via Dashboard**: Once logged in, go to **Settings** → **Security** to update your password. Dashboard settings override the `.env` file once saved.
|
||||
1. **Via Dashboard**: Once logged in, go to **Settings** → **Security** to update your password.
|
||||
2. **Via Environment Variable (legacy seed)**: Setting `ADMIN_PASSWORD` in `.env` before the first start seeds the initial password instead of generating one. It's read only once, for existing deployments; a value left in `.env` is ignored after the node has been configured.
|
||||
|
||||
---
|
||||
|
||||
@@ -123,12 +126,14 @@ Use environment variables for:
|
||||
| -------------------- | --------------------------------- | ------------------------------------ |
|
||||
| `UPSTREAM_BASE_URL` | Upstream API endpoint | — |
|
||||
| `UPSTREAM_API_KEY` | Upstream API key | — |
|
||||
| `ADMIN_PASSWORD` | Dashboard password | (none) |
|
||||
| `ADMIN_PASSWORD` | Legacy seed for the dashboard password (otherwise generated + logged on first start) | (auto-generated) |
|
||||
| `ROUTSTR_SECRET_KEY` | Master key encrypting node secrets at rest. Auto-generated to a key file if unset | (auto-generated) |
|
||||
| `ROUTSTR_SECRET_KEY_FILE` | Path to the generated key file (used when `ROUTSTR_SECRET_KEY` is unset) | `routstr_secret.key` beside the database |
|
||||
| `DATABASE_URL` | Database connection string | `sqlite+aiosqlite:///keys.db` |
|
||||
| `NAME` | Node display name | `ARoutstrNode` |
|
||||
| `DESCRIPTION` | Node description | `A Routstr Node` |
|
||||
| `NPUB` | Nostr public key (bech32) | — |
|
||||
| `NSEC` | Nostr private key | — |
|
||||
| `NSEC` | Legacy seed for the Nostr private key (otherwise set from the admin UI) | — |
|
||||
| `ENABLE_ANALYTICS_SHARING` | Enable usage analytics sharing to Nostr | `true` |
|
||||
| `CASHU_MINTS` | Comma-separated mint URLs | `https://mint.minibits.cash/Bitcoin` |
|
||||
| `RECEIVE_LN_ADDRESS` | Lightning address for withdrawals | — |
|
||||
@@ -137,11 +142,27 @@ Use environment variables for:
|
||||
| `TOR_PROXY_URL` | SOCKS5 proxy for Tor | `socks5://127.0.0.1:9050` |
|
||||
| `CORS_ORIGINS` | Allowed CORS origins | `*` |
|
||||
| `RELAYS` | Nostr relays (comma-separated) | (default set) |
|
||||
| `MODEL_PATHS_REFRESH_INTERVAL_SECONDS` | How often to refresh `/v1/models/paths` discovery data; set `0` to pause the refresh (previously discovered paths keep being served) | `600` |
|
||||
| `ENABLE_MODEL_PATHS_REFRESH` | Kill switch for the background model-path refresh (OpenRouter endpoint fan-out) | `true` |
|
||||
|
||||
### Priority
|
||||
|
||||
Environment variables are read on startup. Dashboard settings override them and persist in the database. Once you change a setting in the dashboard, the env var is ignored for that setting.
|
||||
|
||||
### Secrets at Rest
|
||||
|
||||
The node's Nostr private key (`nsec`) is encrypted in the database using
|
||||
`ROUTSTR_SECRET_KEY`. You don't have to set it: if it's unset, the node generates a
|
||||
key on first start, writes it **beside the database** (the file named by
|
||||
`ROUTSTR_SECRET_KEY_FILE`, default `routstr_secret.key`) so it persists on the same
|
||||
volume as your data, and prints it once.
|
||||
|
||||
**Back up that key** — it lives on the same volume as your database, so include it
|
||||
in your backups. If it is lost or changed, previously encrypted secrets can't be
|
||||
decrypted and must be re-entered — there is no rotation. To keep the key off the
|
||||
data volume, set `ROUTSTR_SECRET_KEY` explicitly (an env value always takes
|
||||
precedence over the file). See also [Deployment](deployment.md).
|
||||
|
||||
---
|
||||
|
||||
## Models
|
||||
@@ -156,3 +177,8 @@ Manage which AI models you offer:
|
||||
- **Create aliases** — friendly names for models
|
||||
|
||||
See [Pricing](pricing.md) for per-model pricing strategies.
|
||||
|
||||
Model path discovery is refreshed in the background and exposed through
|
||||
`/v1/models/paths`. The response groups each client-visible model ID with the
|
||||
provider paths that may appear in chat-completion response metadata. Tune the
|
||||
refresh cadence with `MODEL_PATHS_REFRESH_INTERVAL_SECONDS`.
|
||||
|
||||
@@ -38,7 +38,6 @@ services:
|
||||
- routstr-data:/app/data
|
||||
environment:
|
||||
DATABASE_URL: "sqlite:////app/data/routstr.db"
|
||||
ADMIN_KEY: "your-secure-admin-key"
|
||||
LOG_LEVEL: "info"
|
||||
|
||||
volumes:
|
||||
@@ -87,6 +86,8 @@ services:
|
||||
- ./logs:/app/logs
|
||||
environment:
|
||||
- TOR_PROXY_URL=socks5://tor:9050
|
||||
# Keep the database (and the key file generated beside it) on the volume.
|
||||
- DATABASE_URL=sqlite:////app/data/routstr.db
|
||||
depends_on:
|
||||
- tor
|
||||
|
||||
@@ -125,8 +126,8 @@ services:
|
||||
- UPSTREAM_BASE_URL=https://api.openai.com/v1
|
||||
- UPSTREAM_API_KEY=sk-proj-...
|
||||
|
||||
# Secure the dashboard (recommended)
|
||||
- ADMIN_PASSWORD=your-secure-password
|
||||
# The admin password is generated and logged once on first start; set
|
||||
# ADMIN_PASSWORD here only as a legacy seed for an existing deployment.
|
||||
|
||||
# Node identity
|
||||
- NAME=My Provider Node
|
||||
@@ -134,6 +135,9 @@ services:
|
||||
|
||||
# Lightning withdrawals
|
||||
- RECEIVE_LN_ADDRESS=me@walletofsatoshi.com
|
||||
|
||||
# Keep the database (and the key file generated beside it) on the volume.
|
||||
- DATABASE_URL=sqlite:////app/data/routstr.db
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
```
|
||||
@@ -155,22 +159,36 @@ Example `.env`:
|
||||
```bash
|
||||
UPSTREAM_BASE_URL=https://api.openai.com/v1
|
||||
UPSTREAM_API_KEY=sk-proj-...
|
||||
ADMIN_PASSWORD=change-me
|
||||
# Keep the database (and the key file generated beside it) on the mounted volume.
|
||||
DATABASE_URL=sqlite:////app/data/routstr.db
|
||||
# Encrypts node secrets at rest. Optional — if unset, a key is generated next to
|
||||
# your database (on the same volume) and its file is named once for backup. Set
|
||||
# it explicitly to manage the key yourself.
|
||||
ROUTSTR_SECRET_KEY=
|
||||
NAME=My Provider Node
|
||||
RECEIVE_LN_ADDRESS=me@walletofsatoshi.com
|
||||
```
|
||||
|
||||
!!! note "Secret key persistence"
|
||||
If you leave `ROUTSTR_SECRET_KEY` unset, the node generates one and stores it
|
||||
as `routstr_secret.key` **next to your database**, so it persists on the same
|
||||
volume as your data — just include that volume in your backups. For stronger
|
||||
isolation (keeping the key off the data volume), set `ROUTSTR_SECRET_KEY` from
|
||||
a secrets manager instead.
|
||||
|
||||
See [Configuration](configuration.md) for all available options.
|
||||
|
||||
---
|
||||
|
||||
## Persistence
|
||||
|
||||
Routstr stores all data in `/app/data`:
|
||||
Point `DATABASE_URL` inside `/app/data` (as the examples above do) so everything
|
||||
Routstr persists lands on the mounted volume:
|
||||
|
||||
| Path | Contents |
|
||||
|------|----------|
|
||||
| `keys.db` | SQLite database (settings, API keys, sessions) |
|
||||
| `routstr.db` | SQLite database (settings, API keys, sessions) |
|
||||
| `routstr_secret.key` | Auto-generated master key, written beside the database when `ROUTSTR_SECRET_KEY` is unset |
|
||||
| `.wallet/` | Cashu wallet data (your Bitcoin!) |
|
||||
|
||||
!!! warning "Back Up Your Data"
|
||||
|
||||
@@ -29,19 +29,25 @@ In future versions, you'll be able to run a node that connects to other Routstr
|
||||
Create a `.env` file in the root of the project to store your secrets:
|
||||
|
||||
```bash
|
||||
# Initial Admin Password
|
||||
ADMIN_PASSWORD=mysecretpassword
|
||||
# Encrypts node secrets at rest. Optional — if unset, the node generates a key on
|
||||
# first start and prints it once (back it up).
|
||||
ROUTSTR_SECRET_KEY=
|
||||
|
||||
# Node Identity
|
||||
NAME="My AI Node"
|
||||
DESCRIPTION="Fast access to models"
|
||||
NSEC=yournsec
|
||||
|
||||
# Lightning Payouts
|
||||
RECEIVE_LN_ADDRESS=yourname@wallet.com
|
||||
|
||||
```
|
||||
|
||||
The admin password is generated and logged once on first start (read it from the
|
||||
logs to sign in), and your Nostr identity (`nsec`) is configured afterwards from
|
||||
the admin UI — both are stored encrypted in the database, not in `.env`.
|
||||
(`ADMIN_PASSWORD` / `NSEC` are still read once as a legacy seed for existing
|
||||
deployments.)
|
||||
|
||||
## 2. Start the Node
|
||||
|
||||
The recommended way to run Routstr is using Docker Compose, which handles the node, the UI, and optional services like Tor.
|
||||
@@ -72,7 +78,7 @@ docker compose up -d
|
||||
Open the **Admin Dashboard** at [http://localhost:8000/admin/](http://localhost:8000/admin/).
|
||||
|
||||
!!! note "Login"
|
||||
Use the `ADMIN_PASSWORD` you defined in your `.env` file to log in. If you didn't set one, the dashboard will prompt you to set one on first visit.
|
||||
On first start the node generates an admin password and logs it once — read it from the container logs to sign in. You can change it afterwards from **Settings** → **Security**.
|
||||
|
||||
### Connect Your AI Providers
|
||||
|
||||
|
||||
@@ -485,9 +485,9 @@ back to the requested model's pricing.
|
||||
- ~~End-to-end test with a real Tinfoil SDK client against a Routstr node with
|
||||
`TINFOIL_API_KEY` set.~~ Verified: both non-streaming (header) and streaming
|
||||
(trailer) responses include `model=<name>`.
|
||||
- Streaming requests: usage is delivered as an HTTP trailer. Currently the
|
||||
bearer path finalizes max-cost before streaming begins. Supporting streaming
|
||||
usage would require buffering the response (for X-Cashu) or a deferred
|
||||
finalization (for bearer).
|
||||
- Streaming trailer capture is implemented by buffering the encrypted response
|
||||
in `forward_with_trailer()` and then using the dedicated EHBP payment
|
||||
finalizers for bearer and X-Cashu requests. This provides actual-cost billing
|
||||
today, at the cost of full time-to-last-byte latency for streaming responses.
|
||||
- Whether Tinfoil's `/v1/responses` endpoint also returns usage metrics
|
||||
headers.
|
||||
headers or trailers.
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
"""add model paths table
|
||||
|
||||
Revision ID: 4e0c3d195a49
|
||||
Revises: 9c4d8e2f1a6b
|
||||
Create Date: 2026-07-24 21:14:39.062179
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
import sqlmodel
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "4e0c3d195a49"
|
||||
down_revision = "9c4d8e2f1a6b"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"model_paths",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("model_id", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
||||
sa.Column("path", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
||||
sa.Column("provider_slug", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
||||
sa.Column("provider_type", sqlmodel.sql.sqltypes.AutoString(), nullable=False),
|
||||
sa.Column("endpoint_tag", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
||||
sa.Column("endpoint_name", sqlmodel.sql.sqltypes.AutoString(), nullable=True),
|
||||
sa.Column("upstream_provider_id", sa.Integer(), nullable=False),
|
||||
sa.Column("updated_at", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.ForeignKeyConstraint(
|
||||
["upstream_provider_id"], ["upstream_providers.id"], ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint(
|
||||
"model_id",
|
||||
"path",
|
||||
"upstream_provider_id",
|
||||
name="uq_model_paths_model_path_provider",
|
||||
),
|
||||
)
|
||||
# No standalone index on model_id: the unique constraint's autoindex already
|
||||
# leads on model_id.
|
||||
op.create_index(
|
||||
op.f("ix_model_paths_upstream_provider_id"),
|
||||
"model_paths",
|
||||
["upstream_provider_id"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(op.f("ix_model_paths_upstream_provider_id"), table_name="model_paths")
|
||||
op.drop_table("model_paths")
|
||||
@@ -0,0 +1,62 @@
|
||||
"""add reservation release idempotency records
|
||||
|
||||
Revision ID: 7f2843d3f4e4
|
||||
Revises: fc4fa29630d2
|
||||
Create Date: 2026-07-24 02:06:06.066726
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "7f2843d3f4e4"
|
||||
down_revision = "fc4fa29630d2"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"reservation_releases",
|
||||
sa.Column("id", sa.String(), nullable=False),
|
||||
sa.Column("key_hash", sa.String(), nullable=False),
|
||||
sa.Column("billing_key_hash", sa.String(), nullable=False),
|
||||
sa.Column("reserved_msats", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"status", sa.String(), nullable=False, server_default="active"
|
||||
),
|
||||
sa.Column("created_at", sa.Integer(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reservation_releases_key_hash",
|
||||
"reservation_releases",
|
||||
["key_hash"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reservation_releases_billing_key_hash",
|
||||
"reservation_releases",
|
||||
["billing_key_hash"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_reservation_releases_status_created_at",
|
||||
"reservation_releases",
|
||||
["status", "created_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"ix_reservation_releases_status_created_at",
|
||||
table_name="reservation_releases",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_reservation_releases_billing_key_hash",
|
||||
table_name="reservation_releases",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_reservation_releases_key_hash",
|
||||
table_name="reservation_releases",
|
||||
)
|
||||
op.drop_table("reservation_releases")
|
||||
@@ -0,0 +1,47 @@
|
||||
"""repair missing fee payout checkpoint columns
|
||||
|
||||
Revision ID: 9c4d8e2f1a6b
|
||||
Revises: 7f2843d3f4e4
|
||||
Create Date: 2026-07-25 00:00:00.000000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "9c4d8e2f1a6b"
|
||||
down_revision = "7f2843d3f4e4"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Repair databases stamped past the original checkpoint migration."""
|
||||
conn = op.get_bind()
|
||||
columns = {
|
||||
column["name"] for column in sa.inspect(conn).get_columns("routstr_fees")
|
||||
}
|
||||
|
||||
if "payout_in_progress_msats" not in columns:
|
||||
op.add_column(
|
||||
"routstr_fees",
|
||||
sa.Column(
|
||||
"payout_in_progress_msats",
|
||||
sa.Integer(),
|
||||
nullable=False,
|
||||
server_default="0",
|
||||
),
|
||||
)
|
||||
|
||||
if "payout_started_at" not in columns:
|
||||
op.add_column(
|
||||
"routstr_fees",
|
||||
sa.Column("payout_started_at", sa.Integer(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# The preceding revision already expects both columns. This migration only
|
||||
# repairs schema drift, so downgrading it must preserve the expected schema.
|
||||
pass
|
||||
@@ -0,0 +1,37 @@
|
||||
"""add fee payout checkpoint
|
||||
|
||||
Revision ID: d7e8f9a0b1c2
|
||||
Revises: c6d7e8f9a0b1
|
||||
Create Date: 2026-07-18 00:00:00.000000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "d7e8f9a0b1c2"
|
||||
down_revision = "c6d7e8f9a0b1"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"routstr_fees",
|
||||
sa.Column(
|
||||
"payout_in_progress_msats",
|
||||
sa.Integer(),
|
||||
nullable=False,
|
||||
server_default="0",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"routstr_fees",
|
||||
sa.Column("payout_started_at", sa.Integer(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("routstr_fees", "payout_started_at")
|
||||
op.drop_column("routstr_fees", "payout_in_progress_msats")
|
||||
@@ -0,0 +1,51 @@
|
||||
"""add secrets table
|
||||
|
||||
Revision ID: fc4fa29630d2
|
||||
Revises: d7e8f9a0b1c2
|
||||
Create Date: 2026-07-23 00:00:00.000000
|
||||
|
||||
Creates the node-level singleton secret store (issue #553). Schema only; moving
|
||||
any legacy plaintext into the encrypted/hashed columns happens at bootstrap,
|
||||
where the live ROUTSTR_SECRET_KEY is available. ``nsec_state`` records the vault's
|
||||
ownership of the nsec (legacy | encrypted | cleared), so a cleared identity is
|
||||
never resurrected from a stale legacy ``NSEC`` env var / settings blob on the next
|
||||
boot.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
import sqlmodel
|
||||
from alembic import op
|
||||
|
||||
revision = "fc4fa29630d2"
|
||||
down_revision = "d7e8f9a0b1c2"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"secrets",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"admin_password_hash",
|
||||
sqlmodel.sql.sqltypes.AutoString(),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column(
|
||||
"encrypted_nsec",
|
||||
sqlmodel.sql.sqltypes.AutoString(),
|
||||
nullable=True,
|
||||
),
|
||||
sa.Column(
|
||||
"nsec_state",
|
||||
sqlmodel.sql.sqltypes.AutoString(),
|
||||
nullable=False,
|
||||
server_default="legacy",
|
||||
),
|
||||
sa.Column("updated_at", sa.Integer(), nullable=True),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("secrets")
|
||||
+30
-20
@@ -86,10 +86,12 @@ def get_provider_penalty(provider: "BaseUpstreamProvider") -> float:
|
||||
|
||||
def create_model_mappings(
|
||||
upstreams: list["BaseUpstreamProvider"],
|
||||
overrides_by_id: dict[str, tuple],
|
||||
disabled_model_ids: set[str],
|
||||
overrides_by_key: dict[tuple[str, int], tuple],
|
||||
disabled_model_keys: set[tuple[str, int]],
|
||||
) -> tuple[
|
||||
dict[str, "Model"], dict[str, list["BaseUpstreamProvider"]], dict[str, "Model"]
|
||||
dict[str, "Model"],
|
||||
dict[str, list[tuple["Model", "BaseUpstreamProvider"]]],
|
||||
dict[str, "Model"],
|
||||
]:
|
||||
"""Create optimal model mappings based on cost and provider preferences.
|
||||
|
||||
@@ -97,7 +99,9 @@ def create_model_mappings(
|
||||
and creates three mappings based on cost optimization:
|
||||
|
||||
1. model_instances: alias -> Model (all model aliases mapped to their Model objects)
|
||||
2. provider_map: alias -> List[UpstreamProvider] (sorted list of providers for each alias)
|
||||
2. provider_map: alias -> List[(Model, UpstreamProvider)] (sorted candidate
|
||||
list for each alias; each provider is paired with ITS OWN model so
|
||||
failover can forward and bill the candidate that actually serves)
|
||||
3. unique_models: base_id -> Model (unique models without provider prefixes)
|
||||
|
||||
The algorithm:
|
||||
@@ -107,8 +111,9 @@ def create_model_mappings(
|
||||
|
||||
Args:
|
||||
upstreams: List of all upstream provider instances
|
||||
overrides_by_id: Dict of model overrides from database {model_id: (ModelRow, fee)}
|
||||
disabled_model_ids: Set of model IDs that should be excluded
|
||||
overrides_by_key: Dict of model overrides from database
|
||||
{(model_id_lower, upstream_provider_id): (ModelRow, fee)}
|
||||
disabled_model_keys: Set of provider-scoped model keys that should be excluded
|
||||
|
||||
Returns:
|
||||
Tuple of (model_instances, provider_map, unique_models)
|
||||
@@ -179,14 +184,22 @@ def create_model_mappings(
|
||||
"""Process all models from a given provider."""
|
||||
upstream_prefix = getattr(upstream, "upstream_name", None)
|
||||
provider_key = get_provider_identity(upstream)
|
||||
upstream_db_id = getattr(upstream, "db_id", None)
|
||||
|
||||
for model in upstream.get_cached_models():
|
||||
if not model.enabled or model.id in disabled_model_ids:
|
||||
model_key = (
|
||||
(model.id.lower(), upstream_db_id)
|
||||
if isinstance(upstream_db_id, int)
|
||||
else None
|
||||
)
|
||||
if not model.enabled or (
|
||||
model_key is not None and model_key in disabled_model_keys
|
||||
):
|
||||
continue
|
||||
|
||||
# Apply overrides if present
|
||||
if model.id in overrides_by_id:
|
||||
override_row, provider_fee = overrides_by_id[model.id]
|
||||
# Apply overrides only for this provider's model row.
|
||||
if model_key is not None and model_key in overrides_by_key:
|
||||
override_row, provider_fee = overrides_by_key[model_key]
|
||||
model_to_use = _row_to_model(
|
||||
override_row, apply_provider_fee=True, provider_fee=provider_fee
|
||||
)
|
||||
@@ -237,13 +250,10 @@ def create_model_mappings(
|
||||
|
||||
# Include enabled DB overrides even when provider discovery misses models.
|
||||
# This is important for deployment-based providers like Azure.
|
||||
for model_id, override_data in overrides_by_id.items():
|
||||
if model_id in disabled_model_ids:
|
||||
for (model_id, upstream_provider_id), override_data in overrides_by_key.items():
|
||||
if (model_id, upstream_provider_id) in disabled_model_keys:
|
||||
continue
|
||||
override_row, provider_fee = override_data
|
||||
upstream_provider_id = getattr(override_row, "upstream_provider_id", None)
|
||||
if not isinstance(upstream_provider_id, int):
|
||||
continue
|
||||
|
||||
upstream_for_override = providers_by_db_id.get(upstream_provider_id)
|
||||
if upstream_for_override is None:
|
||||
@@ -321,7 +331,7 @@ def create_model_mappings(
|
||||
|
||||
# Sort candidates and build final maps
|
||||
model_instances: dict[str, "Model"] = {}
|
||||
provider_map: dict[str, list["BaseUpstreamProvider"]] = {}
|
||||
provider_map: dict[str, list[tuple["Model", "BaseUpstreamProvider"]]] = {}
|
||||
|
||||
def alias_priority(model: "Model", alias: str) -> int:
|
||||
"""Rank how strong the mapping of alias->model is.
|
||||
@@ -368,13 +378,13 @@ def create_model_mappings(
|
||||
|
||||
best_model, best_provider = items[0]
|
||||
model_instances[alias] = best_model
|
||||
provider_map[alias] = [p for _, p in items]
|
||||
provider_map[alias] = list(items)
|
||||
|
||||
# Log provider distribution (using top provider for stats)
|
||||
provider_counts: dict[str, int] = {}
|
||||
for providers in provider_map.values():
|
||||
if providers:
|
||||
provider = providers[0]
|
||||
for candidate_list in provider_map.values():
|
||||
if candidate_list:
|
||||
provider = candidate_list[0][1]
|
||||
provider_name = getattr(provider, "upstream_name", "unknown")
|
||||
provider_counts[provider_name] = provider_counts.get(provider_name, 0) + 1
|
||||
|
||||
|
||||
+378
-151
@@ -3,16 +3,25 @@ import hashlib
|
||||
import math
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import case
|
||||
from sqlalchemy import case, inspect
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlmodel import col, select, update
|
||||
|
||||
from .core import get_logger
|
||||
from .core.db import ApiKey, AsyncSession, accumulate_routstr_fee
|
||||
from .core.db import (
|
||||
ApiKey,
|
||||
AsyncSession,
|
||||
ReservationRelease,
|
||||
accumulate_routstr_fee,
|
||||
create_session,
|
||||
)
|
||||
from .core.settings import settings
|
||||
from .payment.cost_calculation import (
|
||||
CostData,
|
||||
@@ -26,15 +35,40 @@ from .wallet import (
|
||||
deserialize_token_from_string,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .payment.models import Model
|
||||
|
||||
logger = get_logger(__name__)
|
||||
payments_logger = get_logger("routstr.payments")
|
||||
|
||||
# Routstr platform fee constants
|
||||
ROUTSTR_FEE_PERCENT: float = 2.1
|
||||
ROUTSTR_LN_ADDRESS: str = "npub130mznv74rxs032peqym6g3wqavh472623mt3z5w73xq9r6qqdufs7ql29s@npub.cash"
|
||||
ROUTSTR_LN_ADDRESS: str = (
|
||||
"npub130mznv74rxs032peqym6g3wqavh472623mt3z5w73xq9r6qqdufs7ql29s@npub.cash"
|
||||
)
|
||||
ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS: int = 900
|
||||
ROUTSTR_FEE_DEFAULT_PAYOUT: int = 200
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReservationSnapshot:
|
||||
release_id: str
|
||||
key_hash: str
|
||||
billing_key_hash: str
|
||||
reserved_msats: int
|
||||
|
||||
|
||||
_current_reservation: ContextVar[ReservationSnapshot | None] = ContextVar(
|
||||
"current_billing_reservation", default=None
|
||||
)
|
||||
|
||||
|
||||
def _clear_current_reservation(snapshot: ReservationSnapshot) -> None:
|
||||
current = _current_reservation.get()
|
||||
if current is not None and current.release_id == snapshot.release_id:
|
||||
_current_reservation.set(None)
|
||||
|
||||
|
||||
# TODO: implement prepaid api key (not like it was before)
|
||||
# PREPAID_API_KEY = os.environ.get("PREPAID_API_KEY", None)
|
||||
# PREPAID_BALANCE = int(os.environ.get("PREPAID_BALANCE", "0")) * 1000 # Convert to msats
|
||||
@@ -592,6 +626,16 @@ async def pay_for_request(
|
||||
},
|
||||
)
|
||||
|
||||
# Create the durable reservation identity before changing aggregate balances.
|
||||
# The row and balance updates commit together, so every reserved amount has one
|
||||
# owner that can reach exactly one terminal state.
|
||||
reservation = ReservationSnapshot(
|
||||
release_id=uuid.uuid4().hex,
|
||||
key_hash=key.hashed_key,
|
||||
billing_key_hash=billing_key.hashed_key,
|
||||
reserved_msats=cost_per_request,
|
||||
)
|
||||
|
||||
# Charge the base cost for the request atomically to avoid race conditions
|
||||
reserved_at_now = int(time.time())
|
||||
stmt = (
|
||||
@@ -653,22 +697,83 @@ async def pay_for_request(
|
||||
child_result = await session.exec(child_stmt) # type: ignore[call-overload]
|
||||
|
||||
if child_result.rowcount == 0:
|
||||
# Build the error before rollback expires ORM attributes.
|
||||
limit_message = (
|
||||
f"Balance limit exceeded: {key.balance_limit} mSats limit. "
|
||||
f"{key.total_spent} already spent ({key.reserved_balance} reserved), "
|
||||
f"{cost_per_request} required for this request."
|
||||
)
|
||||
# The parent reservation update already ran in this transaction.
|
||||
# Roll it back before failover code attempts to restore the previous
|
||||
# reservation; otherwise that later commit can persist both updates.
|
||||
await session.rollback()
|
||||
raise HTTPException(
|
||||
status_code=402,
|
||||
detail={
|
||||
"error": {
|
||||
"message": f"Balance limit exceeded: {key.balance_limit} mSats limit. {key.total_spent} already spent ({key.reserved_balance} reserved), {cost_per_request} required for this request.",
|
||||
"message": limit_message,
|
||||
"type": "insufficient_quota",
|
||||
"code": "balance_limit_exceeded",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
await session.commit()
|
||||
session.add(
|
||||
ReservationRelease(
|
||||
id=reservation.release_id,
|
||||
key_hash=reservation.key_hash,
|
||||
billing_key_hash=reservation.billing_key_hash,
|
||||
reserved_msats=reservation.reserved_msats,
|
||||
status="active",
|
||||
)
|
||||
)
|
||||
# Publish the identity before commit. If the commit succeeds but its
|
||||
# acknowledgement is interrupted, exact cleanup can still recover the
|
||||
# durable row. A definitely failed commit is harmless because every
|
||||
# terminal transition validates that row before touching balances.
|
||||
_current_reservation.set(reservation)
|
||||
try:
|
||||
await session.commit()
|
||||
except BaseException:
|
||||
# The database may have committed even if acknowledgement was cancelled
|
||||
# or the connection failed. Reconcile using a fresh transaction and the
|
||||
# exact durable identity; no upstream request has started yet.
|
||||
try:
|
||||
await session.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
async with create_session() as cleanup_session:
|
||||
record = await cleanup_session.get(
|
||||
ReservationRelease, reservation.release_id
|
||||
)
|
||||
if record is not None and record.status == "active":
|
||||
await _transition_reservation_to_released(
|
||||
reservation,
|
||||
cleanup_session,
|
||||
decrement_requests=True,
|
||||
idempotent_success=True,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to reconcile ambiguous reservation commit",
|
||||
extra={"reservation_id": reservation.release_id},
|
||||
)
|
||||
finally:
|
||||
_clear_current_reservation(reservation)
|
||||
raise
|
||||
|
||||
await session.refresh(billing_key)
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
await session.refresh(key)
|
||||
try:
|
||||
await session.refresh(billing_key)
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
await session.refresh(key)
|
||||
except Exception:
|
||||
# The reservation transaction is already committed and durable. Logging
|
||||
# refresh failures must not make the caller treat it as unreserved.
|
||||
logger.exception(
|
||||
"Reservation committed but post-commit refresh failed",
|
||||
extra={"reservation_id": reservation.release_id},
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Payment processed successfully",
|
||||
@@ -698,89 +803,206 @@ async def pay_for_request(
|
||||
|
||||
|
||||
async def revert_pay_for_request(
|
||||
key: ApiKey, session: AsyncSession, cost_per_request: int
|
||||
key: ApiKey,
|
||||
session: AsyncSession,
|
||||
cost_per_request: int,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> bool:
|
||||
"""Revert a previously reserved payment. Returns True if revert succeeded,
|
||||
False if the reservation was already released (prevents negative reserved_balance)."""
|
||||
billing_key = await get_billing_key(key, session)
|
||||
|
||||
# Keep reserved_at while other reservations remain
|
||||
cleared_reserved_at = case(
|
||||
(col(ApiKey.reserved_balance) - cost_per_request > 0, col(ApiKey.reserved_at)),
|
||||
else_=None,
|
||||
"""Revert the current request's durable reservation exactly once."""
|
||||
snapshot = reservation_snapshot or await get_reservation_snapshot(key, session)
|
||||
await _validate_reservation_snapshot(key, snapshot, session, require_active=False)
|
||||
if cost_per_request != snapshot.reserved_msats:
|
||||
return False
|
||||
return await _transition_reservation_to_released(
|
||||
snapshot,
|
||||
session,
|
||||
decrement_requests=True,
|
||||
idempotent_success=False,
|
||||
)
|
||||
|
||||
stmt = (
|
||||
|
||||
async def _validate_reservation_snapshot(
|
||||
key: ApiKey,
|
||||
snapshot: ReservationSnapshot,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
require_active: bool = True,
|
||||
) -> None:
|
||||
"""Reject cross-request or forged reservation handles before any mutation."""
|
||||
state = inspect(key)
|
||||
identity = state.identity if state is not None else None
|
||||
key_hash = str(identity[0]) if identity else key.__dict__.get("hashed_key")
|
||||
if snapshot.key_hash != key_hash:
|
||||
raise RuntimeError("Billing reservation does not belong to this key")
|
||||
|
||||
persisted_key = await session.get(ApiKey, snapshot.key_hash)
|
||||
if persisted_key is None:
|
||||
raise RuntimeError("Billing reservation key no longer exists")
|
||||
expected_billing_hash = persisted_key.parent_key_hash or persisted_key.hashed_key
|
||||
if snapshot.billing_key_hash != expected_billing_hash:
|
||||
raise RuntimeError("Billing reservation does not belong to this billing key")
|
||||
|
||||
record = await session.get(ReservationRelease, snapshot.release_id)
|
||||
if (
|
||||
record is None
|
||||
or (require_active and record.status != "active")
|
||||
or record.key_hash != snapshot.key_hash
|
||||
or record.billing_key_hash != snapshot.billing_key_hash
|
||||
or record.reserved_msats != snapshot.reserved_msats
|
||||
):
|
||||
raise RuntimeError("Billing reservation record does not match the request")
|
||||
|
||||
|
||||
async def get_reservation_snapshot(
|
||||
key: ApiKey, session: AsyncSession
|
||||
) -> ReservationSnapshot:
|
||||
"""Return the durable reservation created for the current request."""
|
||||
snapshot = _current_reservation.get()
|
||||
if snapshot is None:
|
||||
raise RuntimeError("No billing reservation is associated with this request")
|
||||
await _validate_reservation_snapshot(key, snapshot, session)
|
||||
return snapshot
|
||||
|
||||
|
||||
async def _transition_reservation_to_released(
|
||||
snapshot: ReservationSnapshot,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
decrement_requests: bool,
|
||||
idempotent_success: bool,
|
||||
) -> bool:
|
||||
transition = (
|
||||
update(ReservationRelease)
|
||||
.where(col(ReservationRelease.id) == snapshot.release_id)
|
||||
.where(col(ReservationRelease.status) == "active")
|
||||
.where(col(ReservationRelease.key_hash) == snapshot.key_hash)
|
||||
.where(col(ReservationRelease.billing_key_hash) == snapshot.billing_key_hash)
|
||||
.where(col(ReservationRelease.reserved_msats) == snapshot.reserved_msats)
|
||||
.values(status="released")
|
||||
)
|
||||
transition_result = await session.exec(transition) # type: ignore[call-overload]
|
||||
if transition_result.rowcount != 1:
|
||||
await session.rollback()
|
||||
existing = await session.get(ReservationRelease, snapshot.release_id)
|
||||
return bool(
|
||||
idempotent_success
|
||||
and existing is not None
|
||||
and existing.status == "released"
|
||||
and existing.key_hash == snapshot.key_hash
|
||||
and existing.billing_key_hash == snapshot.billing_key_hash
|
||||
and existing.reserved_msats == snapshot.reserved_msats
|
||||
)
|
||||
|
||||
values: dict[str, object] = {
|
||||
"reserved_balance": col(ApiKey.reserved_balance) - snapshot.reserved_msats,
|
||||
"reserved_at": case(
|
||||
(
|
||||
col(ApiKey.reserved_balance) - snapshot.reserved_msats > 0,
|
||||
col(ApiKey.reserved_at),
|
||||
),
|
||||
else_=None,
|
||||
),
|
||||
}
|
||||
if decrement_requests:
|
||||
values["total_requests"] = col(ApiKey.total_requests) - 1
|
||||
|
||||
release_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
|
||||
.where(col(ApiKey.reserved_balance) >= cost_per_request)
|
||||
.values(
|
||||
reserved_balance=col(ApiKey.reserved_balance) - cost_per_request,
|
||||
reserved_at=cleared_reserved_at,
|
||||
total_requests=col(ApiKey.total_requests) - 1,
|
||||
)
|
||||
.where(col(ApiKey.hashed_key) == snapshot.billing_key_hash)
|
||||
.where(col(ApiKey.reserved_balance) >= snapshot.reserved_msats)
|
||||
.values(**values)
|
||||
)
|
||||
result = await session.exec(release_stmt) # type: ignore[call-overload]
|
||||
if result.rowcount != 1:
|
||||
await session.rollback()
|
||||
return False
|
||||
|
||||
result = await session.exec(stmt) # type: ignore[call-overload]
|
||||
|
||||
# Also decrement total_requests and reserved_balance on the child key if it's different
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
child_stmt = (
|
||||
if snapshot.billing_key_hash != snapshot.key_hash:
|
||||
child_release_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == key.hashed_key)
|
||||
.where(col(ApiKey.reserved_balance) >= cost_per_request)
|
||||
.values(
|
||||
total_requests=col(ApiKey.total_requests) - 1,
|
||||
reserved_balance=col(ApiKey.reserved_balance) - cost_per_request,
|
||||
reserved_at=cleared_reserved_at,
|
||||
)
|
||||
.where(col(ApiKey.hashed_key) == snapshot.key_hash)
|
||||
.where(col(ApiKey.reserved_balance) >= snapshot.reserved_msats)
|
||||
.values(**values)
|
||||
)
|
||||
await session.exec(child_stmt) # type: ignore[call-overload]
|
||||
child_result = await session.exec( # type: ignore[call-overload]
|
||||
child_release_stmt
|
||||
)
|
||||
if child_result.rowcount != 1:
|
||||
await session.rollback()
|
||||
return False
|
||||
|
||||
await session.commit()
|
||||
if result.rowcount == 0:
|
||||
logger.warning(
|
||||
"Revert skipped - reservation already released (no-op to prevent negative reserved_balance)",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"cost_to_revert": cost_per_request,
|
||||
"current_reserved_balance": billing_key.reserved_balance,
|
||||
},
|
||||
)
|
||||
return False
|
||||
await session.refresh(billing_key)
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
await session.refresh(key)
|
||||
payments_logger.info(
|
||||
"REVERT",
|
||||
extra={
|
||||
"event": "revert",
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"cost_reverted": cost_per_request,
|
||||
"balance": billing_key.balance,
|
||||
"reserved_balance": billing_key.reserved_balance,
|
||||
},
|
||||
)
|
||||
_clear_current_reservation(snapshot)
|
||||
return True
|
||||
|
||||
|
||||
async def release_reservation(
|
||||
snapshot: ReservationSnapshot,
|
||||
session: AsyncSession,
|
||||
reserved_msats: int,
|
||||
) -> bool:
|
||||
"""Release one durable reservation exactly once without charging."""
|
||||
if reserved_msats <= 0 or reserved_msats != snapshot.reserved_msats:
|
||||
return False
|
||||
return await _transition_reservation_to_released(
|
||||
snapshot,
|
||||
session,
|
||||
decrement_requests=False,
|
||||
idempotent_success=True,
|
||||
)
|
||||
|
||||
|
||||
async def _claim_reservation_for_charge(
|
||||
snapshot: ReservationSnapshot, session: AsyncSession
|
||||
) -> bool:
|
||||
"""Claim an active reservation in the caller's charge transaction."""
|
||||
statement = (
|
||||
update(ReservationRelease)
|
||||
.where(col(ReservationRelease.id) == snapshot.release_id)
|
||||
.where(col(ReservationRelease.status) == "active")
|
||||
.where(col(ReservationRelease.key_hash) == snapshot.key_hash)
|
||||
.where(col(ReservationRelease.billing_key_hash) == snapshot.billing_key_hash)
|
||||
.where(col(ReservationRelease.reserved_msats) == snapshot.reserved_msats)
|
||||
.values(status="charged")
|
||||
)
|
||||
result = await session.exec(statement) # type: ignore[call-overload]
|
||||
if result.rowcount == 1:
|
||||
_clear_current_reservation(snapshot)
|
||||
return True
|
||||
|
||||
await session.rollback()
|
||||
return False
|
||||
|
||||
|
||||
async def adjust_payment_for_tokens(
|
||||
key: ApiKey,
|
||||
response_data: dict,
|
||||
session: AsyncSession,
|
||||
deducted_max_cost: int,
|
||||
model_obj: "Model | None" = None,
|
||||
provider_fee: float | None = None,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Adjusts the payment based on token usage in the response.
|
||||
This is called after the initial payment and the upstream request is complete.
|
||||
Returns cost data to be included in the response.
|
||||
|
||||
``model_obj`` is the model that actually served the request; it is passed
|
||||
through to ``calculate_cost`` so billing uses the serving candidate's
|
||||
pricing instead of re-deriving it from the response's model string.
|
||||
|
||||
The response's usage object is normalized with the default union parser in
|
||||
``calculate_cost``.
|
||||
"""
|
||||
billing_key = await get_billing_key(key, session)
|
||||
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
|
||||
await _validate_reservation_snapshot(
|
||||
key, reservation, session, require_active=False
|
||||
)
|
||||
# The persisted amount is authoritative if request-level minimum pricing
|
||||
# changed the caller's original estimate.
|
||||
deducted_max_cost = reservation.reserved_msats
|
||||
model = response_data.get("model", "unknown")
|
||||
|
||||
logger.debug(
|
||||
@@ -796,50 +1018,21 @@ async def adjust_payment_for_tokens(
|
||||
)
|
||||
|
||||
async def release_reservation_only() -> None:
|
||||
"""Fallback to release reservation without charging when main update fails."""
|
||||
"""Fallback to release this request's reservation without charging."""
|
||||
try:
|
||||
release_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
|
||||
.where(col(ApiKey.reserved_balance) >= deducted_max_cost)
|
||||
.values(
|
||||
reserved_balance=col(ApiKey.reserved_balance) - deducted_max_cost
|
||||
)
|
||||
released = await release_reservation(
|
||||
reservation, session, reservation.reserved_msats
|
||||
)
|
||||
logger.warning(
|
||||
"Released reservation without charging (fallback)"
|
||||
if released
|
||||
else "Reservation was already finalized; fallback skipped",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"deducted_max_cost": deducted_max_cost,
|
||||
},
|
||||
)
|
||||
result = await session.exec(release_stmt) # type: ignore[call-overload]
|
||||
|
||||
# Also release on child key if it's different
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
child_release_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == key.hashed_key)
|
||||
.where(col(ApiKey.reserved_balance) >= deducted_max_cost)
|
||||
.values(
|
||||
reserved_balance=col(ApiKey.reserved_balance)
|
||||
- deducted_max_cost
|
||||
)
|
||||
)
|
||||
await session.exec(child_release_stmt) # type: ignore[call-overload]
|
||||
|
||||
await session.commit()
|
||||
if result.rowcount == 0: # type: ignore[union-attr]
|
||||
logger.warning(
|
||||
"Release reservation skipped - already released (no-op to prevent negative reserved_balance)",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"deducted_max_cost": deducted_max_cost,
|
||||
},
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Released reservation without charging (fallback)",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"deducted_max_cost": deducted_max_cost,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Failed to release reservation in fallback",
|
||||
@@ -861,7 +1054,17 @@ async def adjust_payment_for_tokens(
|
||||
extra={"error": str(e), "fee_msats": fee_msats},
|
||||
)
|
||||
|
||||
match await calculate_cost(response_data, deducted_max_cost):
|
||||
calculated_cost = await calculate_cost(
|
||||
response_data, deducted_max_cost, model_obj, provider_fee
|
||||
)
|
||||
if not isinstance(calculated_cost, CostDataError):
|
||||
if not await _claim_reservation_for_charge(reservation, session):
|
||||
# A prior charge or release already owns this reservation. Returning
|
||||
# the calculated metadata is safe; the aggregate balances must not
|
||||
# be modified a second time.
|
||||
return calculated_cost.dict()
|
||||
|
||||
match calculated_cost:
|
||||
case MaxCostData() as cost:
|
||||
logger.debug(
|
||||
"Using max cost data (no token adjustment)",
|
||||
@@ -889,8 +1092,10 @@ async def adjust_payment_for_tokens(
|
||||
)
|
||||
|
||||
safe_reserved = case(
|
||||
(col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost),
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
|
||||
@@ -908,8 +1113,10 @@ async def adjust_payment_for_tokens(
|
||||
# Also update total_spent and reserved_balance on the child key if it's different
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
child_safe_reserved = case(
|
||||
(col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost),
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
child_stmt = (
|
||||
@@ -1019,8 +1226,10 @@ async def adjust_payment_for_tokens(
|
||||
)
|
||||
|
||||
exact_safe_reserved = case(
|
||||
(col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost),
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
|
||||
@@ -1038,8 +1247,10 @@ async def adjust_payment_for_tokens(
|
||||
# Also update total_spent and reserved_balance on the child key if it's different
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
child_exact_safe_reserved = case(
|
||||
(col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost),
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
child_stmt = (
|
||||
@@ -1078,31 +1289,45 @@ async def adjust_payment_for_tokens(
|
||||
|
||||
# actual cost exceeded discounted reservation (due to tolerance_percentage)
|
||||
if cost_difference > 0:
|
||||
# Always release the reservation and charge min(actual_cost, balance).
|
||||
# CASE expressions keep this atomic and safe even when the
|
||||
# stale-reservation sweeper has already released the reservation.
|
||||
chargeable = case(
|
||||
(col(ApiKey.balance) >= total_cost_msats, total_cost_msats),
|
||||
else_=col(ApiKey.balance),
|
||||
)
|
||||
overrun_safe_reserved = case(
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
|
||||
finalize_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
|
||||
.values(
|
||||
reserved_balance=overrun_safe_reserved,
|
||||
balance=col(ApiKey.balance) - chargeable,
|
||||
total_spent=col(ApiKey.total_spent) + chargeable,
|
||||
# Lock the billing row so the parent and child record the same
|
||||
# database-determined charge under concurrent finalizations.
|
||||
actual_charge_msats = 0
|
||||
for attempt in range(5):
|
||||
locked_billing_key = (
|
||||
await session.exec(
|
||||
select(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
)
|
||||
).one()
|
||||
observed_balance = locked_billing_key.balance
|
||||
actual_charge_msats = min(observed_balance, total_cost_msats)
|
||||
overrun_safe_reserved = case(
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
)
|
||||
await session.exec(finalize_stmt) # type: ignore[call-overload]
|
||||
finalize_result = await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == billing_key.hashed_key)
|
||||
.where(col(ApiKey.balance) == observed_balance)
|
||||
.values(
|
||||
reserved_balance=overrun_safe_reserved,
|
||||
balance=col(ApiKey.balance) - actual_charge_msats,
|
||||
total_spent=col(ApiKey.total_spent) + actual_charge_msats,
|
||||
)
|
||||
)
|
||||
if finalize_result.rowcount == 1:
|
||||
break
|
||||
await session.rollback()
|
||||
if not await _claim_reservation_for_charge(reservation, session):
|
||||
return cost.dict()
|
||||
else:
|
||||
await session.rollback()
|
||||
raise RuntimeError("Could not atomically finalize cost overrun")
|
||||
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
child_stmt = (
|
||||
@@ -1110,7 +1335,7 @@ async def adjust_payment_for_tokens(
|
||||
.where(col(ApiKey.hashed_key) == key.hashed_key)
|
||||
.values(
|
||||
reserved_balance=overrun_safe_reserved,
|
||||
total_spent=col(ApiKey.total_spent) + min(billing_key.balance, total_cost_msats),
|
||||
total_spent=col(ApiKey.total_spent) + actual_charge_msats,
|
||||
)
|
||||
)
|
||||
await session.exec(child_stmt) # type: ignore[call-overload]
|
||||
@@ -1120,18 +1345,18 @@ async def adjust_payment_for_tokens(
|
||||
await session.refresh(billing_key)
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
await session.refresh(key)
|
||||
cost.total_msats = total_cost_msats
|
||||
cost.total_msats = actual_charge_msats
|
||||
logger.info(
|
||||
"Finalized payment with additional charge",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"charged_amount": total_cost_msats,
|
||||
"charged_amount": actual_charge_msats,
|
||||
"new_balance": billing_key.balance,
|
||||
"model": model,
|
||||
},
|
||||
)
|
||||
await _accumulate_fee(total_cost_msats)
|
||||
await _accumulate_fee(actual_charge_msats)
|
||||
payments_logger.info(
|
||||
"FINALIZE",
|
||||
extra={
|
||||
@@ -1140,7 +1365,7 @@ async def adjust_payment_for_tokens(
|
||||
"billing_key_hash": billing_key.hashed_key[:8] + "...",
|
||||
"model": model,
|
||||
"cost_reserved": deducted_max_cost,
|
||||
"cost_charged": total_cost_msats,
|
||||
"cost_charged": actual_charge_msats,
|
||||
"input_tokens": cost.input_tokens,
|
||||
"output_tokens": cost.output_tokens,
|
||||
"balance": billing_key.balance,
|
||||
@@ -1180,8 +1405,10 @@ async def adjust_payment_for_tokens(
|
||||
)
|
||||
|
||||
refund_safe_reserved = case(
|
||||
(col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost),
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
|
||||
@@ -1199,8 +1426,10 @@ async def adjust_payment_for_tokens(
|
||||
# Also update total_spent and reserved_balance on the child key if it's different
|
||||
if billing_key.hashed_key != key.hashed_key:
|
||||
child_refund_safe_reserved = case(
|
||||
(col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost),
|
||||
(
|
||||
col(ApiKey.reserved_balance) >= deducted_max_cost,
|
||||
col(ApiKey.reserved_balance) - deducted_max_cost,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
child_stmt = (
|
||||
@@ -1375,9 +1604,7 @@ async def periodic_dead_key_prune() -> None:
|
||||
|
||||
try:
|
||||
async with create_session() as session:
|
||||
await prune_dead_api_keys(
|
||||
session, settings.dead_key_min_age_seconds
|
||||
)
|
||||
await prune_dead_api_keys(session, settings.dead_key_min_age_seconds)
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
|
||||
+56
-27
@@ -7,7 +7,7 @@ from typing import Annotated, NoReturn
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlmodel import col, or_, select, update
|
||||
from sqlmodel import col, select, update
|
||||
|
||||
from .auth import get_billing_key, validate_bearer_key
|
||||
from .core.db import (
|
||||
@@ -15,7 +15,10 @@ from .core.db import (
|
||||
AsyncSession,
|
||||
CashuTransaction,
|
||||
get_session,
|
||||
store_cashu_transaction,
|
||||
release_stale_reservations,
|
||||
)
|
||||
from .core.db import (
|
||||
store_cashu_transaction_with_retry as store_cashu_transaction,
|
||||
)
|
||||
from .core.logging import get_logger
|
||||
from .core.settings import settings
|
||||
@@ -106,13 +109,19 @@ async def account_info(
|
||||
# Note: validate_bearer_key already supports refund_address and key_expiry_time params
|
||||
|
||||
|
||||
@router.get("/create")
|
||||
async def create_balance(
|
||||
class BalanceCreateRequest(BaseModel):
|
||||
initial_balance_token: str
|
||||
balance_limit: int | None = None
|
||||
balance_limit_reset: str | None = None
|
||||
validity_date: int | None = None
|
||||
|
||||
|
||||
async def _create_balance(
|
||||
initial_balance_token: str,
|
||||
balance_limit: int | None = None,
|
||||
balance_limit_reset: str | None = None,
|
||||
validity_date: int | None = None,
|
||||
session: AsyncSession = Depends(get_session),
|
||||
balance_limit: int | None,
|
||||
balance_limit_reset: str | None,
|
||||
validity_date: int | None,
|
||||
session: AsyncSession,
|
||||
) -> dict:
|
||||
key = await validate_bearer_key(initial_balance_token, session)
|
||||
|
||||
@@ -132,6 +141,37 @@ async def create_balance(
|
||||
}
|
||||
|
||||
|
||||
@router.post("/create")
|
||||
async def create_balance_from_body(
|
||||
payload: BalanceCreateRequest,
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> dict:
|
||||
return await _create_balance(
|
||||
payload.initial_balance_token,
|
||||
payload.balance_limit,
|
||||
payload.balance_limit_reset,
|
||||
payload.validity_date,
|
||||
session,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/create")
|
||||
async def create_balance(
|
||||
initial_balance_token: str,
|
||||
balance_limit: int | None = None,
|
||||
balance_limit_reset: str | None = None,
|
||||
validity_date: int | None = None,
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> dict:
|
||||
return await _create_balance(
|
||||
initial_balance_token,
|
||||
balance_limit,
|
||||
balance_limit_reset,
|
||||
validity_date,
|
||||
session,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/info")
|
||||
async def wallet_info(
|
||||
key: ApiKey = Depends(get_key_from_header),
|
||||
@@ -321,30 +361,19 @@ async def refund_wallet_endpoint(
|
||||
)
|
||||
|
||||
if key.reserved_balance > 0:
|
||||
# Release the reservation if it is stale
|
||||
cutoff = int(time.time()) - settings.stale_reservation_timeout_seconds
|
||||
stale_release_stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == key.hashed_key)
|
||||
.where(col(ApiKey.reserved_balance) > 0)
|
||||
.where(
|
||||
or_(
|
||||
col(ApiKey.reserved_at).is_(None),
|
||||
col(ApiKey.reserved_at) < cutoff,
|
||||
)
|
||||
)
|
||||
.values(reserved_balance=0, reserved_at=None)
|
||||
# Release only durable reservations old enough to be stale. A newer
|
||||
# request on the same aggregate balance must remain reserved.
|
||||
await release_stale_reservations(
|
||||
session,
|
||||
settings.stale_reservation_timeout_seconds,
|
||||
key_hash=key.hashed_key,
|
||||
)
|
||||
stale_result = await session.exec(stale_release_stmt) # type: ignore[call-overload]
|
||||
await session.commit()
|
||||
|
||||
if stale_result.rowcount == 0:
|
||||
await session.refresh(key)
|
||||
if key.reserved_balance > 0:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Cannot refund key. There are ongoing requests for this api key.",
|
||||
)
|
||||
|
||||
await session.refresh(key)
|
||||
logger.warning(
|
||||
"refund_wallet_endpoint: released stale reservation before refund",
|
||||
extra={
|
||||
|
||||
+104
-46
@@ -20,6 +20,7 @@ from ..wallet import (
|
||||
send_token,
|
||||
slow_filter_spend_proofs,
|
||||
)
|
||||
from . import vault
|
||||
from .db import (
|
||||
ApiKey,
|
||||
CashuTransaction,
|
||||
@@ -28,11 +29,17 @@ from .db import (
|
||||
ModelRow,
|
||||
UpstreamProviderRow,
|
||||
create_session,
|
||||
get_secret,
|
||||
set_admin_password,
|
||||
set_nsec,
|
||||
)
|
||||
from .db import (
|
||||
store_cashu_transaction_with_retry as store_cashu_transaction,
|
||||
)
|
||||
from .log_manager import log_manager
|
||||
from .logging import get_logger
|
||||
from .provider_slugs import allocate_unique_provider_slug
|
||||
from .settings import SettingsService, settings
|
||||
from .settings import SettingsService, derive_npub_from_nsec, settings
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -44,6 +51,13 @@ ADMIN_SESSION_DURATION = 3600
|
||||
MAX_USAGE_ANALYTICS_HOURS = 365 * 24
|
||||
|
||||
|
||||
async def _refresh_provider_model_paths(upstream_provider_id: int) -> None:
|
||||
"""Queue discovery sync without blocking the committed admin mutation."""
|
||||
from ..upstream.model_paths import schedule_model_paths_refresh_for_provider
|
||||
|
||||
await schedule_model_paths_refresh_for_provider(upstream_provider_id)
|
||||
|
||||
|
||||
async def require_admin_api(request: Request) -> None:
|
||||
auth_header = request.headers.get("Authorization")
|
||||
if not auth_header or not auth_header.startswith("Bearer "):
|
||||
@@ -203,8 +217,6 @@ async def get_settings(request: Request) -> dict:
|
||||
data = settings.dict()
|
||||
if "upstream_api_key" in data:
|
||||
data["upstream_api_key"] = "[REDACTED]" if data["upstream_api_key"] else ""
|
||||
if "admin_password" in data:
|
||||
data["admin_password"] = "[REDACTED]" if data["admin_password"] else ""
|
||||
if "nsec" in data:
|
||||
data["nsec"] = "[REDACTED]" if data["nsec"] else ""
|
||||
return data
|
||||
@@ -221,9 +233,10 @@ class PasswordUpdate(BaseModel):
|
||||
|
||||
@admin_router.patch("/api/settings", dependencies=[Depends(require_admin_api)])
|
||||
async def update_settings(request: Request, update: SettingsUpdate) -> dict:
|
||||
# Remove sensitive fields from general settings update
|
||||
# Secrets are not editable through the general settings endpoint; they have
|
||||
# dedicated rotation paths and never reach the settings blob.
|
||||
settings_data = update.root.copy()
|
||||
sensitive_fields = ["admin_password", "upstream_api_key", "nsec"]
|
||||
sensitive_fields = ["upstream_api_key", "nsec"]
|
||||
for field in sensitive_fields:
|
||||
if field in settings_data:
|
||||
del settings_data[field]
|
||||
@@ -238,8 +251,6 @@ async def update_settings(request: Request, update: SettingsUpdate) -> dict:
|
||||
data = new_settings.dict()
|
||||
if "upstream_api_key" in data:
|
||||
data["upstream_api_key"] = "[REDACTED]" if data["upstream_api_key"] else ""
|
||||
if "admin_password" in data:
|
||||
data["admin_password"] = "[REDACTED]" if data["admin_password"] else ""
|
||||
if "nsec" in data:
|
||||
data["nsec"] = "[REDACTED]" if data["nsec"] else ""
|
||||
return data
|
||||
@@ -247,44 +258,63 @@ async def update_settings(request: Request, update: SettingsUpdate) -> dict:
|
||||
|
||||
@admin_router.patch("/api/password", dependencies=[Depends(require_admin_api)])
|
||||
async def update_password(request: Request, password_update: PasswordUpdate) -> dict:
|
||||
current_password = settings.admin_password
|
||||
|
||||
if not current_password:
|
||||
raise HTTPException(status_code=500, detail="Admin password not configured")
|
||||
|
||||
if password_update.current_password != current_password:
|
||||
raise HTTPException(status_code=401, detail="Current password is incorrect")
|
||||
|
||||
# Validate new password
|
||||
new_password = password_update.new_password.strip()
|
||||
if len(new_password) < 6:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="New password must be at least 6 characters"
|
||||
)
|
||||
|
||||
# Update password
|
||||
async with create_session() as session:
|
||||
await SettingsService.update({"admin_password": new_password}, session)
|
||||
secret = await get_secret(session)
|
||||
|
||||
if not secret.admin_password_hash:
|
||||
raise HTTPException(
|
||||
status_code=500, detail="Admin password not configured"
|
||||
)
|
||||
|
||||
if not vault.verify_password(
|
||||
password_update.current_password, secret.admin_password_hash
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=401, detail="Current password is incorrect"
|
||||
)
|
||||
|
||||
# Validate new password
|
||||
new_password = password_update.new_password.strip()
|
||||
if len(new_password) < vault.MIN_PASSWORD_LENGTH:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
"New password must be at least "
|
||||
f"{vault.MIN_PASSWORD_LENGTH} characters"
|
||||
),
|
||||
)
|
||||
|
||||
await set_admin_password(session, new_password)
|
||||
|
||||
return {"ok": True, "message": "Password updated successfully"}
|
||||
|
||||
|
||||
class SetupRequest(BaseModel):
|
||||
password: str
|
||||
class NsecUpdate(BaseModel):
|
||||
nsec: str
|
||||
|
||||
|
||||
@admin_router.post("/api/setup")
|
||||
async def initial_setup(request: Request, payload: SetupRequest) -> dict[str, object]:
|
||||
if settings.admin_password:
|
||||
raise HTTPException(status_code=409, detail="Admin password already set")
|
||||
pw = (payload.password or "").strip()
|
||||
if len(pw) < 8:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Password must be at least 8 characters"
|
||||
)
|
||||
@admin_router.patch("/api/nsec", dependencies=[Depends(require_admin_api)])
|
||||
async def update_nsec(request: Request, payload: NsecUpdate) -> dict[str, object]:
|
||||
# The node's Nostr identity is a secret: it is stored encrypted in the
|
||||
# Secret store, never in the settings blob, so it gets its own endpoint
|
||||
# rather than riding the general settings PATCH (which strips it). An empty
|
||||
# nsec clears the identity.
|
||||
nsec = payload.nsec.strip()
|
||||
npub = ""
|
||||
if nsec:
|
||||
derived = derive_npub_from_nsec(nsec)
|
||||
if not derived:
|
||||
raise HTTPException(status_code=400, detail="Invalid nsec")
|
||||
npub = derived
|
||||
|
||||
async with create_session() as session:
|
||||
await SettingsService.update({"admin_password": pw}, session)
|
||||
return {"ok": True}
|
||||
await set_nsec(session, nsec)
|
||||
|
||||
# Reflect the change in the live runtime so Nostr signing/announcements pick
|
||||
# it up without a restart (mirrors what bootstrap_secrets sets at boot).
|
||||
settings.nsec = nsec
|
||||
settings.npub = npub
|
||||
return {"ok": True, "npub": npub}
|
||||
|
||||
|
||||
class AdminLoginRequest(BaseModel):
|
||||
@@ -295,12 +325,16 @@ class AdminLoginRequest(BaseModel):
|
||||
async def admin_login(
|
||||
request: Request, payload: AdminLoginRequest
|
||||
) -> dict[str, object]:
|
||||
admin_pw = settings.admin_password
|
||||
async with create_session() as session:
|
||||
secret = await get_secret(session)
|
||||
# Read the hash while the session is open; the ORM object is detached
|
||||
# once the context exits and its attributes can no longer be loaded.
|
||||
password_hash = secret.admin_password_hash
|
||||
|
||||
if not admin_pw:
|
||||
if not password_hash:
|
||||
raise HTTPException(status_code=500, detail="Admin password not configured")
|
||||
|
||||
if payload.password != admin_pw:
|
||||
if not vault.verify_password(payload.password, password_hash):
|
||||
raise HTTPException(status_code=401, detail="Invalid password")
|
||||
|
||||
token = secrets.token_urlsafe(32)
|
||||
@@ -411,12 +445,11 @@ async def withdraw(
|
||||
# Get wallet and check balance
|
||||
from .settings import settings as global_settings
|
||||
|
||||
wallet = await get_wallet(
|
||||
withdraw_request.mint_url or global_settings.primary_mint, withdraw_request.unit
|
||||
)
|
||||
effective_mint = withdraw_request.mint_url or global_settings.primary_mint
|
||||
wallet = await get_wallet(effective_mint, withdraw_request.unit)
|
||||
proofs = get_proofs_per_mint_and_unit(
|
||||
wallet,
|
||||
withdraw_request.mint_url or global_settings.primary_mint,
|
||||
effective_mint,
|
||||
withdraw_request.unit,
|
||||
not_reserved=True,
|
||||
)
|
||||
@@ -432,8 +465,27 @@ async def withdraw(
|
||||
raise HTTPException(status_code=400, detail="Insufficient wallet balance")
|
||||
|
||||
token = await send_token(
|
||||
withdraw_request.amount, withdraw_request.unit, withdraw_request.mint_url
|
||||
withdraw_request.amount, withdraw_request.unit, effective_mint
|
||||
)
|
||||
try:
|
||||
await store_cashu_transaction(
|
||||
token=token,
|
||||
amount=withdraw_request.amount,
|
||||
unit=withdraw_request.unit,
|
||||
mint_url=effective_mint,
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="admin",
|
||||
)
|
||||
except Exception:
|
||||
logger.critical(
|
||||
"Admin withdrawal token issued without a persisted audit record",
|
||||
extra={
|
||||
"amount": withdraw_request.amount,
|
||||
"unit": withdraw_request.unit,
|
||||
"mint_url": effective_mint,
|
||||
},
|
||||
)
|
||||
return {"token": token}
|
||||
|
||||
|
||||
@@ -461,7 +513,6 @@ class ModelCreate(BaseModel):
|
||||
async def upsert_provider_model(
|
||||
provider_id: str, payload: ModelCreate
|
||||
) -> dict[str, object]:
|
||||
print(payload)
|
||||
logger.info(
|
||||
f"UPSERT_PROVIDER_MODEL called: provider_id={provider_id}, model_id={payload.id}"
|
||||
)
|
||||
@@ -535,6 +586,7 @@ async def upsert_provider_model(
|
||||
await session.refresh(row)
|
||||
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(provider_pk)
|
||||
return _row_to_model(
|
||||
row, apply_provider_fee=True, provider_fee=provider.provider_fee
|
||||
).dict() # type: ignore
|
||||
@@ -589,6 +641,7 @@ async def delete_provider_model(provider_id: str, model_id: str) -> dict[str, ob
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(provider_pk)
|
||||
return {"ok": True, "deleted_id": model_id}
|
||||
|
||||
|
||||
@@ -608,6 +661,7 @@ async def delete_all_provider_models(provider_id: str) -> dict[str, object]:
|
||||
await session.delete(row) # type: ignore
|
||||
await session.commit()
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(provider_pk)
|
||||
return {"ok": True, "deleted": len(rows)}
|
||||
|
||||
|
||||
@@ -699,6 +753,7 @@ async def batch_override_provider_models(
|
||||
await session.commit()
|
||||
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(provider_pk)
|
||||
return {
|
||||
"ok": True,
|
||||
"count": overridden_count,
|
||||
@@ -899,6 +954,7 @@ async def create_upstream_provider(
|
||||
|
||||
await reinitialize_upstreams()
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(_provider_pk(provider))
|
||||
return _serialize_provider(provider)
|
||||
|
||||
|
||||
@@ -924,6 +980,7 @@ async def update_upstream_provider(
|
||||
|
||||
await reinitialize_upstreams()
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(_provider_pk(provider))
|
||||
return _serialize_provider(provider)
|
||||
|
||||
|
||||
@@ -959,6 +1016,7 @@ async def update_upstream_provider_by_slug(
|
||||
|
||||
await reinitialize_upstreams()
|
||||
await refresh_model_maps()
|
||||
await _refresh_provider_model_paths(_provider_pk(provider))
|
||||
return _serialize_provider(provider)
|
||||
|
||||
|
||||
|
||||
+393
-31
@@ -1,16 +1,19 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import sqlite3
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from enum import Enum
|
||||
from typing import AsyncGenerator
|
||||
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from alembic.util.exc import CommandError
|
||||
from sqlalchemy import UniqueConstraint, delete
|
||||
from sqlalchemy.exc import OperationalError
|
||||
from sqlalchemy import Index, UniqueConstraint, case, delete, or_
|
||||
from sqlalchemy.exc import IntegrityError, OperationalError
|
||||
from sqlalchemy.ext.asyncio.engine import create_async_engine
|
||||
from sqlalchemy.orm import aliased
|
||||
from sqlmodel import Field, Relationship, SQLModel, col, func, select, update
|
||||
@@ -96,32 +99,133 @@ class ApiKey(SQLModel, table=True): # type: ignore
|
||||
|
||||
|
||||
async def reset_all_reserved_balances(session: AsyncSession) -> None:
|
||||
stmt = update(ApiKey).values(reserved_balance=0, reserved_at=None)
|
||||
await session.exec(stmt) # type: ignore[call-overload]
|
||||
"""Release every active durable reservation during explicit startup reset."""
|
||||
await session.exec( # type: ignore[call-overload]
|
||||
update(ReservationRelease)
|
||||
.where(col(ReservationRelease.status) == "active")
|
||||
.values(status="released")
|
||||
)
|
||||
await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey).values(reserved_balance=0, reserved_at=None)
|
||||
)
|
||||
await session.commit()
|
||||
logger.info("Reset reserved balances on startup")
|
||||
|
||||
|
||||
async def release_stale_reservations(
|
||||
session: AsyncSession, max_age_seconds: int
|
||||
session: AsyncSession,
|
||||
max_age_seconds: int,
|
||||
*,
|
||||
key_hash: str | None = None,
|
||||
) -> int:
|
||||
"""Release reservations whose last reserve is older than max_age_seconds.
|
||||
"""
|
||||
"""Release stale durable reservations without touching newer reservations."""
|
||||
cutoff = int(time.time()) - max_age_seconds
|
||||
stmt = (
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.reserved_balance) > 0)
|
||||
.where(col(ApiKey.reserved_at).is_not(None))
|
||||
.where(col(ApiKey.reserved_at) < cutoff)
|
||||
.values(reserved_balance=0, reserved_at=None)
|
||||
query = (
|
||||
select(ReservationRelease)
|
||||
.where(col(ReservationRelease.status) == "active")
|
||||
.where(col(ReservationRelease.created_at) < cutoff)
|
||||
)
|
||||
result = await session.exec(stmt) # type: ignore[call-overload]
|
||||
if key_hash is not None:
|
||||
query = query.where(
|
||||
or_(
|
||||
col(ReservationRelease.key_hash) == key_hash,
|
||||
col(ReservationRelease.billing_key_hash) == key_hash,
|
||||
)
|
||||
)
|
||||
reservations = (await session.exec(query)).all()
|
||||
released = 0
|
||||
|
||||
for reservation in reservations:
|
||||
transition = await session.exec( # type: ignore[call-overload]
|
||||
update(ReservationRelease)
|
||||
.where(col(ReservationRelease.id) == reservation.id)
|
||||
.where(col(ReservationRelease.status) == "active")
|
||||
.values(status="released")
|
||||
)
|
||||
if transition.rowcount != 1:
|
||||
continue
|
||||
|
||||
values = {
|
||||
"reserved_balance": col(ApiKey.reserved_balance)
|
||||
- reservation.reserved_msats,
|
||||
"reserved_at": case(
|
||||
(
|
||||
col(ApiKey.reserved_balance) - reservation.reserved_msats > 0,
|
||||
col(ApiKey.reserved_at),
|
||||
),
|
||||
else_=None,
|
||||
),
|
||||
}
|
||||
parent_result = await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == reservation.billing_key_hash)
|
||||
.where(col(ApiKey.reserved_balance) >= reservation.reserved_msats)
|
||||
.values(**values)
|
||||
)
|
||||
if parent_result.rowcount != 1:
|
||||
await session.rollback()
|
||||
return 0
|
||||
|
||||
if reservation.billing_key_hash != reservation.key_hash:
|
||||
child_result = await session.exec( # type: ignore[call-overload]
|
||||
update(ApiKey)
|
||||
.where(col(ApiKey.hashed_key) == reservation.key_hash)
|
||||
.where(col(ApiKey.reserved_balance) >= reservation.reserved_msats)
|
||||
.values(**values)
|
||||
)
|
||||
if child_result.rowcount != 1:
|
||||
await session.rollback()
|
||||
return 0
|
||||
released += 1
|
||||
|
||||
# Rolling upgrades can leave aggregate reservations created before durable
|
||||
# reservation rows existed. Release only stale aggregates that have no active
|
||||
# durable owner; targeted refund cleanup also heals legacy NULL timestamps.
|
||||
legacy_query = select(ApiKey).where(col(ApiKey.reserved_balance) > 0)
|
||||
if key_hash is None:
|
||||
legacy_query = legacy_query.where(col(ApiKey.reserved_at).is_not(None)).where(
|
||||
col(ApiKey.reserved_at) < cutoff
|
||||
)
|
||||
else:
|
||||
legacy_query = legacy_query.where(
|
||||
or_(
|
||||
col(ApiKey.hashed_key) == key_hash,
|
||||
col(ApiKey.parent_key_hash) == key_hash,
|
||||
)
|
||||
).where(
|
||||
or_(col(ApiKey.reserved_at).is_(None), col(ApiKey.reserved_at) < cutoff)
|
||||
)
|
||||
|
||||
for legacy_key in (await session.exec(legacy_query)).all():
|
||||
active_owner = (
|
||||
await session.exec(
|
||||
select(ReservationRelease.id)
|
||||
.where(col(ReservationRelease.status) == "active")
|
||||
.where(
|
||||
or_(
|
||||
col(ReservationRelease.key_hash) == legacy_key.hashed_key,
|
||||
col(ReservationRelease.billing_key_hash)
|
||||
== legacy_key.hashed_key,
|
||||
)
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
).first()
|
||||
if active_owner is not None:
|
||||
continue
|
||||
legacy_key.reserved_balance = 0
|
||||
legacy_key.reserved_at = None
|
||||
session.add(legacy_key)
|
||||
released += 1
|
||||
|
||||
await session.commit()
|
||||
released = int(result.rowcount or 0)
|
||||
if released:
|
||||
logger.warning(
|
||||
"Released stale balance reservations",
|
||||
extra={"released_keys": released, "max_age_seconds": max_age_seconds},
|
||||
"Released stale reservations",
|
||||
extra={
|
||||
"released_reservations": released,
|
||||
"max_age_seconds": max_age_seconds,
|
||||
},
|
||||
)
|
||||
return released
|
||||
|
||||
@@ -154,9 +258,7 @@ async def prune_dead_api_keys(session: AsyncSession, min_age_seconds: int) -> in
|
||||
.where(col(ApiKey.total_spent) == 0)
|
||||
.where(col(ApiKey.total_requests) == 0)
|
||||
.where(col(ApiKey.parent_key_hash).is_(None))
|
||||
.where(
|
||||
(col(ApiKey.created_at).is_(None)) | (col(ApiKey.created_at) < cutoff)
|
||||
)
|
||||
.where((col(ApiKey.created_at).is_(None)) | (col(ApiKey.created_at) < cutoff))
|
||||
.where(~pending_invoice)
|
||||
.where(~has_children)
|
||||
)
|
||||
@@ -210,6 +312,60 @@ class ModelRow(SQLModel, table=True): # type: ignore
|
||||
upstream_provider: "UpstreamProviderRow" = Relationship(back_populates="models")
|
||||
|
||||
|
||||
class ModelPathRow(SQLModel, table=True): # type: ignore
|
||||
"""Upstream provider path a model is reachable through.
|
||||
|
||||
Discovery/visibility data only. ``model_id`` is intentionally NOT globally
|
||||
unique: it is the client-visible ``/v1/models`` id (``forwarded_model_id or
|
||||
id``) grouped across every provider that exposes the model. A single model
|
||||
can therefore have several rows — one per direct provider path plus one per
|
||||
OpenRouter sub-provider endpoint.
|
||||
"""
|
||||
|
||||
__tablename__ = "model_paths"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"model_id",
|
||||
"path",
|
||||
"upstream_provider_id",
|
||||
name="uq_model_paths_model_path_provider",
|
||||
),
|
||||
)
|
||||
id: int | None = Field(default=None, primary_key=True)
|
||||
# No standalone index on model_id: the unique constraint's autoindex already
|
||||
# leads on model_id, so a second index only adds write amplification.
|
||||
model_id: str = Field(
|
||||
description="Client-visible /v1/models id (forwarded_model_id or id)"
|
||||
)
|
||||
path: str = Field(
|
||||
description=(
|
||||
"Opaque selector containing upstream URL, provider ID, model ID, "
|
||||
"and optional endpoint tag"
|
||||
)
|
||||
)
|
||||
provider_slug: str = Field(
|
||||
description="Public slug of the configured upstream provider"
|
||||
)
|
||||
provider_type: str = Field(description="Configured upstream provider type")
|
||||
endpoint_tag: str | None = Field(
|
||||
default=None,
|
||||
description="Exact OpenRouter endpoint tag used for request-side selection",
|
||||
)
|
||||
endpoint_name: str | None = Field(
|
||||
default=None, description="Human-readable endpoint display name"
|
||||
)
|
||||
upstream_provider_id: int = Field(
|
||||
index=True,
|
||||
foreign_key="upstream_providers.id",
|
||||
ondelete="CASCADE",
|
||||
description="upstream_providers.id this path was discovered from",
|
||||
)
|
||||
updated_at: int = Field(
|
||||
default=0,
|
||||
description="Unix timestamp of the refresh cycle that wrote this row",
|
||||
)
|
||||
|
||||
|
||||
class LightningInvoice(SQLModel, table=True): # type: ignore
|
||||
__tablename__ = "lightning_invoices"
|
||||
|
||||
@@ -287,10 +443,13 @@ async def store_cashu_transaction(
|
||||
created_at: int | None = None,
|
||||
source: str = "x-cashu",
|
||||
api_key_hashed_key: str | None = None,
|
||||
) -> None:
|
||||
transaction_id: str | None = None,
|
||||
log_failure: bool = True,
|
||||
) -> bool:
|
||||
try:
|
||||
async with create_session() as session:
|
||||
tx = CashuTransaction(
|
||||
id=transaction_id or uuid.uuid4().hex,
|
||||
token=token,
|
||||
amount=amount,
|
||||
unit=unit,
|
||||
@@ -304,11 +463,93 @@ async def store_cashu_transaction(
|
||||
)
|
||||
session.add(tx)
|
||||
await session.commit()
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to store cashu transaction: {e} (type={typ})",
|
||||
extra={"error": str(e), "type": typ},
|
||||
)
|
||||
except Exception:
|
||||
if log_failure:
|
||||
logger.critical(
|
||||
"Failed to store Cashu transaction",
|
||||
extra={"type": typ, "request_id": request_id, "source": source},
|
||||
exc_info=True,
|
||||
)
|
||||
raise
|
||||
return True
|
||||
|
||||
|
||||
async def _cashu_transaction_exists(transaction_id: str) -> bool:
|
||||
async with create_session() as session:
|
||||
return await session.get(CashuTransaction, transaction_id) is not None
|
||||
|
||||
|
||||
async def store_cashu_transaction_with_retry(
|
||||
token: str,
|
||||
amount: int,
|
||||
unit: str,
|
||||
mint_url: str | None = None,
|
||||
typ: str = "out",
|
||||
request_id: str | None = None,
|
||||
collected: bool = False,
|
||||
created_at: int | None = None,
|
||||
source: str = "x-cashu",
|
||||
api_key_hashed_key: str | None = None,
|
||||
max_attempts: int = 3,
|
||||
) -> bool:
|
||||
"""Retry a critical Cashu transaction write with bounded backoff."""
|
||||
transaction_id = hashlib.sha256(f"{typ}\0{token}".encode()).hexdigest()
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
return await store_cashu_transaction(
|
||||
token=token,
|
||||
amount=amount,
|
||||
unit=unit,
|
||||
mint_url=mint_url,
|
||||
typ=typ,
|
||||
request_id=request_id,
|
||||
collected=collected,
|
||||
created_at=created_at,
|
||||
source=source,
|
||||
api_key_hashed_key=api_key_hashed_key,
|
||||
transaction_id=transaction_id,
|
||||
log_failure=False,
|
||||
)
|
||||
except IntegrityError as error:
|
||||
try:
|
||||
if await _cashu_transaction_exists(transaction_id):
|
||||
return True
|
||||
except Exception as lookup_error:
|
||||
last_error = lookup_error
|
||||
else:
|
||||
last_error = error
|
||||
except Exception as error:
|
||||
last_error = error
|
||||
|
||||
if last_error is not None:
|
||||
if attempt == max_attempts:
|
||||
break
|
||||
delay = 0.25 * (2 ** (attempt - 1))
|
||||
logger.warning(
|
||||
"Cashu transaction storage failed; retrying",
|
||||
extra={
|
||||
"type": typ,
|
||||
"request_id": request_id,
|
||||
"attempt": attempt,
|
||||
"max_attempts": max_attempts,
|
||||
"retry_delay_seconds": delay,
|
||||
},
|
||||
)
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
logger.critical(
|
||||
"Cashu transaction storage failed after bounded retries",
|
||||
extra={
|
||||
"type": typ,
|
||||
"request_id": request_id,
|
||||
"attempts": max_attempts,
|
||||
"error": str(last_error),
|
||||
},
|
||||
)
|
||||
if last_error is None:
|
||||
raise RuntimeError("Cashu transaction storage failed without an exception")
|
||||
raise last_error
|
||||
|
||||
|
||||
class UpstreamProviderRow(SQLModel, table=True): # type: ignore
|
||||
@@ -346,21 +587,71 @@ class UpstreamProviderRow(SQLModel, table=True): # type: ignore
|
||||
)
|
||||
|
||||
|
||||
class ReservationRelease(SQLModel, table=True): # type: ignore
|
||||
__tablename__ = "reservation_releases"
|
||||
__table_args__ = (
|
||||
Index("ix_reservation_releases_status_created_at", "status", "created_at"),
|
||||
)
|
||||
|
||||
id: str = Field(primary_key=True)
|
||||
key_hash: str = Field(index=True)
|
||||
billing_key_hash: str = Field(index=True)
|
||||
reserved_msats: int
|
||||
status: str = Field(default="active")
|
||||
created_at: int = Field(default_factory=lambda: int(time.time()))
|
||||
|
||||
|
||||
class RoutstrFee(SQLModel, table=True): # type: ignore
|
||||
__tablename__ = "routstr_fees"
|
||||
id: int = Field(default=1, primary_key=True)
|
||||
accumulated_msats: int = Field(default=0)
|
||||
total_paid_msats: int = Field(default=0)
|
||||
last_paid_at: int | None = Field(default=None)
|
||||
payout_in_progress_msats: int = Field(default=0)
|
||||
payout_started_at: int | None = Field(default=None)
|
||||
|
||||
|
||||
class NsecState(str, Enum):
|
||||
"""Ownership state of the node's nsec — an explicit 3-state machine.
|
||||
|
||||
The single ``encrypted_nsec`` column cannot distinguish "never migrated" from
|
||||
"intentionally cleared" (both leave it empty), which let a cleared identity be
|
||||
resurrected from a stale legacy ``NSEC``. This names the three states so the
|
||||
bootstrap branches on ownership rather than inferring it:
|
||||
|
||||
* ``legacy`` — the vault has not taken ownership; a plaintext ``NSEC`` (env or
|
||||
old settings blob) may still exist and should be migrated in once.
|
||||
* ``encrypted`` — the vault owns a ciphertext; decrypt it, never re-read env.
|
||||
* ``cleared`` — the vault owns it but the operator emptied it; stay empty,
|
||||
never re-import from a stale legacy copy.
|
||||
"""
|
||||
|
||||
legacy = "legacy"
|
||||
encrypted = "encrypted"
|
||||
cleared = "cleared"
|
||||
|
||||
|
||||
class Secret(SQLModel, table=True): # type: ignore
|
||||
"""Node-level secrets, stored encrypted/hashed at rest (singleton, id=1).
|
||||
|
||||
The asymmetric column names document the encoding: ``_hash`` is one-way
|
||||
(scrypt, verify only) while ``encrypted_`` is reversible (Fernet). Per-provider
|
||||
upstream keys live on ``upstream_providers``, not here. See ``routstr.core.vault``.
|
||||
"""
|
||||
|
||||
__tablename__ = "secrets"
|
||||
id: int = Field(default=1, primary_key=True)
|
||||
admin_password_hash: str | None = Field(default=None)
|
||||
encrypted_nsec: str | None = Field(default=None)
|
||||
nsec_state: NsecState = Field(default=NsecState.legacy)
|
||||
updated_at: int | None = Field(default=None)
|
||||
|
||||
|
||||
class CliToken(SQLModel, table=True): # type: ignore
|
||||
"""Long-lived authorization token for CLI/agent use against admin endpoints."""
|
||||
|
||||
__tablename__ = "cli_tokens"
|
||||
id: str = Field(
|
||||
primary_key=True, default_factory=lambda: uuid.uuid4().hex
|
||||
)
|
||||
id: str = Field(primary_key=True, default_factory=lambda: uuid.uuid4().hex)
|
||||
token: str = Field(unique=True, index=True, description="Bearer token value")
|
||||
name: str = Field(description="Human-readable label for this token")
|
||||
created_at: int = Field(default_factory=lambda: int(time.time()))
|
||||
@@ -392,18 +683,89 @@ async def get_routstr_fee(session: AsyncSession) -> RoutstrFee:
|
||||
return fee
|
||||
|
||||
|
||||
async def reset_routstr_fee(session: AsyncSession, paid_msats: int) -> None:
|
||||
async def get_secret(session: AsyncSession) -> Secret:
|
||||
secret = await session.get(Secret, 1)
|
||||
if secret is None:
|
||||
secret = Secret(id=1)
|
||||
session.add(secret)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# Another worker created the singleton row between our read and
|
||||
# insert (multiple workers booting against one shared DB). Roll back
|
||||
# and read the row they committed instead of failing startup.
|
||||
await session.rollback()
|
||||
secret = await session.get(Secret, 1)
|
||||
if secret is None:
|
||||
raise
|
||||
return secret
|
||||
await session.refresh(secret)
|
||||
return secret
|
||||
|
||||
|
||||
async def set_admin_password(session: AsyncSession, password: str) -> None:
|
||||
"""Store the admin password as a one-way hash on the Secret singleton."""
|
||||
from .vault import hash_password
|
||||
|
||||
secret = await get_secret(session)
|
||||
secret.admin_password_hash = hash_password(password)
|
||||
secret.updated_at = int(time.time())
|
||||
session.add(secret)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def set_nsec(session: AsyncSession, nsec: str) -> None:
|
||||
"""Store the node's nsec, Fernet-encrypted, on the Secret singleton.
|
||||
|
||||
An empty string clears it (the node then holds no Nostr identity and signs
|
||||
no events). Either way the vault now owns the nsec, so the state moves off
|
||||
``legacy``: a cleared identity (``cleared``) must not be resurrected from a
|
||||
stale legacy ``NSEC`` on the next boot.
|
||||
"""
|
||||
from .vault import encrypt
|
||||
|
||||
secret = await get_secret(session)
|
||||
secret.encrypted_nsec = encrypt(nsec) if nsec else None
|
||||
secret.nsec_state = NsecState.encrypted if nsec else NsecState.cleared
|
||||
secret.updated_at = int(time.time())
|
||||
session.add(secret)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def reset_routstr_fee(session: AsyncSession, paid_msats: int) -> bool:
|
||||
"""Checkpoint a fee payout before making the external payment."""
|
||||
stmt = (
|
||||
update(RoutstrFee)
|
||||
.where(col(RoutstrFee.id) == 1)
|
||||
.where(col(RoutstrFee.payout_in_progress_msats) == 0)
|
||||
.where(col(RoutstrFee.accumulated_msats) >= paid_msats)
|
||||
.values(
|
||||
accumulated_msats=RoutstrFee.accumulated_msats - paid_msats,
|
||||
payout_in_progress_msats=paid_msats,
|
||||
payout_started_at=int(time.time()),
|
||||
)
|
||||
)
|
||||
result = await session.exec(stmt) # type: ignore[call-overload]
|
||||
await session.commit()
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
async def complete_routstr_fee_payout(session: AsyncSession, paid_msats: int) -> bool:
|
||||
"""Mark a checkpointed payout complete after the external payment succeeds."""
|
||||
stmt = (
|
||||
update(RoutstrFee)
|
||||
.where(col(RoutstrFee.id) == 1)
|
||||
.where(col(RoutstrFee.payout_in_progress_msats) == paid_msats)
|
||||
.values(
|
||||
payout_in_progress_msats=0,
|
||||
payout_started_at=None,
|
||||
total_paid_msats=RoutstrFee.total_paid_msats + paid_msats,
|
||||
last_paid_at=int(time.time()),
|
||||
)
|
||||
)
|
||||
await session.exec(stmt) # type: ignore[call-overload]
|
||||
result = await session.exec(stmt) # type: ignore[call-overload]
|
||||
await session.commit()
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
async def balances_for_mint_and_unit(
|
||||
|
||||
+24
-15
@@ -37,7 +37,7 @@ from .exceptions import general_exception_handler, http_exception_handler
|
||||
from .logging import get_logger, setup_logging
|
||||
from .middleware import LoggingMiddleware
|
||||
from .not_found import _NOT_FOUND_HTML, not_found_catch_all # noqa: F401
|
||||
from .settings import SettingsService
|
||||
from .settings import SettingsService, bootstrap_secrets
|
||||
from .settings import settings as global_settings
|
||||
from .version import __version__
|
||||
|
||||
@@ -58,6 +58,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
providers_task = None
|
||||
models_refresh_task = None
|
||||
model_maps_refresh_task = None
|
||||
model_paths_refresh_task = None
|
||||
key_reset_task = None
|
||||
stale_reservation_task = None
|
||||
dead_key_prune_task = None
|
||||
@@ -85,17 +86,20 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
# Initialize application settings (env -> computed -> DB precedence)
|
||||
async with create_session() as session:
|
||||
# Move secrets into the encrypted/hashed store and decrypt the nsec
|
||||
# into the in-memory settings BEFORE initializing settings: the
|
||||
# initialize step strips secrets from the persisted blob, so legacy
|
||||
# plaintext (env or old blob) must be migrated into the Secret store
|
||||
# first or the only copy of a blob-only secret would be lost.
|
||||
# Generates and logs an admin password on a fresh node; fails fast if
|
||||
# a stored secret can't be decrypted.
|
||||
await bootstrap_secrets(session)
|
||||
s = await SettingsService.initialize(session)
|
||||
if s.reset_reserved_balance_on_startup:
|
||||
from .db import reset_all_reserved_balances
|
||||
|
||||
await reset_all_reserved_balances(session)
|
||||
|
||||
if not s.admin_password:
|
||||
logger.warning(
|
||||
f"Admin password is not set. Visit {s.http_url or 'http://localhost:8000'}/admin to set the password."
|
||||
)
|
||||
|
||||
# Apply app metadata from settings
|
||||
try:
|
||||
app.title = s.name
|
||||
@@ -127,6 +131,13 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
refresh_upstreams_models_periodically(get_upstreams)
|
||||
)
|
||||
model_maps_refresh_task = asyncio.create_task(refresh_model_maps_periodically())
|
||||
# Always started: the loop re-reads the enable flag and interval every
|
||||
# iteration, so 0 -> N (or re-enabling) takes effect without a restart.
|
||||
from ..upstream.model_paths import refresh_model_paths_periodically
|
||||
|
||||
model_paths_refresh_task = asyncio.create_task(
|
||||
refresh_model_paths_periodically(get_upstreams)
|
||||
)
|
||||
payout_task = asyncio.create_task(periodic_payout())
|
||||
if global_settings.nsec:
|
||||
nip91_task = asyncio.create_task(announce_provider())
|
||||
@@ -134,9 +145,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
if global_settings.providers_refresh_interval_seconds > 0:
|
||||
providers_task = asyncio.create_task(providers_cache_refresher())
|
||||
key_reset_task = asyncio.create_task(periodic_key_reset())
|
||||
stale_reservation_task = asyncio.create_task(
|
||||
periodic_stale_reservation_sweep()
|
||||
)
|
||||
stale_reservation_task = asyncio.create_task(periodic_stale_reservation_sweep())
|
||||
dead_key_prune_task = asyncio.create_task(periodic_dead_key_prune())
|
||||
auto_topup_task = asyncio.create_task(periodic_auto_topup())
|
||||
refund_sweep_task = asyncio.create_task(periodic_refund_sweep())
|
||||
@@ -173,6 +182,8 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
models_refresh_task.cancel()
|
||||
if model_maps_refresh_task is not None:
|
||||
model_maps_refresh_task.cancel()
|
||||
if model_paths_refresh_task is not None:
|
||||
model_paths_refresh_task.cancel()
|
||||
if key_reset_task is not None:
|
||||
key_reset_task.cancel()
|
||||
if stale_reservation_task is not None:
|
||||
@@ -206,6 +217,8 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]:
|
||||
tasks_to_wait.append(models_refresh_task)
|
||||
if model_maps_refresh_task is not None:
|
||||
tasks_to_wait.append(model_maps_refresh_task)
|
||||
if model_paths_refresh_task is not None:
|
||||
tasks_to_wait.append(model_paths_refresh_task)
|
||||
if key_reset_task is not None:
|
||||
tasks_to_wait.append(key_reset_task)
|
||||
if stale_reservation_task is not None:
|
||||
@@ -242,9 +255,7 @@ class _ImmutableStaticFiles(StaticFiles):
|
||||
async def get_response(self, path: str, scope: Scope) -> StarletteResponse:
|
||||
response = await super().get_response(path, scope)
|
||||
if response.status_code == 200:
|
||||
response.headers["Cache-Control"] = (
|
||||
"public, max-age=31536000, immutable"
|
||||
)
|
||||
response.headers["Cache-Control"] = "public, max-age=31536000, immutable"
|
||||
return response
|
||||
|
||||
|
||||
@@ -318,9 +329,7 @@ if UI_DIST_PATH.exists() and UI_DIST_PATH.is_dir():
|
||||
# Serve the App Router RSC payload for the home page.
|
||||
@app.get("/index.txt", include_in_schema=False)
|
||||
async def serve_root_rsc() -> FileResponse:
|
||||
return FileResponse(
|
||||
UI_DIST_PATH / "index.txt", media_type="text/x-component"
|
||||
)
|
||||
return FileResponse(UI_DIST_PATH / "index.txt", media_type="text/x-component")
|
||||
|
||||
# Next.js is built with `trailingSlash: true`, so all UI page URLs end
|
||||
# with a slash (e.g. `/login/`). The proxy router catches `/{path:path}`
|
||||
|
||||
+254
-31
@@ -3,6 +3,8 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
@@ -26,7 +28,6 @@ class Settings(BaseSettings):
|
||||
# Core
|
||||
upstream_base_url: str = Field(default="", env="UPSTREAM_BASE_URL")
|
||||
upstream_api_key: str = Field(default="", env="UPSTREAM_API_KEY")
|
||||
admin_password: str = Field(default="", env="ADMIN_PASSWORD")
|
||||
|
||||
# Node info
|
||||
name: str = Field(default="ARoutstrNode", env="NAME")
|
||||
@@ -94,10 +95,18 @@ class Settings(BaseSettings):
|
||||
models_refresh_interval_seconds: int = Field(
|
||||
default=360, env="MODELS_REFRESH_INTERVAL_SECONDS"
|
||||
)
|
||||
model_paths_refresh_interval_seconds: int = Field(
|
||||
default=600, env="MODEL_PATHS_REFRESH_INTERVAL_SECONDS"
|
||||
)
|
||||
enable_pricing_refresh: bool = Field(default=True, env="ENABLE_PRICING_REFRESH")
|
||||
enable_models_refresh: bool = Field(default=True, env="ENABLE_MODELS_REFRESH")
|
||||
enable_model_paths_refresh: bool = Field(
|
||||
default=True, env="ENABLE_MODEL_PATHS_REFRESH"
|
||||
)
|
||||
refund_cache_ttl_seconds: int = Field(default=3600, env="REFUND_CACHE_TTL_SECONDS")
|
||||
refund_sweep_ttl_seconds: int = Field(default=604800, env="REFUND_SWEEP_TTL_SECONDS")
|
||||
refund_sweep_ttl_seconds: int = Field(
|
||||
default=604800, env="REFUND_SWEEP_TTL_SECONDS"
|
||||
)
|
||||
|
||||
# Logging
|
||||
log_level: str = Field(default="INFO", env="LOG_LEVEL")
|
||||
@@ -116,9 +125,8 @@ class Settings(BaseSettings):
|
||||
|
||||
# Discovery
|
||||
relays: list[str] = Field(default_factory=list, env="RELAYS")
|
||||
enable_analytics_sharing: bool = Field(
|
||||
default=True, env="ENABLE_ANALYTICS_SHARING"
|
||||
)
|
||||
enable_analytics_sharing: bool = Field(default=True, env="ENABLE_ANALYTICS_SHARING")
|
||||
|
||||
|
||||
def _normalize_settings_data(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Discard unknown keys from persisted settings."""
|
||||
@@ -132,10 +140,70 @@ def _normalize_settings_data(data: dict[str, Any]) -> dict[str, Any]:
|
||||
return normalized
|
||||
|
||||
|
||||
# Secrets are credentials, not config: they live in the encrypted/hashed Secret
|
||||
# store (and decrypted in-memory for runtime use), never in the persisted
|
||||
# settings blob. ``admin_password`` is gone from the model entirely; ``nsec``
|
||||
# remains a live field but is stripped from every blob write so it is never
|
||||
# written back to plaintext. ``upstream_api_key`` is intentionally *not* here:
|
||||
# it has no encrypted home yet (it is node-scoped today but really belongs on a
|
||||
# provider), so stripping it would lose it on the next restart. It stays in the
|
||||
# blob as before; encrypting it is follow-up work. See ``bootstrap_secrets`` and
|
||||
# ``routstr.core.vault``.
|
||||
SECRET_FIELDS = frozenset({"admin_password", "nsec"})
|
||||
|
||||
|
||||
def _strip_secret_fields(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Return a copy of ``data`` without any secret fields (for persistence)."""
|
||||
return {k: v for k, v in data.items() if k not in SECRET_FIELDS}
|
||||
|
||||
|
||||
def _apply_to_live_settings(data: dict[str, Any]) -> None:
|
||||
"""Apply ``data`` onto the live ``settings`` for all in-process importers.
|
||||
|
||||
Secrets are owned exclusively by ``bootstrap_secrets``, which runs first and
|
||||
has already decrypted the authoritative nsec into memory (importing any
|
||||
legacy plaintext on the way). Never re-apply secret fields from env/blob
|
||||
here: a non-empty but stale ``NSEC`` env var would otherwise override an nsec
|
||||
the vault has taken ownership of (e.g. after the operator rotates it in the
|
||||
UI), and an empty one would wipe the live value. Skip them entirely.
|
||||
"""
|
||||
for k, v in data.items():
|
||||
if k in SECRET_FIELDS:
|
||||
continue
|
||||
setattr(settings, k, v)
|
||||
|
||||
|
||||
def _compute_primary_mint(cashu_mints: list[str]) -> str:
|
||||
return cashu_mints[0] if cashu_mints else "https://mint.minibits.cash/Bitcoin"
|
||||
|
||||
|
||||
def derive_npub_from_nsec(nsec: str) -> str | None:
|
||||
"""Derive the npub (bech32) from an nsec or 64-char hex private key, or None.
|
||||
|
||||
Parsing is delegated to :func:`routstr.nostr.listing.nsec_to_keypair`, the
|
||||
single place that knows the nsec/hex formats (and already returns ``None`` on
|
||||
any unusable input); this only bech32-encodes the resulting public key. The
|
||||
contract stays "return None on unusable input", so a bad key never crashes
|
||||
boot.
|
||||
"""
|
||||
try:
|
||||
from nostr.key import PublicKey # type: ignore
|
||||
|
||||
from ..nostr.listing import nsec_to_keypair
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
keypair = nsec_to_keypair(nsec)
|
||||
if keypair is None:
|
||||
return None
|
||||
_privkey_hex, pubkey_hex = keypair
|
||||
|
||||
try:
|
||||
return PublicKey(bytes.fromhex(pubkey_hex)).bech32()
|
||||
except (ValueError, AttributeError):
|
||||
return None
|
||||
|
||||
|
||||
def resolve_bootstrap() -> Settings:
|
||||
base = Settings() # Reads env with custom parse_env_var
|
||||
# Back-compat env mapping
|
||||
@@ -190,23 +258,9 @@ def resolve_bootstrap() -> Settings:
|
||||
pass
|
||||
# Derive NPUB from NSEC if not provided
|
||||
if not base.npub and base.nsec:
|
||||
try:
|
||||
from nostr.key import PrivateKey # type: ignore
|
||||
|
||||
if base.nsec.startswith("nsec"):
|
||||
pk = PrivateKey.from_nsec(base.nsec)
|
||||
elif len(base.nsec) == 64:
|
||||
pk = PrivateKey(bytes.fromhex(base.nsec))
|
||||
else:
|
||||
pk = None
|
||||
if pk is not None:
|
||||
try:
|
||||
base.npub = pk.public_key.bech32()
|
||||
except Exception:
|
||||
# Fallback to hex if bech32 not available
|
||||
base.npub = pk.public_key.hex()
|
||||
except Exception:
|
||||
pass
|
||||
npub = derive_npub_from_nsec(base.nsec)
|
||||
if npub:
|
||||
base.npub = npub
|
||||
if not base.cors_origins:
|
||||
base.cors_origins = ["*"]
|
||||
if not base.primary_mint:
|
||||
@@ -256,15 +310,14 @@ class SettingsService:
|
||||
text(
|
||||
"INSERT INTO settings (id, data, updated_at) VALUES (1, :data, :updated_at)"
|
||||
).bindparams(
|
||||
data=json.dumps(env_resolved.dict()),
|
||||
data=json.dumps(_strip_secret_fields(env_resolved.dict())),
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
)
|
||||
await db_session.commit()
|
||||
cls._current = settings
|
||||
# Update the existing instance in-place for all live importers
|
||||
for k, v in env_resolved.dict().items():
|
||||
setattr(settings, k, v)
|
||||
_apply_to_live_settings(env_resolved.dict())
|
||||
return cls._current
|
||||
|
||||
db_id, db_data, _updated_at = row
|
||||
@@ -281,7 +334,11 @@ class SettingsService:
|
||||
valid_fields = set(env_resolved.dict().keys())
|
||||
merged_dict: dict[str, Any] = dict(env_resolved.dict())
|
||||
merged_dict.update(
|
||||
{k: v for k, v in db_json.items() if v not in (None, "", [], {}) and k in valid_fields}
|
||||
{
|
||||
k: v
|
||||
for k, v in db_json.items()
|
||||
if v not in (None, "", [], {}) and k in valid_fields
|
||||
}
|
||||
)
|
||||
merged_dict = Settings(**merged_dict).dict()
|
||||
|
||||
@@ -291,20 +348,37 @@ class SettingsService:
|
||||
merged_dict.get("cashu_mints", [])
|
||||
)
|
||||
|
||||
if db_json_raw != merged_dict:
|
||||
# Keep npub consistent with the live nsec. bootstrap_secrets has
|
||||
# already run and holds the single authoritative nsec (decrypted from
|
||||
# the encrypted store, or freshly imported). merged_dict starts from
|
||||
# the env/blob, which may carry a STALE nsec — and therefore a stale
|
||||
# derived npub — after the vault took ownership. Derive from the live
|
||||
# value and OVERRIDE, not just fill: otherwise the node keeps the
|
||||
# vault's private key but announces the old env key's npub (npub is a
|
||||
# pure derivation of nsec, never configured independently of it).
|
||||
if settings.nsec:
|
||||
derived_npub = derive_npub_from_nsec(settings.nsec)
|
||||
if derived_npub:
|
||||
merged_dict["npub"] = derived_npub
|
||||
|
||||
# Persist without secrets; compare against the stripped target so a
|
||||
# legacy blob that still carries plaintext secrets gets rewritten
|
||||
# (and thereby sunset) even when its non-secret values are unchanged.
|
||||
persisted = _strip_secret_fields(merged_dict)
|
||||
if db_json_raw != persisted:
|
||||
await db_session.exec( # type: ignore
|
||||
text(
|
||||
"UPDATE settings SET data = :data, updated_at = :updated_at WHERE id = 1"
|
||||
).bindparams(
|
||||
data=json.dumps(merged_dict),
|
||||
data=json.dumps(persisted),
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
# Update the existing instance in-place for all live importers
|
||||
for k, v in merged_dict.items():
|
||||
setattr(settings, k, v)
|
||||
# (keeps the decrypted nsec live in memory).
|
||||
_apply_to_live_settings(merged_dict)
|
||||
cls._current = settings
|
||||
return cls._current
|
||||
|
||||
@@ -326,7 +400,7 @@ class SettingsService:
|
||||
text(
|
||||
"UPDATE settings SET data = :data, updated_at = :updated_at WHERE id = 1"
|
||||
).bindparams(
|
||||
data=json.dumps(candidate.dict()),
|
||||
data=json.dumps(_strip_secret_fields(candidate.dict())),
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
)
|
||||
@@ -355,3 +429,152 @@ class SettingsService:
|
||||
setattr(settings, k, v)
|
||||
cls._current = settings
|
||||
return settings
|
||||
|
||||
|
||||
async def _read_raw_settings_blob(db_session: AsyncSession) -> dict[str, Any]:
|
||||
"""Best-effort read of the raw persisted settings JSON (may not exist yet)."""
|
||||
from sqlmodel import text
|
||||
|
||||
try:
|
||||
result = await db_session.exec( # type: ignore
|
||||
text("SELECT data FROM settings WHERE id = 1")
|
||||
)
|
||||
row = result.first()
|
||||
except Exception:
|
||||
return {}
|
||||
if row is None:
|
||||
return {}
|
||||
(data_str,) = row
|
||||
try:
|
||||
data = json.loads(data_str) if isinstance(data_str, str) else dict(data_str)
|
||||
except Exception:
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _legacy_plaintext(
|
||||
raw_blob: dict[str, Any], env_name: str, blob_key: str
|
||||
) -> str | None:
|
||||
"""Legacy plaintext for a secret: env first, then the old settings blob."""
|
||||
env_value = os.environ.get(env_name)
|
||||
if env_value:
|
||||
return env_value
|
||||
blob_value = raw_blob.get(blob_key)
|
||||
if isinstance(blob_value, str) and blob_value:
|
||||
return blob_value
|
||||
return None
|
||||
|
||||
|
||||
async def bootstrap_secrets(db_session: AsyncSession) -> None:
|
||||
"""Move node secrets into the encrypted/hashed Secret store at startup.
|
||||
|
||||
Per secret:
|
||||
* column already set -> use it (the nsec is decrypted into the in-memory
|
||||
``settings``; a wrong ROUTSTR_SECRET_KEY surfaces as a clear fail-fast).
|
||||
* column empty but legacy plaintext exists (env, or the old settings
|
||||
blob) -> transform it (hash the password / encrypt the nsec) into the
|
||||
column.
|
||||
* nothing (admin password only) -> generate a strong random password,
|
||||
hash it, and log it once with the /admin URL.
|
||||
"""
|
||||
from cryptography.fernet import InvalidToken
|
||||
from sqlmodel import col, update
|
||||
|
||||
from . import vault
|
||||
from .db import NsecState, Secret, get_secret
|
||||
|
||||
raw_blob = await _read_raw_settings_blob(db_session)
|
||||
secret = await get_secret(db_session)
|
||||
changed = False
|
||||
|
||||
# Admin password — one-way scrypt hash.
|
||||
if secret.admin_password_hash is None:
|
||||
legacy_password = _legacy_plaintext(
|
||||
raw_blob, "ADMIN_PASSWORD", "admin_password"
|
||||
)
|
||||
if legacy_password:
|
||||
secret.admin_password_hash = vault.hash_password(legacy_password)
|
||||
changed = True
|
||||
else:
|
||||
generated = secrets.token_urlsafe(24)
|
||||
# Claim the empty slot atomically: only the worker whose UPDATE flips
|
||||
# NULL -> hash owns the generated password and announces it. On a
|
||||
# shared DB a racing worker gets rowcount 0, so it neither clobbers
|
||||
# the winner's hash (which the operator may already be using) nor
|
||||
# prints a second password that would never work.
|
||||
claim_stmt = (
|
||||
update(Secret)
|
||||
.where(col(Secret.id) == 1)
|
||||
.where(col(Secret.admin_password_hash).is_(None))
|
||||
.values(
|
||||
admin_password_hash=vault.hash_password(generated),
|
||||
updated_at=int(time.time()),
|
||||
)
|
||||
)
|
||||
result = await db_session.exec(claim_stmt) # type: ignore[call-overload]
|
||||
await db_session.commit()
|
||||
await db_session.refresh(secret)
|
||||
if result.rowcount == 1:
|
||||
admin_url = (settings.http_url or "http://localhost:8000").rstrip("/")
|
||||
# Print to stdout rather than the logger: the operator must see
|
||||
# this once (e.g. `docker compose logs`), but it must not be
|
||||
# persisted into the on-disk log files the logger also writes to.
|
||||
print(
|
||||
"No admin password set; generated a temporary one (shown "
|
||||
f"only now): {generated}\nLog in at {admin_url}/admin and "
|
||||
"change it from the dashboard settings.",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Nostr nsec — reversible Fernet encryption. ``nsec_state`` is the single
|
||||
# source of truth for ownership, so "intentionally cleared" is never
|
||||
# conflated with "never migrated" (the bug the old bool could not encode).
|
||||
if secret.nsec_state == NsecState.encrypted:
|
||||
# The vault owns the identity: decrypt the ciphertext, never re-read
|
||||
# env/blob. A missing ciphertext here means the row is inconsistent (a
|
||||
# failed write or manual edit); fail fast rather than silently dropping
|
||||
# the identity and falling back to a stale legacy copy.
|
||||
if secret.encrypted_nsec is None:
|
||||
raise RuntimeError(
|
||||
"nsec_state is 'encrypted' but no ciphertext is stored; the "
|
||||
"secrets row is inconsistent. Refusing to boot rather than "
|
||||
"silently resurrecting a stale legacy NSEC."
|
||||
)
|
||||
try:
|
||||
settings.nsec = vault.decrypt(secret.encrypted_nsec)
|
||||
except InvalidToken as exc:
|
||||
raise RuntimeError(
|
||||
"Stored nsec cannot be decrypted with the current "
|
||||
"ROUTSTR_SECRET_KEY. The key changed, or this database came from "
|
||||
"another node. Restore the original ROUTSTR_SECRET_KEY to recover."
|
||||
) from exc
|
||||
elif secret.nsec_state == NsecState.cleared:
|
||||
# The operator emptied the identity via the admin API. A fresh process
|
||||
# has already reloaded a stale ``NSEC`` from env/blob into the live
|
||||
# settings (and may have derived its npub); actively clear both so the
|
||||
# cleared store wins rather than silently resurrecting the old identity.
|
||||
settings.nsec = ""
|
||||
settings.npub = ""
|
||||
else: # NsecState.legacy — the vault has not taken ownership yet.
|
||||
# Import any legacy plaintext (env, or the old settings blob) exactly
|
||||
# once. Encryption at rest is mandatory, but a missing key is
|
||||
# provisioned, not fatal: vault.encrypt generates and persists a master
|
||||
# key (with a loud one-time operator notice) when none was supplied, so
|
||||
# an upgrading node keeps running. The nsec is never stored in plaintext.
|
||||
legacy_nsec = _legacy_plaintext(raw_blob, "NSEC", "nsec")
|
||||
if legacy_nsec:
|
||||
secret.encrypted_nsec = vault.encrypt(legacy_nsec)
|
||||
secret.nsec_state = NsecState.encrypted
|
||||
settings.nsec = legacy_nsec
|
||||
changed = True
|
||||
|
||||
# Derive npub from whatever nsec we now hold, if not already known.
|
||||
if settings.nsec and not settings.npub:
|
||||
npub = derive_npub_from_nsec(settings.nsec)
|
||||
if npub:
|
||||
settings.npub = npub
|
||||
|
||||
if changed:
|
||||
secret.updated_at = int(time.time())
|
||||
db_session.add(secret)
|
||||
await db_session.commit()
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
"""Encrypt/hash/fingerprint helpers for secrets at rest (issue #553).
|
||||
|
||||
Thin wrapper over ``cryptography`` so nothing else in the codebase touches
|
||||
Fernet/scrypt/HMAC directly:
|
||||
|
||||
- :func:`encrypt`/:func:`decrypt` — Fernet symmetric encryption, keyed by the
|
||||
mandatory master key. Ciphertext is self-describing (``fernet:v1:`` prefix) so
|
||||
a value can be told apart from legacy plaintext and so reading it under the
|
||||
wrong key surfaces as a hard error rather than silent corruption.
|
||||
- :func:`hash_password`/:func:`verify_password` — salted scrypt hashing. This is
|
||||
*key-independent*: it never reads the master key, so password login and the
|
||||
recovery script keep working even when the key is missing.
|
||||
|
||||
Key custody is flexible but encryption is not optional. The key comes from the
|
||||
``ROUTSTR_SECRET_KEY`` env var, else a persisted key file
|
||||
(``ROUTSTR_SECRET_KEY_FILE``, defaulting beside the SQLite database so it persists
|
||||
on the same volume as the data); when neither is set, :func:`encrypt` generates
|
||||
one to the key file and prints a one-time notice, so an existing node upgrades
|
||||
without breaking instead of refusing to boot. Reading is strict — :func:`decrypt`
|
||||
never generates a key (a new key could not match existing ciphertext) and fails
|
||||
fast with the generation command when none is configured. A malformed
|
||||
``ROUTSTR_SECRET_KEY`` is an operator error and always fails fast.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import secrets
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from cryptography.fernet import Fernet
|
||||
from sqlalchemy.engine import make_url
|
||||
from sqlalchemy.exc import ArgumentError
|
||||
|
||||
_PREFIX = "fernet:v1:"
|
||||
_GEN_COMMAND = (
|
||||
'python -c "from cryptography.fernet import Fernet; '
|
||||
'print(Fernet.generate_key().decode())"'
|
||||
)
|
||||
|
||||
# Where an auto-generated master key is persisted when the operator supplies no
|
||||
# ``ROUTSTR_SECRET_KEY``. Defaults beside the SQLite database so it rides whatever
|
||||
# volume already persists the data (a container recreate would otherwise generate
|
||||
# a fresh key and be unable to decrypt existing secrets); falls back to the
|
||||
# working directory when the DB location is unknown. Override the exact path with
|
||||
# ``ROUTSTR_SECRET_KEY_FILE``.
|
||||
_KEY_FILE_ENV = "ROUTSTR_SECRET_KEY_FILE"
|
||||
_DEFAULT_KEY_FILE = "routstr_secret.key"
|
||||
|
||||
# Minimum admin-password length, enforced wherever a password is set/changed
|
||||
# (admin endpoints + the recovery script) so the policy lives in one place.
|
||||
MIN_PASSWORD_LENGTH = 8
|
||||
|
||||
# scrypt parameters; packed into each hash so verification is parameter-free.
|
||||
_SCRYPT_N = 2**14
|
||||
_SCRYPT_R = 8
|
||||
_SCRYPT_P = 1
|
||||
_SCRYPT_DKLEN = 32
|
||||
_SCRYPT_SALT_BYTES = 16
|
||||
|
||||
|
||||
def _database_dir() -> Path | None:
|
||||
"""Directory of the SQLite database file, or ``None`` when it has no on-disk
|
||||
location (a non-SQLite URL or ``:memory:``).
|
||||
|
||||
Read from ``DATABASE_URL`` at call time and parsed here rather than importing
|
||||
``routstr.core.db`` — that module builds the engine at import, which the
|
||||
crypto layer must not drag in. Mirrors db.py's ``DATABASE_URL`` default.
|
||||
"""
|
||||
url_str = os.environ.get("DATABASE_URL", "sqlite+aiosqlite:///keys.db")
|
||||
try:
|
||||
url = make_url(url_str)
|
||||
except ArgumentError:
|
||||
return None
|
||||
if url.get_backend_name() != "sqlite" or not url.database:
|
||||
return None
|
||||
if url.database == ":memory:":
|
||||
return None
|
||||
return Path(url.database).parent
|
||||
|
||||
|
||||
def _key_file_path() -> Path:
|
||||
"""Where the auto-generated master key is read from / written to.
|
||||
|
||||
``ROUTSTR_SECRET_KEY_FILE`` wins; otherwise the key sits beside the SQLite
|
||||
database so it persists on the same volume as the data, falling back to the
|
||||
working directory when the DB location is unknown.
|
||||
"""
|
||||
override = os.environ.get(_KEY_FILE_ENV)
|
||||
if override:
|
||||
return Path(override)
|
||||
directory = _database_dir()
|
||||
return (directory or Path()) / _DEFAULT_KEY_FILE
|
||||
|
||||
|
||||
def _read_key_file(path: Path) -> str | None:
|
||||
try:
|
||||
stored = path.read_text().strip()
|
||||
except OSError:
|
||||
return None
|
||||
if not stored:
|
||||
return None
|
||||
_repair_key_file_perms(path)
|
||||
return stored
|
||||
|
||||
|
||||
def _repair_key_file_perms(path: Path) -> None:
|
||||
# A master key must never be group/other-readable. On POSIX, tighten loose
|
||||
# permissions to owner-only (0600) rather than trust — or hard-fail on — a
|
||||
# world-readable key; a friendlier repair keeps an upgrading node booting.
|
||||
if os.name != "posix":
|
||||
return
|
||||
try:
|
||||
mode = path.stat().st_mode
|
||||
except OSError:
|
||||
return
|
||||
if mode & 0o077:
|
||||
try:
|
||||
os.chmod(path, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _load_secret_key() -> str | None:
|
||||
"""The configured key without provisioning: env var, then the key file."""
|
||||
return os.environ.get("ROUTSTR_SECRET_KEY") or _read_key_file(_key_file_path())
|
||||
|
||||
|
||||
def _warn_generated_key(path: Path) -> None:
|
||||
# stdout, not the logger: the operator must see this once (e.g. in
|
||||
# ``docker compose logs``), but it must never be persisted into the on-disk
|
||||
# log files the logger also writes. Mirrors the generated-admin-password
|
||||
# notice so an upgrade cannot silently create an unbacked key.
|
||||
print(
|
||||
"No ROUTSTR_SECRET_KEY was set; generated one to encrypt node secrets at "
|
||||
f"rest and saved it to {path}.\n"
|
||||
"!! BACK UP THIS FILE. If it is lost, the encrypted secrets cannot be "
|
||||
"recovered and will have to be re-entered.\n"
|
||||
"To manage the key yourself (e.g. from a secrets manager) set "
|
||||
"ROUTSTR_SECRET_KEY in the environment instead; the value is in the file "
|
||||
"above.",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
def _generate_and_persist_key(path: Path) -> str:
|
||||
"""Generate a Fernet key, persist it owner-only and atomically, warn once.
|
||||
|
||||
The key is written to a temp file in the same directory, flushed durably,
|
||||
then ``os.link``-ed into place. ``os.link`` publishes the complete file in a
|
||||
single atomic step — a crash mid-write leaves only the temp file (which is
|
||||
removed), never a half-written or empty key at the final path that a later
|
||||
boot would read as corrupt. It also refuses to overwrite an existing key, so
|
||||
a racing worker that generated first keeps ownership (secrets may already be
|
||||
encrypted under its key); the loser adopts that key instead of clobbering it.
|
||||
"""
|
||||
key = Fernet.generate_key().decode()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp_name = tempfile.mkstemp(
|
||||
dir=path.parent, prefix=".routstr_secret.", suffix=".tmp"
|
||||
)
|
||||
tmp = Path(tmp_name)
|
||||
try:
|
||||
with os.fdopen(fd, "w") as handle:
|
||||
handle.write(key) # mkstemp already created it 0600
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
try:
|
||||
os.link(tmp, path)
|
||||
except FileExistsError:
|
||||
# A concurrent worker linked its key in first; adopt theirs rather
|
||||
# than clobber a key that secrets may already be encrypted under.
|
||||
existing = _read_key_file(path)
|
||||
if existing:
|
||||
return existing
|
||||
raise
|
||||
_fsync_dir(path.parent)
|
||||
finally:
|
||||
tmp.unlink(missing_ok=True)
|
||||
_warn_generated_key(path)
|
||||
return key
|
||||
|
||||
|
||||
def _fsync_dir(directory: Path) -> None:
|
||||
# Persist the new directory entry so the linked key survives a crash right
|
||||
# after publish. Best-effort: not every platform lets you fsync a directory.
|
||||
try:
|
||||
dir_fd = os.open(directory, os.O_RDONLY)
|
||||
except OSError:
|
||||
return
|
||||
try:
|
||||
os.fsync(dir_fd)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
os.close(dir_fd)
|
||||
|
||||
|
||||
def ensure_secret_key() -> str:
|
||||
"""Return the master key, provisioning one if the operator supplied none.
|
||||
|
||||
Precedence: the ``ROUTSTR_SECRET_KEY`` env var, then the persisted key file,
|
||||
otherwise a freshly generated key written to the key file (with a one-time
|
||||
operator notice). This keeps encryption at rest mandatory while letting an
|
||||
existing node upgrade without setting a key first. A malformed env key is
|
||||
left to fail at :func:`get_fernet` — it is an operator error, not an unset
|
||||
key, so it must not trigger silent self-provisioning.
|
||||
"""
|
||||
env_key = os.environ.get("ROUTSTR_SECRET_KEY")
|
||||
if env_key:
|
||||
return env_key
|
||||
path = _key_file_path()
|
||||
return _read_key_file(path) or _generate_and_persist_key(path)
|
||||
|
||||
|
||||
def _fernet_from_key(key: str) -> Fernet:
|
||||
try:
|
||||
return Fernet(key.encode())
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise RuntimeError(
|
||||
"ROUTSTR_SECRET_KEY is malformed; it must be a url-safe base64 "
|
||||
"32-byte Fernet key. Generate one with:\n " + _GEN_COMMAND
|
||||
) from exc
|
||||
|
||||
|
||||
def get_fernet() -> Fernet:
|
||||
"""Build a :class:`Fernet` from the configured key (env var or key file).
|
||||
|
||||
Strict: this never generates a key, so already-encrypted ciphertext is never
|
||||
shadowed by a fresh key. A read with no key configured fails fast with the
|
||||
generation command.
|
||||
"""
|
||||
key = _load_secret_key()
|
||||
if not key:
|
||||
raise RuntimeError(
|
||||
"ROUTSTR_SECRET_KEY is not set. It is required to encrypt secrets at "
|
||||
"rest. Generate one with:\n " + _GEN_COMMAND
|
||||
)
|
||||
return _fernet_from_key(key)
|
||||
|
||||
|
||||
def encrypt(plaintext: str) -> str:
|
||||
"""Encrypt ``plaintext`` into a self-describing ``fernet:v1:`` token.
|
||||
|
||||
Provisions a master key (env var, key file, or a freshly generated one) so an
|
||||
upgrading node never has to set one before its first secret is stored; the
|
||||
value is always encrypted, never persisted in plaintext.
|
||||
"""
|
||||
fernet = _fernet_from_key(ensure_secret_key())
|
||||
return _PREFIX + fernet.encrypt(plaintext.encode()).decode()
|
||||
|
||||
|
||||
def is_encrypted(value: str) -> bool:
|
||||
"""True if ``value`` carries the ``fernet:v1:`` prefix this module emits."""
|
||||
return value.startswith(_PREFIX)
|
||||
|
||||
|
||||
def decrypt(ciphertext: str) -> str:
|
||||
"""Decrypt a ``fernet:v1:`` token.
|
||||
|
||||
Raises ``ValueError`` for an unprefixed value (so legacy plaintext is never
|
||||
mistaken for ciphertext) and ``InvalidToken`` when the value was written
|
||||
under a different ``ROUTSTR_SECRET_KEY``.
|
||||
"""
|
||||
if not is_encrypted(ciphertext):
|
||||
raise ValueError("value is not fernet:v1: ciphertext")
|
||||
token = ciphertext[len(_PREFIX) :]
|
||||
return get_fernet().decrypt(token.encode()).decode()
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
"""Salted scrypt hash, self-describing as ``scrypt:n:r:p:salt:hash``."""
|
||||
salt = secrets.token_bytes(_SCRYPT_SALT_BYTES)
|
||||
derived = hashlib.scrypt(
|
||||
password.encode(),
|
||||
salt=salt,
|
||||
n=_SCRYPT_N,
|
||||
r=_SCRYPT_R,
|
||||
p=_SCRYPT_P,
|
||||
dklen=_SCRYPT_DKLEN,
|
||||
)
|
||||
return ":".join(
|
||||
[
|
||||
"scrypt",
|
||||
str(_SCRYPT_N),
|
||||
str(_SCRYPT_R),
|
||||
str(_SCRYPT_P),
|
||||
base64.b64encode(salt).decode(),
|
||||
base64.b64encode(derived).decode(),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def verify_password(password: str, stored: str) -> bool:
|
||||
"""Constant-time check of ``password`` against a :func:`hash_password` value."""
|
||||
try:
|
||||
scheme, n, r, p, salt_b64, hash_b64 = stored.split(":")
|
||||
if scheme != "scrypt":
|
||||
return False
|
||||
n_int, r_int, p_int = int(n), int(r), int(p)
|
||||
# Cap the work factor at the parameters this module emits. scrypt's
|
||||
# memory cost grows with N*r, so an oversized N/r in a tampered or
|
||||
# corrupt stored hash could turn a single login into an OOM/DoS.
|
||||
if n_int > _SCRYPT_N or r_int > _SCRYPT_R or p_int > _SCRYPT_P:
|
||||
return False
|
||||
salt = base64.b64decode(salt_b64)
|
||||
expected = base64.b64decode(hash_b64)
|
||||
derived = hashlib.scrypt(
|
||||
password.encode(),
|
||||
salt=salt,
|
||||
n=n_int,
|
||||
r=r_int,
|
||||
p=p_int,
|
||||
dklen=len(expected),
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
return hmac.compare_digest(derived, expected)
|
||||
@@ -1,4 +1,5 @@
|
||||
import math
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from pydantic.v1 import BaseModel
|
||||
|
||||
@@ -7,6 +8,9 @@ from ..core.settings import settings
|
||||
from .price import sats_usd_price
|
||||
from .usage import normalize_usage, parse_token_count
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .models import Model
|
||||
|
||||
__all__ = [
|
||||
"CostData",
|
||||
"CostDataError",
|
||||
@@ -66,12 +70,23 @@ def _empty_cost(cls: type[CostData] = CostData) -> CostData:
|
||||
async def calculate_cost(
|
||||
response_data: dict,
|
||||
max_cost: int,
|
||||
model_obj: "Model | None" = None,
|
||||
provider_fee: float | None = None,
|
||||
) -> CostData | MaxCostData | CostDataError:
|
||||
"""Calculate the cost of an API request based on token usage.
|
||||
|
||||
Args:
|
||||
response_data: Response data containing usage information
|
||||
max_cost: Maximum cost in millisats
|
||||
model_obj: The model that actually served the request. When given,
|
||||
its pricing is billed directly; without it, pricing is re-derived
|
||||
from the response's model string via the alias map, which resolves
|
||||
to the best-ranked candidate — not necessarily the serving one.
|
||||
provider_fee: The serving provider's fee multiplier, applied on the
|
||||
USD-cost path and the litellm pricing fallback (configured model
|
||||
pricing already carries the fee baked in). Without it, the fee is
|
||||
re-derived from the response's model string, which yields the
|
||||
best-ranked provider's fee.
|
||||
|
||||
Returns:
|
||||
Cost data or error information
|
||||
@@ -157,11 +172,16 @@ async def calculate_cost(
|
||||
},
|
||||
)
|
||||
try:
|
||||
cost_details = usage_data.get("cost_details", {})
|
||||
if not isinstance(cost_details, dict):
|
||||
cost_details = {}
|
||||
input_usd = _coerce_usd(
|
||||
usage_data.get("cost_details", {}).get("input_cost", 0)
|
||||
cost_details.get("input_cost")
|
||||
or cost_details.get("upstream_inference_prompt_cost")
|
||||
)
|
||||
output_usd = _coerce_usd(
|
||||
usage_data.get("cost_details", {}).get("output_cost", 0)
|
||||
cost_details.get("output_cost")
|
||||
or cost_details.get("upstream_inference_completions_cost")
|
||||
)
|
||||
return _calculate_from_usd_cost(
|
||||
usd_cost,
|
||||
@@ -172,6 +192,7 @@ async def calculate_cost(
|
||||
cache_creation_tokens,
|
||||
output_tokens,
|
||||
response_data,
|
||||
provider_fee,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
@@ -185,7 +206,7 @@ async def calculate_cost(
|
||||
|
||||
# Fall back to token-based pricing
|
||||
try:
|
||||
pricing_rates = _get_pricing_rates(response_data)
|
||||
pricing_rates = _get_pricing_rates(response_data, model_obj, provider_fee)
|
||||
except ValueError as e:
|
||||
return CostDataError(message=str(e), code="pricing_error")
|
||||
|
||||
@@ -259,7 +280,17 @@ def _coerce_usd(value: object) -> float:
|
||||
def _resolve_usd_cost(usage_data: dict, response_data: dict) -> float:
|
||||
"""Resolve USD cost with clear priority order.
|
||||
|
||||
Priority: cost_details.total_cost → total_cost → cost (in both usage and response).
|
||||
Priority:
|
||||
|
||||
1. ``cost_details.total_cost``
|
||||
2. ``cost_details.upstream_inference_cost`` (BYOK — see below)
|
||||
3. ``total_cost`` → ``cost`` (in both usage and response)
|
||||
|
||||
**BYOK path (PPQ.AI):** when ``is_byok`` is true the ``usage.cost`` field
|
||||
is only a small (~5 %) routing fee, not the inference cost. The real cost
|
||||
lives in ``cost_details.upstream_inference_cost`` and the provider's
|
||||
balance is debited by ``upstream_inference_cost + byok_fee``. Billing just
|
||||
the fee under-charges by ~20×.
|
||||
"""
|
||||
cost_details = usage_data.get("cost_details")
|
||||
if isinstance(cost_details, dict):
|
||||
@@ -267,6 +298,18 @@ def _resolve_usd_cost(usage_data: dict, response_data: dict) -> float:
|
||||
if cost > 0:
|
||||
return cost
|
||||
|
||||
# PPQ.AI BYOK: upstream_inference_cost is the real inference cost;
|
||||
# usage.cost is only a ~5 % BYOK routing fee. Bill the sum — what PPQ
|
||||
# actually deducts from the balance. For non-BYOK providers (e.g.
|
||||
# OpenRouter) usage.cost already equals upstream_inference_cost, so we
|
||||
# fall through to the normal ``cost`` lookup below.
|
||||
upstream_cost = _coerce_usd(
|
||||
cost_details.get("upstream_inference_cost")
|
||||
)
|
||||
if upstream_cost > 0 and usage_data.get("is_byok"):
|
||||
byok_fee = _coerce_usd(usage_data.get("cost"))
|
||||
return upstream_cost + byok_fee
|
||||
|
||||
for source in [usage_data, response_data]:
|
||||
if not isinstance(source, dict):
|
||||
continue
|
||||
@@ -280,55 +323,106 @@ def _resolve_usd_cost(usage_data: dict, response_data: dict) -> float:
|
||||
|
||||
def _get_pricing_rates(
|
||||
response_data: dict,
|
||||
model_obj: "Model | None",
|
||||
provider_fee: float | None,
|
||||
) -> tuple[float, float, float, float] | None:
|
||||
"""Get model-based pricing rates or None if using fixed pricing.
|
||||
"""Get configured rates, falling back to LiteLLM's model cost map.
|
||||
|
||||
Returns: (input_rate, output_rate, cache_read_rate, cache_write_rate)
|
||||
The served ``model_obj`` (when the caller has it) is billed directly;
|
||||
otherwise the response's model string is resolved through the alias map,
|
||||
which yields the best-ranked candidate rather than the serving one.
|
||||
|
||||
Returns: (input_rate, output_rate, cache_read_rate, cache_write_rate).
|
||||
``None`` means configured fixed pricing should be used by the caller.
|
||||
"""
|
||||
if settings.fixed_pricing:
|
||||
if settings.fixed_pricing and (
|
||||
settings.fixed_per_1k_input_tokens
|
||||
or settings.fixed_per_1k_output_tokens
|
||||
):
|
||||
return None
|
||||
|
||||
from ..proxy import get_model_instance
|
||||
from .models import litellm_cost_entry
|
||||
|
||||
response_model = response_data.get("model", "")
|
||||
model_obj = get_model_instance(response_model)
|
||||
|
||||
if not model_obj:
|
||||
logger.error("Invalid model in response", extra={"response_model": response_model})
|
||||
raise ValueError(f"Invalid model: {response_model}")
|
||||
|
||||
if not model_obj.sats_pricing:
|
||||
logger.error(
|
||||
"Model pricing not defined",
|
||||
extra={"model": response_model, "model_id": response_model},
|
||||
if model_obj is None:
|
||||
logger.warning(
|
||||
"Settling without routed model identity — re-deriving pricing "
|
||||
"from the response's model string via the alias map",
|
||||
extra={"response_model": response_model},
|
||||
)
|
||||
raise ValueError("Model pricing not defined")
|
||||
model_obj = get_model_instance(response_model)
|
||||
|
||||
try:
|
||||
mspp = float(model_obj.sats_pricing.prompt)
|
||||
mspc = float(model_obj.sats_pricing.completion)
|
||||
mscr = float(model_obj.sats_pricing.input_cache_read or 0)
|
||||
mscw = float(model_obj.sats_pricing.input_cache_write or 0)
|
||||
if model_obj and model_obj.sats_pricing:
|
||||
try:
|
||||
mspp = float(model_obj.sats_pricing.prompt)
|
||||
mspc = float(model_obj.sats_pricing.completion)
|
||||
mscr = float(model_obj.sats_pricing.input_cache_read or 0)
|
||||
mscw = float(model_obj.sats_pricing.input_cache_write or 0)
|
||||
|
||||
mspp_1k = mspp * 1_000_000.0
|
||||
mspc_1k = mspc * 1_000_000.0
|
||||
mscr_1k = mscr * 1_000_000.0 if mscr > 0 else mspp_1k
|
||||
mscw_1k = mscw * 1_000_000.0 if mscw > 0 else mspp_1k
|
||||
mspp_1k = mspp * 1_000_000.0
|
||||
mspc_1k = mspc * 1_000_000.0
|
||||
mscr_1k = mscr * 1_000_000.0 if mscr > 0 else mspp_1k
|
||||
mscw_1k = mscw * 1_000_000.0 if mscw > 0 else mspp_1k
|
||||
source = "configured"
|
||||
except Exception as e:
|
||||
logger.error("Invalid pricing data", extra={"error": str(e)})
|
||||
raise ValueError("Invalid pricing data") from e
|
||||
else:
|
||||
pricing_model = (
|
||||
model_obj.forwarded_model_id if model_obj else None
|
||||
) or response_model
|
||||
pricing = litellm_cost_entry(pricing_model)
|
||||
if pricing is None:
|
||||
logger.error(
|
||||
"Model pricing not found in configured models or LiteLLM",
|
||||
extra={
|
||||
"response_model": response_model,
|
||||
"pricing_model": pricing_model,
|
||||
},
|
||||
)
|
||||
raise ValueError(f"Pricing not found for model: {response_model}")
|
||||
|
||||
logger.info(
|
||||
"Applied model-specific pricing",
|
||||
extra={
|
||||
"model": response_model,
|
||||
"input_price_msats_per_1k": mspp_1k,
|
||||
"output_price_msats_per_1k": mspc_1k,
|
||||
"cache_read_price_msats_per_1k": mscr_1k,
|
||||
"cache_write_price_msats_per_1k": mscw_1k,
|
||||
},
|
||||
input_usd = _coerce_usd(pricing.get("input_cost_per_token"))
|
||||
output_usd = _coerce_usd(pricing.get("output_cost_per_token"))
|
||||
if input_usd <= 0 or output_usd <= 0:
|
||||
raise ValueError(f"Incomplete LiteLLM pricing for model: {pricing_model}")
|
||||
|
||||
if provider_fee is None:
|
||||
provider_fee = _resolve_provider_fee(response_model)
|
||||
usd_per_sat = sats_usd_price()
|
||||
mspp_1k = input_usd * provider_fee * 1_000_000.0 / usd_per_sat
|
||||
mspc_1k = output_usd * provider_fee * 1_000_000.0 / usd_per_sat
|
||||
cache_read_usd = _coerce_usd(
|
||||
pricing.get("cache_read_input_token_cost")
|
||||
)
|
||||
return mspp_1k, mspc_1k, mscr_1k, mscw_1k
|
||||
except Exception as e:
|
||||
logger.error("Invalid pricing data", extra={"error": str(e)})
|
||||
raise ValueError("Invalid pricing data") from e
|
||||
cache_write_usd = _coerce_usd(
|
||||
pricing.get("cache_creation_input_token_cost")
|
||||
)
|
||||
mscr_1k = (
|
||||
cache_read_usd * provider_fee * 1_000_000.0 / usd_per_sat
|
||||
if cache_read_usd > 0
|
||||
else mspp_1k
|
||||
)
|
||||
mscw_1k = (
|
||||
cache_write_usd * provider_fee * 1_000_000.0 / usd_per_sat
|
||||
if cache_write_usd > 0
|
||||
else mspp_1k
|
||||
)
|
||||
source = "litellm"
|
||||
|
||||
logger.info(
|
||||
"Applied model-specific pricing",
|
||||
extra={
|
||||
"model": response_model,
|
||||
"pricing_source": source,
|
||||
"input_price_msats_per_1k": mspp_1k,
|
||||
"output_price_msats_per_1k": mspc_1k,
|
||||
"cache_read_price_msats_per_1k": mscr_1k,
|
||||
"cache_write_price_msats_per_1k": mscw_1k,
|
||||
},
|
||||
)
|
||||
return mspp_1k, mspc_1k, mscr_1k, mscw_1k
|
||||
|
||||
|
||||
def _resolve_provider_fee(model_id: str) -> float:
|
||||
@@ -356,9 +450,11 @@ def _calculate_from_usd_cost(
|
||||
cache_creation_tokens: int,
|
||||
output_tokens: int,
|
||||
response_data: dict,
|
||||
provider_fee: float | None,
|
||||
) -> CostData:
|
||||
"""Calculate cost from USD figures, deriving input/output split from tokens."""
|
||||
provider_fee = _resolve_provider_fee(response_data.get("model", ""))
|
||||
if provider_fee is None:
|
||||
provider_fee = _resolve_provider_fee(response_data.get("model", ""))
|
||||
usd_cost = usd_cost * provider_fee
|
||||
input_usd = input_usd * provider_fee
|
||||
output_usd = output_usd * provider_fee
|
||||
@@ -367,8 +463,12 @@ def _calculate_from_usd_cost(
|
||||
cost_in_msats = math.ceil(cost_in_sats * 1000)
|
||||
|
||||
if input_usd > 0 or output_usd > 0:
|
||||
input_msats = int((input_usd * sats_per_usd) * 1000)
|
||||
output_msats = int((output_usd * sats_per_usd) * 1000)
|
||||
# The total is the authoritative billed amount. Allocating that integer
|
||||
# total proportionally avoids losing sub-millisatoshi remainders when
|
||||
# input and output components are each truncated independently.
|
||||
component_usd = input_usd + output_usd
|
||||
input_msats = math.floor(cost_in_msats * input_usd / component_usd)
|
||||
output_msats = cost_in_msats - input_msats
|
||||
else:
|
||||
effective_input_tokens = (
|
||||
input_tokens + cache_read_tokens + cache_creation_tokens
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
from typing import TypedDict
|
||||
|
||||
import httpx
|
||||
from cashu.wallet.wallet import Proof, Wallet
|
||||
|
||||
# The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung or
|
||||
# very slow mint can block a melt (and any caller, e.g. the payout loop)
|
||||
# indefinitely. Bound it here so callers fail instead of hanging forever.
|
||||
MELT_TIMEOUT_SECONDS = 60
|
||||
|
||||
try:
|
||||
from bech32 import bech32_decode, convertbits # type: ignore
|
||||
except ModuleNotFoundError: # pragma: no cover – allow runtime miss
|
||||
@@ -220,10 +226,18 @@ async def raw_send_to_lnurl(
|
||||
if amount:
|
||||
proofs, _ = await wallet.select_to_send(proofs, amount, set_reserved=True)
|
||||
|
||||
_ = await wallet.melt(
|
||||
proofs=proofs,
|
||||
invoice=bolt11_invoice,
|
||||
fee_reserve_sat=melt_quote_resp.fee_reserve,
|
||||
quote_id=melt_quote_resp.quote,
|
||||
)
|
||||
try:
|
||||
_ = await asyncio.wait_for(
|
||||
wallet.melt(
|
||||
proofs=proofs,
|
||||
invoice=bolt11_invoice,
|
||||
fee_reserve_sat=melt_quote_resp.fee_reserve,
|
||||
quote_id=melt_quote_resp.quote,
|
||||
),
|
||||
timeout=MELT_TIMEOUT_SECONDS,
|
||||
)
|
||||
except asyncio.TimeoutError as e:
|
||||
raise LNURLError(
|
||||
f"Melt timed out after {MELT_TIMEOUT_SECONDS}s (mint unresponsive)"
|
||||
) from e
|
||||
return final_amount
|
||||
|
||||
@@ -455,7 +455,9 @@ async def _update_sats_pricing_once() -> None:
|
||||
for m in upstream.get_cached_models()
|
||||
]
|
||||
upstream._models_cache = updated_models
|
||||
upstream._models_by_id = {m.forwarded_model_id or m.id: m for m in updated_models}
|
||||
upstream._models_by_id = {
|
||||
m.forwarded_model_id or m.id: m for m in updated_models
|
||||
}
|
||||
updated_count += len(updated_models)
|
||||
|
||||
if updated_count > 0:
|
||||
@@ -510,9 +512,7 @@ class ModelTestRequest(V2BaseModel):
|
||||
request_data: dict
|
||||
|
||||
|
||||
@models_router.post(
|
||||
"/api/models/test", dependencies=[Depends(_require_admin_api)]
|
||||
)
|
||||
@models_router.post("/api/models/test", dependencies=[Depends(_require_admin_api)])
|
||||
async def test_model(
|
||||
payload: ModelTestRequest,
|
||||
session: AsyncSession = Depends(get_session),
|
||||
@@ -595,6 +595,37 @@ async def test_model(
|
||||
}
|
||||
|
||||
|
||||
@models_router.get("/v1/models/paths")
|
||||
@models_router.get("/v1/models/paths/", include_in_schema=False)
|
||||
async def model_paths() -> dict:
|
||||
"""All models with every upstream provider path they are reachable through."""
|
||||
from ..upstream.model_paths import get_all_model_paths
|
||||
|
||||
return await get_all_model_paths()
|
||||
|
||||
|
||||
@models_router.get("/v1/models/paths/model")
|
||||
@models_router.get("/v1/models/paths/model/", include_in_schema=False)
|
||||
async def model_paths_for_model(model_id: str) -> dict:
|
||||
"""Paths for a single model.
|
||||
|
||||
Uses a query parameter (``?model_id=...``) under a fully static route so
|
||||
model ids containing ``/`` (e.g. ``anthropic/claude-opus-4.6``) need no URL
|
||||
encoding and there is no dynamic-route ambiguity.
|
||||
"""
|
||||
from ..proxy import get_unique_models
|
||||
from ..upstream.model_paths import get_paths_for_model
|
||||
|
||||
result = await get_paths_for_model(model_id)
|
||||
if not result["data"]:
|
||||
advertised_ids = {
|
||||
model.forwarded_model_id or model.id for model in get_unique_models()
|
||||
}
|
||||
if model_id not in advertised_ids:
|
||||
raise HTTPException(status_code=404, detail="Model not found")
|
||||
return result
|
||||
|
||||
|
||||
@models_router.get("/v1/models")
|
||||
@models_router.get("/v1/models/", include_in_schema=False)
|
||||
@models_router.get("/models")
|
||||
|
||||
+137
-64
@@ -7,7 +7,13 @@ from fastapi.responses import Response, StreamingResponse
|
||||
from sqlmodel import select
|
||||
|
||||
from .algorithm import create_model_mappings
|
||||
from .auth import pay_for_request, revert_pay_for_request, validate_bearer_key
|
||||
from .auth import (
|
||||
ReservationSnapshot,
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
revert_pay_for_request,
|
||||
validate_bearer_key,
|
||||
)
|
||||
from .core import get_logger
|
||||
from .core.db import (
|
||||
ApiKey,
|
||||
@@ -37,10 +43,9 @@ logger = get_logger(__name__)
|
||||
proxy_router = APIRouter()
|
||||
|
||||
_upstreams: list[BaseUpstreamProvider] = []
|
||||
_model_instances: dict[str, Model] = {} # All aliases -> Model
|
||||
_provider_map: dict[
|
||||
str, list[BaseUpstreamProvider]
|
||||
] = {} # All aliases -> List[Provider]
|
||||
str, list[tuple[Model, BaseUpstreamProvider]]
|
||||
] = {} # All aliases -> sorted [(candidate Model, its Provider)]
|
||||
_unique_models: dict[str, Model] = {} # Unique model.id -> Model (no duplicates)
|
||||
|
||||
|
||||
@@ -72,32 +77,44 @@ def get_upstreams() -> list[BaseUpstreamProvider]:
|
||||
return _upstreams
|
||||
|
||||
|
||||
def get_model_instance(model_id: str) -> Model | None:
|
||||
"""Get Model instance by ID from global cache."""
|
||||
def get_candidates(
|
||||
model_id: str,
|
||||
) -> list[tuple[Model, BaseUpstreamProvider]] | None:
|
||||
"""Get the sorted (model, provider) candidate list for a model ID.
|
||||
|
||||
Each provider is paired with its own model for the alias, so routing can
|
||||
forward and bill the candidate that actually serves. Version suffixes
|
||||
(e.g. ``-20251222``) are stripped as a retry when the exact ID is
|
||||
unknown, since upstreams may return a specific version of a base model
|
||||
we track.
|
||||
"""
|
||||
if not model_id:
|
||||
return None
|
||||
|
||||
model_id_lower = model_id.lower()
|
||||
# Try exact match first
|
||||
if model := _model_instances.get(model_id_lower):
|
||||
return model
|
||||
if candidates := _provider_map.get(model_id_lower):
|
||||
return candidates
|
||||
|
||||
# Try stripping common version suffixes (e.g., -20251222)
|
||||
# This handles cases where upstream returns a specific version
|
||||
# but we only track the base model name.
|
||||
import re
|
||||
|
||||
base_model_id = re.sub(r"-\d{8}$", "", model_id_lower)
|
||||
if base_model_id != model_id_lower:
|
||||
if model := _model_instances.get(base_model_id):
|
||||
return model
|
||||
if candidates := _provider_map.get(base_model_id):
|
||||
return candidates
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_model_instance(model_id: str) -> Model | None:
|
||||
"""Get the best-ranked Model instance for a model ID."""
|
||||
candidates = get_candidates(model_id)
|
||||
return candidates[0][0] if candidates else None
|
||||
|
||||
|
||||
def get_provider_for_model(model_id: str) -> list[BaseUpstreamProvider] | None:
|
||||
"""Get UpstreamProvider list for model ID from global cache."""
|
||||
return _provider_map.get(model_id.lower())
|
||||
"""Get the sorted UpstreamProvider list for a model ID."""
|
||||
candidates = get_candidates(model_id)
|
||||
return [provider for _, provider in candidates] if candidates else None
|
||||
|
||||
|
||||
def get_unique_models() -> list[Model]:
|
||||
@@ -106,8 +123,13 @@ def get_unique_models() -> list[Model]:
|
||||
|
||||
|
||||
def _is_tinfoil_attestation_path(path: str) -> bool:
|
||||
"""Return True for Tinfoil attestation-bundle proxy paths."""
|
||||
return path in {"attestation", "tee/attestation"}
|
||||
"""Return True for exact Tinfoil attestation routes, with optional slash."""
|
||||
return path in {
|
||||
"attestation",
|
||||
"attestation/",
|
||||
"tee/attestation",
|
||||
"tee/attestation/",
|
||||
}
|
||||
|
||||
|
||||
def _select_unauthenticated_get_upstreams(
|
||||
@@ -132,7 +154,7 @@ async def refresh_model_maps() -> None:
|
||||
"""Refresh global model and provider maps using the cost-based algorithm."""
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
global _model_instances, _provider_map, _unique_models
|
||||
global _provider_map, _unique_models
|
||||
|
||||
async with create_session() as session:
|
||||
# Fetch all providers with their models in a single logical operation
|
||||
@@ -142,24 +164,38 @@ async def refresh_model_maps() -> None:
|
||||
result = await session.exec(query)
|
||||
provider_rows = result.all()
|
||||
|
||||
overrides_by_id: dict[str, tuple[ModelRow, float]] = {}
|
||||
disabled_model_ids: set[str] = set()
|
||||
overrides_by_key: dict[tuple[str, int], tuple[ModelRow, float]] = {}
|
||||
disabled_model_keys: set[tuple[str, int]] = set()
|
||||
|
||||
for provider in provider_rows:
|
||||
if not provider.enabled:
|
||||
continue
|
||||
for model in provider.models:
|
||||
model_key = (model.id.lower(), model.upstream_provider_id)
|
||||
if model.enabled:
|
||||
overrides_by_id[model.id] = (model, provider.provider_fee)
|
||||
overrides_by_key[model_key] = (model, provider.provider_fee)
|
||||
else:
|
||||
disabled_model_ids.add(model.id)
|
||||
disabled_model_keys.add(model_key)
|
||||
|
||||
_model_instances, _provider_map, _unique_models = create_model_mappings(
|
||||
_, _provider_map, _unique_models = create_model_mappings(
|
||||
upstreams=_upstreams,
|
||||
overrides_by_id=overrides_by_id,
|
||||
disabled_model_ids=disabled_model_ids,
|
||||
overrides_by_key=overrides_by_key,
|
||||
disabled_model_keys=disabled_model_keys,
|
||||
)
|
||||
|
||||
# Keep model-path discovery in sync with admin mutations: disabling or
|
||||
# deleting a provider must stop advertising its paths immediately rather
|
||||
# than after the next timed refresh.
|
||||
from .upstream.model_paths import prune_model_paths_for_inactive_providers
|
||||
|
||||
try:
|
||||
await prune_model_paths_for_inactive_providers()
|
||||
except Exception as e: # noqa: BLE001 - discovery sync must not break routing
|
||||
logger.warning(
|
||||
"Failed to prune model paths for inactive providers",
|
||||
extra={"error": str(e), "error_type": type(e).__name__},
|
||||
)
|
||||
|
||||
|
||||
async def refresh_model_maps_periodically() -> None:
|
||||
"""Background task to refresh model maps every minute."""
|
||||
@@ -233,13 +269,10 @@ async def proxy(
|
||||
else:
|
||||
model_id = request_body_dict.get("model", "unknown")
|
||||
|
||||
# /tee/* and /attestation GET requests don't map to models — forward
|
||||
# without model/cost/auth lookups. Tinfoil attestation paths are routed
|
||||
# only to Tinfoil providers so an unrelated upstream's 404 cannot
|
||||
# short-circuit before the attestation proxy is tried.
|
||||
if request.method == "GET" and (
|
||||
path.startswith("tee/") or path.startswith("attestation")
|
||||
):
|
||||
# Exact Tinfoil attestation GET routes don't map to models — forward
|
||||
# without model/cost/auth lookups. Do not prefix-match here: paths such as
|
||||
# /attestationjunk must continue through normal authentication.
|
||||
if request.method == "GET" and _is_tinfoil_attestation_path(path):
|
||||
selected_upstreams = _select_unauthenticated_get_upstreams(path, _upstreams)
|
||||
if not selected_upstreams:
|
||||
return create_error_response(
|
||||
@@ -254,7 +287,10 @@ async def proxy(
|
||||
try:
|
||||
headers = upstream.prepare_headers(dict(request.headers))
|
||||
response = await upstream.forward_get_request(request, path, headers)
|
||||
if response.status_code in [502, 429] and i < len(selected_upstreams) - 1:
|
||||
if (
|
||||
response.status_code in [502, 429]
|
||||
and i < len(selected_upstreams) - 1
|
||||
):
|
||||
logger.warning(
|
||||
"Upstream %s returned %s for unauthenticated GET %s, trying next",
|
||||
upstream.provider_type,
|
||||
@@ -277,25 +313,20 @@ async def proxy(
|
||||
"upstream_error", "All upstreams failed", 502, request=request
|
||||
)
|
||||
|
||||
model_obj = get_model_instance(model_id)
|
||||
candidates = get_candidates(model_id)
|
||||
|
||||
if not model_obj:
|
||||
if not candidates:
|
||||
return create_error_response(
|
||||
"invalid_model", f"Model '{model_id}' not found", 400, request=request
|
||||
)
|
||||
|
||||
upstreams = get_provider_for_model(model_id)
|
||||
if not upstreams:
|
||||
return create_error_response(
|
||||
"invalid_model",
|
||||
f"No provider found for model '{model_id}'",
|
||||
400,
|
||||
request=request,
|
||||
)
|
||||
|
||||
if is_ehbp:
|
||||
upstreams = [upstream for upstream in upstreams if upstream.supports_ehbp]
|
||||
if not upstreams:
|
||||
candidates = [
|
||||
(model, upstream)
|
||||
for model, upstream in candidates
|
||||
if upstream.supports_ehbp
|
||||
]
|
||||
if not candidates:
|
||||
return create_error_response(
|
||||
"unsupported_request",
|
||||
f"No EHBP-capable provider found for model '{model_id}'",
|
||||
@@ -303,9 +334,10 @@ async def proxy(
|
||||
request=request,
|
||||
)
|
||||
|
||||
# todo figure out cost calculation since fallback provider is usually not the same price
|
||||
# Use first provider for initial checks/cost calculation
|
||||
# primary_upstream = upstreams[0]
|
||||
# Reserve/max-cost checks use the best-ranked candidate; the failover loop
|
||||
# below rebinds (model_obj, upstream) per candidate so forwarding and
|
||||
# settlement always use the model of the provider actually being tried.
|
||||
model_obj = candidates[0][0]
|
||||
|
||||
_max_cost_for_model = await get_max_cost_for_model(
|
||||
model=model_id, session=session, model_obj=model_obj
|
||||
@@ -320,7 +352,7 @@ async def proxy(
|
||||
|
||||
if x_cashu := headers.get("x-cashu", None):
|
||||
last_error = None
|
||||
for i, upstream in enumerate(upstreams):
|
||||
for i, (model_obj, upstream) in enumerate(candidates):
|
||||
try:
|
||||
if is_ehbp:
|
||||
if not upstream.supports_ehbp:
|
||||
@@ -358,7 +390,7 @@ async def proxy(
|
||||
"status_code": e.status_code,
|
||||
},
|
||||
)
|
||||
if i == len(upstreams) - 1:
|
||||
if i == len(candidates) - 1:
|
||||
last_error = e
|
||||
continue
|
||||
|
||||
@@ -385,12 +417,12 @@ async def proxy(
|
||||
logger.debug("Processing unauthenticated GET request", extra={"path": path})
|
||||
|
||||
last_error_response = None
|
||||
for i, upstream in enumerate(upstreams):
|
||||
for i, (_, upstream) in enumerate(candidates):
|
||||
try:
|
||||
headers = upstream.prepare_headers(dict(request.headers))
|
||||
response = await upstream.forward_get_request(request, path, headers)
|
||||
|
||||
if response.status_code in [502, 429] and i < len(upstreams) - 1:
|
||||
if response.status_code in [502, 429] and i < len(candidates) - 1:
|
||||
error_message = ""
|
||||
try:
|
||||
if hasattr(response, "body"):
|
||||
@@ -420,22 +452,52 @@ async def proxy(
|
||||
return response
|
||||
except UpstreamError as e:
|
||||
logger.warning(f"Upstream {upstream.provider_type} failed (GET): {e}")
|
||||
if i == len(upstreams) - 1:
|
||||
if i == len(candidates) - 1:
|
||||
last_error_response = create_upstream_error_response(e, request)
|
||||
continue
|
||||
return last_error_response or create_error_response(
|
||||
"upstream_error", "All upstreams failed", 502, request=request
|
||||
)
|
||||
|
||||
reservation_snapshot: ReservationSnapshot | None = None
|
||||
if is_ehbp or request_body_dict:
|
||||
await pay_for_request(key, max_cost_for_model, session)
|
||||
reservation_snapshot = await get_reservation_snapshot(key, session)
|
||||
|
||||
# Tracks request params already removed in response to upstream rejections,
|
||||
# shared across providers so a stripped param stays stripped on failover and
|
||||
# the reactive retry can never loop unboundedly.
|
||||
already_stripped: set[str] = set()
|
||||
|
||||
for i, upstream in enumerate(upstreams):
|
||||
for i, (model_obj, upstream) in enumerate(candidates):
|
||||
if i > 0 and request_body_dict:
|
||||
# The reservation was sized to the previous candidate's envelope;
|
||||
# settlement bills the serving candidate, so a pricier fallback
|
||||
# must be re-reserved at its own max cost before it is tried. A
|
||||
# candidate whose envelope the key cannot cover is rejected, just
|
||||
# as it would be had it been ranked first.
|
||||
candidate_max = await get_max_cost_for_model(
|
||||
model=model_id, session=session, model_obj=model_obj
|
||||
)
|
||||
candidate_max = await calculate_discounted_max_cost(
|
||||
candidate_max, request_body_dict, model_obj=model_obj
|
||||
)
|
||||
candidate_max = max(candidate_max, settings.min_request_msat)
|
||||
if candidate_max > max_cost_for_model:
|
||||
await revert_pay_for_request(
|
||||
key, session, max_cost_for_model, reservation_snapshot
|
||||
)
|
||||
try:
|
||||
await pay_for_request(key, candidate_max, session)
|
||||
except HTTPException:
|
||||
if i == len(candidates) - 1:
|
||||
raise
|
||||
await pay_for_request(key, max_cost_for_model, session)
|
||||
reservation_snapshot = await get_reservation_snapshot(key, session)
|
||||
continue
|
||||
reservation_snapshot = await get_reservation_snapshot(key, session)
|
||||
max_cost_for_model = candidate_max
|
||||
|
||||
headers = upstream.prepare_headers(dict(request.headers))
|
||||
|
||||
try:
|
||||
@@ -462,6 +524,7 @@ async def proxy(
|
||||
max_cost_for_model=max_cost_for_model,
|
||||
session=session,
|
||||
model_obj=model_obj,
|
||||
reservation_snapshot=reservation_snapshot,
|
||||
)
|
||||
elif is_responses_api:
|
||||
response = await upstream.forward_responses_request(
|
||||
@@ -473,6 +536,7 @@ async def proxy(
|
||||
max_cost_for_model,
|
||||
session,
|
||||
model_obj,
|
||||
reservation_snapshot,
|
||||
)
|
||||
else:
|
||||
response = await upstream.forward_request(
|
||||
@@ -484,6 +548,7 @@ async def proxy(
|
||||
max_cost_for_model,
|
||||
session,
|
||||
model_obj,
|
||||
reservation_snapshot,
|
||||
)
|
||||
except UpstreamError:
|
||||
# Let the outer UpstreamError handler manage retry/revert
|
||||
@@ -500,7 +565,9 @@ async def proxy(
|
||||
"max_cost_for_model": max_cost_for_model,
|
||||
},
|
||||
)
|
||||
await revert_pay_for_request(key, session, max_cost_for_model)
|
||||
await revert_pay_for_request(
|
||||
key, session, max_cost_for_model, reservation_snapshot
|
||||
)
|
||||
raise
|
||||
|
||||
# Reactive recovery: some models reject one specific request
|
||||
@@ -536,7 +603,7 @@ async def proxy(
|
||||
if response.status_code != 200:
|
||||
# Check if we should retry (502 Upstream Error or 429 Rate Limit)
|
||||
should_retry = response.status_code in [502, 429, 400, 401, 403, 404]
|
||||
if should_retry and i < len(upstreams) - 1:
|
||||
if should_retry and i < len(candidates) - 1:
|
||||
error_message = ""
|
||||
try:
|
||||
if hasattr(response, "body"):
|
||||
@@ -569,7 +636,9 @@ async def proxy(
|
||||
continue
|
||||
|
||||
# 4xx error (user error), or other non-retryable error, or last provider failed
|
||||
await revert_pay_for_request(key, session, max_cost_for_model)
|
||||
await revert_pay_for_request(
|
||||
key, session, max_cost_for_model, reservation_snapshot
|
||||
)
|
||||
logger.warning(
|
||||
"Upstream request failed, revert payment "
|
||||
"(provider=%s model=%s status=%s path=%s)",
|
||||
@@ -601,8 +670,10 @@ async def proxy(
|
||||
"max_cost_for_model": max_cost_for_model,
|
||||
},
|
||||
)
|
||||
await asyncio.shield(
|
||||
revert_pay_for_request(key, session, max_cost_for_model)
|
||||
# The cancellation has been caught, so complete exact cleanup in
|
||||
# this task before the request-scoped session can be torn down.
|
||||
await revert_pay_for_request(
|
||||
key, session, max_cost_for_model, reservation_snapshot
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -616,13 +687,15 @@ async def proxy(
|
||||
"provider": upstream.provider_type,
|
||||
"model": model_id,
|
||||
"status_code": e.status_code,
|
||||
"retry": i < len(upstreams) - 1,
|
||||
"retry": i < len(candidates) - 1,
|
||||
},
|
||||
)
|
||||
|
||||
# If this was the last provider
|
||||
if i == len(upstreams) - 1:
|
||||
await revert_pay_for_request(key, session, max_cost_for_model)
|
||||
if i == len(candidates) - 1:
|
||||
await revert_pay_for_request(
|
||||
key, session, max_cost_for_model, reservation_snapshot
|
||||
)
|
||||
return create_upstream_error_response(e, request)
|
||||
|
||||
# Otherwise loop continues to next provider
|
||||
|
||||
@@ -4,7 +4,14 @@ import json
|
||||
from sqlmodel import select
|
||||
|
||||
from ..core import get_logger
|
||||
from ..core.db import UpstreamProviderRow, create_session
|
||||
from ..core.db import (
|
||||
CashuTransaction,
|
||||
UpstreamProviderRow,
|
||||
create_session,
|
||||
)
|
||||
from ..core.db import (
|
||||
store_cashu_transaction_with_retry as store_cashu_transaction,
|
||||
)
|
||||
from ..wallet import send_token
|
||||
from .routstr import RoutstrUpstreamProvider
|
||||
|
||||
@@ -123,7 +130,6 @@ async def _check_and_topup(row: UpstreamProviderRow) -> None:
|
||||
},
|
||||
)
|
||||
|
||||
print(amount, mint_url)
|
||||
try:
|
||||
token = await send_token(amount, "sat", mint_url)
|
||||
except Exception as e:
|
||||
@@ -138,6 +144,23 @@ async def _check_and_topup(row: UpstreamProviderRow) -> None:
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
await store_cashu_transaction(
|
||||
token=token,
|
||||
amount=amount,
|
||||
unit="sat",
|
||||
mint_url=mint_url,
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="auto_topup",
|
||||
)
|
||||
except Exception:
|
||||
logger.critical(
|
||||
"Aborting auto top-up because its cashu token could not be persisted",
|
||||
extra={"provider_id": row.id, "mint_url": mint_url},
|
||||
)
|
||||
return
|
||||
|
||||
result = await provider.topup(token)
|
||||
|
||||
if "error" in result:
|
||||
@@ -149,6 +172,26 @@ async def _check_and_topup(row: UpstreamProviderRow) -> None:
|
||||
},
|
||||
)
|
||||
else:
|
||||
async with create_session() as session:
|
||||
transaction = (
|
||||
await session.exec(
|
||||
select(CashuTransaction).where(
|
||||
CashuTransaction.token == token,
|
||||
CashuTransaction.type == "out",
|
||||
CashuTransaction.source == "auto_topup",
|
||||
)
|
||||
)
|
||||
).first()
|
||||
if transaction is None:
|
||||
logger.critical(
|
||||
"Completed auto top-up transaction is missing from the database",
|
||||
extra={"provider_id": row.id, "mint_url": mint_url},
|
||||
)
|
||||
else:
|
||||
transaction.collected = True
|
||||
session.add(transaction)
|
||||
await session.commit()
|
||||
|
||||
logger.info(
|
||||
"Auto top-up completed successfully",
|
||||
extra={
|
||||
|
||||
+333
-121
@@ -14,14 +14,21 @@ from fastapi import BackgroundTasks, HTTPException, Request
|
||||
from fastapi.responses import Response, StreamingResponse
|
||||
from pydantic.v1 import BaseModel
|
||||
|
||||
from ..auth import adjust_payment_for_tokens
|
||||
from ..auth import (
|
||||
ReservationSnapshot,
|
||||
adjust_payment_for_tokens,
|
||||
get_reservation_snapshot,
|
||||
release_reservation,
|
||||
)
|
||||
from ..core import get_logger
|
||||
from ..core.db import (
|
||||
ApiKey,
|
||||
AsyncSession,
|
||||
UpstreamProviderRow,
|
||||
create_session,
|
||||
store_cashu_transaction,
|
||||
)
|
||||
from ..core.db import (
|
||||
store_cashu_transaction_with_retry as store_cashu_transaction,
|
||||
)
|
||||
from ..core.exceptions import UpstreamError
|
||||
from ..core.redaction import redact_org_ids
|
||||
@@ -63,8 +70,7 @@ logger = get_logger(__name__)
|
||||
|
||||
|
||||
def _is_json_content_type(content_type: str | None) -> bool:
|
||||
"""Return True when the upstream response should be parsed as JSON.
|
||||
"""
|
||||
"""Return True when the upstream response should be parsed as JSON."""
|
||||
if not content_type:
|
||||
return False
|
||||
main = content_type.split(";", 1)[0].strip().lower()
|
||||
@@ -227,9 +233,7 @@ class BaseUpstreamProvider:
|
||||
pass
|
||||
if "prompt_tokens" in usage:
|
||||
try:
|
||||
usage["prompt_tokens"] = (
|
||||
int(usage.get("prompt_tokens") or 0) + extra
|
||||
)
|
||||
usage["prompt_tokens"] = int(usage.get("prompt_tokens") or 0) + extra
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
@@ -731,7 +735,9 @@ class BaseUpstreamProvider:
|
||||
and rate_limit.retry_after_seconds is not None
|
||||
and "retry-after" not in {k.lower() for k in headers}
|
||||
):
|
||||
headers["Retry-After"] = str(max(1, math.ceil(rate_limit.retry_after_seconds)))
|
||||
headers["Retry-After"] = str(
|
||||
max(1, math.ceil(rate_limit.retry_after_seconds))
|
||||
)
|
||||
|
||||
if is_json_body:
|
||||
if not content_type:
|
||||
@@ -790,6 +796,56 @@ class BaseUpstreamProvider:
|
||||
media_type="application/json",
|
||||
)
|
||||
|
||||
async def _release_failed_streaming_reservation(
|
||||
self,
|
||||
key: ApiKey,
|
||||
session: AsyncSession,
|
||||
reservation_snapshot: ReservationSnapshot | None,
|
||||
) -> bool:
|
||||
"""Attempt exact release and suppress unsafe settlement retries."""
|
||||
try:
|
||||
await session.rollback()
|
||||
snapshot = reservation_snapshot
|
||||
if snapshot is None:
|
||||
snapshot = await get_reservation_snapshot(key, session)
|
||||
released = await release_reservation(
|
||||
snapshot,
|
||||
session,
|
||||
snapshot.reserved_msats,
|
||||
)
|
||||
if not released:
|
||||
logger.critical(
|
||||
"Billing reservation could not be released",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"reserved_balance": snapshot.reserved_msats,
|
||||
},
|
||||
)
|
||||
# A failed release remains recoverable by the stale-reservation
|
||||
# sweep. Retrying settlement here could charge after an ambiguous
|
||||
# database failure or replace the original stream exception.
|
||||
return True
|
||||
except asyncio.CancelledError:
|
||||
# Preserve the exception that triggered billing cleanup. The stream
|
||||
# propagates it immediately after this helper returns, and stale
|
||||
# reservation cleanup can recover an interrupted release.
|
||||
logger.critical(
|
||||
"Billing reservation release was cancelled",
|
||||
extra={"key_hash": key.hashed_key[:8] + "..."},
|
||||
exc_info=True,
|
||||
)
|
||||
return True
|
||||
except Exception as release_error:
|
||||
logger.critical(
|
||||
"Billing reservation release failed",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"error": str(release_error),
|
||||
},
|
||||
exc_info=True,
|
||||
)
|
||||
return True
|
||||
|
||||
async def handle_streaming_chat_completion(
|
||||
self,
|
||||
response: httpx.Response,
|
||||
@@ -797,6 +853,8 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model: int,
|
||||
background_tasks: BackgroundTasks,
|
||||
requested_model: str | None = None,
|
||||
model_obj: Model | None = None,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> StreamingResponse:
|
||||
"""Handle streaming chat completion responses with token usage tracking and cost adjustment.
|
||||
|
||||
@@ -808,6 +866,15 @@ class BaseUpstreamProvider:
|
||||
Returns:
|
||||
StreamingResponse with cost data injected at the end
|
||||
"""
|
||||
if reservation_snapshot is None:
|
||||
async with create_session() as snapshot_session:
|
||||
snapshot_key = await snapshot_session.get(key.__class__, key.hashed_key)
|
||||
if snapshot_key is None:
|
||||
raise RuntimeError("Billing key disappeared before streaming")
|
||||
reservation_snapshot = await get_reservation_snapshot(
|
||||
snapshot_key, snapshot_session
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"Processing streaming chat completion",
|
||||
extra={
|
||||
@@ -839,6 +906,9 @@ class BaseUpstreamProvider:
|
||||
{"model": last_model_seen or "unknown", "usage": None},
|
||||
new_session,
|
||||
max_cost_for_model,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
usage_finalized = True
|
||||
except Exception:
|
||||
@@ -963,9 +1033,7 @@ class BaseUpstreamProvider:
|
||||
# line so multi-line ``data`` stays valid SSE framing - a bare
|
||||
# second line would otherwise reach the client without its
|
||||
# ``data:`` field and break naive parsers.
|
||||
body = b"".join(
|
||||
b"data: " + ln + b"\n" for ln in data.split(b"\n")
|
||||
)
|
||||
body = b"".join(b"data: " + ln + b"\n" for ln in data.split(b"\n"))
|
||||
yield prefix + body + b"\n"
|
||||
|
||||
try:
|
||||
@@ -1011,48 +1079,46 @@ class BaseUpstreamProvider:
|
||||
adjustment_input,
|
||||
session,
|
||||
max_cost_for_model,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
usage_finalized = True
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Error during usage finalization",
|
||||
except BaseException as e:
|
||||
logger.critical(
|
||||
"Error during usage finalization — CRITICAL",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"error": str(e),
|
||||
},
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
# Fall back so we still emit a non-zero sats cost downstream.
|
||||
cost_data = {
|
||||
"base_msats": 0,
|
||||
"input_msats": 0,
|
||||
"output_msats": 0,
|
||||
"total_msats": 0,
|
||||
"total_usd": 0.0,
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
}
|
||||
# Release is a terminal billing state. Do not enqueue
|
||||
# finalize_db_only from the generator's finally block
|
||||
# and charge this request later.
|
||||
usage_finalized = (
|
||||
await self._release_failed_streaming_reservation(
|
||||
fresh_key,
|
||||
session,
|
||||
reservation_snapshot,
|
||||
)
|
||||
)
|
||||
raise
|
||||
|
||||
if usage_chunk_data is None:
|
||||
if not hasattr(self, "_current_stream_id"):
|
||||
self._current_stream_id = (
|
||||
f"chatcmpl-{uuid.uuid4()}"
|
||||
)
|
||||
self._current_stream_id = f"chatcmpl-{uuid.uuid4()}"
|
||||
usage_chunk_data = {
|
||||
"id": self._current_stream_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"model": last_model_seen or "unknown",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": cost_data.get(
|
||||
"input_tokens", 0
|
||||
),
|
||||
"prompt_tokens": cost_data.get("input_tokens", 0),
|
||||
"completion_tokens": cost_data.get(
|
||||
"output_tokens", 0
|
||||
),
|
||||
"total_tokens": cost_data.get(
|
||||
"input_tokens", 0
|
||||
)
|
||||
"total_tokens": cost_data.get("input_tokens", 0)
|
||||
+ cost_data.get("output_tokens", 0),
|
||||
},
|
||||
}
|
||||
@@ -1107,6 +1173,8 @@ class BaseUpstreamProvider:
|
||||
session: AsyncSession,
|
||||
deducted_max_cost: int,
|
||||
requested_model: str | None = None,
|
||||
model_obj: Model | None = None,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> Response:
|
||||
"""Handle non-streaming chat completion responses with token usage tracking and cost adjustment.
|
||||
|
||||
@@ -1153,6 +1221,9 @@ class BaseUpstreamProvider:
|
||||
response_json,
|
||||
session,
|
||||
deducted_max_cost,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
|
||||
await session.refresh(key)
|
||||
@@ -1248,6 +1319,8 @@ class BaseUpstreamProvider:
|
||||
key: ApiKey,
|
||||
max_cost_for_model: int,
|
||||
requested_model: str | None = None,
|
||||
model_obj: Model | None = None,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> StreamingResponse:
|
||||
"""Handle streaming Responses API responses with token usage tracking and cost adjustment.
|
||||
|
||||
@@ -1291,6 +1364,9 @@ class BaseUpstreamProvider:
|
||||
{"model": last_model_seen or "unknown", "usage": None},
|
||||
new_session,
|
||||
max_cost_for_model,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
usage_finalized = True
|
||||
except Exception:
|
||||
@@ -1375,9 +1451,7 @@ class BaseUpstreamProvider:
|
||||
return
|
||||
# Re-prefix each line so multi-line ``data`` stays valid SSE
|
||||
# framing for the client.
|
||||
body = b"".join(
|
||||
b"data: " + ln + b"\n" for ln in data.split(b"\n")
|
||||
)
|
||||
body = b"".join(b"data: " + ln + b"\n" for ln in data.split(b"\n"))
|
||||
yield prefix + body + b"\n"
|
||||
|
||||
try:
|
||||
@@ -1420,25 +1494,28 @@ class BaseUpstreamProvider:
|
||||
adjustment_input,
|
||||
session,
|
||||
max_cost_for_model,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
usage_finalized = True
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Error during Responses API usage finalization",
|
||||
except BaseException as e:
|
||||
logger.critical(
|
||||
"Error during Responses API usage finalization — CRITICAL",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"error": str(e),
|
||||
},
|
||||
exc_info=True,
|
||||
)
|
||||
cost_data = {
|
||||
"base_msats": 0,
|
||||
"input_msats": 0,
|
||||
"output_msats": 0,
|
||||
"total_msats": 0,
|
||||
"total_usd": 0.0,
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
}
|
||||
usage_finalized = (
|
||||
await self._release_failed_streaming_reservation(
|
||||
fresh_key,
|
||||
session,
|
||||
reservation_snapshot,
|
||||
)
|
||||
)
|
||||
raise
|
||||
|
||||
if usage_chunk_data is None:
|
||||
usage_chunk_data = {
|
||||
@@ -1452,22 +1529,14 @@ class BaseUpstreamProvider:
|
||||
"output_tokens": cost_data.get(
|
||||
"output_tokens", 0
|
||||
),
|
||||
"total_tokens": cost_data.get(
|
||||
"input_tokens", 0
|
||||
)
|
||||
"total_tokens": cost_data.get("input_tokens", 0)
|
||||
+ cost_data.get("output_tokens", 0),
|
||||
},
|
||||
},
|
||||
"usage": {
|
||||
"input_tokens": cost_data.get(
|
||||
"input_tokens", 0
|
||||
),
|
||||
"output_tokens": cost_data.get(
|
||||
"output_tokens", 0
|
||||
),
|
||||
"total_tokens": cost_data.get(
|
||||
"input_tokens", 0
|
||||
)
|
||||
"input_tokens": cost_data.get("input_tokens", 0),
|
||||
"output_tokens": cost_data.get("output_tokens", 0),
|
||||
"total_tokens": cost_data.get("input_tokens", 0)
|
||||
+ cost_data.get("output_tokens", 0),
|
||||
},
|
||||
}
|
||||
@@ -1483,9 +1552,9 @@ class BaseUpstreamProvider:
|
||||
usage_chunk_data["response"]["usage"]["cost"] = (
|
||||
cost_data.get("total_usd", 0.0)
|
||||
)
|
||||
usage_chunk_data["response"]["usage"][
|
||||
"cost_sats"
|
||||
] = sats_cost
|
||||
usage_chunk_data["response"]["usage"]["cost_sats"] = (
|
||||
sats_cost
|
||||
)
|
||||
usage_chunk_data["response"]["usage"][
|
||||
"remaining_balance_msats"
|
||||
] = remaining_balance_msats
|
||||
@@ -1538,6 +1607,8 @@ class BaseUpstreamProvider:
|
||||
session: AsyncSession,
|
||||
deducted_max_cost: int,
|
||||
requested_model: str | None = None,
|
||||
model_obj: Model | None = None,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> Response:
|
||||
"""Handle non-streaming Responses API responses with token usage tracking and cost adjustment.
|
||||
|
||||
@@ -1587,6 +1658,9 @@ class BaseUpstreamProvider:
|
||||
response_json,
|
||||
session,
|
||||
deducted_max_cost,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
|
||||
await session.refresh(key)
|
||||
@@ -1677,7 +1751,13 @@ class BaseUpstreamProvider:
|
||||
raise
|
||||
|
||||
async def _finalize_generic_streaming_payment(
|
||||
self, key_hash: str, max_cost: int, path: str
|
||||
self,
|
||||
key_hash: str,
|
||||
max_cost: int,
|
||||
path: str,
|
||||
model_obj: Model | None,
|
||||
provider_fee: float | None,
|
||||
reservation_snapshot: ReservationSnapshot,
|
||||
) -> None:
|
||||
"""Background task to finalize payment for generic streaming requests."""
|
||||
async with create_session() as session:
|
||||
@@ -1691,11 +1771,16 @@ class BaseUpstreamProvider:
|
||||
|
||||
try:
|
||||
# Finalize with "unknown" model and no usage to release reservation/charge max cost
|
||||
# (no routed identity here by design: the None usage settles at
|
||||
# MaxCostData before any pricing lookup can happen).
|
||||
await adjust_payment_for_tokens(
|
||||
key,
|
||||
{"model": "unknown", "usage": None},
|
||||
session,
|
||||
max_cost,
|
||||
model_obj=model_obj,
|
||||
provider_fee=provider_fee,
|
||||
reservation_snapshot=reservation_snapshot,
|
||||
)
|
||||
logger.debug(
|
||||
"Finalized generic streaming payment in background",
|
||||
@@ -1720,6 +1805,8 @@ class BaseUpstreamProvider:
|
||||
key: ApiKey,
|
||||
max_cost_for_model: int,
|
||||
requested_model: str | None = None,
|
||||
model_obj: Model | None = None,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> StreamingResponse:
|
||||
async def stream_with_cost(
|
||||
max_cost_for_model: int,
|
||||
@@ -1762,9 +1849,7 @@ class BaseUpstreamProvider:
|
||||
_coerce_usd(cd.get("output_cost")),
|
||||
)
|
||||
for field in ("total_cost", "cost"):
|
||||
total_cost = max(
|
||||
total_cost, _coerce_usd(usage_or_root.get(field))
|
||||
)
|
||||
total_cost = max(total_cost, _coerce_usd(usage_or_root.get(field)))
|
||||
|
||||
async def finalize_without_usage() -> bytes | None:
|
||||
nonlocal usage_finalized
|
||||
@@ -1785,12 +1870,29 @@ class BaseUpstreamProvider:
|
||||
fallback,
|
||||
new_session,
|
||||
max_cost_for_model,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
usage_finalized = True
|
||||
return f"event: cost\ndata: {json.dumps({'cost': cost_data})}\n\n".encode()
|
||||
except Exception:
|
||||
usage_finalized = True
|
||||
return None
|
||||
except BaseException as e:
|
||||
logger.critical(
|
||||
"Error during Messages API usage finalization — CRITICAL",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"error": str(e),
|
||||
},
|
||||
exc_info=True,
|
||||
)
|
||||
usage_finalized = (
|
||||
await self._release_failed_streaming_reservation(
|
||||
fresh_key,
|
||||
new_session,
|
||||
reservation_snapshot,
|
||||
)
|
||||
)
|
||||
raise
|
||||
|
||||
try:
|
||||
async for chunk in response.aiter_bytes():
|
||||
@@ -1808,9 +1910,7 @@ class BaseUpstreamProvider:
|
||||
if msg and msg.get("model"):
|
||||
last_model_seen = str(msg.get("model"))
|
||||
|
||||
provider_added = (
|
||||
"provider" not in data
|
||||
)
|
||||
provider_added = "provider" not in data
|
||||
self._apply_provider_field(data)
|
||||
|
||||
if requested_model:
|
||||
@@ -1936,6 +2036,9 @@ class BaseUpstreamProvider:
|
||||
combined_data,
|
||||
new_session,
|
||||
max_cost_for_model,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
|
||||
self.inject_cost_metadata(
|
||||
@@ -1945,8 +2048,23 @@ class BaseUpstreamProvider:
|
||||
usage_finalized = True
|
||||
# Emit the full combined_data as the cost
|
||||
yield f"event: cost\ndata: {json.dumps(combined_data)}\n\n".encode()
|
||||
except Exception:
|
||||
pass
|
||||
except BaseException as e:
|
||||
logger.critical(
|
||||
"Error during Messages API usage finalization — CRITICAL",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"error": str(e),
|
||||
},
|
||||
exc_info=True,
|
||||
)
|
||||
usage_finalized = (
|
||||
await self._release_failed_streaming_reservation(
|
||||
fresh_key,
|
||||
new_session,
|
||||
reservation_snapshot,
|
||||
)
|
||||
)
|
||||
raise
|
||||
|
||||
if not usage_finalized:
|
||||
maybe_cost_event = await finalize_without_usage()
|
||||
@@ -1983,6 +2101,8 @@ class BaseUpstreamProvider:
|
||||
deducted_max_cost: int,
|
||||
path: str,
|
||||
requested_model: str | None = None,
|
||||
model_obj: Model | None = None,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> Response:
|
||||
try:
|
||||
content = await response.aread()
|
||||
@@ -2007,6 +2127,9 @@ class BaseUpstreamProvider:
|
||||
response_json,
|
||||
session,
|
||||
deducted_max_cost,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
|
||||
self.inject_cost_metadata(response_json, cost_data, key)
|
||||
@@ -2054,9 +2177,7 @@ class BaseUpstreamProvider:
|
||||
async def _aggregate_anthropic_events_to_message(
|
||||
self, iterator: AsyncIterator[Any]
|
||||
) -> dict:
|
||||
return await messages_dispatch.aggregate_anthropic_events_to_message(
|
||||
iterator
|
||||
)
|
||||
return await messages_dispatch.aggregate_anthropic_events_to_message(iterator)
|
||||
|
||||
async def _dispatch_anthropic_messages(
|
||||
self,
|
||||
@@ -2082,6 +2203,7 @@ class BaseUpstreamProvider:
|
||||
session: AsyncSession,
|
||||
max_cost_for_model: int,
|
||||
model_obj: Model,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> Response | StreamingResponse:
|
||||
"""Translate /v1/messages to upstream chat/completions via litellm.
|
||||
|
||||
@@ -2101,6 +2223,8 @@ class BaseUpstreamProvider:
|
||||
key,
|
||||
max_cost_for_model,
|
||||
requested_model,
|
||||
model_obj,
|
||||
reservation_snapshot,
|
||||
)
|
||||
|
||||
response_json = messages_dispatch.coerce_litellm_payload(result)
|
||||
@@ -2112,6 +2236,9 @@ class BaseUpstreamProvider:
|
||||
response_json,
|
||||
session,
|
||||
max_cost_for_model,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
self.inject_cost_metadata(response_json, cost_data, key)
|
||||
|
||||
@@ -2151,6 +2278,7 @@ class BaseUpstreamProvider:
|
||||
requested_model,
|
||||
mint,
|
||||
request_id,
|
||||
model_obj,
|
||||
)
|
||||
|
||||
response_json = messages_dispatch.coerce_litellm_payload(result)
|
||||
@@ -2158,10 +2286,14 @@ class BaseUpstreamProvider:
|
||||
if requested_model and "model" in response_json:
|
||||
response_json["model"] = requested_model
|
||||
|
||||
cost_data = await self.get_x_cashu_cost(response_json, max_cost_for_model)
|
||||
cost_data = await self.get_x_cashu_cost(
|
||||
response_json, max_cost_for_model, model_obj
|
||||
)
|
||||
|
||||
if cost_data and "usage" in response_json and isinstance(
|
||||
response_json["usage"], dict
|
||||
if (
|
||||
cost_data
|
||||
and "usage" in response_json
|
||||
and isinstance(response_json["usage"], dict)
|
||||
):
|
||||
response_json["usage"]["cost_sats"] = cost_data.total_msats // 1000
|
||||
self._fold_cache_into_input_tokens(response_json["usage"])
|
||||
@@ -2203,6 +2335,8 @@ class BaseUpstreamProvider:
|
||||
key: ApiKey,
|
||||
max_cost_for_model: int,
|
||||
requested_model: str | None,
|
||||
model_obj: Model | None = None,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> StreamingResponse:
|
||||
"""Re-emit a litellm Anthropic-event iterator as live SSE bytes
|
||||
with cost reconciliation appended at end of stream."""
|
||||
@@ -2237,9 +2371,7 @@ class BaseUpstreamProvider:
|
||||
},
|
||||
)
|
||||
async with create_session() as new_session:
|
||||
fresh_key = await new_session.get(
|
||||
key.__class__, key.hashed_key
|
||||
)
|
||||
fresh_key = await new_session.get(key.__class__, key.hashed_key)
|
||||
if not fresh_key:
|
||||
usage_finalized = True
|
||||
return None
|
||||
@@ -2253,15 +2385,31 @@ class BaseUpstreamProvider:
|
||||
fallback,
|
||||
new_session,
|
||||
max_cost_for_model,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
usage_finalized = True
|
||||
return (
|
||||
f"event: cost\ndata: "
|
||||
f"{json.dumps({'cost': cost_data})}\n\n"
|
||||
f"event: cost\ndata: {json.dumps({'cost': cost_data})}\n\n"
|
||||
).encode()
|
||||
except Exception:
|
||||
usage_finalized = True
|
||||
return None
|
||||
except BaseException as e:
|
||||
logger.critical(
|
||||
"Error during LiteLLM Messages usage finalization — CRITICAL",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"error": str(e),
|
||||
},
|
||||
exc_info=True,
|
||||
)
|
||||
usage_finalized = (
|
||||
await self._release_failed_streaming_reservation(
|
||||
fresh_key,
|
||||
new_session,
|
||||
reservation_snapshot,
|
||||
)
|
||||
)
|
||||
raise
|
||||
|
||||
try:
|
||||
async for annotated in messages_dispatch.stream_annotated_events(
|
||||
@@ -2296,9 +2444,7 @@ class BaseUpstreamProvider:
|
||||
or total_cost > 0
|
||||
):
|
||||
async with create_session() as new_session:
|
||||
fresh_key = await new_session.get(
|
||||
key.__class__, key.hashed_key
|
||||
)
|
||||
fresh_key = await new_session.get(key.__class__, key.hashed_key)
|
||||
if fresh_key:
|
||||
try:
|
||||
rebuilt_usage: dict = {
|
||||
@@ -2326,6 +2472,9 @@ class BaseUpstreamProvider:
|
||||
combined_data,
|
||||
new_session,
|
||||
max_cost_for_model,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
self.inject_cost_metadata(
|
||||
combined_data, cost_data, fresh_key
|
||||
@@ -2335,8 +2484,23 @@ class BaseUpstreamProvider:
|
||||
f"event: cost\ndata: "
|
||||
f"{json.dumps({'cost': cost_data})}\n\n"
|
||||
).encode()
|
||||
except Exception:
|
||||
pass
|
||||
except BaseException as e:
|
||||
logger.critical(
|
||||
"Error during LiteLLM Messages usage finalization — CRITICAL",
|
||||
extra={
|
||||
"key_hash": key.hashed_key[:8] + "...",
|
||||
"error": str(e),
|
||||
},
|
||||
exc_info=True,
|
||||
)
|
||||
usage_finalized = (
|
||||
await self._release_failed_streaming_reservation(
|
||||
fresh_key,
|
||||
new_session,
|
||||
reservation_snapshot,
|
||||
)
|
||||
)
|
||||
raise
|
||||
|
||||
if not usage_finalized:
|
||||
cost_event = await finalize_without_usage()
|
||||
@@ -2347,6 +2511,9 @@ class BaseUpstreamProvider:
|
||||
if not usage_finalized:
|
||||
await finalize_without_usage()
|
||||
raise
|
||||
finally:
|
||||
if not usage_finalized:
|
||||
await finalize_without_usage()
|
||||
|
||||
return StreamingResponse(
|
||||
stream_with_cost(),
|
||||
@@ -2363,6 +2530,7 @@ class BaseUpstreamProvider:
|
||||
requested_model: str | None,
|
||||
mint: str | None,
|
||||
request_id: str | None,
|
||||
model_obj: Model | None = None,
|
||||
) -> StreamingResponse:
|
||||
"""Buffer a litellm stream end-to-end, compute cost, then replay.
|
||||
|
||||
@@ -2454,7 +2622,7 @@ class BaseUpstreamProvider:
|
||||
}
|
||||
try:
|
||||
cost_data = await self.get_x_cashu_cost(
|
||||
response_data, max_cost_for_model
|
||||
response_data, max_cost_for_model, model_obj
|
||||
)
|
||||
if cost_data:
|
||||
refund_amount = messages_dispatch.compute_refund(
|
||||
@@ -2469,8 +2637,7 @@ class BaseUpstreamProvider:
|
||||
)
|
||||
response_headers["X-Cashu"] = refund_token
|
||||
logger.info(
|
||||
"Refund processed for streaming /v1/messages "
|
||||
"via litellm",
|
||||
"Refund processed for streaming /v1/messages via litellm",
|
||||
extra={
|
||||
"refund_amount": refund_amount,
|
||||
"unit": unit,
|
||||
@@ -2508,6 +2675,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model: int,
|
||||
session: AsyncSession,
|
||||
model_obj: Model,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> Response | StreamingResponse:
|
||||
"""Forward authenticated request to upstream service with cost tracking.
|
||||
|
||||
@@ -2542,6 +2710,7 @@ class BaseUpstreamProvider:
|
||||
session=session,
|
||||
max_cost_for_model=max_cost_for_model,
|
||||
model_obj=model_obj,
|
||||
reservation_snapshot=reservation_snapshot,
|
||||
)
|
||||
|
||||
url = self.build_request_url(path, model_obj)
|
||||
@@ -2671,6 +2840,8 @@ class BaseUpstreamProvider:
|
||||
key,
|
||||
max_cost_for_model,
|
||||
requested_model=original_model_id,
|
||||
model_obj=model_obj,
|
||||
reservation_snapshot=reservation_snapshot,
|
||||
)
|
||||
background_tasks = BackgroundTasks()
|
||||
background_tasks.add_task(response.aclose)
|
||||
@@ -2687,6 +2858,8 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model,
|
||||
path,
|
||||
requested_model=original_model_id,
|
||||
model_obj=model_obj,
|
||||
reservation_snapshot=reservation_snapshot,
|
||||
)
|
||||
finally:
|
||||
await response.aclose()
|
||||
@@ -2702,6 +2875,8 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model,
|
||||
path,
|
||||
requested_model=original_model_id,
|
||||
model_obj=model_obj,
|
||||
reservation_snapshot=reservation_snapshot,
|
||||
)
|
||||
finally:
|
||||
await response.aclose()
|
||||
@@ -2751,6 +2926,8 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model,
|
||||
background_tasks,
|
||||
requested_model=original_model_id,
|
||||
model_obj=model_obj,
|
||||
reservation_snapshot=reservation_snapshot,
|
||||
)
|
||||
result.background = background_tasks
|
||||
return result
|
||||
@@ -2764,11 +2941,16 @@ class BaseUpstreamProvider:
|
||||
session,
|
||||
max_cost_for_model,
|
||||
requested_model=original_model_id,
|
||||
model_obj=model_obj,
|
||||
reservation_snapshot=reservation_snapshot,
|
||||
)
|
||||
finally:
|
||||
await response.aclose()
|
||||
await client.aclose()
|
||||
|
||||
if reservation_snapshot is None:
|
||||
reservation_snapshot = await get_reservation_snapshot(key, session)
|
||||
|
||||
background_tasks = BackgroundTasks()
|
||||
background_tasks.add_task(response.aclose)
|
||||
background_tasks.add_task(client.aclose)
|
||||
@@ -2777,6 +2959,9 @@ class BaseUpstreamProvider:
|
||||
key.hashed_key,
|
||||
max_cost_for_model,
|
||||
path,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
@@ -2851,7 +3036,9 @@ class BaseUpstreamProvider:
|
||||
|
||||
supports_ehbp: bool = False
|
||||
|
||||
def get_confidential_inference_profile(self) -> "ConfidentialInferenceProfile | None":
|
||||
def get_confidential_inference_profile(
|
||||
self,
|
||||
) -> "ConfidentialInferenceProfile | None":
|
||||
"""Return provider policy for encrypted/confidential inference forwarding."""
|
||||
return None
|
||||
|
||||
@@ -2879,6 +3066,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model: int,
|
||||
session: AsyncSession,
|
||||
model_obj: Model,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> Response | StreamingResponse:
|
||||
"""Forward authenticated Responses API request to upstream service with cost tracking.
|
||||
|
||||
@@ -3016,6 +3204,8 @@ class BaseUpstreamProvider:
|
||||
key,
|
||||
max_cost_for_model,
|
||||
requested_model=original_model_id,
|
||||
model_obj=model_obj,
|
||||
reservation_snapshot=reservation_snapshot,
|
||||
)
|
||||
background_tasks = BackgroundTasks()
|
||||
background_tasks.add_task(response.aclose)
|
||||
@@ -3031,11 +3221,16 @@ class BaseUpstreamProvider:
|
||||
session,
|
||||
max_cost_for_model,
|
||||
requested_model=original_model_id,
|
||||
model_obj=model_obj,
|
||||
reservation_snapshot=reservation_snapshot,
|
||||
)
|
||||
finally:
|
||||
await response.aclose()
|
||||
await client.aclose()
|
||||
|
||||
if reservation_snapshot is None:
|
||||
reservation_snapshot = await get_reservation_snapshot(key, session)
|
||||
|
||||
background_tasks = BackgroundTasks()
|
||||
background_tasks.add_task(response.aclose)
|
||||
background_tasks.add_task(client.aclose)
|
||||
@@ -3044,6 +3239,9 @@ class BaseUpstreamProvider:
|
||||
key.hashed_key,
|
||||
max_cost_for_model,
|
||||
path,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
reservation_snapshot,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
@@ -3207,13 +3405,19 @@ class BaseUpstreamProvider:
|
||||
)
|
||||
|
||||
async def get_x_cashu_cost(
|
||||
self, response_data: dict, max_cost_for_model: int
|
||||
self,
|
||||
response_data: dict,
|
||||
max_cost_for_model: int,
|
||||
model_obj: Model | None,
|
||||
) -> MaxCostData | CostData | None:
|
||||
"""Calculate cost for X-Cashu payment based on response data.
|
||||
|
||||
Args:
|
||||
response_data: Response data containing model and usage information
|
||||
max_cost_for_model: Maximum cost for the model
|
||||
model_obj: The model that actually served the request; billed
|
||||
directly instead of re-deriving pricing from the upstream's
|
||||
echoed model string
|
||||
|
||||
Returns:
|
||||
Cost data object (MaxCostData or CostData) or None if calculation fails
|
||||
@@ -3227,6 +3431,8 @@ class BaseUpstreamProvider:
|
||||
match await calculate_cost(
|
||||
response_data,
|
||||
max_cost_for_model,
|
||||
model_obj,
|
||||
self.provider_fee,
|
||||
):
|
||||
case MaxCostData() as cost:
|
||||
logger.debug(
|
||||
@@ -3371,6 +3577,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model: int,
|
||||
mint: str | None = None,
|
||||
request_id: str | None = None,
|
||||
model_obj: Model | None = None,
|
||||
) -> StreamingResponse:
|
||||
"""Handle streaming response for X-Cashu payment, calculating refund if needed.
|
||||
|
||||
@@ -3444,7 +3651,7 @@ class BaseUpstreamProvider:
|
||||
response_data = {"usage": usage_data, "model": model}
|
||||
try:
|
||||
cost_data = await self.get_x_cashu_cost(
|
||||
response_data, max_cost_for_model
|
||||
response_data, max_cost_for_model, model_obj
|
||||
)
|
||||
if cost_data:
|
||||
if unit == "msat":
|
||||
@@ -3515,14 +3722,8 @@ class BaseUpstreamProvider:
|
||||
if "provider" not in data_json:
|
||||
self._apply_provider_field(data_json)
|
||||
changed = True
|
||||
if (
|
||||
cost_data
|
||||
and "usage" in data_json
|
||||
and data_json["usage"]
|
||||
):
|
||||
data_json["usage"]["cost_sats"] = (
|
||||
cost_data.total_msats // 1000
|
||||
)
|
||||
if cost_data and "usage" in data_json and data_json["usage"]:
|
||||
data_json["usage"]["cost_sats"] = cost_data.total_msats // 1000
|
||||
changed = True
|
||||
if changed:
|
||||
lines[i] = "data: " + json.dumps(data_json)
|
||||
@@ -3549,6 +3750,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model: int,
|
||||
mint: str | None = None,
|
||||
request_id: str | None = None,
|
||||
model_obj: Model | None = None,
|
||||
) -> Response:
|
||||
"""Handle non-streaming response for X-Cashu payment, calculating refund if needed.
|
||||
|
||||
@@ -3570,7 +3772,9 @@ class BaseUpstreamProvider:
|
||||
try:
|
||||
response_json = json.loads(content_str)
|
||||
self._apply_provider_field(response_json)
|
||||
cost_data = await self.get_x_cashu_cost(response_json, max_cost_for_model)
|
||||
cost_data = await self.get_x_cashu_cost(
|
||||
response_json, max_cost_for_model, model_obj
|
||||
)
|
||||
|
||||
if cost_data and "usage" in response_json:
|
||||
response_json["usage"]["cost_sats"] = cost_data.total_msats // 1000
|
||||
@@ -3699,6 +3903,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model: int,
|
||||
mint: str | None = None,
|
||||
request_id: str | None = None,
|
||||
model_obj: Model | None = None,
|
||||
) -> StreamingResponse | Response:
|
||||
"""Handle chat completion response for X-Cashu payment, detecting streaming vs non-streaming.
|
||||
|
||||
@@ -3742,6 +3947,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model,
|
||||
mint,
|
||||
request_id=request_id,
|
||||
model_obj=model_obj,
|
||||
)
|
||||
else:
|
||||
return await self.handle_x_cashu_non_streaming_response(
|
||||
@@ -3752,6 +3958,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model,
|
||||
mint,
|
||||
request_id=request_id,
|
||||
model_obj=model_obj,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
@@ -3937,6 +4144,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model,
|
||||
mint,
|
||||
request_id=getattr(request.state, "request_id", None),
|
||||
model_obj=model_obj,
|
||||
)
|
||||
background_tasks = BackgroundTasks()
|
||||
background_tasks.add_task(response.aclose)
|
||||
@@ -4229,6 +4437,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model,
|
||||
mint,
|
||||
request_id=getattr(request.state, "request_id", None),
|
||||
model_obj=model_obj,
|
||||
)
|
||||
background_tasks = BackgroundTasks()
|
||||
background_tasks.add_task(response.aclose)
|
||||
@@ -4279,6 +4488,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model: int,
|
||||
mint: str | None = None,
|
||||
request_id: str | None = None,
|
||||
model_obj: Model | None = None,
|
||||
) -> StreamingResponse | Response:
|
||||
"""Handle Responses API completion response for X-Cashu payment.
|
||||
|
||||
@@ -4323,6 +4533,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model,
|
||||
mint,
|
||||
request_id=request_id,
|
||||
model_obj=model_obj,
|
||||
)
|
||||
else:
|
||||
return await self.handle_x_cashu_non_streaming_responses_response(
|
||||
@@ -4333,6 +4544,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model,
|
||||
mint,
|
||||
request_id=request_id,
|
||||
model_obj=model_obj,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
@@ -4360,6 +4572,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model: int,
|
||||
mint: str | None = None,
|
||||
request_id: str | None = None,
|
||||
model_obj: Model | None = None,
|
||||
) -> StreamingResponse:
|
||||
"""Handle streaming Responses API response for X-Cashu payment.
|
||||
|
||||
@@ -4418,7 +4631,7 @@ class BaseUpstreamProvider:
|
||||
response_data = {"usage": usage_data, "model": model}
|
||||
try:
|
||||
cost_data = await self.get_x_cashu_cost(
|
||||
response_data, max_cost_for_model
|
||||
response_data, max_cost_for_model, model_obj
|
||||
)
|
||||
if cost_data:
|
||||
if unit == "msat":
|
||||
@@ -4490,14 +4703,8 @@ class BaseUpstreamProvider:
|
||||
if "provider" not in data_json:
|
||||
self._apply_provider_field(data_json)
|
||||
changed = True
|
||||
if (
|
||||
cost_data
|
||||
and "usage" in data_json
|
||||
and data_json["usage"]
|
||||
):
|
||||
data_json["usage"]["cost_sats"] = (
|
||||
cost_data.total_msats // 1000
|
||||
)
|
||||
if cost_data and "usage" in data_json and data_json["usage"]:
|
||||
data_json["usage"]["cost_sats"] = cost_data.total_msats // 1000
|
||||
changed = True
|
||||
if changed:
|
||||
lines[i] = "data: " + json.dumps(data_json)
|
||||
@@ -4524,6 +4731,7 @@ class BaseUpstreamProvider:
|
||||
max_cost_for_model: int,
|
||||
mint: str | None = None,
|
||||
request_id: str | None = None,
|
||||
model_obj: Model | None = None,
|
||||
) -> Response:
|
||||
"""Handle non-streaming Responses API response for X-Cashu payment."""
|
||||
logger.debug(
|
||||
@@ -4534,7 +4742,9 @@ class BaseUpstreamProvider:
|
||||
try:
|
||||
response_json = json.loads(content_str)
|
||||
self._apply_provider_field(response_json)
|
||||
cost_data = await self.get_x_cashu_cost(response_json, max_cost_for_model)
|
||||
cost_data = await self.get_x_cashu_cost(
|
||||
response_json, max_cost_for_model, model_obj
|
||||
)
|
||||
|
||||
if cost_data and "usage" in response_json:
|
||||
response_json["usage"]["cost_sats"] = cost_data.total_msats // 1000
|
||||
@@ -4986,7 +5196,9 @@ class BaseUpstreamProvider:
|
||||
except Exception:
|
||||
self._models_cache = models_with_fees
|
||||
|
||||
self._models_by_id = {m.forwarded_model_id or m.id: m for m in self._models_cache}
|
||||
self._models_by_id = {
|
||||
m.forwarded_model_id or m.id: m for m in self._models_cache
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
|
||||
@@ -15,7 +15,11 @@ from sqlmodel import col, update
|
||||
|
||||
from ..auth import (
|
||||
ROUTSTR_FEE_PERCENT,
|
||||
ReservationSnapshot,
|
||||
_claim_reservation_for_charge,
|
||||
_validate_reservation_snapshot,
|
||||
get_billing_key,
|
||||
get_reservation_snapshot,
|
||||
payments_logger,
|
||||
)
|
||||
from ..core import get_logger
|
||||
@@ -23,7 +27,9 @@ from ..core.db import (
|
||||
ApiKey,
|
||||
AsyncSession,
|
||||
accumulate_routstr_fee,
|
||||
store_cashu_transaction,
|
||||
)
|
||||
from ..core.db import (
|
||||
store_cashu_transaction_with_retry as store_cashu_transaction,
|
||||
)
|
||||
from ..core.exceptions import UpstreamError
|
||||
from ..core.settings import settings
|
||||
@@ -500,8 +506,14 @@ async def finalize_ehbp_actual_cost_payment(
|
||||
reserved_cost_for_model: int,
|
||||
model_id: str,
|
||||
cost_info: dict,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> None:
|
||||
"""Finalize an EHBP bearer request using clamped provider usage metrics."""
|
||||
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
|
||||
await _validate_reservation_snapshot(key, reservation, session)
|
||||
if not await _claim_reservation_for_charge(reservation, session):
|
||||
return
|
||||
reserved_cost_for_model = reservation.reserved_msats
|
||||
billing_key = await get_billing_key(key, session)
|
||||
key_hash = key.hashed_key
|
||||
billing_key_hash = billing_key.hashed_key
|
||||
@@ -604,6 +616,7 @@ async def finalize_ehbp_max_cost_payment(
|
||||
session: AsyncSession,
|
||||
max_cost_for_model: int,
|
||||
model_id: str,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> None:
|
||||
"""Finalize an EHBP bearer request by charging the reserved max cost.
|
||||
|
||||
@@ -611,6 +624,11 @@ async def finalize_ehbp_max_cost_payment(
|
||||
normal completion handlers, this intentionally charges the pre-reserved max
|
||||
cost and releases the reservation.
|
||||
"""
|
||||
reservation = reservation_snapshot or await get_reservation_snapshot(key, session)
|
||||
await _validate_reservation_snapshot(key, reservation, session)
|
||||
if not await _claim_reservation_for_charge(reservation, session):
|
||||
return
|
||||
max_cost_for_model = reservation.reserved_msats
|
||||
billing_key = await get_billing_key(key, session)
|
||||
key_hash = key.hashed_key
|
||||
billing_key_hash = billing_key.hashed_key
|
||||
@@ -764,6 +782,7 @@ async def forward_ehbp_request(
|
||||
max_cost_for_model: int,
|
||||
session: AsyncSession,
|
||||
model_obj: Model,
|
||||
reservation_snapshot: ReservationSnapshot | None = None,
|
||||
) -> Response | StreamingResponse:
|
||||
"""Forward an EHBP bearer-auth request and finalize billing.
|
||||
|
||||
@@ -881,7 +900,12 @@ async def forward_ehbp_request(
|
||||
# the requested model.
|
||||
billing_model = cost_info.pop("actual_model", None) or model_obj.id
|
||||
await finalize_ehbp_actual_cost_payment(
|
||||
key, session, max_cost_for_model, billing_model, cost_info
|
||||
key,
|
||||
session,
|
||||
max_cost_for_model,
|
||||
billing_model,
|
||||
cost_info,
|
||||
reservation_snapshot,
|
||||
)
|
||||
cost_data = {**cost_info, "total_usd": 0.0}
|
||||
else:
|
||||
@@ -895,7 +919,11 @@ async def forward_ehbp_request(
|
||||
},
|
||||
)
|
||||
await finalize_ehbp_max_cost_payment(
|
||||
key, session, max_cost_for_model, model_obj.id
|
||||
key,
|
||||
session,
|
||||
max_cost_for_model,
|
||||
model_obj.id,
|
||||
reservation_snapshot,
|
||||
)
|
||||
cost_data = {
|
||||
"total_msats": max_cost_for_model,
|
||||
|
||||
+19
-10
@@ -94,12 +94,10 @@ async def get_all_models_with_overrides(
|
||||
provider_result = await session.exec(select(UpstreamProviderRow))
|
||||
providers_by_id = {p.id: p for p in provider_result.all()}
|
||||
|
||||
overrides_by_id: dict[str, tuple[ModelRow, float]] = {
|
||||
row.id: (
|
||||
overrides_by_key: dict[tuple[str, int], tuple[ModelRow, float]] = {
|
||||
(row.id.lower(), row.upstream_provider_id): (
|
||||
row,
|
||||
providers_by_id[row.upstream_provider_id].provider_fee
|
||||
if row.upstream_provider_id in providers_by_id
|
||||
else 1.01,
|
||||
providers_by_id[row.upstream_provider_id].provider_fee,
|
||||
)
|
||||
for row in override_rows
|
||||
if row.upstream_provider_id is not None
|
||||
@@ -107,17 +105,28 @@ async def get_all_models_with_overrides(
|
||||
and providers_by_id[row.upstream_provider_id].enabled
|
||||
}
|
||||
|
||||
all_models: dict[str, Model] = {}
|
||||
all_models: dict[tuple[str, str], Model] = {}
|
||||
|
||||
for upstream in upstreams:
|
||||
upstream_db_id = getattr(upstream, "db_id", None)
|
||||
provider_key = (
|
||||
f"db:{upstream_db_id}"
|
||||
if isinstance(upstream_db_id, int)
|
||||
else f"{getattr(upstream, 'provider_type', '')}|{getattr(upstream, 'base_url', '')}"
|
||||
)
|
||||
for model in upstream.get_cached_models():
|
||||
if model.id in overrides_by_id:
|
||||
override_row, provider_fee = overrides_by_id[model.id]
|
||||
all_models[model.id] = _row_to_model(
|
||||
model_key = (
|
||||
(model.id.lower(), upstream_db_id)
|
||||
if isinstance(upstream_db_id, int)
|
||||
else None
|
||||
)
|
||||
if model_key is not None and model_key in overrides_by_key:
|
||||
override_row, provider_fee = overrides_by_key[model_key]
|
||||
all_models[(model.id.lower(), provider_key)] = _row_to_model(
|
||||
override_row, apply_provider_fee=True, provider_fee=provider_fee
|
||||
)
|
||||
elif model.enabled:
|
||||
all_models[model.id] = model
|
||||
all_models[(model.id.lower(), provider_key)] = model
|
||||
|
||||
return list(all_models.values())
|
||||
|
||||
|
||||
@@ -0,0 +1,807 @@
|
||||
"""Model-path discovery service.
|
||||
|
||||
Exposes every selectable upstream route a Routstr model is reachable through.
|
||||
This PR remains discovery-only: request-side routing will consume the opaque
|
||||
selectors in a follow-up.
|
||||
|
||||
A path is a standard percent-encoded query string containing the configured
|
||||
upstream URL, provider ID, client-visible model ID and, for an exact OpenRouter
|
||||
endpoint, its machine-readable tag::
|
||||
|
||||
url=https%3A%2F%2Fapi.anthropic.com%2Fv1&provider-id=12&model-id=claude-sonnet-4
|
||||
url=https%3A%2F%2Fopenrouter.ai%2Fapi%2Fv1&provider-id=42&model-id=claude-sonnet-4&endpoint=google-vertex%2Fus
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import random
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Callable
|
||||
from urllib.parse import urlencode, urlsplit
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.dialects.sqlite import insert
|
||||
from sqlalchemy.orm import selectinload
|
||||
from sqlmodel import col, delete, select
|
||||
|
||||
from ..core.db import ModelPathRow, ModelRow, UpstreamProviderRow, create_session
|
||||
from ..core.logging import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .base import BaseUpstreamProvider
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Bound the per-model OpenRouter /endpoints fan-out so a provider with hundreds
|
||||
# of models does not open hundreds of concurrent requests every refresh.
|
||||
_OPENROUTER_CONCURRENCY = 5
|
||||
_OPENROUTER_TIMEOUT_SECONDS = 10.0
|
||||
|
||||
# Rows inserted per statement during persist. Keeps each INSERT bounded while
|
||||
# avoiding per-row round-trips that hold SQLite's write lock for ~1s per cycle.
|
||||
_PERSIST_CHUNK_SIZE = 500
|
||||
|
||||
# Admin mutations enqueue provider IDs here instead of running OpenRouter's
|
||||
# per-model endpoint fan-out inside the request. One worker serializes refreshes
|
||||
# and coalesces repeated mutations for the same provider.
|
||||
_scheduled_provider_refresh_ids: set[int] = set()
|
||||
_scheduled_provider_refresh_task: asyncio.Task[None] | None = None
|
||||
|
||||
# Visibility key used across this module: routing carries the provider
|
||||
# dimension everywhere (ModelRow's primary key is (id, upstream_provider_id)),
|
||||
# so all model-id keyed maps here do too, lowercased like proxy.refresh_model_maps.
|
||||
ModelKey = tuple[str, int]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EndpointIdentity:
|
||||
"""Exact OpenRouter endpoint identity returned by ``/endpoints``."""
|
||||
|
||||
tag: str
|
||||
provider_name: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConfiguredProviderIdentity:
|
||||
"""Public identity of one configured upstream provider."""
|
||||
|
||||
id: int
|
||||
slug: str
|
||||
provider_type: str
|
||||
base_url: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DiscoveredPath:
|
||||
"""One model route ready for persistence and API serialization."""
|
||||
|
||||
model_id: str
|
||||
path: str
|
||||
provider: ConfiguredProviderIdentity
|
||||
endpoint_tag: str | None = None
|
||||
endpoint_name: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProviderPathSnapshot:
|
||||
"""Refresh result plus model IDs whose prior rows must survive degradation."""
|
||||
|
||||
paths: tuple[DiscoveredPath, ...]
|
||||
preserve_model_ids: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
def public_provider_url(base_url: str) -> str:
|
||||
"""Mask private IP addresses and URLs with explicit ports."""
|
||||
parsed = urlsplit(base_url)
|
||||
try:
|
||||
if parsed.port is not None:
|
||||
return "http://localhost"
|
||||
except ValueError:
|
||||
# An invalid explicit port must not accidentally leak through.
|
||||
return "http://localhost"
|
||||
|
||||
hostname = parsed.hostname
|
||||
if hostname is None:
|
||||
return base_url
|
||||
try:
|
||||
address = ipaddress.ip_address(hostname)
|
||||
except ValueError:
|
||||
return base_url
|
||||
return "http://localhost" if address.is_private else base_url
|
||||
|
||||
|
||||
def encode_model_path(
|
||||
base_url: str,
|
||||
provider_id: int,
|
||||
model_id: str,
|
||||
endpoint_tag: str | None = None,
|
||||
) -> str:
|
||||
"""Encode the complete upstream route selector advertised to clients."""
|
||||
components: list[tuple[str, str | int]] = [
|
||||
("url", base_url),
|
||||
("provider-id", provider_id),
|
||||
("model-id", model_id),
|
||||
]
|
||||
if endpoint_tag:
|
||||
components.append(("endpoint", endpoint_tag))
|
||||
return urlencode(components)
|
||||
|
||||
|
||||
def _make_http_client() -> httpx.AsyncClient:
|
||||
"""Client factory, separated so tests can substitute a mock transport."""
|
||||
return httpx.AsyncClient()
|
||||
|
||||
|
||||
def is_openrouter_base_url(base_url: str | None) -> bool:
|
||||
"""True when ``base_url`` points at OpenRouter.
|
||||
|
||||
Deliberately separate from ``BaseUpstreamProvider._upstream_accepts_cache_control``:
|
||||
that predicate also returns True for native Anthropic (correct for
|
||||
cache-control, wrong for OpenRouter endpoint discovery). This one keys only
|
||||
on the URL so a ``GenericUpstreamProvider`` aimed at OpenRouter is matched
|
||||
while native Anthropic is not.
|
||||
"""
|
||||
return "openrouter.ai" in (base_url or "")
|
||||
|
||||
|
||||
def exposed_model_id(model: object) -> str:
|
||||
"""Return exactly the ID advertised by ``/v1/models``.
|
||||
|
||||
A forwarded ID is already a public routable alias and must remain intact,
|
||||
including any slash. Without one, ``/v1/models`` exposes the base ID.
|
||||
"""
|
||||
forwarded = getattr(model, "forwarded_model_id", None)
|
||||
if forwarded:
|
||||
return forwarded
|
||||
return public_model_id(getattr(model, "id"))
|
||||
|
||||
|
||||
def public_model_id(model_id: str) -> str:
|
||||
"""Model id exposed by model-path API responses.
|
||||
|
||||
Uses the same rule as ``create_model_mappings.get_base_model_id`` and
|
||||
``resolve_model_alias`` — strip everything before the *first* slash — so
|
||||
the id shown here can be sent back to ``/v1/chat/completions`` verbatim.
|
||||
"""
|
||||
return model_id.split("/", 1)[1] if "/" in model_id else model_id
|
||||
|
||||
|
||||
def openrouter_author_slug(model: object) -> str | None:
|
||||
"""Return a canonical ``author/slug`` for the OpenRouter endpoints API.
|
||||
|
||||
Prefer ``canonical_slug``, then a slash-containing ``id``, then a
|
||||
slash-containing ``forwarded_model_id``. The forwarded id is exactly what
|
||||
the proxy sends upstream for admin-created alias rows (``base.py`` forwards
|
||||
``forwarded_model_id or id``), so it is a valid OpenRouter id when the
|
||||
bare ``id`` is a local alias with no slash.
|
||||
"""
|
||||
canonical = getattr(model, "canonical_slug", None)
|
||||
if canonical and "/" in canonical:
|
||||
return canonical
|
||||
model_id = getattr(model, "id", None)
|
||||
if model_id and "/" in model_id:
|
||||
return model_id
|
||||
forwarded = getattr(model, "forwarded_model_id", None)
|
||||
if forwarded and "/" in forwarded:
|
||||
return forwarded
|
||||
return None
|
||||
|
||||
|
||||
class _RefreshCycleState:
|
||||
"""Per-refresh shared state: fetch dedupe cache and rate-limit latch.
|
||||
|
||||
``endpoint_cache`` dedupes byte-identical ``/endpoints`` fetches when two
|
||||
providers point at the same OpenRouter base URL. ``rate_limited`` latches
|
||||
on the first 429 so the rest of the cycle stops hammering a throttled API;
|
||||
the whole provider result then degrades to "unknown" instead of an empty
|
||||
list, which preserves previously persisted rows.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.endpoint_cache: dict[tuple[str, str], list[EndpointIdentity] | None] = {}
|
||||
self.rate_limited = False
|
||||
|
||||
|
||||
async def _fetch_openrouter_endpoint_subproviders(
|
||||
client: httpx.AsyncClient,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
author_slug: str,
|
||||
semaphore: asyncio.Semaphore,
|
||||
cycle: _RefreshCycleState,
|
||||
) -> list[EndpointIdentity] | None:
|
||||
"""Return exact endpoint identities for one model, or ``None`` when unknown.
|
||||
|
||||
``None`` (not ``[]``) signals a degraded fetch — network failure, rate
|
||||
limit, non-200, or an unparseable payload — so callers can distinguish
|
||||
"this model has no endpoints" from "we could not find out". Failures are
|
||||
logged and swallowed so one model never breaks the whole refresh.
|
||||
"""
|
||||
cache_key = (base_url, author_slug)
|
||||
if cache_key in cycle.endpoint_cache:
|
||||
return cycle.endpoint_cache[cache_key]
|
||||
if cycle.rate_limited:
|
||||
return None
|
||||
|
||||
url = f"{base_url.rstrip('/')}/models/{author_slug}/endpoints"
|
||||
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
|
||||
result: list[EndpointIdentity] | None
|
||||
async with semaphore:
|
||||
try:
|
||||
resp = await client.get(
|
||||
url, headers=headers, timeout=_OPENROUTER_TIMEOUT_SECONDS
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 - isolate per-model failures
|
||||
logger.warning(
|
||||
"OpenRouter endpoint discovery request failed",
|
||||
extra={"author_slug": author_slug, "error": str(e)},
|
||||
)
|
||||
cycle.endpoint_cache[cache_key] = None
|
||||
return None
|
||||
|
||||
if resp.status_code == 429:
|
||||
logger.warning(
|
||||
"OpenRouter endpoint discovery rate-limited; aborting cycle",
|
||||
extra={"author_slug": author_slug},
|
||||
)
|
||||
cycle.rate_limited = True
|
||||
cycle.endpoint_cache[cache_key] = None
|
||||
return None
|
||||
if resp.status_code != 200:
|
||||
logger.warning(
|
||||
"OpenRouter endpoint discovery non-200",
|
||||
extra={"author_slug": author_slug, "status_code": resp.status_code},
|
||||
)
|
||||
cycle.endpoint_cache[cache_key] = None
|
||||
return None
|
||||
|
||||
try:
|
||||
payload = resp.json()
|
||||
data = payload.get("data") if isinstance(payload, dict) else None
|
||||
endpoints = data.get("endpoints") if isinstance(data, dict) else None
|
||||
if not isinstance(endpoints, list):
|
||||
raise ValueError("endpoints must be a list")
|
||||
identities: dict[str, EndpointIdentity] = {}
|
||||
for endpoint in endpoints:
|
||||
if not isinstance(endpoint, dict):
|
||||
continue
|
||||
tag = endpoint.get("tag")
|
||||
if not isinstance(tag, str) or not tag.strip():
|
||||
continue
|
||||
provider_name = endpoint.get("provider_name")
|
||||
identities.setdefault(
|
||||
tag,
|
||||
EndpointIdentity(
|
||||
tag=tag,
|
||||
provider_name=provider_name
|
||||
if isinstance(provider_name, str) and provider_name
|
||||
else None,
|
||||
),
|
||||
)
|
||||
if endpoints and not identities:
|
||||
raise ValueError("endpoints contain no usable tags")
|
||||
result = list(identities.values())
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning(
|
||||
"OpenRouter endpoint discovery bad payload",
|
||||
extra={"author_slug": author_slug, "error": str(e)},
|
||||
)
|
||||
result = None
|
||||
|
||||
cycle.endpoint_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
|
||||
async def _load_model_visibility() -> tuple[
|
||||
dict[ModelKey, ModelRow],
|
||||
set[ModelKey],
|
||||
dict[int, ConfiguredProviderIdentity],
|
||||
]:
|
||||
"""Load the same DB model visibility inputs used by routing.
|
||||
|
||||
``refresh_model_maps`` builds routing from enabled providers, enabled DB
|
||||
override rows, and disabled model keys — all keyed on
|
||||
``(model_id.lower(), upstream_provider_id)`` because ``ModelRow``'s primary
|
||||
key is composite and the same id legitimately exists on several providers.
|
||||
Model-path discovery uses the same keying so disabling a model on one
|
||||
provider never hides it on another, and one provider's
|
||||
``forwarded_model_id`` alias is never applied to a different provider.
|
||||
"""
|
||||
async with create_session() as session:
|
||||
query = select(UpstreamProviderRow).options(
|
||||
selectinload(UpstreamProviderRow.models) # type: ignore[arg-type]
|
||||
)
|
||||
provider_rows = (await session.exec(query)).all()
|
||||
|
||||
overrides_by_key: dict[ModelKey, ModelRow] = {}
|
||||
disabled_model_keys: set[ModelKey] = set()
|
||||
provider_identities: dict[int, ConfiguredProviderIdentity] = {}
|
||||
|
||||
for provider in provider_rows:
|
||||
if not provider.enabled or provider.id is None:
|
||||
continue
|
||||
provider_identities[provider.id] = ConfiguredProviderIdentity(
|
||||
id=provider.id,
|
||||
slug=provider.slug or f"provider-{provider.id}",
|
||||
provider_type=provider.provider_type,
|
||||
base_url=public_provider_url(provider.base_url),
|
||||
)
|
||||
for model in provider.models:
|
||||
key = (model.id.lower(), provider.id)
|
||||
if model.enabled:
|
||||
overrides_by_key[key] = model
|
||||
else:
|
||||
disabled_model_keys.add(key)
|
||||
|
||||
return overrides_by_key, disabled_model_keys, provider_identities
|
||||
|
||||
|
||||
def _apply_model_visibility(
|
||||
upstream: BaseUpstreamProvider,
|
||||
overrides_by_key: dict[ModelKey, ModelRow] | None,
|
||||
disabled_model_keys: set[ModelKey] | None,
|
||||
) -> list[object]:
|
||||
"""Return provider models after DB disabled/override state is applied.
|
||||
|
||||
Only the identity fields (``id``, ``forwarded_model_id``,
|
||||
``canonical_slug``) matter for path discovery, so DB override rows are used
|
||||
directly rather than rebuilt into fully priced ``Model`` objects — the
|
||||
pricing pipeline costs ~0.7ms of event-loop CPU per row for data this
|
||||
module immediately discards.
|
||||
"""
|
||||
overrides_by_key = overrides_by_key or {}
|
||||
disabled_model_keys = disabled_model_keys or set()
|
||||
upstream_provider_id = getattr(upstream, "db_id", None)
|
||||
if not isinstance(upstream_provider_id, int):
|
||||
return [
|
||||
model
|
||||
for model in upstream.get_cached_models()
|
||||
if getattr(model, "enabled", True)
|
||||
]
|
||||
|
||||
visible_models: list[object] = []
|
||||
seen_model_ids: set[str] = set()
|
||||
|
||||
for model in upstream.get_cached_models():
|
||||
model_id = getattr(model, "id", "")
|
||||
key = (model_id.lower(), upstream_provider_id)
|
||||
if not getattr(model, "enabled", True) or key in disabled_model_keys:
|
||||
continue
|
||||
# Apply overrides only for this provider's own model row.
|
||||
override_row = overrides_by_key.get(key)
|
||||
visible: object = model if override_row is None else override_row
|
||||
visible_models.append(visible)
|
||||
seen_model_ids.add(model_id.lower())
|
||||
|
||||
# DB-only override rows for this provider with no cached counterpart.
|
||||
for (model_id_lower, provider_id), override_row in overrides_by_key.items():
|
||||
if provider_id != upstream_provider_id:
|
||||
continue
|
||||
if model_id_lower in seen_model_ids:
|
||||
continue
|
||||
visible_models.append(override_row)
|
||||
seen_model_ids.add(model_id_lower)
|
||||
|
||||
return visible_models
|
||||
|
||||
|
||||
async def _collect_provider_paths(
|
||||
upstream: BaseUpstreamProvider,
|
||||
provider_identity: ConfiguredProviderIdentity,
|
||||
overrides_by_key: dict[ModelKey, ModelRow] | None = None,
|
||||
disabled_model_keys: set[ModelKey] | None = None,
|
||||
cycle: _RefreshCycleState | None = None,
|
||||
) -> ProviderPathSnapshot:
|
||||
"""Collect selectable routes while marking model-level degraded fetches.
|
||||
|
||||
A failed OpenRouter lookup preserves only that model's prior rows. Other
|
||||
models in the same provider still refresh, so a partial outage cannot erase
|
||||
valid discovery data or freeze the entire provider snapshot.
|
||||
"""
|
||||
cycle = cycle or _RefreshCycleState()
|
||||
models = _apply_model_visibility(upstream, overrides_by_key, disabled_model_keys)
|
||||
|
||||
def _base_path(model: object) -> DiscoveredPath:
|
||||
model_id = exposed_model_id(model)
|
||||
return DiscoveredPath(
|
||||
model_id=model_id,
|
||||
path=encode_model_path(
|
||||
provider_identity.base_url, provider_identity.id, model_id
|
||||
),
|
||||
provider=provider_identity,
|
||||
)
|
||||
|
||||
if not is_openrouter_base_url(upstream.base_url):
|
||||
return ProviderPathSnapshot(paths=tuple(_base_path(model) for model in models))
|
||||
|
||||
if not (upstream.provider_type or "").strip():
|
||||
return ProviderPathSnapshot(paths=())
|
||||
|
||||
semaphore = asyncio.Semaphore(_OPENROUTER_CONCURRENCY)
|
||||
async with _make_http_client() as client:
|
||||
|
||||
async def _for_model(
|
||||
model: object,
|
||||
) -> tuple[list[DiscoveredPath], str | None]:
|
||||
model_id = exposed_model_id(model)
|
||||
author_slug = openrouter_author_slug(model)
|
||||
if not author_slug:
|
||||
return [_base_path(model)], None
|
||||
endpoints = await _fetch_openrouter_endpoint_subproviders(
|
||||
client,
|
||||
upstream.base_url,
|
||||
upstream.api_key,
|
||||
author_slug,
|
||||
semaphore,
|
||||
cycle,
|
||||
)
|
||||
if endpoints is None:
|
||||
return [], model_id
|
||||
paths = [_base_path(model)]
|
||||
paths.extend(
|
||||
DiscoveredPath(
|
||||
model_id=model_id,
|
||||
path=encode_model_path(
|
||||
provider_identity.base_url,
|
||||
provider_identity.id,
|
||||
model_id,
|
||||
endpoint.tag,
|
||||
),
|
||||
provider=provider_identity,
|
||||
endpoint_tag=endpoint.tag,
|
||||
endpoint_name=endpoint.provider_name,
|
||||
)
|
||||
for endpoint in endpoints
|
||||
)
|
||||
return paths, None
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(_for_model(model) for model in models), return_exceptions=True
|
||||
)
|
||||
|
||||
paths: list[DiscoveredPath] = []
|
||||
preserve_model_ids: set[str] = set()
|
||||
for model, result in zip(models, results):
|
||||
if isinstance(result, BaseException):
|
||||
model_id = exposed_model_id(model)
|
||||
preserve_model_ids.add(model_id)
|
||||
logger.warning(
|
||||
"OpenRouter endpoint discovery task errored",
|
||||
extra={"provider": upstream.provider_type, "error": str(result)},
|
||||
)
|
||||
continue
|
||||
model_paths, preserved_model_id = result
|
||||
paths.extend(model_paths)
|
||||
if preserved_model_id:
|
||||
preserve_model_ids.add(preserved_model_id)
|
||||
|
||||
return ProviderPathSnapshot(
|
||||
paths=tuple(paths), preserve_model_ids=frozenset(preserve_model_ids)
|
||||
)
|
||||
|
||||
|
||||
async def _persist_provider_paths(
|
||||
upstream_provider_id: int, snapshot: ProviderPathSnapshot
|
||||
) -> None:
|
||||
"""Replace refreshed rows while retaining model-level degraded snapshots."""
|
||||
unique_paths = list(
|
||||
{(path.model_id, path.path): path for path in snapshot.paths}.values()
|
||||
)
|
||||
now = int(time.time())
|
||||
async with create_session() as session:
|
||||
delete_stmt = delete(ModelPathRow).where(
|
||||
col(ModelPathRow.upstream_provider_id) == upstream_provider_id
|
||||
)
|
||||
if snapshot.preserve_model_ids:
|
||||
delete_stmt = delete_stmt.where(
|
||||
col(ModelPathRow.model_id).not_in(sorted(snapshot.preserve_model_ids))
|
||||
)
|
||||
await session.exec(delete_stmt) # type: ignore[call-overload]
|
||||
for start in range(0, len(unique_paths), _PERSIST_CHUNK_SIZE):
|
||||
chunk = unique_paths[start : start + _PERSIST_CHUNK_SIZE]
|
||||
values = [
|
||||
{
|
||||
"model_id": discovered.model_id,
|
||||
"path": discovered.path,
|
||||
"provider_slug": discovered.provider.slug,
|
||||
"provider_type": discovered.provider.provider_type,
|
||||
"endpoint_tag": discovered.endpoint_tag,
|
||||
"endpoint_name": discovered.endpoint_name,
|
||||
"upstream_provider_id": upstream_provider_id,
|
||||
"updated_at": now,
|
||||
}
|
||||
for discovered in chunk
|
||||
]
|
||||
insert_stmt = insert(ModelPathRow).values(values)
|
||||
await session.execute(
|
||||
insert_stmt.on_conflict_do_update(
|
||||
index_elements=["model_id", "path", "upstream_provider_id"],
|
||||
set_={
|
||||
"provider_slug": insert_stmt.excluded.provider_slug,
|
||||
"provider_type": insert_stmt.excluded.provider_type,
|
||||
"endpoint_tag": insert_stmt.excluded.endpoint_tag,
|
||||
"endpoint_name": insert_stmt.excluded.endpoint_name,
|
||||
"updated_at": insert_stmt.excluded.updated_at,
|
||||
},
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def prune_model_paths_for_inactive_providers() -> None:
|
||||
"""Delete paths whose provider is no longer enabled in the database.
|
||||
|
||||
Called from ``refresh_model_maps`` so admin mutations (disable/delete
|
||||
provider) stop advertising a provider's paths immediately instead of
|
||||
waiting for the next timed refresh. Uses the DB as the source of truth, so
|
||||
it is safe at boot even before upstreams initialize.
|
||||
"""
|
||||
async with create_session() as session:
|
||||
enabled_ids = (
|
||||
await session.exec(
|
||||
select(UpstreamProviderRow.id).where(
|
||||
col(UpstreamProviderRow.enabled).is_(True)
|
||||
)
|
||||
)
|
||||
).all()
|
||||
stmt = delete(ModelPathRow)
|
||||
if enabled_ids:
|
||||
stmt = stmt.where(
|
||||
col(ModelPathRow.upstream_provider_id).not_in(
|
||||
[pid for pid in enabled_ids if pid is not None]
|
||||
)
|
||||
)
|
||||
await session.exec(stmt) # type: ignore[call-overload]
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def refresh_model_paths(
|
||||
upstreams: list[BaseUpstreamProvider],
|
||||
) -> None:
|
||||
"""Recompute and persist model paths for every enabled provider.
|
||||
|
||||
One provider's failure is logged and isolated; it must not break the rest.
|
||||
A provider whose paths could not be determined this cycle keeps its
|
||||
previously persisted rows. An empty ``upstreams`` list (e.g. a failed
|
||||
``initialize_upstreams`` at boot) is treated as "unknown" and touches
|
||||
nothing.
|
||||
"""
|
||||
if not upstreams:
|
||||
logger.warning("Skipping model paths refresh: no live upstreams")
|
||||
return
|
||||
|
||||
(
|
||||
overrides_by_key,
|
||||
disabled_model_keys,
|
||||
provider_identities,
|
||||
) = await _load_model_visibility()
|
||||
await prune_model_paths_for_inactive_providers()
|
||||
|
||||
cycle = _RefreshCycleState()
|
||||
for upstream in upstreams:
|
||||
if upstream.db_id is None or upstream.db_id not in provider_identities:
|
||||
continue
|
||||
try:
|
||||
snapshot = await _collect_provider_paths(
|
||||
upstream,
|
||||
provider_identity=provider_identities[upstream.db_id],
|
||||
overrides_by_key=overrides_by_key,
|
||||
disabled_model_keys=disabled_model_keys,
|
||||
cycle=cycle,
|
||||
)
|
||||
if snapshot.preserve_model_ids:
|
||||
logger.warning(
|
||||
"Some model paths are unknown; keeping their previous rows",
|
||||
extra={
|
||||
"provider": upstream.provider_type or upstream.base_url,
|
||||
"db_id": upstream.db_id,
|
||||
"preserved_models": len(snapshot.preserve_model_ids),
|
||||
},
|
||||
)
|
||||
await _persist_provider_paths(upstream.db_id, snapshot)
|
||||
except Exception as e: # noqa: BLE001 - isolate per-provider failures
|
||||
logger.error(
|
||||
"Failed to refresh model paths for provider",
|
||||
extra={
|
||||
"provider": upstream.provider_type or upstream.base_url,
|
||||
"db_id": upstream.db_id,
|
||||
"error": str(e),
|
||||
"error_type": type(e).__name__,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def refresh_model_paths_for_provider(upstream_provider_id: int) -> None:
|
||||
"""Synchronize one provider when model-path discovery is enabled."""
|
||||
if _refresh_interval_seconds() <= 0:
|
||||
return
|
||||
|
||||
from ..proxy import get_upstreams
|
||||
|
||||
matching = [
|
||||
upstream
|
||||
for upstream in get_upstreams()
|
||||
if upstream.db_id == upstream_provider_id
|
||||
]
|
||||
if matching:
|
||||
await refresh_model_paths(matching)
|
||||
else:
|
||||
await prune_model_paths_for_inactive_providers()
|
||||
|
||||
|
||||
async def _drain_scheduled_provider_refreshes() -> None:
|
||||
"""Serialize and coalesce model-path refreshes scheduled by admin writes."""
|
||||
global _scheduled_provider_refresh_task
|
||||
|
||||
try:
|
||||
# Let mutations in the same event-loop turn collapse into one refresh.
|
||||
await asyncio.sleep(0)
|
||||
while _scheduled_provider_refresh_ids:
|
||||
if _refresh_interval_seconds() <= 0:
|
||||
_scheduled_provider_refresh_ids.clear()
|
||||
return
|
||||
provider_id = min(_scheduled_provider_refresh_ids)
|
||||
_scheduled_provider_refresh_ids.remove(provider_id)
|
||||
try:
|
||||
await refresh_model_paths_for_provider(provider_id)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - background best effort
|
||||
logger.warning(
|
||||
"Failed to refresh model paths after admin mutation",
|
||||
extra={
|
||||
"upstream_provider_id": provider_id,
|
||||
"error": str(exc),
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
)
|
||||
finally:
|
||||
_scheduled_provider_refresh_task = None
|
||||
|
||||
|
||||
async def schedule_model_paths_refresh_for_provider(
|
||||
upstream_provider_id: int,
|
||||
) -> None:
|
||||
"""Queue a non-blocking, coalesced refresh after an admin mutation."""
|
||||
global _scheduled_provider_refresh_task
|
||||
|
||||
if _refresh_interval_seconds() <= 0:
|
||||
return
|
||||
_scheduled_provider_refresh_ids.add(upstream_provider_id)
|
||||
if (
|
||||
_scheduled_provider_refresh_task is None
|
||||
or _scheduled_provider_refresh_task.done()
|
||||
):
|
||||
_scheduled_provider_refresh_task = asyncio.create_task(
|
||||
_drain_scheduled_provider_refreshes(),
|
||||
name="model-path-admin-refresh",
|
||||
)
|
||||
|
||||
|
||||
def _refresh_interval_seconds() -> int:
|
||||
"""Current interval, re-read every loop so runtime setting changes apply."""
|
||||
from ..core.settings import settings
|
||||
|
||||
if not getattr(settings, "enable_model_paths_refresh", True):
|
||||
return 0
|
||||
return int(getattr(settings, "model_paths_refresh_interval_seconds", 0) or 0)
|
||||
|
||||
|
||||
async def refresh_model_paths_periodically(
|
||||
upstreams_provider: (
|
||||
Callable[[], list[BaseUpstreamProvider]] | list[BaseUpstreamProvider]
|
||||
),
|
||||
) -> None:
|
||||
"""Background task mirroring ``refresh_upstreams_models_periodically``.
|
||||
|
||||
The interval and enable flag are re-read every iteration, so the refresh
|
||||
can be turned off (or on) and retuned without a restart. While disabled the
|
||||
task idles instead of exiting, so re-enabling takes effect.
|
||||
"""
|
||||
_DISABLED_POLL_SECONDS = 60.0
|
||||
|
||||
def _resolve_upstreams() -> list[BaseUpstreamProvider]:
|
||||
if callable(upstreams_provider):
|
||||
return upstreams_provider()
|
||||
return upstreams_provider
|
||||
|
||||
while True:
|
||||
interval = _refresh_interval_seconds()
|
||||
if interval <= 0:
|
||||
try:
|
||||
await asyncio.sleep(_DISABLED_POLL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
continue
|
||||
|
||||
try:
|
||||
await refresh_model_paths(_resolve_upstreams())
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.error(
|
||||
"Error in model paths refresh loop",
|
||||
extra={"error": str(e), "error_type": type(e).__name__},
|
||||
)
|
||||
|
||||
try:
|
||||
jitter = max(0.0, float(interval) * 0.1)
|
||||
await asyncio.sleep(interval + random.uniform(0, jitter))
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
|
||||
|
||||
def _serialize_path(row: ModelPathRow) -> dict[str, Any]:
|
||||
endpoint = None
|
||||
if row.endpoint_tag or row.endpoint_name:
|
||||
endpoint = {"tag": row.endpoint_tag, "name": row.endpoint_name}
|
||||
return {
|
||||
"path": row.path,
|
||||
"provider": {
|
||||
"id": row.upstream_provider_id,
|
||||
"slug": row.provider_slug,
|
||||
"type": row.provider_type,
|
||||
},
|
||||
"endpoint": endpoint,
|
||||
}
|
||||
|
||||
|
||||
async def get_all_model_paths() -> dict:
|
||||
"""All models with their exact selectable routes."""
|
||||
async with create_session() as session:
|
||||
rows = (
|
||||
await session.exec(
|
||||
select(ModelPathRow).order_by(
|
||||
col(ModelPathRow.model_id),
|
||||
col(ModelPathRow.path),
|
||||
col(ModelPathRow.upstream_provider_id),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
|
||||
grouped: dict[str, list[dict[str, Any]]] = {}
|
||||
seen_paths: dict[str, set[str]] = {}
|
||||
updated_at = 0
|
||||
for row in rows:
|
||||
updated_at = max(updated_at, row.updated_at)
|
||||
if row.path in seen_paths.setdefault(row.model_id, set()):
|
||||
continue
|
||||
seen_paths[row.model_id].add(row.path)
|
||||
grouped.setdefault(row.model_id, []).append(_serialize_path(row))
|
||||
data = [
|
||||
{
|
||||
"id": grouped_model_id,
|
||||
"paths": grouped[grouped_model_id],
|
||||
}
|
||||
for grouped_model_id in sorted(grouped)
|
||||
]
|
||||
return {"data": data, "updated_at": updated_at or None}
|
||||
|
||||
|
||||
async def get_paths_for_model(model_id: str) -> dict:
|
||||
"""Return paths only for the exact model ID advertised by ``/v1/models``."""
|
||||
async with create_session() as session:
|
||||
rows = (
|
||||
await session.exec(
|
||||
select(ModelPathRow)
|
||||
.where(col(ModelPathRow.model_id) == model_id)
|
||||
.order_by(
|
||||
col(ModelPathRow.path),
|
||||
col(ModelPathRow.upstream_provider_id),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
|
||||
seen: set[str] = set()
|
||||
paths: list[dict] = []
|
||||
updated_at = 0
|
||||
for row in rows:
|
||||
updated_at = max(updated_at, row.updated_at)
|
||||
if row.path in seen:
|
||||
continue
|
||||
seen.add(row.path)
|
||||
paths.append(_serialize_path(row))
|
||||
return {"data": paths, "updated_at": updated_at or None}
|
||||
@@ -116,7 +116,7 @@ class TinfoilUpstreamProvider(BaseUpstreamProvider):
|
||||
EHBP-only header used for encrypted POST requests and is not honored
|
||||
for unencrypted GET requests.
|
||||
"""
|
||||
clean_path = path.removeprefix("tee/")
|
||||
clean_path = path.removeprefix("tee/").rstrip("/")
|
||||
if clean_path == "attestation":
|
||||
return await self._proxy_attestation(headers)
|
||||
return await super().forward_get_request(request, path, headers)
|
||||
|
||||
@@ -27,6 +27,16 @@ _READ_BUFSIZE = 65536
|
||||
_DEFAULT_TIMEOUT_SECONDS = 30.0
|
||||
_DEFAULT_CLOSE_TIMEOUT_SECONDS = 1.0
|
||||
_DEFAULT_MAX_RESPONSE_BYTES = 25 * 1024 * 1024
|
||||
_HOP_BY_HOP_HEADERS = {
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"proxy-authenticate",
|
||||
"proxy-authorization",
|
||||
"te",
|
||||
"trailer",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -47,6 +57,19 @@ def _get_header(headers: list[tuple[str, str]], name: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _strip_hop_by_hop_headers(headers: dict[str, str]) -> dict[str, str]:
|
||||
"""Remove connection-specific headers before serializing a new request."""
|
||||
connection_tokens: set[str] = set()
|
||||
for key, value in headers.items():
|
||||
if key.lower() == "connection":
|
||||
connection_tokens.update(
|
||||
token.strip().lower() for token in value.split(",") if token.strip()
|
||||
)
|
||||
|
||||
excluded = _HOP_BY_HOP_HEADERS | connection_tokens
|
||||
return {key: value for key, value in headers.items() if key.lower() not in excluded}
|
||||
|
||||
|
||||
async def forward_with_trailer(
|
||||
*,
|
||||
method: str,
|
||||
@@ -71,6 +94,11 @@ async def forward_with_trailer(
|
||||
if parsed.query:
|
||||
path = f"{path}?{parsed.query}"
|
||||
|
||||
# FastAPI has already decoded the incoming request body. Do not carry the
|
||||
# original connection's framing or other hop-by-hop metadata into the new
|
||||
# upstream connection.
|
||||
headers = _strip_hop_by_hop_headers(headers)
|
||||
|
||||
ssl_ctx = ssl.create_default_context()
|
||||
reader, writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(host, port, ssl=ssl_ctx),
|
||||
@@ -86,7 +114,7 @@ async def forward_with_trailer(
|
||||
header_lines.append("Connection: close")
|
||||
|
||||
for key, value in headers.items():
|
||||
if key.lower() in ("host", "connection"):
|
||||
if key.lower() == "host":
|
||||
continue
|
||||
header_lines.append(f"{key}: {value}")
|
||||
|
||||
|
||||
+168
-96
@@ -14,7 +14,7 @@ from pydantic_core import PydanticUndefined
|
||||
from sqlmodel import col, select, update
|
||||
|
||||
from .core import db, get_logger
|
||||
from .core.db import store_cashu_transaction
|
||||
from .core.db import store_cashu_transaction_with_retry as store_cashu_transaction
|
||||
from .core.settings import settings
|
||||
from .payment.lnurl import raw_send_to_lnurl
|
||||
|
||||
@@ -219,8 +219,12 @@ async def send(amount: int, unit: str, mint_url: str | None = None) -> tuple[int
|
||||
"""Internal send function - returns amount and serialized token"""
|
||||
effective_mint_url = mint_url or settings.primary_mint
|
||||
wallet: Wallet = await get_wallet(effective_mint_url, unit)
|
||||
proofs = get_proofs_per_mint_and_unit(wallet, effective_mint_url, unit)
|
||||
all_proofs = get_proofs_per_mint_and_unit(wallet, effective_mint_url, unit)
|
||||
proofs = [proof for proof in all_proofs if not proof.reserved]
|
||||
# Fallback must compare the requested amount with liquid proofs only. Counting
|
||||
# reserved proofs here can suppress fallback even though they cannot be sent.
|
||||
proofs_for_mint = sum(p.amount for p in proofs)
|
||||
reserved_for_mint = sum(p.amount for p in all_proofs if p.reserved)
|
||||
|
||||
# Fallback: proofs from untrusted source mints are swapped to primary_mint
|
||||
# during receive, so the user's preferred refund_mint_url may have no proofs
|
||||
@@ -232,8 +236,10 @@ async def send(amount: int, unit: str, mint_url: str | None = None) -> tuple[int
|
||||
)
|
||||
effective_mint_url = settings.primary_mint
|
||||
wallet = await get_wallet(effective_mint_url, unit)
|
||||
proofs = get_proofs_per_mint_and_unit(wallet, effective_mint_url, unit)
|
||||
all_proofs = get_proofs_per_mint_and_unit(wallet, effective_mint_url, unit)
|
||||
proofs = [proof for proof in all_proofs if not proof.reserved]
|
||||
proofs_for_mint = sum(p.amount for p in proofs)
|
||||
reserved_for_mint = sum(p.amount for p in all_proofs if p.reserved)
|
||||
|
||||
all_mint_urls = list({k.mint_url for k in wallet.keysets.values()})
|
||||
proof_summary = {
|
||||
@@ -247,7 +253,8 @@ async def send(amount: int, unit: str, mint_url: str | None = None) -> tuple[int
|
||||
raw_proofs_by_keyset[p.id] = raw_proofs_by_keyset.get(p.id, 0) + p.amount
|
||||
logger.info(
|
||||
f"send: proof inventory | mint={effective_mint_url} unit={unit} amount={amount} "
|
||||
f"primary_mint={settings.primary_mint} proofs_for_mint={proofs_for_mint} "
|
||||
f"primary_mint={settings.primary_mint} liquid_proofs_for_mint={proofs_for_mint} "
|
||||
f"reserved_proofs_for_mint={reserved_for_mint} "
|
||||
f"all_mints={all_mint_urls} by_keyset={proof_summary} "
|
||||
f"raw_proofs_by_keyset_id={raw_proofs_by_keyset} "
|
||||
f"total_wallet_proofs={sum(p.amount for p in wallet.proofs)}"
|
||||
@@ -745,11 +752,11 @@ async def credit_balance(
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.debug(
|
||||
"Cashu token successfully redeemed and stored",
|
||||
extra={"amount": amount, "unit": unit, "mint_url": mint_url},
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"Cashu token successfully redeemed and stored",
|
||||
extra={"amount": amount, "unit": unit, "mint_url": mint_url},
|
||||
)
|
||||
return amount
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
@@ -848,7 +855,7 @@ async def fetch_all_balances(
|
||||
"unit": unit,
|
||||
"wallet_balance": proofs_balance,
|
||||
"user_balance": user_balance,
|
||||
"owner_balance": proofs_balance - user_balance if proofs_balance != 0 else 0,
|
||||
"owner_balance": proofs_balance - user_balance,
|
||||
}
|
||||
return result
|
||||
except Exception as e:
|
||||
@@ -904,9 +911,7 @@ async def fetch_all_balances(
|
||||
total_wallet_balance_sats += proofs_balance_sats
|
||||
total_user_balance_sats += user_balance_sats
|
||||
|
||||
owner_balance = 0
|
||||
if total_wallet_balance_sats != 0:
|
||||
owner_balance = total_wallet_balance_sats - total_user_balance_sats
|
||||
owner_balance = total_wallet_balance_sats - total_user_balance_sats
|
||||
|
||||
return (
|
||||
balance_details,
|
||||
@@ -919,104 +924,132 @@ async def fetch_all_balances(
|
||||
async def periodic_payout() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(settings.payout_interval_seconds)
|
||||
print(settings.payout_interval_seconds)
|
||||
if not settings.receive_ln_address:
|
||||
continue
|
||||
try:
|
||||
if not settings.receive_ln_address:
|
||||
continue
|
||||
|
||||
# Include the primary mint even if it is not listed in cashu_mints,
|
||||
# matching fetch_all_balances(); otherwise primary-mint funds never
|
||||
# auto-payout.
|
||||
mint_urls: list[str] = list(settings.cashu_mints)
|
||||
if settings.primary_mint and settings.primary_mint not in mint_urls:
|
||||
mint_urls.append(settings.primary_mint)
|
||||
|
||||
async with db.create_session() as session:
|
||||
for mint_url in settings.cashu_mints:
|
||||
for mint_url in mint_urls:
|
||||
for unit in ["sat", "msat"]:
|
||||
wallet = await get_wallet(mint_url, unit)
|
||||
proofs = get_proofs_per_mint_and_unit(
|
||||
wallet, mint_url, unit, not_reserved=True
|
||||
)
|
||||
proofs = await slow_filter_spend_proofs(proofs, wallet)
|
||||
await asyncio.sleep(5)
|
||||
user_balance = await db.balances_for_mint_and_unit(
|
||||
session, mint_url, unit
|
||||
)
|
||||
if unit == "sat":
|
||||
user_balance = user_balance // 1000
|
||||
proofs_balance = sum(proof.amount for proof in proofs)
|
||||
available_balance = proofs_balance - user_balance
|
||||
# Threshold is configured in sats; convert for msat wallets.
|
||||
min_amount = (
|
||||
settings.min_payout_sat
|
||||
if unit == "sat"
|
||||
else settings.min_payout_sat * 1000
|
||||
)
|
||||
if available_balance > min_amount:
|
||||
amount_received = await raw_send_to_lnurl(
|
||||
wallet,
|
||||
proofs,
|
||||
settings.receive_ln_address,
|
||||
unit,
|
||||
amount=available_balance,
|
||||
# Isolate failures per mint/unit so one slow or failing
|
||||
# mint does not abort payout for every other mint/unit.
|
||||
try:
|
||||
wallet = await get_wallet(mint_url, unit)
|
||||
proofs = get_proofs_per_mint_and_unit(
|
||||
wallet, mint_url, unit, not_reserved=True
|
||||
)
|
||||
logger.info(
|
||||
"Payout sent successfully",
|
||||
proofs = await slow_filter_spend_proofs(proofs, wallet)
|
||||
await asyncio.sleep(5)
|
||||
user_balance = await db.balances_for_mint_and_unit(
|
||||
session, mint_url, unit
|
||||
)
|
||||
if unit == "sat":
|
||||
user_balance = user_balance // 1000
|
||||
proofs_balance = sum(proof.amount for proof in proofs)
|
||||
available_balance = proofs_balance - user_balance
|
||||
# Threshold is configured in sats; convert for msat wallets.
|
||||
min_amount = (
|
||||
settings.min_payout_sat
|
||||
if unit == "sat"
|
||||
else settings.min_payout_sat * 1000
|
||||
)
|
||||
if available_balance > min_amount:
|
||||
amount_received = await raw_send_to_lnurl(
|
||||
wallet,
|
||||
proofs,
|
||||
settings.receive_ln_address,
|
||||
unit,
|
||||
amount=available_balance,
|
||||
)
|
||||
logger.info(
|
||||
"Payout sent successfully",
|
||||
extra={
|
||||
"mint_url": mint_url,
|
||||
"unit": unit,
|
||||
"balance": available_balance,
|
||||
"amount_received": amount_received,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error sending payout: {type(e).__name__}",
|
||||
extra={
|
||||
"error": str(e),
|
||||
"mint_url": mint_url,
|
||||
"unit": unit,
|
||||
"balance": available_balance,
|
||||
"amount_received": amount_received,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Error sending payout: {type(e).__name__}",
|
||||
f"Error in periodic payout cycle: {type(e).__name__}",
|
||||
extra={"error": str(e)},
|
||||
)
|
||||
|
||||
|
||||
async def _refund_sweep_once(cutoff: int) -> None:
|
||||
async with db.create_session() as session:
|
||||
stmt = select(db.CashuTransaction).where(
|
||||
db.CashuTransaction.type == "out",
|
||||
db.CashuTransaction.collected == False, # noqa: E712
|
||||
db.CashuTransaction.swept == False, # noqa: E712
|
||||
db.CashuTransaction.created_at < cutoff,
|
||||
)
|
||||
results = await session.exec(stmt)
|
||||
refunds = results.all()
|
||||
|
||||
for refund in refunds:
|
||||
try:
|
||||
await recieve_token(refund.token)
|
||||
refund.swept = True
|
||||
session.add(refund)
|
||||
logger.info(
|
||||
"Swept uncollected refund",
|
||||
extra={
|
||||
"id": refund.id,
|
||||
"amount": refund.amount,
|
||||
"unit": refund.unit,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
error_msg = str(e).lower()
|
||||
if "already spent" in error_msg:
|
||||
refund.collected = True
|
||||
session.add(refund)
|
||||
logger.info(
|
||||
"Refund already spent (client collected), marking swept",
|
||||
extra={
|
||||
"id": refund.id,
|
||||
},
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Failed to sweep refund",
|
||||
extra={
|
||||
"id": refund.id,
|
||||
"error": str(e),
|
||||
},
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def refund_sweep_once() -> None:
|
||||
"""Sweep eligible uncollected refund tokens once."""
|
||||
cutoff = int(time.time()) - settings.refund_sweep_ttl_seconds
|
||||
await _refund_sweep_once(cutoff)
|
||||
|
||||
|
||||
async def periodic_refund_sweep() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(60 * 60) # every hour
|
||||
try:
|
||||
cutoff = int(time.time()) - settings.refund_sweep_ttl_seconds
|
||||
async with db.create_session() as session:
|
||||
stmt = select(db.CashuTransaction).where(
|
||||
db.CashuTransaction.type == "out",
|
||||
db.CashuTransaction.collected == False, # noqa: E712
|
||||
db.CashuTransaction.swept == False, # noqa: E712
|
||||
db.CashuTransaction.created_at < cutoff,
|
||||
)
|
||||
results = await session.exec(stmt)
|
||||
refunds = results.all()
|
||||
|
||||
for refund in refunds:
|
||||
try:
|
||||
await recieve_token(refund.token)
|
||||
refund.swept = True
|
||||
session.add(refund)
|
||||
logger.info(
|
||||
"Swept uncollected refund",
|
||||
extra={
|
||||
"id": refund.id,
|
||||
"amount": refund.amount,
|
||||
"unit": refund.unit,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
error_msg = str(e).lower()
|
||||
if "already spent" in error_msg:
|
||||
refund.collected = True
|
||||
session.add(refund)
|
||||
logger.info(
|
||||
"Refund already spent (client collected), marking swept",
|
||||
extra={
|
||||
"id": refund.id,
|
||||
},
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Failed to sweep refund",
|
||||
extra={
|
||||
"id": refund.id,
|
||||
"error": str(e),
|
||||
},
|
||||
)
|
||||
await session.commit()
|
||||
await refund_sweep_once()
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Error in periodic refund sweep",
|
||||
@@ -1039,17 +1072,56 @@ async def periodic_routstr_fee_payout() -> None:
|
||||
try:
|
||||
async with db.create_session() as session:
|
||||
fee = await db.get_routstr_fee(session)
|
||||
if fee.payout_in_progress_msats:
|
||||
logger.critical(
|
||||
"Routstr fee payout requires manual reconciliation",
|
||||
extra={
|
||||
"payout_in_progress_msats": fee.payout_in_progress_msats,
|
||||
"payout_started_at": fee.payout_started_at,
|
||||
},
|
||||
)
|
||||
continue
|
||||
|
||||
accumulated_sats = fee.accumulated_msats // 1000
|
||||
if accumulated_sats >= ROUTSTR_FEE_DEFAULT_PAYOUT:
|
||||
wallet = await get_wallet(settings.primary_mint, "sat")
|
||||
proofs = get_proofs_per_mint_and_unit(
|
||||
wallet, settings.primary_mint, "sat", not_reserved=True
|
||||
)
|
||||
amount_received = await raw_send_to_lnurl(
|
||||
wallet, proofs, ROUTSTR_LN_ADDRESS, "sat", amount=accumulated_sats
|
||||
)
|
||||
paid_msats = accumulated_sats * 1000
|
||||
await db.reset_routstr_fee(session, paid_msats)
|
||||
payout_checkpointed = await db.reset_routstr_fee(
|
||||
session, paid_msats
|
||||
)
|
||||
if not payout_checkpointed:
|
||||
logger.warning("Routstr fee payout was already claimed")
|
||||
continue
|
||||
|
||||
try:
|
||||
amount_received = await raw_send_to_lnurl(
|
||||
wallet,
|
||||
proofs,
|
||||
ROUTSTR_LN_ADDRESS,
|
||||
"sat",
|
||||
amount=accumulated_sats,
|
||||
)
|
||||
except Exception:
|
||||
logger.critical(
|
||||
"Routstr fee payout outcome is unknown; manual reconciliation required",
|
||||
extra={"payout_in_progress_msats": paid_msats},
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
|
||||
payout_completed = await db.complete_routstr_fee_payout(
|
||||
session, paid_msats
|
||||
)
|
||||
if not payout_completed:
|
||||
logger.critical(
|
||||
"Routstr fee payout sent but checkpoint was not completed",
|
||||
extra={"payout_in_progress_msats": paid_msats},
|
||||
)
|
||||
continue
|
||||
|
||||
logger.info(
|
||||
"Routstr fee payout sent",
|
||||
extra={
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Operational and recovery scripts for routstr-core (not a shipped package)."""
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Recover admin access by resetting the stored admin password (issue #553).
|
||||
|
||||
The lockout escape hatch for an operator who has lost the admin password. It
|
||||
talks to the ``secrets`` table directly and deliberately does *not* require
|
||||
``ROUTSTR_SECRET_KEY``: the admin password is scrypt-hashed (key-independent),
|
||||
so recovery works even when the encryption key is missing or has changed.
|
||||
|
||||
Two explicit, mutually exclusive actions — running with no arguments only prints
|
||||
help, so the password can't be reset by accident:
|
||||
|
||||
python scripts/reset_admin_password.py --password <new-password>
|
||||
Hash and store <new-password> now.
|
||||
|
||||
python scripts/reset_admin_password.py --regenerate
|
||||
Clear the stored hash; the next node startup generates a fresh random
|
||||
password and logs it once (with the /admin URL).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.db import create_session, get_secret, set_admin_password
|
||||
from routstr.core.vault import MIN_PASSWORD_LENGTH
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="reset_admin_password",
|
||||
description="Reset the node's admin password (recovery from lockout).",
|
||||
)
|
||||
action = parser.add_mutually_exclusive_group()
|
||||
action.add_argument(
|
||||
"--password",
|
||||
metavar="NEW_PASSWORD",
|
||||
help=f"set this as the new admin password (min {MIN_PASSWORD_LENGTH} chars)",
|
||||
)
|
||||
action.add_argument(
|
||||
"--regenerate",
|
||||
action="store_true",
|
||||
help="clear the password so the next startup generates and logs a new one",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
async def apply_reset(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
password: str | None = None,
|
||||
regenerate: bool = False,
|
||||
) -> str:
|
||||
"""Perform the requested reset against ``session``; return a status message."""
|
||||
if password is not None:
|
||||
if len(password) < MIN_PASSWORD_LENGTH:
|
||||
raise ValueError(
|
||||
f"New password must be at least {MIN_PASSWORD_LENGTH} characters"
|
||||
)
|
||||
await set_admin_password(session, password)
|
||||
return "Admin password updated."
|
||||
|
||||
if regenerate:
|
||||
secret = await get_secret(session)
|
||||
secret.admin_password_hash = None
|
||||
secret.updated_at = int(time.time())
|
||||
session.add(secret)
|
||||
await session.commit()
|
||||
return (
|
||||
"Admin password cleared. The next node startup will generate a new "
|
||||
"one and log it once with the /admin URL."
|
||||
)
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
async def _run(password: str | None, regenerate: bool) -> str:
|
||||
async with create_session() as session:
|
||||
return await apply_reset(
|
||||
session, password=password, regenerate=regenerate
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.password is None and not args.regenerate:
|
||||
parser.print_help()
|
||||
return 0
|
||||
|
||||
try:
|
||||
message = asyncio.run(_run(args.password, args.regenerate))
|
||||
except ValueError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
print(message)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Shared pytest configuration for the whole suite.
|
||||
|
||||
A fixed, valid ``ROUTSTR_SECRET_KEY`` is set before any app import so that
|
||||
secret encryption is deterministic across the suite and the mandatory-key
|
||||
fail-fast does not break app-boot tests. Tests that need a different key (or an
|
||||
absent one) override this per-test via ``monkeypatch``.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
# Valid Fernet keys; KEY_A is the suite default, KEY_B is for wrong-key tests.
|
||||
TEST_SECRET_KEY = "l_Tkp-7xmjcQ-IFhr6qhILrU8HPRbEmYMrfSbo_5srU="
|
||||
TEST_SECRET_KEY_ALT = "_Teyrky_iToeDK51Tj1FsI9MJ340_cqKGmeher-a7MQ="
|
||||
|
||||
os.environ.setdefault("ROUTSTR_SECRET_KEY", TEST_SECRET_KEY)
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Tests for admin password auth backed by the hashed Secret store (issue #553).
|
||||
|
||||
Login and password change verify against the one-way ``Secret.admin_password_hash``
|
||||
(scrypt) instead of a plaintext settings field, which also closes the old ``!=``
|
||||
timing-attack comparison. The first hash is written by ``bootstrap_secrets`` at
|
||||
startup (generated, or migrated from a legacy password); here it is seeded
|
||||
directly into the store, then login checks against it and a password change
|
||||
re-hashes so the old password stops working and the new one starts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from httpx import AsyncClient, Response
|
||||
|
||||
from routstr.core.db import create_session, set_admin_password
|
||||
|
||||
|
||||
async def _seed_password(password: str) -> None:
|
||||
# bootstrap_secrets owns first-password creation at startup; tests seed the
|
||||
# hash straight into the store the same way, then exercise login/change.
|
||||
async with create_session() as session:
|
||||
await set_admin_password(session, password)
|
||||
|
||||
|
||||
async def _login(client: AsyncClient, password: str) -> Response:
|
||||
return await client.post("/admin/api/login", json={"password": password})
|
||||
|
||||
|
||||
# --- login -----------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_login_500_when_no_password_configured(
|
||||
integration_client: AsyncClient,
|
||||
) -> None:
|
||||
resp = await _login(integration_client, "anything")
|
||||
assert resp.status_code == 500
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_login_succeeds_with_correct_password(
|
||||
integration_client: AsyncClient,
|
||||
) -> None:
|
||||
await _seed_password("correct horse")
|
||||
resp = await _login(integration_client, "correct horse")
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["ok"] is True
|
||||
assert isinstance(body["token"], str) and body["token"]
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_login_rejects_wrong_password(
|
||||
integration_client: AsyncClient,
|
||||
) -> None:
|
||||
await _seed_password("correct horse")
|
||||
resp = await _login(integration_client, "wrong horse")
|
||||
assert resp.status_code == 401
|
||||
|
||||
|
||||
# --- password change -------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_password_rehashes_so_only_new_works(
|
||||
integration_client: AsyncClient,
|
||||
) -> None:
|
||||
await _seed_password("old password")
|
||||
login = await _login(integration_client, "old password")
|
||||
token = login.json()["token"]
|
||||
integration_client.headers["Authorization"] = f"Bearer {token}"
|
||||
|
||||
resp = await integration_client.patch(
|
||||
"/admin/api/password",
|
||||
json={"current_password": "old password", "new_password": "new password"},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
# Drop admin auth so the login calls aren't treated as authenticated noise.
|
||||
integration_client.headers.pop("Authorization", None)
|
||||
assert (await _login(integration_client, "old password")).status_code == 401
|
||||
assert (await _login(integration_client, "new password")).status_code == 200
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_password_rejects_wrong_current(
|
||||
integration_client: AsyncClient,
|
||||
) -> None:
|
||||
await _seed_password("old password")
|
||||
login = await _login(integration_client, "old password")
|
||||
token = login.json()["token"]
|
||||
integration_client.headers["Authorization"] = f"Bearer {token}"
|
||||
|
||||
resp = await integration_client.patch(
|
||||
"/admin/api/password",
|
||||
json={"current_password": "not the password", "new_password": "new password"},
|
||||
)
|
||||
assert resp.status_code == 401
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_password_rejects_short_new(
|
||||
integration_client: AsyncClient,
|
||||
) -> None:
|
||||
await _seed_password("old password")
|
||||
login = await _login(integration_client, "old password")
|
||||
token = login.json()["token"]
|
||||
integration_client.headers["Authorization"] = f"Bearer {token}"
|
||||
|
||||
resp = await integration_client.patch(
|
||||
"/admin/api/password",
|
||||
json={"current_password": "old password", "new_password": "x"},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Tests for the admin nsec rotation endpoint (issue #553).
|
||||
|
||||
The Nostr identity is a secret: it lives encrypted in the Secret store, never in
|
||||
the settings blob, so it cannot be set through the general settings PATCH. This
|
||||
dedicated endpoint is the supported way to set/rotate/clear it — it encrypts the
|
||||
key at rest, updates the live runtime identity (so signing picks it up without a
|
||||
restart), and derives the npub. Invalid keys are rejected.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from httpx import AsyncClient
|
||||
|
||||
from routstr.core import vault
|
||||
from routstr.core.admin import admin_sessions
|
||||
from routstr.core.db import AsyncSession, get_secret
|
||||
from routstr.core.settings import derive_npub_from_nsec, settings
|
||||
|
||||
# A valid 64-char hex private key (accepted by nsec_to_keypair, as in bootstrap).
|
||||
NSEC_HEX = "1" * 64
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def admin_client(
|
||||
integration_client: AsyncClient,
|
||||
) -> AsyncGenerator[AsyncClient, None]:
|
||||
"""An integration_client pre-authenticated with an admin session token."""
|
||||
token = secrets.token_urlsafe(24)
|
||||
admin_sessions[token] = int(time.time()) + 3600
|
||||
integration_client.headers["Authorization"] = f"Bearer {token}"
|
||||
yield integration_client
|
||||
admin_sessions.pop(token, None)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_nsec_stores_encrypted_and_derives_npub(
|
||||
admin_client: AsyncClient,
|
||||
integration_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(settings, "nsec", "")
|
||||
monkeypatch.setattr(settings, "npub", "")
|
||||
|
||||
resp = await admin_client.patch("/admin/api/nsec", json={"nsec": NSEC_HEX})
|
||||
assert resp.status_code == 200
|
||||
|
||||
expected_npub = derive_npub_from_nsec(NSEC_HEX)
|
||||
assert resp.json() == {"ok": True, "npub": expected_npub}
|
||||
|
||||
# Stored encrypted at rest, decryptable back to the original key.
|
||||
integration_session.expunge_all()
|
||||
secret = await get_secret(integration_session)
|
||||
assert secret.encrypted_nsec is not None
|
||||
assert vault.is_encrypted(secret.encrypted_nsec)
|
||||
assert vault.decrypt(secret.encrypted_nsec) == NSEC_HEX
|
||||
|
||||
# Live runtime identity updated so Nostr signing reflects it without restart.
|
||||
assert settings.nsec == NSEC_HEX
|
||||
assert settings.npub == expected_npub
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_nsec_rejects_invalid_key(
|
||||
admin_client: AsyncClient,
|
||||
integration_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(settings, "nsec", "")
|
||||
monkeypatch.setattr(settings, "npub", "")
|
||||
|
||||
resp = await admin_client.patch(
|
||||
"/admin/api/nsec", json={"nsec": "not-a-real-nsec"}
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
# Nothing stored, live identity untouched.
|
||||
integration_session.expunge_all()
|
||||
secret = await get_secret(integration_session)
|
||||
assert secret.encrypted_nsec is None
|
||||
assert settings.nsec == ""
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_nsec_clears_identity_with_empty_value(
|
||||
admin_client: AsyncClient,
|
||||
integration_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# Start from a node that has an identity...
|
||||
monkeypatch.setattr(settings, "nsec", "")
|
||||
monkeypatch.setattr(settings, "npub", "")
|
||||
set_resp = await admin_client.patch("/admin/api/nsec", json={"nsec": NSEC_HEX})
|
||||
assert set_resp.status_code == 200
|
||||
|
||||
# ...then clear it.
|
||||
clear_resp = await admin_client.patch("/admin/api/nsec", json={"nsec": ""})
|
||||
assert clear_resp.status_code == 200
|
||||
assert clear_resp.json() == {"ok": True, "npub": ""}
|
||||
|
||||
integration_session.expunge_all()
|
||||
secret = await get_secret(integration_session)
|
||||
assert secret.encrypted_nsec is None
|
||||
assert settings.nsec == ""
|
||||
assert settings.npub == ""
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Tests for the admin settings endpoint's handling of secrets (issue #553).
|
||||
|
||||
``admin_password`` is no longer a settings field (it lives only as a one-way
|
||||
hash in the Secret store), so it must never appear in the GET/PATCH payloads.
|
||||
``nsec`` (in-memory at runtime) and ``upstream_api_key`` (still in the settings
|
||||
blob) are both redacted on read and ignored on write — they cannot be set
|
||||
through the general settings endpoint, only through their dedicated paths.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
import time
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from httpx import AsyncClient
|
||||
|
||||
from routstr.core.admin import admin_sessions
|
||||
from routstr.core.db import AsyncSession
|
||||
from routstr.core.settings import SettingsService, settings
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def admin_client(
|
||||
integration_client: AsyncClient,
|
||||
) -> AsyncGenerator[AsyncClient, None]:
|
||||
"""An integration_client pre-authenticated with an admin session token."""
|
||||
token = secrets.token_urlsafe(24)
|
||||
admin_sessions[token] = int(time.time()) + 3600
|
||||
integration_client.headers["Authorization"] = f"Bearer {token}"
|
||||
yield integration_client
|
||||
admin_sessions.pop(token, None)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_settings_omits_admin_password_and_redacts_secrets(
|
||||
admin_client: AsyncClient, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(settings, "nsec", "nsec-secret")
|
||||
monkeypatch.setattr(settings, "upstream_api_key", "sk-secret")
|
||||
|
||||
resp = await admin_client.get("/admin/api/settings")
|
||||
assert resp.status_code == 200
|
||||
|
||||
data = resp.json()
|
||||
assert "admin_password" not in data
|
||||
assert data["nsec"] == "[REDACTED]"
|
||||
assert data["upstream_api_key"] == "[REDACTED]"
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_patch_settings_ignores_secret_fields(
|
||||
admin_client: AsyncClient,
|
||||
integration_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# The PATCH path persists through SettingsService, which needs an
|
||||
# initialized current snapshot and a settings row in the shared test DB.
|
||||
await SettingsService.initialize(integration_session)
|
||||
monkeypatch.setattr(settings, "nsec", "original-nsec")
|
||||
|
||||
resp = await admin_client.patch(
|
||||
"/admin/api/settings",
|
||||
json={
|
||||
"name": "Renamed",
|
||||
"nsec": "attacker-nsec",
|
||||
"upstream_api_key": "attacker-key",
|
||||
"admin_password": "attacker-pw",
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
data = resp.json()
|
||||
assert data["name"] == "Renamed"
|
||||
assert "admin_password" not in data
|
||||
assert data["nsec"] == "[REDACTED]"
|
||||
# The live secret was not overwritten through the general settings endpoint.
|
||||
assert settings.nsec == "original-nsec"
|
||||
@@ -15,6 +15,7 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.auth import ReservationSnapshot
|
||||
from routstr.core.db import ApiKey
|
||||
from routstr.payment.cost_calculation import CostData
|
||||
|
||||
@@ -23,7 +24,7 @@ def _make_key(balance: int, reserved: int) -> ApiKey:
|
||||
return ApiKey(
|
||||
hashed_key=f"test_{uuid.uuid4().hex}",
|
||||
balance=balance,
|
||||
reserved_balance=reserved,
|
||||
reserved_balance=0,
|
||||
total_spent=0,
|
||||
total_requests=1,
|
||||
)
|
||||
@@ -75,6 +76,8 @@ async def test_balance_never_negative_when_cost_exceeds_reservation(
|
||||
key = _make_key(balance=deducted_max_cost, reserved=deducted_max_cost)
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
from routstr.auth import pay_for_request
|
||||
await pay_for_request(key, deducted_max_cost, integration_session)
|
||||
|
||||
response_data = {"model": "test-model", "usage": {"prompt_tokens": 100, "completion_tokens": 100}}
|
||||
|
||||
@@ -82,7 +85,9 @@ async def test_balance_never_negative_when_cost_exceeds_reservation(
|
||||
"routstr.auth.calculate_cost",
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await adjust_payment_for_tokens(key, response_data, integration_session, deducted_max_cost)
|
||||
await adjust_payment_for_tokens(
|
||||
key, response_data, integration_session, deducted_max_cost, None, None
|
||||
)
|
||||
|
||||
await _refresh(integration_session, key)
|
||||
|
||||
@@ -109,6 +114,8 @@ async def test_balance_floor_at_zero_on_overrun(
|
||||
key = _make_key(balance=500, reserved=500)
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
from routstr.auth import pay_for_request
|
||||
await pay_for_request(key, deducted_max_cost, integration_session)
|
||||
|
||||
response_data = {"model": "test-model", "usage": {"prompt_tokens": 50, "completion_tokens": 50}}
|
||||
|
||||
@@ -116,7 +123,9 @@ async def test_balance_floor_at_zero_on_overrun(
|
||||
"routstr.auth.calculate_cost",
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await adjust_payment_for_tokens(key, response_data, integration_session, deducted_max_cost)
|
||||
await adjust_payment_for_tokens(
|
||||
key, response_data, integration_session, deducted_max_cost, None, None
|
||||
)
|
||||
|
||||
await _refresh(integration_session, key)
|
||||
|
||||
@@ -148,6 +157,8 @@ async def test_full_cost_charged_when_balance_sufficient_for_overrun(
|
||||
key = _make_key(balance=2000, reserved=990)
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
from routstr.auth import pay_for_request
|
||||
await pay_for_request(key, deducted_max_cost, integration_session)
|
||||
|
||||
response_data = {"model": "test-model", "usage": {"prompt_tokens": 100, "completion_tokens": 100}}
|
||||
|
||||
@@ -155,7 +166,9 @@ async def test_full_cost_charged_when_balance_sufficient_for_overrun(
|
||||
"routstr.auth.calculate_cost",
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await adjust_payment_for_tokens(key, response_data, integration_session, deducted_max_cost)
|
||||
await adjust_payment_for_tokens(
|
||||
key, response_data, integration_session, deducted_max_cost, None, None
|
||||
)
|
||||
|
||||
await _refresh(integration_session, key)
|
||||
|
||||
@@ -184,7 +197,11 @@ async def test_concurrent_cost_overruns_never_negative(
|
||||
"""Concurrent finalization with cost overruns must never produce negative balance."""
|
||||
import asyncio
|
||||
|
||||
from routstr.auth import adjust_payment_for_tokens, pay_for_request
|
||||
from routstr.auth import (
|
||||
adjust_payment_for_tokens,
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
)
|
||||
from routstr.core.db import create_session
|
||||
|
||||
deducted_max_cost = 990
|
||||
@@ -210,12 +227,14 @@ async def test_concurrent_cost_overruns_never_negative(
|
||||
async with create_session() as session:
|
||||
key_to_reserve = await session.get(ApiKey, key_hash)
|
||||
assert key_to_reserve is not None
|
||||
reservations = []
|
||||
for _ in range(n_requests):
|
||||
await pay_for_request(key_to_reserve, deducted_max_cost, session)
|
||||
reservations.append(await get_reservation_snapshot(key_to_reserve, session))
|
||||
await session.refresh(key_to_reserve)
|
||||
|
||||
# Now finalize all concurrently with cost overrun
|
||||
async def finalize() -> None:
|
||||
async def finalize(reservation: ReservationSnapshot) -> None:
|
||||
response_data = {
|
||||
"model": "test-model",
|
||||
"usage": {"prompt_tokens": 100, "completion_tokens": 100},
|
||||
@@ -223,15 +242,22 @@ async def test_concurrent_cost_overruns_never_negative(
|
||||
async with create_session() as session:
|
||||
fresh_key = await session.get(ApiKey, key_hash)
|
||||
assert fresh_key is not None
|
||||
with patch(
|
||||
"routstr.auth.calculate_cost",
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await adjust_payment_for_tokens(
|
||||
fresh_key, response_data, session, deducted_max_cost
|
||||
)
|
||||
await adjust_payment_for_tokens(
|
||||
fresh_key,
|
||||
response_data,
|
||||
session,
|
||||
deducted_max_cost,
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
await asyncio.gather(*[finalize() for _ in range(n_requests)])
|
||||
# Patch once around the gather: entering the same patch target from
|
||||
# concurrent tasks un-patches in the wrong order and leaks the mock into
|
||||
# every later test in the session.
|
||||
with patch(
|
||||
"routstr.auth.calculate_cost",
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await asyncio.gather(*(finalize(r) for r in reservations))
|
||||
|
||||
async with create_session() as session:
|
||||
final_key = await session.get(ApiKey, key_hash)
|
||||
@@ -272,6 +298,8 @@ async def test_zero_free_balance_overrun_is_safe(
|
||||
key = _make_key(balance=1000, reserved=1000)
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
from routstr.auth import pay_for_request
|
||||
await pay_for_request(key, deducted_max_cost, integration_session)
|
||||
|
||||
response_data = {"model": "test-model", "usage": {"prompt_tokens": 50, "completion_tokens": 100}}
|
||||
|
||||
@@ -279,7 +307,9 @@ async def test_zero_free_balance_overrun_is_safe(
|
||||
"routstr.auth.calculate_cost",
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await adjust_payment_for_tokens(key, response_data, integration_session, deducted_max_cost)
|
||||
await adjust_payment_for_tokens(
|
||||
key, response_data, integration_session, deducted_max_cost, None, None
|
||||
)
|
||||
|
||||
await _refresh(integration_session, key)
|
||||
|
||||
@@ -308,7 +338,11 @@ async def test_parallel_requests_no_free_inference(
|
||||
"""Second parallel finalization must be charged even when first depleted free balance."""
|
||||
import asyncio
|
||||
|
||||
from routstr.auth import adjust_payment_for_tokens
|
||||
from routstr.auth import (
|
||||
adjust_payment_for_tokens,
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
)
|
||||
from routstr.core.db import create_session
|
||||
|
||||
deducted_max_cost = 100
|
||||
@@ -329,14 +363,18 @@ async def test_parallel_requests_no_free_inference(
|
||||
key = ApiKey(
|
||||
hashed_key=key_hash,
|
||||
balance=starting_balance,
|
||||
reserved_balance=deducted_max_cost * 2, # both slots pre-reserved
|
||||
reserved_balance=0,
|
||||
total_spent=0,
|
||||
total_requests=2,
|
||||
)
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
reservations = []
|
||||
for _ in range(2):
|
||||
await pay_for_request(key, deducted_max_cost, session)
|
||||
reservations.append(await get_reservation_snapshot(key, session))
|
||||
|
||||
async def finalize() -> None:
|
||||
async def finalize(reservation: ReservationSnapshot) -> None:
|
||||
response_data = {
|
||||
"model": "test-model",
|
||||
"usage": {"prompt_tokens": 50, "completion_tokens": 100},
|
||||
@@ -344,15 +382,22 @@ async def test_parallel_requests_no_free_inference(
|
||||
async with create_session() as session:
|
||||
fresh_key = await session.get(ApiKey, key_hash)
|
||||
assert fresh_key is not None
|
||||
with patch(
|
||||
"routstr.auth.calculate_cost",
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await adjust_payment_for_tokens(
|
||||
fresh_key, response_data, session, deducted_max_cost
|
||||
)
|
||||
await adjust_payment_for_tokens(
|
||||
fresh_key,
|
||||
response_data,
|
||||
session,
|
||||
deducted_max_cost,
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
await asyncio.gather(finalize(), finalize())
|
||||
# Patch once around the gather: entering the same patch target from two
|
||||
# concurrent tasks un-patches in the wrong order and leaks the mock into
|
||||
# every later test in the session.
|
||||
with patch(
|
||||
"routstr.auth.calculate_cost",
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await asyncio.gather(*(finalize(r) for r in reservations))
|
||||
|
||||
async with create_session() as session:
|
||||
final_key = await session.get(ApiKey, key_hash)
|
||||
|
||||
@@ -77,7 +77,7 @@ async def test_child_key_flow(integration_session: AsyncSession) -> None:
|
||||
|
||||
try:
|
||||
adjustment = await adjust_payment_for_tokens(
|
||||
child_key_db, response_data, integration_session, 500
|
||||
child_key_db, response_data, integration_session, 500, None, None
|
||||
)
|
||||
assert adjustment["total_msats"] == 400
|
||||
|
||||
|
||||
@@ -0,0 +1,666 @@
|
||||
"""Failover requests are billed and forwarded as the provider that served them.
|
||||
|
||||
Covers the whole-system settlement path when two enabled providers expose the
|
||||
same model under different spellings and prices: the routing winner fails with
|
||||
a 502, the fallback provider serves, and the response must be billed at the
|
||||
fallback's configured rate, carry the fallback's model id in the forwarded
|
||||
request body, and echo the fallback's model id to the client.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any, AsyncGenerator
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from httpx import AsyncClient
|
||||
from sqlmodel import select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.db import ApiKey, ReservationRelease
|
||||
from routstr.payment.models import Architecture, Model, Pricing
|
||||
from routstr.proxy import refresh_model_maps
|
||||
from routstr.upstream.base import BaseUpstreamProvider
|
||||
|
||||
CHEAP_BASE_URL = "https://cheap.example.com/v1"
|
||||
EXPENSIVE_BASE_URL = "https://expensive.example.com/v1"
|
||||
THIRD_BASE_URL = "https://third.example.com/v1"
|
||||
|
||||
|
||||
def _make_model(
|
||||
model_id: str,
|
||||
prompt_sats: float,
|
||||
completion_sats: float,
|
||||
max_cost: float = 50.0,
|
||||
) -> Model:
|
||||
"""Build a model whose USD and sats pricing rank consistently."""
|
||||
return Model(
|
||||
id=model_id,
|
||||
name=model_id,
|
||||
created=1,
|
||||
description="test model",
|
||||
context_length=8192,
|
||||
architecture=Architecture(
|
||||
modality="text",
|
||||
input_modalities=["text"],
|
||||
output_modalities=["text"],
|
||||
tokenizer="gpt",
|
||||
instruct_type=None,
|
||||
),
|
||||
pricing=Pricing(
|
||||
prompt=prompt_sats, completion=completion_sats, max_cost=max_cost
|
||||
),
|
||||
sats_pricing=Pricing(
|
||||
prompt=prompt_sats, completion=completion_sats, max_cost=max_cost
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class _StaticProvider(BaseUpstreamProvider):
|
||||
"""Upstream provider with a fixed model catalog and no remote refresh."""
|
||||
|
||||
def __init__(self, base_url: str, api_key: str, fee: float, model: Model) -> None:
|
||||
super().__init__(base_url, api_key, fee)
|
||||
self.provider_type = "custom"
|
||||
self._static_model = model
|
||||
|
||||
def get_cached_models(self) -> list[Model]:
|
||||
return [self._static_model]
|
||||
|
||||
async def refresh_models_cache(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
async def _install_providers(
|
||||
providers: list[_StaticProvider],
|
||||
) -> AsyncGenerator[None, None]:
|
||||
"""Install providers into the routing maps, restoring the originals after."""
|
||||
from routstr import proxy
|
||||
|
||||
original_upstreams = proxy.get_upstreams()
|
||||
with patch("routstr.proxy._upstreams", providers):
|
||||
await refresh_model_maps()
|
||||
yield
|
||||
with patch("routstr.proxy._upstreams", original_upstreams):
|
||||
await refresh_model_maps()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def dual_provider_maps(
|
||||
patched_db_engine: None,
|
||||
) -> AsyncGenerator[tuple[_StaticProvider, _StaticProvider], None]:
|
||||
"""Two same-tail providers under different spellings and prices."""
|
||||
cheap = _StaticProvider(
|
||||
CHEAP_BASE_URL,
|
||||
"key-cheap",
|
||||
1.0,
|
||||
_make_model("prova/dual-model", 0.001, 0.002),
|
||||
)
|
||||
expensive = _StaticProvider(
|
||||
EXPENSIVE_BASE_URL,
|
||||
"key-expensive",
|
||||
1.0,
|
||||
_make_model("provb/dual-model", 0.005, 0.010, max_cost=100.0),
|
||||
)
|
||||
async for _ in _install_providers([cheap, expensive]):
|
||||
yield cheap, expensive
|
||||
|
||||
|
||||
def _upstream_response(request: httpx.Request) -> httpx.Response:
|
||||
"""502 from the cheap (winning) provider; a served completion elsewhere."""
|
||||
if request.url.host == "cheap.example.com":
|
||||
return httpx.Response(
|
||||
502,
|
||||
content=json.dumps({"error": {"message": "bad gateway"}}).encode(),
|
||||
headers={"content-type": "application/json"},
|
||||
)
|
||||
body = {
|
||||
"id": "chatcmpl-served",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "dual-model",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "hi"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 1000,
|
||||
"completion_tokens": 500,
|
||||
"total_tokens": 1500,
|
||||
},
|
||||
}
|
||||
return httpx.Response(
|
||||
200,
|
||||
content=json.dumps(body).encode(),
|
||||
headers={"content-type": "application/json"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_failover_serve_billed_at_serving_providers_rate(
|
||||
authenticated_client: AsyncClient,
|
||||
dual_provider_maps: tuple[_StaticProvider, _StaticProvider],
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""A fallback serve is billed at the fallback's price, not the winner's.
|
||||
|
||||
The cheap provider ranks first for the shared tail; it 502s and the
|
||||
expensive provider serves 1000 input + 500 output tokens. At the serving
|
||||
provider's sats pricing (0.005/0.010 sats per token) that is 10_000 msats;
|
||||
at the winner's (0.001/0.002) it would be 2_000 msats.
|
||||
"""
|
||||
sent_requests: list[httpx.Request] = []
|
||||
|
||||
# Patch the network transport (not AsyncClient.send) so the in-process
|
||||
# ASGI test client is untouched and only the proxy's upstream hop is mocked.
|
||||
async def fake_transport(
|
||||
request: httpx.Request, *args: Any, **kwargs: Any
|
||||
) -> httpx.Response:
|
||||
sent_requests.append(request)
|
||||
return _upstream_response(request)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"httpx.AsyncHTTPTransport.handle_async_request",
|
||||
side_effect=fake_transport,
|
||||
),
|
||||
# cost_calculation binds sats_usd_price at import time, so the price
|
||||
# patch in the app fixture does not reach it; patch its own binding.
|
||||
patch(
|
||||
"routstr.payment.cost_calculation.sats_usd_price",
|
||||
return_value=0.0005,
|
||||
),
|
||||
):
|
||||
response = await authenticated_client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "dual-model",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
|
||||
# Both providers were attempted, cheapest first.
|
||||
assert [r.url.host for r in sent_requests] == [
|
||||
"cheap.example.com",
|
||||
"expensive.example.com",
|
||||
]
|
||||
|
||||
# The fallback must be asked for ITS OWN model spelling, not the winner's.
|
||||
forwarded_body = json.loads(sent_requests[1].content)
|
||||
assert forwarded_body["model"] == "provb/dual-model"
|
||||
|
||||
# The response echo names the model that actually served.
|
||||
assert payload["model"] == "provb/dual-model"
|
||||
|
||||
# Billed at the serving provider's rate: 1000/1000*5000 + 500/1000*10000.
|
||||
assert payload["cost"]["total_msats"] == 10_000
|
||||
|
||||
# The fallback's larger max-cost envelope requires a replacement
|
||||
# reservation. The failed candidate is released, the serving candidate is
|
||||
# charged, and no request-owned reservation remains active.
|
||||
key_hash = authenticated_client._test_api_key.removeprefix("sk-") # type: ignore[attr-defined]
|
||||
records = (
|
||||
await integration_session.exec(
|
||||
select(ReservationRelease).where(ReservationRelease.key_hash == key_hash)
|
||||
)
|
||||
).all()
|
||||
assert sorted(record.status for record in records) == ["charged", "released"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def same_id_provider_maps(
|
||||
patched_db_engine: None,
|
||||
) -> AsyncGenerator[None, None]:
|
||||
"""Two providers exposing the IDENTICAL model id at different prices."""
|
||||
cheap = _StaticProvider(
|
||||
CHEAP_BASE_URL,
|
||||
"key-cheap",
|
||||
1.0,
|
||||
_make_model("dual-model", 0.001, 0.002),
|
||||
)
|
||||
expensive = _StaticProvider(
|
||||
EXPENSIVE_BASE_URL,
|
||||
"key-expensive",
|
||||
1.0,
|
||||
_make_model("dual-model", 0.005, 0.010),
|
||||
)
|
||||
async for _ in _install_providers([cheap, expensive]):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_id_failover_settles_at_serving_price(
|
||||
authenticated_client: AsyncClient,
|
||||
same_id_provider_maps: None,
|
||||
) -> None:
|
||||
"""Settlement must not re-derive pricing from the response's model string.
|
||||
|
||||
Both providers expose the exact same model id, so the forwarded body is
|
||||
identical either way — the only observable difference is the settled
|
||||
amount. The response's model string resolves to the alias winner (cheap),
|
||||
but the expensive provider served, so the bill must be 10_000 msats, not
|
||||
the winner's 2_000.
|
||||
"""
|
||||
sent_requests: list[httpx.Request] = []
|
||||
|
||||
async def fake_transport(
|
||||
request: httpx.Request, *args: Any, **kwargs: Any
|
||||
) -> httpx.Response:
|
||||
sent_requests.append(request)
|
||||
return _upstream_response(request)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"httpx.AsyncHTTPTransport.handle_async_request",
|
||||
side_effect=fake_transport,
|
||||
),
|
||||
patch(
|
||||
"routstr.payment.cost_calculation.sats_usd_price",
|
||||
return_value=0.0005,
|
||||
),
|
||||
):
|
||||
response = await authenticated_client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "dual-model",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert [r.url.host for r in sent_requests] == [
|
||||
"cheap.example.com",
|
||||
"expensive.example.com",
|
||||
]
|
||||
assert response.json()["cost"]["total_msats"] == 10_000
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_version_suffixed_model_id_routes(
|
||||
authenticated_client: AsyncClient,
|
||||
same_id_provider_maps: None,
|
||||
) -> None:
|
||||
"""A version-suffixed request (``…-YYYYMMDD``) routes to the base model.
|
||||
|
||||
Model resolution stripped the suffix but the provider lookup did not, so
|
||||
such requests resolved a model yet found no provider and 400'd. With the
|
||||
unified candidate lookup the strip applies to both.
|
||||
"""
|
||||
|
||||
async def fake_transport(
|
||||
request: httpx.Request, *args: Any, **kwargs: Any
|
||||
) -> httpx.Response:
|
||||
return _upstream_response(request)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"httpx.AsyncHTTPTransport.handle_async_request",
|
||||
side_effect=fake_transport,
|
||||
),
|
||||
patch(
|
||||
"routstr.payment.cost_calculation.sats_usd_price",
|
||||
return_value=0.0005,
|
||||
),
|
||||
):
|
||||
response = await authenticated_client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "dual-model-20260101",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def fee_split_provider_maps(
|
||||
patched_db_engine: None,
|
||||
) -> AsyncGenerator[None, None]:
|
||||
"""Same-tail providers whose fees differ; the serving one charges 1.5x."""
|
||||
cheap = _StaticProvider(
|
||||
CHEAP_BASE_URL,
|
||||
"key-cheap",
|
||||
1.0,
|
||||
_make_model("dual-model", 0.001, 0.002),
|
||||
)
|
||||
expensive = _StaticProvider(
|
||||
EXPENSIVE_BASE_URL,
|
||||
"key-expensive",
|
||||
1.5,
|
||||
_make_model("dual-model", 0.005, 0.010),
|
||||
)
|
||||
async for _ in _install_providers([cheap, expensive]):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_usd_cost_serve_carries_serving_providers_fee(
|
||||
authenticated_client: AsyncClient,
|
||||
fee_split_provider_maps: None,
|
||||
) -> None:
|
||||
"""The USD-cost billing path applies the SERVING provider's fee.
|
||||
|
||||
The upstream that serves reports ``usage.cost`` in USD, so billing goes
|
||||
through the USD-cost path where the provider fee is applied explicitly.
|
||||
The serving provider's fee is 1.5; the alias winner's is 1.0. At 0.001 USD
|
||||
reported cost and 0.0005 USD/sat: 0.001 * 1.5 / 0.0005 = 3 sats = 3000
|
||||
msats (fee 1.0 would give 2000).
|
||||
"""
|
||||
sent_requests: list[httpx.Request] = []
|
||||
|
||||
def usd_cost_response(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.host == "cheap.example.com":
|
||||
return httpx.Response(
|
||||
502,
|
||||
content=json.dumps({"error": {"message": "bad gateway"}}).encode(),
|
||||
headers={"content-type": "application/json"},
|
||||
)
|
||||
body = {
|
||||
"id": "chatcmpl-usd",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "dual-model",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "hi"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 50,
|
||||
"total_tokens": 150,
|
||||
"cost": 0.001,
|
||||
},
|
||||
}
|
||||
return httpx.Response(
|
||||
200,
|
||||
content=json.dumps(body).encode(),
|
||||
headers={"content-type": "application/json"},
|
||||
)
|
||||
|
||||
async def fake_transport(
|
||||
request: httpx.Request, *args: Any, **kwargs: Any
|
||||
) -> httpx.Response:
|
||||
sent_requests.append(request)
|
||||
return usd_cost_response(request)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"httpx.AsyncHTTPTransport.handle_async_request",
|
||||
side_effect=fake_transport,
|
||||
),
|
||||
patch(
|
||||
"routstr.payment.cost_calculation.sats_usd_price",
|
||||
return_value=0.0005,
|
||||
),
|
||||
):
|
||||
response = await authenticated_client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "dual-model",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert [r.url.host for r in sent_requests] == [
|
||||
"cheap.example.com",
|
||||
"expensive.example.com",
|
||||
]
|
||||
assert response.json()["cost"]["total_msats"] == 3_000
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def envelope_split_provider_maps(
|
||||
patched_db_engine: None,
|
||||
) -> AsyncGenerator[None, None]:
|
||||
"""Same-id providers where the fallback's max cost dwarfs the key balance."""
|
||||
cheap = _StaticProvider(
|
||||
CHEAP_BASE_URL,
|
||||
"key-cheap",
|
||||
1.0,
|
||||
_make_model("dual-model", 0.001, 0.002, max_cost=50.0),
|
||||
)
|
||||
expensive = _StaticProvider(
|
||||
EXPENSIVE_BASE_URL,
|
||||
"key-expensive",
|
||||
1.0,
|
||||
_make_model("dual-model", 0.005, 0.010, max_cost=20_000.0),
|
||||
)
|
||||
async for _ in _install_providers([cheap, expensive]):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_failover_beyond_balance_envelope_is_rejected(
|
||||
authenticated_client: AsyncClient,
|
||||
envelope_split_provider_maps: None,
|
||||
) -> None:
|
||||
"""A fallback whose max-cost envelope exceeds the balance is not served.
|
||||
|
||||
Admission and reservation are sized to the best-ranked candidate's max
|
||||
cost. When that candidate fails and the next one's envelope exceeds the
|
||||
key's balance, serving it could settle far beyond what admission allowed,
|
||||
so the request must be rejected (as it would be if the pricier candidate
|
||||
were ranked first) instead of forwarded.
|
||||
"""
|
||||
sent_requests: list[httpx.Request] = []
|
||||
|
||||
async def fake_transport(
|
||||
request: httpx.Request, *args: Any, **kwargs: Any
|
||||
) -> httpx.Response:
|
||||
sent_requests.append(request)
|
||||
return _upstream_response(request)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"httpx.AsyncHTTPTransport.handle_async_request",
|
||||
side_effect=fake_transport,
|
||||
),
|
||||
patch(
|
||||
"routstr.payment.cost_calculation.sats_usd_price",
|
||||
return_value=0.0005,
|
||||
),
|
||||
):
|
||||
response = await authenticated_client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "dual-model",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
# The 20_000-sat envelope exceeds the key's 10_000-sat balance: the
|
||||
# fallback must be rejected before its upstream is ever contacted.
|
||||
assert response.status_code == 402
|
||||
assert [r.url.host for r in sent_requests] == ["cheap.example.com"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def three_candidate_child_maps(
|
||||
patched_db_engine: None,
|
||||
) -> AsyncGenerator[None, None]:
|
||||
"""Second candidate cannot fit the child limit; third restores and serves."""
|
||||
first = _StaticProvider(
|
||||
CHEAP_BASE_URL,
|
||||
"key-first",
|
||||
1.0,
|
||||
_make_model("dual-model", 0.001, 0.002, max_cost=50.0),
|
||||
)
|
||||
too_large = _StaticProvider(
|
||||
EXPENSIVE_BASE_URL,
|
||||
"key-too-large",
|
||||
1.0,
|
||||
_make_model("dual-model", 0.002, 0.003, max_cost=100.0),
|
||||
)
|
||||
third = _StaticProvider(
|
||||
THIRD_BASE_URL,
|
||||
"key-third",
|
||||
1.0,
|
||||
_make_model("dual-model", 0.003, 0.004, max_cost=50.0),
|
||||
)
|
||||
async for _ in _install_providers([first, too_large, third]):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_child_failover_rolls_back_failed_larger_reserve_before_restoring(
|
||||
authenticated_client: AsyncClient,
|
||||
three_candidate_child_maps: None,
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""A failed child guard cannot leak its parent update into restoration."""
|
||||
key_hash = authenticated_client._test_api_key.removeprefix("sk-") # type: ignore[attr-defined]
|
||||
child = await integration_session.get(ApiKey, key_hash)
|
||||
assert child is not None
|
||||
parent = ApiKey(hashed_key="failover-parent", balance=10_000_000)
|
||||
child.parent_key_hash = parent.hashed_key
|
||||
child.balance_limit = 75_000
|
||||
integration_session.add(parent)
|
||||
integration_session.add(child)
|
||||
await integration_session.commit()
|
||||
|
||||
sent_requests: list[httpx.Request] = []
|
||||
|
||||
async def fake_transport(
|
||||
request: httpx.Request, *args: Any, **kwargs: Any
|
||||
) -> httpx.Response:
|
||||
sent_requests.append(request)
|
||||
return _upstream_response(request)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"httpx.AsyncHTTPTransport.handle_async_request",
|
||||
side_effect=fake_transport,
|
||||
),
|
||||
patch(
|
||||
"routstr.payment.cost_calculation.sats_usd_price",
|
||||
return_value=0.0005,
|
||||
),
|
||||
):
|
||||
response = await authenticated_client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "dual-model",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
# The 100-sat candidate is rejected before forwarding; the third serves.
|
||||
assert [request.url.host for request in sent_requests] == [
|
||||
"cheap.example.com",
|
||||
"third.example.com",
|
||||
]
|
||||
|
||||
await integration_session.refresh(parent)
|
||||
await integration_session.refresh(child)
|
||||
assert parent.reserved_balance == 0
|
||||
assert child.reserved_balance == 0
|
||||
assert parent.total_spent == response.json()["cost"]["total_msats"]
|
||||
|
||||
records = (
|
||||
await integration_session.exec(
|
||||
select(ReservationRelease).where(ReservationRelease.key_hash == key_hash)
|
||||
)
|
||||
).all()
|
||||
assert len(records) == 2
|
||||
assert sorted(record.status for record in records) == ["charged", "released"]
|
||||
assert len({record.reserved_msats for record in records}) == 1
|
||||
assert all(record.status != "active" for record in records)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def raised_envelope_provider_maps(
|
||||
patched_db_engine: None,
|
||||
) -> AsyncGenerator[None, None]:
|
||||
"""Same-id providers where the fallback needs a larger, affordable reserve."""
|
||||
cheap = _StaticProvider(
|
||||
CHEAP_BASE_URL,
|
||||
"key-cheap",
|
||||
1.0,
|
||||
_make_model("dual-model", 0.001, 0.002, max_cost=50.0),
|
||||
)
|
||||
expensive = _StaticProvider(
|
||||
EXPENSIVE_BASE_URL,
|
||||
"key-expensive",
|
||||
1.0,
|
||||
_make_model("dual-model", 0.005, 0.010, max_cost=100.0),
|
||||
)
|
||||
async for _ in _install_providers([cheap, expensive]):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.asyncio
|
||||
async def test_failover_reserves_serving_candidates_envelope(
|
||||
authenticated_client: AsyncClient,
|
||||
raised_envelope_provider_maps: None,
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""An affordable pricier fallback is re-reserved, served, and billed.
|
||||
|
||||
The fallback's max cost (100 sats) exceeds the winner's (50 sats) but fits
|
||||
the key's balance, so the reservation is raised to the serving candidate's
|
||||
envelope and the request completes, billed at the serving rate with the
|
||||
unused reserve refunded.
|
||||
"""
|
||||
sent_requests: list[httpx.Request] = []
|
||||
|
||||
async def fake_transport(
|
||||
request: httpx.Request, *args: Any, **kwargs: Any
|
||||
) -> httpx.Response:
|
||||
sent_requests.append(request)
|
||||
return _upstream_response(request)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"httpx.AsyncHTTPTransport.handle_async_request",
|
||||
side_effect=fake_transport,
|
||||
),
|
||||
patch(
|
||||
"routstr.payment.cost_calculation.sats_usd_price",
|
||||
return_value=0.0005,
|
||||
),
|
||||
):
|
||||
response = await authenticated_client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": "dual-model",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert [r.url.host for r in sent_requests] == [
|
||||
"cheap.example.com",
|
||||
"expensive.example.com",
|
||||
]
|
||||
assert response.json()["cost"]["total_msats"] == 10_000
|
||||
|
||||
key_hash = authenticated_client._test_api_key.removeprefix("sk-") # type: ignore[attr-defined]
|
||||
records = (
|
||||
await integration_session.exec(
|
||||
select(ReservationRelease).where(ReservationRelease.key_hash == key_hash)
|
||||
)
|
||||
).all()
|
||||
assert len(records) == 2
|
||||
released = next(record for record in records if record.status == "released")
|
||||
charged = next(record for record in records if record.status == "charged")
|
||||
assert charged.reserved_msats > released.reserved_msats
|
||||
assert all(record.status != "active" for record in records)
|
||||
@@ -38,7 +38,7 @@ async def test_overrun_charges_after_reservation_swept(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""Overrun finalize must charge even when the reservation was already released."""
|
||||
from routstr.auth import adjust_payment_for_tokens
|
||||
from routstr.auth import adjust_payment_for_tokens, pay_for_request
|
||||
|
||||
deducted_max_cost = 990 # discounted reservation
|
||||
actual_token_cost = 1000 # actual cost overruns the reservation
|
||||
@@ -47,6 +47,10 @@ async def test_overrun_charges_after_reservation_swept(
|
||||
key = _make_key(balance=1000, reserved=0)
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(key, deducted_max_cost, integration_session)
|
||||
key.reserved_balance = 0
|
||||
integration_session.add(key)
|
||||
await integration_session.commit()
|
||||
|
||||
response_data = {
|
||||
"model": "test-model",
|
||||
@@ -58,7 +62,7 @@ async def test_overrun_charges_after_reservation_swept(
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await adjust_payment_for_tokens(
|
||||
key, response_data, integration_session, deducted_max_cost
|
||||
key, response_data, integration_session, deducted_max_cost, None, None
|
||||
)
|
||||
|
||||
await integration_session.refresh(key)
|
||||
@@ -79,8 +83,16 @@ async def test_free_response_path_closed_end_to_end(
|
||||
patched_db_engine: None,
|
||||
) -> None:
|
||||
"""A reservation released by the real sweeper must not yield a free response."""
|
||||
from routstr.auth import adjust_payment_for_tokens, pay_for_request
|
||||
from routstr.core.db import create_session, release_stale_reservations
|
||||
from routstr.auth import (
|
||||
adjust_payment_for_tokens,
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
)
|
||||
from routstr.core.db import (
|
||||
ReservationRelease,
|
||||
create_session,
|
||||
release_stale_reservations,
|
||||
)
|
||||
|
||||
deducted_max_cost = 990
|
||||
actual_token_cost = 1000
|
||||
@@ -104,10 +116,15 @@ async def test_free_response_path_closed_end_to_end(
|
||||
key = await session.get(ApiKey, key_hash)
|
||||
assert key is not None
|
||||
await pay_for_request(key, deducted_max_cost, session)
|
||||
snapshot = await get_reservation_snapshot(key, session)
|
||||
await session.refresh(key)
|
||||
assert key.reserved_balance == deducted_max_cost
|
||||
key.reserved_at = int(time.time()) - 10_000
|
||||
record = await session.get(ReservationRelease, snapshot.release_id)
|
||||
assert record is not None
|
||||
record.created_at = int(time.time()) - 10_000
|
||||
session.add(key)
|
||||
session.add(record)
|
||||
await session.commit()
|
||||
|
||||
# Sweeper releases the stale reservation without charging.
|
||||
@@ -129,18 +146,20 @@ async def test_free_response_path_closed_end_to_end(
|
||||
return_value=_cost_data(actual_token_cost),
|
||||
):
|
||||
await adjust_payment_for_tokens(
|
||||
key, response_data, session, deducted_max_cost
|
||||
key,
|
||||
response_data,
|
||||
session,
|
||||
deducted_max_cost,
|
||||
reservation_snapshot=snapshot,
|
||||
)
|
||||
|
||||
async with create_session() as session:
|
||||
final = await session.get(ApiKey, key_hash)
|
||||
assert final is not None
|
||||
|
||||
assert final.total_spent == actual_token_cost, (
|
||||
f"Free response: total_spent={final.total_spent}, expected {actual_token_cost}"
|
||||
)
|
||||
assert final.balance == 1000 - actual_token_cost, (
|
||||
f"Balance not charged after sweep: {final.balance}"
|
||||
)
|
||||
# Stale release is terminal for this reservation. A late finalizer must not
|
||||
# charge aggregate balance that may now belong to a newer request.
|
||||
assert final.total_spent == 0
|
||||
assert final.balance == 1000
|
||||
assert final.balance >= 0
|
||||
assert final.reserved_balance == 0
|
||||
|
||||
@@ -241,8 +241,10 @@ async def test_http_402_response_shape_on_insufficient_balance(
|
||||
mock_upstream.prepare_headers = MagicMock(return_value={})
|
||||
|
||||
with (
|
||||
patch("routstr.proxy.get_model_instance", return_value=mock_model),
|
||||
patch("routstr.proxy.get_provider_for_model", return_value=[mock_upstream]),
|
||||
patch(
|
||||
"routstr.proxy.get_candidates",
|
||||
return_value=[(mock_model, mock_upstream)],
|
||||
),
|
||||
# Patch where it is used (proxy imports it at module level)
|
||||
patch(
|
||||
"routstr.proxy.get_max_cost_for_model",
|
||||
|
||||
@@ -120,7 +120,9 @@ async def test_finalise_releases_reservation_and_charges_balance(
|
||||
response_data = {"model": "test-model", "usage": {"prompt_tokens": 50, "completion_tokens": 50}}
|
||||
|
||||
with patch("routstr.auth.calculate_cost", return_value=cost_data):
|
||||
await adjust_payment_for_tokens(key, response_data, integration_session, cost)
|
||||
await adjust_payment_for_tokens(
|
||||
key, response_data, integration_session, cost, None, None
|
||||
)
|
||||
|
||||
await integration_session.refresh(key)
|
||||
|
||||
|
||||
@@ -141,7 +141,7 @@ async def test_revert_with_zero_reserved_balance_is_noop(
|
||||
Previously this would drive reserved_balance negative. With the floor guard,
|
||||
it should return False and leave reserved_balance at 0.
|
||||
"""
|
||||
from routstr.auth import revert_pay_for_request
|
||||
from routstr.auth import pay_for_request, revert_pay_for_request
|
||||
|
||||
unique_key = f"test_revert_key_{uuid.uuid4().hex[:8]}"
|
||||
test_key = ApiKey(
|
||||
@@ -151,8 +151,12 @@ async def test_revert_with_zero_reserved_balance_is_noop(
|
||||
)
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(test_key, 100, integration_session)
|
||||
test_key.reserved_balance = 0
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
|
||||
# Try to revert more than available — should be a no-op
|
||||
# A stale cleanup already released the aggregate reservation.
|
||||
result = await revert_pay_for_request(test_key, integration_session, 100)
|
||||
|
||||
await integration_session.refresh(test_key)
|
||||
@@ -161,8 +165,8 @@ async def test_revert_with_zero_reserved_balance_is_noop(
|
||||
assert test_key.reserved_balance == 0, (
|
||||
f"Reserved balance should remain 0, got: {test_key.reserved_balance}"
|
||||
)
|
||||
assert test_key.total_requests == 0, (
|
||||
f"Total requests should remain 0, got: {test_key.total_requests}"
|
||||
assert test_key.total_requests == 1, (
|
||||
f"Total requests should remain 1, got: {test_key.total_requests}"
|
||||
)
|
||||
|
||||
|
||||
@@ -171,17 +175,18 @@ async def test_revert_with_sufficient_reserved_balance_succeeds(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""Test that revert_pay_for_request works correctly when there is enough reserved balance."""
|
||||
from routstr.auth import revert_pay_for_request
|
||||
from routstr.auth import pay_for_request, revert_pay_for_request
|
||||
|
||||
unique_key = f"test_revert_ok_{uuid.uuid4().hex[:8]}"
|
||||
test_key = ApiKey(
|
||||
hashed_key=unique_key,
|
||||
balance=5000,
|
||||
reserved_balance=500,
|
||||
total_requests=3,
|
||||
reserved_balance=0,
|
||||
total_requests=2,
|
||||
)
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(test_key, 500, integration_session)
|
||||
|
||||
result = await revert_pay_for_request(test_key, integration_session, 500)
|
||||
|
||||
@@ -202,17 +207,21 @@ async def test_revert_partial_reserved_balance_is_noop(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""Test that reverting more than the current reserved_balance is a no-op."""
|
||||
from routstr.auth import revert_pay_for_request
|
||||
from routstr.auth import pay_for_request, revert_pay_for_request
|
||||
|
||||
unique_key = f"test_revert_partial_{uuid.uuid4().hex[:8]}"
|
||||
test_key = ApiKey(
|
||||
hashed_key=unique_key,
|
||||
balance=5000,
|
||||
reserved_balance=50,
|
||||
total_requests=1,
|
||||
reserved_balance=0,
|
||||
total_requests=0,
|
||||
)
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(test_key, 500, integration_session)
|
||||
test_key.reserved_balance = 50
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
|
||||
# Try to revert 500 when only 50 is reserved — should be no-op
|
||||
result = await revert_pay_for_request(test_key, integration_session, 500)
|
||||
@@ -237,20 +246,28 @@ async def test_double_revert_prevented(
|
||||
This simulates the double-revert scenario where both upstream/base.py
|
||||
and proxy.py attempt to revert the same reservation.
|
||||
"""
|
||||
from routstr.auth import revert_pay_for_request
|
||||
from routstr.auth import (
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
revert_pay_for_request,
|
||||
)
|
||||
|
||||
unique_key = f"test_double_revert_{uuid.uuid4().hex[:8]}"
|
||||
test_key = ApiKey(
|
||||
hashed_key=unique_key,
|
||||
balance=10000,
|
||||
reserved_balance=500,
|
||||
total_requests=5,
|
||||
reserved_balance=0,
|
||||
total_requests=4,
|
||||
)
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(test_key, 500, integration_session)
|
||||
snapshot = await get_reservation_snapshot(test_key, integration_session)
|
||||
|
||||
# First revert — should succeed
|
||||
result1 = await revert_pay_for_request(test_key, integration_session, 500)
|
||||
result1 = await revert_pay_for_request(
|
||||
test_key, integration_session, 500, snapshot
|
||||
)
|
||||
await integration_session.refresh(test_key)
|
||||
|
||||
assert result1 is True
|
||||
@@ -258,7 +275,9 @@ async def test_double_revert_prevented(
|
||||
assert test_key.total_requests == 4
|
||||
|
||||
# Second revert of the same amount — should be no-op
|
||||
result2 = await revert_pay_for_request(test_key, integration_session, 500)
|
||||
result2 = await revert_pay_for_request(
|
||||
test_key, integration_session, 500, snapshot
|
||||
)
|
||||
await integration_session.refresh(test_key)
|
||||
|
||||
assert result2 is False, "Second revert should be a no-op"
|
||||
@@ -279,22 +298,30 @@ async def test_sequential_reverts_never_go_negative(
|
||||
Simulates the double-revert scenario where multiple code paths
|
||||
attempt to revert the same reservation.
|
||||
"""
|
||||
from routstr.auth import revert_pay_for_request
|
||||
from routstr.auth import (
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
revert_pay_for_request,
|
||||
)
|
||||
|
||||
unique_key = f"test_multi_revert_{uuid.uuid4().hex[:8]}"
|
||||
test_key = ApiKey(
|
||||
hashed_key=unique_key,
|
||||
balance=10000,
|
||||
reserved_balance=500,
|
||||
total_requests=5,
|
||||
reserved_balance=0,
|
||||
total_requests=4,
|
||||
)
|
||||
integration_session.add(test_key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(test_key, 500, integration_session)
|
||||
snapshot = await get_reservation_snapshot(test_key, integration_session)
|
||||
|
||||
# Run 5 sequential reverts for the same 500 reservation
|
||||
results = []
|
||||
for _ in range(5):
|
||||
r = await revert_pay_for_request(test_key, integration_session, 500)
|
||||
r = await revert_pay_for_request(
|
||||
test_key, integration_session, 500, snapshot
|
||||
)
|
||||
results.append(r)
|
||||
|
||||
await integration_session.refresh(test_key)
|
||||
@@ -317,7 +344,11 @@ async def test_child_key_revert_floor_guard(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
"""Test that child key reserved_balance also has floor guard on revert."""
|
||||
from routstr.auth import revert_pay_for_request
|
||||
from routstr.auth import (
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
revert_pay_for_request,
|
||||
)
|
||||
|
||||
parent_key_hash = f"test_parent_{uuid.uuid4().hex[:8]}"
|
||||
child_key_hash = f"test_child_{uuid.uuid4().hex[:8]}"
|
||||
@@ -325,22 +356,26 @@ async def test_child_key_revert_floor_guard(
|
||||
parent_key = ApiKey(
|
||||
hashed_key=parent_key_hash,
|
||||
balance=10000,
|
||||
reserved_balance=500,
|
||||
total_requests=3,
|
||||
reserved_balance=0,
|
||||
total_requests=2,
|
||||
)
|
||||
child_key = ApiKey(
|
||||
hashed_key=child_key_hash,
|
||||
balance=0,
|
||||
reserved_balance=500,
|
||||
total_requests=3,
|
||||
reserved_balance=0,
|
||||
total_requests=2,
|
||||
parent_key_hash=parent_key_hash,
|
||||
)
|
||||
integration_session.add(parent_key)
|
||||
integration_session.add(child_key)
|
||||
await integration_session.commit()
|
||||
await pay_for_request(child_key, 500, integration_session)
|
||||
snapshot = await get_reservation_snapshot(child_key, integration_session)
|
||||
|
||||
# First revert succeeds
|
||||
result1 = await revert_pay_for_request(child_key, integration_session, 500)
|
||||
result1 = await revert_pay_for_request(
|
||||
child_key, integration_session, 500, snapshot
|
||||
)
|
||||
await integration_session.refresh(parent_key)
|
||||
await integration_session.refresh(child_key)
|
||||
|
||||
@@ -349,7 +384,9 @@ async def test_child_key_revert_floor_guard(
|
||||
assert child_key.reserved_balance == 0
|
||||
|
||||
# Second revert is a no-op for both parent and child
|
||||
result2 = await revert_pay_for_request(child_key, integration_session, 500)
|
||||
result2 = await revert_pay_for_request(
|
||||
child_key, integration_session, 500, snapshot
|
||||
)
|
||||
await integration_session.refresh(parent_key)
|
||||
await integration_session.refresh(child_key)
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Tests for the ``reset_admin_password`` recovery script (issue #553).
|
||||
|
||||
The script is the lockout escape hatch: it works without ``ROUTSTR_SECRET_KEY``
|
||||
(scrypt hashing is key-independent). Two explicit, mutually exclusive actions —
|
||||
``--password`` sets a new hash now, ``--regenerate`` clears the hash so the next
|
||||
boot generates and logs a fresh one. A bare invocation is informational only and
|
||||
must never touch the database (so nobody resets their password by accident).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core import vault
|
||||
from routstr.core.db import get_secret, set_admin_password
|
||||
from scripts.reset_admin_password import apply_reset, build_parser, main
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_password_sets_a_verifiable_hash(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
await apply_reset(integration_session, password="recover-me-123")
|
||||
|
||||
secret = await get_secret(integration_session)
|
||||
assert secret.admin_password_hash is not None
|
||||
assert vault.verify_password("recover-me-123", secret.admin_password_hash) is True
|
||||
assert secret.updated_at is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_regenerate_clears_the_hash(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
# Start from a node that already has an admin password set.
|
||||
await set_admin_password(integration_session, "old-password-9")
|
||||
assert (await get_secret(integration_session)).admin_password_hash is not None
|
||||
|
||||
await apply_reset(integration_session, regenerate=True)
|
||||
|
||||
secret = await get_secret(integration_session)
|
||||
# Cleared -> the next boot's bootstrap_secrets generates and logs a new one.
|
||||
assert secret.admin_password_hash is None
|
||||
assert secret.updated_at is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_password_below_min_length_is_rejected(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
await set_admin_password(integration_session, "old-password-9")
|
||||
|
||||
with pytest.raises(ValueError, match="8 characters"):
|
||||
await apply_reset(integration_session, password="short")
|
||||
|
||||
# The existing password is untouched by the rejected reset.
|
||||
secret = await get_secret(integration_session)
|
||||
assert vault.verify_password("old-password-9", secret.admin_password_hash or "")
|
||||
|
||||
|
||||
def test_password_and_regenerate_are_mutually_exclusive() -> None:
|
||||
parser = build_parser()
|
||||
with pytest.raises(SystemExit):
|
||||
parser.parse_args(["--password", "abcd1234", "--regenerate"])
|
||||
|
||||
|
||||
def test_no_args_prints_help_and_never_opens_a_session(
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
def _fail() -> None:
|
||||
raise AssertionError("a bare invocation must not touch the database")
|
||||
|
||||
monkeypatch.setattr("scripts.reset_admin_password.create_session", _fail)
|
||||
|
||||
assert main([]) == 0
|
||||
assert "usage" in capsys.readouterr().out.lower()
|
||||
@@ -0,0 +1,484 @@
|
||||
"""Tests for ``bootstrap_secrets`` — moving node secrets into the Secret store.
|
||||
|
||||
Specifies the per-secret bootstrap that runs at startup (issue #553). For both
|
||||
the admin password and the nsec it follows the same three branches: use the
|
||||
column if already set, otherwise migrate any legacy plaintext (env first, then
|
||||
the old settings blob), otherwise — admin password only — generate and log one.
|
||||
A column written under a different ROUTSTR_SECRET_KEY fails fast rather than
|
||||
silently corrupting state.
|
||||
"""
|
||||
|
||||
import json
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, AsyncGenerator
|
||||
|
||||
import pytest
|
||||
from sqlmodel import text
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core import vault
|
||||
from routstr.core.db import NsecState, get_secret, set_nsec
|
||||
from routstr.core.settings import (
|
||||
SettingsService,
|
||||
bootstrap_secrets,
|
||||
derive_npub_from_nsec,
|
||||
settings,
|
||||
)
|
||||
|
||||
# Valid Fernet keys; must match the suite default in tests/conftest.py.
|
||||
TEST_SECRET_KEY = "l_Tkp-7xmjcQ-IFhr6qhILrU8HPRbEmYMrfSbo_5srU="
|
||||
TEST_SECRET_KEY_ALT = "_Teyrky_iToeDK51Tj1FsI9MJ340_cqKGmeher-a7MQ="
|
||||
|
||||
NSEC_HEX = "1" * 64
|
||||
# A different key, standing in for a stale value left behind in env/blob after
|
||||
# the vault has taken ownership of the real one.
|
||||
STALE_NSEC_HEX = "2" * 64
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clean_secret_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""No ambient legacy secrets, and a known in-memory settings baseline."""
|
||||
monkeypatch.delenv("ADMIN_PASSWORD", raising=False)
|
||||
monkeypatch.delenv("NSEC", raising=False)
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY", TEST_SECRET_KEY)
|
||||
monkeypatch.setattr(settings, "nsec", "")
|
||||
monkeypatch.setattr(settings, "npub", "")
|
||||
monkeypatch.setattr(settings, "http_url", "")
|
||||
|
||||
|
||||
async def _create_settings_blob(session: AsyncSession, data: dict) -> None:
|
||||
await session.exec( # type: ignore
|
||||
text(
|
||||
"CREATE TABLE IF NOT EXISTS settings "
|
||||
"(id INTEGER PRIMARY KEY, data TEXT NOT NULL, "
|
||||
"updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)"
|
||||
)
|
||||
)
|
||||
await session.exec( # type: ignore
|
||||
text("INSERT INTO settings (id, data) VALUES (1, :data)").bindparams(
|
||||
data=json.dumps(data)
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
# --- admin password --------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generates_admin_password_when_none(
|
||||
clean_secret_env: None, integration_session: AsyncSession
|
||||
) -> None:
|
||||
await bootstrap_secrets(integration_session)
|
||||
secret = await get_secret(integration_session)
|
||||
assert secret.admin_password_hash is not None
|
||||
assert secret.admin_password_hash.startswith("scrypt:")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admin_password_generation_is_idempotent(
|
||||
clean_secret_env: None, integration_session: AsyncSession
|
||||
) -> None:
|
||||
await bootstrap_secrets(integration_session)
|
||||
first = (await get_secret(integration_session)).admin_password_hash
|
||||
await bootstrap_secrets(integration_session)
|
||||
second = (await get_secret(integration_session)).admin_password_hash
|
||||
assert first is not None and first == second
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hashes_legacy_admin_password_from_env(
|
||||
clean_secret_env: None,
|
||||
integration_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("ADMIN_PASSWORD", "hunter2")
|
||||
await bootstrap_secrets(integration_session)
|
||||
secret = await get_secret(integration_session)
|
||||
assert secret.admin_password_hash is not None
|
||||
assert vault.verify_password("hunter2", secret.admin_password_hash) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hashes_legacy_admin_password_from_blob(
|
||||
clean_secret_env: None, integration_session: AsyncSession
|
||||
) -> None:
|
||||
# No ADMIN_PASSWORD in env, but the old settings blob carries one.
|
||||
await _create_settings_blob(integration_session, {"admin_password": "blobpw"})
|
||||
await bootstrap_secrets(integration_session)
|
||||
secret = await get_secret(integration_session)
|
||||
assert vault.verify_password("blobpw", secret.admin_password_hash or "") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admin_password_race_adopts_winner_without_clobber(
|
||||
clean_secret_env: None,
|
||||
integration_engine: Any,
|
||||
integration_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
# Two workers boot against one shared DB and both read a null admin password.
|
||||
# The first to commit "wins" and shows the operator its generated password. A
|
||||
# worker that read null but lost the race must NOT overwrite the winner's hash
|
||||
# (which the operator may already be logging in with) and must NOT print a
|
||||
# second password that will never work.
|
||||
#
|
||||
# The race window is forced deterministically: a hook fires inside bootstrap's
|
||||
# generate branch (so it only runs once this worker has committed to
|
||||
# generating) and commits the winner's password on a separate connection
|
||||
# before this worker writes its own.
|
||||
import sqlite3
|
||||
|
||||
from routstr.core import settings as settings_mod
|
||||
|
||||
db_file = integration_engine.url.database
|
||||
winner_hash = vault.hash_password("winner-password-123")
|
||||
real_token = settings_mod.secrets.token_urlsafe
|
||||
|
||||
def commit_winner_then_generate(nbytes: int) -> str:
|
||||
conn = sqlite3.connect(db_file)
|
||||
conn.execute(
|
||||
"UPDATE secrets SET admin_password_hash = ? WHERE id = 1", (winner_hash,)
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return real_token(nbytes)
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings_mod.secrets, "token_urlsafe", commit_winner_then_generate
|
||||
)
|
||||
|
||||
await get_secret(integration_session) # row exists, password still null
|
||||
capsys.readouterr() # drop anything emitted before the race resolves
|
||||
await bootstrap_secrets(integration_session)
|
||||
|
||||
secret = await get_secret(integration_session)
|
||||
assert secret.admin_password_hash is not None
|
||||
# The winner's password survives and still verifies — no clobber.
|
||||
assert vault.verify_password("winner-password-123", secret.admin_password_hash)
|
||||
# The losing worker stayed silent — no second generated password was leaked.
|
||||
assert "generated a temporary" not in capsys.readouterr().out
|
||||
|
||||
|
||||
# --- nsec ------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_encrypts_legacy_nsec_from_env_and_derives_npub(
|
||||
clean_secret_env: None,
|
||||
integration_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("NSEC", NSEC_HEX)
|
||||
await bootstrap_secrets(integration_session)
|
||||
secret = await get_secret(integration_session)
|
||||
assert secret.encrypted_nsec is not None
|
||||
assert vault.is_encrypted(secret.encrypted_nsec) is True
|
||||
assert vault.decrypt(secret.encrypted_nsec) == NSEC_HEX
|
||||
# In-memory runtime value is the decrypted nsec, and npub is derived from it.
|
||||
assert settings.nsec == NSEC_HEX
|
||||
assert settings.npub == derive_npub_from_nsec(NSEC_HEX)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_decrypts_existing_nsec_column(
|
||||
clean_secret_env: None, integration_session: AsyncSession
|
||||
) -> None:
|
||||
secret = await get_secret(integration_session)
|
||||
secret.encrypted_nsec = vault.encrypt(NSEC_HEX)
|
||||
secret.nsec_state = NsecState.encrypted
|
||||
integration_session.add(secret)
|
||||
await integration_session.commit()
|
||||
stored = secret.encrypted_nsec
|
||||
|
||||
await bootstrap_secrets(integration_session)
|
||||
reloaded = await get_secret(integration_session)
|
||||
assert settings.nsec == NSEC_HEX
|
||||
# The column is reused, not re-encrypted.
|
||||
assert reloaded.encrypted_nsec == stored
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fail_fast_when_nsec_encrypted_with_different_key(
|
||||
clean_secret_env: None,
|
||||
integration_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# Encrypt the column under the alternate key, then bootstrap under the
|
||||
# suite key -> the value cannot be decrypted -> clear startup failure.
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY", TEST_SECRET_KEY_ALT)
|
||||
secret = await get_secret(integration_session)
|
||||
secret.encrypted_nsec = vault.encrypt(NSEC_HEX)
|
||||
secret.nsec_state = NsecState.encrypted
|
||||
integration_session.add(secret)
|
||||
await integration_session.commit()
|
||||
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY", TEST_SECRET_KEY)
|
||||
with pytest.raises(RuntimeError, match="ROUTSTR_SECRET_KEY"):
|
||||
await bootstrap_secrets(integration_session)
|
||||
|
||||
|
||||
# --- encryption is mandatory, key custody is not: upgrade without a key --------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_legacy_nsec_without_secret_key_generates_and_encrypts(
|
||||
clean_secret_env: None,
|
||||
integration_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
# A node upgrading with a legacy plaintext NSEC but no ROUTSTR_SECRET_KEY must
|
||||
# NOT break. Encryption at rest stays mandatory (the nsec is never persisted
|
||||
# in plaintext), but the key custody is flexible: bootstrap generates a master
|
||||
# key, persists it to the key file, warns loudly, and encrypts the identity —
|
||||
# so the node keeps running instead of refusing to boot.
|
||||
monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False)
|
||||
key_file = tmp_path / "routstr_secret.key"
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file))
|
||||
monkeypatch.setenv("NSEC", NSEC_HEX)
|
||||
|
||||
await bootstrap_secrets(integration_session)
|
||||
|
||||
# A master key was generated and persisted...
|
||||
assert key_file.exists()
|
||||
# ...the nsec is encrypted at rest under it, never stored in plaintext...
|
||||
secret = await get_secret(integration_session)
|
||||
assert secret.encrypted_nsec is not None
|
||||
assert vault.is_encrypted(secret.encrypted_nsec) is True
|
||||
assert vault.decrypt(secret.encrypted_nsec) == NSEC_HEX
|
||||
assert secret.nsec_state == NsecState.encrypted
|
||||
# ...the node holds the live identity (npub derived from it)...
|
||||
assert settings.nsec == NSEC_HEX
|
||||
assert settings.npub == derive_npub_from_nsec(NSEC_HEX)
|
||||
# ...and the operator is loudly told a key was generated and must be backed up
|
||||
# (path shown, but never the key value) so an upgrade cannot silently create
|
||||
# an unbacked key nor leak the key into captured stdout / aggregated logs.
|
||||
out = capsys.readouterr().out
|
||||
assert str(key_file) in out
|
||||
assert key_file.read_text().strip() not in out
|
||||
assert "BACK UP" in out.upper()
|
||||
|
||||
|
||||
# --- boot ordering: rescue legacy blob secrets before they are stripped ----
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_blob_only_nsec_is_migrated_before_blob_is_stripped(
|
||||
clean_secret_env: None, integration_session: AsyncSession
|
||||
) -> None:
|
||||
# Legacy node whose nsec lives ONLY in the settings blob (never in env).
|
||||
# bootstrap_secrets must run *before* SettingsService.initialize strips the
|
||||
# blob, or the only copy of the secret would be lost.
|
||||
await _create_settings_blob(
|
||||
integration_session, {"nsec": NSEC_HEX, "name": "LegacyNode"}
|
||||
)
|
||||
|
||||
await bootstrap_secrets(integration_session)
|
||||
await SettingsService.initialize(integration_session)
|
||||
|
||||
secret = await get_secret(integration_session)
|
||||
# The plaintext nsec has been moved into the encrypted Secret store...
|
||||
assert secret.encrypted_nsec is not None
|
||||
assert vault.decrypt(secret.encrypted_nsec) == NSEC_HEX
|
||||
assert settings.nsec == NSEC_HEX
|
||||
# ...and stripped from the persisted settings blob.
|
||||
row = await integration_session.exec( # type: ignore
|
||||
text("SELECT data FROM settings WHERE id = 1")
|
||||
)
|
||||
blob = json.loads(row.first()[0])
|
||||
assert "nsec" not in blob
|
||||
assert blob["name"] == "LegacyNode"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_does_not_clobber_store_only_nsec(
|
||||
clean_secret_env: None, integration_session: AsyncSession
|
||||
) -> None:
|
||||
# Steady state after migration: the nsec lives ONLY in the encrypted Secret
|
||||
# store (NSEC removed from env, blob already stripped on a previous boot).
|
||||
# bootstrap decrypts it into memory; initialize then re-derives settings from
|
||||
# the secret-free blob and must NOT wipe the live nsec back to empty (or the
|
||||
# node would silently stop signing Nostr announcements).
|
||||
await _create_settings_blob(integration_session, {"name": "LegacyNode"})
|
||||
secret = await get_secret(integration_session)
|
||||
secret.encrypted_nsec = vault.encrypt(NSEC_HEX)
|
||||
secret.nsec_state = NsecState.encrypted
|
||||
integration_session.add(secret)
|
||||
await integration_session.commit()
|
||||
|
||||
await bootstrap_secrets(integration_session)
|
||||
assert settings.nsec == NSEC_HEX # bootstrap decrypted it into memory
|
||||
|
||||
await SettingsService.initialize(integration_session)
|
||||
# The live secret survives initialize even though no env/blob carries it...
|
||||
assert settings.nsec == NSEC_HEX
|
||||
# ...and is still never written back to the persisted blob.
|
||||
row = await integration_session.exec( # type: ignore
|
||||
text("SELECT data FROM settings WHERE id = 1")
|
||||
)
|
||||
assert "nsec" not in json.loads(row.first()[0])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stale_env_nsec_does_not_override_vault_nsec(
|
||||
clean_secret_env: None,
|
||||
integration_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# The vault owns the nsec, but a stale NSEC (e.g. the operator rotated the
|
||||
# key in the UI yet left the old value in .env) is still in the environment.
|
||||
# bootstrap decrypts the store value; initialize must NOT let the stale env
|
||||
# value clobber it, or a restart silently reverts to the old identity.
|
||||
await set_nsec(integration_session, NSEC_HEX)
|
||||
|
||||
monkeypatch.setenv("NSEC", STALE_NSEC_HEX)
|
||||
await _create_settings_blob(integration_session, {"name": "LegacyNode"})
|
||||
|
||||
await bootstrap_secrets(integration_session)
|
||||
await SettingsService.initialize(integration_session)
|
||||
|
||||
# The vault value wins; the stale env value is ignored.
|
||||
assert settings.nsec == NSEC_HEX
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stale_env_nsec_does_not_split_npub_from_vault_nsec(
|
||||
clean_secret_env: None,
|
||||
integration_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# As above, the vault owns the nsec while a stale NSEC lingers in env. The
|
||||
# private key correctly comes from the vault, but the npub must too: if
|
||||
# initialize derives the public key from the stale env nsec, the node ends up
|
||||
# with a private key from the vault and a public key from the old env value,
|
||||
# and anything reading settings.npub announces the wrong Nostr identity.
|
||||
expected_npub = derive_npub_from_nsec(NSEC_HEX)
|
||||
stale_npub = derive_npub_from_nsec(STALE_NSEC_HEX)
|
||||
assert expected_npub and stale_npub and expected_npub != stale_npub # guard
|
||||
|
||||
await set_nsec(integration_session, NSEC_HEX)
|
||||
|
||||
monkeypatch.setenv("NSEC", STALE_NSEC_HEX)
|
||||
await _create_settings_blob(integration_session, {"name": "LegacyNode"})
|
||||
|
||||
await bootstrap_secrets(integration_session)
|
||||
await SettingsService.initialize(integration_session)
|
||||
|
||||
assert settings.nsec == NSEC_HEX
|
||||
assert settings.npub == expected_npub
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cleared_nsec_stays_cleared_across_reboot(
|
||||
clean_secret_env: None,
|
||||
integration_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# An identity was imported from env, then the operator cleared it via the
|
||||
# admin API. The old NSEC is still in env. On the NEXT PROCESS the cleared
|
||||
# identity must stay cleared, not get resurrected from the stale env value.
|
||||
monkeypatch.setenv("NSEC", NSEC_HEX)
|
||||
await bootstrap_secrets(integration_session)
|
||||
assert settings.nsec == NSEC_HEX
|
||||
|
||||
# Clear via the admin path (store empty, vault owns it).
|
||||
await set_nsec(integration_session, "")
|
||||
|
||||
# Simulate a fresh process rather than pre-clearing the live singleton: the
|
||||
# pydantic settings global reloads the (still-stale) NSEC from env and derives
|
||||
# its npub, which is exactly the in-memory state a new boot starts from before
|
||||
# bootstrap runs. The cleared store must win over this stale live value.
|
||||
monkeypatch.setattr(settings, "nsec", NSEC_HEX)
|
||||
monkeypatch.setattr(settings, "npub", derive_npub_from_nsec(NSEC_HEX))
|
||||
|
||||
await bootstrap_secrets(integration_session)
|
||||
|
||||
reloaded = await get_secret(integration_session)
|
||||
assert reloaded.nsec_state == NsecState.cleared
|
||||
assert reloaded.encrypted_nsec is None # not re-imported
|
||||
assert settings.nsec == "" # stays cleared
|
||||
assert settings.npub == "" # and no derived public identity survives
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_keeps_npub_matching_store_only_nsec(
|
||||
clean_secret_env: None, integration_session: AsyncSession
|
||||
) -> None:
|
||||
# Steady state with mandatory encryption: the nsec lives ONLY in the
|
||||
# encrypted Secret store (env carries no NSEC) and the blob has no npub.
|
||||
# bootstrap decrypts the nsec and derives the npub into memory; initialize
|
||||
# then re-derives settings from the npub-less blob and must NOT wipe the npub
|
||||
# back to empty, or the node holds a private key with no matching public key
|
||||
# and silently stops announcing a usable Nostr identity.
|
||||
expected_npub = derive_npub_from_nsec(NSEC_HEX)
|
||||
assert expected_npub # guard: the test key must yield a real npub
|
||||
|
||||
await _create_settings_blob(integration_session, {"name": "LegacyNode"})
|
||||
secret = await get_secret(integration_session)
|
||||
secret.encrypted_nsec = vault.encrypt(NSEC_HEX)
|
||||
secret.nsec_state = NsecState.encrypted
|
||||
integration_session.add(secret)
|
||||
await integration_session.commit()
|
||||
|
||||
await bootstrap_secrets(integration_session)
|
||||
assert settings.npub == expected_npub # bootstrap derived it
|
||||
|
||||
await SettingsService.initialize(integration_session)
|
||||
# The npub still matches the live nsec...
|
||||
assert settings.nsec == NSEC_HEX
|
||||
assert settings.npub == expected_npub
|
||||
# ...and is persisted to the blob (it is public, not a stripped secret).
|
||||
row = await integration_session.exec( # type: ignore
|
||||
text("SELECT data FROM settings WHERE id = 1")
|
||||
)
|
||||
assert json.loads(row.first()[0])["npub"] == expected_npub
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_startup_runs_bootstrap_before_settings_initialize(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# The two tests above prove the migration outcome *given* the call order;
|
||||
# they hardcode that order themselves. This one guards the order at its real
|
||||
# call site — the application lifespan — so a reorder in main.py (which would
|
||||
# strip a blob-only secret before bootstrap could rescue it) is caught.
|
||||
import routstr.core.main as main
|
||||
|
||||
order: list[str] = []
|
||||
|
||||
class _Abort(Exception):
|
||||
pass
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_create_session() -> AsyncGenerator[None, None]:
|
||||
yield None
|
||||
|
||||
async def fake_bootstrap(session: Any) -> None:
|
||||
order.append("bootstrap")
|
||||
|
||||
async def fake_initialize(session: Any) -> None:
|
||||
order.append("initialize")
|
||||
# Stop startup here, before the background-task fan-out (prices, nostr,
|
||||
# upstreams) that we don't want to run in a unit test.
|
||||
raise _Abort()
|
||||
|
||||
async def noop_init_db() -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(main, "configure_litellm", lambda: None)
|
||||
monkeypatch.setattr(main, "register_deepseek_v4_pricing", lambda: None)
|
||||
monkeypatch.setattr(main, "run_migrations", lambda: None)
|
||||
monkeypatch.setattr(main, "init_db", noop_init_db)
|
||||
monkeypatch.setattr(main, "create_session", fake_create_session)
|
||||
monkeypatch.setattr(main, "bootstrap_secrets", fake_bootstrap)
|
||||
monkeypatch.setattr(main.SettingsService, "initialize", fake_initialize)
|
||||
|
||||
with pytest.raises(_Abort):
|
||||
async with main.lifespan(main.app):
|
||||
pass
|
||||
|
||||
assert order == ["bootstrap", "initialize"]
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Tests for the ``Secret`` singleton model (issue #553).
|
||||
|
||||
Specifies the node-level secret store: a single row (``id=1``, like
|
||||
``RoutstrFee``) holding the one-way admin-password hash and the encrypted nsec.
|
||||
``get_secret`` is get-or-create, so callers always get the singleton without
|
||||
worrying whether it has been initialised yet. Encoding of the values themselves
|
||||
lives in ``routstr.core.vault``; here we only assert the row persists and stays
|
||||
a singleton.
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from sqlmodel import select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.db import Secret, get_secret
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_secret_creates_singleton(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
secret = await get_secret(integration_session)
|
||||
assert secret.id == 1
|
||||
# Fresh row carries no secret material yet.
|
||||
assert secret.admin_password_hash is None
|
||||
assert secret.encrypted_nsec is None
|
||||
assert secret.updated_at is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_secret_is_idempotent(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
first = await get_secret(integration_session)
|
||||
second = await get_secret(integration_session)
|
||||
assert first.id == second.id == 1
|
||||
rows = (await integration_session.exec(select(Secret))).all()
|
||||
assert len(rows) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_secret_fields_round_trip(
|
||||
integration_session: AsyncSession,
|
||||
) -> None:
|
||||
secret = await get_secret(integration_session)
|
||||
secret.admin_password_hash = "scrypt:16384:8:1:c2FsdA==:aGFzaA=="
|
||||
secret.encrypted_nsec = "fernet:v1:gAAAAA"
|
||||
secret.updated_at = int(time.time())
|
||||
integration_session.add(secret)
|
||||
await integration_session.commit()
|
||||
|
||||
integration_session.expunge_all()
|
||||
reloaded = await get_secret(integration_session)
|
||||
assert reloaded.admin_password_hash == "scrypt:16384:8:1:c2FsdA==:aGFzaA=="
|
||||
assert reloaded.encrypted_nsec == "fernet:v1:gAAAAA"
|
||||
assert reloaded.updated_at is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_secret_tolerates_concurrent_first_insert(
|
||||
integration_engine: Any,
|
||||
integration_session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# A second worker wins the race and commits the singleton row first.
|
||||
async with AsyncSession(integration_engine, expire_on_commit=False) as other:
|
||||
other.add(Secret(id=1, admin_password_hash="scrypt:from-other-worker"))
|
||||
await other.commit()
|
||||
|
||||
# Reproduce the race window: our session's first read still sees no row, so
|
||||
# it attempts to INSERT a duplicate id=1. The real IntegrityError that follows
|
||||
# must be recovered (roll back, re-read) rather than crashing startup.
|
||||
real_get = integration_session.get
|
||||
calls = {"n": 0}
|
||||
|
||||
async def stale_first_read(model: Any, pk: Any) -> Any:
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
return None
|
||||
return await real_get(model, pk)
|
||||
|
||||
monkeypatch.setattr(integration_session, "get", stale_first_read)
|
||||
|
||||
secret = await get_secret(integration_session)
|
||||
|
||||
# Recovered the other worker's row; no crash, still a single row.
|
||||
assert secret.id == 1
|
||||
assert secret.admin_password_hash == "scrypt:from-other-worker"
|
||||
rows = (await integration_session.exec(select(Secret))).all()
|
||||
assert len(rows) == 1
|
||||
@@ -0,0 +1,82 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.core import admin
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("requested_mint", [None, "https://secondary.example"])
|
||||
async def test_withdraw_uses_effective_mint_and_records_outgoing_transaction(
|
||||
monkeypatch: pytest.MonkeyPatch, requested_mint: str | None
|
||||
) -> None:
|
||||
primary_mint = "https://primary.example"
|
||||
effective_mint = requested_mint or primary_mint
|
||||
wallet = object()
|
||||
proofs = [SimpleNamespace(amount=40), SimpleNamespace(amount=60)]
|
||||
token = "cashuBoutgoing"
|
||||
|
||||
get_wallet = AsyncMock(return_value=wallet)
|
||||
get_proofs = Mock(return_value=proofs)
|
||||
filter_proofs = AsyncMock(return_value=proofs)
|
||||
send_token = AsyncMock(return_value=token)
|
||||
store_transaction = AsyncMock(return_value=True)
|
||||
|
||||
monkeypatch.setattr(admin, "get_wallet", get_wallet)
|
||||
monkeypatch.setattr(admin, "get_proofs_per_mint_and_unit", get_proofs)
|
||||
monkeypatch.setattr(admin, "slow_filter_spend_proofs", filter_proofs)
|
||||
monkeypatch.setattr(admin, "send_token", send_token)
|
||||
monkeypatch.setattr(admin, "store_cashu_transaction", store_transaction)
|
||||
monkeypatch.setattr(admin.settings, "primary_mint", primary_mint)
|
||||
|
||||
result = await admin.withdraw(
|
||||
Mock(),
|
||||
admin.WithdrawRequest(amount=75, mint_url=requested_mint, unit="sat"),
|
||||
)
|
||||
|
||||
assert result == {"token": token}
|
||||
get_wallet.assert_awaited_once_with(effective_mint, "sat")
|
||||
get_proofs.assert_called_once_with(wallet, effective_mint, "sat", not_reserved=True)
|
||||
filter_proofs.assert_awaited_once_with(proofs, wallet)
|
||||
send_token.assert_awaited_once_with(75, "sat", effective_mint)
|
||||
store_transaction.assert_awaited_once_with(
|
||||
token=token,
|
||||
amount=75,
|
||||
unit="sat",
|
||||
mint_url=effective_mint,
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="admin",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_withdraw_returns_issued_token_when_audit_storage_fails(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
mint = "https://primary.example"
|
||||
proofs = [SimpleNamespace(amount=100)]
|
||||
token = "cashuBrecoverable"
|
||||
|
||||
monkeypatch.setattr(admin, "get_wallet", AsyncMock(return_value=object()))
|
||||
monkeypatch.setattr(
|
||||
admin, "get_proofs_per_mint_and_unit", Mock(return_value=proofs)
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
admin, "slow_filter_spend_proofs", AsyncMock(return_value=proofs)
|
||||
)
|
||||
monkeypatch.setattr(admin, "send_token", AsyncMock(return_value=token))
|
||||
monkeypatch.setattr(
|
||||
admin,
|
||||
"store_cashu_transaction",
|
||||
AsyncMock(side_effect=RuntimeError("database unavailable")),
|
||||
)
|
||||
critical = Mock()
|
||||
monkeypatch.setattr(admin.logger, "critical", critical)
|
||||
monkeypatch.setattr(admin.settings, "primary_mint", mint)
|
||||
|
||||
result = await admin.withdraw(Mock(), admin.WithdrawRequest(amount=75))
|
||||
|
||||
assert result == {"token": token}
|
||||
critical.assert_called_once()
|
||||
@@ -137,12 +137,12 @@ def test_create_model_mappings_includes_db_override_for_missing_cached_model(
|
||||
|
||||
model_instances, provider_map, unique_models = create_model_mappings(
|
||||
upstreams=[provider],
|
||||
overrides_by_id={"azure/gpt-4o": (override_row, 1.01)},
|
||||
disabled_model_ids=set(),
|
||||
overrides_by_key={("azure/gpt-4o", 7): (override_row, 1.01)},
|
||||
disabled_model_keys=set(),
|
||||
)
|
||||
|
||||
assert "azure/gpt-4o" in model_instances
|
||||
assert provider_map["azure/gpt-4o"] == [provider]
|
||||
assert [p for _, p in provider_map["azure/gpt-4o"]] == [provider]
|
||||
assert "gpt-4o" in unique_models
|
||||
|
||||
|
||||
@@ -182,11 +182,73 @@ def test_create_model_mappings_dedupes_with_provider_identity_not_provider_type(
|
||||
|
||||
_, provider_map, _ = create_model_mappings(
|
||||
upstreams=[provider_a, provider_b],
|
||||
overrides_by_id={"azure/gpt-4o": (override_row, 1.01)},
|
||||
disabled_model_ids=set(),
|
||||
overrides_by_key={("azure/gpt-4o", 2): (override_row, 1.01)},
|
||||
disabled_model_keys=set(),
|
||||
)
|
||||
|
||||
providers_for_alias = provider_map["azure/gpt-4o"]
|
||||
providers_for_alias = [p for _, p in provider_map["azure/gpt-4o"]]
|
||||
assert provider_a in providers_for_alias
|
||||
assert provider_b in providers_for_alias
|
||||
assert len(providers_for_alias) == 2
|
||||
|
||||
|
||||
def test_create_model_mappings_applies_override_only_to_matching_provider(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Same-id overrides must not add provider-specific aliases to other providers."""
|
||||
provider_a_model = create_test_model("same-id", prompt_price=0.01)
|
||||
provider_a = create_test_provider(
|
||||
"provider-a",
|
||||
"https://provider-a.example/v1",
|
||||
db_id=1,
|
||||
models=[provider_a_model],
|
||||
)
|
||||
provider_b_model = create_test_model("same-id", prompt_price=0.02)
|
||||
provider_b = create_test_provider(
|
||||
"provider-b",
|
||||
"https://provider-b.example/v1",
|
||||
db_id=2,
|
||||
models=[provider_b_model],
|
||||
)
|
||||
|
||||
override_model = create_test_model("same-id", prompt_price=0.001)
|
||||
override_model.alias_ids = ["provider-b-only"]
|
||||
override_row = SimpleNamespace(id="same-id", upstream_provider_id=2, enabled=True)
|
||||
|
||||
def fake_row_to_model(*args, **kwargs) -> Model: # type: ignore[no-untyped-def]
|
||||
return override_model
|
||||
|
||||
monkeypatch.setattr("routstr.payment.models._row_to_model", fake_row_to_model)
|
||||
|
||||
_, provider_map, _ = create_model_mappings(
|
||||
upstreams=[provider_a, provider_b],
|
||||
overrides_by_key={("same-id", 2): (override_row, 1.01)},
|
||||
disabled_model_keys=set(),
|
||||
)
|
||||
|
||||
assert [p for _, p in provider_map["provider-b-only"]] == [provider_b]
|
||||
assert {p for _, p in provider_map["same-id"]} == {provider_a, provider_b}
|
||||
|
||||
|
||||
def test_create_model_mappings_disables_only_matching_provider() -> None:
|
||||
"""Disabled overrides are scoped to the provider row, not the shared model id."""
|
||||
provider_a = create_test_provider(
|
||||
"provider-a",
|
||||
"https://provider-a.example/v1",
|
||||
db_id=1,
|
||||
models=[create_test_model("same-id")],
|
||||
)
|
||||
provider_b = create_test_provider(
|
||||
"provider-b",
|
||||
"https://provider-b.example/v1",
|
||||
db_id=2,
|
||||
models=[create_test_model("same-id")],
|
||||
)
|
||||
|
||||
_, provider_map, _ = create_model_mappings(
|
||||
upstreams=[provider_a, provider_b],
|
||||
overrides_by_key={},
|
||||
disabled_model_keys={("same-id", 2)},
|
||||
)
|
||||
|
||||
assert [p for _, p in provider_map["same-id"]] == [provider_a]
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.core.db import CashuTransaction
|
||||
from routstr.upstream.auto_topup import _check_and_topup
|
||||
|
||||
|
||||
def _row() -> MagicMock:
|
||||
row = MagicMock()
|
||||
row.id = "provider-1"
|
||||
row.base_url = "https://provider.test"
|
||||
row.api_key = "secret"
|
||||
row.provider_settings = json.dumps(
|
||||
{
|
||||
"auto_topup": True,
|
||||
"topup_threshold": 100,
|
||||
"topup_amount_limit": 50,
|
||||
"topup_mint_url": "https://mint.test",
|
||||
}
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
class _Session:
|
||||
def __init__(self, transaction: CashuTransaction) -> None:
|
||||
self.transaction = transaction
|
||||
self.commit = AsyncMock()
|
||||
|
||||
async def __aenter__(self) -> "_Session":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: object) -> None:
|
||||
return None
|
||||
|
||||
async def exec(self, query: object) -> MagicMock:
|
||||
result = MagicMock()
|
||||
result.first.return_value = self.transaction
|
||||
return result
|
||||
|
||||
def add(self, transaction: CashuTransaction) -> None:
|
||||
self.transaction = transaction
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_topup_persists_before_sending_and_marks_success_collected() -> None:
|
||||
provider = MagicMock()
|
||||
provider.get_balance = AsyncMock(return_value=0)
|
||||
provider.topup = AsyncMock(return_value={"balance": 50})
|
||||
transaction = CashuTransaction(
|
||||
token="cashu-token", amount=50, unit="sat", source="auto_topup"
|
||||
)
|
||||
session = _Session(transaction)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.RoutstrUpstreamProvider.from_db_row",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.send_token",
|
||||
AsyncMock(return_value="cashu-token"),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.store_cashu_transaction",
|
||||
AsyncMock(return_value=True),
|
||||
) as store,
|
||||
patch("routstr.upstream.auto_topup.create_session", return_value=session),
|
||||
):
|
||||
await _check_and_topup(_row())
|
||||
|
||||
store.assert_awaited_once_with(
|
||||
token="cashu-token",
|
||||
amount=50,
|
||||
unit="sat",
|
||||
mint_url="https://mint.test",
|
||||
typ="out",
|
||||
collected=False,
|
||||
source="auto_topup",
|
||||
)
|
||||
provider.topup.assert_awaited_once_with("cashu-token")
|
||||
assert transaction.collected is True
|
||||
session.commit.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("outcome", [{"error": "rejected"}, RuntimeError("network")])
|
||||
async def test_auto_topup_failure_leaves_persisted_token_uncollected(
|
||||
outcome: object,
|
||||
) -> None:
|
||||
provider = MagicMock()
|
||||
provider.get_balance = AsyncMock(return_value=0)
|
||||
provider.topup = AsyncMock(
|
||||
side_effect=outcome if isinstance(outcome, Exception) else None,
|
||||
return_value=outcome,
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.RoutstrUpstreamProvider.from_db_row",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.send_token",
|
||||
AsyncMock(return_value="cashu-token"),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.store_cashu_transaction",
|
||||
AsyncMock(return_value=True),
|
||||
),
|
||||
patch("routstr.upstream.auto_topup.create_session") as create_session,
|
||||
):
|
||||
if isinstance(outcome, Exception):
|
||||
with pytest.raises(RuntimeError):
|
||||
await _check_and_topup(_row())
|
||||
else:
|
||||
await _check_and_topup(_row())
|
||||
|
||||
create_session.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_topup_does_not_send_untracked_token() -> None:
|
||||
provider = MagicMock()
|
||||
provider.get_balance = AsyncMock(return_value=0)
|
||||
provider.topup = AsyncMock()
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.RoutstrUpstreamProvider.from_db_row",
|
||||
return_value=provider,
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.send_token",
|
||||
AsyncMock(return_value="cashu-token"),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.auto_topup.store_cashu_transaction",
|
||||
AsyncMock(side_effect=RuntimeError("database unavailable")),
|
||||
),
|
||||
):
|
||||
await _check_and_topup(_row())
|
||||
provider.topup.assert_not_awaited()
|
||||
@@ -0,0 +1,40 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from routstr import balance as balance_module
|
||||
from routstr.core.db import get_session
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_balance_accepts_large_cashu_token_in_post_body(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
token = "cashuA" + "x" * 20_000
|
||||
key = SimpleNamespace(hashed_key="hashed", balance=123_000)
|
||||
validate_bearer_key = AsyncMock(return_value=key)
|
||||
session = AsyncMock()
|
||||
monkeypatch.setattr(balance_module, "validate_bearer_key", validate_bearer_key)
|
||||
|
||||
async def override_get_session(): # type: ignore[no-untyped-def]
|
||||
yield session
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(balance_module.balance_router)
|
||||
app.dependency_overrides[get_session] = override_get_session
|
||||
|
||||
async with AsyncClient(
|
||||
transport=ASGITransport(app=app), # type: ignore[arg-type]
|
||||
base_url="http://test",
|
||||
) as client:
|
||||
response = await client.post(
|
||||
"/v1/balance/create",
|
||||
json={"initial_balance_token": token},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"api_key": "sk-hashed", "balance": 123_000}
|
||||
validate_bearer_key.assert_awaited_once_with(token, session)
|
||||
@@ -0,0 +1,56 @@
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.core.db import store_cashu_transaction
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
OSError("disk full"),
|
||||
RuntimeError("connection lost"),
|
||||
ConnectionRefusedError("database unavailable"),
|
||||
],
|
||||
)
|
||||
async def test_store_cashu_transaction_propagates_commit_errors(
|
||||
error: Exception,
|
||||
) -> None:
|
||||
session = AsyncMock()
|
||||
session.commit.side_effect = error
|
||||
session.__aenter__.return_value = session
|
||||
session.__aexit__.return_value = None
|
||||
|
||||
with (
|
||||
patch("routstr.core.db.create_session", return_value=session),
|
||||
patch("routstr.core.db.logger.critical") as critical,
|
||||
):
|
||||
with pytest.raises(type(error), match=str(error)):
|
||||
await store_cashu_transaction(
|
||||
token="cashuAtest",
|
||||
amount=1_000,
|
||||
unit="sat",
|
||||
mint_url="https://mint.example",
|
||||
typ="out",
|
||||
request_id="request-1",
|
||||
)
|
||||
|
||||
critical.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_store_cashu_transaction_returns_true_after_commit() -> None:
|
||||
session = AsyncMock()
|
||||
session.__aenter__.return_value = session
|
||||
session.__aexit__.return_value = None
|
||||
|
||||
with patch("routstr.core.db.create_session", return_value=session):
|
||||
stored = await store_cashu_transaction(
|
||||
token="cashuAtest",
|
||||
amount=1_000,
|
||||
unit="sat",
|
||||
)
|
||||
|
||||
assert stored is True
|
||||
session.commit.assert_awaited_once()
|
||||
@@ -0,0 +1,91 @@
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlmodel import SQLModel, select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core import db
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cashu_transaction_storage_retries_then_succeeds() -> None:
|
||||
store = AsyncMock(side_effect=[OSError("database locked"), True])
|
||||
sleep = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("routstr.core.db.store_cashu_transaction", store),
|
||||
patch("routstr.core.db.asyncio.sleep", sleep),
|
||||
):
|
||||
stored = await db.store_cashu_transaction_with_retry(
|
||||
token="cashuAretry",
|
||||
amount=100,
|
||||
unit="sat",
|
||||
)
|
||||
|
||||
assert stored is True
|
||||
assert store.await_count == 2
|
||||
sleep.assert_awaited_once_with(0.25)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cashu_transaction_retry_is_idempotent_after_ambiguous_commit() -> None:
|
||||
engine = create_async_engine("sqlite+aiosqlite://")
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(SQLModel.metadata.create_all)
|
||||
|
||||
original_store = db.store_cashu_transaction
|
||||
attempts = 0
|
||||
|
||||
async def ambiguous_store(**kwargs: Any) -> bool:
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
stored = await original_store(**kwargs)
|
||||
if attempts == 1:
|
||||
raise OSError("connection dropped after commit")
|
||||
return stored
|
||||
|
||||
with (
|
||||
patch.object(db, "engine", engine),
|
||||
patch("routstr.core.db.store_cashu_transaction", ambiguous_store),
|
||||
patch("routstr.core.db.asyncio.sleep", AsyncMock()),
|
||||
):
|
||||
stored = await db.store_cashu_transaction_with_retry(
|
||||
token="cashuAambiguous",
|
||||
amount=100,
|
||||
unit="sat",
|
||||
)
|
||||
|
||||
async with AsyncSession(engine) as session:
|
||||
result = await session.exec(select(db.CashuTransaction))
|
||||
transactions = result.all()
|
||||
|
||||
assert stored is True
|
||||
assert attempts == 2
|
||||
assert len(transactions) == 1
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cashu_transaction_storage_raises_after_bounded_retries() -> None:
|
||||
error = OSError("database unavailable")
|
||||
store = AsyncMock(side_effect=error)
|
||||
sleep = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("routstr.core.db.store_cashu_transaction", store),
|
||||
patch("routstr.core.db.asyncio.sleep", sleep),
|
||||
patch("routstr.core.db.logger.critical") as critical,
|
||||
):
|
||||
with pytest.raises(OSError, match="database unavailable"):
|
||||
await db.store_cashu_transaction_with_retry(
|
||||
token="cashuAfail",
|
||||
amount=100,
|
||||
unit="sat",
|
||||
max_attempts=3,
|
||||
)
|
||||
|
||||
assert store.await_count == 3
|
||||
assert [call.args[0] for call in sleep.await_args_list] == [0.25, 0.5]
|
||||
critical.assert_called_once()
|
||||
@@ -465,9 +465,179 @@ async def test_cache_read_only_usd_cost_response_is_billed(
|
||||
assert result.cache_read_input_tokens == 1000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("total_cost", "input_cost", "output_cost", "expected_msats"),
|
||||
[
|
||||
(0.000471, 0.00023451, 0.00023649, 9420),
|
||||
(0.00000004, 0.00000002, 0.00000002, 1),
|
||||
],
|
||||
)
|
||||
async def test_small_usd_cost_components_sum_to_rounded_total(
|
||||
total_cost: float,
|
||||
input_cost: float,
|
||||
output_cost: float,
|
||||
expected_msats: int,
|
||||
) -> None:
|
||||
"""Small USD component costs must retain every billed millisatoshi."""
|
||||
response = {
|
||||
"model": "gpt-4",
|
||||
"usage": {
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"cost_details": {
|
||||
"total_cost": total_cost,
|
||||
"input_cost": input_cost,
|
||||
"output_cost": output_cost,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result = await calculate_cost(response, max_cost=100000)
|
||||
|
||||
assert isinstance(result, CostData)
|
||||
assert result.total_msats == expected_msats
|
||||
assert result.input_msats + result.output_msats == result.total_msats
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openrouter_upstream_inference_cost_components_are_used() -> None:
|
||||
"""OpenRouter component aliases must determine the input/output split."""
|
||||
response = {
|
||||
"model": "gpt-4",
|
||||
"usage": {
|
||||
"prompt_tokens": 375,
|
||||
"completion_tokens": 158,
|
||||
"total_tokens": 533,
|
||||
"cost": 0.00022354,
|
||||
"is_byok": False,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": 286,
|
||||
"cache_write_tokens": 0,
|
||||
},
|
||||
"cost_details": {
|
||||
"upstream_inference_cost": 0.00022354,
|
||||
"upstream_inference_prompt_cost": 0.00004974,
|
||||
"upstream_inference_completions_cost": 0.0001738,
|
||||
},
|
||||
"completion_tokens_details": {"reasoning_tokens": 17},
|
||||
},
|
||||
}
|
||||
|
||||
result = await calculate_cost(response, max_cost=100000)
|
||||
|
||||
assert isinstance(result, CostData)
|
||||
assert result.input_msats == 994
|
||||
assert result.output_msats == 3477
|
||||
assert result.input_msats + result.output_msats == result.total_msats == 4471
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# PPQ.AI BYOK: upstream_inference_cost + BYOK fee billing
|
||||
#
|
||||
# PPQ.AI (bring-your-own-key) returns a small ~5 % BYOK routing fee in
|
||||
# ``usage.cost`` and the real inference cost in
|
||||
# ``cost_details.upstream_inference_cost``. The old code billed only the fee,
|
||||
# under-charging by ~20×. The fix bills ``upstream_inference_cost + byok_fee``
|
||||
# — what PPQ actually deducts from the balance.
|
||||
# Payload numbers are from a live ``glm-5.2-fast`` request (GitHub issue #615).
|
||||
# ============================================================================
|
||||
@pytest.mark.asyncio
|
||||
async def test_ppq_byok_bills_upstream_inference_cost_plus_fee() -> None:
|
||||
"""PPQ.AI BYOK must bill upstream_inference_cost + byok_fee, not just the
|
||||
fee. Mirrors the live request from GitHub issue #615."""
|
||||
response = {
|
||||
"model": "glm-5.2-fast",
|
||||
"usage": {
|
||||
"prompt_tokens": 164371, # includes 159301 cached
|
||||
"completion_tokens": 99,
|
||||
"cost": 0.002260057305, # ~5% BYOK routing fee
|
||||
"is_byok": True,
|
||||
"prompt_tokens_details": {"cached_tokens": 159301},
|
||||
"cost_details": {
|
||||
"upstream_inference_cost": 0.04475361,
|
||||
"upstream_inference_prompt_cost": 0.04410021,
|
||||
"upstream_inference_completions_cost": 0.0006534,
|
||||
},
|
||||
},
|
||||
}
|
||||
result = await calculate_cost(response, max_cost=100000)
|
||||
|
||||
assert isinstance(result, CostData)
|
||||
# The fix bills upstream_inference_cost + byok_fee (~0.047 USD → ~940k
|
||||
# msats), not the fee alone (~0.0023 USD → ~45k msats). ~20× correction.
|
||||
assert result.total_msats == 940274
|
||||
assert result.input_msats + result.output_msats == result.total_msats
|
||||
assert result.input_msats == 926546
|
||||
assert result.output_msats == 13728
|
||||
assert result.total_usd == pytest.approx(0.047013667305)
|
||||
# Token normalisation (OpenAI dialect: cached included in prompt_tokens)
|
||||
assert result.input_tokens == 5070 # 164371 - 159301
|
||||
assert result.cache_read_input_tokens == 159301
|
||||
assert result.output_tokens == 99
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ppq_byok_fee_only_would_undercharge() -> None:
|
||||
"""Sanity check: billing only usage.cost (the BYOK fee) under-charges by
|
||||
~20×. This documents the regression the fix prevents."""
|
||||
response = {
|
||||
"model": "glm-5.2-fast",
|
||||
"usage": {
|
||||
"prompt_tokens": 164371,
|
||||
"completion_tokens": 99,
|
||||
"cost": 0.002260057305, # BYOK fee only — no upstream_inference_cost
|
||||
"is_byok": True,
|
||||
"prompt_tokens_details": {"cached_tokens": 159301},
|
||||
},
|
||||
}
|
||||
result = await calculate_cost(response, max_cost=100000)
|
||||
|
||||
assert isinstance(result, CostData)
|
||||
# Without upstream_inference_cost, only the fee is billed — the old bug.
|
||||
assert result.total_msats == 45202
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Test 13: Missing Usage Block
|
||||
# ============================================================================
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_upstream_cost_uses_litellm_model_pricing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Token usage without an upstream cost is priced from LiteLLM."""
|
||||
monkeypatch.setattr(settings, "fixed_pricing", True)
|
||||
monkeypatch.setattr(settings, "fixed_per_1k_input_tokens", 0)
|
||||
monkeypatch.setattr(settings, "fixed_per_1k_output_tokens", 0)
|
||||
monkeypatch.setattr(
|
||||
"routstr.payment.models.litellm_cost_entry",
|
||||
lambda model: {
|
||||
"input_cost_per_token": 0.000001,
|
||||
"output_cost_per_token": 0.000002,
|
||||
},
|
||||
)
|
||||
response = {
|
||||
"model": "priced-by-litellm",
|
||||
"usage": {
|
||||
"prompt_tokens": 90,
|
||||
"completion_tokens": 80,
|
||||
"total_tokens": 170,
|
||||
"prompt_tokens_details": {"cached_tokens": 0},
|
||||
"completion_tokens_details": {"reasoning_tokens": 74},
|
||||
"prompt_cache_hit_tokens": 0,
|
||||
"prompt_cache_miss_tokens": 90,
|
||||
},
|
||||
}
|
||||
|
||||
result = await calculate_cost(response, max_cost=10000)
|
||||
|
||||
assert isinstance(result, CostData)
|
||||
assert not isinstance(result, MaxCostData)
|
||||
assert result.input_msats == 1800
|
||||
assert result.output_msats == 3200
|
||||
assert result.input_msats + result.output_msats == result.total_msats == 5000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_usage_block(mock_fixed_pricing: None) -> None:
|
||||
"""When usage is missing, return MaxCostData with zero tokens."""
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Coverage tests for admin.py (currently 35%).
|
||||
|
||||
Tests admin endpoints that are testable without full app setup:
|
||||
withdraw validation, authentication guards, and slug validation.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException, Request
|
||||
|
||||
# ===========================================================================
|
||||
# withdraw — validation and edge cases
|
||||
# ===========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_withdraw_rejects_zero_amount() -> None:
|
||||
"""withdraw validation rejects amount <= 0."""
|
||||
from routstr.core.admin import WithdrawRequest, withdraw
|
||||
|
||||
request = Request(scope={"type": "http", "method": "POST"})
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await withdraw(request, WithdrawRequest(amount=0, unit="sat"))
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_withdraw_rejects_negative_amount() -> None:
|
||||
"""withdraw validation rejects negative amounts."""
|
||||
from routstr.core.admin import WithdrawRequest, withdraw
|
||||
|
||||
request = Request(scope={"type": "http", "method": "POST"})
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await withdraw(request, WithdrawRequest(amount=-100, unit="sat"))
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_withdraw_rejects_insufficient_balance() -> None:
|
||||
"""withdraw returns 400 when wallet balance is insufficient."""
|
||||
from routstr.core.admin import WithdrawRequest, withdraw
|
||||
|
||||
request = Request(scope={"type": "http", "method": "POST"})
|
||||
|
||||
with patch("routstr.core.admin.get_wallet") as mock_wallet, \
|
||||
patch("routstr.core.admin.get_proofs_per_mint_and_unit") as mock_proofs, \
|
||||
patch("routstr.core.admin.slow_filter_spend_proofs") as mock_filter:
|
||||
|
||||
mock_w = Mock()
|
||||
mock_w.keysets = {}
|
||||
mock_w.proofs = []
|
||||
mock_wallet.return_value = mock_w
|
||||
mock_proofs.return_value = []
|
||||
mock_filter.return_value = []
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await withdraw(request, WithdrawRequest(amount=1000000, unit="sat"))
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "Insufficient" in str(exc_info.value.detail)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# require_admin_api guard
|
||||
# ===========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_require_admin_rejects_no_session() -> None:
|
||||
"""require_admin_api rejects requests without admin session cookie."""
|
||||
from routstr.core.admin import require_admin_api
|
||||
|
||||
request = Request(scope={
|
||||
"type": "http",
|
||||
"method": "GET",
|
||||
"headers": [],
|
||||
})
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await require_admin_api(request)
|
||||
|
||||
# 401 or 403 depending on auth configuration
|
||||
assert exc_info.value.status_code in (401, 403)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# _validate_slug
|
||||
# ===========================================================================
|
||||
|
||||
def test_validate_slug_accepts_valid() -> None:
|
||||
"""Valid slugs pass validation."""
|
||||
from routstr.core.admin import _validate_slug
|
||||
|
||||
assert _validate_slug("valid-slug") == "valid-slug"
|
||||
assert _validate_slug("valid123") == "valid123"
|
||||
assert _validate_slug("my-provider") == "my-provider"
|
||||
|
||||
|
||||
def test_validate_slug_rejects_spaces() -> None:
|
||||
"""Slugs with spaces are rejected."""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routstr.core.admin import _validate_slug
|
||||
|
||||
with pytest.raises(HTTPException):
|
||||
_validate_slug("invalid slug")
|
||||
|
||||
|
||||
def test_validate_slug_rejects_too_short() -> None:
|
||||
"""Slugs shorter than 3 chars are rejected."""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routstr.core.admin import _validate_slug
|
||||
|
||||
with pytest.raises(HTTPException):
|
||||
_validate_slug("ab")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# admin login endpoint
|
||||
# ===========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_admin_login_requires_payload() -> None:
|
||||
"""admin_login requires a payload — verify it exists."""
|
||||
# Verify the function signature
|
||||
import inspect
|
||||
|
||||
from routstr.core.admin import admin_login
|
||||
sig = inspect.signature(admin_login)
|
||||
params = list(sig.parameters.keys())
|
||||
assert "request" in params
|
||||
assert "payload" in params or len(params) >= 2
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Coverage tests for base.py (currently 41%).
|
||||
|
||||
Tests preparers, builders, accessors, and model cache methods.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.upstream.base import BaseUpstreamProvider
|
||||
|
||||
# ===========================================================================
|
||||
# prepare_headers
|
||||
# ===========================================================================
|
||||
|
||||
def test_prepare_headers_adds_auth() -> None:
|
||||
"""API key is added as Bearer token."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test-key")
|
||||
headers = p.prepare_headers({})
|
||||
|
||||
assert "Authorization" in headers
|
||||
assert headers["Authorization"] == "Bearer sk-test-key"
|
||||
|
||||
|
||||
def test_prepare_headers_preserves_existing() -> None:
|
||||
"""Existing headers are preserved."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test-key")
|
||||
headers = p.prepare_headers({"X-Custom": "value", "Content-Type": "application/json"})
|
||||
|
||||
assert headers["X-Custom"] == "value"
|
||||
assert headers["Content-Type"] == "application/json"
|
||||
|
||||
|
||||
def test_prepare_headers_auth_header_passthrough() -> None:
|
||||
"""Authorization header is handled — verify current behaviour."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test-key")
|
||||
headers = p.prepare_headers({"Authorization": "Bearer user-key"})
|
||||
|
||||
# Currently provider key is used (may be intentional for proxy pattern)
|
||||
assert "Authorization" in headers
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# prepare_params
|
||||
# ===========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prepare_params_passes_through() -> None:
|
||||
"""Query params are preserved by default."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test-key")
|
||||
params = p.prepare_params("/v1/chat/completions", {"temperature": "0.7"})
|
||||
|
||||
assert params["temperature"] == "0.7"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# transform_model_name / normalize_request_path / get_request_base_url
|
||||
# ===========================================================================
|
||||
|
||||
def test_transform_model_name_default_passthrough() -> None:
|
||||
"""Default returns model_id unchanged."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test-key")
|
||||
assert p.transform_model_name("gpt-4") == "gpt-4"
|
||||
assert p.transform_model_name("") == ""
|
||||
|
||||
|
||||
def test_normalize_request_path_passthrough() -> None:
|
||||
"""Default returns path unchanged."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test-key")
|
||||
assert p.normalize_request_path("/v1/chat/completions") == "/v1/chat/completions"
|
||||
|
||||
|
||||
def test_get_request_base_url_default() -> None:
|
||||
"""Default returns the provider's base_url."""
|
||||
p = BaseUpstreamProvider("https://api.test.com/v1", "sk-test-key")
|
||||
url = p.get_request_base_url("/v1/chat/completions")
|
||||
assert url == "https://api.test.com/v1"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# build_request_url
|
||||
# ===========================================================================
|
||||
|
||||
def test_build_request_url_combines_base_and_path() -> None:
|
||||
"""Combines base_url and path."""
|
||||
p = BaseUpstreamProvider("https://api.test.com/v1", "sk-test-key")
|
||||
url = p.build_request_url("/chat/completions")
|
||||
assert "api.test.com" in url
|
||||
assert "/chat/completions" in url
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# get_litellm_provider_prefix / get_provider_metadata
|
||||
# ===========================================================================
|
||||
|
||||
def test_get_litellm_provider_prefix_default() -> None:
|
||||
"""Default returns a string prefix."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test-key")
|
||||
prefix = p.get_litellm_provider_prefix()
|
||||
assert isinstance(prefix, str)
|
||||
|
||||
|
||||
def test_get_provider_metadata_returns_dict() -> None:
|
||||
"""Default metadata has name and capabilities."""
|
||||
metadata = BaseUpstreamProvider.get_provider_metadata()
|
||||
assert isinstance(metadata, dict)
|
||||
assert "name" in metadata
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# from_db_row
|
||||
# ===========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_from_db_row_returns_provider() -> None:
|
||||
"""from_db_row constructs a provider from a valid row."""
|
||||
mock_row = Mock()
|
||||
mock_row.base_url = "https://api.test.com"
|
||||
mock_row.api_key = "sk-test-key"
|
||||
mock_row.slug = "test-slug"
|
||||
mock_row.provider_fee = 1.0
|
||||
mock_row.field_overrides = None
|
||||
mock_row.name = "Test"
|
||||
|
||||
result = BaseUpstreamProvider.from_db_row(mock_row)
|
||||
assert result is not None
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# prepare_request_body
|
||||
# ===========================================================================
|
||||
|
||||
def test_prepare_request_body_with_model() -> None:
|
||||
"""prepare_request_body takes bytes body and Model object."""
|
||||
mock_model = Mock()
|
||||
mock_model.id = "gpt-4"
|
||||
mock_model.forwarded_model_id = None
|
||||
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test-key")
|
||||
|
||||
# None body returns None
|
||||
result = p.prepare_request_body(None, mock_model)
|
||||
assert result is None
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# prepare_responses_request_body
|
||||
# ===========================================================================
|
||||
|
||||
def test_prepare_responses_request_body_none() -> None:
|
||||
"""None body returns None."""
|
||||
model_obj = Mock()
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test-key")
|
||||
result = p.prepare_responses_request_body(None, model_obj)
|
||||
assert result is None
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# _upstream_accepts_cache_control
|
||||
# ===========================================================================
|
||||
|
||||
def test_upstream_accepts_cache_control_default() -> None:
|
||||
"""Default: upstream does NOT accept cache-control."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test-key")
|
||||
assert p._upstream_accepts_cache_control() is False
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# inject_cost_metadata
|
||||
# ===========================================================================
|
||||
|
||||
def test_inject_cost_metadata_adds_metadata() -> None:
|
||||
"""Cost metadata is injected into the response dict."""
|
||||
mock_key = Mock()
|
||||
mock_key.balance_msat = 500000
|
||||
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test-key")
|
||||
data = {"model": "gpt-4", "usage": {"prompt_tokens": 100}}
|
||||
cost_data = {
|
||||
"base_msats": 200000,
|
||||
"input_msats": 100000,
|
||||
"output_msats": 100000,
|
||||
"total_msats": 200000,
|
||||
"total_usd": 0.01,
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 50,
|
||||
}
|
||||
|
||||
p.inject_cost_metadata(data, cost_data, mock_key)
|
||||
|
||||
# Metadata is nested under metadata.routstr.cost
|
||||
assert "metadata" in data or "routstr_cost" in data or "cost" in data
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# _apply_provider_field
|
||||
# ===========================================================================
|
||||
|
||||
def test_apply_provider_field_adds_to_response() -> None:
|
||||
"""Provider field is added to response JSON."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test-key")
|
||||
data = {"id": "chatcmpl-123"}
|
||||
p._apply_provider_field(data)
|
||||
assert "provider" in data
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Additional coverage tests for base.py (41% → target 50%+).
|
||||
|
||||
Tests error message extraction, static helpers, model cache, and cost hooks.
|
||||
|
||||
These test existing correct behavior — all should PASS.
|
||||
"""
|
||||
|
||||
import json
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.upstream.base import BaseUpstreamProvider
|
||||
|
||||
# ===========================================================================
|
||||
# _extract_upstream_error_message
|
||||
# ===========================================================================
|
||||
|
||||
def test_extract_error_from_json_body() -> None:
|
||||
"""Error message is extracted from JSON upstream error response."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
body = json.dumps({"error": {"message": "Model not found", "type": "not_found"}}).encode()
|
||||
|
||||
msg, error_type = p._extract_upstream_error_message(body)
|
||||
|
||||
assert "Model not found" in msg
|
||||
assert error_type == "not_found"
|
||||
|
||||
|
||||
def test_extract_error_from_simple_json() -> None:
|
||||
"""Simple JSON error with direct message key."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
body = json.dumps({"message": "Rate limit exceeded"}).encode()
|
||||
|
||||
msg, error_type = p._extract_upstream_error_message(body)
|
||||
|
||||
assert "Rate limit" in msg
|
||||
|
||||
|
||||
def test_extract_error_from_text_body() -> None:
|
||||
"""Non-JSON text body is returned as-is."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
|
||||
msg, error_type = p._extract_upstream_error_message(b"Internal Server Error")
|
||||
|
||||
assert "Internal Server Error" in msg
|
||||
|
||||
|
||||
def test_extract_error_empty_body() -> None:
|
||||
"""Empty body returns a generic message."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
|
||||
msg, error_type = p._extract_upstream_error_message(b"")
|
||||
|
||||
assert isinstance(msg, str)
|
||||
assert len(msg) > 0
|
||||
|
||||
|
||||
def test_extract_error_simple_error_string_not_parsed() -> None:
|
||||
"""JSON error as plain string (not dict) falls through to generic message."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
body = json.dumps({"error": "Invalid API key"}).encode()
|
||||
|
||||
msg, error_type = p._extract_upstream_error_message(body)
|
||||
|
||||
# Simple error strings not nested in a dict object use generic message
|
||||
assert "Upstream request failed" in msg or "Invalid" in msg
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# on_upstream_error_redirect
|
||||
# ===========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_on_upstream_error_redirect_noop() -> None:
|
||||
"""Default implementation is a no-op for non-redirect statuses."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
await p.on_upstream_error_redirect(402, "Insufficient balance")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_on_upstream_error_redirect_429() -> None:
|
||||
"""429 rate limit passes through (subclasses may override)."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
await p.on_upstream_error_redirect(429, "Rate limited")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# _fold_cache_into_input_tokens (static method)
|
||||
# ===========================================================================
|
||||
|
||||
def test_fold_cache_no_cache_data() -> None:
|
||||
"""Usage without cache details is unchanged."""
|
||||
from routstr.upstream.base import BaseUpstreamProvider
|
||||
|
||||
usage = Mock()
|
||||
usage.prompt_tokens = 100
|
||||
del usage.prompt_tokens_details # No cache details
|
||||
|
||||
BaseUpstreamProvider._fold_cache_into_input_tokens(usage)
|
||||
# Should not modify the usage object when no cache exists
|
||||
|
||||
|
||||
def test_fold_cache_preserves_total() -> None:
|
||||
"""Total prompt tokens remain the same after folding cache."""
|
||||
from routstr.upstream.base import BaseUpstreamProvider
|
||||
|
||||
usage = Mock()
|
||||
usage.prompt_tokens = 100
|
||||
details = Mock()
|
||||
details.cached_tokens = 30
|
||||
usage.prompt_tokens_details = details
|
||||
|
||||
BaseUpstreamProvider._fold_cache_into_input_tokens(usage)
|
||||
# prompt_tokens should still be 100 (total unchanged)
|
||||
assert usage.prompt_tokens == 100
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# get_cached_models / get_cached_model_by_id
|
||||
# ===========================================================================
|
||||
|
||||
def test_get_cached_models_returns_list() -> None:
|
||||
"""get_cached_models always returns a list."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
models = p.get_cached_models()
|
||||
assert isinstance(models, list)
|
||||
|
||||
|
||||
def test_get_cached_model_by_id_unknown_returns_none() -> None:
|
||||
"""Unknown model ID returns None."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
result = p.get_cached_model_by_id("nonexistent-model-xyz-12345")
|
||||
assert result is None
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# get_x_cashu_cost
|
||||
# ===========================================================================
|
||||
|
||||
def test_get_x_cashu_cost_with_usage() -> None:
|
||||
"""Cost is calculated from response data with usage info."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
response_data = {
|
||||
"model": "gpt-4",
|
||||
"usage": {"prompt_tokens": 100, "completion_tokens": 50},
|
||||
}
|
||||
|
||||
result = p.get_x_cashu_cost(response_data, 100000, None)
|
||||
|
||||
# Either returns None (needs more data) or a cost object
|
||||
assert result is not None
|
||||
|
||||
|
||||
def test_get_x_cashu_cost_no_usage() -> None:
|
||||
"""Response without usage returns MaxCostData."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
response_data = {"model": "gpt-4"}
|
||||
|
||||
result = p.get_x_cashu_cost(response_data, 100000, None)
|
||||
|
||||
# Without usage, uses max_cost
|
||||
assert result is not None
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# get_balance
|
||||
# ===========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_balance_raises_not_implemented() -> None:
|
||||
"""Default get_balance raises NotImplementedError (no account support)."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
with pytest.raises(NotImplementedError):
|
||||
await p.get_balance()
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# refresh_models_cache
|
||||
# ===========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_models_cache_no_providers() -> None:
|
||||
"""refresh_models_cache handles empty provider list gracefully."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
# Default implementation may be a no-op or raise
|
||||
try:
|
||||
await p.refresh_models_cache()
|
||||
except Exception:
|
||||
pass # May fail without DB — that's fine
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# fetch_models
|
||||
# ===========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_models_returns_list() -> None:
|
||||
"""fetch_models returns a model list (or empty) for default provider."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
try:
|
||||
result = await p.fetch_models()
|
||||
assert isinstance(result, list)
|
||||
except Exception:
|
||||
pass # May fail without network
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# create_account
|
||||
# ===========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_account_raises_not_implemented() -> None:
|
||||
"""Default create_account raises NotImplementedError."""
|
||||
p = BaseUpstreamProvider("https://api.test.com", "sk-test")
|
||||
with pytest.raises(NotImplementedError):
|
||||
await p.create_account()
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Coverage-filling tests for middleware.py (currently 38% coverage).
|
||||
|
||||
Only LoggingMiddleware and request_id_context exist on main.
|
||||
ConcurrencyLimiterMiddleware + TimeoutMiddleware are on an unmerged branch.
|
||||
"""
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LoggingMiddleware
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_logging_middleware_adds_request_id() -> None:
|
||||
"""Every request gets an x-routstr-request-id header."""
|
||||
from routstr.core.middleware import LoggingMiddleware
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@app.get("/test")
|
||||
async def test_endpoint(request: Request) -> dict:
|
||||
assert hasattr(request.state, "request_id")
|
||||
assert request.state.request_id is not None
|
||||
return {"ok": True}
|
||||
|
||||
app.add_middleware(LoggingMiddleware)
|
||||
|
||||
client = TestClient(app)
|
||||
response = client.get("/test")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "x-routstr-request-id" in response.headers
|
||||
assert len(response.headers["x-routstr-request-id"]) == 36 # UUID4 length
|
||||
|
||||
|
||||
def test_logging_middleware_skips_head_requests() -> None:
|
||||
"""HEAD requests are skipped by _should_log (health probes)."""
|
||||
from routstr.core.middleware import LoggingMiddleware
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@app.head("/test")
|
||||
async def test_endpoint(request: Request) -> dict:
|
||||
return {"ok": True}
|
||||
|
||||
app.add_middleware(LoggingMiddleware)
|
||||
|
||||
client = TestClient(app)
|
||||
response = client.head("/test")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "x-routstr-request-id" in response.headers
|
||||
|
||||
|
||||
def test_logging_middleware_skips_options_requests() -> None:
|
||||
"""OPTIONS requests (CORS preflight) are skipped."""
|
||||
from routstr.core.middleware import LoggingMiddleware
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@app.options("/test")
|
||||
async def test_endpoint(request: Request) -> dict:
|
||||
return {"ok": True}
|
||||
|
||||
app.add_middleware(LoggingMiddleware)
|
||||
|
||||
client = TestClient(app)
|
||||
response = client.options("/test")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "x-routstr-request-id" in response.headers
|
||||
|
||||
|
||||
def test_should_log_rejects_admin_api_prefix() -> None:
|
||||
"""Admin API polling paths are skipped."""
|
||||
from routstr.core.middleware import _should_log
|
||||
|
||||
assert _should_log("GET", "/admin/api/balances") is False
|
||||
assert _should_log("GET", "/admin/api/logs") is False
|
||||
assert _should_log("GET", "/admin/api/providers") is False
|
||||
|
||||
|
||||
def test_should_log_rejects_nextjs_chunks() -> None:
|
||||
"""Next.js static chunks are skipped."""
|
||||
from routstr.core.middleware import _should_log
|
||||
|
||||
assert _should_log("GET", "/_next/static/chunks/main.js") is False
|
||||
assert _should_log("GET", "/_next/data/build-id/page.json") is False
|
||||
|
||||
|
||||
def test_should_log_rejects_exact_paths() -> None:
|
||||
"""Exact paths like /favicon.ico are skipped."""
|
||||
from routstr.core.middleware import _should_log
|
||||
|
||||
assert _should_log("GET", "/favicon.ico") is False
|
||||
assert _should_log("GET", "/v1/wallet/info") is False
|
||||
assert _should_log("GET", "/index.txt") is False
|
||||
assert _should_log("GET", "/login/index.txt") is False
|
||||
|
||||
|
||||
def test_should_log_accepts_normal_paths() -> None:
|
||||
"""Normal API paths are logged."""
|
||||
from routstr.core.middleware import _should_log
|
||||
|
||||
assert _should_log("GET", "/v1/chat/completions") is True
|
||||
assert _should_log("POST", "/v1/chat/completions") is True
|
||||
assert _should_log("GET", "/v1/models") is True
|
||||
assert _should_log("POST", "/api/some-endpoint") is True
|
||||
|
||||
|
||||
def test_should_log_accepts_non_skipped_path() -> None:
|
||||
"""Generic paths not in skip list are logged."""
|
||||
from routstr.core.middleware import _should_log
|
||||
|
||||
assert _should_log("GET", "/some/random/path") is True
|
||||
assert _should_log("POST", "/api/custom") is True
|
||||
|
||||
|
||||
def test_request_id_context_is_contextvar() -> None:
|
||||
"""request_id_context is a ContextVar[str | None] with no default value."""
|
||||
from contextvars import ContextVar
|
||||
|
||||
from routstr.core.middleware import request_id_context
|
||||
|
||||
assert isinstance(request_id_context, ContextVar)
|
||||
# ContextVar without a default raises LookupError when accessed without being set
|
||||
try:
|
||||
val = request_id_context.get()
|
||||
# If it returns, it should be None
|
||||
assert val is None
|
||||
except LookupError:
|
||||
# Expected: ContextVar with no default raises LookupError
|
||||
pass
|
||||
|
||||
|
||||
def test_middleware_exports() -> None:
|
||||
"""Only LoggingMiddleware is exported on main."""
|
||||
from routstr.core.middleware import LoggingMiddleware, request_id_context
|
||||
|
||||
assert LoggingMiddleware is not None
|
||||
assert request_id_context is not None
|
||||
|
||||
|
||||
def test_middleware_skips_health_probe_path() -> None:
|
||||
"""Health probe paths pass through without logging."""
|
||||
from routstr.core.middleware import _should_log
|
||||
|
||||
# HEAD method is always skipped regardless of path
|
||||
assert _should_log("HEAD", "/v1/chat/completions") is False
|
||||
assert _should_log("OPTIONS", "/v1/chat/completions") is False
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Coverage-filling tests for payment/helpers.py (currently 52% coverage).
|
||||
|
||||
Tests the real public API: check_token_balance, get_max_cost_for_model,
|
||||
estimate_tokens, create_error_response, etc.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# check_token_balance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_token_balance_x_cashu_present() -> None:
|
||||
"""X-Cashu header triggers token deserialization and balance check."""
|
||||
from routstr.payment.helpers import check_token_balance
|
||||
|
||||
headers = {"x-cashu": "cashuAtest_token"}
|
||||
body = {"model": "gpt-4"}
|
||||
|
||||
with patch("routstr.payment.helpers.deserialize_token_from_string") as mock_deser:
|
||||
mock_token = Mock()
|
||||
mock_token.amount = 50000
|
||||
mock_token.unit = "sat"
|
||||
mock_deser.return_value = mock_token
|
||||
|
||||
# Should not raise — balance is sufficient
|
||||
check_token_balance(headers, body, 1000)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_token_balance_no_x_cashu_raises() -> None:
|
||||
"""Missing X-Cashu header raises HTTPException (401 on main)."""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routstr.payment.helpers import check_token_balance
|
||||
|
||||
headers: dict[str, str] = {}
|
||||
body = {"model": "gpt-4"}
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
check_token_balance(headers, body, 1000)
|
||||
|
||||
assert exc_info.value.status_code == 401
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_token_balance_insufficient_raises() -> None:
|
||||
"""Token with insufficient balance raises HTTPException 402.
|
||||
|
||||
max_cost_for_model is in msat, so with amount=100 sat (=100,000 msat),
|
||||
max_cost=200,000 msat triggers the insufficient balance check.
|
||||
"""
|
||||
from fastapi import HTTPException
|
||||
|
||||
from routstr.payment.helpers import check_token_balance
|
||||
|
||||
headers = {"x-cashu": "cashuAtest_token"}
|
||||
body = {"model": "gpt-4"}
|
||||
|
||||
with patch("routstr.payment.helpers.deserialize_token_from_string") as mock_deser:
|
||||
mock_token = Mock()
|
||||
mock_token.amount = 100 # 100 sat
|
||||
mock_token.unit = "sat"
|
||||
mock_deser.return_value = mock_token
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
# 200,000 msat > 100,000 msat (100 sat * 1000)
|
||||
check_token_balance(headers, body, 200000)
|
||||
|
||||
assert exc_info.value.status_code == 402
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# estimate_tokens
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_estimate_tokens_empty_messages() -> None:
|
||||
"""Empty message list returns 0 tokens."""
|
||||
from routstr.payment.helpers import estimate_tokens
|
||||
|
||||
result = estimate_tokens([])
|
||||
|
||||
assert result == 0
|
||||
|
||||
|
||||
def test_estimate_tokens_text_content() -> None:
|
||||
"""Text messages are counted."""
|
||||
from routstr.payment.helpers import estimate_tokens
|
||||
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Hello, how are you?"},
|
||||
]
|
||||
|
||||
result = estimate_tokens(messages)
|
||||
|
||||
assert result > 0
|
||||
assert isinstance(result, int)
|
||||
|
||||
|
||||
def test_estimate_tokens_long_text() -> None:
|
||||
"""Longer messages produce higher token counts."""
|
||||
from routstr.payment.helpers import estimate_tokens
|
||||
|
||||
short = estimate_tokens([{"role": "user", "content": "Hi"}])
|
||||
long = estimate_tokens([{"role": "user", "content": "Hello " * 100}])
|
||||
|
||||
assert long > short
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# create_error_response
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_create_error_response_402() -> None:
|
||||
"""402 Payment Required error is properly formatted."""
|
||||
from fastapi import Request
|
||||
|
||||
from routstr.payment.helpers import create_error_response
|
||||
|
||||
request = Request(scope={"type": "http", "method": "GET"})
|
||||
result = create_error_response("insufficient_funds", "Insufficient balance", 402, request)
|
||||
|
||||
assert result.status_code == 402
|
||||
|
||||
|
||||
def test_create_error_response_500() -> None:
|
||||
"""500 Internal Server Error is properly formatted."""
|
||||
from fastapi import Request
|
||||
|
||||
from routstr.payment.helpers import create_error_response
|
||||
|
||||
request = Request(scope={"type": "http", "method": "GET"})
|
||||
result = create_error_response("server_error", "Internal error", 500, request)
|
||||
|
||||
assert result.status_code == 500
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Image token estimation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_image_dimensions_valid_png() -> None:
|
||||
"""_get_image_dimensions returns width and height for a valid PNG."""
|
||||
from routstr.payment.helpers import _get_image_dimensions
|
||||
|
||||
# A minimal 1x1 red PNG (valid minimal file)
|
||||
png = (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
b"\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02"
|
||||
b"\x00\x00\x00\x90wS\xde"
|
||||
b"\x00\x00\x00\x0cIDAT\x08\xd7c\xf8\x0f\x00\x00\x01\x01\x00\x05"
|
||||
b"\x18\xd8N"
|
||||
b"\x00\x00\x00\x00IEND\xaeB`\x82"
|
||||
)
|
||||
|
||||
w, h = _get_image_dimensions(png)
|
||||
assert w == 1
|
||||
assert h == 1
|
||||
|
||||
|
||||
def test_calculate_image_tokens_low_detail() -> None:
|
||||
"""Low detail images are always 85 tokens."""
|
||||
from routstr.payment.helpers import _calculate_image_tokens
|
||||
|
||||
tokens = _calculate_image_tokens(1024, 1024, "low")
|
||||
|
||||
assert tokens == 85
|
||||
|
||||
|
||||
def test_calculate_image_tokens_high_detail() -> None:
|
||||
"""High detail images are scaled and tile-based."""
|
||||
from routstr.payment.helpers import _calculate_image_tokens
|
||||
|
||||
tokens = _calculate_image_tokens(1024, 1024, "high")
|
||||
|
||||
assert tokens > 85
|
||||
assert isinstance(tokens, int)
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Coverage tests for proxy.py (currently 47%).
|
||||
|
||||
Tests request parsing, model extraction, and routing helpers.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
# ===========================================================================
|
||||
# parse_request_body_json
|
||||
# ===========================================================================
|
||||
|
||||
def test_parse_json_valid_body() -> None:
|
||||
"""Valid JSON body is parsed correctly for chat completions."""
|
||||
from routstr.proxy import parse_request_body_json
|
||||
|
||||
body = json.dumps({"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]}).encode()
|
||||
result = parse_request_body_json(body, "/v1/chat/completions")
|
||||
|
||||
assert result["model"] == "gpt-4"
|
||||
assert result["messages"][0]["role"] == "user"
|
||||
|
||||
|
||||
def test_parse_json_invalid_raises_400() -> None:
|
||||
"""Invalid JSON raises HTTPException 400."""
|
||||
from routstr.proxy import parse_request_body_json
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
parse_request_body_json(b"not json", "/v1/chat/completions")
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
def test_parse_json_empty_body() -> None:
|
||||
"""Empty body returns empty dict."""
|
||||
from routstr.proxy import parse_request_body_json
|
||||
|
||||
result = parse_request_body_json(b"", "/v1/chat/completions")
|
||||
assert isinstance(result, dict)
|
||||
assert result == {}
|
||||
|
||||
|
||||
def test_parse_json_responses_path() -> None:
|
||||
"""Responses API path is handled."""
|
||||
from routstr.proxy import parse_request_body_json
|
||||
|
||||
body = json.dumps({"model": "gpt-4", "input": "hello"}).encode()
|
||||
result = parse_request_body_json(body, "/v1/responses")
|
||||
|
||||
assert "model" in result
|
||||
|
||||
|
||||
def test_parse_json_rejects_non_integer_max_tokens() -> None:
|
||||
"""max_tokens must be an integer."""
|
||||
from routstr.proxy import parse_request_body_json
|
||||
|
||||
body = json.dumps({"model": "gpt-4", "max_tokens": "abc"}).encode()
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
parse_request_body_json(body, "/v1/chat/completions")
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# extract_model_from_responses_request
|
||||
# ===========================================================================
|
||||
|
||||
def test_extract_model_from_responses() -> None:
|
||||
"""Model name is extracted from Responses API request."""
|
||||
from routstr.proxy import extract_model_from_responses_request
|
||||
|
||||
body = {"model": "gpt-4o", "input": "test"}
|
||||
model = extract_model_from_responses_request(body)
|
||||
assert model == "gpt-4o"
|
||||
|
||||
|
||||
def test_extract_model_returns_unknown_for_missing() -> None:
|
||||
"""Missing model field returns 'unknown'."""
|
||||
from routstr.proxy import extract_model_from_responses_request
|
||||
|
||||
body = {"input": "test"}
|
||||
model = extract_model_from_responses_request(body)
|
||||
assert model == "unknown"
|
||||
|
||||
|
||||
def test_extract_model_empty_body_returns_unknown() -> None:
|
||||
"""Empty body returns 'unknown'."""
|
||||
from routstr.proxy import extract_model_from_responses_request
|
||||
|
||||
model = extract_model_from_responses_request({})
|
||||
assert model == "unknown"
|
||||
|
||||
|
||||
def test_extract_model_from_input_nested() -> None:
|
||||
"""Model nested in input dict is found."""
|
||||
from routstr.proxy import extract_model_from_responses_request
|
||||
|
||||
body = {"input": {"model": "claude-sonnet", "text": "hi"}}
|
||||
model = extract_model_from_responses_request(body)
|
||||
# The function checks input_data.get("model") for nested
|
||||
assert model in ("claude-sonnet", "unknown")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# get_model_instance / get_provider_for_model / get_unique_models
|
||||
# ===========================================================================
|
||||
|
||||
def test_get_model_instance_unknown_returns_none() -> None:
|
||||
"""Unknown model ID returns None."""
|
||||
from routstr.proxy import get_model_instance
|
||||
|
||||
result = get_model_instance("nonexistent-model-xyz-12345")
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_get_provider_for_model_unknown_returns_none() -> None:
|
||||
"""Unknown model returns None."""
|
||||
from routstr.proxy import get_provider_for_model
|
||||
|
||||
result = get_provider_for_model("nonexistent-model-xyz-12345")
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_get_unique_models_returns_list() -> None:
|
||||
"""get_unique_models always returns a list."""
|
||||
from routstr.proxy import get_unique_models
|
||||
|
||||
result = get_unique_models()
|
||||
assert isinstance(result, list)
|
||||
|
||||
|
||||
def test_get_upstreams_returns_list() -> None:
|
||||
"""get_upstreams returns a list of providers."""
|
||||
from routstr.proxy import get_upstreams
|
||||
|
||||
result = get_upstreams()
|
||||
assert isinstance(result, list)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# parse_request_body_json — nested objects
|
||||
# ===========================================================================
|
||||
|
||||
def test_parse_body_preserves_nested_objects() -> None:
|
||||
"""Nested JSON objects are preserved during parsing."""
|
||||
from routstr.proxy import parse_request_body_json
|
||||
|
||||
body = json.dumps({
|
||||
"model": "claude-3",
|
||||
"messages": [{"role": "system", "content": "You are helpful."}],
|
||||
"temperature": 0.7,
|
||||
"max_tokens": 1024,
|
||||
}).encode()
|
||||
|
||||
result = parse_request_body_json(body, "/v1/chat/completions")
|
||||
assert result["temperature"] == 0.7
|
||||
assert result["max_tokens"] == 1024
|
||||
assert len(result["messages"]) == 1
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import AsyncGenerator
|
||||
from typing import Any, AsyncGenerator
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
@@ -8,7 +9,8 @@ from sqlalchemy.pool import StaticPool
|
||||
from sqlmodel import SQLModel, select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.db import ApiKey
|
||||
from routstr.auth import get_reservation_snapshot, pay_for_request
|
||||
from routstr.core.db import ApiKey, ReservationRelease
|
||||
from routstr.upstream.ehbp import (
|
||||
finalize_ehbp_actual_cost_payment,
|
||||
finalize_ehbp_max_cost_payment,
|
||||
@@ -43,18 +45,38 @@ async def _api_key(session: AsyncSession, hashed_key: str) -> ApiKey | None:
|
||||
).one_or_none()
|
||||
|
||||
|
||||
def _fail_nth_api_key_update(
|
||||
session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
target_update: int,
|
||||
) -> None:
|
||||
"""Return rowcount=0 for one API-key UPDATE without mutating the database."""
|
||||
original_exec = session.exec
|
||||
api_key_updates = 0
|
||||
|
||||
async def exec_with_failure(
|
||||
statement: Any, *args: Any, **kwargs: Any
|
||||
) -> Any:
|
||||
nonlocal api_key_updates
|
||||
table = getattr(statement, "table", None)
|
||||
if getattr(table, "name", None) == "api_keys":
|
||||
api_key_updates += 1
|
||||
if api_key_updates == target_update:
|
||||
return MagicMock(rowcount=0)
|
||||
return await original_exec(statement, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(session, "exec", exec_with_failure)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_actual_cost_payment_updates_balance_and_releases_reserve(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
key = ApiKey(
|
||||
hashed_key="ehbp-actual",
|
||||
balance=10_000,
|
||||
reserved_balance=3_000,
|
||||
reserved_at=123,
|
||||
)
|
||||
key = ApiKey(hashed_key="ehbp-actual", balance=10_000)
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
await pay_for_request(key, 3_000, session)
|
||||
reservation = await get_reservation_snapshot(key, session)
|
||||
|
||||
await finalize_ehbp_actual_cost_payment(
|
||||
key,
|
||||
@@ -68,6 +90,7 @@ async def test_finalize_actual_cost_payment_updates_balance_and_releases_reserve
|
||||
"input_msats": 500,
|
||||
"output_msats": 700,
|
||||
},
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
updated = await _api_key(session, "ehbp-actual")
|
||||
@@ -82,28 +105,22 @@ async def test_finalize_actual_cost_payment_updates_balance_and_releases_reserve
|
||||
async def test_finalize_max_cost_payment_updates_parent_and_child_spend(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
parent = ApiKey(
|
||||
hashed_key="ehbp-parent",
|
||||
balance=10_000,
|
||||
reserved_balance=3_000,
|
||||
reserved_at=123,
|
||||
)
|
||||
parent = ApiKey(hashed_key="ehbp-parent", balance=10_000)
|
||||
child = ApiKey(
|
||||
hashed_key="ehbp-child",
|
||||
balance=0,
|
||||
reserved_balance=3_000,
|
||||
reserved_at=123,
|
||||
parent_key_hash="ehbp-parent",
|
||||
hashed_key="ehbp-child", balance=0, parent_key_hash="ehbp-parent"
|
||||
)
|
||||
session.add(parent)
|
||||
session.add(child)
|
||||
await session.commit()
|
||||
await pay_for_request(child, 3_000, session)
|
||||
reservation = await get_reservation_snapshot(child, session)
|
||||
|
||||
await finalize_ehbp_max_cost_payment(
|
||||
child,
|
||||
session,
|
||||
max_cost_for_model=3_000,
|
||||
model_id="tinfoil/model",
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
updated_parent = await _api_key(session, "ehbp-parent")
|
||||
@@ -123,17 +140,16 @@ async def test_finalize_max_cost_payment_updates_parent_and_child_spend(
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_actual_cost_payment_rolls_back_when_parent_update_matches_no_rows(
|
||||
session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
key = ApiKey(
|
||||
hashed_key="ehbp-missing-parent",
|
||||
balance=10_000,
|
||||
reserved_balance=3_000,
|
||||
reserved_at=123,
|
||||
)
|
||||
key = ApiKey(hashed_key="ehbp-missing-parent", balance=10_000)
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
await session.delete(key)
|
||||
await session.commit()
|
||||
await pay_for_request(key, 3_000, session)
|
||||
reservation = await get_reservation_snapshot(key, session)
|
||||
_fail_nth_api_key_update(session, monkeypatch, target_update=1)
|
||||
rollback_spy = AsyncMock(wraps=session.rollback)
|
||||
monkeypatch.setattr(session, "rollback", rollback_spy)
|
||||
|
||||
await finalize_ehbp_actual_cost_payment(
|
||||
key,
|
||||
@@ -141,45 +157,52 @@ async def test_finalize_actual_cost_payment_rolls_back_when_parent_update_matche
|
||||
reserved_cost_for_model=3_000,
|
||||
model_id="tinfoil/model",
|
||||
cost_info={"total_msats": 1_200},
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
assert await _api_key(session, "ehbp-missing-parent") is None
|
||||
rollback_spy.assert_awaited_once()
|
||||
updated = await _api_key(session, "ehbp-missing-parent")
|
||||
assert updated is not None
|
||||
assert updated.balance == 10_000
|
||||
assert updated.reserved_balance == 3_000
|
||||
assert updated.total_spent == 0
|
||||
release = await session.get(ReservationRelease, reservation.release_id)
|
||||
assert release is not None
|
||||
assert release.status == "active"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_finalize_max_cost_payment_rolls_back_parent_when_child_update_matches_no_rows(
|
||||
session: AsyncSession,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
parent = ApiKey(
|
||||
hashed_key="ehbp-rollback-parent",
|
||||
balance=10_000,
|
||||
reserved_balance=3_000,
|
||||
reserved_at=123,
|
||||
)
|
||||
parent = ApiKey(hashed_key="ehbp-rollback-parent", balance=10_000)
|
||||
child = ApiKey(
|
||||
hashed_key="ehbp-missing-child",
|
||||
balance=0,
|
||||
reserved_balance=3_000,
|
||||
reserved_at=123,
|
||||
parent_key_hash="ehbp-rollback-parent",
|
||||
)
|
||||
session.add(parent)
|
||||
session.add(child)
|
||||
await session.commit()
|
||||
await session.delete(child)
|
||||
await session.commit()
|
||||
await pay_for_request(child, 3_000, session)
|
||||
reservation = await get_reservation_snapshot(child, session)
|
||||
_fail_nth_api_key_update(session, monkeypatch, target_update=2)
|
||||
|
||||
await finalize_ehbp_max_cost_payment(
|
||||
child,
|
||||
session,
|
||||
max_cost_for_model=3_000,
|
||||
model_id="tinfoil/model",
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
updated_parent = await _api_key(session, "ehbp-rollback-parent")
|
||||
assert updated_parent is not None
|
||||
assert updated_parent.balance == 10_000
|
||||
assert updated_parent.reserved_balance == 3_000
|
||||
assert updated_parent.reserved_at == 123
|
||||
assert updated_parent.total_spent == 0
|
||||
assert await _api_key(session, "ehbp-missing-child") is None
|
||||
updated_child = await _api_key(session, "ehbp-missing-child")
|
||||
assert updated_child is not None
|
||||
assert updated_child.reserved_balance == 3_000
|
||||
assert updated_child.total_spent == 0
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlmodel import SQLModel
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr import wallet
|
||||
from routstr.core import db
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _session_context(session: Mock) -> AsyncIterator[Mock]:
|
||||
yield session
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fee_payout_checkpoint_is_atomic_and_durable() -> None:
|
||||
engine = create_async_engine("sqlite+aiosqlite://")
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(SQLModel.metadata.create_all)
|
||||
|
||||
async with AsyncSession(engine) as session:
|
||||
session.add(db.RoutstrFee(id=1, accumulated_msats=5_000))
|
||||
await session.commit()
|
||||
|
||||
assert await db.reset_routstr_fee(session, 5_000) is True
|
||||
assert await db.reset_routstr_fee(session, 5_000) is False
|
||||
|
||||
fee = await db.get_routstr_fee(session)
|
||||
await session.refresh(fee)
|
||||
assert fee.accumulated_msats == 0
|
||||
assert fee.payout_in_progress_msats == 5_000
|
||||
assert fee.total_paid_msats == 0
|
||||
|
||||
assert await db.complete_routstr_fee_payout(session, 5_000) is True
|
||||
await session.refresh(fee)
|
||||
assert fee.payout_in_progress_msats == 0
|
||||
assert fee.total_paid_msats == 5_000
|
||||
assert fee.last_paid_at is not None
|
||||
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fee_payout_checkpoints_before_sending() -> None:
|
||||
session = Mock()
|
||||
fee = SimpleNamespace(
|
||||
accumulated_msats=5_000,
|
||||
payout_in_progress_msats=0,
|
||||
payout_started_at=None,
|
||||
)
|
||||
payout_wallet = Mock()
|
||||
events: list[str] = []
|
||||
|
||||
async def checkpoint(*_args: object) -> bool:
|
||||
events.append("checkpoint")
|
||||
return True
|
||||
|
||||
async def send(*_args: object, **_kwargs: object) -> int:
|
||||
events.append("send")
|
||||
return 5
|
||||
|
||||
async def complete(*_args: object) -> bool:
|
||||
events.append("complete")
|
||||
return True
|
||||
|
||||
with (
|
||||
patch("routstr.auth.ROUTSTR_FEE_DEFAULT_PAYOUT", 1),
|
||||
patch("routstr.auth.ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS", 1),
|
||||
patch("routstr.auth.ROUTSTR_LN_ADDRESS", "fees@example.com"),
|
||||
patch(
|
||||
"routstr.wallet.asyncio.sleep",
|
||||
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
|
||||
),
|
||||
patch("routstr.wallet.db.create_session", return_value=_session_context(session)),
|
||||
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
|
||||
patch("routstr.wallet.db.reset_routstr_fee", side_effect=checkpoint),
|
||||
patch("routstr.wallet.db.complete_routstr_fee_payout", side_effect=complete),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=payout_wallet)),
|
||||
patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[]),
|
||||
patch("routstr.wallet.raw_send_to_lnurl", side_effect=send),
|
||||
):
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await wallet.periodic_routstr_fee_payout()
|
||||
|
||||
assert events == ["checkpoint", "send", "complete"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fee_payout_does_not_retry_an_unresolved_checkpoint() -> None:
|
||||
session = Mock()
|
||||
fee = SimpleNamespace(
|
||||
accumulated_msats=10_000,
|
||||
payout_in_progress_msats=5_000,
|
||||
payout_started_at=123,
|
||||
)
|
||||
|
||||
with (
|
||||
patch("routstr.auth.ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS", 1),
|
||||
patch("routstr.auth.ROUTSTR_LN_ADDRESS", "fees@example.com"),
|
||||
patch(
|
||||
"routstr.wallet.asyncio.sleep",
|
||||
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
|
||||
),
|
||||
patch("routstr.wallet.db.create_session", return_value=_session_context(session)),
|
||||
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
|
||||
patch("routstr.wallet.db.reset_routstr_fee", AsyncMock()) as checkpoint,
|
||||
patch("routstr.wallet.get_wallet", AsyncMock()) as get_wallet,
|
||||
patch("routstr.wallet.raw_send_to_lnurl", AsyncMock()) as send,
|
||||
patch("routstr.wallet.logger.critical") as critical,
|
||||
):
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await wallet.periodic_routstr_fee_payout()
|
||||
|
||||
checkpoint.assert_not_awaited()
|
||||
get_wallet.assert_not_awaited()
|
||||
send.assert_not_awaited()
|
||||
critical.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fee_payout_keeps_checkpoint_when_send_outcome_is_unknown() -> None:
|
||||
session = Mock()
|
||||
fee = SimpleNamespace(
|
||||
accumulated_msats=5_000,
|
||||
payout_in_progress_msats=0,
|
||||
payout_started_at=None,
|
||||
)
|
||||
complete = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("routstr.auth.ROUTSTR_FEE_DEFAULT_PAYOUT", 1),
|
||||
patch("routstr.auth.ROUTSTR_FEE_PAYOUT_INTERVAL_SECONDS", 1),
|
||||
patch("routstr.auth.ROUTSTR_LN_ADDRESS", "fees@example.com"),
|
||||
patch(
|
||||
"routstr.wallet.asyncio.sleep",
|
||||
AsyncMock(side_effect=[None, asyncio.CancelledError()]),
|
||||
),
|
||||
patch("routstr.wallet.db.create_session", return_value=_session_context(session)),
|
||||
patch("routstr.wallet.db.get_routstr_fee", AsyncMock(return_value=fee)),
|
||||
patch("routstr.wallet.db.reset_routstr_fee", AsyncMock(return_value=True)),
|
||||
patch("routstr.wallet.db.complete_routstr_fee_payout", complete),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=Mock())),
|
||||
patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[]),
|
||||
patch(
|
||||
"routstr.wallet.raw_send_to_lnurl",
|
||||
AsyncMock(side_effect=TimeoutError("unknown outcome")),
|
||||
),
|
||||
patch("routstr.wallet.logger.critical") as critical,
|
||||
):
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await wallet.periodic_routstr_fee_payout()
|
||||
|
||||
complete.assert_not_awaited()
|
||||
critical.assert_called_once()
|
||||
@@ -0,0 +1,116 @@
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from alembic.config import Config
|
||||
from alembic.script import ScriptDirectory
|
||||
|
||||
|
||||
def _run_alembic(root: Path, database_url: str, revision: str) -> None:
|
||||
env = os.environ.copy()
|
||||
env["DATABASE_URL"] = database_url
|
||||
subprocess.run(
|
||||
[sys.executable, "-m", "alembic", "upgrade", revision],
|
||||
cwd=root,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def test_fresh_node_migrates_fee_payout_schema_to_head(tmp_path: Path) -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
database_path = tmp_path / "fresh-node.db"
|
||||
database_url = f"sqlite+aiosqlite:///{database_path}"
|
||||
|
||||
_run_alembic(root, database_url, "head")
|
||||
|
||||
with sqlite3.connect(database_path) as connection:
|
||||
version = connection.execute(
|
||||
"SELECT version_num FROM alembic_version"
|
||||
).fetchone()
|
||||
columns = {
|
||||
row[1] for row in connection.execute("PRAGMA table_info(routstr_fees)")
|
||||
}
|
||||
fee = connection.execute(
|
||||
"SELECT id, accumulated_msats, total_paid_msats, last_paid_at, "
|
||||
"payout_in_progress_msats, payout_started_at FROM routstr_fees"
|
||||
).fetchone()
|
||||
|
||||
migration_config = Config(str(root / "alembic.ini"))
|
||||
assert version == (
|
||||
ScriptDirectory.from_config(migration_config).get_current_head(),
|
||||
)
|
||||
assert {
|
||||
"id",
|
||||
"accumulated_msats",
|
||||
"total_paid_msats",
|
||||
"last_paid_at",
|
||||
"payout_in_progress_msats",
|
||||
"payout_started_at",
|
||||
} <= columns
|
||||
assert fee == (1, 0, 0, None, 0, None)
|
||||
|
||||
|
||||
def test_fee_payout_checkpoint_migration_preserves_existing_row(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
database_path = tmp_path / "migration.db"
|
||||
database_url = f"sqlite+aiosqlite:///{database_path}"
|
||||
_run_alembic(root, database_url, "c6d7e8f9a0b1")
|
||||
|
||||
with sqlite3.connect(database_path) as connection:
|
||||
result = connection.execute(
|
||||
"UPDATE routstr_fees SET accumulated_msats = 5000, "
|
||||
"total_paid_msats = 1000, last_paid_at = 123 WHERE id = 1"
|
||||
)
|
||||
assert result.rowcount == 1
|
||||
connection.commit()
|
||||
|
||||
_run_alembic(root, database_url, "head")
|
||||
|
||||
with sqlite3.connect(database_path) as connection:
|
||||
row = connection.execute(
|
||||
"SELECT accumulated_msats, total_paid_msats, last_paid_at, "
|
||||
"payout_in_progress_msats, payout_started_at "
|
||||
"FROM routstr_fees WHERE id = 1"
|
||||
).fetchone()
|
||||
|
||||
assert row == (5000, 1000, 123, 0, None)
|
||||
|
||||
|
||||
def test_fee_payout_checkpoint_repair_restores_columns_missing_at_old_head(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
database_path = tmp_path / "migration.db"
|
||||
database_url = f"sqlite+aiosqlite:///{database_path}"
|
||||
old_head = "7f2843d3f4e4"
|
||||
_run_alembic(root, database_url, old_head)
|
||||
|
||||
# Reproduce a database that was stamped to head after a duplicate-column or
|
||||
# unknown-revision recovery skipped part of the migration chain.
|
||||
with sqlite3.connect(database_path) as connection:
|
||||
connection.execute("ALTER TABLE routstr_fees DROP COLUMN payout_started_at")
|
||||
connection.execute(
|
||||
"ALTER TABLE routstr_fees DROP COLUMN payout_in_progress_msats"
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
_run_alembic(root, database_url, "head")
|
||||
|
||||
with sqlite3.connect(database_path) as connection:
|
||||
columns = {
|
||||
row[1] for row in connection.execute("PRAGMA table_info(routstr_fees)")
|
||||
}
|
||||
row = connection.execute(
|
||||
"SELECT payout_in_progress_msats, payout_started_at "
|
||||
"FROM routstr_fees WHERE id = 1"
|
||||
).fetchone()
|
||||
|
||||
assert {"payout_in_progress_msats", "payout_started_at"} <= columns
|
||||
assert row == (0, None)
|
||||
@@ -11,7 +11,9 @@ async def _fake_session(): # type: ignore[no-untyped-def]
|
||||
yield MagicMock()
|
||||
|
||||
|
||||
def _patches(proof_amount: int = 1000): # type: ignore[no-untyped-def]
|
||||
def _patches( # type: ignore[no-untyped-def]
|
||||
proof_amount: int = 1000, user_balance_msats: int = 0
|
||||
):
|
||||
proof = MagicMock(amount=proof_amount)
|
||||
return [
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=MagicMock())),
|
||||
@@ -25,7 +27,7 @@ def _patches(proof_amount: int = 1000): # type: ignore[no-untyped-def]
|
||||
),
|
||||
patch(
|
||||
"routstr.wallet.db.balances_for_mint_and_unit",
|
||||
AsyncMock(return_value=0),
|
||||
AsyncMock(return_value=user_balance_msats),
|
||||
),
|
||||
patch("routstr.wallet.db.create_session", _fake_session),
|
||||
]
|
||||
@@ -52,6 +54,31 @@ async def test_fetch_all_balances_falls_back_to_primary_mint() -> None:
|
||||
assert total_wallet == 1000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_reports_liability_when_wallet_is_empty() -> None:
|
||||
"""An empty wallet must not hide outstanding user liabilities."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
with patch.object(settings, "cashu_mints", []), patch.object(
|
||||
settings, "primary_mint", "http://primary:3338"
|
||||
):
|
||||
for p in _patches(proof_amount=0, user_balance_msats=5000):
|
||||
p.start()
|
||||
try:
|
||||
details, total_wallet, total_user, owner = await fetch_all_balances(
|
||||
units=["sat"]
|
||||
)
|
||||
finally:
|
||||
patch.stopall()
|
||||
|
||||
assert details[0]["wallet_balance"] == 0
|
||||
assert details[0]["user_balance"] == 5
|
||||
assert details[0]["owner_balance"] == -5
|
||||
assert total_wallet == 0
|
||||
assert total_user == 5
|
||||
assert owner == -5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_all_balances_no_duplicate_primary_mint() -> None:
|
||||
"""primary_mint already in cashu_mints is not inspected twice."""
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
"""raw_send_to_lnurl() must not hang forever on an unresponsive mint.
|
||||
|
||||
The Cashu library issues POST /v1/melt/bolt11 with timeout=None, so a hung
|
||||
mint would block the melt (and the payout loop) indefinitely. raw_send_to_lnurl
|
||||
now wraps wallet.melt() in asyncio.wait_for(MELT_TIMEOUT_SECONDS) and surfaces a
|
||||
timeout as LNURLError instead of hanging.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.payment import lnurl
|
||||
from routstr.payment.lnurl import LNURLError, raw_send_to_lnurl
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_send_to_lnurl_times_out_on_hung_melt() -> None:
|
||||
proofs = [MagicMock(amount=1000)]
|
||||
|
||||
wallet = MagicMock()
|
||||
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
|
||||
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
|
||||
|
||||
async def _hang(**kwargs: object) -> None:
|
||||
await asyncio.sleep(5) # far longer than the patched timeout
|
||||
|
||||
wallet.melt = AsyncMock(side_effect=_hang)
|
||||
|
||||
lnurl_data = {
|
||||
"callback_url": "https://ln.tld/cb",
|
||||
"min_sendable": 1_000,
|
||||
"max_sendable": 100_000_000,
|
||||
}
|
||||
|
||||
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 0.05), patch(
|
||||
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
|
||||
), patch(
|
||||
"routstr.payment.lnurl.get_lnurl_invoice",
|
||||
AsyncMock(return_value=("lnbc1...", {})),
|
||||
):
|
||||
with pytest.raises(LNURLError, match="Melt timed out"):
|
||||
await raw_send_to_lnurl(wallet, proofs, "owner@ln.tld", "sat", amount=1000)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_send_to_lnurl_succeeds_within_timeout() -> None:
|
||||
"""A prompt melt still returns the net amount, unaffected by the guard."""
|
||||
proofs = [MagicMock(amount=1000)]
|
||||
|
||||
wallet = MagicMock()
|
||||
wallet.melt_quote = AsyncMock(return_value=MagicMock(fee_reserve=1, quote="q"))
|
||||
wallet.select_to_send = AsyncMock(return_value=(proofs, None))
|
||||
wallet.melt = AsyncMock(return_value=MagicMock())
|
||||
|
||||
lnurl_data = {
|
||||
"callback_url": "https://ln.tld/cb",
|
||||
"min_sendable": 1_000,
|
||||
"max_sendable": 100_000_000,
|
||||
}
|
||||
|
||||
with patch.object(lnurl, "MELT_TIMEOUT_SECONDS", 5), patch(
|
||||
"routstr.payment.lnurl.get_lnurl_data", AsyncMock(return_value=lnurl_data)
|
||||
), patch(
|
||||
"routstr.payment.lnurl.get_lnurl_invoice",
|
||||
AsyncMock(return_value=("lnbc1...", {})),
|
||||
):
|
||||
paid = await raw_send_to_lnurl(
|
||||
wallet, proofs, "owner@ln.tld", "sat", amount=1000
|
||||
)
|
||||
|
||||
assert paid > 0
|
||||
wallet.melt.assert_awaited_once()
|
||||
@@ -18,6 +18,7 @@ from fastapi.responses import Response, StreamingResponse
|
||||
os.environ.setdefault("UPSTREAM_BASE_URL", "http://test")
|
||||
os.environ.setdefault("UPSTREAM_API_KEY", "test")
|
||||
|
||||
from routstr.auth import ReservationSnapshot # noqa: E402
|
||||
from routstr.core.db import ApiKey # noqa: E402
|
||||
from routstr.payment.cost_calculation import CostData # noqa: E402
|
||||
from routstr.payment.models import Architecture, Model, Pricing # noqa: E402
|
||||
@@ -498,14 +499,27 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
|
||||
yield {"type": "message_stop"}
|
||||
|
||||
fake_cost = {"total_msats": 4321, "total_usd": 0.00015}
|
||||
reservation = ReservationSnapshot(
|
||||
release_id="messages-stream",
|
||||
key_hash=key.hashed_key,
|
||||
billing_key_hash=key.hashed_key,
|
||||
reserved_msats=10_000,
|
||||
)
|
||||
|
||||
captured_cost_call: dict[str, Any] = {}
|
||||
|
||||
async def fake_adjust(
|
||||
fresh_key: Any, combined_data: Any, sess: Any, max_cost: int, usage: Any = None
|
||||
fresh_key: Any,
|
||||
combined_data: Any,
|
||||
sess: Any,
|
||||
max_cost: int,
|
||||
model_obj: Any = None,
|
||||
provider_fee: Any = None,
|
||||
reservation_snapshot: Any = None,
|
||||
) -> dict:
|
||||
captured_cost_call["combined_data"] = combined_data
|
||||
captured_cost_call["max_cost"] = max_cost
|
||||
captured_cost_call["reservation_snapshot"] = reservation_snapshot
|
||||
return fake_cost
|
||||
|
||||
fake_session = MagicMock()
|
||||
@@ -539,6 +553,7 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
|
||||
session=session,
|
||||
max_cost_for_model=10_000,
|
||||
model_obj=model,
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
assert isinstance(result, StreamingResponse)
|
||||
@@ -561,6 +576,7 @@ async def test_streaming_emits_sse_and_reconciles_cost_at_end() -> None:
|
||||
assert combined["usage"]["input_tokens"] == 5
|
||||
assert combined["usage"]["output_tokens"] == 7
|
||||
assert combined["model"] == "openai/gpt-4o-mini"
|
||||
assert captured_cost_call["reservation_snapshot"] is reservation
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -588,12 +604,25 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
|
||||
yield b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
|
||||
|
||||
fake_cost = {"total_msats": 999, "total_usd": 0.0001}
|
||||
reservation = ReservationSnapshot(
|
||||
release_id="messages-byte-stream",
|
||||
key_hash=key.hashed_key,
|
||||
billing_key_hash=key.hashed_key,
|
||||
reserved_msats=10_000,
|
||||
)
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
async def fake_adjust(
|
||||
fresh_key: Any, combined_data: Any, sess: Any, max_cost: int, usage: Any = None
|
||||
fresh_key: Any,
|
||||
combined_data: Any,
|
||||
sess: Any,
|
||||
max_cost: int,
|
||||
model_obj: Any = None,
|
||||
provider_fee: Any = None,
|
||||
reservation_snapshot: Any = None,
|
||||
) -> dict:
|
||||
captured["combined_data"] = combined_data
|
||||
captured["reservation_snapshot"] = reservation_snapshot
|
||||
return fake_cost
|
||||
|
||||
fake_session = MagicMock()
|
||||
@@ -626,6 +655,7 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
|
||||
session=session,
|
||||
max_cost_for_model=10_000,
|
||||
model_obj=model,
|
||||
reservation_snapshot=reservation,
|
||||
)
|
||||
|
||||
assert isinstance(result, StreamingResponse)
|
||||
@@ -649,6 +679,7 @@ async def test_streaming_handles_iterator_yielding_raw_sse_bytes() -> None:
|
||||
assert combined["usage"]["input_tokens"] == 3
|
||||
assert combined["usage"]["output_tokens"] == 4
|
||||
assert combined["model"] == "openai/gpt-4o-mini"
|
||||
assert captured["reservation_snapshot"] is reservation
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,154 @@
|
||||
"""Tests for periodic_payout() resilience fixes.
|
||||
|
||||
Covers two regressions from the auto-payout / primary-mint audit
|
||||
(docs/auto-payout-primary-mint-failure-report.md):
|
||||
|
||||
1. periodic_payout() must include settings.primary_mint even when it is not
|
||||
listed in settings.cashu_mints, matching fetch_all_balances(); otherwise
|
||||
primary-mint funds never auto-payout.
|
||||
2. A failure on one mint/unit must not abort payout for the remaining
|
||||
mint/units in the same cycle (the try/except is now per mint/unit).
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Coroutine
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.wallet import periodic_payout
|
||||
|
||||
# Sentinel interval used to break the otherwise-infinite payout loop after
|
||||
# exactly one full cycle.
|
||||
_INTERVAL = 987
|
||||
|
||||
|
||||
class _LoopBreak(Exception):
|
||||
"""Raised via the patched sleep to stop periodic_payout after one cycle."""
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _fake_session(): # type: ignore[no-untyped-def]
|
||||
yield MagicMock()
|
||||
|
||||
|
||||
def _one_cycle_sleep() -> Callable[[float], Coroutine[Any, Any, None]]:
|
||||
"""Return an async sleep stub that lets exactly one payout cycle run.
|
||||
|
||||
The top-of-loop sleep uses the sentinel interval; the second time it is
|
||||
seen (start of the second cycle) we raise to break out. The inner
|
||||
``asyncio.sleep(5)`` pass-through is ignored.
|
||||
"""
|
||||
seen = {"interval": 0}
|
||||
|
||||
async def _sleep(seconds: float) -> None:
|
||||
if seconds == _INTERVAL:
|
||||
seen["interval"] += 1
|
||||
if seen["interval"] >= 2:
|
||||
raise _LoopBreak()
|
||||
|
||||
return _sleep
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_periodic_payout_includes_primary_mint_not_in_cashu_mints() -> None:
|
||||
"""primary_mint absent from cashu_mints is still paid out."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
get_wallet = AsyncMock(return_value=MagicMock())
|
||||
raw_send = AsyncMock(return_value=1000)
|
||||
|
||||
with patch.object(settings, "cashu_mints", []), patch.object(
|
||||
settings, "primary_mint", "http://primary:3338"
|
||||
), patch.object(settings, "receive_ln_address", "owner@ln.tld"), patch.object(
|
||||
settings, "payout_interval_seconds", _INTERVAL
|
||||
), patch.object(settings, "min_payout_sat", 10), patch(
|
||||
"routstr.wallet.asyncio.sleep", _one_cycle_sleep()
|
||||
), patch("routstr.wallet.db.create_session", _fake_session), patch(
|
||||
"routstr.wallet.get_wallet", get_wallet
|
||||
), patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(return_value=[MagicMock(amount=100_000)]),
|
||||
), patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda proofs, wallet: proofs),
|
||||
), patch(
|
||||
"routstr.wallet.db.balances_for_mint_and_unit", AsyncMock(return_value=0)
|
||||
), patch("routstr.wallet.raw_send_to_lnurl", raw_send):
|
||||
with pytest.raises(_LoopBreak):
|
||||
await periodic_payout()
|
||||
|
||||
processed = {call.args[0] for call in get_wallet.await_args_list}
|
||||
assert processed == {"http://primary:3338"}
|
||||
assert raw_send.await_count >= 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_periodic_payout_isolates_failing_mint() -> None:
|
||||
"""A failing mint does not prevent payout for the other mints."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
async def _get_wallet(mint_url: str, unit: str) -> MagicMock:
|
||||
if mint_url == "http://bad:3338":
|
||||
raise RuntimeError("mint unreachable")
|
||||
return MagicMock()
|
||||
|
||||
get_wallet = AsyncMock(side_effect=_get_wallet)
|
||||
raw_send = AsyncMock(return_value=1000)
|
||||
|
||||
with patch.object(
|
||||
settings, "cashu_mints", ["http://bad:3338", "http://good:3338"]
|
||||
), patch.object(settings, "primary_mint", "http://good:3338"), patch.object(
|
||||
settings, "receive_ln_address", "owner@ln.tld"
|
||||
), patch.object(settings, "payout_interval_seconds", _INTERVAL), patch.object(
|
||||
settings, "min_payout_sat", 10
|
||||
), patch("routstr.wallet.asyncio.sleep", _one_cycle_sleep()), patch(
|
||||
"routstr.wallet.db.create_session", _fake_session
|
||||
), patch("routstr.wallet.get_wallet", get_wallet), patch(
|
||||
"routstr.wallet.get_proofs_per_mint_and_unit",
|
||||
MagicMock(return_value=[MagicMock(amount=100_000)]),
|
||||
), patch(
|
||||
"routstr.wallet.slow_filter_spend_proofs",
|
||||
AsyncMock(side_effect=lambda proofs, wallet: proofs),
|
||||
), patch(
|
||||
"routstr.wallet.db.balances_for_mint_and_unit", AsyncMock(return_value=0)
|
||||
), patch("routstr.wallet.raw_send_to_lnurl", raw_send):
|
||||
with pytest.raises(_LoopBreak):
|
||||
await periodic_payout()
|
||||
|
||||
# The bad mint raised on get_wallet for both units, yet the good mint was
|
||||
# still reached and paid out for both units — failures are isolated.
|
||||
good_calls = [
|
||||
c for c in get_wallet.await_args_list if c.args[0] == "http://good:3338"
|
||||
]
|
||||
assert len(good_calls) == 2 # sat + msat
|
||||
assert raw_send.await_count == 2 # good mint paid for both units
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_periodic_payout_handles_session_creation_failure() -> None:
|
||||
"""A db.create_session failure is logged and the payout loop continues."""
|
||||
from routstr.core.settings import settings
|
||||
|
||||
create_session = MagicMock(side_effect=RuntimeError("db unavailable"))
|
||||
logger = MagicMock()
|
||||
|
||||
with patch.object(settings, "cashu_mints", ["http://mint:3338"]), patch.object(
|
||||
settings, "primary_mint", "http://mint:3338"
|
||||
), patch.object(settings, "receive_ln_address", "owner@ln.tld"), patch.object(
|
||||
settings, "payout_interval_seconds", _INTERVAL
|
||||
), patch(
|
||||
"routstr.wallet.asyncio.sleep", _one_cycle_sleep()
|
||||
), patch(
|
||||
"routstr.wallet.db.create_session", create_session
|
||||
), patch("routstr.wallet.logger", logger):
|
||||
with pytest.raises(_LoopBreak):
|
||||
await periodic_payout()
|
||||
|
||||
create_session.assert_called_once()
|
||||
logger.error.assert_called_once()
|
||||
message = logger.error.call_args.args[0]
|
||||
extra = logger.error.call_args.kwargs["extra"]
|
||||
assert message == "Error in periodic payout cycle: RuntimeError"
|
||||
assert extra == {"error": "db unavailable"}
|
||||
@@ -0,0 +1,126 @@
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import AsyncGenerator
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
from sqlmodel import SQLModel, select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr import wallet
|
||||
from routstr.core.db import CashuTransaction
|
||||
|
||||
|
||||
def _make_engine() -> AsyncEngine:
|
||||
return create_async_engine(
|
||||
"sqlite+aiosqlite://",
|
||||
poolclass=StaticPool,
|
||||
connect_args={"check_same_thread": False},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def refund_db(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> AsyncGenerator[AsyncEngine, None]:
|
||||
engine = _make_engine()
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(SQLModel.metadata.create_all)
|
||||
|
||||
@asynccontextmanager
|
||||
async def create_session() -> AsyncGenerator[AsyncSession, None]:
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
yield session
|
||||
|
||||
monkeypatch.setattr(wallet.db, "create_session", create_session)
|
||||
try:
|
||||
yield engine
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _store(
|
||||
engine: AsyncEngine,
|
||||
*,
|
||||
token: str,
|
||||
created_at: int,
|
||||
typ: str = "out",
|
||||
collected: bool = False,
|
||||
swept: bool = False,
|
||||
) -> None:
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add(
|
||||
CashuTransaction(
|
||||
token=token,
|
||||
amount=10,
|
||||
unit="sat",
|
||||
type=typ,
|
||||
created_at=created_at,
|
||||
collected=collected,
|
||||
swept=swept,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def _transactions(engine: AsyncEngine) -> dict[str, CashuTransaction]:
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
results = await session.exec(select(CashuTransaction))
|
||||
return {transaction.token: transaction for transaction in results.all()}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refund_sweep_only_processes_eligible_outbound_transactions(
|
||||
refund_db: AsyncEngine, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
cutoff = 1_000
|
||||
await _store(refund_db, token="eligible", created_at=cutoff - 1)
|
||||
await _store(refund_db, token="too-new", created_at=cutoff)
|
||||
await _store(
|
||||
refund_db, token="collected", created_at=cutoff - 1, collected=True
|
||||
)
|
||||
await _store(refund_db, token="swept", created_at=cutoff - 1, swept=True)
|
||||
await _store(refund_db, token="inbound", created_at=cutoff - 1, typ="in")
|
||||
receive_token = AsyncMock()
|
||||
monkeypatch.setattr(wallet, "recieve_token", receive_token)
|
||||
|
||||
await wallet._refund_sweep_once(cutoff)
|
||||
|
||||
receive_token.assert_awaited_once_with("eligible")
|
||||
transactions = await _transactions(refund_db)
|
||||
assert transactions["eligible"].swept is True
|
||||
assert transactions["too-new"].swept is False
|
||||
assert transactions["collected"].collected is True
|
||||
assert transactions["collected"].swept is False
|
||||
assert transactions["swept"].swept is True
|
||||
assert transactions["inbound"].swept is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refund_sweep_persists_success_and_isolates_token_failures(
|
||||
refund_db: AsyncEngine, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
cutoff = 1_000
|
||||
for token in ("success", "already-spent", "temporary-failure"):
|
||||
await _store(refund_db, token=token, created_at=cutoff - 1)
|
||||
|
||||
async def receive_token(token: str) -> None:
|
||||
if token == "already-spent":
|
||||
raise RuntimeError("Token already spent")
|
||||
if token == "temporary-failure":
|
||||
raise RuntimeError("mint unavailable")
|
||||
|
||||
receive = AsyncMock(side_effect=receive_token)
|
||||
monkeypatch.setattr(wallet, "recieve_token", receive)
|
||||
|
||||
await wallet._refund_sweep_once(cutoff)
|
||||
|
||||
assert receive.await_count == 3
|
||||
transactions = await _transactions(refund_db)
|
||||
assert transactions["success"].swept is True
|
||||
assert transactions["success"].collected is False
|
||||
assert transactions["already-spent"].collected is True
|
||||
assert transactions["already-spent"].swept is False
|
||||
assert transactions["temporary-failure"].collected is False
|
||||
assert transactions["temporary-failure"].swept is False
|
||||
@@ -38,7 +38,8 @@ async def test_attestation_get_routes_directly_to_tinfoil_provider(
|
||||
monkeypatch.setattr(proxy_module, "_upstreams", [non_tinfoil, tinfoil])
|
||||
|
||||
async with AsyncClient(
|
||||
transport=ASGITransport(app=proxy_app), base_url="http://test" # type: ignore[arg-type]
|
||||
transport=ASGITransport(app=proxy_app), # type: ignore[arg-type]
|
||||
base_url="http://test",
|
||||
) as client:
|
||||
response = await client.get("/attestation")
|
||||
|
||||
@@ -69,7 +70,8 @@ async def test_tee_attestation_get_routes_directly_to_tinfoil_provider(
|
||||
monkeypatch.setattr(proxy_module, "_upstreams", [non_tinfoil, tinfoil])
|
||||
|
||||
async with AsyncClient(
|
||||
transport=ASGITransport(app=proxy_app), base_url="http://test" # type: ignore[arg-type]
|
||||
transport=ASGITransport(app=proxy_app), # type: ignore[arg-type]
|
||||
base_url="http://test",
|
||||
) as client:
|
||||
response = await client.get("/tee/attestation")
|
||||
|
||||
@@ -79,6 +81,48 @@ async def test_tee_attestation_get_routes_directly_to_tinfoil_provider(
|
||||
tinfoil.forward_get_request.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["attestation/", "tee/attestation/"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_attestation_trailing_slash_routes_directly_to_tinfoil(
|
||||
monkeypatch: pytest.MonkeyPatch, proxy_app: FastAPI, path: str
|
||||
) -> None:
|
||||
tinfoil = MagicMock()
|
||||
tinfoil.provider_type = "tinfoil"
|
||||
tinfoil.prepare_headers = MagicMock(return_value={})
|
||||
tinfoil.forward_get_request = AsyncMock(return_value=Response(status_code=200))
|
||||
monkeypatch.setattr(proxy_module, "_upstreams", [tinfoil])
|
||||
|
||||
async with AsyncClient(
|
||||
transport=ASGITransport(app=proxy_app), # type: ignore[arg-type]
|
||||
base_url="http://test",
|
||||
) as client:
|
||||
response = await client.get(f"/{path}")
|
||||
|
||||
assert response.status_code == 200
|
||||
tinfoil.forward_get_request.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["attestation/foo", "attestationjunk"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_attestation_prefix_does_not_bypass_authentication(
|
||||
monkeypatch: pytest.MonkeyPatch, proxy_app: FastAPI, path: str
|
||||
) -> None:
|
||||
tinfoil = MagicMock()
|
||||
tinfoil.provider_type = "tinfoil"
|
||||
tinfoil.forward_get_request = AsyncMock()
|
||||
monkeypatch.setattr(proxy_module, "_upstreams", [tinfoil])
|
||||
|
||||
async with AsyncClient(
|
||||
transport=ASGITransport(app=proxy_app), # type: ignore[arg-type]
|
||||
base_url="http://test",
|
||||
) as client:
|
||||
response = await client.get(f"/{path}")
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json()["error"]["type"] == "invalid_model"
|
||||
tinfoil.forward_get_request.assert_not_awaited()
|
||||
|
||||
|
||||
def test_attestation_upstream_selection_is_tinfoil_only() -> None:
|
||||
non_tinfoil = MagicMock(provider_type="openai")
|
||||
tinfoil = MagicMock(provider_type="tinfoil")
|
||||
@@ -90,6 +134,8 @@ def test_attestation_upstream_selection_is_tinfoil_only() -> None:
|
||||
"tee/attestation", [non_tinfoil, tinfoil]
|
||||
) == [tinfoil]
|
||||
assert proxy_module._select_unauthenticated_get_upstreams(
|
||||
"tee/other", [non_tinfoil, tinfoil]
|
||||
"attestation/", [non_tinfoil, tinfoil]
|
||||
) == [tinfoil]
|
||||
assert proxy_module._select_unauthenticated_get_upstreams(
|
||||
"attestationjunk", [non_tinfoil, tinfoil]
|
||||
) == [non_tinfoil, tinfoil]
|
||||
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
from collections.abc import AsyncIterator
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
from sqlmodel import SQLModel, select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.db import CashuTransaction
|
||||
from routstr.wallet import refund_sweep_once
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def session_factory(
|
||||
tmp_path: Path,
|
||||
) -> AsyncIterator[async_sessionmaker[AsyncSession]]:
|
||||
engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'refunds.db'}")
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(SQLModel.metadata.create_all)
|
||||
factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
yield factory
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _insert(
|
||||
factory: async_sessionmaker[AsyncSession], *rows: CashuTransaction
|
||||
) -> None:
|
||||
async with factory() as session:
|
||||
session.add_all(rows)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def _load(
|
||||
factory: async_sessionmaker[AsyncSession],
|
||||
) -> dict[str, CashuTransaction]:
|
||||
async with factory() as session:
|
||||
result = await session.exec(select(CashuTransaction))
|
||||
return {row.token: row for row in result.all()}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refund_sweep_only_processes_expired_eligible_outgoing_tokens(
|
||||
session_factory: async_sessionmaker[AsyncSession],
|
||||
) -> None:
|
||||
rows = [
|
||||
CashuTransaction(
|
||||
token="eligible", amount=1, unit="sat", type="out", created_at=800
|
||||
),
|
||||
CashuTransaction(
|
||||
token="fresh", amount=1, unit="sat", type="out", created_at=950
|
||||
),
|
||||
CashuTransaction(
|
||||
token="boundary", amount=1, unit="sat", type="out", created_at=900
|
||||
),
|
||||
CashuTransaction(
|
||||
token="collected",
|
||||
amount=1,
|
||||
unit="sat",
|
||||
type="out",
|
||||
created_at=800,
|
||||
collected=True,
|
||||
),
|
||||
CashuTransaction(
|
||||
token="swept", amount=1, unit="sat", type="out", created_at=800, swept=True
|
||||
),
|
||||
CashuTransaction(
|
||||
token="incoming", amount=1, unit="sat", type="in", created_at=800
|
||||
),
|
||||
]
|
||||
await _insert(session_factory, *rows)
|
||||
receive = AsyncMock()
|
||||
with (
|
||||
patch("routstr.wallet.db.create_session", side_effect=session_factory),
|
||||
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
|
||||
patch("routstr.wallet.time.time", return_value=1000),
|
||||
patch("routstr.wallet.recieve_token", receive),
|
||||
):
|
||||
await refund_sweep_once()
|
||||
|
||||
receive.assert_awaited_once_with("eligible")
|
||||
loaded = await _load(session_factory)
|
||||
assert loaded["eligible"].swept is True
|
||||
assert loaded["fresh"].swept is False
|
||||
assert loaded["boundary"].swept is False
|
||||
assert loaded["collected"].swept is False
|
||||
assert loaded["swept"].swept is True
|
||||
assert loaded["incoming"].swept is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("error", "collected"),
|
||||
[
|
||||
(RuntimeError("token already spent"), True),
|
||||
(RuntimeError("mint unavailable"), False),
|
||||
],
|
||||
)
|
||||
async def test_refund_sweep_records_terminal_but_not_transient_failures(
|
||||
session_factory: async_sessionmaker[AsyncSession], error: Exception, collected: bool
|
||||
) -> None:
|
||||
await _insert(
|
||||
session_factory,
|
||||
CashuTransaction(
|
||||
token="refund", amount=1, unit="sat", type="out", created_at=800
|
||||
),
|
||||
)
|
||||
with (
|
||||
patch("routstr.wallet.db.create_session", side_effect=session_factory),
|
||||
patch("routstr.wallet.settings.refund_sweep_ttl_seconds", 100),
|
||||
patch("routstr.wallet.time.time", return_value=1000),
|
||||
patch("routstr.wallet.recieve_token", AsyncMock(side_effect=error)),
|
||||
):
|
||||
await refund_sweep_once()
|
||||
|
||||
refund = (await _load(session_factory))["refund"]
|
||||
assert refund.collected is collected
|
||||
assert refund.swept is False
|
||||
+132
-1
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
@@ -6,7 +7,15 @@ from sqlalchemy.ext.asyncio import create_async_engine
|
||||
from sqlmodel import text
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.core.settings import Settings, SettingsService
|
||||
from routstr.core.settings import Settings, SettingsService, settings
|
||||
|
||||
NSEC_HEX = "1" * 64
|
||||
|
||||
|
||||
async def _read_settings_blob(session: AsyncSession) -> dict:
|
||||
"""Return the raw persisted settings JSON (id=1) as a dict."""
|
||||
row = await session.exec(text("SELECT data FROM settings WHERE id = 1")) # type: ignore
|
||||
return json.loads(row.first()[0])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -120,3 +129,125 @@ async def test_settings_initialize_discards_unknown_keys() -> None:
|
||||
assert '"enable_analytics_sharing": true' in stored_data
|
||||
assert "nostr_analytics_enabled" not in stored_data
|
||||
assert "unknown_key" not in stored_data
|
||||
|
||||
|
||||
# ── Secret fields are never written to the settings blob (issue #553) ────────
|
||||
|
||||
|
||||
def test_settings_model_drops_admin_password_field() -> None:
|
||||
# admin_password now lives only as a one-way hash in the Secret store; it is
|
||||
# no longer a settings field at all.
|
||||
assert "admin_password" not in Settings.__fields__
|
||||
# nsec remains a runtime value held in memory; upstream_api_key is ordinary
|
||||
# config that still lives in the persisted blob.
|
||||
assert "nsec" in Settings.__fields__
|
||||
assert "upstream_api_key" in Settings.__fields__
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_secret_fields_kept_in_memory_but_not_persisted(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# bootstrap_secrets (which runs first at boot) owns the nsec and has already
|
||||
# decrypted it into memory; simulate that live value. initialize must keep it
|
||||
# in memory for runtime consumers yet never write it to the settings blob.
|
||||
monkeypatch.setattr(settings, "nsec", NSEC_HEX)
|
||||
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
s = await SettingsService.initialize(session)
|
||||
|
||||
# Runtime consumers still see the live secret value.
|
||||
assert s.nsec == NSEC_HEX
|
||||
|
||||
# ...but it is never written to the settings blob.
|
||||
blob = await _read_settings_blob(session)
|
||||
assert "nsec" not in blob
|
||||
assert "admin_password" not in blob
|
||||
# Non-secret derived/public values are still persisted.
|
||||
assert blob["npub"] == s.npub
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upstream_api_key_survives_persistence(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# upstream_api_key is provider-scoped config, not a vault secret: it has no
|
||||
# encrypted home yet, so it must stay in the settings blob. Stripping it
|
||||
# would load it once, rewrite the blob without it, and lose it on the next
|
||||
# restart. Guard the on-disk survival path: blob-only value, no env.
|
||||
monkeypatch.delenv("UPSTREAM_API_KEY", raising=False)
|
||||
monkeypatch.setattr(settings, "upstream_api_key", "")
|
||||
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
await SettingsService.initialize(session)
|
||||
await session.exec( # type: ignore
|
||||
text("UPDATE settings SET data = :d WHERE id = 1").bindparams(
|
||||
d=json.dumps({"name": "LegacyNode", "upstream_api_key": "sk-only-in-db"})
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
# A reload must not drop the key from the blob...
|
||||
await SettingsService.initialize(session)
|
||||
blob = await _read_settings_blob(session)
|
||||
assert blob["upstream_api_key"] == "sk-only-in-db"
|
||||
# ...and it stays live for the proxy hot path.
|
||||
assert settings.upstream_api_key == "sk-only-in-db"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_existing_blob_secrets_are_stripped_on_initialize(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("NSEC", raising=False)
|
||||
monkeypatch.delenv("UPSTREAM_API_KEY", raising=False)
|
||||
monkeypatch.delenv("ADMIN_PASSWORD", raising=False)
|
||||
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
await SettingsService.initialize(session)
|
||||
|
||||
# Simulate a legacy row that still carries plaintext secrets in the blob.
|
||||
await session.exec( # type: ignore
|
||||
text("UPDATE settings SET data = :d WHERE id = 1").bindparams(
|
||||
d=json.dumps(
|
||||
{
|
||||
"name": "LegacyNode",
|
||||
"admin_password": "pw",
|
||||
"nsec": NSEC_HEX,
|
||||
"upstream_api_key": "sk-legacy",
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
await SettingsService.initialize(session)
|
||||
|
||||
blob = await _read_settings_blob(session)
|
||||
assert "admin_password" not in blob
|
||||
assert "nsec" not in blob
|
||||
# Non-secret values survive the migration, including upstream_api_key,
|
||||
# which is not vaulted yet and so must stay in the blob.
|
||||
assert blob["name"] == "LegacyNode"
|
||||
assert blob["upstream_api_key"] == "sk-legacy"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_does_not_persist_secret_fields(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("NSEC", NSEC_HEX)
|
||||
monkeypatch.setattr(settings, "nsec", "")
|
||||
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
await SettingsService.initialize(session)
|
||||
await SettingsService.update({"name": "Updated"}, session)
|
||||
|
||||
blob = await _read_settings_blob(session)
|
||||
assert blob["name"] == "Updated"
|
||||
assert "nsec" not in blob
|
||||
assert "admin_password" not in blob
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Settlement bills the model and provider fee that actually served.
|
||||
|
||||
Covers ``calculate_cost``'s served-identity parameters: a passed ``model_obj``
|
||||
is billed directly instead of re-deriving pricing from the response's model
|
||||
string through the alias map (which yields the best-ranked candidate, not the
|
||||
serving one), and a passed ``provider_fee`` is applied on the USD-cost path
|
||||
instead of the best-ranked provider's fee. The string/alias fallbacks remain
|
||||
for callers without routed identity.
|
||||
"""
|
||||
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("UPSTREAM_BASE_URL", "http://test")
|
||||
os.environ.setdefault("UPSTREAM_API_KEY", "test")
|
||||
|
||||
from routstr.payment.cost_calculation import CostData, calculate_cost
|
||||
from routstr.payment.models import Architecture, Model, Pricing
|
||||
|
||||
|
||||
def _make_model(
|
||||
model_id: str, prompt_sats: float, completion_sats: float
|
||||
) -> Model:
|
||||
return Model(
|
||||
id=model_id,
|
||||
name=model_id,
|
||||
created=0,
|
||||
description="",
|
||||
context_length=64000,
|
||||
architecture=Architecture(
|
||||
modality="text->text",
|
||||
input_modalities=["text"],
|
||||
output_modalities=["text"],
|
||||
tokenizer="Other",
|
||||
instruct_type=None,
|
||||
),
|
||||
pricing=Pricing(prompt=prompt_sats, completion=completion_sats),
|
||||
sats_pricing=Pricing(prompt=prompt_sats, completion=completion_sats),
|
||||
)
|
||||
|
||||
|
||||
WINNER = _make_model("dual-model", 0.001, 0.002)
|
||||
SERVED = _make_model("dual-model", 0.005, 0.010)
|
||||
|
||||
RESPONSE = {
|
||||
"model": "dual-model",
|
||||
"usage": {
|
||||
"prompt_tokens": 1000,
|
||||
"completion_tokens": 500,
|
||||
"total_tokens": 1500,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def patch_sats_usd_price() -> None: # type: ignore[misc]
|
||||
with patch(
|
||||
"routstr.payment.cost_calculation.sats_usd_price", return_value=5.0e-4
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_served_model_pricing_wins_over_alias_lookup() -> None:
|
||||
"""With ``model_obj`` given, the alias map is not consulted for pricing."""
|
||||
with patch(
|
||||
"routstr.proxy.get_model_instance", return_value=WINNER
|
||||
) as alias_lookup:
|
||||
result = await calculate_cost(
|
||||
dict(RESPONSE), max_cost=100_000, model_obj=SERVED
|
||||
)
|
||||
|
||||
assert isinstance(result, CostData)
|
||||
# 1000/1000 * 5000 + 500/1000 * 10000 msats at the SERVED model's rates.
|
||||
assert result.total_msats == 10_000
|
||||
alias_lookup.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_string_fallback_still_prices_without_model_obj() -> None:
|
||||
"""Callers without routed identity keep the alias-map string lookup."""
|
||||
with patch("routstr.proxy.get_model_instance", return_value=WINNER):
|
||||
result = await calculate_cost(dict(RESPONSE), max_cost=100_000)
|
||||
|
||||
assert isinstance(result, CostData)
|
||||
assert result.total_msats == 2_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_usd_cost_path_applies_given_provider_fee() -> None:
|
||||
"""The USD-cost path bills the serving provider's fee when supplied."""
|
||||
from unittest.mock import Mock
|
||||
|
||||
response = dict(RESPONSE)
|
||||
response["usage"] = dict(RESPONSE["usage"], cost=0.001) # type: ignore[arg-type]
|
||||
|
||||
best_ranked = Mock(provider_fee=1.0)
|
||||
with patch(
|
||||
"routstr.proxy.get_provider_for_model", return_value=[best_ranked]
|
||||
):
|
||||
result = await calculate_cost(
|
||||
response, max_cost=100_000, model_obj=SERVED, provider_fee=1.5
|
||||
)
|
||||
|
||||
assert isinstance(result, CostData)
|
||||
# 0.001 USD * fee 1.5 / 0.0005 USD-per-sat = 3 sats = 3000 msats.
|
||||
assert result.total_msats == 3_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_usd_cost_path_falls_back_to_best_ranked_fee() -> None:
|
||||
"""Without a supplied fee, the alias-map provider lookup still applies."""
|
||||
from unittest.mock import Mock
|
||||
|
||||
response = dict(RESPONSE)
|
||||
response["usage"] = dict(RESPONSE["usage"], cost=0.001) # type: ignore[arg-type]
|
||||
|
||||
best_ranked = Mock(provider_fee=2.0)
|
||||
with patch(
|
||||
"routstr.proxy.get_provider_for_model", return_value=[best_ranked]
|
||||
):
|
||||
result = await calculate_cost(response, max_cost=100_000)
|
||||
|
||||
assert isinstance(result, CostData)
|
||||
assert result.total_msats == 4_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_x_cashu_cost_uses_served_model_not_upstream_echo() -> None:
|
||||
"""``get_x_cashu_cost`` bills the routed model, not the raw model echo.
|
||||
|
||||
X-Cashu handlers do not rewrite the upstream's echoed model string, so
|
||||
without the routed model the settle would look up whatever wire name the
|
||||
upstream reported. With ``model_obj`` given, the echo must be irrelevant.
|
||||
"""
|
||||
from routstr.upstream import GenericUpstreamProvider
|
||||
|
||||
provider = GenericUpstreamProvider("http://upstream.example", "key", 1.0)
|
||||
response = dict(RESPONSE, model="totally-unknown-wire-name")
|
||||
|
||||
with patch(
|
||||
"routstr.proxy.get_model_instance", return_value=WINNER
|
||||
) as alias_lookup:
|
||||
cost = await provider.get_x_cashu_cost(
|
||||
response, max_cost_for_model=100_000, model_obj=SERVED
|
||||
)
|
||||
|
||||
assert cost is not None
|
||||
assert cost.total_msats == 10_000
|
||||
alias_lookup.assert_not_called()
|
||||
@@ -16,13 +16,14 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
||||
import pytest
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
from sqlmodel import SQLModel
|
||||
from sqlmodel import SQLModel, select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from routstr.auth import pay_for_request
|
||||
from routstr.balance import refund_wallet_endpoint
|
||||
from routstr.core.db import (
|
||||
ApiKey,
|
||||
ReservationRelease,
|
||||
release_stale_reservations,
|
||||
reset_all_reserved_balances,
|
||||
)
|
||||
@@ -150,6 +151,39 @@ async def test_release_stale_reservations_releases_old(session: AsyncSession) ->
|
||||
assert key.reserved_at is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_targeted_parent_cleanup_releases_child_owned_reservation(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
parent = ApiKey(hashed_key="stale-parent", balance=5_000)
|
||||
child = ApiKey(
|
||||
hashed_key="stale-child", parent_key_hash=parent.hashed_key, balance=0
|
||||
)
|
||||
session.add_all([parent, child])
|
||||
await session.commit()
|
||||
await pay_for_request(child, 1_000, session)
|
||||
reservation = (
|
||||
await session.exec(
|
||||
select(ReservationRelease).where(
|
||||
ReservationRelease.key_hash == child.hashed_key
|
||||
)
|
||||
)
|
||||
).one()
|
||||
reservation.created_at = int(time.time()) - 1_000
|
||||
session.add(reservation)
|
||||
await session.commit()
|
||||
|
||||
released = await release_stale_reservations(
|
||||
session, max_age_seconds=300, key_hash=parent.hashed_key
|
||||
)
|
||||
|
||||
assert released == 1
|
||||
await session.refresh(parent)
|
||||
await session.refresh(child)
|
||||
assert parent.reserved_balance == 0
|
||||
assert child.reserved_balance == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_stale_reservations_keeps_fresh(session: AsyncSession) -> None:
|
||||
key = ApiKey(
|
||||
@@ -352,11 +386,15 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
|
||||
upstream.forward_request = AsyncMock(side_effect=asyncio.CancelledError())
|
||||
|
||||
session = MagicMock()
|
||||
reservation_snapshot = MagicMock()
|
||||
revert_mock = AsyncMock(return_value=True)
|
||||
|
||||
with (
|
||||
patch.object(proxy_module, "get_model_instance", return_value=MagicMock()),
|
||||
patch.object(proxy_module, "get_provider_for_model", return_value=[upstream]),
|
||||
patch.object(
|
||||
proxy_module,
|
||||
"get_candidates",
|
||||
return_value=[(MagicMock(), upstream)],
|
||||
),
|
||||
patch.object(
|
||||
proxy_module, "get_max_cost_for_model", AsyncMock(return_value=1_000)
|
||||
),
|
||||
@@ -370,9 +408,16 @@ async def test_proxy_reverts_reservation_on_client_disconnect() -> None:
|
||||
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
|
||||
),
|
||||
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
|
||||
patch.object(
|
||||
proxy_module,
|
||||
"get_reservation_snapshot",
|
||||
AsyncMock(return_value=reservation_snapshot),
|
||||
),
|
||||
patch.object(proxy_module, "revert_pay_for_request", revert_mock),
|
||||
):
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await proxy_module.proxy(request, "v1/chat/completions", session=session)
|
||||
|
||||
revert_mock.assert_awaited_once_with(key, session, 1_000)
|
||||
revert_mock.assert_awaited_once_with(
|
||||
key, session, 1_000, reservation_snapshot
|
||||
)
|
||||
|
||||
@@ -4,6 +4,7 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.auth import ReservationSnapshot
|
||||
from routstr.core.db import ApiKey
|
||||
from routstr.upstream.base import BaseUpstreamProvider
|
||||
|
||||
@@ -67,6 +68,12 @@ async def test_stream_with_id_injection() -> None:
|
||||
max_cost_for_model=100,
|
||||
background_tasks=background_tasks,
|
||||
requested_model="test-model",
|
||||
reservation_snapshot=ReservationSnapshot(
|
||||
release_id="test-release",
|
||||
key_hash="test_hash",
|
||||
billing_key_hash="test_hash",
|
||||
reserved_msats=100,
|
||||
),
|
||||
)
|
||||
|
||||
results = []
|
||||
|
||||
@@ -0,0 +1,448 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncGenerator
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
from sqlmodel import SQLModel
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
import routstr.auth as auth_module
|
||||
from routstr.auth import (
|
||||
ReservationSnapshot,
|
||||
adjust_payment_for_tokens,
|
||||
get_reservation_snapshot,
|
||||
pay_for_request,
|
||||
release_reservation,
|
||||
)
|
||||
from routstr.core.db import ApiKey, ReservationRelease
|
||||
from routstr.payment.cost_calculation import MaxCostData
|
||||
from routstr.upstream.base import BaseUpstreamProvider
|
||||
|
||||
|
||||
async def _engine() -> AsyncEngine:
|
||||
engine = create_async_engine("sqlite+aiosqlite://")
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(SQLModel.metadata.create_all)
|
||||
return engine
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_reservation_is_durable_and_idempotent() -> None:
|
||||
engine = await _engine()
|
||||
key = ApiKey(hashed_key="key", balance=1_000)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
await pay_for_request(key, 500, session)
|
||||
snapshot = await get_reservation_snapshot(key, session)
|
||||
|
||||
record = await session.get(ReservationRelease, snapshot.release_id)
|
||||
assert record is not None and record.status == "active"
|
||||
assert await release_reservation(snapshot, session, 500) is True
|
||||
assert await release_reservation(snapshot, session, 500) is True
|
||||
|
||||
await session.refresh(key)
|
||||
await session.refresh(record)
|
||||
assert key.reserved_balance == 0
|
||||
assert key.reserved_at is None
|
||||
assert record.status == "released"
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_only_owns_its_concurrent_reservation() -> None:
|
||||
engine = await _engine()
|
||||
key = ApiKey(hashed_key="key", balance=1_000)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
|
||||
await pay_for_request(key, 400, session)
|
||||
first = await get_reservation_snapshot(key, session)
|
||||
await pay_for_request(key, 400, session)
|
||||
second = await get_reservation_snapshot(key, session)
|
||||
|
||||
assert first.release_id != second.release_id
|
||||
assert await release_reservation(first, session, 400) is True
|
||||
assert await release_reservation(first, session, 400) is True
|
||||
await session.refresh(key)
|
||||
assert key.reserved_balance == 400
|
||||
|
||||
assert await release_reservation(second, session, 400) is True
|
||||
await session.refresh(key)
|
||||
assert key.reserved_balance == 0
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_updates_parent_and_child_atomically() -> None:
|
||||
engine = await _engine()
|
||||
parent = ApiKey(hashed_key="parent", balance=1_000)
|
||||
child = ApiKey(hashed_key="child", parent_key_hash="parent", balance=0)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add_all([parent, child])
|
||||
await session.commit()
|
||||
await pay_for_request(child, 500, session)
|
||||
snapshot = await get_reservation_snapshot(child, session)
|
||||
|
||||
assert await release_reservation(snapshot, session, 500) is True
|
||||
await session.refresh(parent)
|
||||
await session.refresh(child)
|
||||
assert (parent.reserved_balance, child.reserved_balance) == (0, 0)
|
||||
assert (parent.reserved_at, child.reserved_at) == (None, None)
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_rolls_back_partial_parent_child_update() -> None:
|
||||
engine = await _engine()
|
||||
parent = ApiKey(hashed_key="parent", balance=1_000)
|
||||
child = ApiKey(hashed_key="child", parent_key_hash="parent", balance=0)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add_all([parent, child])
|
||||
await session.commit()
|
||||
await pay_for_request(child, 500, session)
|
||||
snapshot = await get_reservation_snapshot(child, session)
|
||||
child.reserved_balance = 100
|
||||
session.add(child)
|
||||
await session.commit()
|
||||
|
||||
assert await release_reservation(snapshot, session, 500) is False
|
||||
await session.refresh(parent)
|
||||
await session.refresh(child)
|
||||
record = await session.get(ReservationRelease, snapshot.release_id)
|
||||
assert (parent.reserved_balance, child.reserved_balance) == (500, 100)
|
||||
assert record is not None and record.status == "active"
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_commit_failure_cannot_release_charged_reservation() -> None:
|
||||
engine = await _engine()
|
||||
key = ApiKey(hashed_key="key", balance=1_000)
|
||||
cost = MaxCostData(
|
||||
base_msats=500,
|
||||
input_msats=0,
|
||||
output_msats=0,
|
||||
total_msats=500,
|
||||
)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
await pay_for_request(key, 500, session)
|
||||
snapshot = await get_reservation_snapshot(key, session)
|
||||
|
||||
with (
|
||||
patch("routstr.auth.calculate_cost", AsyncMock(return_value=cost)),
|
||||
patch.object(
|
||||
session,
|
||||
"refresh",
|
||||
AsyncMock(side_effect=SQLAlchemyError("post-commit refresh failed")),
|
||||
),
|
||||
):
|
||||
with pytest.raises(SQLAlchemyError, match="post-commit refresh failed"):
|
||||
await adjust_payment_for_tokens(key, {}, session, 500)
|
||||
|
||||
await session.rollback()
|
||||
assert await release_reservation(snapshot, session, 500) is False
|
||||
charged_key = await session.get(ApiKey, "key")
|
||||
record = await session.get(ReservationRelease, snapshot.release_id)
|
||||
assert charged_key is not None
|
||||
assert (charged_key.balance, charged_key.reserved_balance) == (500, 0)
|
||||
assert record is not None and record.status == "charged"
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generic_background_settlement_uses_explicit_reservation() -> None:
|
||||
engine = await _engine()
|
||||
provider = BaseUpstreamProvider(
|
||||
base_url="https://api.example.com", api_key="test-key", provider_fee=1.0
|
||||
)
|
||||
key = ApiKey(hashed_key="generic-key", balance=1_000)
|
||||
cost = MaxCostData(
|
||||
base_msats=500,
|
||||
input_msats=0,
|
||||
output_msats=0,
|
||||
total_msats=500,
|
||||
)
|
||||
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add(key)
|
||||
await session.commit()
|
||||
await pay_for_request(key, 500, session)
|
||||
snapshot = await get_reservation_snapshot(key, session)
|
||||
|
||||
context_token = auth_module._current_reservation.set(None)
|
||||
try:
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.base.create_session",
|
||||
side_effect=lambda: AsyncSession(engine, expire_on_commit=False),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.base.adjust_payment_for_tokens",
|
||||
auth_module.adjust_payment_for_tokens,
|
||||
),
|
||||
patch("routstr.auth.calculate_cost", AsyncMock(return_value=cost)),
|
||||
):
|
||||
await provider._finalize_generic_streaming_payment(
|
||||
key.hashed_key,
|
||||
500,
|
||||
"audio/speech",
|
||||
model_obj=None,
|
||||
provider_fee=provider.provider_fee,
|
||||
reservation_snapshot=snapshot,
|
||||
)
|
||||
finally:
|
||||
auth_module._current_reservation.reset(context_token)
|
||||
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
settled_key = await session.get(ApiKey, key.hashed_key)
|
||||
record = await session.get(ReservationRelease, snapshot.release_id)
|
||||
assert settled_key is not None
|
||||
assert (settled_key.balance, settled_key.reserved_balance) == (500, 0)
|
||||
assert record is not None and record.status == "charged"
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_release_is_terminal_and_suppresses_background_charge() -> None:
|
||||
provider = BaseUpstreamProvider(
|
||||
base_url="https://api.example.com", api_key="test-key"
|
||||
)
|
||||
|
||||
async def aiter_bytes() -> AsyncGenerator[bytes, None]:
|
||||
yield b"data: [DONE]\n\n"
|
||||
|
||||
upstream_response = MagicMock()
|
||||
upstream_response.status_code = 200
|
||||
upstream_response.headers = {"content-type": "text/event-stream"}
|
||||
upstream_response.aiter_bytes = aiter_bytes
|
||||
|
||||
key = MagicMock(spec=ApiKey)
|
||||
key.hashed_key = "test-key-hash"
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=key)
|
||||
session.rollback = AsyncMock()
|
||||
session_context = MagicMock()
|
||||
session_context.__aenter__ = AsyncMock(return_value=session)
|
||||
session_context.__aexit__ = AsyncMock(return_value=None)
|
||||
release = AsyncMock(return_value=True)
|
||||
reservation_snapshot = MagicMock()
|
||||
reservation_snapshot.reserved_msats = 500
|
||||
background_tasks = MagicMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"routstr.upstream.base.adjust_payment_for_tokens",
|
||||
AsyncMock(side_effect=SQLAlchemyError("database unavailable")),
|
||||
),
|
||||
patch(
|
||||
"routstr.upstream.base.get_reservation_snapshot",
|
||||
AsyncMock(return_value=reservation_snapshot),
|
||||
),
|
||||
patch("routstr.upstream.base.release_reservation", release),
|
||||
patch("routstr.upstream.base.create_session", return_value=session_context),
|
||||
):
|
||||
response = await provider.handle_streaming_chat_completion(
|
||||
response=upstream_response,
|
||||
key=key,
|
||||
max_cost_for_model=500,
|
||||
background_tasks=background_tasks,
|
||||
)
|
||||
|
||||
with pytest.raises(SQLAlchemyError, match="database unavailable"):
|
||||
async for _ in response.body_iterator:
|
||||
pass
|
||||
|
||||
session.rollback.assert_awaited_once()
|
||||
release.assert_awaited_once_with(reservation_snapshot, session, 500)
|
||||
background_tasks.add_task.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"release_outcome",
|
||||
[True, False, RuntimeError("release failed"), asyncio.CancelledError()],
|
||||
)
|
||||
async def test_responses_streaming_releases_and_raises_on_billing_failure(
|
||||
release_outcome: bool | BaseException,
|
||||
) -> None:
|
||||
provider = BaseUpstreamProvider(
|
||||
base_url="https://api.example.com", api_key="test-key"
|
||||
)
|
||||
|
||||
async def aiter_bytes() -> AsyncGenerator[bytes, None]:
|
||||
yield (
|
||||
b'data: {"type":"response.completed","response":{"model":"test",'
|
||||
b'"usage":{"input_tokens":1,"output_tokens":1}}}\n\n'
|
||||
)
|
||||
yield b"data: [DONE]\n\n"
|
||||
|
||||
upstream_response = MagicMock(
|
||||
status_code=200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
upstream_response.aiter_bytes = aiter_bytes
|
||||
key = MagicMock(spec=ApiKey)
|
||||
key.hashed_key = "responses-key"
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=key)
|
||||
session.rollback = AsyncMock()
|
||||
session_context = MagicMock()
|
||||
session_context.__aenter__ = AsyncMock(return_value=session)
|
||||
session_context.__aexit__ = AsyncMock(return_value=None)
|
||||
snapshot = ReservationSnapshot(
|
||||
release_id="responses-release",
|
||||
key_hash=key.hashed_key,
|
||||
billing_key_hash=key.hashed_key,
|
||||
reserved_msats=500,
|
||||
)
|
||||
release = (
|
||||
AsyncMock(side_effect=release_outcome)
|
||||
if isinstance(release_outcome, BaseException)
|
||||
else AsyncMock(return_value=release_outcome)
|
||||
)
|
||||
adjust = AsyncMock(side_effect=SQLAlchemyError("database unavailable"))
|
||||
|
||||
with (
|
||||
patch("routstr.upstream.base.adjust_payment_for_tokens", adjust),
|
||||
patch("routstr.upstream.base.release_reservation", release),
|
||||
patch("routstr.upstream.base.create_session", return_value=session_context),
|
||||
):
|
||||
response = await provider.handle_streaming_responses_completion(
|
||||
response=upstream_response,
|
||||
key=key,
|
||||
max_cost_for_model=500,
|
||||
reservation_snapshot=snapshot,
|
||||
)
|
||||
emitted = bytearray()
|
||||
with pytest.raises(SQLAlchemyError, match="database unavailable"):
|
||||
async for chunk in response.body_iterator:
|
||||
if isinstance(chunk, str):
|
||||
emitted.extend(chunk.encode())
|
||||
else:
|
||||
emitted.extend(bytes(chunk))
|
||||
|
||||
assert b'"total_msats": 0' not in emitted
|
||||
adjust.assert_awaited_once()
|
||||
session.rollback.assert_awaited_once()
|
||||
release.assert_awaited_once_with(snapshot, session, 500)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("via_litellm", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"release_outcome",
|
||||
[True, False, RuntimeError("release failed"), asyncio.CancelledError()],
|
||||
)
|
||||
async def test_messages_streaming_releases_and_raises_on_billing_failure(
|
||||
via_litellm: bool,
|
||||
release_outcome: bool | BaseException,
|
||||
) -> None:
|
||||
provider = BaseUpstreamProvider(
|
||||
base_url="https://api.example.com", api_key="test-key"
|
||||
)
|
||||
key = MagicMock(spec=ApiKey)
|
||||
key.hashed_key = "messages-key"
|
||||
session = MagicMock()
|
||||
session.get = AsyncMock(return_value=key)
|
||||
session.rollback = AsyncMock()
|
||||
session_context = MagicMock()
|
||||
session_context.__aenter__ = AsyncMock(return_value=session)
|
||||
session_context.__aexit__ = AsyncMock(return_value=None)
|
||||
snapshot = ReservationSnapshot(
|
||||
release_id=f"messages-{'litellm' if via_litellm else 'native'}",
|
||||
key_hash=key.hashed_key,
|
||||
billing_key_hash=key.hashed_key,
|
||||
reserved_msats=500,
|
||||
)
|
||||
release = (
|
||||
AsyncMock(side_effect=release_outcome)
|
||||
if isinstance(release_outcome, BaseException)
|
||||
else AsyncMock(return_value=release_outcome)
|
||||
)
|
||||
adjust = AsyncMock(side_effect=SQLAlchemyError("database unavailable"))
|
||||
|
||||
async def native_chunks() -> AsyncGenerator[bytes, None]:
|
||||
yield (
|
||||
b'event: message_start\ndata: {"type":"message_start","message":'
|
||||
b'{"model":"test","usage":{"input_tokens":1,"output_tokens":0}}}\n\n'
|
||||
)
|
||||
yield b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
|
||||
|
||||
async def litellm_chunks() -> AsyncGenerator[dict, None]:
|
||||
yield {
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"model": "test",
|
||||
"usage": {"input_tokens": 1, "output_tokens": 0},
|
||||
},
|
||||
}
|
||||
yield {"type": "message_stop"}
|
||||
|
||||
with (
|
||||
patch("routstr.upstream.base.adjust_payment_for_tokens", adjust),
|
||||
patch("routstr.upstream.base.release_reservation", release),
|
||||
patch("routstr.upstream.base.create_session", return_value=session_context),
|
||||
):
|
||||
if via_litellm:
|
||||
response = provider._stream_litellm_messages(
|
||||
iterator=litellm_chunks(),
|
||||
key=key,
|
||||
max_cost_for_model=500,
|
||||
requested_model=None,
|
||||
reservation_snapshot=snapshot,
|
||||
)
|
||||
else:
|
||||
upstream_response = MagicMock(
|
||||
status_code=200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
upstream_response.aiter_bytes = native_chunks
|
||||
response = await provider.handle_streaming_messages_completion(
|
||||
response=upstream_response,
|
||||
key=key,
|
||||
max_cost_for_model=500,
|
||||
reservation_snapshot=snapshot,
|
||||
)
|
||||
|
||||
with pytest.raises(SQLAlchemyError, match="database unavailable"):
|
||||
async for _ in response.body_iterator:
|
||||
pass
|
||||
|
||||
adjust.assert_awaited_once()
|
||||
session.rollback.assert_awaited_once()
|
||||
release.assert_awaited_once_with(snapshot, session, 500)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_key_reservation_snapshot_is_rejected_without_mutation() -> None:
|
||||
engine = await _engine()
|
||||
first = ApiKey(hashed_key="first", balance=1_000)
|
||||
second = ApiKey(hashed_key="second", balance=1_000)
|
||||
async with AsyncSession(engine, expire_on_commit=False) as session:
|
||||
session.add(first)
|
||||
session.add(second)
|
||||
await session.commit()
|
||||
await pay_for_request(first, 500, session)
|
||||
snapshot = await get_reservation_snapshot(first, session)
|
||||
|
||||
with pytest.raises(RuntimeError, match="does not belong"):
|
||||
await adjust_payment_for_tokens(
|
||||
second,
|
||||
{"model": "test", "usage": None},
|
||||
session,
|
||||
500,
|
||||
reservation_snapshot=snapshot,
|
||||
)
|
||||
|
||||
await session.refresh(first)
|
||||
await session.refresh(second)
|
||||
assert first.reserved_balance == 500
|
||||
assert second.reserved_balance == 0
|
||||
|
||||
await engine.dispose()
|
||||
@@ -24,6 +24,7 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from routstr.auth import ReservationSnapshot
|
||||
from routstr.core.db import ApiKey
|
||||
from routstr.upstream import base
|
||||
from routstr.upstream.base import BaseUpstreamProvider
|
||||
@@ -67,6 +68,12 @@ async def _drive(chunks: list[bytes], requested_model: str | None = None) -> lis
|
||||
max_cost_for_model=100,
|
||||
background_tasks=MagicMock(),
|
||||
requested_model=requested_model,
|
||||
reservation_snapshot=ReservationSnapshot(
|
||||
release_id="test-release",
|
||||
key_hash="test_hash",
|
||||
billing_key_hash="test_hash",
|
||||
reserved_msats=100,
|
||||
),
|
||||
)
|
||||
|
||||
out: list[bytes] = []
|
||||
|
||||
@@ -89,6 +89,7 @@ class TestParseTinfoilUsageMetrics:
|
||||
result = parse_tinfoil_usage_metrics(
|
||||
"prompt=1,completion=1,total=2,model=kimi-k2-6"
|
||||
)
|
||||
assert result is not None
|
||||
assert result["model"] == "kimi-k2-6"
|
||||
|
||||
def test_model_with_extra_fields(self) -> None:
|
||||
@@ -97,6 +98,7 @@ class TestParseTinfoilUsageMetrics:
|
||||
"cached_prompt_tokens=64,uncached_prompt_tokens=5,"
|
||||
"model=kimi-k2-6"
|
||||
)
|
||||
assert result is not None
|
||||
assert result["prompt_tokens"] == 69
|
||||
assert result["completion_tokens"] == 20
|
||||
assert result["total_tokens"] == 89
|
||||
|
||||
@@ -66,6 +66,54 @@ async def test_forward_with_trailer_captures_usage_trailer(
|
||||
writer.wait_closed.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_forward_with_trailer_strips_hop_by_hop_headers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
response = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"
|
||||
reader = FakeReader([response])
|
||||
writer = FakeWriter()
|
||||
monkeypatch.setattr(
|
||||
"routstr.upstream.tinfoil_trailer.asyncio.open_connection",
|
||||
AsyncMock(return_value=(reader, writer)),
|
||||
)
|
||||
|
||||
await forward_with_trailer(
|
||||
method="POST",
|
||||
url="https://enclave.tinfoil.sh/v1/chat/completions",
|
||||
headers={
|
||||
"Authorization": "Bearer upstream",
|
||||
"Connection": "keep-alive, X-Client-Hop",
|
||||
"Keep-Alive": "timeout=5",
|
||||
"Proxy-Authenticate": "Basic",
|
||||
"Proxy-Authorization": "Basic secret",
|
||||
"TE": "trailers",
|
||||
"Trailer": "X-Usage",
|
||||
"Transfer-Encoding": "chunked",
|
||||
"Upgrade": "websocket",
|
||||
"X-Client-Hop": "remove-me",
|
||||
"X-End-To-End": "preserve-me",
|
||||
},
|
||||
body=b"opaque",
|
||||
)
|
||||
|
||||
serialized_headers = writer.written.split(b"\r\n\r\n", 1)[0].lower()
|
||||
for name in (
|
||||
b"keep-alive",
|
||||
b"proxy-authenticate",
|
||||
b"proxy-authorization",
|
||||
b"te:",
|
||||
b"trailer:",
|
||||
b"transfer-encoding",
|
||||
b"upgrade:",
|
||||
b"x-client-hop",
|
||||
):
|
||||
assert name not in serialized_headers
|
||||
assert b"connection: close" in serialized_headers
|
||||
assert b"content-length: 6" in serialized_headers
|
||||
assert b"x-end-to-end: preserve-me" in serialized_headers
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_forward_with_trailer_enforces_response_size_limit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
||||
@@ -331,6 +331,7 @@ async def test_5xx_wrapped_rate_limit_is_classified(
|
||||
@pytest.mark.asyncio
|
||||
async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
|
||||
from routstr import proxy as proxy_module
|
||||
from routstr.auth import ReservationSnapshot
|
||||
from routstr.core.db import ApiKey
|
||||
from routstr.core.exceptions import UpstreamError
|
||||
|
||||
@@ -359,11 +360,20 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
|
||||
)
|
||||
|
||||
session = MagicMock()
|
||||
reservation = ReservationSnapshot(
|
||||
release_id="rate-limit-release",
|
||||
key_hash=key.hashed_key,
|
||||
billing_key_hash=key.hashed_key,
|
||||
reserved_msats=1_000,
|
||||
)
|
||||
revert_mock = AsyncMock(return_value=True)
|
||||
|
||||
with (
|
||||
patch.object(proxy_module, "get_model_instance", return_value=MagicMock()),
|
||||
patch.object(proxy_module, "get_provider_for_model", return_value=[upstream]),
|
||||
patch.object(
|
||||
proxy_module,
|
||||
"get_candidates",
|
||||
return_value=[(MagicMock(), upstream)],
|
||||
),
|
||||
patch.object(
|
||||
proxy_module, "get_max_cost_for_model", AsyncMock(return_value=1_000)
|
||||
),
|
||||
@@ -377,6 +387,11 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
|
||||
proxy_module, "get_bearer_token_key", AsyncMock(return_value=key)
|
||||
),
|
||||
patch.object(proxy_module, "pay_for_request", AsyncMock(return_value=1_000)),
|
||||
patch.object(
|
||||
proxy_module,
|
||||
"get_reservation_snapshot",
|
||||
AsyncMock(return_value=reservation),
|
||||
),
|
||||
patch.object(proxy_module, "revert_pay_for_request", revert_mock),
|
||||
):
|
||||
response = await proxy_module.proxy(
|
||||
@@ -393,4 +408,4 @@ async def test_proxy_loop_surfaces_rate_limit_and_reverts_once() -> None:
|
||||
assert RAW_ORG_ID not in serialized
|
||||
assert "org-[REDACTED]" in serialized
|
||||
# Single upstream failed -> reservation reverted exactly once (no double-charge).
|
||||
revert_mock.assert_awaited_once_with(key, session, 1_000)
|
||||
revert_mock.assert_awaited_once_with(key, session, 1_000, reservation)
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
"""Tests for ``routstr.core.vault`` — the secret encrypt/hash/fingerprint helpers.
|
||||
|
||||
Specifies the primitives that the rest of the secret-storage work (issue #553)
|
||||
builds on, independent of any database or app wiring:
|
||||
|
||||
- ``encrypt``/``decrypt`` — Fernet symmetric encryption emitting self-describing
|
||||
``fernet:v1:`` ciphertext, so a value can be told apart from legacy plaintext
|
||||
and from ciphertext written under a different ``ROUTSTR_SECRET_KEY`` (which
|
||||
surfaces as a hard ``InvalidToken`` rather than silent corruption).
|
||||
- ``hash_password``/``verify_password`` — salted scrypt hashing that is
|
||||
*key-independent* (does not depend on ``ROUTSTR_SECRET_KEY``), so password
|
||||
login and the recovery script keep working even if the key is lost.
|
||||
- a missing/malformed ``ROUTSTR_SECRET_KEY`` fails fast with the generation
|
||||
command in the message.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from cryptography.fernet import InvalidToken
|
||||
|
||||
from routstr.core import vault
|
||||
|
||||
# Two distinct, valid Fernet keys held fixed so ciphertext/fingerprints are
|
||||
# reproducible across runs and we can exercise the wrong-key path.
|
||||
KEY_A = "l_Tkp-7xmjcQ-IFhr6qhILrU8HPRbEmYMrfSbo_5srU="
|
||||
KEY_B = "_Teyrky_iToeDK51Tj1FsI9MJ340_cqKGmeher-a7MQ="
|
||||
|
||||
|
||||
def _use_key(monkeypatch: pytest.MonkeyPatch, key: str) -> None:
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY", key)
|
||||
|
||||
|
||||
# --- encrypt / decrypt -----------------------------------------------------
|
||||
|
||||
|
||||
def test_encrypt_decrypt_round_trips(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_use_key(monkeypatch, KEY_A)
|
||||
assert vault.decrypt(vault.encrypt("nsec1secret")) == "nsec1secret"
|
||||
|
||||
|
||||
def test_encrypt_emits_self_describing_prefix(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_use_key(monkeypatch, KEY_A)
|
||||
assert vault.encrypt("x").startswith("fernet:v1:")
|
||||
|
||||
|
||||
def test_encrypt_is_non_deterministic(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# Fernet embeds a random IV/timestamp: equal plaintext -> different
|
||||
# ciphertext. This is exactly why upstream-key equality needs a blind index.
|
||||
_use_key(monkeypatch, KEY_A)
|
||||
assert vault.encrypt("same") != vault.encrypt("same")
|
||||
|
||||
|
||||
def test_is_encrypted_distinguishes_ciphertext_from_plaintext(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_use_key(monkeypatch, KEY_A)
|
||||
assert vault.is_encrypted(vault.encrypt("x")) is True
|
||||
assert vault.is_encrypted("sk-plaintext-api-key") is False
|
||||
assert vault.is_encrypted("") is False
|
||||
|
||||
|
||||
def test_decrypt_rejects_unprefixed_value(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# Guards the migration paths: a legacy plaintext value must never be
|
||||
# mistaken for ciphertext and "decrypted".
|
||||
_use_key(monkeypatch, KEY_A)
|
||||
with pytest.raises(ValueError):
|
||||
vault.decrypt("not-encrypted")
|
||||
|
||||
|
||||
def test_decrypt_with_wrong_key_raises(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# The fail-fast signal: ciphertext written under KEY_A cannot be read under
|
||||
# KEY_B -> InvalidToken (bootstrap turns this into a clear startup error).
|
||||
_use_key(monkeypatch, KEY_A)
|
||||
token = vault.encrypt("secret")
|
||||
_use_key(monkeypatch, KEY_B)
|
||||
with pytest.raises(InvalidToken):
|
||||
vault.decrypt(token)
|
||||
|
||||
|
||||
# --- password hashing (key-independent) ------------------------------------
|
||||
|
||||
|
||||
def test_hash_and_verify_password(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_use_key(monkeypatch, KEY_A)
|
||||
stored = vault.hash_password("correct horse")
|
||||
assert vault.verify_password("correct horse", stored) is True
|
||||
assert vault.verify_password("wrong", stored) is False
|
||||
|
||||
|
||||
def test_password_hash_is_salted(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_use_key(monkeypatch, KEY_A)
|
||||
a = vault.hash_password("pw")
|
||||
b = vault.hash_password("pw")
|
||||
assert a != b
|
||||
assert vault.verify_password("pw", a) is True
|
||||
assert vault.verify_password("pw", b) is True
|
||||
|
||||
|
||||
def test_verify_password_rejects_malformed_stored_value(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# A garbage or non-scrypt stored value must verify to False, never raise.
|
||||
_use_key(monkeypatch, KEY_A)
|
||||
assert vault.verify_password("pw", "") is False
|
||||
assert vault.verify_password("pw", "not-a-hash") is False
|
||||
assert vault.verify_password("pw", "bcrypt:1:2:3:x:y") is False
|
||||
|
||||
|
||||
def test_password_hashing_is_key_independent(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# scrypt does not use ROUTSTR_SECRET_KEY, so login and the recovery script
|
||||
# work even when the key is missing.
|
||||
monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False)
|
||||
stored = vault.hash_password("pw")
|
||||
assert vault.verify_password("pw", stored) is True
|
||||
|
||||
|
||||
# --- fail-fast on missing/malformed key ------------------------------------
|
||||
|
||||
|
||||
def test_decrypt_without_any_key_fails_fast_with_generation_command(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
# Reading secrets is strict: with no key in env AND no key file, decrypt
|
||||
# fails fast with the generation command rather than silently minting a new
|
||||
# key (a fresh key could never match already-encrypted ciphertext). Only the
|
||||
# encrypt path auto-provisions; the read path never does.
|
||||
monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False)
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(tmp_path / "absent.key"))
|
||||
with pytest.raises(RuntimeError) as exc:
|
||||
vault.decrypt("fernet:v1:not-real-ciphertext")
|
||||
msg = str(exc.value)
|
||||
assert "ROUTSTR_SECRET_KEY" in msg
|
||||
assert "Fernet.generate_key" in msg
|
||||
|
||||
|
||||
def test_malformed_key_fails_fast(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY", "not-a-valid-fernet-key")
|
||||
with pytest.raises(RuntimeError):
|
||||
vault.encrypt("x")
|
||||
|
||||
|
||||
def test_malformed_env_key_does_not_self_provision(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
# A malformed env key is an operator mistake, not an unset key: it must fail
|
||||
# loudly, never silently generate a different key to a file (which would hide
|
||||
# the mistake and could brick secrets the operator meant to key differently).
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY", "not-a-valid-fernet-key")
|
||||
key_file = tmp_path / "routstr_secret.key"
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file))
|
||||
with pytest.raises(RuntimeError):
|
||||
vault.encrypt("x")
|
||||
assert not key_file.exists()
|
||||
|
||||
|
||||
# --- auto-provisioned key file (non-breaking upgrade path) -----------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def generated_key_file(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
|
||||
"""No env key; the key file points at a fresh, empty tmp location.
|
||||
|
||||
Exercises what an existing node hits when it upgrades without setting
|
||||
ROUTSTR_SECRET_KEY: the master key is auto-generated and persisted here so
|
||||
boot does not break, while secrets are still never written in plaintext.
|
||||
"""
|
||||
monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False)
|
||||
key_file = tmp_path / "routstr_secret.key"
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file))
|
||||
return key_file
|
||||
|
||||
|
||||
def test_encrypt_without_key_generates_and_persists_key_file(
|
||||
generated_key_file: Path,
|
||||
) -> None:
|
||||
# Encryption at rest stays mandatory, but a missing key is provisioned rather
|
||||
# than fatal: encrypt generates a key, writes it to the key file (owner-only),
|
||||
# and the value round-trips — decrypt, still with no env key, reads the same
|
||||
# file key back.
|
||||
key_file = generated_key_file
|
||||
assert not key_file.exists()
|
||||
|
||||
token = vault.encrypt("nsec1secret")
|
||||
|
||||
assert token.startswith("fernet:v1:")
|
||||
assert key_file.exists()
|
||||
assert key_file.stat().st_mode & 0o077 == 0 # not group/other-accessible
|
||||
assert vault.decrypt(token) == "nsec1secret"
|
||||
|
||||
|
||||
def test_generated_key_warns_operator_with_path_not_value(
|
||||
generated_key_file: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
# The notice names the file to back up and shouts the back-up imperative so an
|
||||
# upgrading operator cannot miss it, but it MUST NOT echo the key value: the
|
||||
# secret lives in the 0600 file, and printing it would leak it into captured
|
||||
# stdout / aggregated container logs.
|
||||
key_file = generated_key_file
|
||||
vault.encrypt("x")
|
||||
|
||||
out = capsys.readouterr().out
|
||||
assert "ROUTSTR_SECRET_KEY" in out
|
||||
assert str(key_file) in out
|
||||
assert key_file.read_text().strip() not in out
|
||||
assert "BACK UP" in out.upper()
|
||||
|
||||
|
||||
def test_existing_key_file_is_reused_and_warns_only_once(
|
||||
generated_key_file: Path, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
# Once the key exists, later encrypts reuse it (never rotate a key that
|
||||
# secrets were already encrypted under) and stay silent (no repeated notice).
|
||||
key_file = generated_key_file
|
||||
vault.encrypt("a")
|
||||
first_key = key_file.read_text()
|
||||
capsys.readouterr() # drain the one-time notice
|
||||
|
||||
vault.encrypt("b")
|
||||
|
||||
assert key_file.read_text() == first_key
|
||||
assert capsys.readouterr().out == ""
|
||||
|
||||
|
||||
def test_generated_key_is_published_atomically(
|
||||
generated_key_file: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# The key must appear at its final path only as a complete file: it is written
|
||||
# to a temp file and atomically linked into place. If the publish (link) step
|
||||
# fails — e.g. the process crashes — the destination must be absent, never a
|
||||
# half-written or empty file that a later boot would read as a corrupt key and
|
||||
# then refuse to decrypt every secret. No temp debris is left behind.
|
||||
key_file = generated_key_file
|
||||
|
||||
def boom(src: object, dst: object) -> None:
|
||||
raise OSError("crash during atomic publish")
|
||||
|
||||
monkeypatch.setattr(vault.os, "link", boom)
|
||||
|
||||
with pytest.raises(OSError):
|
||||
vault.encrypt("x")
|
||||
|
||||
assert not key_file.exists()
|
||||
assert list(key_file.parent.iterdir()) == []
|
||||
|
||||
|
||||
def test_racing_worker_adopts_winners_key_without_clobber(
|
||||
generated_key_file: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# Two workers auto-generate a key concurrently. The first to link "wins" and
|
||||
# its key is the one on disk; a worker that loses the link race must adopt the
|
||||
# winner's key — secrets may already be encrypted under it — rather than
|
||||
# clobber it or crash. Force the race deterministically: the winner publishes
|
||||
# its key at the final path just as this worker tries to link, so os.link
|
||||
# raises FileExistsError and the loser reads the winner's key back.
|
||||
key_file = generated_key_file
|
||||
|
||||
def winner_links_first(src: object, dst: object) -> None:
|
||||
key_file.write_text(KEY_A) # the winner's already-published key
|
||||
raise FileExistsError
|
||||
|
||||
monkeypatch.setattr(vault.os, "link", winner_links_first)
|
||||
|
||||
token = vault.encrypt("secret")
|
||||
|
||||
# The winner's key stays put and the loser encrypted under it, so the value
|
||||
# round-trips under KEY_A even though this worker had generated its own key.
|
||||
assert key_file.read_text().strip() == KEY_A
|
||||
assert vault.decrypt(token) == "secret"
|
||||
# The losing worker left no temp debris behind.
|
||||
assert [p.name for p in key_file.parent.iterdir()] == [key_file.name]
|
||||
|
||||
|
||||
def test_loose_key_file_perms_are_tightened_on_read(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
# A key file left group/other-readable (e.g. written under a loose umask
|
||||
# before this hardening, or by a careless operator) is a leaked-secret risk.
|
||||
# Reading it repairs the permissions to owner-only rather than trusting a
|
||||
# world-readable master key, while still using the key so boot is not broken.
|
||||
monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False)
|
||||
key_file = tmp_path / "routstr_secret.key"
|
||||
key_file.write_text(KEY_A)
|
||||
key_file.chmod(0o644)
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file))
|
||||
|
||||
token = vault.encrypt("secret") # reads the loose file, repairs its perms
|
||||
|
||||
assert key_file.stat().st_mode & 0o077 == 0
|
||||
assert vault.decrypt(token) == "secret"
|
||||
|
||||
|
||||
def test_env_key_takes_precedence_over_key_file(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
# An explicit env key wins over a persisted file key (an operator-supplied
|
||||
# key from a secrets manager overrides the auto-generated one), and the file
|
||||
# is left untouched.
|
||||
key_file = tmp_path / "routstr_secret.key"
|
||||
key_file.write_text(KEY_B)
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY_FILE", str(key_file))
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY", KEY_A)
|
||||
|
||||
token = vault.encrypt("x")
|
||||
|
||||
assert vault.decrypt(token) == "x" # env key (A) decrypts it
|
||||
monkeypatch.setenv("ROUTSTR_SECRET_KEY", KEY_B)
|
||||
with pytest.raises(InvalidToken):
|
||||
vault.decrypt(token) # the file key (B) does not
|
||||
assert key_file.read_text() == KEY_B # file key never used or overwritten
|
||||
|
||||
|
||||
def test_generated_key_defaults_beside_the_database(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
# With no env key and no explicit key-file path, the key is provisioned next
|
||||
# to the SQLite database, so it rides whatever volume already persists the
|
||||
# data instead of landing in the working directory (where a container
|
||||
# recreate would lose it and brick decryption).
|
||||
monkeypatch.delenv("ROUTSTR_SECRET_KEY", raising=False)
|
||||
monkeypatch.delenv("ROUTSTR_SECRET_KEY_FILE", raising=False)
|
||||
monkeypatch.chdir(tmp_path) # isolate the working-dir fallback from the repo
|
||||
db_dir = tmp_path / "data"
|
||||
db_dir.mkdir()
|
||||
monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{db_dir}/routstr.db")
|
||||
|
||||
token = vault.encrypt("beside-the-db")
|
||||
|
||||
assert (db_dir / "routstr_secret.key").exists()
|
||||
assert not (tmp_path / "routstr_secret.key").exists() # not the working dir
|
||||
assert vault.decrypt(token) == "beside-the-db"
|
||||
@@ -15,6 +15,7 @@ from routstr.wallet import (
|
||||
get_balance,
|
||||
is_mint_connection_error,
|
||||
recieve_token,
|
||||
send,
|
||||
send_token,
|
||||
)
|
||||
|
||||
@@ -26,7 +27,11 @@ async def test_get_balance() -> None:
|
||||
mock_wallet.load_mint = AsyncMock()
|
||||
mock_wallet.load_proofs = AsyncMock()
|
||||
|
||||
with patch("routstr.wallet.Wallet.with_db", return_value=mock_wallet):
|
||||
# Reset the module-level wallet cache so a real wallet cached by an earlier
|
||||
# test (e.g. an unmocked admin-withdraw path) can't shadow the mock here.
|
||||
with patch("routstr.wallet._wallets", {}), patch(
|
||||
"routstr.wallet.Wallet.with_db", return_value=mock_wallet
|
||||
):
|
||||
balance = await get_balance("sat")
|
||||
assert balance == 50000
|
||||
|
||||
@@ -147,6 +152,69 @@ async def test_send_token() -> None:
|
||||
assert token == "test_token"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_falls_back_when_preferred_mint_has_only_reserved_balance() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
preferred_wallet = Mock(keysets={}, proofs=[])
|
||||
preferred_wallet.select_to_send = AsyncMock()
|
||||
primary_wallet = Mock(keysets={}, proofs=[])
|
||||
primary_wallet.select_to_send = AsyncMock()
|
||||
primary_wallet.serialize_proofs = AsyncMock(return_value="primary-token")
|
||||
primary_wallet.set_reserved_for_send = AsyncMock()
|
||||
|
||||
preferred_liquid = Mock(amount=500, reserved=False)
|
||||
preferred_reserved = Mock(amount=600, reserved=True)
|
||||
primary_liquid = Mock(amount=1000, reserved=False)
|
||||
primary_wallet.select_to_send.return_value = ([primary_liquid], None)
|
||||
|
||||
async def get_wallet(mint_url: str, unit: str) -> Mock:
|
||||
assert unit == "sat"
|
||||
return primary_wallet if mint_url == "http://primary:3338" else preferred_wallet
|
||||
|
||||
def get_proofs(wallet: Mock, mint_url: str, unit: str) -> list[Mock]:
|
||||
assert unit == "sat"
|
||||
if wallet is primary_wallet:
|
||||
assert mint_url == "http://primary:3338"
|
||||
return [primary_liquid]
|
||||
assert mint_url == "http://preferred:3338"
|
||||
return [preferred_liquid, preferred_reserved]
|
||||
|
||||
with (
|
||||
patch.object(settings, "primary_mint", "http://primary:3338"),
|
||||
patch("routstr.wallet.get_wallet", side_effect=get_wallet),
|
||||
patch("routstr.wallet.get_proofs_per_mint_and_unit", side_effect=get_proofs),
|
||||
):
|
||||
amount, token = await send(1000, "sat", "http://preferred:3338")
|
||||
|
||||
assert (amount, token) == (1000, "primary-token")
|
||||
preferred_wallet.select_to_send.assert_not_awaited()
|
||||
primary_wallet.select_to_send.assert_awaited_once_with(
|
||||
[primary_liquid], 1000, set_reserved=False, include_fees=False
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_primary_with_only_reserved_proofs_still_raises() -> None:
|
||||
from routstr.core.settings import settings
|
||||
|
||||
wallet = Mock(keysets={}, proofs=[])
|
||||
wallet.select_to_send = AsyncMock(side_effect=RuntimeError("balance too low"))
|
||||
reserved = Mock(amount=1000, reserved=True)
|
||||
|
||||
with (
|
||||
patch.object(settings, "primary_mint", "http://primary:3338"),
|
||||
patch("routstr.wallet.get_wallet", AsyncMock(return_value=wallet)),
|
||||
patch("routstr.wallet.get_proofs_per_mint_and_unit", return_value=[reserved]),
|
||||
pytest.raises(RuntimeError, match="balance too low"),
|
||||
):
|
||||
await send(1000, "sat", "http://primary:3338")
|
||||
|
||||
wallet.select_to_send.assert_awaited_once_with(
|
||||
[], 1000, set_reserved=False, include_fees=False
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_credit_balance() -> None:
|
||||
token_data = {
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Additional money-path coverage tests for wallet.py (86% → target 90%).
|
||||
|
||||
Tests error classification, periodic task structure, and token operations.
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# ===========================================================================
|
||||
# is_mint_connection_error
|
||||
# ===========================================================================
|
||||
|
||||
def test_is_mint_connection_error_true() -> None:
|
||||
"""Connection errors are detected."""
|
||||
from routstr.wallet import is_mint_connection_error
|
||||
|
||||
assert is_mint_connection_error(ConnectionRefusedError("refused")) is True
|
||||
assert is_mint_connection_error(TimeoutError("timeout")) is True
|
||||
|
||||
|
||||
def test_is_mint_connection_error_false() -> None:
|
||||
"""Non-connection errors are not flagged."""
|
||||
from routstr.wallet import is_mint_connection_error
|
||||
|
||||
assert is_mint_connection_error(ValueError("bad data")) is False
|
||||
assert is_mint_connection_error(KeyError("missing key")) is False
|
||||
assert is_mint_connection_error(RuntimeError("something broke")) is False
|
||||
assert is_mint_connection_error(AttributeError("no attr")) is False
|
||||
# OSError is NOT a connection error unless it's a subclass
|
||||
assert is_mint_connection_error(OSError("generic")) is False
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# classify_redemption_error
|
||||
# ===========================================================================
|
||||
|
||||
def test_classify_redemption_error_token_consumed() -> None:
|
||||
"""Token already spent returns token_consumed classification."""
|
||||
from routstr.wallet import TokenConsumedError, classify_redemption_error
|
||||
|
||||
result = classify_redemption_error(
|
||||
TokenConsumedError("Token was already redeemed")
|
||||
)
|
||||
assert result is not None
|
||||
assert result[0] == "token_consumed"
|
||||
assert result[1] == 500
|
||||
|
||||
|
||||
def test_classify_redemption_error_mint_connection() -> None:
|
||||
"""Mint connection error is classified correctly."""
|
||||
from routstr.wallet import classify_redemption_error
|
||||
|
||||
result = classify_redemption_error(
|
||||
ConnectionRefusedError("Connection refused")
|
||||
)
|
||||
assert result is not None
|
||||
# Should classify as mint_connection or return error tuple
|
||||
assert isinstance(result, tuple)
|
||||
assert len(result) >= 3
|
||||
|
||||
|
||||
def test_classify_redemption_error_unclassified() -> None:
|
||||
"""Generic errors are classified as cashu_error with 400 status."""
|
||||
from routstr.wallet import classify_redemption_error
|
||||
|
||||
result = classify_redemption_error(ValueError("unexpected"))
|
||||
# classify_redemption_error classifies all unrecognized errors
|
||||
# as cashu_error with a generic message
|
||||
assert result is not None
|
||||
assert result[0] == "cashu_error"
|
||||
assert result[1] == 400
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Store readiness: store_cashu_transaction succeeds
|
||||
# ===========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_store_cashu_transaction_succeeds_normally() -> None:
|
||||
"""Normal store_cashu_transaction returns True on success."""
|
||||
from routstr.core.db import store_cashu_transaction
|
||||
|
||||
with patch("routstr.core.db.create_session") as mock_create:
|
||||
mock_session = AsyncMock()
|
||||
mock_session.commit = AsyncMock()
|
||||
mock_session.__aenter__ = AsyncMock(return_value=mock_session)
|
||||
mock_session.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_create.return_value = mock_session
|
||||
|
||||
result = await store_cashu_transaction(
|
||||
token="cashuAtest",
|
||||
amount=1000,
|
||||
unit="sat",
|
||||
typ="in",
|
||||
request_id="req-test",
|
||||
)
|
||||
|
||||
assert result is True
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# get_balance
|
||||
# ===========================================================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_balance_returns_integer() -> None:
|
||||
"""get_balance returns an integer balance from wallet."""
|
||||
from routstr.wallet import get_balance
|
||||
|
||||
mock_wallet = Mock()
|
||||
mock_wallet.available_balance = Mock(amount=50000)
|
||||
mock_wallet.load_mint = AsyncMock()
|
||||
mock_wallet.load_proofs = AsyncMock()
|
||||
|
||||
with (
|
||||
patch("routstr.wallet._wallets", {}),
|
||||
patch("routstr.wallet.Wallet.with_db", return_value=mock_wallet),
|
||||
):
|
||||
balance = await get_balance("sat")
|
||||
assert isinstance(balance, int)
|
||||
assert balance == 50000
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Periodic task structure verification
|
||||
# ===========================================================================
|
||||
|
||||
def test_periodic_payout_has_loop_and_error_handling() -> None:
|
||||
"""periodic_payout runs in a loop with error handling."""
|
||||
import inspect
|
||||
|
||||
from routstr import wallet
|
||||
|
||||
source = inspect.getsource(wallet.periodic_payout)
|
||||
assert "while True" in source
|
||||
assert "except" in source, "Must have error handling"
|
||||
|
||||
|
||||
def test_periodic_refund_sweep_has_error_handling() -> None:
|
||||
"""Refund sweep catches errors to stay alive."""
|
||||
import inspect
|
||||
|
||||
from routstr import wallet
|
||||
|
||||
source = inspect.getsource(wallet.periodic_refund_sweep)
|
||||
assert "while True" in source
|
||||
assert "except" in source, "Must have error handling"
|
||||
|
||||
|
||||
def test_periodic_routstr_fee_payout_structure() -> None:
|
||||
"""Fee payout loop handles missing LN address gracefully."""
|
||||
import inspect
|
||||
|
||||
from routstr import wallet
|
||||
|
||||
source = inspect.getsource(wallet.periodic_routstr_fee_payout)
|
||||
# Returns early if ROUTSTR_LN_ADDRESS not set
|
||||
assert "ROUTSTR_LN_ADDRESS" in source
|
||||
assert "return" in source or "skip" in source.lower()
|
||||
+3
-3
@@ -4,15 +4,15 @@ FROM base AS deps
|
||||
RUN apk add --no-cache libc6-compat
|
||||
WORKDIR /app
|
||||
|
||||
RUN corepack enable pnpm && corepack prepare pnpm@latest --activate
|
||||
RUN corepack enable pnpm && corepack prepare pnpm@10.15.0 --activate
|
||||
|
||||
COPY package.json pnpm-lock.yaml* ./
|
||||
COPY package.json pnpm-lock.yaml* pnpm-workspace.yaml* ./
|
||||
RUN pnpm install --frozen-lockfile
|
||||
|
||||
FROM base AS builder
|
||||
WORKDIR /app
|
||||
|
||||
RUN corepack enable pnpm && corepack prepare pnpm@latest --activate
|
||||
RUN corepack enable pnpm && corepack prepare pnpm@10.15.0 --activate
|
||||
|
||||
COPY --from=deps /app/node_modules ./node_modules
|
||||
COPY . .
|
||||
|
||||
+3
-3
@@ -6,14 +6,14 @@ RUN apk add --no-cache libc6-compat
|
||||
WORKDIR /app
|
||||
|
||||
# Copy package files
|
||||
COPY package.json pnpm-lock.yaml* ./
|
||||
RUN corepack enable pnpm && corepack prepare pnpm@latest --activate
|
||||
COPY package.json pnpm-lock.yaml* pnpm-workspace.yaml* ./
|
||||
RUN corepack enable pnpm && corepack prepare pnpm@10.15.0 --activate
|
||||
RUN pnpm install --frozen-lockfile
|
||||
|
||||
# Build the UI
|
||||
FROM base AS builder
|
||||
WORKDIR /app
|
||||
RUN corepack enable pnpm && corepack prepare pnpm@latest --activate
|
||||
RUN corepack enable pnpm && corepack prepare pnpm@10.15.0 --activate
|
||||
COPY --from=deps /app/node_modules ./node_modules
|
||||
COPY . .
|
||||
|
||||
|
||||
@@ -86,7 +86,7 @@ export function AddModelForm({
|
||||
|
||||
return (
|
||||
<Dialog open={isOpen} onOpenChange={handleClose}>
|
||||
<DialogContent className='max-h-[90vh] overflow-y-auto sm:max-w-[600px]'>
|
||||
<DialogContent className='max-h-[90dvh] overflow-y-auto sm:max-w-[600px]'>
|
||||
<DialogHeader>
|
||||
<DialogTitle className='flex items-center gap-2'>
|
||||
<Plus className='h-5 w-5' />
|
||||
|
||||
@@ -438,7 +438,7 @@ export function AddProviderModelDialog({
|
||||
|
||||
return (
|
||||
<Dialog open={isOpen} onOpenChange={(open) => !open && onClose()}>
|
||||
<DialogContent className='max-h-[90vh] overflow-y-auto sm:max-w-[720px]'>
|
||||
<DialogContent className='max-h-[90dvh] overflow-y-auto sm:max-w-[720px]'>
|
||||
<DialogHeader>
|
||||
<DialogTitle className='flex items-center gap-2'>
|
||||
<Plus className='h-4 w-4' />
|
||||
|
||||
@@ -177,7 +177,7 @@ export function CollectModelsDialog({
|
||||
|
||||
return (
|
||||
<Dialog open={isOpen} onOpenChange={handleClose}>
|
||||
<DialogContent className='max-h-[80vh] sm:max-w-[700px]'>
|
||||
<DialogContent className='max-h-[80dvh] sm:max-w-[700px]'>
|
||||
<DialogHeader>
|
||||
<DialogTitle className='flex items-center gap-2'>
|
||||
<Download className='h-5 w-5' />
|
||||
|
||||
@@ -25,7 +25,7 @@ export function CostCalculatorDialog({
|
||||
}: CostCalculatorDialogProps) {
|
||||
return (
|
||||
<Dialog open={isOpen} onOpenChange={onClose}>
|
||||
<DialogContent className='max-h-[90vh] overflow-y-auto sm:max-w-[700px]'>
|
||||
<DialogContent className='max-h-[90dvh] overflow-y-auto sm:max-w-[700px]'>
|
||||
<DialogHeader>
|
||||
<DialogTitle className='flex items-center gap-2'>
|
||||
<Calculator className='h-5 w-5' />
|
||||
|
||||
@@ -100,7 +100,7 @@ export function EditGroupForm({
|
||||
|
||||
return (
|
||||
<Dialog open={isOpen} onOpenChange={handleClose}>
|
||||
<DialogContent className='max-h-[90vh] overflow-y-auto sm:max-w-[700px]'>
|
||||
<DialogContent className='max-h-[90dvh] overflow-y-auto sm:max-w-[700px]'>
|
||||
<DialogHeader>
|
||||
<DialogTitle className='flex items-center gap-2'>
|
||||
<Users className='h-5 w-5' />
|
||||
|
||||
@@ -91,25 +91,27 @@ export function CashuPaymentWorkflow({
|
||||
setIsCreatingKey(true);
|
||||
|
||||
try {
|
||||
const params = new URLSearchParams({
|
||||
const requestPayload: {
|
||||
initial_balance_token: string;
|
||||
balance_limit?: number;
|
||||
balance_limit_reset?: string;
|
||||
validity_date?: number;
|
||||
} = {
|
||||
initial_balance_token: initialToken.trim(),
|
||||
});
|
||||
if (balanceLimit) params.append('balance_limit', balanceLimit);
|
||||
};
|
||||
if (balanceLimit) requestPayload.balance_limit = Number(balanceLimit);
|
||||
if (balanceLimitReset)
|
||||
params.append('balance_limit_reset', balanceLimitReset);
|
||||
requestPayload.balance_limit_reset = balanceLimitReset;
|
||||
if (validityDate) {
|
||||
const timestamp = Math.floor(
|
||||
requestPayload.validity_date = Math.floor(
|
||||
new Date(validityDate + 'T23:59:59').getTime() / 1000
|
||||
);
|
||||
params.append('validity_date', timestamp.toString());
|
||||
}
|
||||
const response = await fetch(
|
||||
`${baseUrl}/v1/balance/create?${params.toString()}`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}
|
||||
);
|
||||
const response = await fetch(`${baseUrl}/v1/balance/create`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(requestPayload),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(errorText || 'Failed to create API key');
|
||||
|
||||
@@ -213,7 +213,7 @@ export function ProviderCard({
|
||||
</CardHeader>
|
||||
|
||||
<Dialog open={isKeyModalOpen} onOpenChange={setIsKeyModalOpen}>
|
||||
<DialogContent className='max-h-[90vh] overflow-y-auto sm:max-w-[500px]'>
|
||||
<DialogContent className='max-h-[90dvh] overflow-y-auto sm:max-w-[500px]'>
|
||||
<DialogHeader>
|
||||
<DialogTitle>
|
||||
{provider.api_key
|
||||
|
||||
@@ -53,7 +53,7 @@ export function ProviderFormDialogContent({
|
||||
availableMints,
|
||||
}: ProviderFormDialogContentProps) {
|
||||
return (
|
||||
<DialogContent className='max-h-[90vh] overflow-y-auto sm:max-w-[500px]'>
|
||||
<DialogContent className='max-h-[90dvh] overflow-y-auto sm:max-w-[500px]'>
|
||||
<DialogHeader>
|
||||
<DialogTitle>{title}</DialogTitle>
|
||||
<DialogDescription>{description}</DialogDescription>
|
||||
|
||||
@@ -112,7 +112,7 @@ export function ProviderBalance({
|
||||
</Button>
|
||||
|
||||
<Dialog open={isTopupDialogOpen} onOpenChange={handleCloseDialog}>
|
||||
<DialogContent className='max-h-[90vh] overflow-y-auto sm:max-w-md'>
|
||||
<DialogContent className='max-h-[90dvh] overflow-y-auto sm:max-w-md'>
|
||||
<DialogHeader>
|
||||
<DialogTitle>Top Up Balance</DialogTitle>
|
||||
<DialogDescription>
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user